From 16bb9a5ba074f59e2a757facf9aebf4114c5fcab Mon Sep 17 00:00:00 2001 From: Chris Dragan Date: Fri, 28 May 2021 03:26:07 -0700 Subject: [PATCH] misc: mods: update from Perforce Change-Id: If7d0606ab444ba4dabc8ef317120d1d4393ef871 Signed-off-by: Chris Dragan Reviewed-on: https://git-master.nvidia.com/r/c/linux-nvidia/+/2536359 Reviewed-by: svcacv Reviewed-by: svc_kernel_abi Reviewed-by: Bharat Nihalani Reviewed-by: mobile promotions GVS: Gerrit_Virtual_Submit Tested-by: mobile promotions --- drivers/misc/mods/mods_config.h | 33 ++++++++--------- drivers/misc/mods/mods_internal.h | 11 ++++-- drivers/misc/mods/mods_krnl.c | 12 ++++--- drivers/misc/mods/mods_mem.c | 59 +++++++++++++++++++++---------- drivers/misc/mods/mods_pci.c | 17 +++------ 5 files changed, 77 insertions(+), 55 deletions(-) diff --git a/drivers/misc/mods/mods_config.h b/drivers/misc/mods/mods_config.h index 5a186e9b..a475d46e 100644 --- a/drivers/misc/mods/mods_config.h +++ b/drivers/misc/mods/mods_config.h @@ -2,7 +2,7 @@ /* * mods_config.h - This file is part of NVIDIA MODS kernel driver. * - * Copyright (c) 2008-2020, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2008-2021, NVIDIA CORPORATION. All rights reserved. * * NVIDIA MODS kernel driver is free software: you can redistribute it and/or * modify it under the terms of the GNU General Public License, @@ -23,9 +23,13 @@ #define MODS_KERNEL_VERSION LINUX_VERSION_CODE -#if KERNEL_VERSION(2, 6, 30) <= MODS_KERNEL_VERSION +#if KERNEL_VERSION(2, 6, 30) <= MODS_KERNEL_VERSION && \ + KERNEL_VERSION(4, 16, 0) > MODS_KERNEL_VERSION && \ + defined(CONFIG_X86) # define MODS_HAS_DMA_OPS 1 -#else +#endif + +#if KERNEL_VERSION(2, 6, 30) > MODS_KERNEL_VERSION # define MODS_HASNT_PCI_RESCAN_BUS 1 #endif @@ -60,12 +64,6 @@ # define MODS_HASNT_PCI_LOCK_RESCAN_REMOVE 1 #endif -#if defined(CONFIG_ARM64) && \ - KERNEL_VERSION(3, 15, 0) <= MODS_KERNEL_VERSION && \ - KERNEL_VERSION(4, 14, 0) > MODS_KERNEL_VERSION -# define MODS_HAS_NONCOH_DMA_OPS 1 -#endif - #if KERNEL_VERSION(3, 16, 0) <= MODS_KERNEL_VERSION && \ defined(CONFIG_VT_HW_CONSOLE_BINDING) # define MODS_HAS_CONSOLE_BINDING 1 @@ -75,24 +73,27 @@ # define MODS_HAS_PNV_PCI_GET_NPU_DEV 1 #endif -#if KERNEL_VERSION(4, 12, 0) <= MODS_KERNEL_VERSION && !defined(CONFIG_PPC64) -# define MODS_HAS_SET_MEMORY_HEADER 1 +#if KERNEL_VERSION(4, 12, 0) <= MODS_KERNEL_VERSION && \ + KERNEL_VERSION(4, 13, 0) > MODS_KERNEL_VERSION && \ + defined(CONFIG_X86) +# define MODS_HAS_ASM_SET_MEMORY_HEADER 1 #endif #if KERNEL_VERSION(4, 13, 0) <= MODS_KERNEL_VERSION # define MODS_HAS_FLR_SUPPORT -#if KERNEL_VERSION(4, 17, 0) <= MODS_KERNEL_VERSION -# define MODS_PCIE_FLR_HAS_ERR -#endif +# define MODS_HAS_SET_MEMORY_HEADER 1 #endif #if KERNEL_VERSION(4, 14, 0) <= MODS_KERNEL_VERSION # define MODS_HAS_KERNEL_WRITE #endif -#if KERNEL_VERSION(4, 16, 0) > MODS_KERNEL_VERSION && defined(CONFIG_X86) -# define MODS_HAS_MAP_SG_ATTRS +#if KERNEL_VERSION(4, 17, 0) <= MODS_KERNEL_VERSION +# define MODS_PCIE_FLR_HAS_ERR #endif +#if defined(CONFIG_ARM64) && KERNEL_VERSION(5, 10, 0) <= MODS_KERNEL_VERSION +# define MODS_HAS_ARM64_READ_FTR_REG 1 +#endif #endif /* _MODS_CONFIG_H_ */ diff --git a/drivers/misc/mods/mods_internal.h b/drivers/misc/mods/mods_internal.h index 9c640d13..5b9d7b5a 100644 --- a/drivers/misc/mods/mods_internal.h +++ b/drivers/misc/mods/mods_internal.h @@ -32,8 +32,10 @@ #include "mods_config.h" #include "mods.h" -#ifdef MODS_HAS_SET_MEMORY_HEADER +#ifdef MODS_HAS_ASM_SET_MEMORY_HEADER #include +#elif defined(MODS_HAS_SET_MEMORY_HEADER) +#include #endif #ifndef true @@ -473,6 +475,11 @@ int esc_mods_iommu_dma_unmap_memory(struct mods_client *client, int esc_mods_memory_barrier(struct mods_client *client); #endif +#ifdef CONFIG_ARM64 +int esc_mods_flush_cpu_cache_range(struct mods_client *client, + struct MODS_FLUSH_CPU_CACHE_RANGE *p); +#endif + #if defined(CONFIG_PPC64) /* ppc64 */ int esc_mods_set_ppc_tce_bypass(struct mods_client *client, @@ -614,8 +621,6 @@ int esc_mods_reset_assert(struct mods_client *client, struct MODS_RESET_HANDLE *p); int esc_mods_get_rst_handle(struct mods_client *client, struct MODS_GET_RESET_HANDLE *p); -int esc_mods_flush_cpu_cache_range(struct mods_client *client, - struct MODS_FLUSH_CPU_CACHE_RANGE *p); int esc_mods_dma_alloc_coherent(struct mods_client *client, struct MODS_DMA_COHERENT_MEM_HANDLE *p); int esc_mods_dma_free_coherent(struct mods_client *client, diff --git a/drivers/misc/mods/mods_krnl.c b/drivers/misc/mods/mods_krnl.c index 4e295e8a..7aa6e281 100644 --- a/drivers/misc/mods/mods_krnl.c +++ b/drivers/misc/mods/mods_krnl.c @@ -2360,11 +2360,6 @@ static long mods_krnl_ioctl(struct file *fp, esc_mods_bpmp_init_pcie_ep_pll, MODS_INIT_PCIE_EP_PLL); break; - case MODS_ESC_FLUSH_CPU_CACHE_RANGE: - MODS_IOCTL_NORETVAL(MODS_ESC_FLUSH_CPU_CACHE_RANGE, - esc_mods_flush_cpu_cache_range, - MODS_FLUSH_CPU_CACHE_RANGE); - break; case MODS_ESC_DMA_ALLOC_COHERENT: MODS_IOCTL(MODS_ESC_DMA_ALLOC_COHERENT, esc_mods_dma_alloc_coherent, @@ -2443,6 +2438,13 @@ static long mods_krnl_ioctl(struct file *fp, esc_mods_memory_barrier); break; #endif +#ifdef CONFIG_ARM64 + case MODS_ESC_FLUSH_CPU_CACHE_RANGE: + MODS_IOCTL_NORETVAL(MODS_ESC_FLUSH_CPU_CACHE_RANGE, + esc_mods_flush_cpu_cache_range, + MODS_FLUSH_CPU_CACHE_RANGE); + break; +#endif #ifdef CONFIG_ARCH_TEGRA case MODS_ESC_DMABUF_GET_PHYSICAL_ADDRESS: MODS_IOCTL(MODS_ESC_DMABUF_GET_PHYSICAL_ADDRESS, diff --git a/drivers/misc/mods/mods_mem.c b/drivers/misc/mods/mods_mem.c index bd3a8e13..34569c26 100644 --- a/drivers/misc/mods/mods_mem.c +++ b/drivers/misc/mods/mods_mem.c @@ -2,7 +2,7 @@ /* * mods_mem.c - This file is part of NVIDIA MODS kernel driver. * - * Copyright (c) 2008-2020, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2008-2021, NVIDIA CORPORATION. All rights reserved. * * NVIDIA MODS kernel driver is free software: you can redistribute it and/or * modify it under the terms of the GNU General Public License, @@ -27,6 +27,10 @@ #include #endif +#ifdef CONFIG_ARCH64 +#include +#endif + static int mods_post_alloc(struct mods_client *client, struct MODS_PHYS_CHUNK *chunk, u64 phys_addr, @@ -2232,26 +2236,42 @@ failed: LOG_EXT(); return err; } +#endif +#ifdef CONFIG_ARM64 static void clear_contiguous_cache(struct mods_client *client, u64 virt_start, u64 phys_start, u32 size) { +#ifdef CONFIG_ARCH_TEGRA + __flush_dcache_area((void *)(size_t)(virt_start), size); +#else + /* __flush_dcache_area is not exported in upstream kernels */ + u64 end = virt_start + size; + u64 cur; + u32 d_line_shift = 4; /* Fallback for kernel 5.9 or older */ + u64 d_size; + +#ifdef MODS_HAS_ARM64_READ_FTR_REG + { + const u64 ctr_el0 = read_sanitised_ftr_reg(SYS_CTR_EL0); + + d_line_shift = cpuid_feature_extract_unsigned_field(ctr_el0, + CTR_DMINLINE_SHIFT); + } +#endif + + d_size = 4 << d_line_shift; + cur = virt_start & ~(d_size - 1); + do { + asm volatile("dc civac, %0" : : "r" (cur) : "memory"); + } while (cur += d_size, cur < end); +#endif + cl_debug(DEBUG_MEM_DETAILED, "clear cache virt 0x%llx phys 0x%llx size 0x%x\n", virt_start, phys_start, size); - -#ifdef CONFIG_ARM64 - /* Flush L1 cache */ - __flush_dcache_area((void *)(size_t)(virt_start), size); -#else - /* Flush L1 cache */ - __cpuc_flush_dcache_area((void *)(size_t)(virt_start), size); - - /* Now flush L2 cache. */ - outer_flush_range(phys_start, phys_start + size); -#endif } static void clear_entry_cache_mappings(struct mods_client *client, @@ -2341,10 +2361,15 @@ int esc_mods_flush_cpu_cache_range(struct mods_client *client, LOG_ENT(); if (irqs_disabled() || in_interrupt() || - p->virt_addr_start > p->virt_addr_end || - p->flags == MODS_INVALIDATE_CPU_CACHE) { + p->virt_addr_start > p->virt_addr_end) { - cl_debug(DEBUG_MEM_DETAILED, "cannot clear cache\n"); + cl_debug(DEBUG_MEM_DETAILED, "cannot flush cache\n"); + LOG_EXT(); + return -EINVAL; + } + + if (p->flags == MODS_INVALIDATE_CPU_CACHE) { + cl_debug(DEBUG_MEM_DETAILED, "cannot invalidate cache\n"); LOG_EXT(); return -EINVAL; } @@ -2397,7 +2422,6 @@ int esc_mods_flush_cpu_cache_range(struct mods_client *client, LOG_EXT(); return OK; } - #endif static int mods_post_alloc(struct mods_client *client, @@ -2452,8 +2476,7 @@ static int mods_post_alloc(struct mods_client *client, /* On systems with SWIOTLB active, disable default DMA mapping * because we don't support scatter-gather lists. */ -#if defined(CONFIG_SWIOTLB) && defined(MODS_HAS_DMA_OPS) && \ - defined(MODS_HAS_MAP_SG_ATTRS) +#if defined(CONFIG_SWIOTLB) && defined(MODS_HAS_DMA_OPS) const struct dma_map_ops *ops = get_dma_ops(&dev->dev); if (ops->map_sg == swiotlb_map_sg_attrs) diff --git a/drivers/misc/mods/mods_pci.c b/drivers/misc/mods/mods_pci.c index 4012bd91..7140fac8 100644 --- a/drivers/misc/mods/mods_pci.c +++ b/drivers/misc/mods/mods_pci.c @@ -2,7 +2,7 @@ /* * mods_pci.c - This file is part of NVIDIA MODS kernel driver. * - * Copyright (c) 2008-2020, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2008-2021, NVIDIA CORPORATION. All rights reserved. * * NVIDIA MODS kernel driver is free software: you can redistribute it and/or * modify it under the terms of the GNU General Public License, @@ -820,8 +820,7 @@ int esc_mods_get_iommu_state_2(struct mods_client *client, { #if !defined(CONFIG_SWIOTLB) state->state = MODS_SWIOTLB_DISABLED; -#elif defined(MODS_HAS_DMA_OPS) && \ - (defined(MODS_HAS_NONCOH_DMA_OPS) || defined(MODS_HAS_MAP_SG_ATTRS)) +#elif defined(MODS_HAS_DMA_OPS) const struct dma_map_ops *ops; struct pci_dev *dev; @@ -837,22 +836,14 @@ int esc_mods_get_iommu_state_2(struct mods_client *client, ops = get_dma_ops(&dev->dev); -#if defined(MODS_HAS_NONCOH_DMA_OPS) - state->state = (ops != &noncoherent_swiotlb_dma_ops && - ops != &coherent_swiotlb_dma_ops) - ? MODS_SWIOTLB_DISABLED : MODS_SWIOTLB_ACTIVE; -#else state->state = ops->map_sg != swiotlb_map_sg_attrs ? MODS_SWIOTLB_DISABLED : MODS_SWIOTLB_ACTIVE; -#endif + pci_dev_put(dev); LOG_EXT(); -#elif defined(CONFIG_PPC64) || defined(CONFIG_ARM64) - /* No way to detect, assume SW I/O TLB is disabled on ppc64/arm64 */ - state->state = MODS_SWIOTLB_DISABLED; #else - /* No way to detect on old kernel */ + /* No way to detect it */ state->state = MODS_SWIOTLB_INDETERMINATE; #endif return OK;