From 4cf8c8066974d494c498747bd901a4c07c6a8cd5 Mon Sep 17 00:00:00 2001 From: Laxman Dewangan Date: Tue, 11 Apr 2023 05:47:21 +0000 Subject: [PATCH] nvmap: Copy drivers and headers from kernel/nvidia Copy the driver and header sources of the nvmap to kernel/nvidia-oot from kernel/nvidia as part of removing the dependency of kernel/nvidia for OOT drivers. The latest (few) git history of the files copied are b7a355916 video: tegra: nvmap: Fix type casting issue 2128c5433 video: tegra: nvmap: Fix type casting issues 0cd082559 video: tegra: nvmap: Change peer vm id data type 4bd7ece67 tegra: nvmap: mark ivm carveout pages occupied e86f3630a video: tegra: nvmap: Fix type casting issue c43a23e58 video: tegra: nvmap: Fix type casting issue ca1dda22e video: tegra: nvmap: Fix type casting issue 1f567abfe video: tegra: nvmap: Fix wrap up condition 29db4d31c video: tegra: nvmap: Remove unnecessary debugfs fe72f1413 video: tegra: nvmap: Remove get_drv_data() call 3b0fc79e7 video: tegra: nvmap: Fix coverity defect 3cc0ce41b video: tegra: nvmap: Fix coverity defect 6da39e966 video: tegra: nvmap: Fix WARN_ON condition a16351ff1 video: tegra: nvmap: Remove dead code 9993f2d2d video: tegra: nvmap: Update print level 6066a2077 video: tegra: nvmap: Remove nvmap_debug_lru_allocations_show 3cdf2b7ba video: tegra: nvmap: Add kernel version check 716ded4fc video: tegra: nvmap: Initialize the return value 9b6c1b4ab video: tegra: nvmap: Correct debugfs code 33e70118b video: tegra: nvmap: Fix Cert-C error handling bug 7b960ed79 video: tegra: nvmap: Fix Cert-C error handling bug 945dc1471 video: tegra: nvmap: Fix Cert-C error handling bug 31e572de2 video: tegra: nvmap: Fix Cert-C error handling bug 1f25cbf68 video: tegra: nvmap: Fix Cert-C error handling bug fa5428107 video: tegra: nvmap: Remove nvmap_handle_get_from_fd df73f2208 video: tegra: nvmap: Protect kmap/kunmap code 9842e7c6a video: tegra: nvmap: Remove t19x dma_buf map/unmap 06dff1a8d video: tegra: nvmap: Remove unnecessary export symbols 6f097f86b video: tegra: nvmap: Fix Cert-C error handling bug f14171608 video: tegra: nvmap: load nvmap for T23x compatible platforms 266812814 video: tegra: nvmap: Get rid of NVMAP_CONFIG_KSTABLE_KERNEL 1b38c0887 nvmap: Don't use NV_BUILD_KERNEL_OPTIONS 0ab8dc032 video: tegra: nvmap: Reintroduce NVMAP_CONFIG_VPR_RESIZE cc8db9797 driver: platform: tegra: Separate out vpr code 28955d95c video/tegra: nvmap: Enable build as OOT module 876d1fbb8 video: tegra: nvmap: Remove IS_ENABLED check 5ea30867a nvmap: Add support to build as module from OOT kernel a71ad020e video: tegra: nvmap: Protect tegra_vpr args under config e70061cc1 video: tegra: nvmap: Do not export cvnas_dev d2a26ff36 video: tegra: nvmap: Include missing header 692e4f682 video: tegra: nvmap: Update page coloring algo 2b9dbb911 video: tegra: nvmap: Check for return value de8de12b6 video: tegra: nvmap: Enable legacy init support 65d478158 video: tegra: nvmap: Remove dependency of cvnas 38bdd6f05 video: tegra: nvmap: Make nvmap as loadable module 9668e410b video: tegra: nvmap: Enable handle as ID 11c6cbd23 tegra: nvmap: Fix build for Linux v5.18 fbd95c3ab linux: nvmap: change ivm_handle to u32 eb1e2c302 video: tegra: nvmap: Fix NVSCIIPC support 022689b29 tegra: nvmap: return error if handle as ID enabled but id is fd 19e5106ed video: tegra: nvmap: Don't treat ivm as reserved mem carveouts Bug 4038415 Change-Id: I7108aec3b8532fe79c9423c2835744b1213719e8 Signed-off-by: Laxman Dewangan --- drivers/video/tegra/nvmap/Makefile | 54 +- .../video/tegra/nvmap/Makefile.memory.configs | 221 +++ .../tegra/nvmap/include/linux/nvmap_exports.h | 23 + drivers/video/tegra/nvmap/nvmap_alloc.c | 1150 ++++++++++++ drivers/video/tegra/nvmap/nvmap_cache.c | 431 +++++ drivers/video/tegra/nvmap/nvmap_cache_maint.S | 236 +++ drivers/video/tegra/nvmap/nvmap_carveout.c | 270 +++ drivers/video/tegra/nvmap/nvmap_core.c | 355 ++++ drivers/video/tegra/nvmap/nvmap_dev.c | 1557 +++++++++++++++++ drivers/video/tegra/nvmap/nvmap_dmabuf.c | 666 +++++++ drivers/video/tegra/nvmap/nvmap_fault.c | 261 +++ drivers/video/tegra/nvmap/nvmap_handle.c | 511 ++++++ drivers/video/tegra/nvmap/nvmap_heap.c | 607 +++++++ drivers/video/tegra/nvmap/nvmap_heap.h | 79 + drivers/video/tegra/nvmap/nvmap_id_array.c | 62 + drivers/video/tegra/nvmap/nvmap_init.c | 979 +++++++++++ drivers/video/tegra/nvmap/nvmap_init_t19x.c | 62 + drivers/video/tegra/nvmap/nvmap_ioctl.c | 1434 +++++++++++++++ drivers/video/tegra/nvmap/nvmap_ioctl.h | 79 + .../video/tegra/nvmap/nvmap_kasan_wrapper.c | 46 + drivers/video/tegra/nvmap/nvmap_mm.c | 84 + drivers/video/tegra/nvmap/nvmap_pp.c | 774 ++++++++ drivers/video/tegra/nvmap/nvmap_priv.h | 904 ++++++++++ drivers/video/tegra/nvmap/nvmap_sci_ipc.c | 365 ++++ drivers/video/tegra/nvmap/nvmap_sci_ipc.h | 37 + drivers/video/tegra/nvmap/nvmap_stats.c | 106 ++ drivers/video/tegra/nvmap/nvmap_stats.h | 45 + drivers/video/tegra/nvmap/nvmap_tag.c | 113 ++ include/linux/nvmap.h | 113 ++ include/linux/nvmap_t19x.h | 38 + include/trace/events/nvmap.h | 870 +++++++++ include/uapi/linux/nvmap.h | 366 ++++ 32 files changed, 12890 insertions(+), 8 deletions(-) create mode 100644 drivers/video/tegra/nvmap/Makefile.memory.configs create mode 100644 drivers/video/tegra/nvmap/include/linux/nvmap_exports.h create mode 100644 drivers/video/tegra/nvmap/nvmap_alloc.c create mode 100644 drivers/video/tegra/nvmap/nvmap_cache.c create mode 100644 drivers/video/tegra/nvmap/nvmap_cache_maint.S create mode 100644 drivers/video/tegra/nvmap/nvmap_carveout.c create mode 100644 drivers/video/tegra/nvmap/nvmap_core.c create mode 100644 drivers/video/tegra/nvmap/nvmap_dev.c create mode 100644 drivers/video/tegra/nvmap/nvmap_dmabuf.c create mode 100644 drivers/video/tegra/nvmap/nvmap_fault.c create mode 100644 drivers/video/tegra/nvmap/nvmap_handle.c create mode 100644 drivers/video/tegra/nvmap/nvmap_heap.c create mode 100644 drivers/video/tegra/nvmap/nvmap_heap.h create mode 100644 drivers/video/tegra/nvmap/nvmap_id_array.c create mode 100644 drivers/video/tegra/nvmap/nvmap_init.c create mode 100644 drivers/video/tegra/nvmap/nvmap_init_t19x.c create mode 100644 drivers/video/tegra/nvmap/nvmap_ioctl.c create mode 100644 drivers/video/tegra/nvmap/nvmap_ioctl.h create mode 100644 drivers/video/tegra/nvmap/nvmap_kasan_wrapper.c create mode 100644 drivers/video/tegra/nvmap/nvmap_mm.c create mode 100644 drivers/video/tegra/nvmap/nvmap_pp.c create mode 100644 drivers/video/tegra/nvmap/nvmap_priv.h create mode 100644 drivers/video/tegra/nvmap/nvmap_sci_ipc.c create mode 100644 drivers/video/tegra/nvmap/nvmap_sci_ipc.h create mode 100644 drivers/video/tegra/nvmap/nvmap_stats.c create mode 100644 drivers/video/tegra/nvmap/nvmap_stats.h create mode 100644 drivers/video/tegra/nvmap/nvmap_tag.c create mode 100644 include/linux/nvmap.h create mode 100644 include/linux/nvmap_t19x.h create mode 100644 include/trace/events/nvmap.h create mode 100644 include/uapi/linux/nvmap.h diff --git a/drivers/video/tegra/nvmap/Makefile b/drivers/video/tegra/nvmap/Makefile index d18d535d..584536bf 100644 --- a/drivers/video/tegra/nvmap/Makefile +++ b/drivers/video/tegra/nvmap/Makefile @@ -1,9 +1,47 @@ -# SPDX-License-Identifier: GPL-2.0 -# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +GCOV_PROFILE := y -# NOTE: Do not change or add anything in this makefile. -# The source code and makefile rules are copied from the -# kernel/nvidia/drivers/video/tegra/nvmap. This file is -# just place-holder for empty makefile to avoid any build -# issue when copy is not done from command line and building -# the tree independent of source copy. +subdir-ccflags-y := -Werror + +include $(srctree.nvidia)/drivers/video/tegra/nvmap/Makefile.memory.configs + +ccflags-y += -I$(srctree.nvidia)/include/ + +ifeq ($(NVMAP_CONFIG), y) +# Build NvMap only when NVMAP_CONFIG is set to y +nvmap-y := nvmap_core.o \ + nvmap_alloc.o \ + nvmap_cache.o \ + nvmap_dev.o \ + nvmap_dmabuf.o \ + nvmap_fault.o \ + nvmap_handle.o \ + nvmap_heap.o \ + nvmap_ioctl.o \ + nvmap_init.o \ + nvmap_tag.o \ + nvmap_mm.o \ + nvmap_stats.o \ + nvmap_carveout.o \ + nvmap_kasan_wrapper.o +nvmap-$(NVMAP_CONFIG_HANDLE_AS_ID) += nvmap_id_array.o + +nvmap-$(NVMAP_CONFIG_SCIIPC) += nvmap_sci_ipc.o + +ifeq ($(NVMAP_CONFIG_PAGE_POOLS), y) +nvmap-y += nvmap_pp.o +endif #NVMAP_CONFIG_PAGE_POOLS + +ifneq ($(filter y,$(CONFIG_ARCH_TEGRA_19x_SOC) $(CONFIG_ARCH_TEGRA_194_SOC)),) +nvmap-y += nvmap_init_t19x.o +endif #CONFIG_ARCH_TEGRA_19x_SOC or CONFIG_ARCH_TEGRA_194_SOC + +KASAN_SANITIZE_nvmap_kasan_wrapper.o := n + +ifeq ($(NVMAP_CONFIG_LOADABLE_MODULE), y) +nvmap-y += nvmap_cache_maint.o +obj-m += nvmap.o +else +# Not a loadable module +obj-y += nvmap.o +endif #NVMAP_CONFIG_LOADABLE_MODULE +endif #NVMAP_CONFIG diff --git a/drivers/video/tegra/nvmap/Makefile.memory.configs b/drivers/video/tegra/nvmap/Makefile.memory.configs new file mode 100644 index 00000000..2eb230b0 --- /dev/null +++ b/drivers/video/tegra/nvmap/Makefile.memory.configs @@ -0,0 +1,221 @@ +# Copyright (c) 2021-2022, NVIDIA CORPORATION. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms and conditions of the GNU General Public License, +# version 2, as published by the Free Software Foundation. +# +# This program is distributed in the hope it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# + +# This file consists of 4 sections +# Section 1: This section is for doing prerequisite check. +# It checkes whether the prereq configs are enabled or not. +# If not, return error. +# +# Section 2: This section declare all configs with the default values +# just similar to Kconfig +# +# Section 3: This section consists of checks for kernel versions and +# actual values of these configs for corresponding kernel +# version. This is the place where we can enable/disable or +# set values to configs as per kernel version. +# +# Section 4: This section creates ccflags based upon the values specified +# in the section 2. These ccflags would be actually used in the +# source files. This section also takes care of the dependencies +# between the configs. +# + +################################################################################ +ifeq ($(CONFIG_ARCH_TEGRA), y) +# Section 1 +# For dma_buf support CONFIG_DMA_SHARED_BUFFER needs be enabled +ifneq ($(CONFIG_DMA_SHARED_BUFFER),y) +$(error CONFIG_DMA_SHARED_BUFFER is not set) +endif + +################################################################################ +# Section 2 +# This config is used to include the memory management driver for the +# Tegra GPU, multimedia and display subsystems +NVMAP_CONFIG := y + +# Config to reduce the alloction overhead, which is significant +# for uncached, writecombine and inner cacheable memories as it +# involves changing page attributes during every allocation per page +# and flushing cache. Alloc time is reduced by allcoating the pages +# ahead and keeping them aside. The reserved pages would be released +# when system is low on memory and acquired back during release of +# memory. +NVMAP_CONFIG_PAGE_POOLS := y + +# Config to include some debugging info in the page pools. This +# adds a bit of unnecessary overhead so only enable this is you +# suspect there is an issue with the nvmap page pools. +NVMAP_CONFIG_PAGE_POOL_DEBUG := n + +# Config for page pool size in pages +NVMAP_CONFIG_PAGE_POOL_SIZE := 0x0 + +# Config to enable page coloring +# Page coloring rearranges the pages allocated based on the color +# of the page. It can improve memory access performance. +# The coloring option enable can optionally overallocate a portion of +# reqeusted allcoation size to improve the probabilty of better +# page coloring. If unsure, say Y. +NVMAP_CONFIG_COLOR_PAGES := y + +# Config for FD number to start allocation from +# NvMap handles are represented with FD's in the user processes. +# To avoid Linux FD usage limitations, NvMap allocates FD starting +# from this number. +NVMAP_CONFIG_FD_START := 0x400 + +# Config for enabling deferred FD recycle +# A released nvmap handle would release memory and FD. This FD +# can be reused immediately for subsequent nvmap allocation req in +# the same process. Any buggy code in client process that continues to +# use FD of released allocation would continue to use new allocation +# and can lead to undesired consequences, which can be hard to debug. +# Enabling this option would defer recycling FD for longer time and +# allows debugging incorrect FD references by clients by returning errors +# for the accesses that occur after handle/FD release. +NVMAP_CONFIG_DEFER_FD_RECYCLE := n + +# Config for FD number to start free FD recycle +# Once last allocated FD reaches this number, allocation of subsequent +# FD's start from NVMAP_START_FD. +NVMAP_CONFIG_DEFER_FD_RECYCLE_MAX_FD := 0x8000 + +# Config for enabling nvmap mapping with SciIpc secure buffer sharing +# Enable nvmap mapping with SciIpc secure buffer sharing. +# Supports nvmap ioctls to get Unique SciIpcId and attach +# it with nvmap_handle. +# Suppports getting nvmap_handle from SciIpcId passed via ioctl. +NVMAP_CONFIG_SCIIPC := n + +# Config for enabling NvMap as OOT module +NVMAP_CONFIG_LOADABLE_MODULE := n + +# Config for enabling PROCRANK functionality +NVMAP_CONFIG_PROCRANK := y + +# Config for enabling VPR resize functionality +NVMAP_CONFIG_VPR_RESIZE := n + +# Config for enabling few debugfs which would impact the NvMap performance +# There are few debugfs which would impact NvMap performance. +# Disable this when perf regression is observed. +NVMAP_CONFIG_DEBUG_MAPS := n + +# This is fallback option to support handle as FD +# To support handle as ID, set this to n +# This config is useful to debug issue if its due to handle as ID or FD +NVMAP_CONFIG_HANDLE_AS_FD := n + +# Config for kstable/OOT kernel +# This is useful when any kstable/OOT specific checks are needed +NVMAP_CONFIG_UPSTREAM_KERNEL := n +################################################################################ +# Section 3 +# Enable/Disable configs based upon the kernel version +# Specify the values which are different from the default values + +ifdef CONFIG_TEGRA_VPR +# For 4.9 +NVMAP_CONFIG_VPR_RESIZE := y +else +# For 5.10+ +NVMAP_CONFIG_LOADABLE_MODULE := y +NVMAP_CONFIG_PROCRANK := n +ifneq ($(NVMAP_CONFIG_HANDLE_AS_FD),y) +NVMAP_CONFIG_HANDLE_AS_ID := y +NVMAP_CONFIG_FD_START := 0x0 +endif +NVMAP_CONFIG_SCIIPC := y +# For OOT build +ifeq ($(CONFIG_TEGRA_OOT_MODULE),m) +NVMAP_CONFIG_UPSTREAM_KERNEL := y +NVMAP_CONFIG_COLOR_PAGES := n +endif +endif +################################################################################ +# Section 4 +# This section creates ccflags based upon the values specified +# in the section 2. These ccflags would be actually used in the +# source files. + +ifeq ($(NVMAP_CONFIG),y) +# All other flags make sense only when NVMAP_CONFIG is enabled +ccflags-y += -DNVMAP_CONFIG + +ifeq ($(NVMAP_CONFIG_PAGE_POOLS),y) +ccflags-y += -DNVMAP_CONFIG_PAGE_POOLS + +# NVMAP_CONFIG_PAGE_POOL_DEBUG depends upon NVMAP_CONFIG_PAGE_POOLS +ifeq ($(NVMAP_CONFIG_PAGE_POOL_DEBUG),y) +ccflags-y += -DNVMAP_CONFIG_PAGE_POOL_DEBUG +endif #NVMAP_CONFIG_PAGE_POOL_DEBUG + +# NVMAP_CONFIG_PAGE_POOL_SIZE depends upon NVMAP_CONFIG_PAGE_POOLS +ifdef NVMAP_CONFIG_PAGE_POOL_SIZE +ccflags-y += -DNVMAP_CONFIG_PAGE_POOL_SIZE=${NVMAP_CONFIG_PAGE_POOL_SIZE} +endif #NVMAP_CONFIG_PAGE_POOL_SIZE +endif #NVMAP_CONFIG_PAGE_POOLS + +# NVMAP_CONFIG_COLOR_PAGES depends upon CONFIG_ARM64_4K_PAGES +ifeq ($(CONFIG_ARM64_4K_PAGES),y) +ifeq ($(NVMAP_CONFIG_COLOR_PAGES),y) +ccflags-y += -DNVMAP_CONFIG_COLOR_PAGES +endif #CONFIG_ARM64_4K_PAGES +endif #NVMAP_CONFIG_COLOR_PAGES + +ifdef NVMAP_CONFIG_FD_START +ccflags-y += -DNVMAP_CONFIG_FD_START=${NVMAP_CONFIG_FD_START} +endif #NVMAP_CONFIG_FD_START + +ifeq ($(NVMAP_CONFIG_DEFER_FD_RECYCLE),y) +ccflags-y += -DNVMAP_CONFIG_DEFER_FD_RECYCLE +# NVMAP_CONFIG_DEFER_FD_RECYCLE_MAX_FD depends upon CONFIG_NVMAP_DEFER_FD_RECYCLE +ifdef NVMAP_CONFIG_DEFER_FD_RECYCLE_MAX_FD +ccflags-y += -DNVMAP_CONFIG_DEFER_FD_RECYCLE_MAX_FD=${NVMAP_CONFIG_DEFER_FD_RECYCLE_MAX_FD} +endif #NVMAP_CONFIG_DEFER_FD_RECYCLE_MAX_FD +endif #NVMAP_CONFIG_DEFER_FD_RECYCLE + +ifeq ($(NVMAP_CONFIG_SCIIPC),y) +ccflags-y += -DNVMAP_CONFIG_SCIIPC +endif #NVMAP_CONFIG_SCIIPC + +ifeq ($(NVMAP_CONFIG_VPR_RESIZE),y) +ccflags-y += -DNVMAP_CONFIG_VPR_RESIZE +endif #NVMAP_CONFIG_VPR_RESIZE + +ifeq ($(NVMAP_CONFIG_LOADABLE_MODULE),y) +ccflags-y += -DNVMAP_LOADABLE_MODULE +endif #NVMAP_CONFIG_LOADABLE_MODULE + +ifeq ($(NVMAP_CONFIG_UPSTREAM_KERNEL),y) +ccflags-y += -DNVMAP_UPSTREAM_KERNEL +endif #NVMAP_CONFIG_UPSTREAM_KERNEL + +ifeq ($(NVMAP_CONFIG_PROCRANK),y) +ccflags-y += -DNVMAP_CONFIG_PROCRANK +endif #NVMAP_CONFIG_PROCRANK + +ifeq ($(NVMAP_CONFIG_DEBUG_MAPS),y) +ccflags-y += -DNVMAP_CONFIG_DEBUG_MAPS +endif #NVMAP_CONFIG_DEBUG_MAPS + +ifeq ($(NVMAP_CONFIG_HANDLE_AS_ID),y) +ccflags-y += -DNVMAP_CONFIG_HANDLE_AS_ID +endif #NVMAP_CONFIG_HANDLE_AS_ID + +ifeq ($(CONFIG_TEGRA_CVNAS),y) +ccflags-y += -DCVNAS_BUILTIN +endif #CONFIG_TEGRA_CVNAS + +endif #NVMAP_CONFIG +endif #CONFIG_ARCH_TEGRA diff --git a/drivers/video/tegra/nvmap/include/linux/nvmap_exports.h b/drivers/video/tegra/nvmap/include/linux/nvmap_exports.h new file mode 100644 index 00000000..4676a993 --- /dev/null +++ b/drivers/video/tegra/nvmap/include/linux/nvmap_exports.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + */ +#ifndef __NVMAP_EXPORTS_H +#define __NVMAP_EXPORTS_H + +void *nvmap_dma_alloc_attrs(struct device *dev, size_t size, + dma_addr_t *dma_handle, gfp_t flag, + unsigned long attrs); +void nvmap_dma_free_attrs(struct device *dev, size_t size, void *cpu_addr, + dma_addr_t dma_handle, unsigned long attrs); +extern struct device tegra_vpr_dev; +#endif /* __NVMAP_EXPORTS_H */ diff --git a/drivers/video/tegra/nvmap/nvmap_alloc.c b/drivers/video/tegra/nvmap/nvmap_alloc.c new file mode 100644 index 00000000..46b3dd33 --- /dev/null +++ b/drivers/video/tegra/nvmap/nvmap_alloc.c @@ -0,0 +1,1150 @@ +/* + * drivers/video/tegra/nvmap/nvmap_alloc.c + * + * Handle allocation and freeing routines for nvmap + * + * Copyright (c) 2011-2022, NVIDIA CORPORATION. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + */ + +#define pr_fmt(fmt) "%s: " fmt, __func__ + +#include +#include +#include +#include +#if KERNEL_VERSION(4, 15, 0) > LINUX_VERSION_CODE +#include +#else +#include +#endif +#include + +#ifndef NVMAP_LOADABLE_MODULE +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0) +#include +#endif +#endif /* !NVMAP_LOADABLE_MODULE */ + +#include "nvmap_priv.h" + +bool nvmap_convert_carveout_to_iovmm; +bool nvmap_convert_iovmm_to_carveout; + +u32 nvmap_max_handle_count; +u64 nvmap_big_page_allocs; +u64 nvmap_total_page_allocs; + +/* handles may be arbitrarily large (16+MiB), and any handle allocated from + * the kernel (i.e., not a carveout handle) includes its array of pages. to + * preserve kmalloc space, if the array of pages exceeds PAGELIST_VMALLOC_MIN, + * the array is allocated using vmalloc. */ +#define PAGELIST_VMALLOC_MIN (PAGE_SIZE) + +void *nvmap_altalloc(size_t len) +{ + if (len > PAGELIST_VMALLOC_MIN) + return vzalloc(len); + else + return kzalloc(len, GFP_KERNEL); +} + +void nvmap_altfree(void *ptr, size_t len) +{ + if (!ptr) + return; + + if (len > PAGELIST_VMALLOC_MIN) + vfree(ptr); + else + kfree(ptr); +} + +static struct page *nvmap_alloc_pages_exact(gfp_t gfp, size_t size) +{ + struct page *page, *p, *e; + unsigned int order; + + order = get_order(size); + page = alloc_pages(gfp, order); + + if (!page) + return NULL; + + split_page(page, order); + e = nth_page(page, (1 << order)); + for (p = nth_page(page, (size >> PAGE_SHIFT)); p < e; p++) + __free_page(p); + + return page; +} + +static uint s_nr_colors = 1; +module_param_named(nr_colors, s_nr_colors, uint, 0644); + +#define NVMAP_MAX_COLORS 16 + +struct color_list { + u32 *counts; + u32 *heads; + u32 *list; + struct page **pages; + u32 page_count; + u32 length; +}; + +static struct color_list *alloc_color_list(u32 nr_pages, u32 nr_colors) +{ + struct color_list *list; + u32 *temp = NULL; + u32 nr_u32; + + list = kzalloc(sizeof(struct color_list), GFP_KERNEL); + if (!list) + return NULL; + + list->pages = vmalloc(nr_pages * sizeof(struct page *)); + if (!list->pages) { + kfree(list); + return NULL; + } + + /* Allocate counts, heads, and list with a single allocation */ + nr_u32 = nr_pages + 2 * nr_colors; + temp = vmalloc(nr_u32 * sizeof(u32)); + if (!temp) + goto fail; + + memset(&temp[0], 0, 2 * nr_colors * sizeof(u32)); + list->counts = &temp[0]; + list->heads = &temp[nr_colors]; + list->list = &temp[2 * nr_colors]; + + list->page_count = nr_pages; + + return list; +fail: + if (list->pages) + vfree(list->pages); + kfree(list); + return NULL; +} + +static void free_color_list(struct color_list *list) +{ + vfree(list->pages); + vfree(list->counts); /* Frees counts, heads, and list */ + kfree(list); +} + +static struct page *list_pop_page(struct color_list *list, u32 color, char *who) +{ + u32 i; + + /* Debug check */ + if ((list->counts[color] == 0) || (list->counts[color] > 1 << 31)) { + pr_err("list_pop_page: OVER FREE!\n"); + pr_err(" called from: %s\n", who); + for (i = 0; i < s_nr_colors; i++) + pr_err(" color = %d: %d\n", i, list->counts[i]); + BUG(); + } + i = list->heads[color]; + list->heads[color] = list->list[i]; + list->counts[color]--; + return list->pages[i]; +} + +struct nvmap_alloc_state { + u32 nr_colors; + u32 (*addr_to_color)(uintptr_t phys); + u32 tile; + u32 output_count; + u32 nr_pages; + u32 max_color_per_tile; + struct color_list *list; +}; + +#define CHANNEL_MASK_0 0x27af5200 +#define CHANNEL_MASK_1 0x563ca400 +#define CHANNEL_MASK_2 0x3f264800 +#define CHANNEL_MASK_3 0xe2443000 +#define BANK_MASK_0 0x5ca78400 +#define BANK_MASK_1 0xe5724800 +#define BANK_MASK_2 0x973bb000 + +#define BIT_N(a, n) \ + (((a) >> (n)) & 1) + +#define BITS_XOR_9_TO_31(a) \ + (BIT_N((a), 9) ^ BIT_N((a), 10) ^ BIT_N((a), 11) ^ BIT_N((a), 12) ^ \ + BIT_N((a), 13) ^ BIT_N((a), 14) ^ BIT_N((a), 15) ^ BIT_N((a), 16) ^ \ + BIT_N((a), 17) ^ BIT_N((a), 18) ^ BIT_N((a), 19) ^ BIT_N((a), 20) ^ \ + BIT_N((a), 21) ^ BIT_N((a), 22) ^ BIT_N((a), 23) ^ BIT_N((a), 24) ^ \ + BIT_N((a), 25) ^ BIT_N((a), 26) ^ BIT_N((a), 27) ^ BIT_N((a), 28) ^ \ + BIT_N((a), 29) ^ BIT_N((a), 30) ^ BIT_N((a), 31)) + +#define BITS_XOR_10_TO_31(a) \ + (BIT_N((a), 10) ^ BIT_N((a), 11) ^ BIT_N((a), 12) ^ \ + BIT_N((a), 13) ^ BIT_N((a), 14) ^ BIT_N((a), 15) ^ BIT_N((a), 16) ^ \ + BIT_N((a), 17) ^ BIT_N((a), 18) ^ BIT_N((a), 19) ^ BIT_N((a), 20) ^ \ + BIT_N((a), 21) ^ BIT_N((a), 22) ^ BIT_N((a), 23) ^ BIT_N((a), 24) ^ \ + BIT_N((a), 25) ^ BIT_N((a), 26) ^ BIT_N((a), 27) ^ BIT_N((a), 28) ^ \ + BIT_N((a), 29) ^ BIT_N((a), 30) ^ BIT_N((a), 31)) + +static u32 addr_to_color_t19x(uintptr_t phys) +{ + int color, chan, bank; + u32 addr = (u32)phys; + u32 xaddr = (u32)(phys >> 4); + + + chan = (BITS_XOR_9_TO_31(addr & CHANNEL_MASK_0) << 0); + chan |= (BITS_XOR_9_TO_31(addr & CHANNEL_MASK_1) << 1); + chan |= (BITS_XOR_9_TO_31(addr & CHANNEL_MASK_2) << 2); + chan |= (BITS_XOR_9_TO_31(addr & CHANNEL_MASK_3) << 3); + + bank = (BITS_XOR_10_TO_31(xaddr & BANK_MASK_0) << 0); + bank |= (BITS_XOR_10_TO_31(xaddr & BANK_MASK_1) << 1); + bank |= (BITS_XOR_10_TO_31(xaddr & BANK_MASK_2) << 2); + + WARN_ON(chan > 15); + WARN_ON(bank > 7); + /* It is preferable to color pages based on even/odd banks + * as well. To limit the number of colors to 16, bank info + * is not used in page coloring. + */ + color = chan; + + return color; +} + +static struct color_list *init_color_list(struct nvmap_alloc_state *state, + u32 nr_pages) +{ + struct color_list *list; + u32 color, i, page_index = 0; + gfp_t gfp = GFP_NVMAP | __GFP_ZERO; + + list = alloc_color_list(nr_pages, state->nr_colors); + if (!list) + return NULL; + +#ifdef NVMAP_CONFIG_PAGE_POOLS + /* Allocated page from nvmap page pool if possible */ + page_index = nvmap_page_pool_alloc_lots(&nvmap_dev->pool, list->pages, nr_pages); +#endif + /* Fall back to general page allocator */ + for (i = page_index; i < nr_pages; i++) { + list->pages[i] = nvmap_alloc_pages_exact(gfp, PAGE_SIZE); + if (!list->pages[i]) + goto fail; + } + /* Clean the cache for any page that didn't come from the page pool */ + if (page_index < nr_pages) + nvmap_clean_cache(&list->pages[page_index], + nr_pages - page_index); + + /* Create linked list of colors and compute the histogram */ + for (i = 0; i < nr_pages; i++) { + color = state->addr_to_color((uintptr_t) + page_to_phys(list->pages[i])); + list->list[i] = list->heads[color]; + list->heads[color] = i; + list->counts[color]++; + } + return list; +fail: + while (i--) + __free_page(list->pages[i]); + free_color_list(list); + return NULL; +} + +static void smooth_pages(struct color_list *list, u32 nr_extra, u32 nr_colors) +{ + u32 i, j, color, max; + u32 counts[NVMAP_MAX_COLORS] = {0}; + + if (nr_extra == 0) + return; + + /* Determine which colors need to be freed */ + for (i = 0; i < nr_extra; i++) { + /* Find the max */ + max = 0; + color = 0; + for (j = 0; j < nr_colors; j++) { + if (list->counts[j] - counts[j] > max) { + color = j; + max = list->counts[j] - counts[j]; + } + } + counts[color]++; + } + + /* Iterate through 0...nr_extra-1 in psuedorandom order */ + do { + /* Pop the max off and free it */ + for (color = 0; color < nr_colors; color++) { + while (counts[color]) { + __free_page(list_pop_page(list, + color, "smooth_pages")); + counts[color]--; + nr_extra--; + } + } + } while (nr_extra > 0); +} + +static void add_perfect(struct nvmap_alloc_state *state, u32 nr_pages, + struct page **out_pages) +{ + u32 i; + u32 color; + struct page *page; + uintptr_t virt_addr; + + + /* create a perfect tile */ + for (i = 0; + i < state->nr_colors && state->output_count < nr_pages; + i++) { + virt_addr = (i + (state->tile * state->nr_colors)) * PAGE_SIZE; + color = state->addr_to_color(virt_addr); + page = list_pop_page(state->list, color, "perfect"); + out_pages[state->output_count++] = page; + } +} + +static void add_imperfect(struct nvmap_alloc_state *state, u32 nr_pages, + struct page **out_pages) +{ + u32 i, j; + u32 max_count; + u32 color; + struct page *page; + uintptr_t virt_addr; + u32 counts[NVMAP_MAX_COLORS] = {0}; + + /* Determine which colors will go into the tile */ + for (i = 0; i < state->nr_colors; i++) { + max_count = 0; + color = 0; + for (j = 0; j < state->nr_colors; j++) { + u32 left = state->list->counts[j] - counts[j]; + + if (left > max_count && + counts[j] < state->max_color_per_tile) { + max_count = left; + color = j; + } + } + counts[color]++; + } + + /* Arrange the colors into the tile */ + for (i = 0; + i < state->nr_colors && state->output_count < nr_pages; + i++) { + virt_addr = (i + (i * state->nr_colors)) * PAGE_SIZE; + color = state->addr_to_color(virt_addr); + /* Find a substitute color */ + if (counts[color] == 0) { + /* Find the color used the most in the tile */ + max_count = 0; + for (j = 0; j < state->nr_colors; j++) { + if (counts[j] > max_count) { + max_count = counts[j]; + color = j; + } + } + } + page = list_pop_page(state->list, color, "imperfect"); + out_pages[state->output_count++] = page; + counts[color]--; + } +} + +static int alloc_colored(u32 nr_pages, + struct page **out_pages, u32 chipid) +{ + struct nvmap_alloc_state state; + u32 nr_alloc, max_count, min_count; + u32 nr_tiles, nr_perfect, nr_imperfect; + int dither_state; + u32 i; + + state.nr_colors = s_nr_colors; + state.addr_to_color = addr_to_color_t19x; + + /* Allocate pages for full 32-page tiles */ + nr_tiles = (nr_pages + state.nr_colors - 1) / state.nr_colors; + /* Overallocate pages by 1/16th */ + nr_alloc = state.nr_colors * nr_tiles; + nr_alloc += nr_alloc >> 4; + + /* Create lists of each page color */ + state.list = init_color_list(&state, nr_alloc); + if (!state.list) + return -ENOMEM; + + /* Smooth out the histogram by freeing over allocated pages */ + smooth_pages(state.list, nr_alloc - state.nr_colors * nr_tiles, + state.nr_colors); + + max_count = 0; + min_count = state.list->counts[0]; + for (i = 0; i < state.nr_colors; i++) { + if (state.list->counts[i] > max_count) + max_count = state.list->counts[i]; + if (state.list->counts[i] < min_count) + min_count = state.list->counts[i]; + } + + /* + * Compute the number of perfect / imperfect tiles and the maximum + * number of pages with the same color can be in a tile + * + * Perfect tile: A tile which consist of one page of each color i.e. 16 pages, + * each of different color + * Imperfect tile: A tile which is not perfect i.e. at least some color will repeat + * max_color_per_tile: How many max times any color can be present in a tile + */ + if (min_count == 0) { + /* + * If there is no page of at least one color, then not a sigle perefect tile can be + * created. The max color pages would need to be distributed equally among all + * tiles. + */ + nr_perfect = 0; + state.max_color_per_tile = max_count / nr_tiles; + if (max_count % nr_tiles != 0) + (state.max_color_per_tile)++; + } else if (min_count == nr_tiles) { + /* + * If pages with each color are at least the number of tiles, then all of the tiles + * can be perfect. + */ + nr_perfect = nr_tiles; + state.max_color_per_tile = 1; + } else { + /* + * Some of the tiles can be perfect and remaining will be imperfect. + * min_count number of perfect tiles can be created, hence the min_count number of + * pages + * having max color would be present in the perfect tiles, The remaining pages would + * be distributed equally among the imperfect tiles. + */ + nr_perfect = min_count; + nr_imperfect = nr_tiles - nr_perfect; + state.max_color_per_tile = (max_count - nr_perfect) / nr_imperfect; + if ((max_count - nr_perfect) % nr_imperfect != 0) + (state.max_color_per_tile)++; + } + nr_imperfect = nr_tiles - nr_perfect; + + /* Output tiles */ + dither_state = nr_perfect - nr_imperfect; + state.output_count = 0; + for (state.tile = 0; state.tile < nr_tiles; state.tile++) { + if (dither_state > 0) { + add_perfect(&state, nr_pages, out_pages); + dither_state -= nr_imperfect; + } else { + add_imperfect(&state, nr_pages, out_pages); + dither_state += nr_perfect; + } + } + + /* Free extra pages created when the buffer does not + * fill the last tile + */ + for (i = 0; i < state.nr_colors; i++) + while (state.list->counts[i] > 0) + __free_page(list_pop_page(state.list, i, "free")); + + free_color_list(state.list); + + return 0; +} + +static int handle_page_alloc(struct nvmap_client *client, + struct nvmap_handle *h, bool contiguous) +{ + size_t size = h->size; + size_t nr_page = size >> PAGE_SHIFT; + int i = 0, page_index = 0, allocated = 0; + struct page **pages; + gfp_t gfp = GFP_NVMAP | __GFP_ZERO; +#ifdef CONFIG_ARM64_4K_PAGES +#ifdef NVMAP_CONFIG_PAGE_POOLS + int pages_per_big_pg = NVMAP_PP_BIG_PAGE_SIZE >> PAGE_SHIFT; +#else + int pages_per_big_pg = 0; +#endif +#endif /* CONFIG_ARM64_4K_PAGES */ +#if KERNEL_VERSION(4, 15, 0) > LINUX_VERSION_CODE + static u32 chipid; +#else + static u8 chipid; +#endif + + if (!chipid) { +#ifdef NVMAP_CONFIG_COLOR_PAGES +#if KERNEL_VERSION(4, 15, 0) > LINUX_VERSION_CODE + chipid = tegra_hidrev_get_chipid(tegra_read_chipid()); +#else + chipid = tegra_get_chip_id(); +#endif + if (chipid == TEGRA194) + s_nr_colors = 16; +#endif + } + + pages = nvmap_altalloc(nr_page * sizeof(*pages)); + if (!pages) + return -ENOMEM; + + if (contiguous) { + struct page *page; + page = nvmap_alloc_pages_exact(gfp, size); + if (!page) + goto fail; + + for (i = 0; i < nr_page; i++) + pages[i] = nth_page(page, i); + + } else { +#ifdef CONFIG_ARM64_4K_PAGES +#ifdef NVMAP_CONFIG_PAGE_POOLS + /* Get as many big pages from the pool as possible. */ + page_index = nvmap_page_pool_alloc_lots_bp(&nvmap_dev->pool, pages, + nr_page); + pages_per_big_pg = nvmap_dev->pool.pages_per_big_pg; +#endif + /* Try to allocate big pages from page allocator */ + for (i = page_index; + i < nr_page && pages_per_big_pg > 1 && (nr_page - i) >= pages_per_big_pg; + i += pages_per_big_pg, page_index += pages_per_big_pg) { + struct page *page; + int idx; + /* + * set the gfp not to trigger direct/kswapd reclaims and + * not to use emergency reserves. + */ + gfp_t gfp_no_reclaim = (gfp | __GFP_NOMEMALLOC) & ~__GFP_RECLAIM; + + page = nvmap_alloc_pages_exact(gfp_no_reclaim, + pages_per_big_pg << PAGE_SHIFT); + if (!page) + break; + + for (idx = 0; idx < pages_per_big_pg; idx++) + pages[i + idx] = nth_page(page, idx); + nvmap_clean_cache(&pages[i], pages_per_big_pg); + } + nvmap_big_page_allocs += page_index; +#endif /* CONFIG_ARM64_4K_PAGES */ + if (s_nr_colors <= 1) { +#ifdef NVMAP_CONFIG_PAGE_POOLS + /* Get as many pages from the pool as possible. */ + page_index += nvmap_page_pool_alloc_lots( + &nvmap_dev->pool, &pages[page_index], + nr_page - page_index); +#endif + allocated = page_index; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 13, 0) + if (page_index < nr_page) + allocated = __alloc_pages_bulk(gfp, numa_mem_id(), NULL, + nr_page, NULL, pages); +#endif + for (i = allocated; i < nr_page; i++) { + pages[i] = nvmap_alloc_pages_exact(gfp, + PAGE_SIZE); + if (!pages[i]) + goto fail; + } + } else if (page_index < nr_page) { + if (alloc_colored(nr_page - page_index, &pages[page_index], chipid)) + goto fail; + page_index = nr_page; + } + nvmap_total_page_allocs += nr_page; + } + + /* + * Make sure any data in the caches is cleaned out before + * passing these pages to userspace. Many nvmap clients assume that + * the buffers are clean as soon as they are allocated. nvmap + * clients can pass the buffer to hardware as it is without any + * explicit cache maintenance. + */ + if (page_index < nr_page) + nvmap_clean_cache(&pages[page_index], nr_page - page_index); + + h->pgalloc.pages = pages; + h->pgalloc.contig = contiguous; + atomic_set(&h->pgalloc.ndirty, 0); + return 0; + +fail: + while (i--) + __free_page(pages[i]); + nvmap_altfree(pages, nr_page * sizeof(*pages)); + wmb(); + return -ENOMEM; +} + +static struct device *nvmap_heap_pgalloc_dev(unsigned long type) +{ + int ret = -EINVAL; + struct device *dma_dev; + + ret = 0; + + if (ret || (type != NVMAP_HEAP_CARVEOUT_VPR)) + return ERR_PTR(-EINVAL); + + dma_dev = dma_dev_from_handle(type); +#ifdef NVMAP_CONFIG_VPR_RESIZE + if (!IS_ERR(dma_dev)) { + ret = dma_set_resizable_heap_floor_size(dma_dev, 0); + if (ret) + return ERR_PTR(ret); + } +#endif + return dma_dev; +} + +static int nvmap_heap_pgalloc(struct nvmap_client *client, + struct nvmap_handle *h, unsigned long type) +{ + size_t size = h->size; + struct page **pages; + struct device *dma_dev; +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0) + dma_addr_t pa = DMA_ERROR_CODE; +#else + dma_addr_t pa = DMA_MAPPING_ERROR; +#endif + + dma_dev = nvmap_heap_pgalloc_dev(type); + if (IS_ERR(dma_dev)) + return PTR_ERR(dma_dev); + + pages = dma_alloc_attrs(dma_dev, size, &pa, + GFP_KERNEL, DMA_ALLOC_FREE_ATTR); + if (dma_mapping_error(dma_dev, pa)) + return -ENOMEM; + + h->pgalloc.pages = pages; + h->pgalloc.contig = 0; + atomic_set(&h->pgalloc.ndirty, 0); + return 0; +} + +static int nvmap_heap_pgfree(struct nvmap_handle *h) +{ + size_t size = h->size; + struct device *dma_dev; + dma_addr_t pa = ~(dma_addr_t)0; + + dma_dev = nvmap_heap_pgalloc_dev(h->heap_type); + if (IS_ERR(dma_dev)) + return PTR_ERR(dma_dev); + + dma_free_attrs(dma_dev, size, h->pgalloc.pages, pa, + DMA_ALLOC_FREE_ATTR); + h->pgalloc.pages = NULL; + return 0; +} + +static bool nvmap_cpu_map_is_allowed(struct nvmap_handle *handle) +{ + if (handle->heap_type & NVMAP_HEAP_CARVEOUT_VPR) + return false; + else + return handle->heap_type & nvmap_dev->dynamic_dma_map_mask; +} + +static void alloc_handle(struct nvmap_client *client, + struct nvmap_handle *h, unsigned int type) +{ + unsigned int carveout_mask = NVMAP_HEAP_CARVEOUT_MASK; + unsigned int iovmm_mask = NVMAP_HEAP_IOVMM; + int ret; + + /* type should only be non-zero and in power of 2. */ + BUG_ON((!type) || (type & (type - 1))); + + if (nvmap_convert_carveout_to_iovmm) { + carveout_mask &= ~NVMAP_HEAP_CARVEOUT_GENERIC; + iovmm_mask |= NVMAP_HEAP_CARVEOUT_GENERIC; + } else if (nvmap_convert_iovmm_to_carveout) { + if (type & NVMAP_HEAP_IOVMM) { + type &= ~NVMAP_HEAP_IOVMM; + type |= NVMAP_HEAP_CARVEOUT_GENERIC; + } + } + + if (type & carveout_mask) { + struct nvmap_heap_block *b; + + b = nvmap_carveout_alloc(client, h, type, NULL); + if (b) { + h->heap_type = type; + h->heap_pgalloc = false; + /* barrier to ensure all handle alloc data + * is visible before alloc is seen by other + * processors. + */ + mb(); + h->alloc = true; + + /* Clear the allocated buffer */ + if (nvmap_cpu_map_is_allowed(h)) { + void *cpu_addr; + + cpu_addr = memremap(b->base, h->size, + MEMREMAP_WB); + if (cpu_addr != NULL) { + memset(cpu_addr, 0, h->size); + __dma_flush_area(cpu_addr, h->size); + memunmap(cpu_addr); + } + } + + return; + } + ret = nvmap_heap_pgalloc(client, h, type); + if (ret) + return; + h->heap_type = NVMAP_HEAP_CARVEOUT_VPR; + h->heap_pgalloc = true; + mb(); + h->alloc = true; + } else if (type & iovmm_mask) { + ret = handle_page_alloc(client, h, + h->userflags & NVMAP_HANDLE_PHYS_CONTIG); + if (ret) + return; + h->heap_type = NVMAP_HEAP_IOVMM; + h->heap_pgalloc = true; + mb(); + h->alloc = true; + } +} + +static int alloc_handle_from_va(struct nvmap_client *client, + struct nvmap_handle *h, + ulong vaddr, + u32 flags) +{ + size_t nr_page = h->size >> PAGE_SHIFT; + struct page **pages; + int ret = 0; + struct mm_struct *mm = current->mm; + + pages = nvmap_altalloc(nr_page * sizeof(*pages)); + if (IS_ERR_OR_NULL(pages)) + return PTR_ERR(pages); + + nvmap_acquire_mmap_read_lock(mm); + ret = nvmap_get_user_pages(vaddr & PAGE_MASK, nr_page, pages, true, + (flags & NVMAP_HANDLE_RO) ? 0 : FOLL_WRITE); + nvmap_release_mmap_read_lock(mm); + if (ret) { + nvmap_altfree(pages, nr_page * sizeof(*pages)); + return ret; + } + + if (flags & NVMAP_HANDLE_RO) + h->is_ro = true; + + nvmap_clean_cache(&pages[0], nr_page); + h->pgalloc.pages = pages; + atomic_set(&h->pgalloc.ndirty, 0); + h->heap_type = NVMAP_HEAP_IOVMM; + h->heap_pgalloc = true; + h->from_va = true; + mb(); + h->alloc = true; + return ret; +} + +/* small allocations will try to allocate from generic OS memory before + * any of the limited heaps, to increase the effective memory for graphics + * allocations, and to reduce fragmentation of the graphics heaps with + * sub-page splinters */ +static const unsigned int heap_policy_small[] = { + NVMAP_HEAP_CARVEOUT_VPR, + NVMAP_HEAP_CARVEOUT_MASK, + NVMAP_HEAP_IOVMM, + 0, +}; + +static const unsigned int heap_policy_large[] = { + NVMAP_HEAP_CARVEOUT_VPR, + NVMAP_HEAP_IOVMM, + NVMAP_HEAP_CARVEOUT_MASK, + 0, +}; + +static const unsigned int heap_policy_excl[] = { + NVMAP_HEAP_CARVEOUT_IVM, + NVMAP_HEAP_CARVEOUT_VIDMEM, + 0, +}; + +int nvmap_alloc_handle(struct nvmap_client *client, + struct nvmap_handle *h, unsigned int heap_mask, + size_t align, + u8 kind, + unsigned int flags, + unsigned int peer) +{ + const unsigned int *alloc_policy; + size_t nr_page; + int err = -ENOMEM; + int tag, i; + bool alloc_from_excl = false; + + h = nvmap_handle_get(h); + + if (!h) + return -EINVAL; + + if (h->alloc) { + nvmap_handle_put(h); + return -EEXIST; + } + + nvmap_stats_inc(NS_TOTAL, h->size); + nvmap_stats_inc(NS_ALLOC, h->size); + trace_nvmap_alloc_handle(client, h, + h->size, heap_mask, align, flags, + nvmap_stats_read(NS_TOTAL), + nvmap_stats_read(NS_ALLOC)); + h->userflags = flags; + nr_page = ((h->size + PAGE_SIZE - 1) >> PAGE_SHIFT); + /* Force mapping to uncached for VPR memory. */ + if (heap_mask & (NVMAP_HEAP_CARVEOUT_VPR | ~nvmap_dev->cpu_access_mask)) + h->flags = NVMAP_HANDLE_UNCACHEABLE; + else + h->flags = (flags & NVMAP_HANDLE_CACHE_FLAG); + h->align = max_t(size_t, align, L1_CACHE_BYTES); + h->peer = peer; + tag = flags >> 16; + + if (!tag && client && !client->tag_warned) { + char task_comm[TASK_COMM_LEN]; + client->tag_warned = 1; + get_task_comm(task_comm, client->task); + pr_err("PID %d: %s: WARNING: " + "All NvMap Allocations must have a tag " + "to identify the subsystem allocating memory." + "Please pass the tag to the API call" + " NvRmMemHanldeAllocAttr() or relevant. \n", + client->task->pid, task_comm); + } + + /* + * If user specifies one of the exclusive carveouts, allocation + * from no other heap should be allowed. + */ + for (i = 0; i < ARRAY_SIZE(heap_policy_excl); i++) { + if (!(heap_mask & heap_policy_excl[i])) + continue; + + if (heap_mask & ~(heap_policy_excl[i])) { + pr_err("%s alloc mixes exclusive heap %d and other heaps\n", + current->group_leader->comm, heap_policy_excl[i]); + err = -EINVAL; + goto out; + } + alloc_from_excl = true; + } + + if (!heap_mask) { + err = -EINVAL; + goto out; + } + + alloc_policy = alloc_from_excl ? heap_policy_excl : + (nr_page == 1) ? heap_policy_small : heap_policy_large; + + while (!h->alloc && *alloc_policy) { + unsigned int heap_type; + + heap_type = *alloc_policy++; + heap_type &= heap_mask; + + if (!heap_type) + continue; + + heap_mask &= ~heap_type; + + while (heap_type && !h->alloc) { + unsigned int heap; + + /* iterate possible heaps MSB-to-LSB, since higher- + * priority carveouts will have higher usage masks */ + heap = 1 << __fls(heap_type); + alloc_handle(client, h, heap); + heap_type &= ~heap; + } + } + +out: + if (h->alloc) { + if (client->kernel_client) + nvmap_stats_inc(NS_KALLOC, h->size); + else + nvmap_stats_inc(NS_UALLOC, h->size); + NVMAP_TAG_TRACE(trace_nvmap_alloc_handle_done, + NVMAP_TP_ARGS_CHR(client, h, NULL)); + err = 0; + } else { + nvmap_stats_dec(NS_TOTAL, h->size); + nvmap_stats_dec(NS_ALLOC, h->size); + } + nvmap_handle_put(h); + return err; +} + +int nvmap_alloc_handle_from_va(struct nvmap_client *client, + struct nvmap_handle *h, + ulong addr, + unsigned int flags) +{ + int err = -ENOMEM; + int tag; + + h = nvmap_handle_get(h); + if (!h) + return -EINVAL; + + if (h->alloc) { + nvmap_handle_put(h); + return -EEXIST; + } + + h->userflags = flags; + h->flags = (flags & NVMAP_HANDLE_CACHE_FLAG); + h->align = PAGE_SIZE; + tag = flags >> 16; + + if (!tag && client && !client->tag_warned) { + char task_comm[TASK_COMM_LEN]; + client->tag_warned = 1; + get_task_comm(task_comm, client->task); + pr_err("PID %d: %s: WARNING: " + "All NvMap Allocations must have a tag " + "to identify the subsystem allocating memory." + "Please pass the tag to the API call" + " NvRmMemHanldeAllocAttr() or relevant. \n", + client->task->pid, task_comm); + } + + err = alloc_handle_from_va(client, h, addr, flags); + if (err) { + pr_err("alloc_handle_from_va failed %d", err); + nvmap_handle_put(h); + return -EINVAL; + } + + if (h->alloc) { + NVMAP_TAG_TRACE(trace_nvmap_alloc_handle_done, + NVMAP_TP_ARGS_CHR(client, h, NULL)); + err = 0; + } + nvmap_handle_put(h); + return err; +} + +void _nvmap_handle_free(struct nvmap_handle *h) +{ + unsigned int i, nr_page, page_index = 0; + struct nvmap_handle_dmabuf_priv *curr, *next; + + list_for_each_entry_safe(curr, next, &h->dmabuf_priv, list) { + curr->priv_release(curr->priv); + list_del(&curr->list); +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0) + kzfree(curr); +#else + kfree_sensitive(curr); +#endif + } + + if (nvmap_handle_remove(nvmap_dev, h) != 0) + return; + + if (!h->alloc) + goto out; + + nvmap_stats_inc(NS_RELEASE, h->size); + nvmap_stats_dec(NS_TOTAL, h->size); + if (!h->heap_pgalloc) { + if (h->vaddr) { + void *addr = h->vaddr; + + addr -= (h->carveout->base & ~PAGE_MASK); + iounmap((void __iomem *)addr); + } + + nvmap_heap_free(h->carveout); + nvmap_kmaps_dec(h); + h->carveout = NULL; + h->vaddr = NULL; + goto out; + } else { + int ret = nvmap_heap_pgfree(h); + if (!ret) + goto out; + } + + nr_page = DIV_ROUND_UP(h->size, PAGE_SIZE); + + BUG_ON(h->size & ~PAGE_MASK); + BUG_ON(!h->pgalloc.pages); + + if (h->vaddr) { + nvmap_kmaps_dec(h); + vunmap(h->vaddr); + + h->vaddr = NULL; + } + + for (i = 0; i < nr_page; i++) + h->pgalloc.pages[i] = nvmap_to_page(h->pgalloc.pages[i]); + +#ifdef NVMAP_CONFIG_PAGE_POOLS + if (!h->from_va) + page_index = nvmap_page_pool_fill_lots(&nvmap_dev->pool, + h->pgalloc.pages, nr_page); +#endif + + for (i = page_index; i < nr_page; i++) { + if (h->from_va) + put_page(h->pgalloc.pages[i]); + else + __free_page(h->pgalloc.pages[i]); + } + + nvmap_altfree(h->pgalloc.pages, nr_page * sizeof(struct page *)); + +out: + NVMAP_TAG_TRACE(trace_nvmap_destroy_handle, + NULL, get_current()->pid, 0, NVMAP_TP_ARGS_H(h)); + kfree(h); +} + +void nvmap_free_handle(struct nvmap_client *client, + struct nvmap_handle *handle, bool is_ro) +{ + struct nvmap_handle_ref *ref; + struct nvmap_handle *h; + + nvmap_ref_lock(client); + + ref = __nvmap_validate_locked(client, handle, is_ro); + if (!ref) { + nvmap_ref_unlock(client); + return; + } + + BUG_ON(!ref->handle); + h = ref->handle; + + if (atomic_dec_return(&ref->dupes)) { + NVMAP_TAG_TRACE(trace_nvmap_free_handle, + NVMAP_TP_ARGS_CHR(client, h, ref)); + nvmap_ref_unlock(client); + goto out; + } + + smp_rmb(); + rb_erase(&ref->node, &client->handle_refs); + client->handle_count--; + atomic_dec(&ref->handle->share_count); + + nvmap_ref_unlock(client); + + if (h->owner == client) + h->owner = NULL; + + if (is_ro) + dma_buf_put(ref->handle->dmabuf_ro); + else + dma_buf_put(ref->handle->dmabuf); + NVMAP_TAG_TRACE(trace_nvmap_free_handle, + NVMAP_TP_ARGS_CHR(client, h, ref)); + kfree(ref); + +out: + BUG_ON(!atomic_read(&h->ref)); + nvmap_handle_put(h); +} + +bool is_nvmap_id_ro(struct nvmap_client *client, int id) +{ + struct nvmap_handle_info *info = NULL; + struct dma_buf *dmabuf = NULL; + + if (WARN_ON(!client)) + return ERR_PTR(-EINVAL); + + if (client->ida) + dmabuf = nvmap_id_array_get_dmabuf_from_id(client->ida, + id); + else + dmabuf = dma_buf_get(id); + + if (IS_ERR_OR_NULL(dmabuf)) + return false; + + if (dmabuf_is_nvmap(dmabuf)) + info = dmabuf->priv; + dma_buf_put(dmabuf); + + return (info != NULL) ? info->is_ro : false; + +} +void nvmap_free_handle_from_fd(struct nvmap_client *client, + int id) +{ + bool is_ro = is_nvmap_id_ro(client, id); + struct nvmap_handle *handle; + struct dma_buf *dmabuf = NULL; + int handle_ref = 0; + long dmabuf_ref = 0; + + handle = nvmap_handle_get_from_id(client, id); + if (IS_ERR_OR_NULL(handle)) + return; + + if (client->ida) + nvmap_id_array_id_release(client->ida, id); + + nvmap_free_handle(client, handle, is_ro); + nvmap_handle_put(handle); + + if (handle) { + dmabuf = is_ro ? handle->dmabuf_ro : handle->dmabuf; + handle_ref = atomic_read(&handle->ref); + dmabuf_ref = dmabuf ? atomic_long_read(&dmabuf->file->f_count) : 0; + } + + trace_refcount_free_handle(handle, dmabuf, handle_ref, dmabuf_ref, + is_ro ? "RO" : "RW"); +} diff --git a/drivers/video/tegra/nvmap/nvmap_cache.c b/drivers/video/tegra/nvmap/nvmap_cache.c new file mode 100644 index 00000000..9f0bf9c4 --- /dev/null +++ b/drivers/video/tegra/nvmap/nvmap_cache.c @@ -0,0 +1,431 @@ +/* + * drivers/video/tegra/nvmap/nvmap_cache.c + * + * Copyright (c) 2011-2022, NVIDIA CORPORATION. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + */ + +#define pr_fmt(fmt) "nvmap: %s() " fmt, __func__ + +#include +#include +#include +#include +#if KERNEL_VERSION(4, 15, 0) > LINUX_VERSION_CODE +#include +#else +#include +#endif + +#include +#ifdef NVMAP_LOADABLE_MODULE +__weak struct arm64_ftr_reg arm64_ftr_reg_ctrel0; +#endif /*NVMAP_LOADABLE_MODULE */ + +#include + +#include "nvmap_priv.h" + +/* + * FIXME: + * + * __clean_dcache_page() is only available on ARM64 (well, we haven't + * implemented it on ARMv7). + */ +void nvmap_clean_cache_page(struct page *page) +{ + __clean_dcache_area_poc(page_address(page), PAGE_SIZE); +} + +void nvmap_clean_cache(struct page **pages, int numpages) +{ + int i; + + /* Not technically a flush but that's what nvmap knows about. */ + nvmap_stats_inc(NS_CFLUSH_DONE, numpages << PAGE_SHIFT); + trace_nvmap_cache_flush(numpages << PAGE_SHIFT, + nvmap_stats_read(NS_ALLOC), + nvmap_stats_read(NS_CFLUSH_RQ), + nvmap_stats_read(NS_CFLUSH_DONE)); + + for (i = 0; i < numpages; i++) + nvmap_clean_cache_page(pages[i]); +} + +void inner_cache_maint(unsigned int op, void *vaddr, size_t size) +{ + if (op == NVMAP_CACHE_OP_WB_INV) + __dma_flush_area(vaddr, size); + else if (op == NVMAP_CACHE_OP_INV) + __dma_map_area(vaddr, size, DMA_FROM_DEVICE); + else + __dma_map_area(vaddr, size, DMA_TO_DEVICE); +} + +static void heap_page_cache_maint( + struct nvmap_handle *h, unsigned long start, unsigned long end, + unsigned int op, bool inner, bool outer, bool clean_only_dirty) +{ + /* Don't perform cache maint for RO mapped buffers */ + if (h->from_va && h->is_ro) + return; + + if (h->userflags & NVMAP_HANDLE_CACHE_SYNC) { + /* + * zap user VA->PA mappings so that any access to the pages + * will result in a fault and can be marked dirty + */ + nvmap_handle_mkclean(h, start, end-start); + nvmap_zap_handle(h, start, end - start); + } + + if (inner) { + if (!h->vaddr) { + if (__nvmap_mmap(h)) + __nvmap_munmap(h, h->vaddr); + else + goto per_page_cache_maint; + } + /* Fast inner cache maintenance using single mapping */ + inner_cache_maint(op, h->vaddr + start, end - start); + if (!outer) + return; + /* Skip per-page inner maintenance in loop below */ + inner = false; + + } +per_page_cache_maint: + + while (start < end) { + struct page *page; + phys_addr_t paddr; + unsigned long next; + unsigned long off; + size_t size; + int ret; + + page = nvmap_to_page(h->pgalloc.pages[start >> PAGE_SHIFT]); + next = min(((start + PAGE_SIZE) & PAGE_MASK), end); + off = start & ~PAGE_MASK; + size = next - start; + paddr = page_to_phys(page) + off; + + ret = nvmap_cache_maint_phys_range(op, paddr, paddr + size, + inner, outer); + WARN_ON(ret != 0); + start = next; + } +} + +struct cache_maint_op { + phys_addr_t start; + phys_addr_t end; + unsigned int op; + struct nvmap_handle *h; + bool inner; + bool outer; + bool clean_only_dirty; +}; + +int nvmap_cache_maint_phys_range(unsigned int op, phys_addr_t pstart, + phys_addr_t pend, int inner, int outer) +{ + void __iomem *io_addr; + phys_addr_t loop; + + if (!inner) + goto do_outer; + + loop = pstart; + while (loop < pend) { + phys_addr_t next = (loop + PAGE_SIZE) & PAGE_MASK; + void *base; + next = min(next, pend); + io_addr = __ioremap(loop, PAGE_SIZE, PG_PROT_KERNEL); + if (io_addr == NULL) + return -ENOMEM; + base = (__force void *)io_addr + (loop & ~PAGE_MASK); + inner_cache_maint(op, base, next - loop); + iounmap(io_addr); + loop = next; + } + +do_outer: + return 0; +} + +static int do_cache_maint(struct cache_maint_op *cache_work) +{ + phys_addr_t pstart = cache_work->start; + phys_addr_t pend = cache_work->end; + int err = 0; + struct nvmap_handle *h = cache_work->h; + unsigned int op = cache_work->op; + + if (!h || !h->alloc) + return -EFAULT; + + wmb(); + if (h->flags == NVMAP_HANDLE_UNCACHEABLE || + h->flags == NVMAP_HANDLE_WRITE_COMBINE || pstart == pend) + goto out; + + trace_nvmap_cache_maint(h->owner, h, pstart, pend, op, pend - pstart); + if (pstart > h->size || pend > h->size) { + pr_warn("cache maintenance outside handle\n"); + err = -EINVAL; + goto out; + } + + if (h->heap_pgalloc) { + heap_page_cache_maint(h, pstart, pend, op, true, + (h->flags == NVMAP_HANDLE_INNER_CACHEABLE) ? + false : true, cache_work->clean_only_dirty); + goto out; + } + + pstart += h->carveout->base; + pend += h->carveout->base; + + err = nvmap_cache_maint_phys_range(op, pstart, pend, true, + h->flags != NVMAP_HANDLE_INNER_CACHEABLE); + +out: + if (!err) { + nvmap_stats_inc(NS_CFLUSH_DONE, pend - pstart); + } + + trace_nvmap_cache_flush(pend - pstart, + nvmap_stats_read(NS_ALLOC), + nvmap_stats_read(NS_CFLUSH_RQ), + nvmap_stats_read(NS_CFLUSH_DONE)); + + return 0; +} + +static void nvmap_handle_get_cacheability(struct nvmap_handle *h, + bool *inner, bool *outer) +{ + *inner = h->flags == NVMAP_HANDLE_CACHEABLE || + h->flags == NVMAP_HANDLE_INNER_CACHEABLE; + *outer = h->flags == NVMAP_HANDLE_CACHEABLE; +} + +int __nvmap_do_cache_maint(struct nvmap_client *client, + struct nvmap_handle *h, + unsigned long start, unsigned long end, + unsigned int op, bool clean_only_dirty) +{ + int err; + struct cache_maint_op cache_op; + + h = nvmap_handle_get(h); + if (!h) + return -EFAULT; + + if ((start >= h->size) || (end > h->size)) { + pr_debug("%s start: %ld end: %ld h->size: %zu\n", __func__, + start, end, h->size); + nvmap_handle_put(h); + return -EFAULT; + } + + if (!(h->heap_type & nvmap_dev->cpu_access_mask)) { + pr_debug("%s heap_type %u access_mask 0x%x\n", __func__, + h->heap_type, nvmap_dev->cpu_access_mask); + nvmap_handle_put(h); + return -EPERM; + } + + nvmap_kmaps_inc(h); + if (op == NVMAP_CACHE_OP_INV) + op = NVMAP_CACHE_OP_WB_INV; + + /* clean only dirty is applicable only for Write Back operation */ + if (op != NVMAP_CACHE_OP_WB) + clean_only_dirty = false; + + cache_op.h = h; + cache_op.start = start ? start : 0; + cache_op.end = end ? end : h->size; + cache_op.op = op; + nvmap_handle_get_cacheability(h, &cache_op.inner, &cache_op.outer); + cache_op.clean_only_dirty = clean_only_dirty; + + nvmap_stats_inc(NS_CFLUSH_RQ, end - start); + err = do_cache_maint(&cache_op); + nvmap_kmaps_dec(h); + nvmap_handle_put(h); + return err; +} + +int __nvmap_cache_maint(struct nvmap_client *client, + struct nvmap_cache_op_64 *op) +{ + struct vm_area_struct *vma; + struct nvmap_vma_priv *priv; + struct nvmap_handle *handle; + unsigned long start; + unsigned long end; + int err = 0; + + if (!op->addr || op->op < NVMAP_CACHE_OP_WB || + op->op > NVMAP_CACHE_OP_WB_INV) + return -EINVAL; + + handle = nvmap_handle_get_from_id(client, op->handle); + if (IS_ERR_OR_NULL(handle)) + return -EINVAL; + + nvmap_acquire_mmap_read_lock(current->mm); + + vma = find_vma(current->active_mm, (unsigned long)op->addr); + if (!vma || !is_nvmap_vma(vma) || + (ulong)op->addr < vma->vm_start || + (ulong)op->addr >= vma->vm_end || + op->len > vma->vm_end - (ulong)op->addr) { + err = -EADDRNOTAVAIL; + goto out; + } + + priv = (struct nvmap_vma_priv *)vma->vm_private_data; + + if (priv->handle != handle) { + err = -EFAULT; + goto out; + } + + start = (unsigned long)op->addr - vma->vm_start + + (vma->vm_pgoff << PAGE_SHIFT); + end = start + op->len; + + err = __nvmap_do_cache_maint(client, priv->handle, start, end, op->op, + false); +out: + nvmap_release_mmap_read_lock(current->mm); + nvmap_handle_put(handle); + return err; +} + +/* + * Perform cache op on the list of memory regions within passed handles. + * A memory region within handle[i] is identified by offsets[i], sizes[i] + * + * sizes[i] == 0 is a special case which causes handle wide operation, + * this is done by replacing offsets[i] = 0, sizes[i] = handles[i]->size. + * So, the input arrays sizes, offsets are not guaranteed to be read-only + * + * This will optimze the op if it can. + * In the case that all the handles together are larger than the inner cache + * maint threshold it is possible to just do an entire inner cache flush. + * + * NOTE: this omits outer cache operations which is fine for ARM64 + */ +static int __nvmap_do_cache_maint_list(struct nvmap_handle **handles, + u64 *offsets, u64 *sizes, int op, u32 nr_ops, + bool is_32) +{ + u32 i; + u64 total = 0; + u64 thresh = ~0; + + WARN(!IS_ENABLED(CONFIG_ARM64), + "cache list operation may not function properly"); + + for (i = 0; i < nr_ops; i++) { + bool inner, outer; + u32 *sizes_32 = (u32 *)sizes; + u64 size = is_32 ? sizes_32[i] : sizes[i]; + + nvmap_handle_get_cacheability(handles[i], &inner, &outer); + + if (!inner && !outer) + continue; + + if ((op == NVMAP_CACHE_OP_WB) && nvmap_handle_track_dirty(handles[i])) + total += atomic_read(&handles[i]->pgalloc.ndirty); + else + total += size ? size : handles[i]->size; + } + + if (!total) + return 0; + + /* Full flush in the case the passed list is bigger than our + * threshold. */ + if (total >= thresh) { + for (i = 0; i < nr_ops; i++) { + if (handles[i]->userflags & + NVMAP_HANDLE_CACHE_SYNC) { + nvmap_handle_mkclean(handles[i], 0, + handles[i]->size); + nvmap_zap_handle(handles[i], 0, + handles[i]->size); + } + } + + nvmap_stats_inc(NS_CFLUSH_RQ, total); + nvmap_stats_inc(NS_CFLUSH_DONE, thresh); + trace_nvmap_cache_flush(total, + nvmap_stats_read(NS_ALLOC), + nvmap_stats_read(NS_CFLUSH_RQ), + nvmap_stats_read(NS_CFLUSH_DONE)); + } else { + for (i = 0; i < nr_ops; i++) { + u32 *offs_32 = (u32 *)offsets, *sizes_32 = (u32 *)sizes; + u64 size = is_32 ? sizes_32[i] : sizes[i]; + u64 offset = is_32 ? offs_32[i] : offsets[i]; + int err; + + size = size ?: handles[i]->size; + offset = offset ?: 0; + err = __nvmap_do_cache_maint(handles[i]->owner, + handles[i], offset, + offset + size, + op, false); + if (err) { + pr_err("cache maint per handle failed [%d]\n", + err); + return err; + } + } + } + + return 0; +} + +#if (LINUX_VERSION_CODE > KERNEL_VERSION(4, 9, 0)) +static const struct soc_device_attribute tegra194_soc = { + .soc_id = "TEGRA194", +}; + +static const struct soc_device_attribute tegra234_soc = { + .soc_id = "TEGRA234", +}; +#endif +inline int nvmap_do_cache_maint_list(struct nvmap_handle **handles, + u64 *offsets, u64 *sizes, int op, u32 nr_ops, + bool is_32) +{ + /* + * As io-coherency is enabled by default from T194 onwards, + * Don't do cache maint from CPU side. The HW, SCF will do. + */ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0)) + if (!(tegra_get_chip_id() == TEGRA194)) +#else + if (!soc_device_match(&tegra194_soc) && + !soc_device_match(&tegra234_soc)) +#endif + return __nvmap_do_cache_maint_list(handles, + offsets, sizes, op, nr_ops, is_32); + return 0; +} diff --git a/drivers/video/tegra/nvmap/nvmap_cache_maint.S b/drivers/video/tegra/nvmap/nvmap_cache_maint.S new file mode 100644 index 00000000..c471aa04 --- /dev/null +++ b/drivers/video/tegra/nvmap/nvmap_cache_maint.S @@ -0,0 +1,236 @@ +/* + * Copyright (c) 2021-2022, NVIDIA CORPORATION. All rights reserved. + * + * Cache maintenance + * + * Derived from Linux kernel source file arch/arm64/mm/cache.S + * Copyright (C) 2001 Deep Blue Solutions Ltd. + * Copyright (C) 2012 ARM Ltd. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 18, 0) +#define NVMAP_SYM_FUNC_START(func) SYM_FUNC_START(__ip_##func) +#define NVMAP_SYM_FUNC_END(func) SYM_FUNC_END(__ip_##func) +#define NVMAP_SYM_FUNC_ALIAS(func) SYM_FUNC_ALIAS(func, __ip_##func) +#else +#define NVMAP_SYM_FUNC_START(func) SYM_FUNC_START_PI(func) +#define NVMAP_SYM_FUNC_END(func) SYM_FUNC_END_PI(func) +#define NVMAP_SYM_FUNC_ALIAS(func) +#endif + +/* + * invalidate_icache_range(start,end) + * + * Ensure that the I cache is invalid within specified region. + * + * - start - virtual start address of region + * - end - virtual end address of region + */ +SYM_FUNC_START(invalidate_icache_range) +alternative_if ARM64_HAS_CACHE_DIC + mov x0, xzr + isb + ret +alternative_else_nop_endif + + uaccess_ttbr0_enable x2, x3, x4 + + invalidate_icache_by_line x0, x1, x2, x3, 2f + mov x0, xzr +1: + uaccess_ttbr0_disable x1, x2 + ret +2: + mov x0, #-EFAULT + b 1b +SYM_FUNC_END(invalidate_icache_range) + +/* + * __flush_dcache_area(kaddr, size) + * + * Ensure that any D-cache lines for the interval [kaddr, kaddr+size) + * are cleaned and invalidated to the PoC. + * + * - kaddr - kernel address + * - size - size in question + */ +NVMAP_SYM_FUNC_START(__flush_dcache_area) +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 14, 0) + add x1, x0, x1 +#endif + dcache_by_line_op civac, sy, x0, x1, x2, x3 + ret +NVMAP_SYM_FUNC_END(__flush_dcache_area) +NVMAP_SYM_FUNC_ALIAS(__flush_dcache_area) + +/* + * __clean_dcache_area_pou(kaddr, size) + * + * Ensure that any D-cache lines for the interval [kaddr, kaddr+size) + * are cleaned to the PoU. + * + * - kaddr - kernel address + * - size - size in question + */ +SYM_FUNC_START(__clean_dcache_area_pou) +alternative_if ARM64_HAS_CACHE_IDC + dsb ishst + ret +alternative_else_nop_endif +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 14, 0) + add x1, x0, x1 +#endif + dcache_by_line_op cvau, ish, x0, x1, x2, x3 + ret +SYM_FUNC_END(__clean_dcache_area_pou) + +/* + * __inval_dcache_area(kaddr, size) + * + * Ensure that any D-cache lines for the interval [kaddr, kaddr+size) + * are invalidated. Any partial lines at the ends of the interval are + * also cleaned to PoC to prevent data loss. + * + * - kaddr - kernel address + * - size - size in question + */ +SYM_FUNC_START_LOCAL(__dma_inv_area) +NVMAP_SYM_FUNC_START(__inval_dcache_area) + /* FALLTHROUGH */ + +/* + * __dma_inv_area(start, size) + * - start - virtual start address of region + * - size - size in question + */ + add x1, x1, x0 + dcache_line_size x2, x3 + sub x3, x2, #1 + tst x1, x3 // end cache line aligned? + bic x1, x1, x3 + b.eq 1f + dc civac, x1 // clean & invalidate D / U line +1: tst x0, x3 // start cache line aligned? + bic x0, x0, x3 + b.eq 2f + dc civac, x0 // clean & invalidate D / U line + b 3f +2: dc ivac, x0 // invalidate D / U line +3: add x0, x0, x2 + cmp x0, x1 + b.lo 2b + dsb sy + ret +NVMAP_SYM_FUNC_END(__inval_dcache_area) +NVMAP_SYM_FUNC_ALIAS(__inval_dcache_area) +SYM_FUNC_END(__dma_inv_area) + +/* + * __clean_dcache_area_poc(kaddr, size) + * + * Ensure that any D-cache lines for the interval [kaddr, kaddr+size) + * are cleaned to the PoC. + * + * - kaddr - kernel address + * - size - size in question + */ +SYM_FUNC_START_LOCAL(__dma_clean_area) +NVMAP_SYM_FUNC_START(__clean_dcache_area_poc) + /* FALLTHROUGH */ + +/* + * __dma_clean_area(start, size) + * - start - virtual start address of region + * - size - size in question + */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 14, 0) + add x1, x0, x1 +#endif + dcache_by_line_op cvac, sy, x0, x1, x2, x3 + ret +NVMAP_SYM_FUNC_END(__clean_dcache_area_poc) +NVMAP_SYM_FUNC_ALIAS(__clean_dcache_area_poc) +SYM_FUNC_END(__dma_clean_area) + +/* + * __clean_dcache_area_pop(kaddr, size) + * + * Ensure that any D-cache lines for the interval [kaddr, kaddr+size) + * are cleaned to the PoP. + * + * - kaddr - kernel address + * - size - size in question + */ +NVMAP_SYM_FUNC_START(__clean_dcache_area_pop) + alternative_if_not ARM64_HAS_DCPOP + b __clean_dcache_area_poc + alternative_else_nop_endif +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 14, 0) + add x1, x0, x1 +#endif + dcache_by_line_op cvap, sy, x0, x1, x2, x3 + ret +NVMAP_SYM_FUNC_END(__clean_dcache_area_pop) +NVMAP_SYM_FUNC_ALIAS(__clean_dcache_area_pop) + +/* + * __dma_flush_area(start, size) + * + * clean & invalidate D / U line + * + * - start - virtual start address of region + * - size - size in question + */ +NVMAP_SYM_FUNC_START(__dma_flush_area) +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 14, 0) + add x1, x0, x1 +#endif + dcache_by_line_op civac, sy, x0, x1, x2, x3 + ret +NVMAP_SYM_FUNC_END(__dma_flush_area) +NVMAP_SYM_FUNC_ALIAS(__dma_flush_area) + +/* + * __dma_map_area(start, size, dir) + * - start - kernel virtual start address + * - size - size of region + * - dir - DMA direction + */ +NVMAP_SYM_FUNC_START(__dma_map_area) + cmp w2, #DMA_FROM_DEVICE + b.eq __dma_inv_area + b __dma_clean_area +NVMAP_SYM_FUNC_END(__dma_map_area) +NVMAP_SYM_FUNC_ALIAS(__dma_map_area) + +/* + * __dma_unmap_area(start, size, dir) + * - start - kernel virtual start address + * - size - size of region + * - dir - DMA direction + */ +NVMAP_SYM_FUNC_START(__dma_unmap_area) + cmp w2, #DMA_TO_DEVICE + b.ne __dma_inv_area + ret +NVMAP_SYM_FUNC_END(__dma_unmap_area) +NVMAP_SYM_FUNC_ALIAS(__dma_unmap_area) diff --git a/drivers/video/tegra/nvmap/nvmap_carveout.c b/drivers/video/tegra/nvmap/nvmap_carveout.c new file mode 100644 index 00000000..ac703d58 --- /dev/null +++ b/drivers/video/tegra/nvmap/nvmap_carveout.c @@ -0,0 +1,270 @@ +/* + * drivers/video/tegra/nvmap/nvmap_dev.c + * + * Interface with nvmap carveouts + * + * Copyright (c) 2011-2022, NVIDIA CORPORATION. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + */ + +#include + +#include "nvmap_priv.h" + +extern struct nvmap_device *nvmap_dev; + +extern const struct file_operations debug_clients_fops; +extern const struct file_operations debug_allocations_fops; +extern const struct file_operations debug_all_allocations_fops; +extern const struct file_operations debug_orphan_handles_fops; +extern const struct file_operations debug_maps_fops; +#ifdef NVMAP_CONFIG_DEBUG_MAPS +extern const struct file_operations debug_device_list_fops; +#endif /* NVMAP_CONFIG_DEBUG_MAPS */ + +int nvmap_create_carveout(const struct nvmap_platform_carveout *co) +{ + int i, err = 0; + struct nvmap_carveout_node *node; + + mutex_lock(&nvmap_dev->carveout_lock); + if (!nvmap_dev->heaps) { + int nr_heaps; + + nvmap_dev->nr_carveouts = 0; + if (nvmap_dev->plat) + nr_heaps = nvmap_dev->plat->nr_carveouts + 1; + else + nr_heaps = 1; + nvmap_dev->heaps = kzalloc(sizeof(struct nvmap_carveout_node) * + nr_heaps, GFP_KERNEL); + if (!nvmap_dev->heaps) { + err = -ENOMEM; + pr_err("couldn't allocate carveout memory\n"); + goto out; + } + nvmap_dev->nr_heaps = nr_heaps; + } else if (nvmap_dev->nr_carveouts >= nvmap_dev->nr_heaps) { + node = krealloc(nvmap_dev->heaps, + sizeof(*node) * (nvmap_dev->nr_carveouts + 1), + GFP_KERNEL); + if (!node) { + err = -ENOMEM; + pr_err("nvmap heap array resize failed\n"); + goto out; + } + nvmap_dev->heaps = node; + nvmap_dev->nr_heaps = nvmap_dev->nr_carveouts + 1; + } + + for (i = 0; i < nvmap_dev->nr_heaps; i++) + if ((co->usage_mask != NVMAP_HEAP_CARVEOUT_IVM) && + (nvmap_dev->heaps[i].heap_bit & co->usage_mask)) { + pr_err("carveout %s already exists\n", co->name); + err = -EEXIST; + goto out; + } + + node = &nvmap_dev->heaps[nvmap_dev->nr_carveouts]; + + node->base = round_up(co->base, PAGE_SIZE); + node->size = round_down(co->size - + (node->base - co->base), PAGE_SIZE); + if (!co->size) + goto out; + + node->carveout = nvmap_heap_create( + nvmap_dev->dev_user.this_device, co, + node->base, node->size, node); + + if (!node->carveout) { + err = -ENOMEM; + pr_err("couldn't create %s\n", co->name); + goto out; + } + node->index = nvmap_dev->nr_carveouts; + nvmap_dev->nr_carveouts++; + node->heap_bit = co->usage_mask; + + if (!IS_ERR_OR_NULL(nvmap_dev->debug_root)) { + struct dentry *heap_root = + debugfs_create_dir(co->name, nvmap_dev->debug_root); + if (!IS_ERR_OR_NULL(heap_root)) { + debugfs_create_file("clients", S_IRUGO, + heap_root, + (void *)(uintptr_t)node->heap_bit, + &debug_clients_fops); + debugfs_create_file("allocations", S_IRUGO, + heap_root, + (void *)(uintptr_t)node->heap_bit, + &debug_allocations_fops); + debugfs_create_file("all_allocations", S_IRUGO, + heap_root, + (void *)(uintptr_t)node->heap_bit, + &debug_all_allocations_fops); + debugfs_create_file("orphan_handles", S_IRUGO, + heap_root, + (void *)(uintptr_t)node->heap_bit, + &debug_orphan_handles_fops); + debugfs_create_file("maps", S_IRUGO, + heap_root, + (void *)(uintptr_t)node->heap_bit, + &debug_maps_fops); + debugfs_create_bool("no_cpu_access", S_IRUGO, + heap_root, (bool *)&co->no_cpu_access); +#ifdef NVMAP_CONFIG_DEBUG_MAPS + debugfs_create_file("device_list", S_IRUGO, + heap_root, + (void *)(uintptr_t)node->heap_bit, + &debug_device_list_fops); +#endif /* NVMAP_CONFIG_DEBUG_MAPS */ + nvmap_heap_debugfs_init(heap_root, + node->carveout); + } + } +out: + mutex_unlock(&nvmap_dev->carveout_lock); + return err; +} + +#ifdef NVMAP_CONFIG_DEBUG_MAPS +struct nvmap_device_list *nvmap_is_device_present(char *device_name, u32 heap_type) +{ + struct rb_node *node = NULL; + int i; + + if (heap_type == NVMAP_HEAP_IOVMM) { + node = nvmap_dev->device_names.rb_node; + } else { + for (i = 0; i < nvmap_dev->nr_carveouts; i++) { + if ((heap_type & nvmap_dev->heaps[i].heap_bit) && + nvmap_dev->heaps[i].carveout) { + node = nvmap_dev->heaps[i].carveout->device_names.rb_node; + break; + } + } + } + while (node) { + struct nvmap_device_list *dl = container_of(node, + struct nvmap_device_list, node); + if (strcmp(dl->device_name, device_name) > 0) + node = node->rb_left; + else if (strcmp(dl->device_name, device_name) < 0) + node = node->rb_right; + else + return dl; + } + return NULL; +} + +void nvmap_add_device_name(char *device_name, u64 dma_mask, u32 heap_type) +{ + struct rb_root *root = NULL; + struct rb_node **new = NULL, *parent = NULL; + struct nvmap_device_list *dl = NULL; + int i; + + if (heap_type == NVMAP_HEAP_IOVMM) { + root = &nvmap_dev->device_names; + } else { + for (i = 0; i < nvmap_dev->nr_carveouts; i++) { + if ((heap_type & nvmap_dev->heaps[i].heap_bit) && + nvmap_dev->heaps[i].carveout) { + root = &nvmap_dev->heaps[i].carveout->device_names; + break; + } + } + } + if (root) { + new = &(root->rb_node); + while (*new) { + dl = container_of(*new, struct nvmap_device_list, node); + parent = *new; + if (strcmp(dl->device_name, device_name) > 0) + new = &((*new)->rb_left); + else if (strcmp(dl->device_name, device_name) < 0) + new = &((*new)->rb_right); + } + dl = kzalloc(sizeof(*dl), GFP_KERNEL); + if (!dl) + return; + dl->device_name = kzalloc(strlen(device_name) + 1, GFP_KERNEL); + if (!dl->device_name) + return; + strcpy(dl->device_name, device_name); + dl->dma_mask = dma_mask; + rb_link_node(&dl->node, parent, new); + rb_insert_color(&dl->node, root); + } +} + +void nvmap_remove_device_name(char *device_name, u32 heap_type) +{ + struct nvmap_device_list *dl = NULL; + int i; + + dl = nvmap_is_device_present(device_name, heap_type); + if (dl) { + if (heap_type == NVMAP_HEAP_IOVMM) { + rb_erase(&dl->node, + &nvmap_dev->device_names); + kfree(dl->device_name); + kfree(dl); + return; + } + for (i = 0; i < nvmap_dev->nr_carveouts; i++) { + if ((heap_type & nvmap_dev->heaps[i].heap_bit) && + nvmap_dev->heaps[i].carveout) { + rb_erase(&dl->node, + &nvmap_dev->heaps[i].carveout->device_names); + kfree(dl->device_name); + kfree(dl); + return; + } + } + } +} +#endif /* NVMAP_CONFIG_DEBUG_MAPS */ + +static +struct nvmap_heap_block *do_nvmap_carveout_alloc(struct nvmap_client *client, + struct nvmap_handle *handle, + unsigned long type, + phys_addr_t *start) +{ + struct nvmap_carveout_node *co_heap; + struct nvmap_device *dev = nvmap_dev; + int i; + + for (i = 0; i < dev->nr_carveouts; i++) { + struct nvmap_heap_block *block; + co_heap = &dev->heaps[i]; + + if (!(co_heap->heap_bit & type)) + continue; + + if (type & NVMAP_HEAP_CARVEOUT_IVM) + handle->size = ALIGN(handle->size, NVMAP_IVM_ALIGNMENT); + + block = nvmap_heap_alloc(co_heap->carveout, handle, start); + if (block) + return block; + } + return NULL; +} + +struct nvmap_heap_block *nvmap_carveout_alloc(struct nvmap_client *client, + struct nvmap_handle *handle, + unsigned long type, + phys_addr_t *start) +{ + return do_nvmap_carveout_alloc(client, handle, type, start); +} diff --git a/drivers/video/tegra/nvmap/nvmap_core.c b/drivers/video/tegra/nvmap/nvmap_core.c new file mode 100644 index 00000000..ce978df6 --- /dev/null +++ b/drivers/video/tegra/nvmap/nvmap_core.c @@ -0,0 +1,355 @@ +/* + * drivers/video/tegra/nvmap/nvmap_core.c + * + * Memory manager for Tegra GPU + * + * Copyright (c) 2009-2022, NVIDIA CORPORATION. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + */ + +#define pr_fmt(fmt) "nvmap: %s() " fmt, __func__ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include + +#include "nvmap_priv.h" + +static phys_addr_t handle_phys(struct nvmap_handle *h) +{ + if (h->heap_pgalloc) + BUG(); + return h->carveout->base; +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0) +void *__nvmap_kmap(struct nvmap_handle *h, unsigned int pagenum) +{ + phys_addr_t paddr; + unsigned long kaddr; + void __iomem *addr; + pgprot_t prot; + + if (!virt_addr_valid(h)) + return NULL; + + h = nvmap_handle_get(h); + if (!h) + return NULL; + /* + * If the handle is RO and virtual mapping is requested in + * kernel address space, return error. + */ + if (h->from_va && h->is_ro) + goto put_handle; + + if (!h->alloc) + goto put_handle; + + if (!(h->heap_type & nvmap_dev->cpu_access_mask)) + goto put_handle; + + nvmap_kmaps_inc(h); + if (pagenum >= h->size >> PAGE_SHIFT) + goto out; + + if (h->vaddr) { + kaddr = (unsigned long)h->vaddr + pagenum * PAGE_SIZE; + } else { + prot = nvmap_pgprot(h, PG_PROT_KERNEL); + if (h->heap_pgalloc) + paddr = page_to_phys(nvmap_to_page( + h->pgalloc.pages[pagenum])); + else + paddr = h->carveout->base + pagenum * PAGE_SIZE; + + addr = __ioremap(paddr, PAGE_SIZE, prot); + if (addr == NULL) + goto out; + kaddr = (unsigned long)addr; + } + return (void *)kaddr; +out: + nvmap_kmaps_dec(h); +put_handle: + nvmap_handle_put(h); + return NULL; +} + +void __nvmap_kunmap(struct nvmap_handle *h, unsigned int pagenum, + void *addr) +{ + phys_addr_t paddr; + + if (!h || !h->alloc || + WARN_ON(!virt_addr_valid(h)) || + WARN_ON(!addr) || + !(h->heap_type & nvmap_dev->cpu_access_mask)) + return; + + if (WARN_ON(pagenum >= h->size >> PAGE_SHIFT)) + return; + + if (h->vaddr && (h->vaddr == (addr - pagenum * PAGE_SIZE))) + goto out; + + if (h->heap_pgalloc) + paddr = page_to_phys(nvmap_to_page(h->pgalloc.pages[pagenum])); + else + paddr = h->carveout->base + pagenum * PAGE_SIZE; + + if (h->flags != NVMAP_HANDLE_UNCACHEABLE && + h->flags != NVMAP_HANDLE_WRITE_COMBINE) { + __dma_flush_area(addr, PAGE_SIZE); + outer_flush_range(paddr, paddr + PAGE_SIZE); /* FIXME */ + } + iounmap((void __iomem *)addr); +out: + nvmap_kmaps_dec(h); + nvmap_handle_put(h); +} +#endif + +void *__nvmap_mmap(struct nvmap_handle *h) +{ + pgprot_t prot; + void *vaddr; + unsigned long adj_size; + struct page **pages = NULL; + int i = 0; + + if (!virt_addr_valid(h)) + return NULL; + + h = nvmap_handle_get(h); + if (!h) + return NULL; + /* + * If the handle is RO and virtual mapping is requested in + * kernel address space, return error. + */ + if (h->from_va && h->is_ro) + goto put_handle; + + + if (!h->alloc) + goto put_handle; + + if (!(h->heap_type & nvmap_dev->cpu_access_mask)) + goto put_handle; + + if (h->vaddr) + return h->vaddr; + + nvmap_kmaps_inc(h); + prot = nvmap_pgprot(h, PG_PROT_KERNEL); + + if (h->heap_pgalloc) { + pages = nvmap_pages(h->pgalloc.pages, h->size >> PAGE_SHIFT); + if (!pages) + goto out; + + vaddr = vmap(pages, h->size >> PAGE_SHIFT, VM_MAP, prot); + if (!vaddr && !h->vaddr) + goto out; + nvmap_altfree(pages, (h->size >> PAGE_SHIFT) * sizeof(*pages)); + + if (vaddr && atomic_long_cmpxchg((atomic_long_t *)&h->vaddr, 0, (long)vaddr)) { + nvmap_kmaps_dec(h); + vunmap(vaddr); + } + return h->vaddr; + } + + /* carveout - explicitly map the pfns into a vmalloc area */ + adj_size = h->carveout->base & ~PAGE_MASK; + adj_size += h->size; + adj_size = PAGE_ALIGN(adj_size); + + if (pfn_valid(__phys_to_pfn(h->carveout->base & PAGE_MASK))) { + unsigned long pfn; + struct page *page; + int nr_pages; + + pfn = ((h->carveout->base) >> PAGE_SHIFT); + page = pfn_to_page(pfn); + nr_pages = h->size >> PAGE_SHIFT; + + pages = vmalloc(nr_pages * sizeof(*pages)); + if (!pages) + goto out; + + for (i = 0; i < nr_pages; i++) + pages[i] = nth_page(page, i); + + vaddr = vmap(pages, nr_pages, VM_MAP, prot); + } else { + vaddr = (__force void *)__ioremap(h->carveout->base, adj_size, + prot); + } + if (vaddr == NULL) + goto out; + + if (vaddr && atomic_long_cmpxchg((atomic_long_t *)&h->vaddr, + 0, (long)vaddr)) { + vaddr -= (h->carveout->base & ~PAGE_MASK); + /* + * iounmap calls vunmap for vmalloced address, hence + * takes care of vmap/__ioremap freeing part. + */ + iounmap((void __iomem *)vaddr); + nvmap_kmaps_dec(h); + } + + if (pages) + vfree(pages); + + /* leave the handle ref count incremented by 1, so that + * the handle will not be freed while the kernel mapping exists. + * nvmap_handle_put will be called by unmapping this address */ + return h->vaddr; +out: + if (pages) + vfree(pages); + nvmap_kmaps_dec(h); +put_handle: + nvmap_handle_put(h); + return NULL; +} + +void __nvmap_munmap(struct nvmap_handle *h, void *addr) +{ + if (!h || !h->alloc || + WARN_ON(!virt_addr_valid(h)) || + WARN_ON(!addr) || + !(h->heap_type & nvmap_dev->cpu_access_mask)) + return; + + nvmap_handle_put(h); +} + +/* + * NOTE: this does not ensure the continued existence of the underlying + * dma_buf. If you want ensure the existence of the dma_buf you must get an + * nvmap_handle_ref as that is what tracks the dma_buf refs. + */ +struct nvmap_handle *nvmap_handle_get(struct nvmap_handle *h) +{ + int cnt; + + if (WARN_ON(!virt_addr_valid(h))) { + pr_err("%s: invalid handle\n", current->group_leader->comm); + return NULL; + } + + cnt = atomic_inc_return(&h->ref); + NVMAP_TAG_TRACE(trace_nvmap_handle_get, h, cnt); + + if (unlikely(cnt <= 1)) { + pr_err("%s: %s attempt to get a freed handle\n", + __func__, current->group_leader->comm); + atomic_dec(&h->ref); + return NULL; + } + + return h; +} + +void nvmap_handle_put(struct nvmap_handle *h) +{ + int cnt; + + if (WARN_ON(!virt_addr_valid(h))) + return; + cnt = atomic_dec_return(&h->ref); + NVMAP_TAG_TRACE(trace_nvmap_handle_put, h, cnt); + + if (WARN_ON(cnt < 0)) { + pr_err("%s: %s put to negative references\n", + __func__, current->comm); + } else if (cnt == 0) + _nvmap_handle_free(h); +} + +struct sg_table *__nvmap_sg_table(struct nvmap_client *client, + struct nvmap_handle *h) +{ + struct sg_table *sgt = NULL; + int err, npages; + struct page **pages; + + if (!virt_addr_valid(h)) + return ERR_PTR(-EINVAL); + + h = nvmap_handle_get(h); + if (!h) + return ERR_PTR(-EINVAL); + + if (!h->alloc) { + err = -EINVAL; + goto put_handle; + } + + npages = PAGE_ALIGN(h->size) >> PAGE_SHIFT; + sgt = kzalloc(sizeof(*sgt), GFP_KERNEL); + if (!sgt) { + err = -ENOMEM; + goto err; + } + + if (!h->heap_pgalloc) { + phys_addr_t paddr = handle_phys(h); + struct page *page = phys_to_page(paddr); + + err = sg_alloc_table(sgt, 1, GFP_KERNEL); + if (err) + goto err; + + sg_set_page(sgt->sgl, page, h->size, offset_in_page(paddr)); + } else { + pages = nvmap_pages(h->pgalloc.pages, npages); + if (!pages) { + err = -ENOMEM; + goto err; + } + err = sg_alloc_table_from_pages(sgt, pages, + npages, 0, h->size, GFP_KERNEL); + nvmap_altfree(pages, npages * sizeof(*pages)); + if (err) + goto err; + } + nvmap_handle_put(h); + return sgt; + +err: + kfree(sgt); +put_handle: + nvmap_handle_put(h); + return ERR_PTR(err); +} + +void __nvmap_free_sg_table(struct nvmap_client *client, + struct nvmap_handle *h, struct sg_table *sgt) +{ + sg_free_table(sgt); + kfree(sgt); +} diff --git a/drivers/video/tegra/nvmap/nvmap_dev.c b/drivers/video/tegra/nvmap/nvmap_dev.c new file mode 100644 index 00000000..3610607a --- /dev/null +++ b/drivers/video/tegra/nvmap/nvmap_dev.c @@ -0,0 +1,1557 @@ +/* + * drivers/video/tegra/nvmap/nvmap_dev.c + * + * User-space interface to nvmap + * + * Copyright (c) 2011-2022, NVIDIA CORPORATION. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef CVNAS_BUILTIN +#include +#include +#endif /* CVNAS_BUILTIN */ + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 15, 0) +#include +#else +#include +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0) +#include +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0) +#include +#endif + +#include +#include + +#define CREATE_TRACE_POINTS +#include + +#include "nvmap_priv.h" +#include "nvmap_heap.h" +#include "nvmap_ioctl.h" + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0) +#include +#endif + +#define NVMAP_CARVEOUT_KILLER_RETRY_TIME 100 /* msecs */ + +struct nvmap_device *nvmap_dev; +EXPORT_SYMBOL(nvmap_dev); +ulong nvmap_init_time; + +static struct device_dma_parameters nvmap_dma_parameters = { + .max_segment_size = UINT_MAX, +}; + +static int nvmap_open(struct inode *inode, struct file *filp); +static int nvmap_release(struct inode *inode, struct file *filp); +static long nvmap_ioctl(struct file *filp, unsigned int cmd, unsigned long arg); +static int nvmap_map(struct file *filp, struct vm_area_struct *vma); +#if !defined(CONFIG_MMU) +static unsigned nvmap_mmap_capabilities(struct file *filp); +#endif + +static const struct file_operations nvmap_user_fops = { + .owner = THIS_MODULE, + .open = nvmap_open, + .release = nvmap_release, + .unlocked_ioctl = nvmap_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = nvmap_ioctl, +#endif + .mmap = nvmap_map, +#if !defined(CONFIG_MMU) + .mmap_capabilities = nvmap_mmap_capabilities, +#endif +}; + +static const struct file_operations debug_handles_by_pid_fops; + +struct nvmap_pid_data { + struct rb_node node; + pid_t pid; + struct kref refcount; + struct dentry *handles_file; +}; + +static void nvmap_pid_release_locked(struct kref *kref) +{ + struct nvmap_pid_data *p = container_of(kref, struct nvmap_pid_data, + refcount); + debugfs_remove(p->handles_file); + rb_erase(&p->node, &nvmap_dev->pids); + kfree(p); +} + +static void nvmap_pid_get_locked(struct nvmap_device *dev, pid_t pid) +{ + struct rb_root *root = &dev->pids; + struct rb_node **new = &(root->rb_node), *parent = NULL; + struct nvmap_pid_data *p; + char name[16]; + + while (*new) { + p = container_of(*new, struct nvmap_pid_data, node); + parent = *new; + + if (p->pid > pid) { + new = &((*new)->rb_left); + } else if (p->pid < pid) { + new = &((*new)->rb_right); + } else { + kref_get(&p->refcount); + return; + } + } + + if (snprintf(name, sizeof(name), "%d", pid) < 0) + return; + + p = kzalloc(sizeof(*p), GFP_KERNEL); + if (!p) + return; + + p->pid = pid; + kref_init(&p->refcount); + p->handles_file = debugfs_create_file(name, S_IRUGO, + dev->handles_by_pid, p, + &debug_handles_by_pid_fops); + + if (IS_ERR_OR_NULL(p->handles_file)) { + kfree(p); + } else { + rb_link_node(&p->node, parent, new); + rb_insert_color(&p->node, root); + } +} + +static struct nvmap_pid_data *nvmap_pid_find_locked(struct nvmap_device *dev, + pid_t pid) +{ + struct rb_node *node = dev->pids.rb_node; + + while (node) { + struct nvmap_pid_data *p = container_of(node, + struct nvmap_pid_data, node); + + if (p->pid > pid) + node = node->rb_left; + else if (p->pid < pid) + node = node->rb_right; + else + return p; + } + return NULL; +} + +static void nvmap_pid_put_locked(struct nvmap_device *dev, pid_t pid) +{ + struct nvmap_pid_data *p = nvmap_pid_find_locked(dev, pid); + if (p) + kref_put(&p->refcount, nvmap_pid_release_locked); +} + +struct nvmap_client *__nvmap_create_client(struct nvmap_device *dev, + const char *name) +{ + struct nvmap_client *client; + struct task_struct *task; + + if (WARN_ON(!dev)) + return NULL; + + client = kzalloc(sizeof(*client), GFP_KERNEL); + if (!client) + return NULL; + + client->name = name; + client->kernel_client = true; + client->handle_refs = RB_ROOT; + + get_task_struct(current->group_leader); + task_lock(current->group_leader); + /* don't bother to store task struct for kernel threads, + they can't be killed anyway */ + if (current->flags & PF_KTHREAD) { + put_task_struct(current->group_leader); + task = NULL; + } else { + task = current->group_leader; + } + task_unlock(current->group_leader); + client->task = task; + + mutex_init(&client->ref_lock); + atomic_set(&client->count, 1); + + mutex_lock(&dev->clients_lock); + list_add(&client->list, &dev->clients); + if (!IS_ERR_OR_NULL(dev->handles_by_pid)) { + pid_t pid = nvmap_client_pid(client); + nvmap_pid_get_locked(dev, pid); + } + mutex_unlock(&dev->clients_lock); + return client; +} + +static void destroy_client(struct nvmap_client *client) +{ + struct rb_node *n; + + if (!client) + return; + + mutex_lock(&nvmap_dev->clients_lock); + if (!IS_ERR_OR_NULL(nvmap_dev->handles_by_pid)) { + pid_t pid = nvmap_client_pid(client); + nvmap_pid_put_locked(nvmap_dev, pid); + } + list_del(&client->list); + mutex_unlock(&nvmap_dev->clients_lock); + + while ((n = rb_first(&client->handle_refs))) { + struct nvmap_handle_ref *ref; + int dupes; + + ref = rb_entry(n, struct nvmap_handle_ref, node); + smp_rmb(); + if (ref->handle->owner == client) + ref->handle->owner = NULL; + + if (ref->is_ro) + dma_buf_put(ref->handle->dmabuf_ro); + else + dma_buf_put(ref->handle->dmabuf); + rb_erase(&ref->node, &client->handle_refs); + atomic_dec(&ref->handle->share_count); + + dupes = atomic_read(&ref->dupes); + while (dupes--) + nvmap_handle_put(ref->handle); + + kfree(ref); + } + + if (client->task) + put_task_struct(client->task); + + kfree(client); +} + +static int nvmap_open(struct inode *inode, struct file *filp) +{ + struct miscdevice *miscdev = filp->private_data; + struct nvmap_device *dev = dev_get_drvdata(miscdev->parent); + struct nvmap_client *priv; + int ret; + __attribute__((unused)) struct rlimit old_rlim, new_rlim; + + ret = nonseekable_open(inode, filp); + if (unlikely(ret)) + return ret; + + BUG_ON(dev != nvmap_dev); + priv = __nvmap_create_client(dev, "user"); + if (!priv) + return -ENOMEM; + trace_nvmap_open(priv, priv->name); + + priv->kernel_client = false; + + nvmap_id_array_init(&priv->id_array); + +#ifdef NVMAP_CONFIG_HANDLE_AS_ID + priv->ida = &priv->id_array; +#else + priv->ida = NULL; +#endif + + filp->private_data = priv; + return 0; +} + +static int nvmap_release(struct inode *inode, struct file *filp) +{ + struct nvmap_client *priv = filp->private_data; + + if(!priv) + return 0; + + nvmap_id_array_exit(&priv->id_array); + +#ifdef NVMAP_CONFIG_HANDLE_AS_ID + priv->ida = NULL; +#endif + + trace_nvmap_release(priv, priv->name); + + if (!atomic_dec_return(&priv->count)) + destroy_client(priv); + + return 0; +} + +static int nvmap_map(struct file *filp, struct vm_area_struct *vma) +{ + char task_comm[TASK_COMM_LEN]; + + get_task_comm(task_comm, current); + pr_debug("error: mmap not supported on nvmap file, pid=%d, %s\n", + task_tgid_nr(current), task_comm); + return -EPERM; +} + +static long nvmap_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + int err = 0; + void __user *uarg = (void __user *)arg; + + if (_IOC_TYPE(cmd) != NVMAP_IOC_MAGIC) + return -ENOTTY; + + if (_IOC_NR(cmd) > NVMAP_IOC_MAXNR) + return -ENOTTY; + + if (_IOC_DIR(cmd) & _IOC_READ) + err = !ACCESS_OK(VERIFY_WRITE, uarg, _IOC_SIZE(cmd)); + if (!err && (_IOC_DIR(cmd) & _IOC_WRITE)) + err = !ACCESS_OK(VERIFY_READ, uarg, _IOC_SIZE(cmd)); + + if (err) + return -EFAULT; + + err = -ENOTTY; + + switch (cmd) { + case NVMAP_IOC_CREATE: + case NVMAP_IOC_CREATE_64: + case NVMAP_IOC_FROM_FD: + err = nvmap_ioctl_create(filp, cmd, uarg); + break; + + case NVMAP_IOC_FROM_VA: + err = nvmap_ioctl_create_from_va(filp, uarg); + break; + + case NVMAP_IOC_GET_FD: + err = nvmap_ioctl_getfd(filp, uarg); + break; + + case NVMAP_IOC_GET_IVM_HEAPS: + err = nvmap_ioctl_get_ivc_heap(filp, uarg); + break; + + case NVMAP_IOC_FROM_IVC_ID: + err = nvmap_ioctl_create_from_ivc(filp, uarg); + break; + + case NVMAP_IOC_GET_IVC_ID: + err = nvmap_ioctl_get_ivcid(filp, uarg); + break; + + case NVMAP_IOC_ALLOC: + err = nvmap_ioctl_alloc(filp, uarg); + break; + + case NVMAP_IOC_ALLOC_IVM: + err = nvmap_ioctl_alloc_ivm(filp, uarg); + break; + + case NVMAP_IOC_VPR_FLOOR_SIZE: + err = nvmap_ioctl_vpr_floor_size(filp, uarg); + break; + + case NVMAP_IOC_FREE: + err = nvmap_ioctl_free(filp, arg); + break; + + case NVMAP_IOC_DUP_HANDLE: + err = nvmap_ioctl_dup_handle(filp, uarg); + break; + +#ifdef CONFIG_COMPAT + case NVMAP_IOC_WRITE_32: + case NVMAP_IOC_READ_32: + err = nvmap_ioctl_rw_handle(filp, cmd == NVMAP_IOC_READ_32, + uarg, sizeof(struct nvmap_rw_handle_32)); + break; +#endif + + case NVMAP_IOC_WRITE: + case NVMAP_IOC_READ: + err = nvmap_ioctl_rw_handle(filp, cmd == NVMAP_IOC_READ, uarg, + sizeof(struct nvmap_rw_handle)); + break; + +#ifdef CONFIG_COMPAT + case NVMAP_IOC_CACHE_32: + err = nvmap_ioctl_cache_maint(filp, uarg, + sizeof(struct nvmap_cache_op_32)); + break; +#endif + + case NVMAP_IOC_CACHE: + err = nvmap_ioctl_cache_maint(filp, uarg, + sizeof(struct nvmap_cache_op)); + break; + + case NVMAP_IOC_CACHE_64: + err = nvmap_ioctl_cache_maint(filp, uarg, + sizeof(struct nvmap_cache_op_64)); + break; + + case NVMAP_IOC_CACHE_LIST: + err = nvmap_ioctl_cache_maint_list(filp, uarg); + break; + + case NVMAP_IOC_GUP_TEST: + err = nvmap_ioctl_gup_test(filp, uarg); + break; + + case NVMAP_IOC_FROM_ID: + case NVMAP_IOC_GET_ID: + pr_warn("NVMAP_IOC_GET_ID/FROM_ID pair is deprecated. " + "Use the pair NVMAP_IOC_GET_FD/FROM_FD.\n"); + break; + + case NVMAP_IOC_SET_TAG_LABEL: + err = nvmap_ioctl_set_tag_label(filp, uarg); + break; + + case NVMAP_IOC_GET_AVAILABLE_HEAPS: + err = nvmap_ioctl_get_available_heaps(filp, uarg); + break; + + case NVMAP_IOC_GET_HEAP_SIZE: + err = nvmap_ioctl_get_heap_size(filp, uarg); + break; + + case NVMAP_IOC_PARAMETERS: + err = nvmap_ioctl_get_handle_parameters(filp, uarg); + break; + + case NVMAP_IOC_GET_SCIIPCID: + err = nvmap_ioctl_get_sci_ipc_id(filp, uarg); + break; + + case NVMAP_IOC_HANDLE_FROM_SCIIPCID: + err = nvmap_ioctl_handle_from_sci_ipc_id(filp, uarg); + break; + + case NVMAP_IOC_QUERY_HEAP_PARAMS: + err = nvmap_ioctl_query_heap_params(filp, uarg); + break; + + default: + pr_warn("Unknown NVMAP_IOC = 0x%x\n", cmd); + } + return err; +} + +#define DEBUGFS_OPEN_FOPS_STATIC(name) \ +static int nvmap_debug_##name##_open(struct inode *inode, \ + struct file *file) \ +{ \ + return single_open(file, nvmap_debug_##name##_show, \ + inode->i_private); \ +} \ +\ +static const struct file_operations debug_##name##_fops = { \ + .open = nvmap_debug_##name##_open, \ + .read = seq_read, \ + .llseek = seq_lseek, \ + .release = single_release, \ +} + +#define DEBUGFS_OPEN_FOPS(name) \ +static int nvmap_debug_##name##_open(struct inode *inode, \ + struct file *file) \ +{ \ + return single_open(file, nvmap_debug_##name##_show, \ + inode->i_private); \ +} \ +\ +const struct file_operations debug_##name##_fops = { \ + .open = nvmap_debug_##name##_open, \ + .read = seq_read, \ + .llseek = seq_lseek, \ + .release = single_release, \ +} + +#define K(x) (x >> 10) + +static void client_stringify(struct nvmap_client *client, struct seq_file *s) +{ + char task_comm[TASK_COMM_LEN]; + if (!client->task) { + seq_printf(s, "%-18s %18s %8u", client->name, "kernel", 0); + return; + } + get_task_comm(task_comm, client->task); + seq_printf(s, "%-18s %18s %8u", client->name, task_comm, + client->task->pid); +} + +static void allocations_stringify(struct nvmap_client *client, + struct seq_file *s, u32 heap_type) +{ + struct rb_node *n; + unsigned int pin_count = 0; + struct nvmap_device *dev = nvmap_dev; + + nvmap_ref_lock(client); + mutex_lock(&dev->tags_lock); + n = rb_first(&client->handle_refs); + for (; n != NULL; n = rb_next(n)) { + struct nvmap_handle_ref *ref = + rb_entry(n, struct nvmap_handle_ref, node); + struct nvmap_handle *handle = ref->handle; + if (handle->alloc && handle->heap_type == heap_type) { + phys_addr_t base = heap_type == NVMAP_HEAP_IOVMM ? 0 : + handle->heap_pgalloc ? 0 : + (handle->carveout->base); + size_t size = K(handle->size); + int i = 0; + +next_page: + if ((heap_type == NVMAP_HEAP_CARVEOUT_VPR) && handle->heap_pgalloc) { + base = page_to_phys(handle->pgalloc.pages[i++]); + size = K(PAGE_SIZE); + } + + seq_printf(s, + "%-18s %-18s %8llx %10zuK %8x %6u %6u %6u %6u %6u %6u %8pK %s\n", + "", "", + (unsigned long long)base, size, + handle->userflags, + atomic_read(&handle->ref), + atomic_read(&ref->dupes), + pin_count, + atomic_read(&handle->kmap_count), + atomic_read(&handle->umap_count), + atomic_read(&handle->share_count), + handle, + __nvmap_tag_name(dev, handle->userflags >> 16)); + + if ((heap_type == NVMAP_HEAP_CARVEOUT_VPR) && handle->heap_pgalloc) { + i++; + if (i < (handle->size >> PAGE_SHIFT)) + goto next_page; + } + } + } + mutex_unlock(&dev->tags_lock); + nvmap_ref_unlock(client); +} + +bool is_nvmap_memory_available(size_t size, uint32_t heap) +{ + unsigned long total_num_pages; + unsigned int carveout_mask = NVMAP_HEAP_CARVEOUT_MASK; + unsigned int iovmm_mask = NVMAP_HEAP_IOVMM; + struct nvmap_device *dev = nvmap_dev; + bool heap_present = false; + int i; + + if (!heap) + return false; + + if (nvmap_convert_carveout_to_iovmm) { + carveout_mask &= ~NVMAP_HEAP_CARVEOUT_GENERIC; + iovmm_mask |= NVMAP_HEAP_CARVEOUT_GENERIC; + } else if (nvmap_convert_iovmm_to_carveout) { + if (heap & NVMAP_HEAP_IOVMM) { + heap &= ~NVMAP_HEAP_IOVMM; + heap |= NVMAP_HEAP_CARVEOUT_GENERIC; + } + } + + if (heap & iovmm_mask) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 0, 0) + total_num_pages = totalram_pages(); +#else + total_num_pages = totalram_pages; +#endif + if ((size >> PAGE_SHIFT) > total_num_pages) { + pr_debug("Requested size is more than available memory\n"); + pr_debug("Requested size : %lu B, Available memory : %lu B\n", size, + total_num_pages << PAGE_SHIFT); + return false; + } + return true; + } + + for (i = 0; i < dev->nr_carveouts; i++) { + struct nvmap_carveout_node *co_heap; + struct nvmap_heap *h; + + co_heap = &dev->heaps[i]; + if (!(co_heap->heap_bit & heap)) + continue; + + heap_present = true; + h = co_heap->carveout; + if (size > h->free_size) { + pr_debug("Requested size is more than available memory"); + pr_debug("Requested size : %lu B, Available memory : %lu B\n", size, + h->free_size); + return false; + } + break; + } + return heap_present; +} + +/* compute the total amount of handle physical memory that is mapped + * into client's virtual address space. Remember that vmas list is + * sorted in ascending order of handle offsets. + * NOTE: This function should be called while holding handle's lock mutex. + */ +static void nvmap_get_client_handle_mss(struct nvmap_client *client, + struct nvmap_handle *handle, u64 *total) +{ + struct nvmap_vma_list *vma_list = NULL; + struct vm_area_struct *vma = NULL; + u64 end_offset = 0, vma_start_offset, vma_size; + int64_t overlap_size; + + *total = 0; + list_for_each_entry(vma_list, &handle->vmas, list) { + + if (client->task->pid == vma_list->pid) { + vma = vma_list->vma; + vma_size = vma->vm_end - vma->vm_start; + + vma_start_offset = vma->vm_pgoff << PAGE_SHIFT; + if (end_offset < vma_start_offset + vma_size) { + *total += vma_size; + + overlap_size = end_offset - vma_start_offset; + if (overlap_size > 0) + *total -= overlap_size; + end_offset = vma_start_offset + vma_size; + } + } + } +} + +static void maps_stringify(struct nvmap_client *client, + struct seq_file *s, u32 heap_type) +{ + struct rb_node *n; + struct nvmap_vma_list *vma_list = NULL; + struct vm_area_struct *vma = NULL; + u64 total_mapped_size, vma_size; + + nvmap_ref_lock(client); + n = rb_first(&client->handle_refs); + for (; n != NULL; n = rb_next(n)) { + struct nvmap_handle_ref *ref = + rb_entry(n, struct nvmap_handle_ref, node); + struct nvmap_handle *handle = ref->handle; + if (handle->alloc && handle->heap_type == heap_type) { + phys_addr_t base = heap_type == NVMAP_HEAP_IOVMM ? 0 : + handle->heap_pgalloc ? 0 : + (handle->carveout->base); + size_t size = K(handle->size); + int i = 0; + +next_page: + if ((heap_type == NVMAP_HEAP_CARVEOUT_VPR) && handle->heap_pgalloc) { + base = page_to_phys(handle->pgalloc.pages[i++]); + size = K(PAGE_SIZE); + } + + seq_printf(s, + "%-18s %-18s %8llx %10zuK %8x %6u %16pK " + "%12s %12s ", + "", "", + (unsigned long long)base, K(handle->size), + handle->userflags, + atomic_read(&handle->share_count), + handle, "", ""); + + if ((heap_type == NVMAP_HEAP_CARVEOUT_VPR) && handle->heap_pgalloc) { + i++; + if (i < (handle->size >> PAGE_SHIFT)) + goto next_page; + } + + mutex_lock(&handle->lock); + nvmap_get_client_handle_mss(client, handle, + &total_mapped_size); + seq_printf(s, "%6lluK\n", K(total_mapped_size)); + + list_for_each_entry(vma_list, &handle->vmas, list) { + + if (vma_list->pid == client->task->pid) { + vma = vma_list->vma; + vma_size = vma->vm_end - vma->vm_start; + seq_printf(s, + "%-18s %-18s %8s %11s %8s %6s %16s " + "%-12lx-%12lx %6lluK\n", + "", "", "", "", "", "", "", + vma->vm_start, vma->vm_end, + K(vma_size)); + } + } + mutex_unlock(&handle->lock); + } + } + nvmap_ref_unlock(client); +} + +static void nvmap_get_client_mss(struct nvmap_client *client, + u64 *total, u32 heap_type) +{ + struct rb_node *n; + + *total = 0; + nvmap_ref_lock(client); + n = rb_first(&client->handle_refs); + for (; n != NULL; n = rb_next(n)) { + struct nvmap_handle_ref *ref = + rb_entry(n, struct nvmap_handle_ref, node); + struct nvmap_handle *handle = ref->handle; + if (handle->alloc && handle->heap_type == heap_type) + *total += handle->size / + atomic_read(&handle->share_count); + } + nvmap_ref_unlock(client); +} + +#define PSS_SHIFT 12 +static void nvmap_get_total_mss(u64 *pss, u64 *total, u32 heap_type) +{ + int i; + struct rb_node *n; + struct nvmap_device *dev = nvmap_dev; + + *total = 0; + if (pss) + *pss = 0; + if (!dev) + return; + spin_lock(&dev->handle_lock); + n = rb_first(&dev->handles); + for (; n != NULL; n = rb_next(n)) { + struct nvmap_handle *h = + rb_entry(n, struct nvmap_handle, node); + + if (!h || !h->alloc || h->heap_type != heap_type) + continue; + + *total += h->size; + if (!pss) + continue; + + for (i = 0; i < h->size >> PAGE_SHIFT; i++) { + struct page *page = nvmap_to_page(h->pgalloc.pages[i]); + + if (page_mapcount(page) > 0) + *pss += PAGE_SIZE; + } + } + spin_unlock(&dev->handle_lock); +} + +static int nvmap_debug_allocations_show(struct seq_file *s, void *unused) +{ + u64 total; + struct nvmap_client *client; + u32 heap_type = (u32)(uintptr_t)s->private; + + mutex_lock(&nvmap_dev->clients_lock); + seq_printf(s, "%-18s %18s %8s %11s\n", + "CLIENT", "PROCESS", "PID", "SIZE"); + seq_printf(s, "%-18s %18s %8s %11s %8s %6s %6s %6s %6s %6s %6s %8s\n", + "", "", "BASE", "SIZE", "FLAGS", "REFS", + "DUPES", "PINS", "KMAPS", "UMAPS", "SHARE", "UID"); + list_for_each_entry(client, &nvmap_dev->clients, list) { + u64 client_total; + client_stringify(client, s); + nvmap_get_client_mss(client, &client_total, heap_type); + seq_printf(s, " %10lluK\n", K(client_total)); + allocations_stringify(client, s, heap_type); + seq_printf(s, "\n"); + } + mutex_unlock(&nvmap_dev->clients_lock); + nvmap_get_total_mss(NULL, &total, heap_type); + seq_printf(s, "%-18s %-18s %8s %10lluK\n", "total", "", "", K(total)); + return 0; +} + +DEBUGFS_OPEN_FOPS(allocations); + +static int nvmap_debug_free_size_show(struct seq_file *s, void *unused) +{ + unsigned long free_mem = 0; + + if (system_heap_free_mem(&free_mem)) + seq_printf(s, "Error while fetching free size of IOVMM memory\n"); + else + seq_printf(s, "Max allocatable IOVMM memory: %lu bytes\n", free_mem); + return 0; +} +DEBUGFS_OPEN_FOPS(free_size); + +#ifdef NVMAP_CONFIG_DEBUG_MAPS +static int nvmap_debug_device_list_show(struct seq_file *s, void *unused) +{ + u32 heap_type = (u32)(uintptr_t)s->private; + struct rb_node *n = NULL; + struct nvmap_device_list *dl = NULL; + int i; + + if (heap_type == NVMAP_HEAP_IOVMM) { + n = rb_first(&nvmap_dev->device_names); + } else { + /* Iterate over all heaps to find the matching heap */ + for (i = 0; i < nvmap_dev->nr_carveouts; i++) { + if (heap_type & nvmap_dev->heaps[i].heap_bit) { + if (nvmap_dev->heaps[i].carveout) { + n = rb_first(&nvmap_dev->heaps[i].carveout->device_names); + break; + } + } + } + } + if (n) { + seq_printf(s, "Device list is\n"); + for (; n != NULL; n = rb_next(n)) { + dl = rb_entry(n, struct nvmap_device_list, node); + seq_printf(s, "%s %llu\n", dl->device_name, dl->dma_mask); + } + } + return 0; +} +DEBUGFS_OPEN_FOPS(device_list); +#endif /* NVMAP_CONFIG_DEBUG_MAPS */ + +static int nvmap_debug_all_allocations_show(struct seq_file *s, void *unused) +{ + u32 heap_type = (u32)(uintptr_t)s->private; + struct rb_node *n; + + + spin_lock(&nvmap_dev->handle_lock); + seq_printf(s, "%8s %11s %9s %6s %6s %6s %6s %8s\n", + "BASE", "SIZE", "USERFLAGS", "REFS", + "KMAPS", "UMAPS", "SHARE", "UID"); + + /* for each handle */ + n = rb_first(&nvmap_dev->handles); + for (; n != NULL; n = rb_next(n)) { + struct nvmap_handle *handle = + rb_entry(n, struct nvmap_handle, node); + int i = 0; + + if (handle->alloc && handle->heap_type == heap_type) { + phys_addr_t base = heap_type == NVMAP_HEAP_IOVMM ? 0 : + handle->heap_pgalloc ? 0 : + (handle->carveout->base); + size_t size = K(handle->size); + +next_page: + if ((heap_type == NVMAP_HEAP_CARVEOUT_VPR) && handle->heap_pgalloc) { + base = page_to_phys(handle->pgalloc.pages[i++]); + size = K(PAGE_SIZE); + } + + seq_printf(s, + "%8llx %10zuK %9x %6u %6u %6u %6u %8p\n", + (unsigned long long)base, K(handle->size), + handle->userflags, + atomic_read(&handle->ref), + atomic_read(&handle->kmap_count), + atomic_read(&handle->umap_count), + atomic_read(&handle->share_count), + handle); + + if ((heap_type == NVMAP_HEAP_CARVEOUT_VPR) && handle->heap_pgalloc) { + i++; + if (i < (handle->size >> PAGE_SHIFT)) + goto next_page; + } + } + } + + spin_unlock(&nvmap_dev->handle_lock); + + return 0; +} + +DEBUGFS_OPEN_FOPS(all_allocations); + +static int nvmap_debug_orphan_handles_show(struct seq_file *s, void *unused) +{ + u32 heap_type = (u32)(uintptr_t)s->private; + struct rb_node *n; + + + spin_lock(&nvmap_dev->handle_lock); + seq_printf(s, "%8s %11s %9s %6s %6s %6s %8s\n", + "BASE", "SIZE", "USERFLAGS", "REFS", + "KMAPS", "UMAPS", "UID"); + + /* for each handle */ + n = rb_first(&nvmap_dev->handles); + for (; n != NULL; n = rb_next(n)) { + struct nvmap_handle *handle = + rb_entry(n, struct nvmap_handle, node); + int i = 0; + + if (handle->alloc && handle->heap_type == heap_type && + !atomic_read(&handle->share_count)) { + phys_addr_t base = heap_type == NVMAP_HEAP_IOVMM ? 0 : + handle->heap_pgalloc ? 0 : + (handle->carveout->base); + size_t size = K(handle->size); + +next_page: + if ((heap_type == NVMAP_HEAP_CARVEOUT_VPR) && handle->heap_pgalloc) { + base = page_to_phys(handle->pgalloc.pages[i++]); + size = K(PAGE_SIZE); + } + + seq_printf(s, + "%8llx %10zuK %9x %6u %6u %6u %8p\n", + (unsigned long long)base, K(handle->size), + handle->userflags, + atomic_read(&handle->ref), + atomic_read(&handle->kmap_count), + atomic_read(&handle->umap_count), + handle); + + if ((heap_type == NVMAP_HEAP_CARVEOUT_VPR) && handle->heap_pgalloc) { + i++; + if (i < (handle->size >> PAGE_SHIFT)) + goto next_page; + } + } + } + + spin_unlock(&nvmap_dev->handle_lock); + + return 0; +} + +DEBUGFS_OPEN_FOPS(orphan_handles); + +static int nvmap_debug_maps_show(struct seq_file *s, void *unused) +{ + u64 total; + struct nvmap_client *client; + u32 heap_type = (u32)(uintptr_t)s->private; + + mutex_lock(&nvmap_dev->clients_lock); + seq_printf(s, "%-18s %18s %8s %11s\n", + "CLIENT", "PROCESS", "PID", "SIZE"); + seq_printf(s, "%-18s %18s %8s %11s %8s %6s %9s %21s %18s\n", + "", "", "BASE", "SIZE", "FLAGS", "SHARE", "UID", + "MAPS", "MAPSIZE"); + + list_for_each_entry(client, &nvmap_dev->clients, list) { + u64 client_total; + client_stringify(client, s); + nvmap_get_client_mss(client, &client_total, heap_type); + seq_printf(s, " %10lluK\n", K(client_total)); + maps_stringify(client, s, heap_type); + seq_printf(s, "\n"); + } + mutex_unlock(&nvmap_dev->clients_lock); + + nvmap_get_total_mss(NULL, &total, heap_type); + seq_printf(s, "%-18s %-18s %8s %10lluK\n", "total", "", "", K(total)); + return 0; +} + +DEBUGFS_OPEN_FOPS(maps); + +static int nvmap_debug_clients_show(struct seq_file *s, void *unused) +{ + u64 total; + struct nvmap_client *client; + ulong heap_type = (ulong)s->private; + + mutex_lock(&nvmap_dev->clients_lock); + seq_printf(s, "%-18s %18s %8s %11s\n", + "CLIENT", "PROCESS", "PID", "SIZE"); + list_for_each_entry(client, &nvmap_dev->clients, list) { + u64 client_total; + client_stringify(client, s); + nvmap_get_client_mss(client, &client_total, heap_type); + seq_printf(s, " %10lluK\n", K(client_total)); + } + mutex_unlock(&nvmap_dev->clients_lock); + nvmap_get_total_mss(NULL, &total, heap_type); + seq_printf(s, "%-18s %18s %8s %10lluK\n", "total", "", "", K(total)); + return 0; +} + +DEBUGFS_OPEN_FOPS(clients); + +static int nvmap_debug_handles_by_pid_show_client(struct seq_file *s, + struct nvmap_client *client) +{ + struct rb_node *n; + int ret = 0; + + nvmap_ref_lock(client); + n = rb_first(&client->handle_refs); + for (; n != NULL; n = rb_next(n)) { + struct nvmap_handle_ref *ref = rb_entry(n, + struct nvmap_handle_ref, node); + struct nvmap_handle *handle = ref->handle; + struct nvmap_debugfs_handles_entry entry; + u64 total_mapped_size; + int i = 0; + + if (!handle->alloc) + continue; + + mutex_lock(&handle->lock); + nvmap_get_client_handle_mss(client, handle, &total_mapped_size); + mutex_unlock(&handle->lock); + + entry.base = handle->heap_type == NVMAP_HEAP_IOVMM ? 0 : + handle->heap_pgalloc ? 0 : + (handle->carveout->base); + entry.size = handle->size; + entry.flags = handle->userflags; + entry.share_count = atomic_read(&handle->share_count); + entry.mapped_size = total_mapped_size; + +next_page: + if ((handle->heap_type == NVMAP_HEAP_CARVEOUT_VPR) && handle->heap_pgalloc) { + entry.base = page_to_phys(handle->pgalloc.pages[i++]); + entry.size = K(PAGE_SIZE); + } + + seq_printf(s, "%llu %12llu %8u %8u %10llu\n", entry.base, entry.size, + entry.flags, entry.share_count, entry.mapped_size); + + if ((handle->heap_type == NVMAP_HEAP_CARVEOUT_VPR) && handle->heap_pgalloc) { + i++; + if (i < (handle->size >> PAGE_SHIFT)) + goto next_page; + } + } + nvmap_ref_unlock(client); + + return ret; +} + +static int nvmap_debug_handles_by_pid_show(struct seq_file *s, void *unused) +{ + struct nvmap_pid_data *p = s->private; + struct nvmap_client *client; + struct nvmap_debugfs_handles_header header; + int ret = 0; + + header.version = 1; + seq_printf(s, "%s: %u\n", "header.version", header.version); + seq_printf(s, "%s %8s %8s %12s %8s\n", "base", + "size", "flags", "share_count", "mapped_size"); + mutex_lock(&nvmap_dev->clients_lock); + + list_for_each_entry(client, &nvmap_dev->clients, list) { + if (nvmap_client_pid(client) != p->pid) + continue; + + ret = nvmap_debug_handles_by_pid_show_client(s, client); + if (ret < 0) + break; + } + + mutex_unlock(&nvmap_dev->clients_lock); + return ret; +} + +DEBUGFS_OPEN_FOPS_STATIC(handles_by_pid); + +#define PRINT_MEM_STATS_NOTE(x) \ +do { \ + seq_printf(s, "Note: total memory is precise account of pages " \ + "allocated by NvMap.\nIt doesn't match with all clients " \ + "\"%s\" accumulated as shared memory \nis accounted in " \ + "full in each clients \"%s\" that shared memory.\n", #x, #x); \ +} while (0) + +#ifdef NVMAP_CONFIG_PROCRANK +struct procrank_stats { + struct vm_area_struct *vma; + u64 pss; +}; + +static int procrank_pte_entry(pte_t *pte, unsigned long addr, unsigned long end, + struct mm_walk *walk) +{ + struct procrank_stats *mss = walk->private; + struct vm_area_struct *vma = mss->vma; + struct page *page = NULL; + int mapcount; + + if (pte_present(*pte)) + page = vm_normal_page(vma, addr, *pte); + else if (is_swap_pte(*pte)) { + swp_entry_t swpent = pte_to_swp_entry(*pte); + + if (is_migration_entry(swpent)) + page = migration_entry_to_page(swpent); + } + + if (!page) + return 0; + + mapcount = page_mapcount(page); + if (mapcount >= 2) + mss->pss += (PAGE_SIZE << PSS_SHIFT) / mapcount; + else + mss->pss += (PAGE_SIZE << PSS_SHIFT); + + return 0; +} + +#ifndef PTRACE_MODE_READ_FSCREDS +#define PTRACE_MODE_READ_FSCREDS PTRACE_MODE_READ +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0) +static void nvmap_iovmm_get_client_mss(struct nvmap_client *client, u64 *pss, + u64 *total) +{ + struct rb_node *n; + struct nvmap_vma_list *tmp; + struct procrank_stats mss; + struct mm_walk procrank_walk = { + .pte_entry = procrank_pte_entry, + .private = &mss, + }; + struct mm_struct *mm; + + memset(&mss, 0, sizeof(mss)); + *pss = *total = 0; + + mm = mm_access(client->task, + PTRACE_MODE_READ_FSCREDS); + if (!mm || IS_ERR(mm)) return; + + nvmap_acquire_mmap_read_lock(mm); + procrank_walk.mm = mm; + + nvmap_ref_lock(client); + n = rb_first(&client->handle_refs); + for (; n != NULL; n = rb_next(n)) { + struct nvmap_handle_ref *ref = + rb_entry(n, struct nvmap_handle_ref, node); + struct nvmap_handle *h = ref->handle; + + if (!h || !h->alloc || !h->heap_pgalloc) + continue; + + mutex_lock(&h->lock); + list_for_each_entry(tmp, &h->vmas, list) { + if (client->task->pid == tmp->pid) { + mss.vma = tmp->vma; + walk_page_range(tmp->vma->vm_start, + tmp->vma->vm_end, + &procrank_walk); + } + } + mutex_unlock(&h->lock); + *total += h->size / atomic_read(&h->share_count); + } + + nvmap_release_mmap_read_lock(mm); + mmput(mm); + *pss = (mss.pss >> PSS_SHIFT); + nvmap_ref_unlock(client); +} +#else +static void nvmap_iovmm_get_client_mss(struct nvmap_client *client, u64 *pss, + u64 *total) +{ + struct mm_walk_ops wk_ops = { + .pte_entry = procrank_pte_entry, + }; + struct rb_node *n; + struct nvmap_vma_list *tmp; + struct procrank_stats mss; + struct mm_walk procrank_walk = { + .ops = &wk_ops, + .private = &mss, + }; + struct mm_struct *mm; + + memset(&mss, 0, sizeof(mss)); + *pss = *total = 0; + + mm = mm_access(client->task, + PTRACE_MODE_READ_FSCREDS); + if (!mm || IS_ERR(mm)) return; + + nvmap_acquire_mmap_read_lock(mm); + procrank_walk.mm = mm; + + nvmap_ref_lock(client); + n = rb_first(&client->handle_refs); + for (; n != NULL; n = rb_next(n)) { + struct nvmap_handle_ref *ref = + rb_entry(n, struct nvmap_handle_ref, node); + struct nvmap_handle *h = ref->handle; + + if (!h || !h->alloc || !h->heap_pgalloc) + continue; + + mutex_lock(&h->lock); + list_for_each_entry(tmp, &h->vmas, list) { + if (client->task->pid == tmp->pid) { + mss.vma = tmp->vma; + walk_page_range(procrank_walk.mm, + tmp->vma->vm_start, + tmp->vma->vm_end, + procrank_walk.ops, + procrank_walk.private); + } + } + mutex_unlock(&h->lock); + *total += h->size / atomic_read(&h->share_count); + } + + nvmap_release_mmap_read_lock(mm); + mmput(mm); + *pss = (mss.pss >> PSS_SHIFT); + nvmap_ref_unlock(client); +} +#endif + +static int nvmap_debug_iovmm_procrank_show(struct seq_file *s, void *unused) +{ + u64 pss, total; + struct nvmap_client *client; + struct nvmap_device *dev = s->private; + u64 total_memory, total_pss; + + mutex_lock(&dev->clients_lock); + seq_printf(s, "%-18s %18s %8s %11s %11s\n", + "CLIENT", "PROCESS", "PID", "PSS", "SIZE"); + list_for_each_entry(client, &dev->clients, list) { + client_stringify(client, s); + nvmap_iovmm_get_client_mss(client, &pss, &total); + seq_printf(s, " %10lluK %10lluK\n", K(pss), K(total)); + } + mutex_unlock(&dev->clients_lock); + + nvmap_get_total_mss(&total_pss, &total_memory, NVMAP_HEAP_IOVMM); + seq_printf(s, "%-18s %18s %8s %10lluK %10lluK\n", + "total", "", "", K(total_pss), K(total_memory)); + return 0; +} + +DEBUGFS_OPEN_FOPS(iovmm_procrank); +#endif /* NVMAP_CONFIG_PROCRANK */ + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 15, 0) +ulong nvmap_iovmm_get_used_pages(void) +{ + u64 total; + + nvmap_get_total_mss(NULL, &total, NVMAP_HEAP_IOVMM); + return total >> PAGE_SHIFT; +} +#endif + +static void nvmap_iovmm_debugfs_init(void) +{ + if (!IS_ERR_OR_NULL(nvmap_dev->debug_root)) { + struct dentry *iovmm_root = + debugfs_create_dir("iovmm", nvmap_dev->debug_root); + if (!IS_ERR_OR_NULL(iovmm_root)) { + debugfs_create_file("clients", S_IRUGO, iovmm_root, + (void *)(uintptr_t)NVMAP_HEAP_IOVMM, + &debug_clients_fops); + debugfs_create_file("allocations", S_IRUGO, iovmm_root, + (void *)(uintptr_t)NVMAP_HEAP_IOVMM, + &debug_allocations_fops); + debugfs_create_file("all_allocations", S_IRUGO, + iovmm_root, (void *)(uintptr_t)NVMAP_HEAP_IOVMM, + &debug_all_allocations_fops); + debugfs_create_file("orphan_handles", S_IRUGO, + iovmm_root, (void *)(uintptr_t)NVMAP_HEAP_IOVMM, + &debug_orphan_handles_fops); + debugfs_create_file("maps", S_IRUGO, iovmm_root, + (void *)(uintptr_t)NVMAP_HEAP_IOVMM, + &debug_maps_fops); + debugfs_create_file("free_size", S_IRUGO, iovmm_root, + (void *)(uintptr_t)NVMAP_HEAP_IOVMM, + &debug_free_size_fops); +#ifdef NVMAP_CONFIG_DEBUG_MAPS + debugfs_create_file("device_list", S_IRUGO, iovmm_root, + (void *)(uintptr_t)NVMAP_HEAP_IOVMM, + &debug_device_list_fops); +#endif /* NVMAP_CONFIG_DEBUG_MAPS */ + +#ifdef NVMAP_CONFIG_PROCRANK + debugfs_create_file("procrank", S_IRUGO, iovmm_root, + nvmap_dev, &debug_iovmm_procrank_fops); +#endif + } + } +} + +static bool nvmap_is_iommu_present(void) +{ + struct device_node *np; + struct property *prop; + + np = of_find_node_by_name(NULL, "iommu"); + while (np) { + prop = of_find_property(np, "status", NULL); + if (prop && !strcmp(prop->value, "okay")) { + of_node_put(np); + return true; + } + of_node_put(np); + np = of_find_node_by_name(np, "iommu"); + } + + return false; +} + +int __init nvmap_probe(struct platform_device *pdev) +{ + struct nvmap_platform_data *plat; + struct nvmap_device *dev; + struct dentry *nvmap_debug_root; + unsigned int i; + int e; + int generic_carveout_present = 0; + ulong start_time = sched_clock(); + + if (WARN_ON(nvmap_dev != NULL)) { + dev_err(&pdev->dev, "only one nvmap device may be present\n"); + e = -ENODEV; + goto finish; + } + + dev = devm_kzalloc(&pdev->dev, sizeof(*dev), GFP_KERNEL); + if (!dev) { + dev_err(&pdev->dev, "out of memory for device\n"); + e = -ENOMEM; + goto finish; + } + + nvmap_init(pdev); + + plat = pdev->dev.platform_data; +#ifndef NVMAP_LOADABLE_MODULE + if (!plat) { + dev_err(&pdev->dev, "no platform data?\n"); + e = -ENODEV; + goto finish; + } +#endif /* !NVMAP_LOADABLE_MODULE */ + + nvmap_dev = dev; + nvmap_dev->plat = plat; + + /* + * dma_parms need to be set with desired max_segment_size to avoid + * DMA map API returning multiple IOVA's for the buffer size > 64KB. + */ + pdev->dev.dma_parms = &nvmap_dma_parameters; + dev->dev_user.minor = MISC_DYNAMIC_MINOR; + dev->dev_user.name = "nvmap"; + dev->dev_user.fops = &nvmap_user_fops; + dev->dev_user.parent = &pdev->dev; + dev->handles = RB_ROOT; + +#ifdef NVMAP_CONFIG_PAGE_POOLS + e = nvmap_page_pool_init(dev); + if (e) + goto fail; +#endif + + spin_lock_init(&dev->handle_lock); + INIT_LIST_HEAD(&dev->clients); + dev->pids = RB_ROOT; + mutex_init(&dev->clients_lock); + INIT_LIST_HEAD(&dev->lru_handles); + spin_lock_init(&dev->lru_lock); + dev->tags = RB_ROOT; + mutex_init(&dev->tags_lock); + mutex_init(&dev->carveout_lock); + + e = misc_register(&dev->dev_user); + if (e) { + dev_err(&pdev->dev, "unable to register miscdevice %s\n", + dev->dev_user.name); + goto fail; + } + + nvmap_debug_root = debugfs_create_dir("nvmap", NULL); + nvmap_dev->debug_root = nvmap_debug_root; + if (IS_ERR_OR_NULL(nvmap_debug_root)) + dev_err(&pdev->dev, "couldn't create debug files\n"); + else { + debugfs_create_u32("max_handle_count", S_IRUGO, + nvmap_debug_root, &nvmap_max_handle_count); + nvmap_dev->handles_by_pid = debugfs_create_dir("handles_by_pid", + nvmap_debug_root); +#if defined(CONFIG_DEBUG_FS) + debugfs_create_ulong("nvmap_init_time", S_IRUGO | S_IWUSR, + nvmap_dev->debug_root, &nvmap_init_time); +#endif + } + nvmap_dev->dynamic_dma_map_mask = ~0U; + nvmap_dev->cpu_access_mask = ~0U; + if (plat) + for (i = 0; i < plat->nr_carveouts; i++) + nvmap_create_carveout(&plat->carveouts[i]); +#ifdef NVMAP_CONFIG_DEBUG_MAPS + nvmap_dev->device_names = RB_ROOT; +#endif /* NVMAP_CONFIG_DEBUG_MAPS */ + nvmap_iovmm_debugfs_init(); +#ifdef NVMAP_CONFIG_PAGE_POOLS + nvmap_page_pool_debugfs_init(nvmap_dev->debug_root); +#endif + nvmap_stats_init(nvmap_debug_root); + platform_set_drvdata(pdev, dev); + + for (i = 0; i < dev->nr_carveouts; i++) + if (dev->heaps[i].heap_bit & NVMAP_HEAP_CARVEOUT_GENERIC) + generic_carveout_present = 1; + + if (generic_carveout_present) { + if (!iommu_present(&platform_bus_type) && + !nvmap_is_iommu_present()) + nvmap_convert_iovmm_to_carveout = 1; + else if (!of_property_read_bool(pdev->dev.of_node, + "dont-convert-iovmm-to-carveout")) + nvmap_convert_iovmm_to_carveout = 1; + } else { + nvmap_convert_carveout_to_iovmm = 1; + } + +#ifdef NVMAP_CONFIG_PAGE_POOLS + if (nvmap_convert_iovmm_to_carveout) + nvmap_page_pool_fini(dev); +#endif + + e = nvmap_sci_ipc_init(); + if (e) + goto fail_heaps; + +#ifdef CVNAS_BUILTIN + if (tegra_get_chip_id() == TEGRA194) { + phys_addr_t cvs_base = nvcvnas_get_cvsram_base(); + size_t cvs_size = nvcvnas_get_cvsram_size(); + + e = nvmap_register_cvsram_carveout(NULL, + cvs_base, cvs_size, + nvcvnas_busy, nvcvnas_idle); + if (e) { + dev_err(&pdev->dev, "failed to register cvsram carveout\n"); + goto fail_sci_ipc; + } + } +#endif /* CVNAS_BUILTIN */ + + goto finish; +#ifdef CVNAS_BUILTIN +fail_sci_ipc: + nvmap_sci_ipc_exit(); +#endif /* CVNAS_BUILTIN */ +fail_heaps: + debugfs_remove_recursive(nvmap_dev->debug_root); + for (i = 0; i < dev->nr_carveouts; i++) { + struct nvmap_carveout_node *node = &dev->heaps[i]; + nvmap_heap_destroy(node->carveout); + } +fail: +#ifdef NVMAP_CONFIG_PAGE_POOLS + nvmap_page_pool_fini(nvmap_dev); +#endif + kfree(dev->heaps); + if (dev->dev_user.minor != MISC_DYNAMIC_MINOR) + misc_deregister(&dev->dev_user); + nvmap_dev = NULL; +finish: + nvmap_init_time += sched_clock() - start_time; + return e; +} + +int nvmap_remove(struct platform_device *pdev) +{ + struct nvmap_device *dev = platform_get_drvdata(pdev); + struct rb_node *n; + struct nvmap_handle *h; + int i; + +#ifdef NVMAP_CONFIG_SCIIPC + nvmap_sci_ipc_exit(); +#endif + debugfs_remove_recursive(dev->debug_root); + misc_deregister(&dev->dev_user); +#ifdef NVMAP_CONFIG_PAGE_POOLS + nvmap_page_pool_clear(); + nvmap_page_pool_fini(nvmap_dev); +#endif + while ((n = rb_first(&dev->handles))) { + h = rb_entry(n, struct nvmap_handle, node); + rb_erase(&h->node, &dev->handles); + kfree(h); + } + + for (i = 0; i < dev->nr_carveouts; i++) { + struct nvmap_carveout_node *node = &dev->heaps[i]; + nvmap_heap_destroy(node->carveout); + } + kfree(dev->heaps); + + nvmap_dev = NULL; + return 0; +} diff --git a/drivers/video/tegra/nvmap/nvmap_dmabuf.c b/drivers/video/tegra/nvmap/nvmap_dmabuf.c new file mode 100644 index 00000000..873fda1d --- /dev/null +++ b/drivers/video/tegra/nvmap/nvmap_dmabuf.c @@ -0,0 +1,666 @@ +/* + * dma_buf exporter for nvmap + * + * Copyright (c) 2012-2022, NVIDIA CORPORATION. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + */ + +#define pr_fmt(fmt) "nvmap: %s() " fmt, __func__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 18, 0) +#include +#endif + +#include + +#include "nvmap_priv.h" +#include "nvmap_ioctl.h" + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0) +#define NVMAP_DMABUF_ATTACH nvmap_dmabuf_attach +#else +#define NVMAP_DMABUF_ATTACH __nvmap_dmabuf_attach +#endif + +static int __nvmap_dmabuf_attach(struct dma_buf *dmabuf, struct device *dev, + struct dma_buf_attachment *attach) +{ + struct nvmap_handle_info *info = dmabuf->priv; + + trace_nvmap_dmabuf_attach(dmabuf, dev); + + dev_dbg(dev, "%s() 0x%p\n", __func__, info->handle); + return 0; +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0) +static int nvmap_dmabuf_attach(struct dma_buf *dmabuf, + struct dma_buf_attachment *attach) +{ + return __nvmap_dmabuf_attach(dmabuf, attach->dev, attach); +} +#endif + +static void nvmap_dmabuf_detach(struct dma_buf *dmabuf, + struct dma_buf_attachment *attach) +{ + struct nvmap_handle_info *info = dmabuf->priv; + + trace_nvmap_dmabuf_detach(dmabuf, attach->dev); + + dev_dbg(attach->dev, "%s() 0x%p\n", __func__, info->handle); +} + +static inline bool access_vpr_phys(struct device *dev) +{ + if (!iommu_get_domain_for_dev(dev)) + return true; + + /* + * Assumes gpu nodes always have DT entry, this is valid as device + * specifying access-vpr-phys will do so through its DT entry. + */ + if (!dev->of_node) + return false; + + return !!of_find_property(dev->of_node, "access-vpr-phys", NULL); +} + +static struct sg_table *nvmap_dmabuf_map_dma_buf(struct dma_buf_attachment *attach, + enum dma_data_direction dir) +{ + struct nvmap_handle_info *info = attach->dmabuf->priv; + int ents = 0; + struct sg_table *sgt; +#ifdef NVMAP_CONFIG_DEBUG_MAPS + char *device_name = NULL; + u32 heap_type; + u64 dma_mask; +#endif /* NVMAP_CONFIG_DEBUG_MAPS */ + DEFINE_DMA_ATTRS(attrs); + + trace_nvmap_dmabuf_map_dma_buf(attach->dmabuf, attach->dev); + + /* + * If the exported buffer is foreign buffer(alloc_from_va) and + * has RO access, don't map it in device space. + * Return error as no access. + */ + if (info->handle->from_va && info->handle->is_ro && + (dir != DMA_TO_DEVICE)) + return ERR_PTR(-EACCES); + + nvmap_lru_reset(info->handle); + mutex_lock(&info->maps_lock); + + atomic_inc(&info->handle->pin); + + sgt = __nvmap_sg_table(NULL, info->handle); + if (IS_ERR(sgt)) { + atomic_dec(&info->handle->pin); + mutex_unlock(&info->maps_lock); + return sgt; + } + + if (!info->handle->alloc) { + goto err_map; + } else if (!(nvmap_dev->dynamic_dma_map_mask & + info->handle->heap_type)) { + sg_dma_address(sgt->sgl) = info->handle->carveout->base; + } else if (info->handle->heap_type == NVMAP_HEAP_CARVEOUT_VPR && + access_vpr_phys(attach->dev)) { + sg_dma_address(sgt->sgl) = 0; + } else { + dma_set_attr(DMA_ATTR_SKIP_CPU_SYNC, __DMA_ATTR(attrs)); + ents = dma_map_sg_attrs(attach->dev, sgt->sgl, + sgt->nents, dir, __DMA_ATTR(attrs)); + if (ents <= 0) + goto err_map; + } + + attach->priv = sgt; + +#ifdef NVMAP_CONFIG_DEBUG_MAPS + /* Insert device name into the carveout's device name rb tree */ + heap_type = info->handle->heap_type; + device_name = (char *)dev_name(attach->dev); + dma_mask = *(attach->dev->dma_mask); + if (device_name && !nvmap_is_device_present(device_name, heap_type)) { + /* If the device name is not already present in the tree, then only add */ + nvmap_add_device_name(device_name, dma_mask, heap_type); + } +#endif /* NVMAP_CONFIG_DEBUG_MAPS */ + mutex_unlock(&info->maps_lock); + return sgt; + +err_map: + __nvmap_free_sg_table(NULL, info->handle, sgt); + atomic_dec(&info->handle->pin); + mutex_unlock(&info->maps_lock); + return ERR_PTR(-ENOMEM); +} + +static void nvmap_dmabuf_unmap_dma_buf(struct dma_buf_attachment *attach, + struct sg_table *sgt, + enum dma_data_direction dir) +{ + struct nvmap_handle_info *info = attach->dmabuf->priv; +#ifdef NVMAP_CONFIG_DEBUG_MAPS + char *device_name = NULL; + u32 heap_type = 0; +#endif /* NVMAP_CONFIG_DEBUG_MAPS */ + + trace_nvmap_dmabuf_unmap_dma_buf(attach->dmabuf, attach->dev); + + mutex_lock(&info->maps_lock); + if (!atomic_add_unless(&info->handle->pin, -1, 0)) { + mutex_unlock(&info->maps_lock); + WARN(1, "Unpinning handle that has yet to be pinned!\n"); + return; + } + + if (!(nvmap_dev->dynamic_dma_map_mask & info->handle->heap_type)) { + sg_dma_address(sgt->sgl) = 0; + } else if (info->handle->heap_type == NVMAP_HEAP_CARVEOUT_VPR && + access_vpr_phys(attach->dev)) { + sg_dma_address(sgt->sgl) = 0; + } else { + dma_unmap_sg_attrs(attach->dev, + sgt->sgl, sgt->nents, + dir, DMA_ATTR_SKIP_CPU_SYNC); + } + __nvmap_free_sg_table(NULL, info->handle, sgt); + +#ifdef NVMAP_CONFIG_DEBUG_MAPS + /* Remove the device name from the list of carveout accessing devices */ + heap_type = info->handle->heap_type; + device_name = (char *)dev_name(attach->dev); + if (device_name) + nvmap_remove_device_name(device_name, heap_type); +#endif /* NVMAP_CONFIG_DEBUG_MAPS */ + mutex_unlock(&info->maps_lock); +} + +static void nvmap_dmabuf_release(struct dma_buf *dmabuf) +{ + struct nvmap_handle_info *info = dmabuf->priv; + + trace_nvmap_dmabuf_release(info->handle->owner ? + info->handle->owner->name : "unknown", + info->handle, + dmabuf); + + mutex_lock(&info->handle->lock); + if (info->is_ro) { + BUG_ON(dmabuf != info->handle->dmabuf_ro); + info->handle->dmabuf_ro = NULL; + } else { + BUG_ON(dmabuf != info->handle->dmabuf); + info->handle->dmabuf = NULL; + } + mutex_unlock(&info->handle->lock); + + nvmap_handle_put(info->handle); + kfree(info); +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0) +static int __nvmap_dmabuf_end_cpu_access(struct dma_buf *dmabuf, + enum dma_data_direction dir) +{ + struct nvmap_handle_info *info = dmabuf->priv; + + trace_nvmap_dmabuf_end_cpu_access(dmabuf, 0, dmabuf->size); + return __nvmap_do_cache_maint(NULL, info->handle, + 0, dmabuf->size, + NVMAP_CACHE_OP_WB, false); +} + +static int __nvmap_dmabuf_begin_cpu_access(struct dma_buf *dmabuf, + enum dma_data_direction dir) +{ + struct nvmap_handle_info *info = dmabuf->priv; + + trace_nvmap_dmabuf_begin_cpu_access(dmabuf, 0, dmabuf->size); + return __nvmap_do_cache_maint(NULL, info->handle, 0, dmabuf->size, + NVMAP_CACHE_OP_WB_INV, false); +} +#define NVMAP_DMABUF_BEGIN_CPU_ACCESS __nvmap_dmabuf_begin_cpu_access +#define NVMAP_DMABUF_END_CPU_ACCESS __nvmap_dmabuf_end_cpu_access +#else +static int nvmap_dmabuf_begin_cpu_access(struct dma_buf *dmabuf, + size_t start, size_t len, + enum dma_data_direction dir) +{ + struct nvmap_handle_info *info = dmabuf->priv; + + trace_nvmap_dmabuf_begin_cpu_access(dmabuf, start, len); + return __nvmap_do_cache_maint(NULL, info->handle, start, start + len, + NVMAP_CACHE_OP_WB_INV, false); +} + +static void nvmap_dmabuf_end_cpu_access(struct dma_buf *dmabuf, + size_t start, size_t len, + enum dma_data_direction dir) +{ + struct nvmap_handle_info *info = dmabuf->priv; + + trace_nvmap_dmabuf_end_cpu_access(dmabuf, start, len); + __nvmap_do_cache_maint(NULL, info->handle, + start, start + len, + NVMAP_CACHE_OP_WB, false); +} +#define NVMAP_DMABUF_BEGIN_CPU_ACCESS nvmap_dmabuf_begin_cpu_access +#define NVMAP_DMABUF_END_CPU_ACCESS nvmap_dmabuf_end_cpu_access +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0) +static void *nvmap_dmabuf_kmap(struct dma_buf *dmabuf, unsigned long page_num) +{ + struct nvmap_handle_info *info = dmabuf->priv; + + trace_nvmap_dmabuf_kmap(dmabuf); + return __nvmap_kmap(info->handle, page_num); +} + +static void nvmap_dmabuf_kunmap(struct dma_buf *dmabuf, + unsigned long page_num, void *addr) +{ + struct nvmap_handle_info *info = dmabuf->priv; + + trace_nvmap_dmabuf_kunmap(dmabuf); + __nvmap_kunmap(info->handle, page_num, addr); +} + +static void *nvmap_dmabuf_kmap_atomic(struct dma_buf *dmabuf, + unsigned long page_num) +{ + WARN(1, "%s() can't be called from atomic\n", __func__); + return NULL; +} +#endif + +int __nvmap_map(struct nvmap_handle *h, struct vm_area_struct *vma) +{ + struct nvmap_vma_priv *priv; + + h = nvmap_handle_get(h); + if (!h) + return -EINVAL; + + if (!(h->heap_type & nvmap_dev->cpu_access_mask)) { + nvmap_handle_put(h); + return -EPERM; + } + + /* + * If the handle is RO and RW mapping is requested, then + * return error. + */ + if (h->from_va && h->is_ro && (vma->vm_flags & VM_WRITE)) { + nvmap_handle_put(h); + return -EPERM; + } + /* + * Don't allow mmap on VPR memory as it would be mapped + * as device memory. User space shouldn't be accessing + * device memory. + */ + if (h->heap_type == NVMAP_HEAP_CARVEOUT_VPR) { + nvmap_handle_put(h); + return -EPERM; + } + + priv = kzalloc(sizeof(*priv), GFP_KERNEL); + if (!priv) { + nvmap_handle_put(h); + return -ENOMEM; + } + priv->handle = h; + + vma->vm_flags |= VM_SHARED | VM_DONTEXPAND | + VM_DONTDUMP | VM_DONTCOPY | + (h->heap_pgalloc ? 0 : VM_PFNMAP); + vma->vm_ops = &nvmap_vma_ops; + BUG_ON(vma->vm_private_data != NULL); + vma->vm_private_data = priv; + vma->vm_page_prot = nvmap_pgprot(h, vma->vm_page_prot); + nvmap_vma_open(vma); + return 0; +} + +static int nvmap_dmabuf_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma) +{ + struct nvmap_handle_info *info = dmabuf->priv; + + trace_nvmap_dmabuf_mmap(dmabuf); + + return __nvmap_map(info->handle, vma); +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0) +static void *nvmap_dmabuf_vmap(struct dma_buf *dmabuf) +{ + struct nvmap_handle_info *info = dmabuf->priv; + + trace_nvmap_dmabuf_vmap(dmabuf); + + /* Don't allow vmap on RO buffers */ + if (info->is_ro) + return ERR_PTR(-EPERM); + + return __nvmap_mmap(info->handle); +} + +static void nvmap_dmabuf_vunmap(struct dma_buf *dmabuf, void *vaddr) +{ + struct nvmap_handle_info *info = dmabuf->priv; + + trace_nvmap_dmabuf_vunmap(dmabuf); + __nvmap_munmap(info->handle, vaddr); +} +#else +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 18, 0) +static int nvmap_dmabuf_vmap(struct dma_buf *dmabuf, struct iosys_map *map) +#else +static int nvmap_dmabuf_vmap(struct dma_buf *dmabuf, struct dma_buf_map *map) +#endif +{ + struct nvmap_handle_info *info = dmabuf->priv; + void *res; + int ret = 0; + + trace_nvmap_dmabuf_vmap(dmabuf); + + /* Don't allow vmap on RO buffers */ + if (info->is_ro) + return -EPERM; + + res = __nvmap_mmap(info->handle); + if (res != NULL) { + map->vaddr = res; + map->is_iomem = false; + } + else { + ret = -ENOMEM; + } + return ret; +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 18, 0) +static void nvmap_dmabuf_vunmap(struct dma_buf *dmabuf, struct iosys_map *map) +#else +static void nvmap_dmabuf_vunmap(struct dma_buf *dmabuf, struct dma_buf_map *map) +#endif +{ + struct nvmap_handle_info *info = dmabuf->priv; + + trace_nvmap_dmabuf_vunmap(dmabuf); + __nvmap_munmap(info->handle, info->handle->vaddr); +} +#endif + +static struct dma_buf_ops nvmap_dma_buf_ops = { + .attach = NVMAP_DMABUF_ATTACH, + .detach = nvmap_dmabuf_detach, + .map_dma_buf = nvmap_dmabuf_map_dma_buf, + .unmap_dma_buf = nvmap_dmabuf_unmap_dma_buf, + .release = nvmap_dmabuf_release, + .begin_cpu_access = NVMAP_DMABUF_BEGIN_CPU_ACCESS, + .end_cpu_access = NVMAP_DMABUF_END_CPU_ACCESS, +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0) + .kmap_atomic = nvmap_dmabuf_kmap_atomic, + .kmap = nvmap_dmabuf_kmap, + .kunmap = nvmap_dmabuf_kunmap, +#elif LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0) + .map_atomic = nvmap_dmabuf_kmap_atomic, + .map = nvmap_dmabuf_kmap, + .unmap = nvmap_dmabuf_kunmap, +#endif + .mmap = nvmap_dmabuf_mmap, + .vmap = nvmap_dmabuf_vmap, + .vunmap = nvmap_dmabuf_vunmap, +#if LINUX_VERSION_CODE > KERNEL_VERSION(5, 4, 0) + .cache_sgt_mapping = true, +#endif + +}; + +static char dmabuf_name[] = "nvmap_dmabuf"; + +bool dmabuf_is_nvmap(struct dma_buf *dmabuf) +{ + return dmabuf->exp_name == dmabuf_name; +} + +static struct dma_buf *__dma_buf_export(struct nvmap_handle_info *info, + size_t size, bool ro_buf) +{ + DEFINE_DMA_BUF_EXPORT_INFO(exp_info); + + exp_info.priv = info; + exp_info.ops = &nvmap_dma_buf_ops; + exp_info.size = size; + + if (ro_buf) { + exp_info.flags = O_RDONLY; + } else { + exp_info.flags = O_RDWR; + } + +#ifndef NVMAP_UPSTREAM_KERNEL + /* Disable defer unmap feature only for kstable */ + exp_info.exp_flags = DMABUF_CAN_DEFER_UNMAP | + DMABUF_SKIP_CACHE_SYNC; +#endif /* !NVMAP_UPSTREAM_KERNEL */ + exp_info.exp_name = dmabuf_name; + + return dma_buf_export(&exp_info); +} + +/* + * Make a dmabuf object for an nvmap handle. + */ +struct dma_buf *__nvmap_make_dmabuf(struct nvmap_client *client, + struct nvmap_handle *handle, bool ro_buf) +{ + int err; + struct dma_buf *dmabuf; + struct nvmap_handle_info *info; + + info = kzalloc(sizeof(*info), GFP_KERNEL); + if (!info) { + err = -ENOMEM; + goto err_nomem; + } + info->handle = handle; + info->is_ro = ro_buf; + INIT_LIST_HEAD(&info->maps); + mutex_init(&info->maps_lock); + + dmabuf = __dma_buf_export(info, handle->size, ro_buf); + if (IS_ERR(dmabuf)) { + err = PTR_ERR(dmabuf); + goto err_export; + } + nvmap_handle_get(handle); + + trace_nvmap_make_dmabuf(client->name, handle, dmabuf); + return dmabuf; + +err_export: + kfree(info); +err_nomem: + return ERR_PTR(err); +} + +int __nvmap_dmabuf_fd(struct nvmap_client *client, + struct dma_buf *dmabuf, int flags) +{ +#if !defined(NVMAP_CONFIG_HANDLE_AS_ID) && !defined(NVMAP_LOADABLE_MODULE) + int start_fd = NVMAP_CONFIG_FD_START; +#endif + int ret; + +#ifdef NVMAP_CONFIG_DEFER_FD_RECYCLE + if (client->next_fd < NVMAP_CONFIG_FD_START) + client->next_fd = NVMAP_CONFIG_FD_START; + start_fd = client->next_fd++; + if (client->next_fd >= NVMAP_CONFIG_DEFER_FD_RECYCLE_MAX_FD) + client->next_fd = NVMAP_CONFIG_FD_START; +#endif + if (!dmabuf || !dmabuf->file) + return -EINVAL; + /* Allocate fd from start_fd(>=1024) onwards to overcome + * __FD_SETSIZE limitation issue for select(), + * pselect() syscalls. + */ +#if defined(NVMAP_LOADABLE_MODULE) || defined(NVMAP_CONFIG_HANDLE_AS_ID) + ret = get_unused_fd_flags(flags); +#else + ret = __alloc_fd(current->files, start_fd, sysctl_nr_open, flags); +#endif + if (ret == -EMFILE) + pr_err_ratelimited("NvMap: FD limit is crossed for uid %d\n", + from_kuid(current_user_ns(), current_uid())); + return ret; +} + +static struct dma_buf *__nvmap_dmabuf_export(struct nvmap_client *client, + struct nvmap_handle *handle, bool is_ro) +{ + struct dma_buf *buf; + + handle = nvmap_handle_get(handle); + if (!handle) + return ERR_PTR(-EINVAL); + if (is_ro) + buf = handle->dmabuf_ro; + else + buf = handle->dmabuf; + + if (WARN(!buf, "Attempting to get a freed dma_buf!\n")) { + nvmap_handle_put(handle); + return NULL; + } + + get_dma_buf(buf); + + /* + * Don't want to take out refs on the handle here. + */ + nvmap_handle_put(handle); + + return buf; +} + +int nvmap_get_dmabuf_fd(struct nvmap_client *client, struct nvmap_handle *h, + bool is_ro) +{ + int fd; + struct dma_buf *dmabuf; + + dmabuf = __nvmap_dmabuf_export(client, h, is_ro); + if (IS_ERR(dmabuf)) + return PTR_ERR(dmabuf); + + fd = __nvmap_dmabuf_fd(client, dmabuf, O_CLOEXEC); + if (fd < 0) + dma_buf_put(dmabuf); + return fd; +} + +/* + * Returns the nvmap handle ID associated with the passed dma_buf's fd. This + * does not affect the ref count of the dma_buf. + * NOTE: Callers of this utility function must invoke nvmap_handle_put after + * using the returned nvmap_handle. Call to nvmap_handle_get is required in + * this utility function to avoid race conditions in code where nvmap_handle + * returned by this function is freed concurrently while the caller is still + * using it. + */ +struct nvmap_handle *nvmap_handle_get_from_dmabuf_fd( + struct nvmap_client *client, int fd) +{ + struct nvmap_handle *handle = ERR_PTR(-EINVAL); + struct dma_buf *dmabuf; + struct nvmap_handle_info *info; + + dmabuf = dma_buf_get(fd); + if (IS_ERR(dmabuf)) + return ERR_CAST(dmabuf); + if (dmabuf_is_nvmap(dmabuf)) { + info = dmabuf->priv; + handle = info->handle; + if (!nvmap_handle_get(handle)) + handle = ERR_PTR(-EINVAL); + } + dma_buf_put(dmabuf); + return handle; +} + +bool is_nvmap_dmabuf_fd_ro(int fd) +{ + struct dma_buf *dmabuf; + struct nvmap_handle_info *info = NULL; + + dmabuf = dma_buf_get(fd); + if (IS_ERR(dmabuf)) { + return false; + } + if (dmabuf_is_nvmap(dmabuf)) { + info = dmabuf->priv; + } + dma_buf_put(dmabuf); + + return (info != NULL) ? info->is_ro : false; +} + +/* + * Duplicates a generic dma_buf fd. nvmap dma_buf fd has to be duplicated + * using existing code paths to preserve memory accounting behavior, so this + * function returns -EINVAL on dma_buf fds created by nvmap. + */ +int nvmap_dmabuf_duplicate_gen_fd(struct nvmap_client *client, + struct dma_buf *dmabuf) +{ + int ret = 0; + + if (dmabuf_is_nvmap(dmabuf)) { + ret = -EINVAL; + goto error; + } + + ret = __nvmap_dmabuf_fd(client, dmabuf, O_CLOEXEC); + if (ret < 0) + goto error; + + return ret; + +error: + dma_buf_put(dmabuf); + return ret; +} diff --git a/drivers/video/tegra/nvmap/nvmap_fault.c b/drivers/video/tegra/nvmap/nvmap_fault.c new file mode 100644 index 00000000..e51a0b5b --- /dev/null +++ b/drivers/video/tegra/nvmap/nvmap_fault.c @@ -0,0 +1,261 @@ +/* + * drivers/video/tegra/nvmap/nvmap_fault.c + * + * Copyright (c) 2011-2022, NVIDIA CORPORATION. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + */ + +#define pr_fmt(fmt) "%s: " fmt, __func__ + +#include +#include + +#include "nvmap_priv.h" + +static void nvmap_vma_close(struct vm_area_struct *vma); + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0) +#define __atomic_add_unless atomic_fetch_add_unless +static vm_fault_t nvmap_vma_fault(struct vm_fault *vmf); +#elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0) +static int nvmap_vma_fault(struct vm_fault *vmf); +#else +static int nvmap_vma_fault(struct vm_area_struct *vma, struct vm_fault *vmf); +#endif + +struct vm_operations_struct nvmap_vma_ops = { + .open = nvmap_vma_open, + .close = nvmap_vma_close, + .fault = nvmap_vma_fault, +}; + +int is_nvmap_vma(struct vm_area_struct *vma) +{ + return vma->vm_ops == &nvmap_vma_ops; +} + +/* to ensure that the backing store for the VMA isn't freed while a fork'd + * reference still exists, nvmap_vma_open increments the reference count on + * the handle, and nvmap_vma_close decrements it. alternatively, we could + * disallow copying of the vma, or behave like pmem and zap the pages. FIXME. +*/ +void nvmap_vma_open(struct vm_area_struct *vma) +{ + struct nvmap_vma_priv *priv; + struct nvmap_handle *h; + struct nvmap_vma_list *vma_list, *tmp; + struct list_head *tmp_head = NULL; + pid_t current_pid = task_tgid_nr(current); + bool vma_pos_found = false; + size_t nr_page, i; + int vma_open_count; + + priv = vma->vm_private_data; + BUG_ON(!priv); + BUG_ON(!priv->handle); + + h = priv->handle; + nvmap_umaps_inc(h); + + mutex_lock(&h->lock); + vma_open_count = atomic_inc_return(&priv->count); + if (vma_open_count == 1 && h->heap_pgalloc) { + nr_page = h->size >> PAGE_SHIFT; + for (i = 0; i < nr_page; i++) { + struct page *page = nvmap_to_page(h->pgalloc.pages[i]); + /* This is necessry to avoid page being accounted + * under NR_FILE_MAPPED. This way NR_FILE_MAPPED would + * be fully accounted under NR_FILE_PAGES. This allows + * Android low mem killer detect low memory condition + * precisely. + * This has a side effect of inaccurate pss accounting + * for NvMap memory mapped into user space. Android + * procrank and NvMap Procrank both would have same + * issue. Subtracting NvMap_Procrank pss from + * procrank pss would give non-NvMap pss held by process + * and adding NvMap memory used by process represents + * entire memroy consumption by the process. + */ + atomic_inc(&page->_mapcount); + } + } + mutex_unlock(&h->lock); + + vma_list = kmalloc(sizeof(*vma_list), GFP_KERNEL); + if (vma_list) { + mutex_lock(&h->lock); + tmp_head = &h->vmas; + + /* insert vma into handle's vmas list in the increasing order of + * handle offsets + */ + list_for_each_entry(tmp, &h->vmas, list) { + /* if vma exists in list, just increment refcount */ + if (tmp->vma == vma) { + atomic_inc(&tmp->ref); + kfree(vma_list); + goto unlock; + } + + if (!vma_pos_found && (current_pid == tmp->pid)) { + if (vma->vm_pgoff < tmp->vma->vm_pgoff) { + tmp_head = &tmp->list; + vma_pos_found = true; + } else { + tmp_head = tmp->list.next; + } + } + } + + vma_list->vma = vma; + vma_list->pid = current_pid; + vma_list->save_vm_flags = vma->vm_flags; + atomic_set(&vma_list->ref, 1); + list_add_tail(&vma_list->list, tmp_head); +unlock: + mutex_unlock(&h->lock); + } else { + WARN(1, "vma not tracked"); + } +} + +static void nvmap_vma_close(struct vm_area_struct *vma) +{ + struct nvmap_vma_priv *priv = vma->vm_private_data; + struct nvmap_vma_list *vma_list; + struct nvmap_handle *h; + bool vma_found = false; + size_t nr_page, i; + + if (!priv) + return; + + BUG_ON(!priv->handle); + + h = priv->handle; + nr_page = h->size >> PAGE_SHIFT; + + mutex_lock(&h->lock); + list_for_each_entry(vma_list, &h->vmas, list) { + if (vma_list->vma != vma) + continue; + if (atomic_dec_return(&vma_list->ref) == 0) { + list_del(&vma_list->list); + kfree(vma_list); + } + vma_found = true; + break; + } + BUG_ON(!vma_found); + nvmap_umaps_dec(h); + + if (__atomic_add_unless(&priv->count, -1, 0) == 1) { + if (h->heap_pgalloc) { + for (i = 0; i < nr_page; i++) { + struct page *page; + page = nvmap_to_page(h->pgalloc.pages[i]); + atomic_dec(&page->_mapcount); + } + } + mutex_unlock(&h->lock); + if (priv->handle) + nvmap_handle_put(priv->handle); + vma->vm_private_data = NULL; + kfree(priv); + } else { + mutex_unlock(&h->lock); + } +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0) +static vm_fault_t nvmap_vma_fault(struct vm_fault *vmf) +#define vm_insert_pfn vmf_insert_pfn +#elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0) +static int nvmap_vma_fault(struct vm_fault *vmf) +#else +static int nvmap_vma_fault(struct vm_area_struct *vma, struct vm_fault *vmf) +#endif +{ + struct page *page; + struct nvmap_vma_priv *priv; + unsigned long offs; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0) + struct vm_area_struct *vma = vmf->vma; + unsigned long vmf_address = vmf->address; +#else + void __user *vmf_address = vmf->virtual_address; +#endif + + offs = (unsigned long)(vmf_address - vma->vm_start); + priv = vma->vm_private_data; + if (!priv || !priv->handle || !priv->handle->alloc) + return VM_FAULT_SIGBUS; + + offs += priv->offs; + /* if the VMA was split for some reason, vm_pgoff will be the VMA's + * offset from the original VMA */ + offs += (vma->vm_pgoff << PAGE_SHIFT); + + if (offs >= priv->handle->size) + return VM_FAULT_SIGBUS; + + if (!priv->handle->heap_pgalloc) { + unsigned long pfn; + BUG_ON(priv->handle->carveout->base & ~PAGE_MASK); + pfn = ((priv->handle->carveout->base + offs) >> PAGE_SHIFT); + if (!pfn_valid(pfn)) { + vm_insert_pfn(vma, + (unsigned long)vmf_address, pfn); + return VM_FAULT_NOPAGE; + } + /* CMA memory would get here */ + page = pfn_to_page(pfn); + } else { + void *kaddr; + + offs >>= PAGE_SHIFT; + if (atomic_read(&priv->handle->pgalloc.reserved)) + return VM_FAULT_SIGBUS; + page = nvmap_to_page(priv->handle->pgalloc.pages[offs]); + + if (PageAnon(page) && (vma->vm_flags & VM_SHARED)) + return VM_FAULT_SIGSEGV; + + if (!nvmap_handle_track_dirty(priv->handle)) + goto finish; + + mutex_lock(&priv->handle->lock); + if (nvmap_page_dirty(priv->handle->pgalloc.pages[offs])) { + mutex_unlock(&priv->handle->lock); + goto finish; + } + + /* inner cache maint */ + kaddr = kmap(page); + BUG_ON(!kaddr); + inner_cache_maint(NVMAP_CACHE_OP_WB_INV, kaddr, PAGE_SIZE); + kunmap(page); + + if (priv->handle->flags & NVMAP_HANDLE_INNER_CACHEABLE) + goto make_dirty; + +make_dirty: + nvmap_page_mkdirty(&priv->handle->pgalloc.pages[offs]); + atomic_inc(&priv->handle->pgalloc.ndirty); + mutex_unlock(&priv->handle->lock); + } + +finish: + if (page) + get_page(page); + vmf->page = page; + return (page) ? 0 : VM_FAULT_SIGBUS; +} diff --git a/drivers/video/tegra/nvmap/nvmap_handle.c b/drivers/video/tegra/nvmap/nvmap_handle.c new file mode 100644 index 00000000..d729c467 --- /dev/null +++ b/drivers/video/tegra/nvmap/nvmap_handle.c @@ -0,0 +1,511 @@ +/* + * drivers/video/tegra/nvmap/nvmap_handle.c + * + * Handle allocation and freeing routines for nvmap + * + * Copyright (c) 2009-2022, NVIDIA CORPORATION. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + */ + +#define pr_fmt(fmt) "%s: " fmt, __func__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if KERNEL_VERSION(4, 15, 0) > LINUX_VERSION_CODE +#include +#else +#include +#endif + +#include + +#include + +#include "nvmap_priv.h" +#include "nvmap_ioctl.h" + +/* + * Verifies that the passed ID is a valid handle ID. Then the passed client's + * reference to the handle is returned. + * + * Note: to call this function make sure you own the client ref lock. + */ +struct nvmap_handle_ref *__nvmap_validate_locked(struct nvmap_client *c, + struct nvmap_handle *h, + bool is_ro) +{ + struct rb_node *n = c->handle_refs.rb_node; + + while (n) { + struct nvmap_handle_ref *ref; + ref = rb_entry(n, struct nvmap_handle_ref, node); + if ((ref->handle == h) && (ref->is_ro == is_ro)) + return ref; + else if ((uintptr_t)h > (uintptr_t)ref->handle) + n = n->rb_right; + else + n = n->rb_left; + } + + return NULL; +} +/* adds a newly-created handle to the device master tree */ +void nvmap_handle_add(struct nvmap_device *dev, struct nvmap_handle *h) +{ + struct rb_node **p; + struct rb_node *parent = NULL; + + spin_lock(&dev->handle_lock); + p = &dev->handles.rb_node; + while (*p) { + struct nvmap_handle *b; + + parent = *p; + b = rb_entry(parent, struct nvmap_handle, node); + if (h > b) + p = &parent->rb_right; + else + p = &parent->rb_left; + } + rb_link_node(&h->node, parent, p); + rb_insert_color(&h->node, &dev->handles); + nvmap_lru_add(h); + spin_unlock(&dev->handle_lock); +} + +/* remove a handle from the device's tree of all handles; called + * when freeing handles. */ +int nvmap_handle_remove(struct nvmap_device *dev, struct nvmap_handle *h) +{ + spin_lock(&dev->handle_lock); + + /* re-test inside the spinlock if the handle really has no clients; + * only remove the handle if it is unreferenced */ + if (atomic_add_return(0, &h->ref) > 0) { + spin_unlock(&dev->handle_lock); + return -EBUSY; + } + smp_rmb(); + BUG_ON(atomic_read(&h->ref) < 0); + BUG_ON(atomic_read(&h->pin) != 0); + + nvmap_lru_del(h); + rb_erase(&h->node, &dev->handles); + + spin_unlock(&dev->handle_lock); + return 0; +} + +/* Validates that a handle is in the device master tree and that the + * client has permission to access it. */ +struct nvmap_handle *nvmap_validate_get(struct nvmap_handle *id) +{ + struct nvmap_handle *h = NULL; + struct rb_node *n; + + spin_lock(&nvmap_dev->handle_lock); + + n = nvmap_dev->handles.rb_node; + + while (n) { + h = rb_entry(n, struct nvmap_handle, node); + if (h == id) { + h = nvmap_handle_get(h); + spin_unlock(&nvmap_dev->handle_lock); + return h; + } + if (id > h) + n = n->rb_right; + else + n = n->rb_left; + } + spin_unlock(&nvmap_dev->handle_lock); + return NULL; +} + +static void add_handle_ref(struct nvmap_client *client, + struct nvmap_handle_ref *ref) +{ + struct rb_node **p, *parent = NULL; + + nvmap_ref_lock(client); + p = &client->handle_refs.rb_node; + while (*p) { + struct nvmap_handle_ref *node; + parent = *p; + node = rb_entry(parent, struct nvmap_handle_ref, node); + if (ref->handle > node->handle) + p = &parent->rb_right; + else + p = &parent->rb_left; + } + rb_link_node(&ref->node, parent, p); + rb_insert_color(&ref->node, &client->handle_refs); + client->handle_count++; + if (client->handle_count > nvmap_max_handle_count) + nvmap_max_handle_count = client->handle_count; + atomic_inc(&ref->handle->share_count); + nvmap_ref_unlock(client); +} + +struct nvmap_handle_ref *nvmap_create_handle_from_va(struct nvmap_client *client, + ulong vaddr, size_t size, + u32 flags) +{ + struct vm_area_struct *vma; + struct nvmap_handle_ref *ref; + vm_flags_t vm_flags; + struct mm_struct *mm = current->mm; + + /* don't allow non-page aligned addresses. */ + if (vaddr & ~PAGE_MASK) + return ERR_PTR(-EINVAL); + + nvmap_acquire_mmap_read_lock(mm); + vma = find_vma(mm, vaddr); + if (unlikely(!vma)) { + nvmap_release_mmap_read_lock(mm); + return ERR_PTR(-EINVAL); + } + + if (!size) + size = vma->vm_end - vaddr; + + /* Don't allow exuberantly large sizes. */ + if (!is_nvmap_memory_available(size, NVMAP_HEAP_IOVMM)) { + pr_debug("Cannot allocate %zu bytes.\n", size); + nvmap_release_mmap_read_lock(mm); + return ERR_PTR(-ENOMEM); + } + + vm_flags = vma->vm_flags; + nvmap_release_mmap_read_lock(mm); + + /* + * If buffer is malloc/mprotect as RO but alloc flag is not passed + * as RO, don't create handle. + */ + if (!(vm_flags & VM_WRITE) && !(flags & NVMAP_HANDLE_RO)) + return ERR_PTR(-EINVAL); + + ref = nvmap_create_handle(client, size, flags & NVMAP_HANDLE_RO); + if (!IS_ERR(ref)) + ref->handle->orig_size = size; + + return ref; +} + +struct nvmap_handle_ref *nvmap_create_handle(struct nvmap_client *client, + size_t size, bool ro_buf) +{ + void *err = ERR_PTR(-ENOMEM); + struct nvmap_handle *h; + struct nvmap_handle_ref *ref = NULL; + struct dma_buf *dmabuf; + + if (!client) + return ERR_PTR(-EINVAL); + + if (!size) + return ERR_PTR(-EINVAL); + + h = kzalloc(sizeof(*h), GFP_KERNEL); + if (!h) + return ERR_PTR(-ENOMEM); + + ref = kzalloc(sizeof(*ref), GFP_KERNEL); + if (!ref) + goto ref_alloc_fail; + + atomic_set(&h->ref, 1); + atomic_set(&h->pin, 0); + h->owner = client; + BUG_ON(!h->owner); + h->orig_size = size; + h->size = PAGE_ALIGN(size); + h->flags = NVMAP_HANDLE_WRITE_COMBINE; + h->peer = NVMAP_IVM_INVALID_PEER; + mutex_init(&h->lock); + INIT_LIST_HEAD(&h->vmas); + INIT_LIST_HEAD(&h->lru); + INIT_LIST_HEAD(&h->dmabuf_priv); + + /* + * This takes out 1 ref on the dambuf. This corresponds to the + * handle_ref that gets automatically made by nvmap_create_handle(). + */ + dmabuf = __nvmap_make_dmabuf(client, h, ro_buf); + if (IS_ERR(dmabuf)) { + err = dmabuf; + goto make_dmabuf_fail; + } + if (!ro_buf) + h->dmabuf = dmabuf; + else + h->dmabuf_ro = dmabuf; + + nvmap_handle_add(nvmap_dev, h); + + /* + * Major assumption here: the dma_buf object that the handle contains + * is created with a ref count of 1. + */ + atomic_set(&ref->dupes, 1); + ref->handle = h; + add_handle_ref(client, ref); + if (ro_buf) + ref->is_ro = true; + else + ref->is_ro = false; + trace_nvmap_create_handle(client, client->name, h, size, ref); + return ref; + +make_dmabuf_fail: + kfree(ref); +ref_alloc_fail: + kfree(h); + return err; +} + +struct nvmap_handle_ref *nvmap_try_duplicate_by_ivmid( + struct nvmap_client *client, u64 ivm_id, + struct nvmap_heap_block **block) +{ + struct nvmap_handle *h = NULL; + struct nvmap_handle_ref *ref = NULL; + struct rb_node *n; + + spin_lock(&nvmap_dev->handle_lock); + + n = nvmap_dev->handles.rb_node; + for (n = rb_first(&nvmap_dev->handles); n; n = rb_next(n)) { + h = rb_entry(n, struct nvmap_handle, node); + if (h->ivm_id == ivm_id) { + BUG_ON(!virt_addr_valid(h)); + /* get handle's ref only if non-zero */ + if (atomic_inc_not_zero(&h->ref) == 0) { + *block = h->carveout; + /* strip handle's block and fail duplication */ + h->carveout = NULL; + break; + } + spin_unlock(&nvmap_dev->handle_lock); + goto found; + } + } + + spin_unlock(&nvmap_dev->handle_lock); + /* handle is either freed or being freed, don't duplicate it */ + goto finish; + + /* + * From this point, handle and its buffer are valid and won't be + * freed as a reference is taken on it. The dmabuf can still be + * freed anytime till reference is taken on it below. + */ +found: + mutex_lock(&h->lock); + /* + * Save this block. If dmabuf's reference is not held in time, + * this can be reused to avoid the delay to free the buffer + * in this old handle and allocate it for a new handle from + * the ivm allocation ioctl. + */ + *block = h->carveout; + if (!h->dmabuf) + goto fail; + BUG_ON(!h->dmabuf->file); + /* This is same as get_dma_buf() if file->f_count was non-zero */ + if (atomic_long_inc_not_zero(&h->dmabuf->file->f_count) == 0) + goto fail; + mutex_unlock(&h->lock); + + /* h->dmabuf can't be NULL anymore. Duplicate the handle. */ + ref = nvmap_duplicate_handle(client, h, true, false); + /* put the extra ref taken using get_dma_buf. */ + dma_buf_put(h->dmabuf); +finish: + return ref; +fail: + /* free handle but not its buffer */ + h->carveout = NULL; + mutex_unlock(&h->lock); + nvmap_handle_put(h); + return NULL; +} + +struct nvmap_handle_ref *nvmap_duplicate_handle(struct nvmap_client *client, + struct nvmap_handle *h, bool skip_val, + bool is_ro) +{ + struct nvmap_handle_ref *ref = NULL; + + BUG_ON(!client); + + if (!skip_val) + /* on success, the reference count for the handle should be + * incremented, so the success paths will not call + * nvmap_handle_put */ + h = nvmap_validate_get(h); + + if (!h) { + pr_debug("%s duplicate handle failed\n", + current->group_leader->comm); + return ERR_PTR(-EPERM); + } + + if (!h->alloc) { + pr_err("%s duplicating unallocated handle\n", + current->group_leader->comm); + nvmap_handle_put(h); + return ERR_PTR(-EINVAL); + } + + nvmap_ref_lock(client); + ref = __nvmap_validate_locked(client, h, is_ro); + + if (ref) { + atomic_inc(&ref->dupes); + nvmap_ref_unlock(client); + goto out; + } + + nvmap_ref_unlock(client); + + ref = kzalloc(sizeof(*ref), GFP_KERNEL); + if (!ref) { + nvmap_handle_put(h); + return ERR_PTR(-ENOMEM); + } + + atomic_set(&ref->dupes, 1); + ref->handle = h; + add_handle_ref(client, ref); + + if (is_ro) { + ref->is_ro = true; + get_dma_buf(h->dmabuf_ro); + } else { + ref->is_ro = false; + get_dma_buf(h->dmabuf); + } + +out: + NVMAP_TAG_TRACE(trace_nvmap_duplicate_handle, + NVMAP_TP_ARGS_CHR(client, h, ref)); + return ref; +} + +struct nvmap_handle_ref *nvmap_create_handle_from_id( + struct nvmap_client *client, u32 id) +{ + struct nvmap_handle *handle; + struct nvmap_handle_ref *ref; + + if (WARN_ON(!client)) + return ERR_PTR(-EINVAL); + + if (is_nvmap_id_ro(client, id)) + return nvmap_dup_handle_ro(client, id); + + handle = nvmap_handle_get_from_id(client, id); + if (IS_ERR_OR_NULL(handle)) { + /* fd might be dmabuf fd received from parent process. + * Its entry is not made in id_array. + */ + handle = nvmap_handle_get_from_dmabuf_fd(client, id); + if (IS_ERR(handle)) + return ERR_CAST(handle); + } + + ref = nvmap_duplicate_handle(client, handle, false, false); + nvmap_handle_put(handle); + return ref; +} + +struct nvmap_handle_ref *nvmap_create_handle_from_fd( + struct nvmap_client *client, int fd) +{ + struct nvmap_handle *handle; + struct nvmap_handle_ref *ref; + bool is_ro; + + if (WARN_ON(!client)) + return ERR_PTR(-EINVAL); + + handle = nvmap_handle_get_from_dmabuf_fd(client, fd); + if (IS_ERR(handle)) + return ERR_CAST(handle); + + is_ro = is_nvmap_dmabuf_fd_ro(fd); + if (is_ro) + ref = nvmap_duplicate_handle(client, handle, false, true); + else + ref = nvmap_duplicate_handle(client, handle, false, false); + + nvmap_handle_put(handle); + return ref; +} + +struct nvmap_handle_ref *nvmap_dup_handle_ro(struct nvmap_client *client, + int id) +{ + struct nvmap_handle *h; + struct nvmap_handle_ref *ref = NULL; + bool dmabuf_created = false; + + if (!client) + return ERR_PTR(-EINVAL); + + h = nvmap_handle_get_from_id(client, id); + if (IS_ERR_OR_NULL(h)) { + /* fd might be dmabuf fd received from parent process. + * Its entry is not made in id_array. + */ + h = nvmap_handle_get_from_dmabuf_fd(client, id); + if (IS_ERR(h)) + return ERR_CAST(h); + } + + if (h->dmabuf_ro == NULL) { + h->dmabuf_ro = __nvmap_make_dmabuf(client, h, true); + if (IS_ERR(h->dmabuf_ro)) { + nvmap_handle_put(h); + return ERR_CAST(h->dmabuf_ro); + } + dmabuf_created = true; + } + + ref = nvmap_duplicate_handle(client, h, false, true); + if (!ref) { + nvmap_handle_put(h); + return ref; + } + /* + * When new dmabuf created (only RO dmabuf is getting created in this function) + * it's counter is incremented one extra time in nvmap_duplicate_handle. Hence + * decrement it by one. + */ + if (dmabuf_created) + dma_buf_put(h->dmabuf_ro); + + nvmap_handle_put(h); + + return ref; +} diff --git a/drivers/video/tegra/nvmap/nvmap_heap.c b/drivers/video/tegra/nvmap/nvmap_heap.c new file mode 100644 index 00000000..66f7b511 --- /dev/null +++ b/drivers/video/tegra/nvmap/nvmap_heap.c @@ -0,0 +1,607 @@ +/* + * drivers/video/tegra/nvmap/nvmap_heap.c + * + * GPU heap allocator. + * + * Copyright (c) 2011-2022, NVIDIA Corporation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + */ + +#define pr_fmt(fmt) "%s: " fmt, __func__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0) +#include +#endif + +#include +#include +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0) +#include +#else +#include +#endif + +#include "nvmap_priv.h" +#include "nvmap_heap.h" + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0) +#include "include/linux/nvmap_exports.h" +#endif + +/* + * "carveouts" are platform-defined regions of physically contiguous memory + * which are not managed by the OS. A platform may specify multiple carveouts, + * for either small special-purpose memory regions or reserved regions of main + * system memory. + * + * The carveout allocator returns allocations which are physically contiguous. + */ + +static struct kmem_cache *heap_block_cache; + +struct list_block { + struct nvmap_heap_block block; + struct list_head all_list; + unsigned int mem_prot; + phys_addr_t orig_addr; + size_t size; + size_t align; + struct nvmap_heap *heap; + struct list_head free_list; +}; + +struct device *dma_dev_from_handle(unsigned long type) +{ + int i; + struct nvmap_carveout_node *co_heap; + + for (i = 0; i < nvmap_dev->nr_carveouts; i++) { + co_heap = &nvmap_dev->heaps[i]; + + if (!(co_heap->heap_bit & type)) + continue; + + return co_heap->carveout->dma_dev; + } + return ERR_PTR(-ENODEV); +} + +int nvmap_query_heap_peer(struct nvmap_heap *heap, unsigned int *peer) +{ + if (!heap || !heap->is_ivm) + return -EINVAL; + *peer = heap->peer; + return 0; +} + +size_t nvmap_query_heap_size(struct nvmap_heap *heap) +{ + if (!heap) + return 0; + + return heap->len; +} + +void nvmap_heap_debugfs_init(struct dentry *heap_root, struct nvmap_heap *heap) +{ + if (sizeof(heap->base) == sizeof(u64)) + debugfs_create_x64("base", S_IRUGO, + heap_root, (u64 *)&heap->base); + else + debugfs_create_x32("base", S_IRUGO, + heap_root, (u32 *)&heap->base); + if (sizeof(heap->len) == sizeof(u64)) + debugfs_create_x64("size", S_IRUGO, + heap_root, (u64 *)&heap->len); + else + debugfs_create_x32("size", S_IRUGO, + heap_root, (u32 *)&heap->len); + if (sizeof(heap->free_size) == sizeof(u64)) + debugfs_create_x64("free_size", S_IRUGO, + heap_root, (u64 *)&heap->free_size); + else + debugfs_create_x32("free_size", S_IRUGO, + heap_root, (u32 *)&heap->free_size); +} + +static phys_addr_t nvmap_alloc_mem(struct nvmap_heap *h, size_t len, + phys_addr_t *start) +{ +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0) + phys_addr_t pa = DMA_ERROR_CODE; +#else + phys_addr_t pa = DMA_MAPPING_ERROR; +#endif + struct device *dev = h->dma_dev; + + if (len > UINT_MAX) { + dev_err(dev, "%s: %d alloc size is out of range\n", + __func__, __LINE__); + return DMA_ERROR_CODE; + } + +#ifdef CONFIG_TEGRA_VIRTUALIZATION + if (start && h->is_ivm) { + void *ret; + pa = h->base + (*start); +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0) + ret = dma_mark_declared_memory_occupied(dev, pa, len, + DMA_ATTR_ALLOC_EXACT_SIZE); +#else + ret = nvmap_dma_mark_declared_memory_occupied(dev, pa, len); +#endif + if (IS_ERR(ret)) { + dev_err(dev, "Failed to reserve (%pa) len(%zu)\n", + &pa, len); + return DMA_ERROR_CODE; + } + } else +#endif + { +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0) + (void)dma_alloc_attrs(dev, len, &pa, + GFP_KERNEL, DMA_ATTR_ALLOC_EXACT_SIZE); +#else + (void)nvmap_dma_alloc_attrs(dev, len, &pa, + GFP_KERNEL, DMA_ATTR_ALLOC_EXACT_SIZE); +#endif + if (!dma_mapping_error(dev, pa)) { +#ifdef NVMAP_CONFIG_VPR_RESIZE + int ret; + + dev_dbg(dev, "Allocated addr (%pa) len(%zu)\n", + &pa, len); + if (!dma_is_coherent_dev(dev) && h->cma_dev) { + ret = nvmap_cache_maint_phys_range( + NVMAP_CACHE_OP_WB, pa, pa + len, + true, true); + if (!ret) + return pa; + + dev_err(dev, "cache WB on (%pa, %zu) failed\n", + &pa, len); + } +#endif + dev_dbg(dev, "Allocated addr (%pa) len(%zu)\n", + &pa, len); + } + } + + return pa; +} + +static void nvmap_free_mem(struct nvmap_heap *h, phys_addr_t base, + size_t len) +{ + struct device *dev = h->dma_dev; + + dev_dbg(dev, "Free base (%pa) size (%zu)\n", &base, len); + + if (len > UINT_MAX) { + dev_err(dev, "%s: %d freeing length out of range\n", + __func__, __LINE__); + return; + } + +#ifdef CONFIG_TEGRA_VIRTUALIZATION + if (h->is_ivm && !h->can_alloc) { +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0) + dma_mark_declared_memory_unoccupied(dev, base, len, + DMA_ATTR_ALLOC_EXACT_SIZE); +#else + nvmap_dma_mark_declared_memory_unoccupied(dev, base, len); +#endif + } else +#endif + { +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0) + dma_free_attrs(dev, len, + (void *)(uintptr_t)base, + (dma_addr_t)base, DMA_ATTR_ALLOC_EXACT_SIZE); +#else + nvmap_dma_free_attrs(dev, len, + (void *)(uintptr_t)base, + (dma_addr_t)base, + DMA_ATTR_ALLOC_EXACT_SIZE); +#endif + } +} + +/* + * base_max limits position of allocated chunk in memory. + * if base_max is 0 then there is no such limitation. + */ +static struct nvmap_heap_block *do_heap_alloc(struct nvmap_heap *heap, + size_t len, size_t align, + unsigned int mem_prot, + phys_addr_t base_max, + phys_addr_t *start) +{ + struct list_block *heap_block = NULL; + dma_addr_t dev_base; + struct device *dev = heap->dma_dev; + + /* since pages are only mappable with one cache attribute, + * and most allocations from carveout heaps are DMA coherent + * (i.e., non-cacheable), round cacheable allocations up to + * a page boundary to ensure that the physical pages will + * only be mapped one way. */ + if (mem_prot == NVMAP_HANDLE_CACHEABLE || + mem_prot == NVMAP_HANDLE_INNER_CACHEABLE) { + align = max_t(size_t, align, PAGE_SIZE); + len = PAGE_ALIGN(len); + } + + if (heap->is_ivm) + align = max_t(size_t, align, NVMAP_IVM_ALIGNMENT); + + heap_block = kmem_cache_zalloc(heap_block_cache, GFP_KERNEL); + if (!heap_block) { + dev_err(dev, "%s: failed to alloc heap block %s\n", + __func__, dev_name(dev)); + goto fail_heap_block_alloc; + } + + dev_base = nvmap_alloc_mem(heap, len, start); + if (dma_mapping_error(dev, dev_base)) { + dev_err(dev, "failed to alloc mem of size (%zu)\n", + len); +#ifdef NVMAP_CONFIG_VPR_RESIZE + if (dma_is_coherent_dev(dev)) { + struct dma_coherent_stats stats; + + dma_get_coherent_stats(dev, &stats); + dev_err(dev, "used:%zu,curr_size:%zu max:%zu\n", + stats.used, stats.size, stats.max); + } +#endif + goto fail_dma_alloc; + } + + heap_block->block.base = dev_base; + heap_block->orig_addr = dev_base; + heap_block->size = len; + + list_add_tail(&heap_block->all_list, &heap->all_list); + heap_block->heap = heap; + heap->free_size -= len; + heap_block->mem_prot = mem_prot; + heap_block->align = align; + return &heap_block->block; + +fail_dma_alloc: + kmem_cache_free(heap_block_cache, heap_block); +fail_heap_block_alloc: + return NULL; +} + +static void do_heap_free(struct nvmap_heap_block *block) +{ + struct list_block *b = container_of(block, struct list_block, block); + struct nvmap_heap *heap = b->heap; + + list_del(&b->all_list); + + heap->free_size += b->size; + nvmap_free_mem(heap, block->base, b->size); + kmem_cache_free(heap_block_cache, b); +} + +/* nvmap_heap_alloc: allocates a block of memory of len bytes, aligned to + * align bytes. */ +struct nvmap_heap_block *nvmap_heap_alloc(struct nvmap_heap *h, + struct nvmap_handle *handle, + phys_addr_t *start) +{ + struct nvmap_heap_block *b; + size_t len = handle->size; + size_t align = handle->align; + unsigned int prot = handle->flags; + + mutex_lock(&h->lock); + + if (h->is_ivm) { /* Is IVM carveout? */ + /* Check if this correct IVM heap */ + if (handle->peer != h->peer) { + mutex_unlock(&h->lock); + return NULL; + } else { + if (h->can_alloc && start) { + /* If this partition does actual allocation, it + * should not specify start_offset. + */ + mutex_unlock(&h->lock); + return NULL; + } else if (!h->can_alloc && !start) { + /* If this partition does not do actual + * allocation, it should specify start_offset. + */ + mutex_unlock(&h->lock); + return NULL; + } + } + } + + /* + * If this HEAP has pm_ops defined and powering on the + * RAM attached with the HEAP returns error, don't + * allocate from the heap and return NULL. + */ + if (h->pm_ops.busy) { + if (h->pm_ops.busy() < 0) { + pr_err("Unable to power on the heap device\n"); + mutex_unlock(&h->lock); + return NULL; + } + } + + align = max_t(size_t, align, L1_CACHE_BYTES); + b = do_heap_alloc(h, len, align, prot, 0, start); + if (b) { + b->handle = handle; + handle->carveout = b; + /* Generate IVM for partition that can alloc */ + if (h->is_ivm && h->can_alloc) { + unsigned int offs = (b->base - h->base); + + BUG_ON(offs & (NVMAP_IVM_ALIGNMENT - 1)); + BUG_ON((offs >> ffs(NVMAP_IVM_ALIGNMENT)) & + ~((1 << NVMAP_IVM_OFFSET_WIDTH) - 1)); + BUG_ON(h->vm_id & ~(NVMAP_IVM_IVMID_MASK)); + /* So, page alignment is sufficient check. + */ + BUG_ON(len & ~(PAGE_MASK)); + + /* Copy offset from IVM mem pool in nvmap handle. + * The offset will be exported via ioctl. + */ + handle->offs = offs; + + handle->ivm_id = ((u64)h->vm_id << NVMAP_IVM_IVMID_SHIFT); + handle->ivm_id |= (((offs >> (ffs(NVMAP_IVM_ALIGNMENT) - 1)) & + ((1ULL << NVMAP_IVM_OFFSET_WIDTH) - 1)) << + NVMAP_IVM_OFFSET_SHIFT); + handle->ivm_id |= (len >> PAGE_SHIFT); + } + } + mutex_unlock(&h->lock); + return b; +} + +struct nvmap_heap *nvmap_block_to_heap(struct nvmap_heap_block *b) +{ + struct list_block *lb; + lb = container_of(b, struct list_block, block); + return lb->heap; +} + +/* nvmap_heap_free: frees block b*/ +void nvmap_heap_free(struct nvmap_heap_block *b) +{ + struct nvmap_heap *h; + struct list_block *lb; + + if (!b) + return; + + h = nvmap_block_to_heap(b); + mutex_lock(&h->lock); + + lb = container_of(b, struct list_block, block); + nvmap_flush_heap_block(NULL, b, lb->size, lb->mem_prot); + do_heap_free(b); + /* + * If this HEAP has pm_ops defined and powering off the + * RAM attached with the HEAP returns error, raise warning. + */ + if (h->pm_ops.idle) { + if (h->pm_ops.idle() < 0) + WARN_ON(1); + } + + mutex_unlock(&h->lock); +} + +/* nvmap_heap_create: create a heap object of len bytes, starting from + * address base. + */ +struct nvmap_heap *nvmap_heap_create(struct device *parent, + const struct nvmap_platform_carveout *co, + phys_addr_t base, size_t len, void *arg) +{ + struct nvmap_heap *h; + + h = kzalloc(sizeof(*h), GFP_KERNEL); + if (!h) { + dev_err(parent, "%s: out of memory\n", __func__); + return NULL; + } + + h->dma_dev = co->dma_dev; + if (co->cma_dev) { +#ifdef CONFIG_DMA_CMA +#ifdef NVMAP_CONFIG_VPR_RESIZE + struct dma_contiguous_stats stats; + + if (dma_get_contiguous_stats(co->cma_dev, &stats)) + goto fail; + + base = stats.base; + len = stats.size; + h->cma_dev = co->cma_dev; +#endif +#else + dev_err(parent, "invalid resize config for carveout %s\n", + co->name); + goto fail; +#endif + } else if (!co->init_done) { + int err; + + /* declare Non-CMA heap */ +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0) + err = dma_declare_coherent_memory(h->dma_dev, 0, base, len, + DMA_MEMORY_NOMAP); +#else + err = nvmap_dma_declare_coherent_memory(h->dma_dev, 0, base, len, + DMA_MEMORY_NOMAP); +#endif +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0) + if (!err) { +#else + if (err & DMA_MEMORY_NOMAP) { +#endif + dev_info(parent, + "%s :dma coherent mem declare %pa,%zu\n", + co->name, &base, len); + } else { + dev_err(parent, + "%s: dma coherent declare fail %pa,%zu\n", + co->name, &base, len); + goto fail; + } + } + + dev_set_name(h->dma_dev, "%s", co->name); + dma_set_coherent_mask(h->dma_dev, DMA_BIT_MASK(64)); + h->name = co->name; + h->arg = arg; + h->base = base; + h->can_alloc = !!co->can_alloc; + h->is_ivm = co->is_ivm; + h->len = len; + h->free_size = len; + h->peer = co->peer; + h->vm_id = co->vmid; + if (co->pm_ops.busy) + h->pm_ops.busy = co->pm_ops.busy; + + if (co->pm_ops.idle) + h->pm_ops.idle = co->pm_ops.idle; + + INIT_LIST_HEAD(&h->all_list); + mutex_init(&h->lock); +#ifdef NVMAP_CONFIG_DEBUG_MAPS + h->device_names = RB_ROOT; +#endif /* NVMAP_CONFIG_DEBUG_MAPS */ + if (!co->no_cpu_access && co->usage_mask != NVMAP_HEAP_CARVEOUT_VPR + && nvmap_cache_maint_phys_range(NVMAP_CACHE_OP_WB_INV, + base, base + len, true, true)) { + dev_err(parent, "cache flush failed\n"); + goto fail; + } + wmb(); + + if (co->disable_dynamic_dma_map) + nvmap_dev->dynamic_dma_map_mask &= ~co->usage_mask; + + if (co->no_cpu_access) + nvmap_dev->cpu_access_mask &= ~co->usage_mask; + + dev_info(parent, "created heap %s base 0x%px size (%zuKiB)\n", + co->name, (void *)(uintptr_t)base, len/1024); + return h; +fail: + kfree(h); + return NULL; +} + +/* nvmap_heap_destroy: frees all resources in heap */ +void nvmap_heap_destroy(struct nvmap_heap *heap) +{ + WARN_ON(!list_empty(&heap->all_list)); + while (!list_empty(&heap->all_list)) { + struct list_block *l; + l = list_first_entry(&heap->all_list, struct list_block, + all_list); + list_del(&l->all_list); + kmem_cache_free(heap_block_cache, l); + } + kfree(heap); +} + +int nvmap_heap_init(void) +{ + ulong start_time = sched_clock(); + + heap_block_cache = KMEM_CACHE(list_block, 0); + if (!heap_block_cache) { + pr_err("%s: unable to create heap block cache\n", __func__); + return -ENOMEM; + } + pr_info("%s: created heap block cache\n", __func__); + nvmap_init_time += sched_clock() - start_time; + return 0; +} + +void nvmap_heap_deinit(void) +{ + if (heap_block_cache) + kmem_cache_destroy(heap_block_cache); + + heap_block_cache = NULL; +} + +/* + * This routine is used to flush the carveout memory from cache. + * Why cache flush is needed for carveout? Consider the case, where a piece of + * carveout is allocated as cached and released. After this, if the same memory is + * allocated for uncached request and the memory is not flushed out from cache. + * In this case, the client might pass this to H/W engine and it could start modify + * the memory. As this was cached earlier, it might have some portion of it in cache. + * During cpu request to read/write other memory, the cached portion of this memory + * might get flushed back to main memory and would cause corruptions, if it happens + * after H/W writes data to memory. + * + * But flushing out the memory blindly on each carveout allocation is redundant. + * + * In order to optimize the carveout buffer cache flushes, the following + * strategy is used. + * + * The whole Carveout is flushed out from cache during its initialization. + * During allocation, carveout buffers are not flused from cache. + * During deallocation, carveout buffers are flushed, if they were allocated as cached. + * if they were allocated as uncached/writecombined, no cache flush is needed. + * Just draining store buffers is enough. + */ +int nvmap_flush_heap_block(struct nvmap_client *client, + struct nvmap_heap_block *block, size_t len, unsigned int prot) +{ + phys_addr_t phys = block->base; + phys_addr_t end = block->base + len; + int ret = 0; + + if (prot == NVMAP_HANDLE_UNCACHEABLE || prot == NVMAP_HANDLE_WRITE_COMBINE) + goto out; + + ret = nvmap_cache_maint_phys_range(NVMAP_CACHE_OP_WB_INV, phys, end, + true, prot != NVMAP_HANDLE_INNER_CACHEABLE); + if (ret) + goto out; +out: + wmb(); + return ret; +} diff --git a/drivers/video/tegra/nvmap/nvmap_heap.h b/drivers/video/tegra/nvmap/nvmap_heap.h new file mode 100644 index 00000000..1527b6b2 --- /dev/null +++ b/drivers/video/tegra/nvmap/nvmap_heap.h @@ -0,0 +1,79 @@ +/* + * drivers/video/tegra/nvmap/nvmap_heap.h + * + * GPU heap allocator. + * + * Copyright (c) 2010-2022, NVIDIA Corporation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + */ + +#ifndef __NVMAP_HEAP_H +#define __NVMAP_HEAP_H + +struct device; +struct nvmap_heap; +struct nvmap_client; + +struct nvmap_heap_block { + phys_addr_t base; + unsigned int type; + struct nvmap_handle *handle; +}; + +struct nvmap_heap { + struct list_head all_list; + struct mutex lock; + const char *name; + void *arg; + /* heap base */ + phys_addr_t base; + /* heap size */ + size_t len; + size_t free_size; + struct device *cma_dev; + struct device *dma_dev; + bool is_ivm; + bool can_alloc; /* Used only if is_ivm == true */ + unsigned int peer; /* Used only if is_ivm == true */ + unsigned int vm_id; /* Used only if is_ivm == true */ + struct nvmap_pm_ops pm_ops; +#ifdef NVMAP_CONFIG_DEBUG_MAPS + struct rb_root device_names; +#endif /* NVMAP_CONFIG_DEBUG_MAPS */ +}; + +struct nvmap_heap *nvmap_heap_create(struct device *parent, + const struct nvmap_platform_carveout *co, + phys_addr_t base, size_t len, void *arg); + +void nvmap_heap_destroy(struct nvmap_heap *heap); + +struct nvmap_heap_block *nvmap_heap_alloc(struct nvmap_heap *heap, + struct nvmap_handle *handle, + phys_addr_t *start); + +struct nvmap_heap *nvmap_block_to_heap(struct nvmap_heap_block *b); + +void nvmap_heap_free(struct nvmap_heap_block *block); + +int __init nvmap_heap_init(void); + +void nvmap_heap_deinit(void); + +int nvmap_flush_heap_block(struct nvmap_client *client, + struct nvmap_heap_block *block, size_t len, unsigned int prot); + +void nvmap_heap_debugfs_init(struct dentry *heap_root, struct nvmap_heap *heap); + +int nvmap_query_heap_peer(struct nvmap_heap *heap, unsigned int *peer); +size_t nvmap_query_heap_size(struct nvmap_heap *heap); + +#endif diff --git a/drivers/video/tegra/nvmap/nvmap_id_array.c b/drivers/video/tegra/nvmap/nvmap_id_array.c new file mode 100644 index 00000000..86b96835 --- /dev/null +++ b/drivers/video/tegra/nvmap/nvmap_id_array.c @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2021-2022 NVIDIA Corporation + */ + +#include +#include +#include "nvmap_priv.h" + +#define XA_START (U32_MAX / 2) +/* + * Initialize xarray mapping + */ +void nvmap_id_array_init(struct xarray *id_arr) +{ + xa_init_flags(id_arr, XA_FLAGS_ALLOC1); +} + +/* + * Remove id to dma_buf mapping + */ +void nvmap_id_array_exit(struct xarray *id_arr) +{ + xa_destroy(id_arr); +} + +/* + * Create mapping between the id(NvRmMemHandle) and dma_buf + */ +int nvmap_id_array_id_alloc(struct xarray *id_arr, u32 *id, struct dma_buf *dmabuf) +{ + if (!id_arr || !dmabuf) + return -EINVAL; + + return xa_alloc(id_arr, id, dmabuf, + XA_LIMIT(XA_START, U32_MAX), GFP_KERNEL); +} + +/* + * Clear mapping between the id(NvRmMemHandle) and dma_buf + */ +struct dma_buf *nvmap_id_array_id_release(struct xarray *id_arr, u32 id) +{ + if (!id_arr || !id) + return NULL; + + return xa_erase(id_arr, id); +} + +/* + * Return dma_buf from the id. + */ +struct dma_buf *nvmap_id_array_get_dmabuf_from_id(struct xarray *id_arr, u32 id) +{ + struct dma_buf *dmabuf; + + dmabuf = xa_load(id_arr, id); + if (!IS_ERR_OR_NULL(dmabuf)) + get_dma_buf(dmabuf); + + return dmabuf; +} diff --git a/drivers/video/tegra/nvmap/nvmap_init.c b/drivers/video/tegra/nvmap/nvmap_init.c new file mode 100644 index 00000000..ba2a571d --- /dev/null +++ b/drivers/video/tegra/nvmap/nvmap_init.c @@ -0,0 +1,979 @@ +/* + * Copyright (c) 2014-2022, NVIDIA CORPORATION. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * Portions derived from Linux kernel source file kernel/dma/coherent.c + */ + +#define pr_fmt(fmt) "%s: " fmt, __func__ + +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(NVMAP_LOADABLE_MODULE) +#include +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0) +#include +#endif + +#include +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0) +#include +#else +#include +#include +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0) +#include "include/linux/nvmap_exports.h" +#endif + +#include "nvmap_priv.h" + +#ifdef CONFIG_TEGRA_VIRTUALIZATION +#include +#include +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0) +#ifdef CONFIG_ARM_DMA_IOMMU_ALIGNMENT +#define DMA_BUF_ALIGNMENT CONFIG_ARM_DMA_IOMMU_ALIGNMENT +#else +#define DMA_BUF_ALIGNMENT 8 +#endif +#endif /* LINUX_VERSION_CODE */ + +#ifndef NVMAP_UPSTREAM_KERNEL +#ifndef NVMAP_CONFIG_VPR_RESIZE +extern phys_addr_t tegra_vpr_start; +extern phys_addr_t tegra_vpr_size; +extern bool tegra_vpr_resize; +#endif /* NVMAP_CONFIG_VPR_RESIZE */ +#endif /* !NVMAP_UPSTREAM_KERNEL */ + +struct device __weak tegra_generic_dev; + +struct device __weak tegra_vpr_dev; +EXPORT_SYMBOL(tegra_vpr_dev); + +struct device __weak tegra_generic_cma_dev; +struct device __weak tegra_vpr_cma_dev; + +#ifdef NVMAP_LOADABLE_MODULE +static struct platform_device *pdev; +#endif /* NVMAP_LOADABLE_MODULE */ + +#ifdef NVMAP_CONFIG_VPR_RESIZE +struct dma_resize_notifier_ops __weak vpr_dev_ops; + +static struct dma_declare_info generic_dma_info = { + .name = "generic", + .size = 0, + .notifier.ops = NULL, +}; + +static struct dma_declare_info vpr_dma_info = { + .name = "vpr", + .size = SZ_32M, + .notifier.ops = &vpr_dev_ops, +}; +#endif + +__weak const struct of_device_id nvmap_of_ids[] = { + { .compatible = "nvidia,carveouts" }, + { .compatible = "nvidia,carveouts-t18x" }, + { } +}; + +static struct nvmap_platform_carveout nvmap_carveouts[] = { + [0] = { + .name = "generic-0", + .usage_mask = NVMAP_HEAP_CARVEOUT_GENERIC, + .base = 0, + .size = 0, + .dma_dev = &tegra_generic_dev, + .cma_dev = &tegra_generic_cma_dev, +#ifdef NVMAP_CONFIG_VPR_RESIZE + .dma_info = &generic_dma_info, +#endif + }, + [1] = { + .name = "vpr", + .usage_mask = NVMAP_HEAP_CARVEOUT_VPR, + .base = 0, + .size = 0, + .dma_dev = &tegra_vpr_dev, + .cma_dev = &tegra_vpr_cma_dev, +#ifdef NVMAP_CONFIG_VPR_RESIZE + .dma_info = &vpr_dma_info, +#endif + .enable_static_dma_map = true, + }, + [2] = { + .name = "vidmem", + .usage_mask = NVMAP_HEAP_CARVEOUT_VIDMEM, + .base = 0, + .size = 0, + .disable_dynamic_dma_map = true, + .no_cpu_access = true, + }, + [3] = { + .name = "fsi", + .usage_mask = NVMAP_HEAP_CARVEOUT_FSI, + .base = 0, + .size = 0, + }, + /* Need uninitialized entries for IVM carveouts */ + [4] = { + .name = NULL, + .usage_mask = NVMAP_HEAP_CARVEOUT_IVM, + }, + [5] = { + .name = NULL, + .usage_mask = NVMAP_HEAP_CARVEOUT_IVM, + }, + [6] = { + .name = NULL, + .usage_mask = NVMAP_HEAP_CARVEOUT_IVM, + }, + [7] = { + .name = NULL, + .usage_mask = NVMAP_HEAP_CARVEOUT_IVM, + }, +}; + +static struct nvmap_platform_data nvmap_data = { + .carveouts = nvmap_carveouts, + .nr_carveouts = 4, +}; + +static struct nvmap_platform_carveout *nvmap_get_carveout_pdata(const char *name) +{ + struct nvmap_platform_carveout *co; + for (co = nvmap_carveouts; + co < nvmap_carveouts + ARRAY_SIZE(nvmap_carveouts); co++) { + int i = min_t(int, strcspn(name, "_"), strcspn(name, "-")); + /* handle IVM carveouts */ + if ((co->usage_mask == NVMAP_HEAP_CARVEOUT_IVM) && !co->name) + goto found; + + if (strncmp(co->name, name, i)) + continue; +found: + co->dma_dev = co->dma_dev ? co->dma_dev : &co->dev; + return co; + } + pr_err("not enough space for all nvmap carveouts\n"); + return NULL; +} + +int nvmap_register_vidmem_carveout(struct device *dma_dev, + phys_addr_t base, size_t size) +{ + struct nvmap_platform_carveout *vidmem_co; + + if (!base || !size || (base != PAGE_ALIGN(base)) || + (size != PAGE_ALIGN(size))) + return -EINVAL; + + vidmem_co = nvmap_get_carveout_pdata("vidmem"); + if (!vidmem_co) + return -ENODEV; + + if (vidmem_co->base || vidmem_co->size) + return -EEXIST; + + vidmem_co->base = base; + vidmem_co->size = size; + if (dma_dev) + vidmem_co->dma_dev = dma_dev; + return nvmap_create_carveout(vidmem_co); +} +EXPORT_SYMBOL(nvmap_register_vidmem_carveout); + +#ifdef CONFIG_TEGRA_VIRTUALIZATION +static int __init nvmap_populate_ivm_carveout(struct device *dev) +{ + char *name; + const __be32 *prop; + int ret = 0; + struct nvmap_platform_carveout *co; + struct of_phandle_iterator it; + struct tegra_hv_ivm_cookie *ivm; + unsigned long long id; + unsigned int guestid; + + if (!of_phandle_iterator_init(&it, dev->of_node, "memory-region", NULL, 0)) { + while (!of_phandle_iterator_next(&it) && it.node) { + if (of_device_is_available(it.node) && + of_device_is_compatible(it.node, "nvidia,ivm_carveout") > 0) { + co = nvmap_get_carveout_pdata("nvidia,ivm_carveout"); + if (!co) { + ret = -ENOMEM; + goto err; + } + + if (hyp_read_gid(&guestid)) { + pr_err("failed to read gid\n"); + ret = -EINVAL; + goto err; + } + + prop = of_get_property(it.node, "ivm", NULL); + if (!prop) { + pr_err("failed to read ivm property\n"); + ret = -EINVAL; + goto err; + } + + id = of_read_number(prop + 1, 1); + if (id > UINT_MAX) { + ret = -EINVAL; + goto err; + } + ivm = tegra_hv_mempool_reserve(id); + if (IS_ERR_OR_NULL(ivm)) { + pr_err("failed to reserve IVM memory pool %llu\n", id); + ret = -ENOMEM; + goto err; + } + /* XXX: Are these the available fields from IVM cookie? */ + co->base = (phys_addr_t)ivm->ipa; + co->peer = ivm->peer_vmid; + co->size = ivm->size; + co->vmid = guestid; + + if (!co->base || !co->size) { + ret = -EINVAL; + goto fail; + } + + /* See if this VM can allocate (or just create handle from ID) + * generated by peer partition + */ + prop = of_get_property(it.node, "alloc", NULL); + if (!prop) { + pr_err("failed to read alloc property\n"); + ret = -EINVAL; + goto fail; + } + + name = kzalloc(32, GFP_KERNEL); + if (!name) { + ret = -ENOMEM; + goto fail; + } + + co->can_alloc = of_read_number(prop, 1); + co->is_ivm = true; + sprintf(name, "ivm%02u%02u%02d", co->vmid, co->peer, co->can_alloc); + pr_info("IVM carveout IPA:%p, size=%zu, peer vmid=%u, name=%s\n", + (void *)(uintptr_t)co->base, co->size, co->peer, name); + co->name = name; + nvmap_data.nr_carveouts++; + + } + } + } + return 0; +fail: + co->base = 0; + co->peer = 0; + co->size = 0; + co->vmid = 0; +err: + return ret; + +} +#endif /* CONFIG_TEGRA_VIRTUALIZATION */ + +/* + * This requires proper kernel arguments to have been passed. + */ +#ifndef NVMAP_UPSTREAM_KERNEL +static int __nvmap_init_legacy(struct device *dev) +{ +#ifndef NVMAP_CONFIG_VPR_RESIZE + /* VPR */ + if (!nvmap_carveouts[1].base) { + nvmap_carveouts[1].base = tegra_vpr_start; + nvmap_carveouts[1].size = tegra_vpr_size; + nvmap_carveouts[1].cma_dev = NULL; + } +#endif /* NVMAP_CONFIG_VPR_RESIZE */ + + return 0; +} +#endif /* !NVMAP_UPSTREAM_KERNEL */ + +static int __nvmap_init_dt(struct platform_device *pdev) +{ + if (!of_match_device(nvmap_of_ids, &pdev->dev)) { + pr_err("Missing DT entry!\n"); + return -EINVAL; + } + +#ifndef NVMAP_UPSTREAM_KERNEL + /* For VM_2 we need carveout. So, enabling it here */ + __nvmap_init_legacy(&pdev->dev); +#endif /* !NVMAP_UPSTREAM_KERNEL */ + + pdev->dev.platform_data = &nvmap_data; + + return 0; +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0) +static inline struct page **nvmap_kvzalloc_pages(u32 count) +{ + if (count * sizeof(struct page *) <= PAGE_SIZE) + return kzalloc(count * sizeof(struct page *), GFP_KERNEL); + else + return vzalloc(count * sizeof(struct page *)); +} + +static void *__nvmap_dma_alloc_from_coherent(struct device *dev, + struct dma_coherent_mem_replica *mem, + size_t size, + dma_addr_t *dma_handle, + unsigned long attrs, + unsigned long start) +{ + int order = get_order(size); + unsigned long flags; + unsigned int count, i = 0, j = 0; + unsigned int alloc_size; + unsigned long align, pageno; + void *addr = NULL; + struct page **pages = NULL; + int do_memset = 0; + int *bitmap_nos = NULL; + + if (dma_get_attr(DMA_ATTR_ALLOC_EXACT_SIZE, attrs)) + count = PAGE_ALIGN(size) >> PAGE_SHIFT; + else + count = 1 << order; + + if (!count) + return NULL; + + bitmap_nos = vzalloc(count * sizeof(int)); + if (!bitmap_nos) { + dev_err(dev, "failed to allocate memory\n"); + return NULL; + } + if ((mem->flags & DMA_MEMORY_NOMAP) && + dma_get_attr(DMA_ATTR_ALLOC_SINGLE_PAGES, attrs)) { + alloc_size = 1; + pages = nvmap_kvzalloc_pages(count); + if (!pages) + return NULL; + } else { + alloc_size = count; + } + + spin_lock_irqsave(&mem->spinlock, flags); + + if (unlikely(size > (mem->size << PAGE_SHIFT))) + goto err; + + if ((mem->flags & DMA_MEMORY_NOMAP) && + dma_get_attr(DMA_ATTR_ALLOC_SINGLE_PAGES, attrs)) { + align = 0; + } else { + if (order > DMA_BUF_ALIGNMENT) + align = (1 << DMA_BUF_ALIGNMENT) - 1; + else + align = (1 << order) - 1; + } + + while (count) { + pageno = bitmap_find_next_zero_area(mem->bitmap, mem->size, + start, alloc_size, align); + + if (pageno >= mem->size) + goto err; + + count -= alloc_size; + if (pages) + pages[i++] = pfn_to_page(mem->pfn_base + pageno); + bitmap_set(mem->bitmap, pageno, alloc_size); + bitmap_nos[j++] = pageno; + } + + /* + * Memory was found in the coherent area. + */ + *dma_handle = mem->device_base + (pageno << PAGE_SHIFT); + if (!(mem->flags & DMA_MEMORY_NOMAP)) { + addr = mem->virt_base + (pageno << PAGE_SHIFT); + do_memset = 1; + } else if (dma_get_attr(DMA_ATTR_ALLOC_SINGLE_PAGES, attrs)) { + addr = pages; + } + + spin_unlock_irqrestore(&mem->spinlock, flags); + + if (do_memset) + memset(addr, 0, size); + + kvfree(bitmap_nos); + return addr; +err: + while (j--) + bitmap_clear(mem->bitmap, bitmap_nos[j], alloc_size); + + spin_unlock_irqrestore(&mem->spinlock, flags); + kvfree(pages); + kvfree(bitmap_nos); + return NULL; +} + +void *nvmap_dma_alloc_attrs(struct device *dev, size_t size, + dma_addr_t *dma_handle, + gfp_t flag, unsigned long attrs) +{ + struct dma_coherent_mem_replica *mem; + + if (!dev || !dev->dma_mem) + return NULL; + + WARN_ON_ONCE(!dev->coherent_dma_mask); + + mem = (struct dma_coherent_mem_replica *)(dev->dma_mem); + + return __nvmap_dma_alloc_from_coherent(dev, mem, size, dma_handle, + attrs, 0); +} +EXPORT_SYMBOL(nvmap_dma_alloc_attrs); + +void nvmap_dma_free_attrs(struct device *dev, size_t size, void *cpu_addr, + dma_addr_t dma_handle, unsigned long attrs) +{ + void *mem_addr; + unsigned long flags; + unsigned int pageno; + struct dma_coherent_mem_replica *mem; + + if (!dev || !dev->dma_mem) + return; + + mem = (struct dma_coherent_mem_replica *)(dev->dma_mem); + if ((mem->flags & DMA_MEMORY_NOMAP) && + dma_get_attr(DMA_ATTR_ALLOC_SINGLE_PAGES, attrs)) { + struct page **pages = cpu_addr; + int i; + + spin_lock_irqsave(&mem->spinlock, flags); + for (i = 0; i < (size >> PAGE_SHIFT); i++) { + pageno = page_to_pfn(pages[i]) - mem->pfn_base; + if (WARN_ONCE(pageno > mem->size, + "invalid pageno:%d\n", pageno)) + continue; + bitmap_clear(mem->bitmap, pageno, 1); + } + spin_unlock_irqrestore(&mem->spinlock, flags); + kvfree(pages); + return; + } + + if (mem->flags & DMA_MEMORY_NOMAP) + mem_addr = (void *)(uintptr_t)mem->device_base; + else + mem_addr = mem->virt_base; + + if (mem && cpu_addr >= mem_addr && + cpu_addr - mem_addr < mem->size << PAGE_SHIFT) { + unsigned int page = (cpu_addr - mem_addr) >> PAGE_SHIFT; + unsigned long flags; + unsigned int count; + + if (DMA_ATTR_ALLOC_EXACT_SIZE & attrs) + count = PAGE_ALIGN(size) >> PAGE_SHIFT; + else + count = 1 << get_order(size); + + spin_lock_irqsave(&mem->spinlock, flags); + bitmap_clear(mem->bitmap, page, count); + spin_unlock_irqrestore(&mem->spinlock, flags); + } +} +EXPORT_SYMBOL(nvmap_dma_free_attrs); + +void *nvmap_dma_mark_declared_memory_occupied(struct device *dev, + dma_addr_t device_addr, size_t size) +{ + struct dma_coherent_mem_replica *mem; + unsigned long flags; + int pos, err; + + if (!dev || !dev->dma_mem) + return ERR_PTR(-EINVAL); + + mem = (struct dma_coherent_mem_replica *)(dev->dma_mem); + + size += device_addr & ~PAGE_MASK; + + spin_lock_irqsave(&mem->spinlock, flags); + pos = PFN_DOWN(device_addr - mem->device_base); + err = bitmap_allocate_region(mem->bitmap, pos, get_order(size)); + spin_unlock_irqrestore(&mem->spinlock, flags); + + if (err != 0) + return ERR_PTR(err); + return mem->virt_base + (pos << PAGE_SHIFT); +} + +void nvmap_dma_mark_declared_memory_unoccupied(struct device *dev, + dma_addr_t device_addr, size_t size) +{ + struct dma_coherent_mem_replica *mem; + unsigned long flags; + int pos; + + if (!dev || !dev->dma_mem) + return; + + mem = (struct dma_coherent_mem_replica *)(dev->dma_mem); + + size += device_addr & ~PAGE_MASK; + + spin_lock_irqsave(&mem->spinlock, flags); + pos = PFN_DOWN(device_addr - mem->device_base); + bitmap_release_region(mem->bitmap, pos, get_order(size)); + spin_unlock_irqrestore(&mem->spinlock, flags); +} + +static void nvmap_dma_release_coherent_memory(struct dma_coherent_mem_replica *mem) +{ + if (!mem) + return; + if (!(mem->flags & DMA_MEMORY_NOMAP)) + memunmap(mem->virt_base); + kfree(mem->bitmap); + kfree(mem); +} + +static int nvmap_dma_assign_coherent_memory(struct device *dev, + struct dma_coherent_mem_replica *mem) +{ + if (!dev) + return -ENODEV; + + if (dev->dma_mem) + return -EBUSY; + + dev->dma_mem = (struct dma_coherent_mem *)mem; + return 0; +} + +static int nvmap_dma_init_coherent_memory( + phys_addr_t phys_addr, dma_addr_t device_addr, size_t size, int flags, + struct dma_coherent_mem_replica **mem) +{ + struct dma_coherent_mem_replica *dma_mem = NULL; + void *mem_base = NULL; + int pages = size >> PAGE_SHIFT; + int bitmap_size = BITS_TO_LONGS(pages) * sizeof(long); + int ret; + + if (!size) + return -EINVAL; + + if (!(flags & DMA_MEMORY_NOMAP)) { + mem_base = memremap(phys_addr, size, MEMREMAP_WC); + if (!mem_base) + return -EINVAL; + } + + dma_mem = kzalloc(sizeof(struct dma_coherent_mem_replica), GFP_KERNEL); + if (!dma_mem) { + ret = -ENOMEM; + goto err_memunmap; + } + + dma_mem->bitmap = kzalloc(bitmap_size, GFP_KERNEL); + if (!dma_mem->bitmap) { + ret = -ENOMEM; + goto err_free_dma_mem; + } + + dma_mem->virt_base = mem_base; + dma_mem->device_base = device_addr; + dma_mem->pfn_base = PFN_DOWN(phys_addr); + dma_mem->size = pages; + dma_mem->flags = flags; + spin_lock_init(&dma_mem->spinlock); + + *mem = dma_mem; + return 0; + +err_free_dma_mem: + kfree(dma_mem); + +err_memunmap: + memunmap(mem_base); + return ret; +} + +int nvmap_dma_declare_coherent_memory(struct device *dev, phys_addr_t phys_addr, + dma_addr_t device_addr, size_t size, int flags) +{ + struct dma_coherent_mem_replica *mem; + int ret; + + ret = nvmap_dma_init_coherent_memory(phys_addr, device_addr, size, flags, &mem); + if (ret) + return ret; + + ret = nvmap_dma_assign_coherent_memory(dev, mem); + if (ret) + nvmap_dma_release_coherent_memory(mem); + return ret; +} +#endif /* LINUX_VERSION_CODE */ + +static int __init nvmap_co_device_init(struct reserved_mem *rmem, + struct device *dev) +{ + struct nvmap_platform_carveout *co = rmem->priv; + int err = 0; + + if (!co) + return -ENODEV; + + /* if co size is 0, => co is not present. So, skip init. */ + if (!co->size) + return 0; + + if (!co->cma_dev) { +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0) + err = dma_declare_coherent_memory(co->dma_dev, 0, + co->base, co->size, + DMA_MEMORY_NOMAP); +#else + err = nvmap_dma_declare_coherent_memory(co->dma_dev, 0, + co->base, co->size, + DMA_MEMORY_NOMAP); +#endif +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0) + if (!err) { +#else + if (err & DMA_MEMORY_NOMAP) { +#endif + dev_info(dev, + "%s :dma coherent mem declare %pa,%zu\n", + co->name, &co->base, co->size); + co->init_done = true; + err = 0; + } else + dev_err(dev, + "%s :dma coherent mem declare fail %pa,%zu,err:%d\n", + co->name, &co->base, co->size, err); + } else { +#ifdef NVMAP_CONFIG_VPR_RESIZE +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0) + /* + * When vpr memory is reserved, kmemleak tries to scan vpr + * memory for pointers. vpr memory should not be accessed + * from cpu so avoid scanning it. When vpr memory is removed, + * the memblock_remove() API ensures that kmemleak won't scan + * a removed block. + */ + if (!strncmp(co->name, "vpr", 3)) + kmemleak_no_scan(__va(co->base)); +#endif + + co->dma_info->cma_dev = co->cma_dev; + err = dma_declare_coherent_resizable_cma_memory( + co->dma_dev, co->dma_info); + if (err) + dev_err(dev, "%s coherent memory declaration failed\n", + co->name); + else +#endif + co->init_done = true; + } + return err; +} + +static void nvmap_co_device_release(struct reserved_mem *rmem,struct device *dev) +{ + struct nvmap_platform_carveout *co = rmem->priv; + + if (!co) + return; + + if (co->usage_mask == NVMAP_HEAP_CARVEOUT_IVM) + kfree(co->name); +} + +static const struct reserved_mem_ops nvmap_co_ops = { + .device_init = nvmap_co_device_init, + .device_release = nvmap_co_device_release, +}; + +#ifndef NVMAP_LOADABLE_MODULE +int __init nvmap_co_setup(struct reserved_mem *rmem) +{ + struct nvmap_platform_carveout *co; + int ret = 0; +#ifdef NVMAP_CONFIG_VPR_RESIZE + struct cma *cma; +#endif + ulong start = sched_clock(); + + co = nvmap_get_carveout_pdata(rmem->name); + if (!co) + return ret; + + rmem->ops = &nvmap_co_ops; + rmem->priv = co; + + co->base = rmem->base; + co->size = rmem->size; + +#ifdef NVMAP_CONFIG_VPR_RESIZE + if (!of_get_flat_dt_prop(rmem->fdt_node, "reusable", NULL) || + of_get_flat_dt_prop(rmem->fdt_node, "no-map", NULL)) + goto skip_cma; + + WARN_ON(!rmem->base); + if (dev_get_cma_area(co->cma_dev)) { + pr_info("cma area initialed in legacy way already\n"); + goto finish; + } +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0) + ret = cma_init_reserved_mem(rmem->base, rmem->size, 0, + rmem->name, &cma); +#else + ret = cma_init_reserved_mem(rmem->base, rmem->size, 0, &cma); +#endif + if (ret) { + pr_info("cma_init_reserved_mem fails for %s\n", rmem->name); + goto finish; + } + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0) + dma_contiguous_early_fixup_vpr(rmem->base, rmem->size); + if (co->cma_dev) + co->cma_dev->cma_area = cma; +#else + dma_contiguous_early_fixup(rmem->base, rmem->size); + dev_set_cma_area(co->cma_dev, cma); +#endif + pr_debug("tegra-carveouts carveout=%s %pa@%pa\n", + rmem->name, &rmem->size, &rmem->base); + goto finish; + +skip_cma: +#endif + co->cma_dev = NULL; +#ifdef NVMAP_CONFIG_VPR_RESIZE +finish: +#endif + nvmap_init_time += sched_clock() - start; + return ret; +} +EXPORT_SYMBOL(nvmap_co_setup); +#else +int __init nvmap_co_setup(struct reserved_mem *rmem) +{ + struct nvmap_platform_carveout *co; + ulong start = sched_clock(); + int ret = 0; + + co = nvmap_get_carveout_pdata(rmem->name); + if (!co) + return ret; + + rmem->ops = &nvmap_co_ops; + rmem->priv = co; + + co->base = rmem->base; + co->size = rmem->size; + co->cma_dev = NULL; + nvmap_init_time += sched_clock() - start; + return ret; +} +#endif /* !NVMAP_LOADABLE_MODULE */ + +RESERVEDMEM_OF_DECLARE(nvmap_co, "nvidia,generic_carveout", nvmap_co_setup); +#ifndef NVMAP_LOADABLE_MODULE +RESERVEDMEM_OF_DECLARE(nvmap_vpr_co, "nvidia,vpr-carveout", nvmap_co_setup); +RESERVEDMEM_OF_DECLARE(nvmap_fsi_co, "nvidia,fsi-carveout", nvmap_co_setup); +#endif /* !NVMAP_LOADABLE_MODULE */ + +/* + * Fills in the platform data either from the device tree or with the + * legacy path. + */ +int __init nvmap_init(struct platform_device *pdev) +{ + int err; + struct reserved_mem rmem; +#ifdef NVMAP_LOADABLE_MODULE + struct reserved_mem *rmem2; + struct device_node *np = pdev->dev.of_node; + struct of_phandle_iterator it; + const char *compp; + + if (!of_phandle_iterator_init(&it, np, "memory-region", NULL, 0)) { + while (!of_phandle_iterator_next(&it) && it.node) { + if (of_device_is_available(it.node) && + !of_device_is_compatible(it.node, "nvidia,ivm_carveout")) { + rmem2 = of_reserved_mem_lookup(it.node); + if (!rmem2) { + if (!of_property_read_string(it.node, "compatible", &compp)) + pr_err("unable to acquire memory-region: %s\n", + compp); + return -EINVAL; + } + nvmap_co_setup(rmem2); + } + } + } +#endif /* NVMAP_LOADABLE_MODULE */ + + if (pdev->dev.of_node) { + err = __nvmap_init_dt(pdev); + if (err) + return err; + } + + err = of_reserved_mem_device_init(&pdev->dev); + if (err) + pr_debug("reserved_mem_device_init fails, try legacy init\n"); + + /* try legacy init */ + if (!nvmap_carveouts[0].init_done) { + rmem.priv = &nvmap_carveouts[0]; + err = nvmap_co_device_init(&rmem, &pdev->dev); + if (err) + goto end; + } + + if (!nvmap_carveouts[1].init_done) { + rmem.priv = &nvmap_carveouts[1]; + err = nvmap_co_device_init(&rmem, &pdev->dev); + if (err) + goto end; + } + +#ifdef CONFIG_TEGRA_VIRTUALIZATION + err = nvmap_populate_ivm_carveout(&pdev->dev); +#endif /* CONFIG_TEGRA_VIRTUALIZATION */ + +end: + return err; +} + +#ifdef NVMAP_LOADABLE_MODULE +static bool nvmap_is_carveout_node_present(void) +{ + struct device_node *np; + + np = of_find_node_by_name(NULL, "tegra-carveouts"); + if (of_device_is_available(np)) { + of_node_put(np); + return true; + } + of_node_put(np); + return false; +} +#endif /* NVMAP_LOADABLE_MODULE */ + +static struct platform_driver __refdata nvmap_driver = { + .probe = nvmap_probe, + .remove = nvmap_remove, + + .driver = { + .name = "tegra-carveouts", + .owner = THIS_MODULE, +#ifndef NVMAP_LOADABLE_MODULE + .of_match_table = nvmap_of_ids, +#endif /* !NVMAP_LOADABLE_MODULE */ + .probe_type = PROBE_PREFER_ASYNCHRONOUS, + .suppress_bind_attrs = true, + }, +}; + +static int __init nvmap_init_driver(void) +{ + int e = 0; + + e = nvmap_heap_init(); + if (e) + goto fail; + +#ifdef NVMAP_LOADABLE_MODULE + if (!(of_machine_is_compatible("nvidia,tegra186") || + of_machine_is_compatible("nvidia,tegra194") || + of_machine_is_compatible("nvidia,tegra234") || + of_machine_is_compatible("nvidia,tegra239") || + of_machine_is_compatible("nvidia,tegra23x") || + of_machine_is_compatible("nvidia,tegra232"))) { + nvmap_heap_deinit(); + return -ENODEV; + } +#endif /* NVMAP_LOADABLE_MODULE */ + e = platform_driver_register(&nvmap_driver); + if (e) { + nvmap_heap_deinit(); + goto fail; + } + +#ifdef NVMAP_LOADABLE_MODULE + if (!nvmap_is_carveout_node_present()) { + pdev = platform_device_register_simple("tegra-carveouts", -1, NULL, 0); + if (IS_ERR(pdev)) { + platform_driver_unregister(&nvmap_driver); + nvmap_heap_deinit(); + return PTR_ERR(pdev); + } + } +#endif /* NVMAP_LOADABLE_MODULE */ + +fail: + return e; +} + +#ifdef NVMAP_LOADABLE_MODULE +module_init(nvmap_init_driver); +#else +fs_initcall(nvmap_init_driver); +#endif /* NVMAP_LOADABLE_MODULE */ + +static void __exit nvmap_exit_driver(void) +{ +#ifdef NVMAP_LOADABLE_MODULE + if (!nvmap_is_carveout_node_present()) + platform_device_unregister(pdev); +#endif /* NVMAP_LOADABLE_MODULE */ + platform_driver_unregister(&nvmap_driver); + nvmap_heap_deinit(); + nvmap_dev = NULL; +} +module_exit(nvmap_exit_driver); +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 16, 0) +MODULE_IMPORT_NS(DMA_BUF); +#endif +MODULE_DESCRIPTION("NvMap: Nvidia Tegra Memory Management Driver"); +MODULE_AUTHOR("Puneet Saxena "); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/video/tegra/nvmap/nvmap_init_t19x.c b/drivers/video/tegra/nvmap/nvmap_init_t19x.c new file mode 100644 index 00000000..dd5ed97c --- /dev/null +++ b/drivers/video/tegra/nvmap/nvmap_init_t19x.c @@ -0,0 +1,62 @@ +/* + * drivers/video/tegra/nvmap/nvmap_init_t19x.c + * + * Copyright (c) 2016-2022, NVIDIA CORPORATION. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + */ + +#define pr_fmt(fmt) "nvmap: %s() " fmt, __func__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nvmap_priv.h" + +bool nvmap_version_t19x; + +const struct of_device_id nvmap_of_ids[] = { + { .compatible = "nvidia,carveouts" }, + { .compatible = "nvidia,carveouts-t18x" }, + { .compatible = "nvidia,carveouts-t19x" }, + { } +}; + +int nvmap_register_cvsram_carveout(struct device *dma_dev, + phys_addr_t base, size_t size, int (*busy)(void), + int (*idle)(void)) +{ + static struct nvmap_platform_carveout cvsram = { + .name = "cvsram", + .usage_mask = NVMAP_HEAP_CARVEOUT_CVSRAM, + .disable_dynamic_dma_map = true, + .no_cpu_access = true, + }; + + cvsram.pm_ops.busy = busy; + cvsram.pm_ops.idle = idle; + + if (!base || !size || (base != PAGE_ALIGN(base)) || + (size != PAGE_ALIGN(size))) + return -EINVAL; + cvsram.base = base; + cvsram.size = size; + + cvsram.dma_dev = &cvsram.dev; + return nvmap_create_carveout(&cvsram); +} +EXPORT_SYMBOL(nvmap_register_cvsram_carveout); diff --git a/drivers/video/tegra/nvmap/nvmap_ioctl.c b/drivers/video/tegra/nvmap/nvmap_ioctl.c new file mode 100644 index 00000000..78c5a8c3 --- /dev/null +++ b/drivers/video/tegra/nvmap/nvmap_ioctl.c @@ -0,0 +1,1434 @@ +/* + * drivers/video/tegra/nvmap/nvmap_ioctl.c + * + * User-space interface to nvmap + * + * Copyright (c) 2011-2022, NVIDIA CORPORATION. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + */ + +#define pr_fmt(fmt) "nvmap: %s() " fmt, __func__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#ifdef NVMAP_CONFIG_SCIIPC +#include +#include +#include "nvmap_sci_ipc.h" +#endif + +#include "nvmap_ioctl.h" +#include "nvmap_priv.h" +#include "nvmap_heap.h" + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0) +#include +#ifndef NVMAP_LOADABLE_MODULE +#include +#endif /* !NVMAP_LOADABLE_MODULE */ +#endif + +extern struct device tegra_vpr_dev; + +static ssize_t rw_handle(struct nvmap_client *client, struct nvmap_handle *h, + int is_read, unsigned long h_offs, + unsigned long sys_addr, unsigned long h_stride, + unsigned long sys_stride, unsigned long elem_size, + unsigned long count); + +struct nvmap_handle *nvmap_handle_get_from_id(struct nvmap_client *client, + u32 id) +{ + struct nvmap_handle *handle = ERR_PTR(-EINVAL); + struct nvmap_handle_info *info; + struct dma_buf *dmabuf; + + if (WARN_ON(!client)) + return ERR_PTR(-EINVAL); + + if (client->ida) { + dmabuf = dma_buf_get((int)id); + /* + * id is dmabuf fd created from foreign dmabuf + * but handle as ID is enabled, hence it doesn't belong + * to nvmap_handle, bail out early. + */ + if (!IS_ERR_OR_NULL(dmabuf)) { + dma_buf_put(dmabuf); + return NULL; + } + + dmabuf = nvmap_id_array_get_dmabuf_from_id(client->ida, id); + } else { + dmabuf = dma_buf_get((int)id); + } + if (IS_ERR_OR_NULL(dmabuf)) + return ERR_CAST(dmabuf); + + if (dmabuf_is_nvmap(dmabuf)) { + info = dmabuf->priv; + handle = info->handle; + if (!nvmap_handle_get(handle)) + handle = ERR_PTR(-EINVAL); + } + + dma_buf_put(dmabuf); + + if (!IS_ERR(handle)) + return handle; + + return NULL; +} + +static int nvmap_install_fd(struct nvmap_client *client, + struct nvmap_handle *handle, int fd, void __user *arg, + void *op, size_t op_size, bool free, struct dma_buf *dmabuf) +{ + int err = 0; + struct nvmap_handle_info *info; + + if (!dmabuf) { + err = -EFAULT; + goto dmabuf_fail; + } + info = dmabuf->priv; + if (IS_ERR_VALUE((uintptr_t)fd)) { + err = fd; + goto fd_fail; + } + + if (copy_to_user(arg, op, op_size)) { + err = -EFAULT; + goto copy_fail; + } + + fd_install(fd, dmabuf->file); + return err; + +copy_fail: + put_unused_fd(fd); +fd_fail: + if (dmabuf) + dma_buf_put(dmabuf); + if (free && handle) + nvmap_free_handle(client, handle, info->is_ro); +dmabuf_fail: + return err; +} + +int nvmap_ioctl_getfd(struct file *filp, void __user *arg) +{ + struct nvmap_handle *handle = NULL; + struct nvmap_create_handle op; + struct nvmap_client *client = filp->private_data; + struct dma_buf *dmabuf; + int ret = 0; + bool is_ro; + + if (copy_from_user(&op, arg, sizeof(op))) + return -EFAULT; + + is_ro = is_nvmap_id_ro(client, op.handle); + + handle = nvmap_handle_get_from_id(client, op.handle); + if (!IS_ERR_OR_NULL(handle)) { + op.fd = nvmap_get_dmabuf_fd(client, handle, is_ro); + nvmap_handle_put(handle); + dmabuf = IS_ERR_VALUE((uintptr_t)op.fd) ? + NULL : (is_ro ? handle->dmabuf_ro : handle->dmabuf); + } else { + /* + * if we get an error, the fd might be non-nvmap dmabuf fd. + * Don't attach nvmap handle with this fd. + */ + dmabuf = dma_buf_get(op.handle); + if (IS_ERR(dmabuf)) + return PTR_ERR(dmabuf); + op.fd = nvmap_dmabuf_duplicate_gen_fd(client, dmabuf); + } + + ret = nvmap_install_fd(client, handle, + op.fd, arg, &op, sizeof(op), 0, dmabuf); + + if (!ret && !IS_ERR_OR_NULL(handle)) + trace_refcount_getfd(handle, dmabuf, + atomic_read(&handle->ref), + atomic_long_read(&dmabuf->file->f_count), + is_ro ? "RO" : "RW"); + return ret; +} + +int nvmap_ioctl_alloc(struct file *filp, void __user *arg) +{ + struct nvmap_alloc_handle op; + struct nvmap_client *client = filp->private_data; + struct nvmap_handle *handle; + struct dma_buf *dmabuf = NULL; + bool is_ro; + int err; + + if (copy_from_user(&op, arg, sizeof(op))) + return -EFAULT; + + if (op.align & (op.align - 1)) + return -EINVAL; + + if (!op.handle) + return -EINVAL; + + handle = nvmap_handle_get_from_id(client, op.handle); + if (IS_ERR_OR_NULL(handle)) + return -EINVAL; + + if (!is_nvmap_memory_available(handle->size, op.heap_mask)) { + nvmap_handle_put(handle); + return -ENOMEM; + } + + /* user-space handles are aligned to page boundaries, to prevent + * data leakage. */ + op.align = max_t(size_t, op.align, PAGE_SIZE); + + err = nvmap_alloc_handle(client, handle, op.heap_mask, op.align, + 0, /* no kind */ + op.flags & (~NVMAP_HANDLE_KIND_SPECIFIED), + NVMAP_IVM_INVALID_PEER); + nvmap_handle_put(handle); + is_ro = is_nvmap_id_ro(client, op.handle); + dmabuf = is_ro ? handle->dmabuf_ro : handle->dmabuf; + + if (!err) + trace_refcount_alloc(handle, dmabuf, + atomic_read(&handle->ref), + atomic_long_read(&dmabuf->file->f_count), + is_ro ? "RO" : "RW"); + return err; +} + +int nvmap_ioctl_alloc_ivm(struct file *filp, void __user *arg) +{ + struct nvmap_client *client = filp->private_data; + struct nvmap_alloc_ivm_handle op; + struct nvmap_handle *handle; + int err; + + if (copy_from_user(&op, arg, sizeof(op))) + return -EFAULT; + + if (op.align & (op.align - 1)) + return -EINVAL; + + handle = nvmap_handle_get_from_id(client, op.handle); + if (IS_ERR_OR_NULL(handle)) + return -EINVAL; + + /* user-space handles are aligned to page boundaries, to prevent + * data leakage. */ + op.align = max_t(size_t, op.align, PAGE_SIZE); + + err = nvmap_alloc_handle(client, handle, op.heap_mask, op.align, + 0, /* no kind */ + op.flags & (~NVMAP_HANDLE_KIND_SPECIFIED), + op.peer); + nvmap_handle_put(handle); + return err; +} + +int nvmap_ioctl_vpr_floor_size(struct file *filp, void __user *arg) +{ + int err=0; + u32 floor_size; + + if (copy_from_user(&floor_size, arg, sizeof(floor_size))) + return -EFAULT; +#ifdef NVMAP_CONFIG_VPR_RESIZE + err = dma_set_resizable_heap_floor_size(&tegra_vpr_dev, floor_size); +#endif + return err; +} + +int nvmap_ioctl_create(struct file *filp, unsigned int cmd, void __user *arg) +{ + struct nvmap_create_handle op; + struct nvmap_handle_ref *ref = NULL; + struct nvmap_client *client = filp->private_data; + struct dma_buf *dmabuf = NULL; + struct nvmap_handle *handle = NULL; + int fd = -1, ret = 0; + u32 id = 0; + bool is_ro = false; + + if (copy_from_user(&op, arg, sizeof(op))) + return -EFAULT; + + if (!client) + return -ENODEV; + + if (cmd == NVMAP_IOC_CREATE) + op.size64 = op.size; + + if ((cmd == NVMAP_IOC_CREATE) || (cmd == NVMAP_IOC_CREATE_64)) { + ref = nvmap_create_handle(client, op.size64, false); + if (!IS_ERR(ref)) + ref->handle->orig_size = op.size64; + } else if (cmd == NVMAP_IOC_FROM_FD) { + is_ro = is_nvmap_dmabuf_fd_ro(op.fd); + ref = nvmap_create_handle_from_fd(client, op.fd); + /* if we get an error, the fd might be non-nvmap dmabuf fd */ + if (IS_ERR_OR_NULL(ref)) { + dmabuf = dma_buf_get(op.fd); + if (IS_ERR(dmabuf)) + return PTR_ERR(dmabuf); + fd = nvmap_dmabuf_duplicate_gen_fd(client, dmabuf); + if (fd < 0) + return fd; + } + } else { + return -EINVAL; + } + + if (!IS_ERR(ref)) { + handle = ref->handle; + dmabuf = is_ro ? handle->dmabuf_ro : handle->dmabuf; + + if (client->ida) { + + if (nvmap_id_array_id_alloc(client->ida, + &id, dmabuf) < 0) { + if (dmabuf) + dma_buf_put(dmabuf); + nvmap_free_handle(client, handle, is_ro); + return -ENOMEM; + } + if (cmd == NVMAP_IOC_CREATE_64) + op.handle64 = id; + else + op.handle = id; + + if (copy_to_user(arg, &op, sizeof(op))) { + if (dmabuf) + dma_buf_put(dmabuf); + nvmap_free_handle(client, handle, is_ro); + nvmap_id_array_id_release(client->ida, id); + return -EFAULT; + } + ret = 0; + goto out; + } + + fd = nvmap_get_dmabuf_fd(client, ref->handle, is_ro); + } else if (!dmabuf) { + return PTR_ERR(ref); + } + + if (cmd == NVMAP_IOC_CREATE_64) + op.handle64 = fd; + else + op.handle = fd; + + ret = nvmap_install_fd(client, handle, fd, + arg, &op, sizeof(op), 1, dmabuf); + +out: + if (!ret && !IS_ERR_OR_NULL(handle)) { + if (cmd == NVMAP_IOC_FROM_FD) + trace_refcount_create_handle_from_fd(handle, dmabuf, + atomic_read(&handle->ref), + atomic_long_read(&dmabuf->file->f_count), + is_ro ? "RO" : "RW"); + else + trace_refcount_create_handle(handle, dmabuf, + atomic_read(&handle->ref), + atomic_long_read(&dmabuf->file->f_count), + is_ro ? "RO" : "RW"); + } + + return ret; +} + +int nvmap_ioctl_create_from_va(struct file *filp, void __user *arg) +{ + int fd = -1; + u32 id = 0; + int err; + struct nvmap_create_handle_from_va op; + struct nvmap_handle_ref *ref = NULL; + struct nvmap_client *client = filp->private_data; + struct dma_buf *dmabuf = NULL; + struct nvmap_handle *handle = NULL; + bool is_ro = false; + + if (copy_from_user(&op, arg, sizeof(op))) + return -EFAULT; + + if (!client) + return -ENODEV; + + is_ro = op.flags & NVMAP_HANDLE_RO; + ref = nvmap_create_handle_from_va(client, op.va, + op.size ? op.size : op.size64, + op.flags); + if (IS_ERR(ref)) + return PTR_ERR(ref); + handle = ref->handle; + + err = nvmap_alloc_handle_from_va(client, handle, + op.va, op.flags); + if (err) { + nvmap_free_handle(client, handle, is_ro); + return err; + } + + dmabuf = is_ro ? ref->handle->dmabuf_ro : ref->handle->dmabuf; + if (client->ida) { + + err = nvmap_id_array_id_alloc(client->ida, &id, + dmabuf); + if (err < 0) { + if (dmabuf) + dma_buf_put(dmabuf); + nvmap_free_handle(client, ref->handle, is_ro); + return -ENOMEM; + } + op.handle = id; + if (copy_to_user(arg, &op, sizeof(op))) { + if (dmabuf) + dma_buf_put(dmabuf); + nvmap_free_handle(client, ref->handle, is_ro); + nvmap_id_array_id_release(client->ida, id); + return -EFAULT; + } + err = 0; + goto out; + } + + fd = nvmap_get_dmabuf_fd(client, ref->handle, is_ro); + + op.handle = fd; + + err = nvmap_install_fd(client, ref->handle, fd, + arg, &op, sizeof(op), 1, dmabuf); + +out: + if (!err) + trace_refcount_create_handle_from_va(handle, dmabuf, + atomic_read(&handle->ref), + atomic_long_read(&dmabuf->file->f_count), + is_ro ? "RO" : "RW"); + + return err; +} + +int nvmap_ioctl_rw_handle(struct file *filp, int is_read, void __user *arg, + size_t op_size) +{ + struct nvmap_client *client = filp->private_data; + struct nvmap_rw_handle __user *uarg = arg; + struct nvmap_rw_handle op; +#ifdef CONFIG_COMPAT + struct nvmap_rw_handle_32 __user *uarg32 = arg; + struct nvmap_rw_handle_32 op32; +#endif + struct nvmap_handle *h; + ssize_t copied; + int err = 0; + unsigned long addr, offset, elem_size, hmem_stride, user_stride; + unsigned long count; + int handle; + +#ifdef CONFIG_COMPAT + if (op_size == sizeof(op32)) { + if (copy_from_user(&op32, arg, sizeof(op32))) + return -EFAULT; + addr = op32.addr; + handle = op32.handle; + offset = op32.offset; + elem_size = op32.elem_size; + hmem_stride = op32.hmem_stride; + user_stride = op32.user_stride; + count = op32.count; + } else +#endif + { + if (copy_from_user(&op, arg, sizeof(op))) + return -EFAULT; + addr = op.addr; + handle = op.handle; + offset = op.offset; + elem_size = op.elem_size; + hmem_stride = op.hmem_stride; + user_stride = op.user_stride; + count = op.count; + } + + if (!addr || !count || !elem_size) + return -EINVAL; + + h = nvmap_handle_get_from_id(client, handle); + if (IS_ERR_OR_NULL(h)) + return -EINVAL; + + /* Don't allow write on RO handle */ + if (!is_read && is_nvmap_id_ro(client, handle)) { + nvmap_handle_put(h); + return -EPERM; + } + + if (is_read && h->heap_type == NVMAP_HEAP_CARVEOUT_VPR) { + nvmap_handle_put(h); + return -EPERM; + } + + /* + * If Buffer is RO and write operation is asked from the buffer, + * return error. + */ + if (h->is_ro && !is_read) { + nvmap_handle_put(h); + return -EPERM; + } + + nvmap_kmaps_inc(h); + trace_nvmap_ioctl_rw_handle(client, h, is_read, offset, + addr, hmem_stride, + user_stride, elem_size, count); + copied = rw_handle(client, h, is_read, offset, + addr, hmem_stride, + user_stride, elem_size, count); + nvmap_kmaps_dec(h); + + if (copied < 0) { + err = copied; + copied = 0; + } else if (copied < (count * elem_size)) + err = -EINTR; + +#ifdef CONFIG_COMPAT + if (op_size == sizeof(op32)) + __put_user(copied, &uarg32->count); + else +#endif + __put_user(copied, &uarg->count); + + nvmap_handle_put(h); + + return err; +} + +int nvmap_ioctl_cache_maint(struct file *filp, void __user *arg, int op_size) +{ + struct nvmap_client *client = filp->private_data; + struct nvmap_cache_op op; + struct nvmap_cache_op_64 op64; +#ifdef CONFIG_COMPAT + struct nvmap_cache_op_32 op32; +#endif + +#ifdef CONFIG_COMPAT + if (op_size == sizeof(op32)) { + if (copy_from_user(&op32, arg, sizeof(op32))) + return -EFAULT; + op64.addr = op32.addr; + op64.handle = op32.handle; + op64.len = op32.len; + op64.op = op32.op; + } else +#endif + { + if (op_size == sizeof(op)) { + if (copy_from_user(&op, arg, sizeof(op))) + return -EFAULT; + op64.addr = op.addr; + op64.handle = op.handle; + op64.len = op.len; + op64.op = op.op; + } else { + if (copy_from_user(&op64, arg, sizeof(op64))) + return -EFAULT; + } + } + + return __nvmap_cache_maint(client, &op64); +} + +int nvmap_ioctl_free(struct file *filp, unsigned long arg) +{ + struct nvmap_client *client = filp->private_data; + struct dma_buf *dmabuf = NULL; + + if (!arg || IS_ERR_OR_NULL(client)) + return 0; + + nvmap_free_handle_from_fd(client, arg); + + if (client->ida) { + dmabuf = dma_buf_get(arg); + /* id is dmabuf fd created from foreign dmabuf */ + if (!IS_ERR_OR_NULL(dmabuf)) { + dma_buf_put(dmabuf); + goto close_fd; + } + return 0; + } +close_fd: + return SYS_CLOSE(arg); +} + +static ssize_t rw_handle(struct nvmap_client *client, struct nvmap_handle *h, + int is_read, unsigned long h_offs, + unsigned long sys_addr, unsigned long h_stride, + unsigned long sys_stride, unsigned long elem_size, + unsigned long count) +{ + ssize_t copied = 0; + void *tmp = NULL; + void *addr; + int ret = 0; + + if (!(h->heap_type & nvmap_dev->cpu_access_mask)) + return -EPERM; + + if (!elem_size || !count) + return -EINVAL; + + if (!h->alloc) + return -EFAULT; + + if (elem_size == h_stride && elem_size == sys_stride && (h_offs % 8 == 0)) { + elem_size *= count; + h_stride = elem_size; + sys_stride = elem_size; + count = 1; + } + + if (elem_size > h->size || + h_offs >= h->size || + elem_size > sys_stride || + elem_size > h_stride || + sys_stride > (h->size - h_offs) / count || + h_offs + h_stride * (count - 1) + elem_size > h->size) + return -EINVAL; + + if (!h->vaddr) { + if (!__nvmap_mmap(h)) + return -ENOMEM; + __nvmap_munmap(h, h->vaddr); + } + + addr = h->vaddr + h_offs; + + /* Allocate buffer to cache data for VPR write */ + if (!is_read && h->heap_type == NVMAP_HEAP_CARVEOUT_VPR) { + tmp = vmalloc(elem_size); + if (!tmp) + return -ENOMEM; + } + + while (count--) { + if (h_offs + elem_size > h->size) { + pr_warn("read/write outside of handle\n"); + ret = -EFAULT; + break; + } + if (is_read && + !(h->userflags & NVMAP_HANDLE_CACHE_SYNC_AT_RESERVE)) + __nvmap_do_cache_maint(client, h, h_offs, + h_offs + elem_size, NVMAP_CACHE_OP_INV, false); + + if (is_read) + ret = copy_to_user((void __user *)sys_addr, addr, elem_size); + else { + if (h->heap_type == NVMAP_HEAP_CARVEOUT_VPR) { + ret = copy_from_user(tmp, (void __user *)sys_addr, + elem_size); + if (!ret) + kasan_memcpy_toio((void __iomem *)addr, tmp, elem_size); + } else + ret = copy_from_user(addr, (void __user *)sys_addr, elem_size); + } + + if (ret) + break; + + if (!is_read && + !(h->userflags & NVMAP_HANDLE_CACHE_SYNC_AT_RESERVE)) + __nvmap_do_cache_maint(client, h, h_offs, + h_offs + elem_size, NVMAP_CACHE_OP_WB_INV, + false); + + copied += elem_size; + sys_addr += sys_stride; + h_offs += h_stride; + addr += h_stride; + } + + /* Release the buffer used for VPR write */ + if (!is_read && h->heap_type == NVMAP_HEAP_CARVEOUT_VPR && tmp) + vfree(tmp); + + return ret ?: copied; +} + +int nvmap_ioctl_get_ivcid(struct file *filp, void __user *arg) +{ + struct nvmap_client *client = filp->private_data; + struct nvmap_create_handle op; + struct nvmap_handle *h = NULL; + + if (copy_from_user(&op, arg, sizeof(op))) + return -EFAULT; + + h = nvmap_handle_get_from_id(client, op.ivm_handle); + if (IS_ERR_OR_NULL(h)) + return -EINVAL; + + if (!h->alloc) { /* || !h->ivm_id) { */ + nvmap_handle_put(h); + return -EFAULT; + } + + op.ivm_id = h->ivm_id; + + nvmap_handle_put(h); + + return copy_to_user(arg, &op, sizeof(op)) ? -EFAULT : 0; +} + +int nvmap_ioctl_get_ivc_heap(struct file *filp, void __user *arg) +{ + struct nvmap_device *dev = nvmap_dev; + int i; + unsigned int heap_mask = 0; + + for (i = 0; i < dev->nr_carveouts; i++) { + struct nvmap_carveout_node *co_heap = &dev->heaps[i]; + unsigned int peer; + + if (!(co_heap->heap_bit & NVMAP_HEAP_CARVEOUT_IVM)) + continue; + + if (nvmap_query_heap_peer(co_heap->carveout, &peer) < 0) + return -EINVAL; + + heap_mask |= BIT(peer); + } + + if (copy_to_user(arg, &heap_mask, sizeof(heap_mask))) + return -EFAULT; + + return 0; +} + +int nvmap_ioctl_create_from_ivc(struct file *filp, void __user *arg) +{ + struct nvmap_create_handle op; + struct nvmap_handle_ref *ref; + struct nvmap_client *client = filp->private_data; + int fd; + phys_addr_t offs; + size_t size = 0; + unsigned int peer; + struct nvmap_heap_block *block = NULL; + + /* First create a new handle and then fake carveout allocation */ + if (copy_from_user(&op, arg, sizeof(op))) + return -EFAULT; + + if (!client) + return -ENODEV; + + ref = nvmap_try_duplicate_by_ivmid(client, op.ivm_id, &block); + if (!ref) { + /* + * See nvmap_heap_alloc() for encoding details. + */ + offs = ((op.ivm_id & + ~((u64)NVMAP_IVM_IVMID_MASK << NVMAP_IVM_IVMID_SHIFT)) >> + NVMAP_IVM_LENGTH_WIDTH) << (ffs(NVMAP_IVM_ALIGNMENT) - 1); + size = (op.ivm_id & + ((1ULL << NVMAP_IVM_LENGTH_WIDTH) - 1)) << PAGE_SHIFT; + peer = (op.ivm_id >> NVMAP_IVM_IVMID_SHIFT); + + ref = nvmap_create_handle(client, size, false); + if (IS_ERR(ref)) { + nvmap_heap_free(block); + return PTR_ERR(ref); + } + ref->handle->orig_size = size; + + ref->handle->peer = peer; + if (!block) + block = nvmap_carveout_alloc(client, ref->handle, + NVMAP_HEAP_CARVEOUT_IVM, &offs); + if (!block) { + nvmap_free_handle(client, ref->handle, false); + return -ENOMEM; + } + + ref->handle->heap_type = NVMAP_HEAP_CARVEOUT_IVM; + ref->handle->heap_pgalloc = false; + ref->handle->ivm_id = op.ivm_id; + ref->handle->carveout = block; + block->handle = ref->handle; + mb(); + ref->handle->alloc = true; + NVMAP_TAG_TRACE(trace_nvmap_alloc_handle_done, + NVMAP_TP_ARGS_CHR(client, ref->handle, ref)); + } + if (client->ida) { + u32 id = 0; + + if (nvmap_id_array_id_alloc(client->ida, &id, + ref->handle->dmabuf) < 0) { + if (ref->handle->dmabuf) + dma_buf_put(ref->handle->dmabuf); + nvmap_free_handle(client, ref->handle, false); + return -ENOMEM; + } + op.ivm_handle = id; + if (copy_to_user(arg, &op, sizeof(op))) { + if (ref->handle->dmabuf) + dma_buf_put(ref->handle->dmabuf); + nvmap_free_handle(client, ref->handle, false); + nvmap_id_array_id_release(client->ida, id); + return -EFAULT; + } + return 0; + } + + fd = nvmap_get_dmabuf_fd(client, ref->handle, false); + + op.ivm_handle = fd; + return nvmap_install_fd(client, ref->handle, fd, + arg, &op, sizeof(op), 1, ref->handle->dmabuf); +} + +int nvmap_ioctl_cache_maint_list(struct file *filp, void __user *arg) +{ + struct nvmap_client *client = filp->private_data; + struct nvmap_cache_op_list op; + u32 *handle_ptr; + u64 *offset_ptr; + u64 *size_ptr; + struct nvmap_handle **refs; + int err = 0; + u32 i, n_unmarshal_handles = 0, count = 0; + size_t bytes; + size_t elem_size; + bool is_32; + + if (copy_from_user(&op, arg, sizeof(op))) + return -EFAULT; + + if (!op.nr || op.nr > UINT_MAX / sizeof(u32)) + return -EINVAL; + + bytes = op.nr * sizeof(*refs); + if (!ACCESS_OK(VERIFY_READ, (const void __user *)op.handles, op.nr * sizeof(u32))) + return -EFAULT; + + elem_size = (op.op & NVMAP_ELEM_SIZE_U64) ? + sizeof(u64) : sizeof(u32); + op.op &= ~NVMAP_ELEM_SIZE_U64; + is_32 = elem_size == sizeof(u32) ? 1 : 0; + + bytes += 2 * op.nr * elem_size; + bytes += op.nr * sizeof(u32); + refs = nvmap_altalloc(bytes); + if (!refs) { + pr_err("memory allocation failed\n"); + return -ENOMEM; + } + + offset_ptr = (u64 *)(refs + op.nr); + size_ptr = (u64 *)(((uintptr_t)offset_ptr) + op.nr * elem_size); + handle_ptr = (u32 *)(((uintptr_t)size_ptr) + op.nr * elem_size); + + if (!op.handles || !op.offsets || !op.sizes) { + pr_err("pointers are invalid\n"); + err = -EINVAL; + goto free_mem; + } + + if (!IS_ALIGNED((ulong)offset_ptr, elem_size) || + !IS_ALIGNED((ulong)size_ptr, elem_size) || + !IS_ALIGNED((ulong)handle_ptr, sizeof(u32))) { + pr_err("pointers are not properly aligned!!\n"); + err = -EINVAL; + goto free_mem; + } + + if (copy_from_user(handle_ptr, (void __user *)op.handles, + op.nr * sizeof(u32))) { + pr_err("Can't copy from user pointer op.handles\n"); + err = -EFAULT; + goto free_mem; + } + + if (copy_from_user(offset_ptr, (void __user *)op.offsets, + op.nr * elem_size)) { + pr_err("Can't copy from user pointer op.offsets\n"); + err = -EFAULT; + goto free_mem; + } + + if (copy_from_user(size_ptr, (void __user *)op.sizes, + op.nr * elem_size)) { + pr_err("Can't copy from user pointer op.sizes\n"); + err = -EFAULT; + goto free_mem; + } + + for (i = 0; i < op.nr; i++) { + refs[i] = nvmap_handle_get_from_id(client, handle_ptr[i]); + if (IS_ERR_OR_NULL(refs[i])) { + pr_err("invalid handle_ptr[%d] = %u\n", + i, handle_ptr[i]); + err = -EINVAL; + goto free_mem; + } + if (!(refs[i]->heap_type & nvmap_dev->cpu_access_mask)) { + pr_err("heap %x can't be accessed from cpu\n", + refs[i]->heap_type); + err = -EPERM; + goto free_mem; + } + + n_unmarshal_handles++; + } + + /* + * Either all handles should have NVMAP_HANDLE_CACHE_SYNC_AT_RESERVE + * or none should have it. + */ + for (i = 0; i < op.nr; i++) + if (refs[i]->userflags & NVMAP_HANDLE_CACHE_SYNC_AT_RESERVE) + count++; + + if (op.nr && count % op.nr) { + pr_err("incorrect CACHE_SYNC_AT_RESERVE mix of handles\n"); + err = -EINVAL; + goto free_mem; + } + + /* + * when NVMAP_HANDLE_CACHE_SYNC_AT_RESERVE is specified mix can cause + * cache WB_INV at unreserve op on iovmm handles increasing overhead. + * So, either all handles should have pages from carveout or from iovmm. + */ + if (count) { + for (i = 0; i < op.nr; i++) + if (refs[i]->heap_pgalloc) + count++; + + if (op.nr && count % op.nr) { + pr_err("all or none of the handles should be from heap\n"); + err = -EINVAL; + goto free_mem; + } + } + + err = nvmap_do_cache_maint_list(refs, offset_ptr, size_ptr, + op.op, op.nr, is_32); + +free_mem: + for (i = 0; i < n_unmarshal_handles; i++) + nvmap_handle_put(refs[i]); + nvmap_altfree(refs, bytes); + return err; +} + +int nvmap_ioctl_gup_test(struct file *filp, void __user *arg) +{ + struct nvmap_client *client = filp->private_data; + int err = -EINVAL; + struct nvmap_gup_test op; + struct vm_area_struct *vma; + struct nvmap_handle *handle; + size_t i; + size_t nr_page; + struct page **pages; + struct mm_struct *mm = current->mm; + + if (copy_from_user(&op, arg, sizeof(op))) + return -EFAULT; + op.result = 1; + + nvmap_acquire_mmap_read_lock(mm); + vma = find_vma(mm, op.va); + if (unlikely(!vma) || (unlikely(op.va < vma->vm_start )) || + unlikely(op.va >= vma->vm_end)) { + nvmap_release_mmap_read_lock(mm); + goto exit; + } + + handle = nvmap_handle_get_from_id(client, op.handle); + if (IS_ERR_OR_NULL(handle)) { + nvmap_release_mmap_read_lock(mm); + goto exit; + } + + if (vma->vm_end - vma->vm_start != handle->size) { + pr_err("handle size(0x%zx) and vma size(0x%lx) don't match\n", + handle->size, vma->vm_end - vma->vm_start); + nvmap_release_mmap_read_lock(mm); + goto put_handle; + } + + err = -ENOMEM; + nr_page = handle->size >> PAGE_SHIFT; + pages = nvmap_altalloc(nr_page * sizeof(*pages)); + if (IS_ERR_OR_NULL(pages)) { + err = PTR_ERR(pages); + nvmap_release_mmap_read_lock(mm); + goto put_handle; + } + + err = nvmap_get_user_pages(op.va & PAGE_MASK, nr_page, pages, false, 0); + if (err) { + nvmap_release_mmap_read_lock(mm); + goto free_pages; + } + + nvmap_release_mmap_read_lock(mm); + + for (i = 0; i < nr_page; i++) { + if (handle->pgalloc.pages[i] != pages[i]) { + pr_err("page pointers don't match, %p %p\n", + handle->pgalloc.pages[i], pages[i]); + op.result = 0; + } + } + + if (op.result) + err = 0; + + if (copy_to_user(arg, &op, sizeof(op))) + err = -EFAULT; + + for (i = 0; i < nr_page; i++) { + put_page(pages[i]); + } +free_pages: + nvmap_altfree(pages, nr_page * sizeof(*pages)); +put_handle: + nvmap_handle_put(handle); +exit: + pr_info("GUP Test %s\n", err ? "failed" : "passed"); + return err; +} + +int nvmap_ioctl_set_tag_label(struct file *filp, void __user *arg) +{ + struct nvmap_set_tag_label op; + struct nvmap_device *dev = nvmap_dev; + int err; + + if (copy_from_user(&op, arg, sizeof(op))) + return -EFAULT; + + if (op.len > NVMAP_TAG_LABEL_MAXLEN) + op.len = NVMAP_TAG_LABEL_MAXLEN; + + if (op.len) + err = nvmap_define_tag(dev, op.tag, + (const char __user *)op.addr, op.len); + else + err = nvmap_remove_tag(dev, op.tag); + + return err; +} + +int nvmap_ioctl_get_available_heaps(struct file *filp, void __user *arg) +{ + struct nvmap_available_heaps op; + int i; + + memset(&op, 0, sizeof(op)); + + for (i = 0; i < nvmap_dev->nr_carveouts; i++) + op.heaps |= nvmap_dev->heaps[i].heap_bit; + + if (copy_to_user(arg, &op, sizeof(op))) { + pr_err("copy_to_user failed\n"); + return -EINVAL; + } + + return 0; +} + +int nvmap_ioctl_get_heap_size(struct file *filp, void __user *arg) +{ + struct nvmap_heap_size op; + struct nvmap_heap *heap; + int i; + memset(&op, 0, sizeof(op)); + + if (copy_from_user(&op, arg, sizeof(op))) + return -EFAULT; + + for (i = 0; i < nvmap_dev->nr_carveouts; i++) { + if (op.heap & nvmap_dev->heaps[i].heap_bit) { + heap = nvmap_dev->heaps[i].carveout; + op.size = nvmap_query_heap_size(heap); + if (copy_to_user(arg, &op, sizeof(op))) + return -EFAULT; + return 0; + } + } + return -ENODEV; + +} + +int nvmap_ioctl_get_handle_parameters(struct file *filp, void __user *arg) +{ + struct nvmap_client *client = filp->private_data; + struct nvmap_handle_parameters op; + struct nvmap_handle *handle; + bool is_ro = false; + + if (copy_from_user(&op, arg, sizeof(op))) + return -EFAULT; + + handle = nvmap_handle_get_from_id(client, op.handle); + if (IS_ERR_OR_NULL(handle)) + goto exit; + + if (!handle->alloc) { + op.heap = 0; + } else { + op.heap = handle->heap_type; + } + + /* heap_number, only valid for IVM carveout */ + op.heap_number = handle->peer; + + op.size = handle->size; + + if (handle->userflags & NVMAP_HANDLE_PHYS_CONTIG) { + op.contig = 1U; + } else { + op.contig = 0U; + } + + op.align = handle->align; + + op.offset = handle->offs; + + op.coherency = handle->flags; + + is_ro = is_nvmap_id_ro(client, op.handle); + if (is_ro) + op.access_flags = NVMAP_HANDLE_RO; + + nvmap_handle_put(handle); + + if (copy_to_user(arg, &op, sizeof(op))) + return -EFAULT; + return 0; + +exit: + return -ENODEV; +} + +#ifdef NVMAP_CONFIG_SCIIPC +int nvmap_ioctl_get_sci_ipc_id(struct file *filp, void __user *arg) +{ + struct nvmap_client *client = filp->private_data; + NvSciIpcEndpointVuid pr_vuid, lclu_vuid; + struct nvmap_handle *handle = NULL; + struct nvmap_sciipc_map op; + struct dma_buf *dmabuf = NULL; + bool is_ro = false; + int ret = 0; + + if (copy_from_user(&op, arg, sizeof(op))) + return -EFAULT; + + handle = nvmap_handle_get_from_id(client, op.handle); + if (IS_ERR_OR_NULL(handle)) + return -ENODEV; + + is_ro = is_nvmap_id_ro(client, op.handle); + + /* Cannot create RW handle from RO handle */ + if (is_ro && (op.flags != PROT_READ)) { + ret = -EPERM; + goto exit; + } + + ret = nvmap_validate_sci_ipc_params(client, op.auth_token, + &pr_vuid, &lclu_vuid); + if (ret) + goto exit; + + ret = nvmap_create_sci_ipc_id(client, handle, op.flags, + &op.sci_ipc_id, pr_vuid, is_ro); + if (ret) + goto exit; + + if (copy_to_user(arg, &op, sizeof(op))) { + pr_err("copy_to_user failed\n"); + ret = -EINVAL; + } +exit: + nvmap_handle_put(handle); + dmabuf = is_ro ? handle->dmabuf_ro : handle->dmabuf; + + if (!ret) + trace_refcount_get_sci_ipc_id(handle, dmabuf, + atomic_read(&handle->ref), + atomic_long_read(&dmabuf->file->f_count), + is_ro ? "RO" : "RW"); + return ret; +} + +int nvmap_ioctl_handle_from_sci_ipc_id(struct file *filp, void __user *arg) +{ + struct nvmap_client *client = filp->private_data; + NvSciIpcEndpointVuid pr_vuid, lclu_vuid; + struct nvmap_sciipc_map op; + int ret = 0; + + if (copy_from_user(&op, arg, sizeof(op))) { + ret = -EFAULT; + goto exit; + } + + ret = nvmap_validate_sci_ipc_params(client, op.auth_token, + &pr_vuid, &lclu_vuid); + if (ret) + goto exit; + + ret = nvmap_get_handle_from_sci_ipc_id(client, op.flags, + op.sci_ipc_id, lclu_vuid, &op.handle); + if (ret) + goto exit; + + if (copy_to_user(arg, &op, sizeof(op))) { + pr_err("copy_to_user failed\n"); + ret = -EINVAL; + } +exit: + return ret; +} +#else +int nvmap_ioctl_get_sci_ipc_id(struct file *filp, void __user *arg) +{ + return -EPERM; +} +int nvmap_ioctl_handle_from_sci_ipc_id(struct file *filp, void __user *arg) +{ + return -EPERM; +} +#endif + +/* + * This function calculates allocatable free memory using following formula: + * free_mem = avail mem - cma free - (avail mem - cma free) / 16 + * The CMA memory is not allocatable by NvMap for regular allocations and it + * is part of Available memory reported, so subtract it from available memory. + * NvMap allocates 1/16 extra memory in page coloring, so subtract it as well. + */ +int system_heap_free_mem(unsigned long *mem_val) +{ + long available_mem = 0; + unsigned long free_mem = 0; + unsigned long cma_free = 0; + + available_mem = si_mem_available(); + if (available_mem <= 0) { + *mem_val = 0; + return 0; + } + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0) + cma_free = global_zone_page_state(NR_FREE_CMA_PAGES) << PAGE_SHIFT; +#else + cma_free = global_page_state(NR_FREE_CMA_PAGES) << PAGE_SHIFT; +#endif + if ((available_mem << PAGE_SHIFT) < cma_free) { + *mem_val = 0; + return 0; + } + free_mem = (available_mem << PAGE_SHIFT) - cma_free; +#ifdef NVMAP_CONFIG_COLOR_PAGES + free_mem = free_mem - (free_mem >> 4); +#endif /* NVMAP_CONFIG_COLOR_PAGES */ + *mem_val = free_mem; + return 0; +} + +static unsigned long system_heap_total_mem(void) +{ + struct sysinfo sys_heap; + + si_meminfo(&sys_heap); + + return sys_heap.totalram << PAGE_SHIFT; +} + +int nvmap_ioctl_query_heap_params(struct file *filp, void __user *arg) +{ + unsigned int carveout_mask = NVMAP_HEAP_CARVEOUT_MASK; + unsigned int iovmm_mask = NVMAP_HEAP_IOVMM; + struct nvmap_query_heap_params op; + struct nvmap_heap *heap; + unsigned int type; + int ret = 0; + int i; + unsigned long free_mem = 0; + + memset(&op, 0, sizeof(op)); + if (copy_from_user(&op, arg, sizeof(op))) { + ret = -EFAULT; + goto exit; + } + + type = op.heap_mask; + WARN_ON(type & (type - 1)); + + if (nvmap_convert_carveout_to_iovmm) { + carveout_mask &= ~NVMAP_HEAP_CARVEOUT_GENERIC; + iovmm_mask |= NVMAP_HEAP_CARVEOUT_GENERIC; + } else if (nvmap_convert_iovmm_to_carveout) { + if (type & NVMAP_HEAP_IOVMM) { + type &= ~NVMAP_HEAP_IOVMM; + type |= NVMAP_HEAP_CARVEOUT_GENERIC; + } + } + + /* To Do: select largest free block */ + op.largest_free_block = PAGE_SIZE; + + if (type & NVMAP_HEAP_CARVEOUT_MASK) { + for (i = 0; i < nvmap_dev->nr_carveouts; i++) { + if (type & nvmap_dev->heaps[i].heap_bit) { + heap = nvmap_dev->heaps[i].carveout; + op.total = nvmap_query_heap_size(heap); + op.free = heap->free_size; + break; + } + } + /* If queried heap is not present */ + if (i >= nvmap_dev->nr_carveouts) + return -ENODEV; + + } else if (type & iovmm_mask) { + op.total = system_heap_total_mem(); + ret = system_heap_free_mem(&free_mem); + if (ret) + goto exit; + op.free = free_mem; + } + + if (copy_to_user(arg, &op, sizeof(op))) + ret = -EFAULT; +exit: + return ret; +} + +int nvmap_ioctl_dup_handle(struct file *filp, void __user *arg) +{ + struct nvmap_client *client = filp->private_data; + struct nvmap_handle_ref *ref = NULL; + struct nvmap_handle *handle = NULL; + struct nvmap_duplicate_handle op; + struct dma_buf *dmabuf = NULL; + int fd = -1, ret = 0; + u32 id = 0; + bool is_ro = false; + + if (copy_from_user(&op, arg, sizeof(op))) + return -EFAULT; + + if (!client) + return -ENODEV; + + /* Don't allow duplicating RW handle from RO handle */ + if (is_nvmap_id_ro(client, op.handle) && + op.access_flags != NVMAP_HANDLE_RO) + return -EPERM; + + is_ro = (op.access_flags == NVMAP_HANDLE_RO); + if (!is_ro) + ref = nvmap_create_handle_from_id(client, op.handle); + else + ref = nvmap_dup_handle_ro(client, op.handle); + + if (!IS_ERR(ref)) { + dmabuf = is_ro ? ref->handle->dmabuf_ro : ref->handle->dmabuf; + handle = ref->handle; + if (client->ida) { + if (nvmap_id_array_id_alloc(client->ida, + &id, dmabuf) < 0) { + if (dmabuf) + dma_buf_put(dmabuf); + if (handle) + nvmap_free_handle(client, handle, + is_ro); + return -ENOMEM; + } + op.dup_handle = id; + + if (copy_to_user(arg, &op, sizeof(op))) { + if (dmabuf) + dma_buf_put(dmabuf); + if (handle) + nvmap_free_handle(client, handle, + is_ro); + nvmap_id_array_id_release(client->ida, id); + return -EFAULT; + } + ret = 0; + goto out; + } + fd = nvmap_get_dmabuf_fd(client, ref->handle, is_ro); + } else { + /* if we get an error, the fd might be non-nvmap dmabuf fd */ + dmabuf = dma_buf_get(op.handle); + if (IS_ERR(dmabuf)) + return PTR_ERR(dmabuf); + fd = nvmap_dmabuf_duplicate_gen_fd(client, dmabuf); + if (fd < 0) + return PTR_ERR(ref); + } + + op.dup_handle = fd; + + ret = nvmap_install_fd(client, handle, + op.dup_handle, arg, &op, sizeof(op), 0, dmabuf); +out: + if (!ret && !IS_ERR_OR_NULL(handle)) + trace_refcount_dup_handle(handle, dmabuf, + atomic_read(&handle->ref), + atomic_long_read(&dmabuf->file->f_count), + is_ro ? "RO" : "RW"); + return ret; +} diff --git a/drivers/video/tegra/nvmap/nvmap_ioctl.h b/drivers/video/tegra/nvmap/nvmap_ioctl.h new file mode 100644 index 00000000..271f0122 --- /dev/null +++ b/drivers/video/tegra/nvmap/nvmap_ioctl.h @@ -0,0 +1,79 @@ +/* + * drivers/video/tegra/nvmap/nvmap_ioctl.h + * + * ioctl declarations for nvmap + * + * Copyright (c) 2010-2019, NVIDIA CORPORATION. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + */ + +#ifndef __VIDEO_TEGRA_NVMAP_IOCTL_H +#define __VIDEO_TEGRA_NVMAP_IOCTL_H + +#include + +int nvmap_ioctl_pinop(struct file *filp, bool is_pin, void __user *arg, + bool is32); + +int nvmap_ioctl_getid(struct file *filp, void __user *arg); + +int nvmap_ioctl_get_ivcid(struct file *filp, void __user *arg); + +int nvmap_ioctl_getfd(struct file *filp, void __user *arg); + +int nvmap_ioctl_alloc(struct file *filp, void __user *arg); + +int nvmap_ioctl_alloc_kind(struct file *filp, void __user *arg); + +int nvmap_ioctl_alloc_ivm(struct file *filp, void __user *arg); + +int nvmap_ioctl_vpr_floor_size(struct file *filp, void __user *arg); + +int nvmap_ioctl_free(struct file *filp, unsigned long arg); + +int nvmap_ioctl_create(struct file *filp, unsigned int cmd, void __user *arg); + +int nvmap_ioctl_create_from_va(struct file *filp, void __user *arg); + +int nvmap_ioctl_create_from_ivc(struct file *filp, void __user *arg); + +int nvmap_ioctl_get_ivc_heap(struct file *filp, void __user *arg); + +int nvmap_map_into_caller_ptr(struct file *filp, void __user *arg, bool is32); + +int nvmap_ioctl_cache_maint(struct file *filp, void __user *arg, int size); + +int nvmap_ioctl_rw_handle(struct file *filp, int is_read, void __user *arg, + size_t op_size); + +int nvmap_ioctl_cache_maint_list(struct file *filp, void __user *arg); + +int nvmap_ioctl_gup_test(struct file *filp, void __user *arg); + +int nvmap_ioctl_set_tag_label(struct file *filp, void __user *arg); + +int nvmap_ioctl_get_available_heaps(struct file *filp, void __user *arg); + +int nvmap_ioctl_get_heap_size(struct file *filp, void __user *arg); +void kasan_memcpy_toio(void __iomem *to, const void *from, + size_t count); + +int nvmap_ioctl_get_handle_parameters(struct file *filp, void __user *arg); + +int nvmap_ioctl_get_sci_ipc_id(struct file *filp, void __user *arg); + +int nvmap_ioctl_handle_from_sci_ipc_id(struct file *filp, void __user *arg); + +int nvmap_ioctl_query_heap_params(struct file *filp, void __user *arg); + +int nvmap_ioctl_dup_handle(struct file *filp, void __user *arg); + +#endif /* __VIDEO_TEGRA_NVMAP_IOCTL_H */ diff --git a/drivers/video/tegra/nvmap/nvmap_kasan_wrapper.c b/drivers/video/tegra/nvmap/nvmap_kasan_wrapper.c new file mode 100644 index 00000000..ce09edee --- /dev/null +++ b/drivers/video/tegra/nvmap/nvmap_kasan_wrapper.c @@ -0,0 +1,46 @@ +/* + * drivers/video/tegra/nvmap/nvmap_kasan_wrapper.c + * + * place to add wrapper function to drop kasan scan + * + * Copyright (c) 2018-2022, NVIDIA CORPORATION. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + */ +#include +#include +#include +#include "nvmap_ioctl.h" + +void kasan_memcpy_toio(void __iomem *to, + const void *from, size_t count) +{ + while (count && (!IS_ALIGNED((unsigned long)to, 8) || + !IS_ALIGNED((unsigned long)from, 8))) { + __raw_writeb(*(u8 *)from, to); + from++; + to++; + count--; + } + + while (count >= 8) { + __raw_writeq(*(u64 *)from, to); + from += 8; + to += 8; + count -= 8; + } + + while (count) { + __raw_writeb(*(u8 *)from, to); + from++; + to++; + count--; + } +} diff --git a/drivers/video/tegra/nvmap/nvmap_mm.c b/drivers/video/tegra/nvmap/nvmap_mm.c new file mode 100644 index 00000000..640511f0 --- /dev/null +++ b/drivers/video/tegra/nvmap/nvmap_mm.c @@ -0,0 +1,84 @@ +/* + * drivers/video/tegra/nvmap/nvmap_mm.c + * + * Some MM related functionality specific to nvmap. + * + * Copyright (c) 2013-2021, NVIDIA CORPORATION. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + */ + +#include +#include + +#include + +#include "nvmap_priv.h" + + +#ifndef NVMAP_LOADABLE_MODULE +void nvmap_zap_handle(struct nvmap_handle *handle, u64 offset, u64 size) +{ + struct list_head *vmas; + struct nvmap_vma_list *vma_list; + struct vm_area_struct *vma; + + if (!handle->heap_pgalloc) + return; + + /* if no dirty page is present, no need to zap */ + if (nvmap_handle_track_dirty(handle) && !atomic_read(&handle->pgalloc.ndirty)) + return; + + if (!size) { + offset = 0; + size = handle->size; + } + + size = PAGE_ALIGN((offset & ~PAGE_MASK) + size); + + mutex_lock(&handle->lock); + vmas = &handle->vmas; + list_for_each_entry(vma_list, vmas, list) { + struct nvmap_vma_priv *priv; + size_t vm_size = size; + + vma = vma_list->vma; + priv = vma->vm_private_data; + if ((offset + size) > (vma->vm_end - vma->vm_start)) + vm_size = vma->vm_end - vma->vm_start - offset; + + if (priv->offs || vma->vm_pgoff) + /* vma mapping starts in the middle of handle memory. + * zapping needs special care. zap entire range for now. + * FIXME: optimze zapping. + */ + zap_page_range(vma, vma->vm_start, +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0) + vma->vm_end - vma->vm_start); +#else + vma->vm_end - vma->vm_start, NULL); +#endif + else + zap_page_range(vma, vma->vm_start + offset, +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0) + vm_size); +#else + vm_size, NULL); +#endif + } + mutex_unlock(&handle->lock); +} +#else +void nvmap_zap_handle(struct nvmap_handle *handle, u64 offset, u64 size) +{ + pr_debug("%s is not supported!\n", __func__); +} +#endif /* !NVMAP_LOADABLE_MODULE */ diff --git a/drivers/video/tegra/nvmap/nvmap_pp.c b/drivers/video/tegra/nvmap/nvmap_pp.c new file mode 100644 index 00000000..981a2bf9 --- /dev/null +++ b/drivers/video/tegra/nvmap/nvmap_pp.c @@ -0,0 +1,774 @@ +/* + * drivers/video/tegra/nvmap/nvmap_pp.c + * + * Manage page pools to speed up page allocation. + * + * Copyright (c) 2009-2022, NVIDIA CORPORATION. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + */ + +#define pr_fmt(fmt) "%s: " fmt, __func__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0) +#include +#include +#endif + +#include + +#include "nvmap_priv.h" + +#define NVMAP_TEST_PAGE_POOL_SHRINKER 1 +#define PENDING_PAGES_SIZE (SZ_1M / PAGE_SIZE) + +static bool enable_pp = 1; +static u32 pool_size; + +static struct task_struct *background_allocator; +static DECLARE_WAIT_QUEUE_HEAD(nvmap_bg_wait); + +#ifdef NVMAP_CONFIG_PAGE_POOL_DEBUG +static inline void __pp_dbg_var_add(u64 *dbg_var, u32 nr) +{ + *dbg_var += nr; +} +#else +#define __pp_dbg_var_add(dbg_var, nr) +#endif + +#define pp_alloc_add(pool, nr) __pp_dbg_var_add(&(pool)->allocs, nr) +#define pp_fill_add(pool, nr) __pp_dbg_var_add(&(pool)->fills, nr) +#define pp_hit_add(pool, nr) __pp_dbg_var_add(&(pool)->hits, nr) +#define pp_miss_add(pool, nr) __pp_dbg_var_add(&(pool)->misses, nr) + +static int __nvmap_page_pool_fill_lots_locked(struct nvmap_page_pool *pool, + struct page **pages, u32 nr); + +static inline struct page *get_zero_list_page(struct nvmap_page_pool *pool) +{ + struct page *page; + + trace_get_zero_list_page(pool->to_zero); + + if (list_empty(&pool->zero_list)) + return NULL; + + page = list_first_entry(&pool->zero_list, struct page, lru); + list_del(&page->lru); + + pool->to_zero--; + + return page; +} + +static inline struct page *get_page_list_page(struct nvmap_page_pool *pool) +{ + struct page *page; + + trace_get_page_list_page(pool->count); + + if (list_empty(&pool->page_list)) + return NULL; + + page = list_first_entry(&pool->page_list, struct page, lru); + list_del(&page->lru); + + pool->count--; + + return page; +} + +#ifdef CONFIG_ARM64_4K_PAGES +static inline struct page *get_page_list_page_bp(struct nvmap_page_pool *pool) +{ + struct page *page; + + if (list_empty(&pool->page_list_bp)) + return NULL; + + page = list_first_entry(&pool->page_list_bp, struct page, lru); + list_del(&page->lru); + + pool->count -= pool->pages_per_big_pg; + pool->big_page_count -= pool->pages_per_big_pg; + + return page; +} +#endif /* CONFIG_ARM64_4K_PAGES */ + +static inline bool nvmap_bg_should_run(struct nvmap_page_pool *pool) +{ + return !list_empty(&pool->zero_list); +} + +static void nvmap_pp_zero_pages(struct page **pages, int nr) +{ + int i; + + for (i = 0; i < nr; i++) { + clear_highpage(pages[i]); + nvmap_clean_cache_page(pages[i]); + } + + trace_nvmap_pp_zero_pages(nr); +} + +static void nvmap_pp_do_background_zero_pages(struct nvmap_page_pool *pool) +{ + int i; + struct page *page; + int ret; + /* + * Statically declared array of pages to be zeroed in a batch, + * local to this thread but too big for the stack. + */ + static struct page *pending_zero_pages[PENDING_PAGES_SIZE]; + + rt_mutex_lock(&pool->lock); + for (i = 0; i < PENDING_PAGES_SIZE; i++) { + page = get_zero_list_page(pool); + if (page == NULL) + break; + pending_zero_pages[i] = page; + pool->under_zero++; + } + rt_mutex_unlock(&pool->lock); + + nvmap_pp_zero_pages(pending_zero_pages, i); + + rt_mutex_lock(&pool->lock); + ret = __nvmap_page_pool_fill_lots_locked(pool, pending_zero_pages, i); + pool->under_zero -= i; + rt_mutex_unlock(&pool->lock); + + trace_nvmap_pp_do_background_zero_pages(ret, i); + + for (; ret < i; ret++) + __free_page(pending_zero_pages[ret]); +} + +/* + * This thread fills the page pools with zeroed pages. We avoid releasing the + * pages directly back into the page pools since we would then have to zero + * them ourselves. Instead it is easier to just reallocate zeroed pages. This + * happens in the background so that the overhead of allocating zeroed pages is + * not directly seen by userspace. Of course if the page pools are empty user + * space will suffer. + */ +static int nvmap_background_zero_thread(void *arg) +{ + struct nvmap_page_pool *pool = &nvmap_dev->pool; +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0) + struct sched_param param = { .sched_priority = 0 }; +#endif + + pr_info("PP zeroing thread starting.\n"); + + set_freezable(); +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0) + sched_setscheduler(current, SCHED_NORMAL, ¶m); +#else + sched_set_normal(current, MAX_NICE); +#endif + + while (!kthread_should_stop()) { + while (nvmap_bg_should_run(pool)) + nvmap_pp_do_background_zero_pages(pool); + + wait_event_freezable(nvmap_bg_wait, + nvmap_bg_should_run(pool) || + kthread_should_stop()); + } + + return 0; +} + +#ifdef NVMAP_CONFIG_PAGE_POOL_DEBUG +static void nvmap_pgcount(struct page *page, bool incr) +{ + page_ref_add(page, incr ? 1 : -1); +} +#endif /* NVMAP_CONFIG_PAGE_POOL_DEBUG */ + +/* + * Free the passed number of pages from the page pool. This happens regardless + * of whether the page pools are enabled. This lets one disable the page pools + * and then free all the memory therein. + * + * FIXME: Pages in pending_zero_pages[] can still be unreleased. + */ +static ulong nvmap_page_pool_free_pages_locked(struct nvmap_page_pool *pool, + ulong nr_pages) +{ + struct page *page; + bool use_page_list = false; +#ifdef CONFIG_ARM64_4K_PAGES + bool use_page_list_bp = false; + int i; +#endif /* CONFIG_ARM64_4K_PAGES */ + + pr_debug("req to release pages=%ld\n", nr_pages); + + while (nr_pages) { + +#ifdef CONFIG_ARM64_4K_PAGES + if (use_page_list_bp) + page = get_page_list_page_bp(pool); + else if (use_page_list) +#else + if (use_page_list) +#endif /* CONFIG_ARM64_4K_PAGES */ + page = get_page_list_page(pool); + else + page = get_zero_list_page(pool); + + if (!page) { + if (!use_page_list) { + use_page_list = true; + continue; + } +#ifdef CONFIG_ARM64_4K_PAGES + else if (!use_page_list_bp) { + use_page_list_bp = true; + continue; + } +#endif /* CONFIG_ARM64_4K_PAGES */ + break; + } + +#ifdef CONFIG_ARM64_4K_PAGES + if (use_page_list_bp) { + for (i = 0; i < pool->pages_per_big_pg; i++) + __free_page(nth_page(page, i)); + pr_debug("released %d pages\n", pool->pages_per_big_pg); + if (nr_pages > pool->pages_per_big_pg) + nr_pages -= pool->pages_per_big_pg; + else + nr_pages = 0; + } else { +#endif /* CONFIG_ARM64_4K_PAGES */ + __free_page(page); + nr_pages--; + pr_debug("released 1 page\n"); +#ifdef CONFIG_ARM64_4K_PAGES + } +#endif /* CONFIG_ARM64_4K_PAGES */ + } + + pr_debug("remaining pages to release=%ld\n", nr_pages); + return nr_pages; +} + +/* + * Alloc a bunch of pages from the page pool. This will alloc as many as it can + * and return the number of pages allocated. Pages are placed into the passed + * array in a linear fashion starting from index 0. + */ +int nvmap_page_pool_alloc_lots(struct nvmap_page_pool *pool, + struct page **pages, u32 nr) +{ + u32 ind = 0; + u32 non_zero_idx; + u32 non_zero_cnt = 0; + + if (!enable_pp || !nr) + return 0; + + rt_mutex_lock(&pool->lock); + + while (ind < nr) { + struct page *page = NULL; + + if (!non_zero_cnt) + page = get_page_list_page(pool); + + if (!page) { + page = get_zero_list_page(pool); + if (!page) + break; + if (!non_zero_cnt) + non_zero_idx = ind; + non_zero_cnt++; + } + + pages[ind++] = page; +#ifdef NVMAP_CONFIG_PAGE_POOL_DEBUG + nvmap_pgcount(page, false); + BUG_ON(page_count(page) != 1); +#endif /* NVMAP_CONFIG_PAGE_POOL_DEBUG */ + } + + rt_mutex_unlock(&pool->lock); + + /* Zero non-zeroed pages, if any */ + if (non_zero_cnt) + nvmap_pp_zero_pages(&pages[non_zero_idx], non_zero_cnt); + + pp_alloc_add(pool, ind); + pp_hit_add(pool, ind); + pp_miss_add(pool, nr - ind); + + trace_nvmap_pp_alloc_lots(ind, nr); + + return ind; +} + +#ifdef CONFIG_ARM64_4K_PAGES +int nvmap_page_pool_alloc_lots_bp(struct nvmap_page_pool *pool, + struct page **pages, u32 nr) +{ + u32 ind = 0, nr_pages = nr; + struct page *page; + + if (!enable_pp || pool->pages_per_big_pg <= 1 || + nr_pages < pool->pages_per_big_pg) + return 0; + + rt_mutex_lock(&pool->lock); + + while (nr_pages - ind >= pool->pages_per_big_pg) { + int i; + + page = get_page_list_page_bp(pool); + if (!page) + break; + + for (i = 0; i < pool->pages_per_big_pg; i++) + pages[ind + i] = nth_page(page, i); + + ind += pool->pages_per_big_pg; + } + + rt_mutex_unlock(&pool->lock); + return ind; +} + +static bool nvmap_is_big_page(struct nvmap_page_pool *pool, + struct page **pages, int idx, int nr) +{ + int i; + struct page *page = pages[idx]; + + if (pool->pages_per_big_pg <= 1) + return false; + + if (nr - idx < pool->pages_per_big_pg) + return false; + + /* Allow coalescing pages at big page boundary only */ + if (page_to_phys(page) & (pool->big_pg_sz - 1)) + return false; + + for (i = 1; i < pool->pages_per_big_pg; i++) + if (pages[idx + i] != nth_page(page, i)) + break; + + return i == pool->pages_per_big_pg ? true: false; +} +#endif /* CONFIG_ARM64_4K_PAGES */ + +/* + * Fill a bunch of pages into the page pool. This will fill as many as it can + * and return the number of pages filled. Pages are used from the start of the + * passed page pointer array in a linear fashion. + * + * You must lock the page pool before using this. + */ +static int __nvmap_page_pool_fill_lots_locked(struct nvmap_page_pool *pool, + struct page **pages, u32 nr) +{ + int real_nr; + int pages_to_fill; + int ind = 0; + + if (!enable_pp) + return 0; + + BUG_ON(pool->count > pool->max); + real_nr = min_t(u32, pool->max - pool->count, nr); + pages_to_fill = real_nr; + if (real_nr == 0) + return 0; + + while (real_nr > 0) { +#ifdef NVMAP_CONFIG_PAGE_POOL_DEBUG + nvmap_pgcount(pages[ind], true); + BUG_ON(page_count(pages[ind]) != 2); +#endif /* NVMAP_CONFIG_PAGE_POOL_DEBUG */ + +#ifdef CONFIG_ARM64_4K_PAGES + if (nvmap_is_big_page(pool, pages, ind, pages_to_fill)) { + list_add_tail(&pages[ind]->lru, &pool->page_list_bp); + ind += pool->pages_per_big_pg; + real_nr -= pool->pages_per_big_pg; + pool->big_page_count += pool->pages_per_big_pg; + } else { +#endif /* CONFIG_ARM64_4K_PAGES */ + list_add_tail(&pages[ind++]->lru, &pool->page_list); + real_nr--; +#ifdef CONFIG_ARM64_4K_PAGES + } +#endif /* CONFIG_ARM64_4K_PAGES */ + } + + pool->count += ind; + BUG_ON(pool->count > pool->max); + pp_fill_add(pool, ind); + + return ind; +} + +u32 nvmap_page_pool_fill_lots(struct nvmap_page_pool *pool, + struct page **pages, u32 nr) +{ + u32 ret = 0; + u32 i; + u32 save_to_zero; + + rt_mutex_lock(&pool->lock); + + save_to_zero = pool->to_zero; + + ret = min(nr, pool->max - pool->count - pool->to_zero - pool->under_zero); + + for (i = 0; i < ret; i++) { + /* If page has additonal referecnces, Don't add it into + * page pool. get_user_pages() on mmap'ed nvmap handle can + * hold a refcount on the page. These pages can't be + * reused till the additional refs are dropped. + */ + if (page_count(pages[i]) > 1) { + __free_page(pages[i]); + } else { + list_add_tail(&pages[i]->lru, &pool->zero_list); + pool->to_zero++; + } + } + + if (pool->to_zero) + wake_up_interruptible(&nvmap_bg_wait); + ret = i; + + trace_nvmap_pp_fill_zero_lots(save_to_zero, pool->to_zero, + ret, nr); + + rt_mutex_unlock(&pool->lock); + + return ret; +} + +ulong nvmap_page_pool_get_unused_pages(void) +{ + int total = 0; + + if (!nvmap_dev) + return 0; + + total = nvmap_dev->pool.count + nvmap_dev->pool.to_zero; + + return total; +} + +/* + * Remove and free to the system all the pages currently in the page + * pool. This operation will happen even if the page pools are disabled. + */ +int nvmap_page_pool_clear(void) +{ + struct nvmap_page_pool *pool = &nvmap_dev->pool; + + rt_mutex_lock(&pool->lock); + + (void)nvmap_page_pool_free_pages_locked(pool, pool->count + pool->to_zero); + + /* For some reason, if an error occured... */ + if (!list_empty(&pool->page_list) || !list_empty(&pool->zero_list)) { + rt_mutex_unlock(&pool->lock); + return -ENOMEM; + } + + rt_mutex_unlock(&pool->lock); + + return 0; +} + +/* + * Resizes the page pool to the passed size. If the passed size is 0 then + * all associated resources are released back to the system. This operation + * will only occur if the page pools are enabled. + */ +static void nvmap_page_pool_resize(struct nvmap_page_pool *pool, u32 size) +{ + u64 curr; + + rt_mutex_lock(&pool->lock); + + curr = nvmap_page_pool_get_unused_pages(); + if (curr > size) + (void)nvmap_page_pool_free_pages_locked(pool, curr - size); + + pr_debug("page pool resized to %d from %d pages\n", size, pool->max); + pool->max = size; + + rt_mutex_unlock(&pool->lock); +} + +static unsigned long nvmap_page_pool_count_objects(struct shrinker *shrinker, + struct shrink_control *sc) +{ + return nvmap_page_pool_get_unused_pages(); +} + +static unsigned long nvmap_page_pool_scan_objects(struct shrinker *shrinker, + struct shrink_control *sc) +{ + unsigned long remaining; + + pr_debug("sh_pages=%lu", sc->nr_to_scan); + + rt_mutex_lock(&nvmap_dev->pool.lock); + remaining = nvmap_page_pool_free_pages_locked( + &nvmap_dev->pool, sc->nr_to_scan); + rt_mutex_unlock(&nvmap_dev->pool.lock); + + return (remaining == sc->nr_to_scan) ? \ + SHRINK_STOP : (sc->nr_to_scan - remaining); +} + +static struct shrinker nvmap_page_pool_shrinker = { + .count_objects = nvmap_page_pool_count_objects, + .scan_objects = nvmap_page_pool_scan_objects, + .seeks = 1, +}; + +static void shrink_page_pools(unsigned long *total_pages, unsigned long *available_pages) +{ + struct shrink_control sc; + + if (*total_pages == 0) { + sc.gfp_mask = GFP_KERNEL; + sc.nr_to_scan = 0; + *total_pages = nvmap_page_pool_count_objects(NULL, &sc); + } + sc.nr_to_scan = *total_pages; + nvmap_page_pool_scan_objects(NULL, &sc); + *available_pages = nvmap_page_pool_count_objects(NULL, &sc); +} + +#if NVMAP_TEST_PAGE_POOL_SHRINKER +static int shrink_pp; +static int shrink_set(const char *arg, const struct kernel_param *kp) +{ + int cpu = smp_processor_id(); + unsigned long long t1, t2; + unsigned long total_pages, available_pages; + + param_set_int(arg, kp); + + if (shrink_pp) { + total_pages = shrink_pp; + t1 = cpu_clock(cpu); + shrink_page_pools(&total_pages, &available_pages); + t2 = cpu_clock(cpu); + pr_debug("shrink page pools: time=%lldns, " + "total_pages_released=%lu, free_pages_available=%lu", + t2-t1, total_pages, available_pages); + } + return 0; +} + +static int shrink_get(char *buff, const struct kernel_param *kp) +{ + return param_get_int(buff, kp); +} + +static struct kernel_param_ops shrink_ops = { + .get = shrink_get, + .set = shrink_set, +}; + +module_param_cb(shrink_page_pools, &shrink_ops, &shrink_pp, 0644); +#endif + +static int enable_pp_set(const char *arg, const struct kernel_param *kp) +{ + int ret; + + ret = param_set_bool(arg, kp); + if (ret) + return ret; + + if (!enable_pp) + nvmap_page_pool_clear(); + + return 0; +} + +static int enable_pp_get(char *buff, const struct kernel_param *kp) +{ + return param_get_bool(buff, kp); +} + +static struct kernel_param_ops enable_pp_ops = { + .get = enable_pp_get, + .set = enable_pp_set, +}; + +module_param_cb(enable_page_pools, &enable_pp_ops, &enable_pp, 0644); + +static int pool_size_set(const char *arg, const struct kernel_param *kp) +{ + int ret = param_set_uint(arg, kp); + + if (!ret && (pool_size != nvmap_dev->pool.max)) + nvmap_page_pool_resize(&nvmap_dev->pool, pool_size); + + return ret; +} + +static int pool_size_get(char *buff, const struct kernel_param *kp) +{ + return param_get_int(buff, kp); +} + +static struct kernel_param_ops pool_size_ops = { + .get = pool_size_get, + .set = pool_size_set, +}; + +module_param_cb(pool_size, &pool_size_ops, &pool_size, 0644); + +int nvmap_page_pool_debugfs_init(struct dentry *nvmap_root) +{ + struct dentry *pp_root; + + if (!nvmap_root) + return -ENODEV; + + pp_root = debugfs_create_dir("pagepool", nvmap_root); + if (!pp_root) + return -ENODEV; + + debugfs_create_u32("page_pool_available_pages", + S_IRUGO, pp_root, + &nvmap_dev->pool.count); + debugfs_create_u32("page_pool_pages_to_zero", + S_IRUGO, pp_root, + &nvmap_dev->pool.to_zero); +#ifdef CONFIG_ARM64_4K_PAGES + debugfs_create_u32("page_pool_available_big_pages", + S_IRUGO, pp_root, + &nvmap_dev->pool.big_page_count); + debugfs_create_u32("page_pool_big_page_size", + S_IRUGO, pp_root, + &nvmap_dev->pool.big_pg_sz); + debugfs_create_u64("total_big_page_allocs", + S_IRUGO, pp_root, + &nvmap_big_page_allocs); +#endif /* CONFIG_ARM64_4K_PAGES */ + debugfs_create_u64("total_page_allocs", + S_IRUGO, pp_root, + &nvmap_total_page_allocs); + +#ifdef NVMAP_CONFIG_PAGE_POOL_DEBUG + debugfs_create_u64("page_pool_allocs", + S_IRUGO, pp_root, + &nvmap_dev->pool.allocs); + debugfs_create_u64("page_pool_fills", + S_IRUGO, pp_root, + &nvmap_dev->pool.fills); + debugfs_create_u64("page_pool_hits", + S_IRUGO, pp_root, + &nvmap_dev->pool.hits); + debugfs_create_u64("page_pool_misses", + S_IRUGO, pp_root, + &nvmap_dev->pool.misses); +#endif + + return 0; +} + +int nvmap_page_pool_init(struct nvmap_device *dev) +{ + struct sysinfo info; + struct nvmap_page_pool *pool = &dev->pool; + + memset(pool, 0x0, sizeof(*pool)); + rt_mutex_init(&pool->lock); + INIT_LIST_HEAD(&pool->page_list); + INIT_LIST_HEAD(&pool->zero_list); +#ifdef CONFIG_ARM64_4K_PAGES + INIT_LIST_HEAD(&pool->page_list_bp); + + pool->big_pg_sz = NVMAP_PP_BIG_PAGE_SIZE; + pool->pages_per_big_pg = NVMAP_PP_BIG_PAGE_SIZE >> PAGE_SHIFT; +#endif /* CONFIG_ARM64_4K_PAGES */ + + si_meminfo(&info); + pr_info("Total RAM pages: %lu\n", info.totalram); + + if (!NVMAP_CONFIG_PAGE_POOL_SIZE) + /* The ratio is pool pages per 1K ram pages. + * So, the >> 10 */ + pool->max = (info.totalram * NVMAP_PP_POOL_SIZE) >> 10; + else + pool->max = NVMAP_CONFIG_PAGE_POOL_SIZE; + + if (pool->max >= info.totalram) + goto fail; + pool_size = pool->max; + + pr_info("nvmap page pool size: %u pages (%u MB)\n", pool->max, + (pool->max * info.mem_unit) >> 20); + + background_allocator = kthread_run(nvmap_background_zero_thread, + NULL, "nvmap-bz"); + if (IS_ERR(background_allocator)) + goto fail; + + register_shrinker(&nvmap_page_pool_shrinker); + + return 0; +fail: + nvmap_page_pool_fini(dev); + return -ENOMEM; +} + +int nvmap_page_pool_fini(struct nvmap_device *dev) +{ + struct nvmap_page_pool *pool = &dev->pool; + + /* + * if background allocator is not initialzed or not + * properly initialized, then shrinker is also not + * registered + */ + if (!IS_ERR_OR_NULL(background_allocator)) { + unregister_shrinker(&nvmap_page_pool_shrinker); + kthread_stop(background_allocator); + background_allocator = NULL; + } + + WARN_ON(!list_empty(&pool->page_list)); + + return 0; +} diff --git a/drivers/video/tegra/nvmap/nvmap_priv.h b/drivers/video/tegra/nvmap/nvmap_priv.h new file mode 100644 index 00000000..7b3b1029 --- /dev/null +++ b/drivers/video/tegra/nvmap/nvmap_priv.h @@ -0,0 +1,904 @@ +/* + * drivers/video/tegra/nvmap/nvmap.h + * + * GPU memory management driver for Tegra + * + * Copyright (c) 2009-2022, NVIDIA CORPORATION. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + */ + +#ifndef __VIDEO_TEGRA_NVMAP_NVMAP_H +#define __VIDEO_TEGRA_NVMAP_NVMAP_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#ifndef CONFIG_ARM64 +#include +#endif +#include "nvmap_heap.h" +#include "nvmap_stats.h" + +#include + +#define DMA_ERROR_CODE (~(dma_addr_t)0) + +#define __DMA_ATTR(attrs) attrs +#define DEFINE_DMA_ATTRS(attrs) unsigned long attrs = 0 + +/** + * dma_set_attr - set a specific attribute + * @attr: attribute to set + * @attrs: struct dma_attrs (may be NULL) + */ +#define dma_set_attr(attr, attrs) (attrs |= attr) + +/** + * dma_get_attr - check for a specific attribute + * @attr: attribute to set + * @attrs: struct dma_attrs (may be NULL) + */ +#define dma_get_attr(attr, attrs) (attrs & attr) + +#define NVMAP_TAG_LABEL_MAXLEN (63 - sizeof(struct nvmap_tag_entry)) + +#define NVMAP_TP_ARGS_H(handle) \ + handle, \ + atomic_read(&handle->share_count), \ + handle->heap_type == NVMAP_HEAP_IOVMM ? 0 : \ + (handle->carveout ? handle->carveout->base : 0), \ + handle->size, \ + (handle->userflags & 0xFFFF), \ + (handle->userflags >> 16), \ + __nvmap_tag_name(nvmap_dev, handle->userflags >> 16) + +#define NVMAP_TP_ARGS_CHR(client, handle, ref) \ + client, \ + client ? nvmap_client_pid((struct nvmap_client *)client) : 0, \ + (ref) ? atomic_read(&((struct nvmap_handle_ref *)ref)->dupes) : 1, \ + NVMAP_TP_ARGS_H(handle) + +#define NVMAP_TAG_TRACE(x, ...) \ +do { \ + if (x##_enabled()) { \ + mutex_lock(&nvmap_dev->tags_lock); \ + x(__VA_ARGS__); \ + mutex_unlock(&nvmap_dev->tags_lock); \ + } \ +} while (0) + +#define GFP_NVMAP (GFP_KERNEL | __GFP_HIGHMEM | __GFP_NOWARN) + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0) + +/* + * DMA_ATTR_ALLOC_EXACT_SIZE: This tells the DMA-mapping + * subsystem to allocate the exact number of pages + */ +#define DMA_ATTR_ALLOC_EXACT_SIZE (DMA_ATTR_PRIVILEGED << 2) + +#define DMA_MEMORY_NOMAP 0x02 + +#endif /* LINUX_VERSION_CODE */ + +#ifdef NVMAP_LOADABLE_MODULE + +#ifdef NVMAP_UPSTREAM_KERNEL +/* + * DMA_ATTR_READ_ONLY: for DMA memory allocations, attempt to map + * memory as read-only for the device. CPU access will still be + * read-write. This corresponds to the direction being DMA_TO_DEVICE + * instead of DMA_BIDIRECTIONAL. + */ +#define DMA_ATTR_READ_ONLY (DMA_ATTR_PRIVILEGED << 12) + +/* DMA_ATTR_WRITE_ONLY: This tells the DMA-mapping subsystem + * to map as write-only + */ +#define DMA_ATTR_WRITE_ONLY (DMA_ATTR_PRIVILEGED << 13) + +#endif /* NVMAP_UPSTREAM_KERNEL */ +#endif /* NVMAP_LOADABLE_MODULE */ + +#define DMA_ALLOC_FREE_ATTR DMA_ATTR_ALLOC_SINGLE_PAGES +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0) +#define ACCESS_OK(type, addr, size) access_ok(type, addr, size) +#define SYS_CLOSE(arg) sys_close(arg) +#elif LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0) +#define ACCESS_OK(type, addr, size) access_ok(addr, size) +#define SYS_CLOSE(arg) ksys_close(arg) +#else +#define ACCESS_OK(type, addr, size) access_ok(addr, size) +#define SYS_CLOSE(arg) close_fd(arg) +#endif + +struct page; +struct nvmap_device; + +void _nvmap_handle_free(struct nvmap_handle *h); +/* holds max number of handles allocted per process at any time */ +extern u32 nvmap_max_handle_count; +extern u64 nvmap_big_page_allocs; +extern u64 nvmap_total_page_allocs; + +extern bool nvmap_convert_iovmm_to_carveout; +extern bool nvmap_convert_carveout_to_iovmm; + +extern struct vm_operations_struct nvmap_vma_ops; + +#ifdef CONFIG_ARM64 +#define PG_PROT_KERNEL PAGE_KERNEL +#define FLUSH_DCACHE_AREA __flush_dcache_area +#define outer_flush_range(s, e) +#define outer_inv_range(s, e) +#define outer_clean_range(s, e) +#define outer_flush_all() +#define outer_clean_all() +extern void __clean_dcache_page(struct page *); +extern void __clean_dcache_area_poc(void *addr, size_t len); +#else +#define PG_PROT_KERNEL pgprot_kernel +#define FLUSH_DCACHE_AREA __cpuc_flush_dcache_area +extern void __flush_dcache_page(struct address_space *, struct page *); +#endif + +struct nvmap_vma_list { + struct list_head list; + struct vm_area_struct *vma; + unsigned long save_vm_flags; + pid_t pid; + atomic_t ref; +}; + +struct nvmap_carveout_node { + unsigned int heap_bit; + struct nvmap_heap *carveout; + int index; + phys_addr_t base; + size_t size; +}; + +/* handles allocated using shared system memory (either IOVMM- or high-order + * page allocations */ +struct nvmap_pgalloc { + struct page **pages; + bool contig; /* contiguous system memory */ + atomic_t reserved; + atomic_t ndirty; /* count number of dirty pages */ +}; + +#ifdef NVMAP_CONFIG_DEBUG_MAPS +struct nvmap_device_list { + struct rb_node node; + u64 dma_mask; + char *device_name; +}; +#endif /* NVMAP_CONFIG_DEBUG_MAPS */ + +/* bit 31-29: IVM peer + * bit 28-16: offset (aligned to 32K) + * bit 15-00: len (aligned to page_size) + */ +#define NVMAP_IVM_LENGTH_SHIFT (0) +#define NVMAP_IVM_LENGTH_WIDTH (16) +#define NVMAP_IVM_LENGTH_MASK ((1 << NVMAP_IVM_LENGTH_WIDTH) - 1) +#define NVMAP_IVM_OFFSET_SHIFT (NVMAP_IVM_LENGTH_SHIFT + NVMAP_IVM_LENGTH_WIDTH) +#define NVMAP_IVM_OFFSET_WIDTH (13) +#define NVMAP_IVM_OFFSET_MASK ((1 << NVMAP_IVM_OFFSET_WIDTH) - 1) +#define NVMAP_IVM_IVMID_SHIFT (NVMAP_IVM_OFFSET_SHIFT + NVMAP_IVM_OFFSET_WIDTH) +#define NVMAP_IVM_IVMID_WIDTH (3) +#define NVMAP_IVM_IVMID_MASK ((1 << NVMAP_IVM_IVMID_WIDTH) - 1) +#define NVMAP_IVM_ALIGNMENT (SZ_32K) + +struct nvmap_handle_dmabuf_priv { + void *priv; + struct device *dev; + void (*priv_release)(void *priv); + struct list_head list; +}; + +struct nvmap_handle { + struct rb_node node; /* entry on global handle tree */ + atomic_t ref; /* reference count (i.e., # of duplications) */ + atomic_t pin; /* pin count */ + u32 flags; /* caching flags */ + size_t size; /* padded (as-allocated) size */ + size_t orig_size; /* original (as-requested) size */ + size_t align; + struct nvmap_client *owner; + struct dma_buf *dmabuf; + struct dma_buf *dmabuf_ro; + union { + struct nvmap_pgalloc pgalloc; + struct nvmap_heap_block *carveout; + }; + bool heap_pgalloc; /* handle is page allocated (sysmem / iovmm) */ + bool alloc; /* handle has memory allocated */ + bool from_va; /* handle memory is from VA */ + u32 heap_type; /* handle heap is allocated from */ + u32 userflags; /* flags passed from userspace */ + void *vaddr; /* mapping used inside kernel */ + struct list_head vmas; /* list of all user vma's */ + atomic_t umap_count; /* number of outstanding maps from user */ + atomic_t kmap_count; /* number of outstanding map from kernel */ + atomic_t share_count; /* number of processes sharing the handle */ + struct list_head lru; /* list head to track the lru */ + struct mutex lock; + struct list_head dmabuf_priv; + u64 ivm_id; + unsigned int peer; /* Peer VM number */ + int offs; /* Offset in IVM mem pool */ + /* + * To be set only in handle created from VA case if the handle is + * read-only. + */ + bool is_ro; +}; + +struct nvmap_handle_info { + struct nvmap_handle *handle; + struct list_head maps; + struct mutex maps_lock; + bool is_ro; +}; + +struct nvmap_tag_entry { + struct rb_node node; + atomic_t ref; /* reference count (i.e., # of duplications) */ + u32 tag; +}; + +/* handle_ref objects are client-local references to an nvmap_handle; + * they are distinct objects so that handles can be unpinned and + * unreferenced the correct number of times when a client abnormally + * terminates */ +struct nvmap_handle_ref { + struct nvmap_handle *handle; + struct rb_node node; + atomic_t dupes; /* number of times to free on file close */ + bool is_ro; +}; + +#if defined(NVMAP_CONFIG_PAGE_POOLS) +/* + * This is the default ratio defining pool size. It can be thought of as pool + * size in either MB per GB or KB per MB. That means the max this number can + * be is 1024 (all physical memory - not a very good idea) or 0 (no page pool + * at all). + */ +#define NVMAP_PP_POOL_SIZE (128) + +#ifdef CONFIG_ARM64_4K_PAGES +#define NVMAP_PP_BIG_PAGE_SIZE (0x10000) +#endif /* CONFIG_ARM64_4K_PAGES */ +struct nvmap_page_pool { + struct rt_mutex lock; + u32 count; /* Number of pages in the page & dirty list. */ + u32 max; /* Max no. of pages in all lists. */ + u32 to_zero; /* Number of pages on the zero list */ + u32 under_zero; /* Number of pages getting zeroed */ +#ifdef CONFIG_ARM64_4K_PAGES + u32 big_pg_sz; /* big page size supported(64k, etc.) */ + u32 big_page_count; /* Number of zeroed big pages avaialble */ + u32 pages_per_big_pg; /* Number of pages in big page */ +#endif /* CONFIG_ARM64_4K_PAGES */ + struct list_head page_list; + struct list_head zero_list; +#ifdef CONFIG_ARM64_4K_PAGES + struct list_head page_list_bp; +#endif /* CONFIG_ARM64_4K_PAGES */ + +#ifdef NVMAP_CONFIG_PAGE_POOL_DEBUG + u64 allocs; + u64 fills; + u64 hits; + u64 misses; +#endif +}; + +int nvmap_page_pool_init(struct nvmap_device *dev); +int nvmap_page_pool_fini(struct nvmap_device *dev); +struct page *nvmap_page_pool_alloc(struct nvmap_page_pool *pool); +int nvmap_page_pool_alloc_lots(struct nvmap_page_pool *pool, + struct page **pages, u32 nr); +#ifdef CONFIG_ARM64_4K_PAGES +int nvmap_page_pool_alloc_lots_bp(struct nvmap_page_pool *pool, + struct page **pages, u32 nr); +#endif /* CONFIG_ARM64_4K_PAGES */ +u32 nvmap_page_pool_fill_lots(struct nvmap_page_pool *pool, + struct page **pages, u32 nr); +int nvmap_page_pool_clear(void); +int nvmap_page_pool_debugfs_init(struct dentry *nvmap_root); +#endif + +#define NVMAP_IVM_INVALID_PEER (-1) + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0) +#ifndef NVMAP_CONFIG_HANDLE_AS_ID +struct xarray {}; +#endif /* !NVMAP_CONFIG_HANDLE_AS_ID */ +#endif /* KERNEL_VERSION < 5.10 */ + +struct nvmap_client { + const char *name; + struct rb_root handle_refs; + struct mutex ref_lock; + bool kernel_client; + atomic_t count; + struct task_struct *task; + struct list_head list; + u32 handle_count; + u32 next_fd; + int warned; + int tag_warned; + struct xarray id_array; + struct xarray *ida; +}; + +struct nvmap_vma_priv { + struct nvmap_handle *handle; + size_t offs; + atomic_t count; /* number of processes cloning the VMA */ +}; + +struct nvmap_device { + struct rb_root handles; + spinlock_t handle_lock; + struct miscdevice dev_user; + struct nvmap_carveout_node *heaps; + int nr_heaps; + int nr_carveouts; +#ifdef NVMAP_CONFIG_PAGE_POOLS + struct nvmap_page_pool pool; +#endif + struct list_head clients; + struct rb_root pids; + struct mutex clients_lock; + struct list_head lru_handles; + spinlock_t lru_lock; + struct dentry *handles_by_pid; + struct dentry *debug_root; + struct nvmap_platform_data *plat; + struct rb_root tags; + struct mutex tags_lock; + struct mutex carveout_lock; /* needed to serialize carveout creation */ + u32 dynamic_dma_map_mask; + u32 cpu_access_mask; +#ifdef NVMAP_CONFIG_DEBUG_MAPS + struct rb_root device_names; +#endif /* NVMAP_CONFIG_DEBUG_MAPS */ +}; + + +extern struct nvmap_device *nvmap_dev; +extern ulong nvmap_init_time; + +static inline void nvmap_ref_lock(struct nvmap_client *priv) +{ + mutex_lock(&priv->ref_lock); +} + +static inline void nvmap_ref_unlock(struct nvmap_client *priv) +{ + mutex_unlock(&priv->ref_lock); +} + +static inline void nvmap_acquire_mmap_read_lock(struct mm_struct *mm) +{ +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0) + down_read(&mm->mmap_sem); +#else + down_read(&mm->mmap_lock); +#endif +} + +static inline void nvmap_release_mmap_read_lock(struct mm_struct *mm) +{ +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0) + up_read(&mm->mmap_sem); +#else + up_read(&mm->mmap_lock); +#endif +} + +static inline pgprot_t nvmap_pgprot(struct nvmap_handle *h, pgprot_t prot) +{ + if (h->flags == NVMAP_HANDLE_UNCACHEABLE) { +#ifdef CONFIG_ARM64 + if (h->heap_type != NVMAP_HEAP_CARVEOUT_VPR && + h->owner && !h->owner->warned) { + char task_comm[TASK_COMM_LEN]; + h->owner->warned = 1; + get_task_comm(task_comm, h->owner->task); + pr_err("PID %d: %s: TAG: 0x%04x WARNING: " + "NVMAP_HANDLE_WRITE_COMBINE " + "should be used in place of " + "NVMAP_HANDLE_UNCACHEABLE on ARM64\n", + h->owner->task->pid, task_comm, + h->userflags >> 16); + } +#endif + return pgprot_noncached(prot); + } + else if (h->flags == NVMAP_HANDLE_WRITE_COMBINE) + return pgprot_writecombine(prot); + return prot; +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0) +struct dma_coherent_mem_replica { + void *virt_base; + dma_addr_t device_base; + unsigned long pfn_base; +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0) + size_t size; +#else + int size; +#endif + int flags; + unsigned long *bitmap; + spinlock_t spinlock; + bool use_dev_dma_pfn_offset; +}; + +int nvmap_dma_declare_coherent_memory(struct device *dev, phys_addr_t phys_addr, + dma_addr_t device_addr, size_t size, int flags); +#endif +int nvmap_probe(struct platform_device *pdev); +int nvmap_remove(struct platform_device *pdev); +int nvmap_init(struct platform_device *pdev); + +int nvmap_create_carveout(const struct nvmap_platform_carveout *co); +int nvmap_co_setup(struct reserved_mem *rmem); + +struct device *dma_dev_from_handle(unsigned long type); +struct nvmap_heap_block *nvmap_carveout_alloc(struct nvmap_client *dev, + struct nvmap_handle *handle, + unsigned long type, + phys_addr_t *start); + +struct nvmap_carveout_node; + +struct nvmap_handle *nvmap_handle_get(struct nvmap_handle *h); +void nvmap_handle_put(struct nvmap_handle *h); + +struct nvmap_handle_ref *__nvmap_validate_locked(struct nvmap_client *priv, + struct nvmap_handle *h, + bool is_ro); + +struct nvmap_handle *nvmap_validate_get(struct nvmap_handle *h); + +struct nvmap_handle_ref *nvmap_create_handle(struct nvmap_client *client, + size_t size, bool ro_buf); + +struct nvmap_handle_ref *nvmap_create_handle_from_va(struct nvmap_client *client, + ulong addr, size_t size, + unsigned int access_flags); + +struct nvmap_handle_ref *nvmap_dup_handle_ro(struct nvmap_client *client, + int fd); + +bool is_nvmap_dmabuf_fd_ro(int fd); + +bool is_nvmap_id_ro(struct nvmap_client *client, int id); + +struct nvmap_handle_ref *nvmap_duplicate_handle(struct nvmap_client *client, + struct nvmap_handle *h, bool skip_val, + bool is_ro); + +struct nvmap_handle_ref *nvmap_try_duplicate_by_ivmid( + struct nvmap_client *client, u64 ivm_id, + struct nvmap_heap_block **block); + +struct nvmap_handle_ref *nvmap_create_handle_from_id( + struct nvmap_client *client, u32 id); + +struct nvmap_handle_ref *nvmap_create_handle_from_fd( + struct nvmap_client *client, int fd); + +void inner_cache_maint(unsigned int op, void *vaddr, size_t size); +void outer_cache_maint(unsigned int op, phys_addr_t paddr, size_t size); + +int nvmap_alloc_handle(struct nvmap_client *client, + struct nvmap_handle *h, unsigned int heap_mask, + size_t align, u8 kind, + unsigned int flags, unsigned int peer); + +int nvmap_alloc_handle_from_va(struct nvmap_client *client, + struct nvmap_handle *h, + ulong addr, + unsigned int flags); + +void nvmap_free_handle(struct nvmap_client *c, struct nvmap_handle *h, bool is_ro); + +void nvmap_free_handle_from_fd(struct nvmap_client *c, int fd); + +int nvmap_handle_remove(struct nvmap_device *dev, struct nvmap_handle *h); + +void nvmap_handle_add(struct nvmap_device *dev, struct nvmap_handle *h); + +int is_nvmap_vma(struct vm_area_struct *vma); + +int nvmap_get_dmabuf_fd(struct nvmap_client *client, struct nvmap_handle *h, + bool is_ro); +struct nvmap_handle *nvmap_handle_get_from_dmabuf_fd( + struct nvmap_client *client, int fd); +int nvmap_dmabuf_duplicate_gen_fd(struct nvmap_client *client, + struct dma_buf *dmabuf); + +int nvmap_get_handle_param(struct nvmap_client *client, + struct nvmap_handle_ref *ref, u32 param, u64 *result); + +struct nvmap_handle *nvmap_handle_get_from_fd(int fd); + +/* MM definitions. */ +extern void v7_flush_kern_cache_all(void); +extern void v7_clean_kern_cache_all(void *); + +void nvmap_clean_cache(struct page **pages, int numpages); +void nvmap_clean_cache_page(struct page *page); +void nvmap_flush_cache(struct page **pages, int numpages); +int nvmap_cache_maint_phys_range(unsigned int op, phys_addr_t pstart, + phys_addr_t pend, int inner, int outer); + +int nvmap_do_cache_maint_list(struct nvmap_handle **handles, u64 *offsets, + u64 *sizes, int op, u32 nr_ops, bool is_32); +int __nvmap_cache_maint(struct nvmap_client *client, + struct nvmap_cache_op_64 *op); + +/* Internal API to support dmabuf */ +struct dma_buf *__nvmap_make_dmabuf(struct nvmap_client *client, + struct nvmap_handle *handle, bool ro_buf); +struct sg_table *__nvmap_sg_table(struct nvmap_client *client, + struct nvmap_handle *h); +void __nvmap_free_sg_table(struct nvmap_client *client, + struct nvmap_handle *h, struct sg_table *sgt); +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0) +void *__nvmap_kmap(struct nvmap_handle *h, unsigned int pagenum); +void __nvmap_kunmap(struct nvmap_handle *h, unsigned int pagenum, void *addr); +#endif +void *__nvmap_mmap(struct nvmap_handle *h); +void __nvmap_munmap(struct nvmap_handle *h, void *addr); +int __nvmap_map(struct nvmap_handle *h, struct vm_area_struct *vma); +int __nvmap_do_cache_maint(struct nvmap_client *client, struct nvmap_handle *h, + unsigned long start, unsigned long end, + unsigned int op, bool clean_only_dirty); +struct nvmap_client *__nvmap_create_client(struct nvmap_device *dev, + const char *name); +int __nvmap_dmabuf_fd(struct nvmap_client *client, + struct dma_buf *dmabuf, int flags); + +int nvmap_dmabuf_stash_init(void); + +void *nvmap_altalloc(size_t len); +void nvmap_altfree(void *ptr, size_t len); + +static inline struct page *nvmap_to_page(struct page *page) +{ + return (struct page *)((unsigned long)page & ~3UL); +} + +static inline bool nvmap_page_dirty(struct page *page) +{ + return (unsigned long)page & 1UL; +} + +static inline bool nvmap_page_mkdirty(struct page **page) +{ + if (nvmap_page_dirty(*page)) + return false; + *page = (struct page *)((unsigned long)*page | 1UL); + return true; +} + +static inline bool nvmap_page_mkclean(struct page **page) +{ + if (!nvmap_page_dirty(*page)) + return false; + *page = (struct page *)((unsigned long)*page & ~1UL); + return true; +} + +/* + * FIXME: assume user space requests for reserve operations + * are page aligned + */ +static inline int nvmap_handle_mk(struct nvmap_handle *h, + u32 offset, u32 size, + bool (*fn)(struct page **), + bool locked) +{ + int i, nchanged = 0; + u32 start_page = offset >> PAGE_SHIFT; + u32 end_page = PAGE_ALIGN(offset + size) >> PAGE_SHIFT; + + if (!locked) + mutex_lock(&h->lock); + if (h->heap_pgalloc && + (offset < h->size) && + (size <= h->size) && + (offset <= (h->size - size))) { + for (i = start_page; i < end_page; i++) + nchanged += fn(&h->pgalloc.pages[i]) ? 1 : 0; + } + if (!locked) + mutex_unlock(&h->lock); + return nchanged; +} + +static inline void nvmap_handle_mkclean(struct nvmap_handle *h, + u32 offset, u32 size) +{ + int nchanged; + + if (h->heap_pgalloc && !atomic_read(&h->pgalloc.ndirty)) + return; + if (size == 0) + size = h->size; + + nchanged = nvmap_handle_mk(h, offset, size, nvmap_page_mkclean, false); + if (h->heap_pgalloc) + atomic_sub(nchanged, &h->pgalloc.ndirty); +} + +static inline void _nvmap_handle_mkdirty(struct nvmap_handle *h, + u32 offset, u32 size) +{ + int nchanged; + + if (h->heap_pgalloc && + (atomic_read(&h->pgalloc.ndirty) == (h->size >> PAGE_SHIFT))) + return; + + nchanged = nvmap_handle_mk(h, offset, size, nvmap_page_mkdirty, true); + if (h->heap_pgalloc) + atomic_add(nchanged, &h->pgalloc.ndirty); +} + +static inline struct page **nvmap_pages(struct page **pg_pages, u32 nr_pages) +{ + struct page **pages; + int i; + + pages = nvmap_altalloc(sizeof(*pages) * nr_pages); + if (!pages) + return NULL; + + for (i = 0; i < nr_pages; i++) + pages[i] = nvmap_to_page(pg_pages[i]); + + return pages; +} + +void nvmap_zap_handle(struct nvmap_handle *handle, u64 offset, u64 size); + +void nvmap_vma_open(struct vm_area_struct *vma); + +int nvmap_reserve_pages(struct nvmap_handle **handles, u64 *offsets, + u64 *sizes, u32 nr, u32 op, bool is_32); + +static inline void nvmap_kmaps_inc(struct nvmap_handle *h) +{ + mutex_lock(&h->lock); + atomic_inc(&h->kmap_count); + mutex_unlock(&h->lock); +} + +static inline void nvmap_kmaps_inc_no_lock(struct nvmap_handle *h) +{ + atomic_inc(&h->kmap_count); +} + +static inline void nvmap_kmaps_dec(struct nvmap_handle *h) +{ + atomic_dec(&h->kmap_count); +} + +static inline void nvmap_umaps_inc(struct nvmap_handle *h) +{ + mutex_lock(&h->lock); + atomic_inc(&h->umap_count); + mutex_unlock(&h->lock); +} + +static inline void nvmap_umaps_dec(struct nvmap_handle *h) +{ + atomic_dec(&h->umap_count); +} + +static inline void nvmap_lru_add(struct nvmap_handle *h) +{ + spin_lock(&nvmap_dev->lru_lock); + BUG_ON(!list_empty(&h->lru)); + list_add_tail(&h->lru, &nvmap_dev->lru_handles); + spin_unlock(&nvmap_dev->lru_lock); +} + +static inline void nvmap_lru_del(struct nvmap_handle *h) +{ + spin_lock(&nvmap_dev->lru_lock); + list_del(&h->lru); + INIT_LIST_HEAD(&h->lru); + spin_unlock(&nvmap_dev->lru_lock); +} + +static inline void nvmap_lru_reset(struct nvmap_handle *h) +{ + spin_lock(&nvmap_dev->lru_lock); + BUG_ON(list_empty(&h->lru)); + list_del(&h->lru); + list_add_tail(&h->lru, &nvmap_dev->lru_handles); + spin_unlock(&nvmap_dev->lru_lock); +} + +static inline bool nvmap_handle_track_dirty(struct nvmap_handle *h) +{ + if (!h->heap_pgalloc) + return false; + + return h->userflags & (NVMAP_HANDLE_CACHE_SYNC | + NVMAP_HANDLE_CACHE_SYNC_AT_RESERVE); +} + +struct nvmap_tag_entry *nvmap_search_tag_entry(struct rb_root *root, u32 tag); + +int nvmap_define_tag(struct nvmap_device *dev, u32 tag, + const char __user *name, u32 len); + +int nvmap_remove_tag(struct nvmap_device *dev, u32 tag); + +/* must hold tag_lock */ +static inline char *__nvmap_tag_name(struct nvmap_device *dev, u32 tag) +{ + struct nvmap_tag_entry *entry; + + entry = nvmap_search_tag_entry(&dev->tags, tag); + return entry ? (char *)(entry + 1) : ""; +} +static inline pid_t nvmap_client_pid(struct nvmap_client *client) +{ + return client->task ? client->task->pid : 0; +} + +/* must be called with mmap_sem held for read or write */ +static inline int nvmap_get_user_pages(ulong vaddr, + size_t nr_page, struct page **pages, + bool is_user_flags, u32 user_foll_flags) +{ + u32 foll_flags = FOLL_FORCE; + struct vm_area_struct *vma; + vm_flags_t vm_flags; + long user_pages = 0; + int ret = 0; + + vma = find_vma(current->mm, vaddr); + if (vma) { + if (is_user_flags) { + foll_flags |= user_foll_flags; + } else { + vm_flags = vma->vm_flags; + /* + * If the vaddr points to writable page then only + * pass FOLL_WRITE flag + */ + if (vm_flags & VM_WRITE) + foll_flags |= FOLL_WRITE; + } + pr_debug("vaddr %lu is_user_flags %d user_foll_flags %x foll_flags %x.\n", + vaddr, is_user_flags?1:0, user_foll_flags, foll_flags); + user_pages = get_user_pages(vaddr & PAGE_MASK, nr_page, + foll_flags, pages, NULL); + } + if (user_pages != nr_page) { + ret = user_pages < 0 ? user_pages : -ENOMEM; + pr_err("get_user_pages requested/got: %zu/%ld]\n", nr_page, + user_pages); + while (--user_pages >= 0) + put_page(pages[user_pages]); + } + return ret; +} + +#define device_node_from_iter(iter) \ + iter.node + +extern struct of_device_id __nvmapcache_of_table; + +#define NVMAP_CACHE_OF_DECLARE(compat, fn) \ + _OF_DECLARE(nvmapcache, nvmapcache_of, compat, fn, \ + nvmap_setup_chip_cache_fn) + +#ifdef NVMAP_CONFIG_SCIIPC +int nvmap_sci_ipc_init(void); +void nvmap_sci_ipc_exit(void); +#else +__weak int nvmap_sci_ipc_init(void) +{ + return 0; +} +__weak void nvmap_sci_ipc_exit(void) +{ +} +#endif + +#ifdef NVMAP_CONFIG_HANDLE_AS_ID +void nvmap_id_array_init(struct xarray *xarr); +void nvmap_id_array_exit(struct xarray *xarr); +struct dma_buf *nvmap_id_array_get_dmabuf_from_id(struct xarray *xarr, u32 id); +int nvmap_id_array_id_alloc(struct xarray *xarr, u32 *id, struct dma_buf *dmabuf); +struct dma_buf *nvmap_id_array_id_release(struct xarray *xarr, u32 id); +#else +static inline void nvmap_id_array_init(struct xarray *xarr) +{ + +} + +static inline void nvmap_id_array_exit(struct xarray *xarr) +{ + +} + +static inline struct dma_buf *nvmap_id_array_get_dmabuf_from_id(struct xarray *xarr, u32 id) +{ + return NULL; +} + +static inline int nvmap_id_array_id_alloc(struct xarray *xarr, u32 *id, struct dma_buf *dmabuf) +{ + return 0; +} + +static inline struct dma_buf *nvmap_id_array_id_release(struct xarray *xarr, u32 id) +{ + return NULL; +} +#endif +void *nvmap_dmabuf_get_drv_data(struct dma_buf *dmabuf, + struct device *dev); +bool is_nvmap_memory_available(size_t size, uint32_t heap); +int system_heap_free_mem(unsigned long *mem_val); + +#ifdef NVMAP_CONFIG_DEBUG_MAPS +struct nvmap_device_list *nvmap_is_device_present(char *device_name, u32 heap_type); +void nvmap_add_device_name(char *device_name, u64 dma_mask, u32 heap_type); +void nvmap_remove_device_name(char *device_name, u32 heap_type); +#endif /* NVMAP_CONFIG_DEBUG_MAPS */ + +bool dmabuf_is_nvmap(struct dma_buf *dmabuf); +struct nvmap_handle *nvmap_handle_get_from_id(struct nvmap_client *client, + u32 id); +void *nvmap_dma_mark_declared_memory_occupied(struct device *dev, + dma_addr_t device_addr, size_t size); +void nvmap_dma_mark_declared_memory_unoccupied(struct device *dev, + dma_addr_t device_addr, size_t size); +#endif /* __VIDEO_TEGRA_NVMAP_NVMAP_H */ diff --git a/drivers/video/tegra/nvmap/nvmap_sci_ipc.c b/drivers/video/tegra/nvmap/nvmap_sci_ipc.c new file mode 100644 index 00000000..56f3551f --- /dev/null +++ b/drivers/video/tegra/nvmap/nvmap_sci_ipc.c @@ -0,0 +1,365 @@ +/* + * drivers/video/tegra/nvmap/nvmap_sci_ipc.c + * + * mapping between nvmap_hnadle and sci_ipc entery + * + * Copyright (c) 2019-2022, NVIDIA CORPORATION. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + */ + +#define pr_fmt(fmt) "%s: " fmt, __func__ + +#include +#include +#include +#include +#include + +#include +#include + +#include +#include "nvmap_priv.h" +#include "nvmap_sci_ipc.h" + +struct nvmap_sci_ipc { + struct rb_root entries; + struct mutex mlock; + struct list_head free_sid_list; +}; + +struct free_sid_node { + struct list_head list; + u32 sid; +}; + +/* An rb-tree root node for holding sci_ipc_id of clients */ +struct nvmap_sci_ipc_entry { + struct rb_node entry; + struct nvmap_client *client; + struct nvmap_handle *handle; + u32 sci_ipc_id; + u64 peer_vuid; + u32 flags; + u32 refcount; +}; + +static struct nvmap_sci_ipc *nvmapsciipc; + +int nvmap_validate_sci_ipc_params(struct nvmap_client *client, + NvSciIpcEndpointAuthToken auth_token, + NvSciIpcEndpointVuid *pr_vuid, + NvSciIpcEndpointVuid *lu_vuid) +{ + NvSciError err = NvSciError_Success; + NvSciIpcTopoId pr_topoid; + int ret = 0; + + err = NvSciIpcEndpointValidateAuthTokenLinuxCurrent(auth_token, + lu_vuid); + if (err != NvSciError_Success) { + ret = -EINVAL; + goto out; + } + + err = NvSciIpcEndpointMapVuid(*lu_vuid, &pr_topoid, pr_vuid); + if (err != NvSciError_Success) { + ret = -EINVAL; + goto out; + } + +out: + return ret; +} + +static u32 nvmap_unique_sci_ipc_id(void) +{ + static atomic_t unq_id = { 0 }; + u32 id; + + if (!list_empty(&nvmapsciipc->free_sid_list)) { + struct free_sid_node *fnode = list_first_entry( + &nvmapsciipc->free_sid_list, + typeof(*fnode), + list); + + id = fnode->sid; + list_del(&fnode->list); + kfree(fnode); + goto ret_id; + } + + id = atomic_add_return(2, &unq_id); + +ret_id: + WARN_ON(id == 0); + return id; +} + +static struct nvmap_sci_ipc_entry *nvmap_search_sci_ipc_entry( + struct rb_root *root, + struct nvmap_handle *h, + u32 flags, + NvSciIpcEndpointVuid peer_vuid) +{ + struct rb_node *node; /* top of the tree */ + struct nvmap_sci_ipc_entry *entry; + + for (node = rb_first(root); node; node = rb_next(node)) { + entry = rb_entry(node, struct nvmap_sci_ipc_entry, entry); + + if (entry && entry->handle == h + && entry->flags == flags + && entry->peer_vuid == peer_vuid) + return entry; + } + return NULL; +} + +static void nvmap_insert_sci_ipc_entry(struct rb_root *root, + struct nvmap_sci_ipc_entry *new) +{ + struct nvmap_sci_ipc_entry *entry; + struct rb_node *parent = NULL; + u32 sid = new->sci_ipc_id; + struct rb_node **link; + + link = &root->rb_node; + /* Go to the bottom of the tree */ + while (*link) { + parent = *link; + entry = rb_entry(parent, struct nvmap_sci_ipc_entry, entry); + + if (entry->sci_ipc_id > sid) + link = &parent->rb_left; + else + link = &parent->rb_right; + } + + /* Put the new node there */ + rb_link_node(&new->entry, parent, link); + rb_insert_color(&new->entry, root); +} + +int nvmap_create_sci_ipc_id(struct nvmap_client *client, + struct nvmap_handle *h, + u32 flags, + u32 *sci_ipc_id, + NvSciIpcEndpointVuid peer_vuid, + bool is_ro) +{ + struct nvmap_sci_ipc_entry *new_entry; + struct nvmap_sci_ipc_entry *entry; + int ret = -EINVAL; + u32 id; + + mutex_lock(&nvmapsciipc->mlock); + + entry = nvmap_search_sci_ipc_entry(&nvmapsciipc->entries, + h, flags, peer_vuid); + if (entry) { + entry->refcount++; + *sci_ipc_id = entry->sci_ipc_id; + pr_debug("%d: matched Sci_Ipc_Id:%u\n", __LINE__, *sci_ipc_id); + ret = 0; + goto unlock; + } else { + new_entry = kzalloc(sizeof(*new_entry), GFP_KERNEL); + if (!new_entry) { + ret = -ENOMEM; + goto unlock; + } + id = nvmap_unique_sci_ipc_id(); + *sci_ipc_id = id; + new_entry->sci_ipc_id = id; + new_entry->client = client; + new_entry->handle = h; + new_entry->peer_vuid = peer_vuid; + new_entry->flags = flags; + new_entry->refcount = 1; + + pr_debug("%d: New Sci_ipc_id %d entry->pr_vuid: %llu entry->flags: %u new_entry->handle:0x%p\n", + __LINE__, new_entry->sci_ipc_id, new_entry->peer_vuid, + new_entry->flags, new_entry->handle); + + nvmap_insert_sci_ipc_entry(&nvmapsciipc->entries, new_entry); + ret = 0; + } +unlock: + mutex_unlock(&nvmapsciipc->mlock); + if (!ret) + (void)nvmap_handle_get(h); + + return ret; +} + +static struct nvmap_sci_ipc_entry *nvmap_find_entry_for_id(struct rb_root *es, + u32 id) +{ + struct nvmap_sci_ipc_entry *e = NULL; + struct rb_node *n; + + for (n = rb_first(es); n; n = rb_next(n)) { + e = rb_entry(n, struct nvmap_sci_ipc_entry, entry); + if (e && e->sci_ipc_id == id) + goto found; + } +found: + return e; +} + +int nvmap_get_handle_from_sci_ipc_id(struct nvmap_client *client, u32 flags, + u32 sci_ipc_id, NvSciIpcEndpointVuid localu_vuid, u32 *handle) +{ + bool is_ro = (flags == PROT_READ) ? true : false, dmabuf_created = false; + struct nvmap_handle_ref *ref = NULL; + struct nvmap_sci_ipc_entry *entry; + struct dma_buf *dmabuf = NULL; + struct nvmap_handle *h; + int ret = 0; + int fd; + + mutex_lock(&nvmapsciipc->mlock); + + pr_debug("%d: Sci_Ipc_Id %d local_vuid: %llu flags: %u\n", + __LINE__, sci_ipc_id, localu_vuid, flags); + + entry = nvmap_find_entry_for_id(&nvmapsciipc->entries, sci_ipc_id); + if ((entry == NULL) || (entry->handle == NULL) || + (entry->peer_vuid != localu_vuid) || (entry->flags != flags)) { + + pr_debug("%d: No matching Sci_Ipc_Id %d found\n", + __LINE__, sci_ipc_id); + + ret = -EINVAL; + goto unlock; + } + + h = entry->handle; + + if (is_ro && h->dmabuf_ro == NULL) { + h->dmabuf_ro = __nvmap_make_dmabuf(client, h, true); + if (IS_ERR(h->dmabuf_ro)) { + ret = PTR_ERR(h->dmabuf_ro); + goto unlock; + } + dmabuf_created = true; + } + + ref = nvmap_duplicate_handle(client, h, false, is_ro); + if (!ref) { + ret = -EINVAL; + goto unlock; + } + nvmap_handle_put(h); + /* + * When new dmabuf created (only RO dmabuf is getting created in this function) + * it's counter is incremented one extra time in nvmap_duplicate_handle. Hence + * decrement it by one. + */ + if (dmabuf_created) + dma_buf_put(h->dmabuf_ro); + if (!IS_ERR(ref)) { + u32 id = 0; + + dmabuf = is_ro ? h->dmabuf_ro : h->dmabuf; + if (client->ida) { + if (nvmap_id_array_id_alloc(client->ida, &id, dmabuf) < 0) { + if (dmabuf) + dma_buf_put(dmabuf); + nvmap_free_handle(client, h, is_ro); + ret = -ENOMEM; + goto unlock; + } + if (!id) + *handle = 0; + else + *handle = id; + } else { + fd = nvmap_get_dmabuf_fd(client, h, is_ro); + if (IS_ERR_VALUE((uintptr_t)fd)) { + if (dmabuf) + dma_buf_put(dmabuf); + nvmap_free_handle(client, h, is_ro); + ret = -EINVAL; + goto unlock; + } + *handle = fd; + fd_install(fd, dmabuf->file); + } + } + entry->refcount--; + if (entry->refcount == 0U) { + struct free_sid_node *free_node; + + rb_erase(&entry->entry, &nvmapsciipc->entries); + free_node = kzalloc(sizeof(*free_node), GFP_KERNEL); + if (free_node == NULL) { + ret = -ENOMEM; + kfree(entry); + goto unlock; + } + free_node->sid = entry->sci_ipc_id; + list_add_tail(&free_node->list, &nvmapsciipc->free_sid_list); + kfree(entry); + } +unlock: + mutex_unlock(&nvmapsciipc->mlock); + + if (!ret) { + if (!client->ida) + trace_refcount_create_handle_from_sci_ipc_id(h, dmabuf, + atomic_read(&h->ref), + atomic_long_read(&dmabuf->file->f_count), + is_ro ? "RO" : "RW"); + else + trace_refcount_get_handle_from_sci_ipc_id(h, dmabuf, + atomic_read(&h->ref), + is_ro ? "RO" : "RW"); + } + + return ret; +} + +int nvmap_sci_ipc_init(void) +{ + nvmapsciipc = kzalloc(sizeof(*nvmapsciipc), GFP_KERNEL); + if (!nvmapsciipc) + return -ENOMEM; + nvmapsciipc->entries = RB_ROOT; + INIT_LIST_HEAD(&nvmapsciipc->free_sid_list); + mutex_init(&nvmapsciipc->mlock); + + return 0; +} + +void nvmap_sci_ipc_exit(void) +{ + struct nvmap_sci_ipc_entry *e; + struct free_sid_node *fnode, *temp; + struct rb_node *n; + + mutex_lock(&nvmapsciipc->mlock); + while ((n = rb_first(&nvmapsciipc->entries))) { + e = rb_entry(n, struct nvmap_sci_ipc_entry, entry); + rb_erase(&e->entry, &nvmapsciipc->entries); + kfree(e); + } + + list_for_each_entry_safe(fnode, temp, &nvmapsciipc->free_sid_list, list) { + list_del(&fnode->list); + kfree(fnode); + } + + mutex_unlock(&nvmapsciipc->mlock); + kfree(nvmapsciipc); + nvmapsciipc = NULL; +} diff --git a/drivers/video/tegra/nvmap/nvmap_sci_ipc.h b/drivers/video/tegra/nvmap/nvmap_sci_ipc.h new file mode 100644 index 00000000..398404c0 --- /dev/null +++ b/drivers/video/tegra/nvmap/nvmap_sci_ipc.h @@ -0,0 +1,37 @@ +/* + * drivers/video/tegra/nvmap/nvmap_sci_ipc.c + * + * mapping between nvmap_hnadle and sci_ipc entery + * + * Copyright (c) 2019-2021, NVIDIA CORPORATION. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + */ +#ifndef __VIDEO_TEGRA_NVMAP_SCI_IPC_H +#define __VIDEO_TEGRA_NVMAP_SCI_IPC_H + +int nvmap_validate_sci_ipc_params(struct nvmap_client *client, + NvSciIpcEndpointAuthToken auth_token, + NvSciIpcEndpointVuid *pr_vuid, + NvSciIpcEndpointVuid *localusr_vuid); + +int nvmap_create_sci_ipc_id(struct nvmap_client *client, + struct nvmap_handle *h, + u32 flags, + u32 *sci_ipc_id, + NvSciIpcEndpointVuid pr_vuid, + bool is_ro); + +int nvmap_get_handle_from_sci_ipc_id(struct nvmap_client *client, + u32 flags, + u32 sci_ipc_id, + NvSciIpcEndpointVuid localusr_vuid, + u32 *h); +#endif /* __VIDEO_TEGRA_NVMAP_SCI_IPC_H */ diff --git a/drivers/video/tegra/nvmap/nvmap_stats.c b/drivers/video/tegra/nvmap/nvmap_stats.c new file mode 100644 index 00000000..7be8aeef --- /dev/null +++ b/drivers/video/tegra/nvmap/nvmap_stats.c @@ -0,0 +1,106 @@ +/* + * drivers/video/tegra/nvmap/nvmap_stats.c + * + * Nvmap Stats keeping + * + * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + */ + +#include + +#include "nvmap_priv.h" + +struct nvmap_stats nvmap_stats; + +static int nvmap_stats_reset(void *data, u64 val) +{ + int i; + + if (val) { + atomic64_set(&nvmap_stats.collect, 0); + for (i = 0; i < NS_NUM; i++) { + if (i == NS_TOTAL) + continue; + atomic64_set(&nvmap_stats.stats[i], 0); + } + } + return 0; +} + +static int nvmap_stats_get(void *data, u64 *val) +{ + atomic64_t *ptr = data; + + *val = atomic64_read(ptr); + return 0; +} + +static int nvmap_stats_set(void *data, u64 val) +{ + atomic64_t *ptr = data; + + atomic64_set(ptr, val); + return 0; +} + +DEFINE_SIMPLE_ATTRIBUTE(reset_stats_fops, NULL, nvmap_stats_reset, "%llu\n"); +DEFINE_SIMPLE_ATTRIBUTE(stats_fops, nvmap_stats_get, nvmap_stats_set, "%llu\n"); + +void nvmap_stats_init(struct dentry *nvmap_debug_root) +{ + struct dentry *stats_root; + +#define CREATE_DF(x, y) \ + debugfs_create_file(#x, S_IRUGO, stats_root, &y, &stats_fops); + + stats_root = debugfs_create_dir("stats", nvmap_debug_root); + if (!IS_ERR_OR_NULL(stats_root)) { + CREATE_DF(alloc, nvmap_stats.stats[NS_ALLOC]); + CREATE_DF(release, nvmap_stats.stats[NS_RELEASE]); + CREATE_DF(ualloc, nvmap_stats.stats[NS_UALLOC]); + CREATE_DF(urelease, nvmap_stats.stats[NS_URELEASE]); + CREATE_DF(kalloc, nvmap_stats.stats[NS_KALLOC]); + CREATE_DF(krelease, nvmap_stats.stats[NS_KRELEASE]); + CREATE_DF(cflush_rq, nvmap_stats.stats[NS_CFLUSH_RQ]); + CREATE_DF(cflush_done, nvmap_stats.stats[NS_CFLUSH_DONE]); + CREATE_DF(ucflush_rq, nvmap_stats.stats[NS_UCFLUSH_RQ]); + CREATE_DF(ucflush_done, nvmap_stats.stats[NS_UCFLUSH_DONE]); + CREATE_DF(kcflush_rq, nvmap_stats.stats[NS_KCFLUSH_RQ]); + CREATE_DF(kcflush_done, nvmap_stats.stats[NS_KCFLUSH_DONE]); + CREATE_DF(total_memory, nvmap_stats.stats[NS_TOTAL]); + + debugfs_create_file("collect", S_IRUGO | S_IWUSR, + stats_root, &nvmap_stats.collect, &stats_fops); + debugfs_create_file("reset", S_IWUSR, + stats_root, NULL, &reset_stats_fops); + } + +#undef CREATE_DF +} + +void nvmap_stats_inc(enum nvmap_stats_t stat, size_t size) +{ + if (atomic64_read(&nvmap_stats.collect) || stat == NS_TOTAL) + atomic64_add(size, &nvmap_stats.stats[stat]); +} + +void nvmap_stats_dec(enum nvmap_stats_t stat, size_t size) +{ + if (atomic64_read(&nvmap_stats.collect) || stat == NS_TOTAL) + atomic64_sub(size, &nvmap_stats.stats[stat]); +} + +u64 nvmap_stats_read(enum nvmap_stats_t stat) +{ + return atomic64_read(&nvmap_stats.stats[stat]); +} + diff --git a/drivers/video/tegra/nvmap/nvmap_stats.h b/drivers/video/tegra/nvmap/nvmap_stats.h new file mode 100644 index 00000000..3e2d3e80 --- /dev/null +++ b/drivers/video/tegra/nvmap/nvmap_stats.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + */ + +#ifndef __VIDEO_TEGRA_NVMAP_STATS_H +#define __VIDEO_TEGRA_NVMAP_STATS_H + +enum nvmap_stats_t { + NS_ALLOC = 0, + NS_RELEASE, + NS_UALLOC, + NS_URELEASE, + NS_KALLOC, + NS_KRELEASE, + NS_CFLUSH_RQ, + NS_CFLUSH_DONE, + NS_UCFLUSH_RQ, + NS_UCFLUSH_DONE, + NS_KCFLUSH_RQ, + NS_KCFLUSH_DONE, + NS_TOTAL, + NS_NUM, +}; + +struct nvmap_stats { + atomic64_t stats[NS_NUM]; + atomic64_t collect; +}; + +extern struct nvmap_stats nvmap_stats; + +void nvmap_stats_init(struct dentry *nvmap_debug_root); +void nvmap_stats_inc(enum nvmap_stats_t, size_t size); +void nvmap_stats_dec(enum nvmap_stats_t, size_t size); +u64 nvmap_stats_read(enum nvmap_stats_t); +#endif /* __VIDEO_TEGRA_NVMAP_STATS_H */ diff --git a/drivers/video/tegra/nvmap/nvmap_tag.c b/drivers/video/tegra/nvmap/nvmap_tag.c new file mode 100644 index 00000000..0efd8ecc --- /dev/null +++ b/drivers/video/tegra/nvmap/nvmap_tag.c @@ -0,0 +1,113 @@ +/* + * drivers/video/tegra/nvmap/nvmap_tag.c + * + * Allocation tag routines for nvmap + * + * Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + */ + +#define pr_fmt(fmt) "%s: " fmt, __func__ + +#include + +#include + +#include "nvmap_priv.h" + +struct nvmap_tag_entry *nvmap_search_tag_entry(struct rb_root *root, u32 tag) +{ + struct rb_node *node = root->rb_node; /* top of the tree */ + struct nvmap_tag_entry *entry; + + while (node) { + entry = rb_entry(node, struct nvmap_tag_entry, node); + + if (entry->tag > tag) + node = node->rb_left; + else if (entry->tag < tag) + node = node->rb_right; + else + return entry; /* Found it */ + } + return NULL; +} + + +static void nvmap_insert_tag_entry(struct rb_root *root, + struct nvmap_tag_entry *new) +{ + struct rb_node **link = &root->rb_node; + struct rb_node *parent = NULL; + struct nvmap_tag_entry *entry; + u32 tag = new->tag; + + /* Go to the bottom of the tree */ + while (*link) { + parent = *link; + entry = rb_entry(parent, struct nvmap_tag_entry, node); + + if (entry->tag > tag) + link = &parent->rb_left; + else + link = &parent->rb_right; + } + + /* Put the new node there */ + rb_link_node(&new->node, parent, link); + rb_insert_color(&new->node, root); +} + + +int nvmap_define_tag(struct nvmap_device *dev, u32 tag, + const char __user *name, u32 len) +{ + struct nvmap_tag_entry *new; + struct nvmap_tag_entry *old; + + new = kzalloc(sizeof(struct nvmap_tag_entry) + len + 1, GFP_KERNEL); + if (!new) + return -ENOMEM; + + if (copy_from_user(new + 1, name, len)) { + kfree(new); + return -EFAULT; + } + + new->tag = tag; + + mutex_lock(&dev->tags_lock); + old = nvmap_search_tag_entry(&dev->tags, tag); + if (old) { + rb_replace_node(&old->node, &new->node, &dev->tags); + kfree(old); + } else { + nvmap_insert_tag_entry(&dev->tags, new); + } + mutex_unlock(&dev->tags_lock); + + return 0; +} + +int nvmap_remove_tag(struct nvmap_device *dev, u32 tag) +{ + struct nvmap_tag_entry *old; + + mutex_lock(&dev->tags_lock); + old = nvmap_search_tag_entry(&dev->tags, tag); + if (old){ + rb_erase(&old->node, &dev->tags); + kfree(old); + } + mutex_unlock(&dev->tags_lock); + + return 0; +} diff --git a/include/linux/nvmap.h b/include/linux/nvmap.h new file mode 100644 index 00000000..6710b52a --- /dev/null +++ b/include/linux/nvmap.h @@ -0,0 +1,113 @@ +/* + * include/linux/nvmap.h + * + * structure declarations for nvmem and nvmap user-space ioctls + * + * Copyright (c) 2009-2022, NVIDIA CORPORATION. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + */ + +#ifndef _LINUX_NVMAP_H +#define _LINUX_NVMAP_H + +#include +#include +#include +#include +#include +#include + +#define NVMAP_HEAP_IOVMM (1ul<<30) + +/* common carveout heaps */ +#define NVMAP_HEAP_CARVEOUT_VPR (1ul<<28) +#define NVMAP_HEAP_CARVEOUT_TSEC (1ul<<27) +#define NVMAP_HEAP_CARVEOUT_VIDMEM (1ul<<26) +#define NVMAP_HEAP_CARVEOUT_FSI (1ul<<2) +#define NVMAP_HEAP_CARVEOUT_IVM (1ul<<1) +#define NVMAP_HEAP_CARVEOUT_GENERIC (1ul<<0) + +#define NVMAP_HEAP_CARVEOUT_MASK (NVMAP_HEAP_IOVMM - 1) + +/* allocation flags */ +#define NVMAP_HANDLE_UNCACHEABLE (0x0ul << 0) +#define NVMAP_HANDLE_WRITE_COMBINE (0x1ul << 0) +#define NVMAP_HANDLE_INNER_CACHEABLE (0x2ul << 0) +#define NVMAP_HANDLE_CACHEABLE (0x3ul << 0) +#define NVMAP_HANDLE_CACHE_FLAG (0x3ul << 0) + +#define NVMAP_HANDLE_SECURE (0x1ul << 2) +#define NVMAP_HANDLE_KIND_SPECIFIED (0x1ul << 3) +#define NVMAP_HANDLE_COMPR_SPECIFIED (0x1ul << 4) +#define NVMAP_HANDLE_ZEROED_PAGES (0x1ul << 5) +#define NVMAP_HANDLE_PHYS_CONTIG (0x1ul << 6) +#define NVMAP_HANDLE_CACHE_SYNC (0x1ul << 7) +#define NVMAP_HANDLE_CACHE_SYNC_AT_RESERVE (0x1ul << 8) +#define NVMAP_HANDLE_RO (0x1ul << 9) + +#ifdef NVMAP_CONFIG_PAGE_POOLS +ulong nvmap_page_pool_get_unused_pages(void); +#else +static inline ulong nvmap_page_pool_get_unused_pages(void) +{ + return 0; +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 15, 0) +ulong nvmap_iovmm_get_used_pages(void); +#endif + +int nvmap_register_vidmem_carveout(struct device *dma_dev, + phys_addr_t base, size_t size); + +/* + * A heap can be mapped to memory other than DRAM. + * The HW, controls the memory, can be power gated/ungated + * based upon the clients using the memory. + * if no client/alloc happens from the memory, the HW needs + * to be power gated. Similarly it should power ungated if + * alloc happens from the memory. + * int (*busy)(void) - trigger runtime power ungate + * int (*idle)(void) - trigger runtime power gate + */ +struct nvmap_pm_ops { + int (*busy)(void); + int (*idle)(void); +}; + +struct nvmap_platform_carveout { + const char *name; + unsigned int usage_mask; + phys_addr_t base; + size_t size; + struct device *cma_dev; + bool resize; + struct device *dma_dev; + struct device dev; + struct dma_declare_info *dma_info; + bool is_ivm; + unsigned int peer; + unsigned int vmid; + int can_alloc; + bool enable_static_dma_map; + bool disable_dynamic_dma_map; + bool no_cpu_access; /* carveout can't be accessed from cpu at all */ + bool init_done; /* FIXME: remove once all caveouts use reserved-memory */ + struct nvmap_pm_ops pm_ops; +}; + +struct nvmap_platform_data { + const struct nvmap_platform_carveout *carveouts; + unsigned int nr_carveouts; +}; + +#endif /* _LINUX_NVMAP_H */ diff --git a/include/linux/nvmap_t19x.h b/include/linux/nvmap_t19x.h new file mode 100644 index 00000000..7be131dc --- /dev/null +++ b/include/linux/nvmap_t19x.h @@ -0,0 +1,38 @@ +/* + * include/linux/nvmap_t19x.h + * + * structure declarations for nvmem and nvmap user-space ioctls + * + * Copyright (c) 2009-2022, NVIDIA CORPORATION. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + */ + +#include +#include +#include +#include + +#ifndef _LINUX_NVMAP_T19x_H +#define _LINUX_NVMAP_T19x_H + +#define NVMAP_HEAP_CARVEOUT_CVSRAM (1ul<<25) + +int nvmap_register_cvsram_carveout(struct device *dma_dev, + phys_addr_t base, size_t size, + int (*pmops_busy)(void), int (*pmops_idle)(void)); + +struct nvmap_handle_t19x { + atomic_t nc_pin; /* no. of pins from non io coherent devices */ +}; + +extern bool nvmap_version_t19x; + +#endif /* _LINUX_NVMAP_T19x_H */ diff --git a/include/trace/events/nvmap.h b/include/trace/events/nvmap.h new file mode 100644 index 00000000..65a5e548 --- /dev/null +++ b/include/trace/events/nvmap.h @@ -0,0 +1,870 @@ +/* + * include/trace/events/nvmap.h + * + * NvMap event logging to ftrace. + * + * Copyright (c) 2012-2022, NVIDIA CORPORATION. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM nvmap + +#if !defined(_TRACE_NVMAP_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_NVMAP_H + +#include +#include +#include +#include + +struct nvmap_handle; +struct nvmap_handle_ref; +struct nvmap_client; + +DECLARE_EVENT_CLASS(nvmap, + TP_PROTO(struct nvmap_client *client, const char *name), + TP_ARGS(client, name), + TP_STRUCT__entry( + __field(struct nvmap_client *, client) + __string(sname, name) + ), + TP_fast_assign( + __entry->client = client; + __assign_str(sname, name) + ), + TP_printk("client=%p, name=%s", + __entry->client, __get_str(sname)) +); + +DEFINE_EVENT(nvmap, nvmap_open, + TP_PROTO(struct nvmap_client *client, const char *name), + TP_ARGS(client, name) +); + +DEFINE_EVENT(nvmap, nvmap_release, + TP_PROTO(struct nvmap_client *client, const char *name), + TP_ARGS(client, name) +); + +TRACE_EVENT(nvmap_create_handle, + TP_PROTO(struct nvmap_client *client, + const char *name, + struct nvmap_handle *h, + u32 size, + struct nvmap_handle_ref *ref + ), + + TP_ARGS(client, name, h, size, ref), + + TP_STRUCT__entry( + __field(struct nvmap_client *, client) + __string(sname, name) + __field(struct nvmap_handle *, h) + __field(u32, size) + __field(struct nvmap_handle_ref *, ref) + ), + + TP_fast_assign( + __entry->client = client; + __assign_str(sname, name) + __entry->h = h; + __entry->size = size; + __entry->ref = ref; + ), + + TP_printk("client=%p, name=%s, handle=%p, size=%d, ref=%p", + __entry->client, __get_str(sname), + __entry->h, __entry->size, __entry->ref) +); + +TRACE_EVENT(nvmap_alloc_handle, + TP_PROTO(struct nvmap_client *client, + struct nvmap_handle *handle, + size_t size, + u32 heap_mask, + u32 align, + u32 flags, + u64 total, + u64 alloc + ), + + TP_ARGS(client, handle, size, heap_mask, align, flags, total, alloc), + + TP_STRUCT__entry( + __field(struct nvmap_client *, client) + __field(struct nvmap_handle *, handle) + __field(size_t, size) + __field(u32, heap_mask) + __field(u32, align) + __field(u32, flags) + __field(u64, total) + __field(u64, alloc) + ), + + TP_fast_assign( + __entry->client = client; + __entry->handle = handle; + __entry->size = size; + __entry->heap_mask = heap_mask; + __entry->align = align; + __entry->flags = flags; + __entry->total = total; + __entry->alloc = alloc; + ), + + TP_printk("client=%p, id=0x%p, size=%zu, heap_mask=0x%x, align=%d, flags=0x%x, total=%llu, alloc=%llu", + __entry->client, __entry->handle, __entry->size, + __entry->heap_mask, __entry->align, __entry->flags, + (unsigned long long)__entry->total, + (unsigned long long)__entry->alloc) +); + + +DECLARE_EVENT_CLASS(nvmap_handle_summary, + TP_PROTO(struct nvmap_client *client, pid_t pid, u32 dupes, + struct nvmap_handle *handle, u32 share, u64 base, + size_t size, u32 flags, u32 tag, const char *tag_name), + TP_ARGS(client, pid, dupes, handle, share, base, size, flags, tag, tag_name), + + TP_STRUCT__entry( + __field(struct nvmap_client *, client) + __field(pid_t, pid) + __field(u32, dupes) + __field(struct nvmap_handle *, handle) + __field(u32, share) + __field(u64, base) + __field(size_t, size) + __field(u32, flags) + __field(u32, tag) + __string(tag_name, tag_name) + ), + + TP_fast_assign( + __entry->client = client; + __entry->pid = pid; + __entry->dupes = dupes; + __entry->handle = handle; + __entry->share = share; + __entry->base = base; + __entry->size = size; + __entry->flags = flags; + __entry->tag = tag; + __assign_str(tag_name, tag_name) + ), + + TP_printk("client=0x%p pid=%d dupes=%u handle=0x%p share=%u base=%llx size=%zu flags=0x%x tag=0x%x %s", + __entry->client, + __entry->pid, + __entry->dupes, + __entry->handle, + __entry->share, + __entry->base, + __entry->size, + __entry->flags, + __entry->tag, + __get_str(tag_name) + ) +); + +DEFINE_EVENT(nvmap_handle_summary, + nvmap_alloc_handle_done, + TP_PROTO(struct nvmap_client *client, pid_t pid, u32 dupes, + struct nvmap_handle *handle, u32 share, u64 base, + size_t size, u32 flags, u32 tag, const char *tag_name), + TP_ARGS(client, pid, dupes, handle, share, base, size, flags, tag, tag_name) +); + + +DEFINE_EVENT(nvmap_handle_summary, + nvmap_duplicate_handle, + TP_PROTO(struct nvmap_client *client, pid_t pid, u32 dupes, + struct nvmap_handle *handle, u32 share, u64 base, + size_t size, u32 flags, u32 tag, const char *tag_name), + TP_ARGS(client, pid, dupes, handle, share, base, size, flags, tag, tag_name) +); + +DEFINE_EVENT(nvmap_handle_summary, + nvmap_free_handle, + TP_PROTO(struct nvmap_client *client, pid_t pid, u32 dupes, + struct nvmap_handle *handle, u32 share, u64 base, + size_t size, u32 flags, u32 tag, const char *tag_name), + TP_ARGS(client, pid, dupes, handle, share, base, size, flags, tag, tag_name) +); + +DEFINE_EVENT(nvmap_handle_summary, + nvmap_destroy_handle, + TP_PROTO(struct nvmap_client *client, pid_t pid, u32 dupes, + struct nvmap_handle *handle, u32 share, u64 base, + size_t size, u32 flags, u32 tag, const char *tag_name), + TP_ARGS(client, pid, dupes, handle, share, base, size, flags, tag, tag_name) +); + +TRACE_EVENT(nvmap_cache_maint, + TP_PROTO(struct nvmap_client *client, + struct nvmap_handle *h, + ulong start, + ulong end, + u32 op, + size_t size + ), + + TP_ARGS(client, h, start, end, op, size), + + TP_STRUCT__entry( + __field(struct nvmap_client *, client) + __field(struct nvmap_handle *, h) + __field(ulong, start) + __field(ulong, end) + __field(u32, op) + __field(size_t, size) + ), + + TP_fast_assign( + __entry->client = client; + __entry->h = h; + __entry->start = start; + __entry->end = end; + __entry->op = op; + __entry->size = size; + ), + + TP_printk("client=%p, h=%p, start=0x%lx, end=0x%lx, op=%d, size=%zu", + __entry->client, __entry->h, __entry->start, + __entry->end, __entry->op, __entry->size) +); + +TRACE_EVENT(nvmap_cache_flush, + TP_PROTO(size_t size, + u64 alloc_rq, + u64 total_rq, + u64 total_done + ), + + TP_ARGS(size, alloc_rq, total_rq, total_done), + + TP_STRUCT__entry( + __field(size_t, size) + __field(u64, alloc_rq) + __field(u64, total_rq) + __field(u64, total_done) + ), + + TP_fast_assign( + __entry->size = size; + __entry->alloc_rq = alloc_rq; + __entry->total_rq = total_rq; + __entry->total_done = total_done; + ), + + TP_printk("size=%zu, alloc_rq=%llu, total_rq=%llu, total_done=%llu", + __entry->size, + (unsigned long long)__entry->alloc_rq, + (unsigned long long)__entry->total_rq, + (unsigned long long)__entry->total_done) +); + +TRACE_EVENT(nvmap_map_into_caller_ptr, + TP_PROTO(struct nvmap_client *client, + struct nvmap_handle *h, + u32 offset, + u32 length, + u32 flags + ), + + TP_ARGS(client, h, offset, length, flags), + + TP_STRUCT__entry( + __field(struct nvmap_client *, client) + __field(struct nvmap_handle *, h) + __field(u32, offset) + __field(u32, length) + __field(u32, flags) + ), + + TP_fast_assign( + __entry->client = client; + __entry->h = h; + __entry->offset = offset; + __entry->length = length; + __entry->flags = flags; + ), + + TP_printk("client=%p, h=%p, offset=%d, length=%d, flags=0x%x", + __entry->client, __entry->h, __entry->offset, + __entry->length, __entry->flags) +); + +TRACE_EVENT(nvmap_ioctl_rw_handle, + TP_PROTO(struct nvmap_client *client, + struct nvmap_handle *h, + u32 is_read, + u32 offset, + unsigned long addr, + u32 mem_stride, + u32 user_stride, + u32 elem_size, + u32 count + ), + + TP_ARGS(client, h, is_read, offset, addr, mem_stride, + user_stride, elem_size, count), + + TP_STRUCT__entry( + __field(struct nvmap_client *, client) + __field(struct nvmap_handle *, h) + __field(u32, is_read) + __field(u32, offset) + __field(unsigned long, addr) + __field(u32, mem_stride) + __field(u32, user_stride) + __field(u32, elem_size) + __field(u32, count) + ), + + TP_fast_assign( + __entry->client = client; + __entry->h = h; + __entry->is_read = is_read; + __entry->offset = offset; + __entry->addr = addr; + __entry->mem_stride = mem_stride; + __entry->user_stride = user_stride; + __entry->elem_size = elem_size; + __entry->count = count; + ), + + TP_printk("client=%p, h=%p, is_read=%d, offset=%d, addr=0x%lx," + "mem_stride=%d, user_stride=%d, elem_size=%d, count=%d", + __entry->client, __entry->h, __entry->is_read, __entry->offset, + __entry->addr, __entry->mem_stride, __entry->user_stride, + __entry->elem_size, __entry->count) +); + +TRACE_EVENT(nvmap_ioctl_pinop, + TP_PROTO(struct nvmap_client *client, + u32 is_pin, + u32 count, + struct nvmap_handle **ids + ), + + TP_ARGS(client, is_pin, count, ids), + + TP_STRUCT__entry( + __field(struct nvmap_client *, client) + __field(u32, is_pin) + __field(u32, count) + __field(struct nvmap_handle **, ids) + __dynamic_array(struct nvmap_handle *, ids, count) + ), + + TP_fast_assign( + __entry->client = client; + __entry->is_pin = is_pin; + __entry->count = count; + __entry->ids = ids; + memcpy(__get_dynamic_array(ids), ids, + sizeof(struct nvmap_handle *) * count); + ), + + TP_printk("client=%p, is_pin=%d, count=%d, ids=[%s]", + __entry->client, __entry->is_pin, __entry->count, + __print_hex(__get_dynamic_array(ids), __entry->ids ? + sizeof(struct nvmap_handle *) * __entry->count : 0)) +); + +DECLARE_EVENT_CLASS(handle_get_put, + TP_PROTO(struct nvmap_handle *handle, u32 ref_count), + + TP_ARGS(handle, ref_count), + + TP_STRUCT__entry( + __field(struct nvmap_handle *, handle) + __field(u32, ref_count) + ), + + TP_fast_assign( + __entry->handle = handle; + __entry->ref_count = ref_count; + ), + + TP_printk("ref=%u handle=%p", + __entry->ref_count, + __entry->handle + ) +); + +DEFINE_EVENT(handle_get_put, nvmap_handle_get, + TP_PROTO(struct nvmap_handle *handle, u32 ref_count), + TP_ARGS(handle, ref_count) +); + +DEFINE_EVENT(handle_get_put, nvmap_handle_put, + TP_PROTO(struct nvmap_handle *handle, u32 ref_count), + TP_ARGS(handle, ref_count) +); + +DECLARE_EVENT_CLASS(pin_unpin, + TP_PROTO(struct nvmap_client *client, + const char *name, + struct nvmap_handle *h, + u32 pin_count + ), + + TP_ARGS(client, name, h, pin_count), + + TP_STRUCT__entry( + __field(struct nvmap_client *, client) + __string(sname, name) + __field(struct nvmap_handle *, h) + __field(u32, pin_count) + ), + + TP_fast_assign( + __entry->client = client; + __assign_str(sname, name) + __entry->h = h; + __entry->pin_count = pin_count; + ), + + TP_printk("client=%p, name=%s, h=%p, pin_count=%d", + __entry->client, __get_str(sname), + __entry->h, __entry->pin_count) +); + +DEFINE_EVENT(pin_unpin, nvmap_pin, + TP_PROTO(struct nvmap_client *client, + const char *name, + struct nvmap_handle *h, + u32 pin_count + ), + TP_ARGS(client, name, h, pin_count) +); + +DEFINE_EVENT(pin_unpin, nvmap_unpin, + TP_PROTO(struct nvmap_client *client, + const char *name, + struct nvmap_handle *h, + u32 pin_count + ), + TP_ARGS(client, name, h, pin_count) +); + +DEFINE_EVENT(pin_unpin, handle_unpin_error, + TP_PROTO(struct nvmap_client *client, + const char *name, + struct nvmap_handle *h, + u32 pin_count + ), + TP_ARGS(client, name, h, pin_count) +); + +/* + * Nvmap dmabuf events + */ +DECLARE_EVENT_CLASS(nvmap_dmabuf_2, + TP_PROTO(struct dma_buf *dbuf, + struct device *dev + ), + + TP_ARGS(dbuf, dev), + + TP_STRUCT__entry( + __field(struct dma_buf *, dbuf) + __string(name, dev_name(dev)) + ), + + TP_fast_assign( + __entry->dbuf = dbuf; + __assign_str(name, dev_name(dev)); + ), + + TP_printk("dmabuf=%p, device=%s", + __entry->dbuf, __get_str(name) + ) +); + +DECLARE_EVENT_CLASS(nvmap_dmabuf_1, + TP_PROTO(struct dma_buf *dbuf), + + TP_ARGS(dbuf), + + TP_STRUCT__entry( + __field(struct dma_buf *, dbuf) + ), + + TP_fast_assign( + __entry->dbuf = dbuf; + ), + + TP_printk("dmabuf=%p", + __entry->dbuf + ) +); + +DEFINE_EVENT(nvmap_dmabuf_2, nvmap_dmabuf_attach, + TP_PROTO(struct dma_buf *dbuf, + struct device *dev + ), + TP_ARGS(dbuf, dev) +); + +DEFINE_EVENT(nvmap_dmabuf_2, nvmap_dmabuf_detach, + TP_PROTO(struct dma_buf *dbuf, + struct device *dev + ), + TP_ARGS(dbuf, dev) +); + +DEFINE_EVENT(nvmap_dmabuf_2, nvmap_dmabuf_map_dma_buf, + TP_PROTO(struct dma_buf *dbuf, + struct device *dev + ), + TP_ARGS(dbuf, dev) +); + +DEFINE_EVENT(nvmap_dmabuf_2, nvmap_dmabuf_unmap_dma_buf, + TP_PROTO(struct dma_buf *dbuf, + struct device *dev + ), + TP_ARGS(dbuf, dev) +); + +DEFINE_EVENT(nvmap_dmabuf_1, nvmap_dmabuf_mmap, + TP_PROTO(struct dma_buf *dbuf), + TP_ARGS(dbuf) +); + +DEFINE_EVENT(nvmap_dmabuf_1, nvmap_dmabuf_vmap, + TP_PROTO(struct dma_buf *dbuf), + TP_ARGS(dbuf) +); + +DEFINE_EVENT(nvmap_dmabuf_1, nvmap_dmabuf_vunmap, + TP_PROTO(struct dma_buf *dbuf), + TP_ARGS(dbuf) +); + +DEFINE_EVENT(nvmap_dmabuf_1, nvmap_dmabuf_kmap, + TP_PROTO(struct dma_buf *dbuf), + TP_ARGS(dbuf) +); + +DEFINE_EVENT(nvmap_dmabuf_1, nvmap_dmabuf_kunmap, + TP_PROTO(struct dma_buf *dbuf), + TP_ARGS(dbuf) +); + +DECLARE_EVENT_CLASS(nvmap_dmabuf_cpu_access, + TP_PROTO(struct dma_buf *dbuf, + size_t start, + size_t len), + + TP_ARGS(dbuf, start, len), + + TP_STRUCT__entry( + __field(struct dma_buf *, dbuf) + __field(size_t, start) + __field(size_t, len) + ), + + TP_fast_assign( + __entry->dbuf = dbuf; + __entry->start = start; + __entry->len = len; + ), + + TP_printk("dmabuf=%p, start=%zd len=%zd", + __entry->dbuf, __entry->start, __entry->len + ) +); + +DEFINE_EVENT(nvmap_dmabuf_cpu_access, nvmap_dmabuf_begin_cpu_access, + TP_PROTO(struct dma_buf *dbuf, + size_t start, + size_t len), + TP_ARGS(dbuf, start, len) +); + +DEFINE_EVENT(nvmap_dmabuf_cpu_access, nvmap_dmabuf_end_cpu_access, + TP_PROTO(struct dma_buf *dbuf, + size_t start, + size_t len), + TP_ARGS(dbuf, start, len) +); + +DECLARE_EVENT_CLASS(nvmap_dmabuf_make_release, + TP_PROTO(const char *cli, + struct nvmap_handle *h, + struct dma_buf *dbuf + ), + + TP_ARGS(cli, h, dbuf), + + TP_STRUCT__entry( + __field(const char *, cli) + __field(struct nvmap_handle *, h) + __field(struct dma_buf *, dbuf) + ), + + TP_fast_assign( + __entry->cli = cli; + __entry->h = h; + __entry->dbuf = dbuf; + ), + + TP_printk("cli=%s handle=%p dmabuf=%p", + __entry->cli, __entry->h, __entry->dbuf + ) +); + +DEFINE_EVENT(nvmap_dmabuf_make_release, nvmap_make_dmabuf, + TP_PROTO(const char *cli, + struct nvmap_handle *h, + struct dma_buf *dbuf + ), + TP_ARGS(cli, h, dbuf) +); + +DEFINE_EVENT(nvmap_dmabuf_make_release, nvmap_dmabuf_release, + TP_PROTO(const char *cli, + struct nvmap_handle *h, + struct dma_buf *dbuf + ), + TP_ARGS(cli, h, dbuf) +); + +TRACE_EVENT(pp_clean_cache, + TP_PROTO(u32 dirty_pages, + size_t cache_maint_th, + int cache_maint_by_set_ways + ), + + TP_ARGS(dirty_pages, cache_maint_th, cache_maint_by_set_ways), + + TP_STRUCT__entry( + __field(u32, dirty_pages) + __field(size_t, cache_maint_th) + __field(int, cache_maint_by_set_ways) + ), + + TP_fast_assign( + __entry->dirty_pages = dirty_pages; + __entry->cache_maint_th = cache_maint_th >> PAGE_SHIFT; + __entry->cache_maint_by_set_ways = cache_maint_by_set_ways; + ), + + TP_printk("dirty_pages=%u, cache_maint_th=%zu, cache_maint_by_set_ways=%d", + __entry->dirty_pages, __entry->cache_maint_th, + __entry->cache_maint_by_set_ways) +); + +DECLARE_EVENT_CLASS(nvmap_get_list_page, + TP_PROTO(u32 count), + + TP_ARGS(count), + + TP_STRUCT__entry( + __field(u32, count) + ), + + TP_fast_assign( + __entry->count = count; + ), + + TP_printk("pages left in list=%u", __entry->count) +); + +DEFINE_EVENT(nvmap_get_list_page, get_zero_list_page, + TP_PROTO(u32 count), + TP_ARGS(count) +); + +DEFINE_EVENT(nvmap_get_list_page, get_page_list_page, + TP_PROTO(u32 count), + TP_ARGS(count) +); + +TRACE_EVENT(nvmap_pp_zero_pages, + TP_PROTO(u32 count), + + TP_ARGS(count), + + TP_STRUCT__entry( + __field(u32, count) + ), + + TP_fast_assign( + __entry->count = count; + ), + + TP_printk("no. of pages zeroed=%u", __entry->count) +); + +TRACE_EVENT(nvmap_pp_do_background_zero_pages, + TP_PROTO(u32 inserted, u32 zeroed), + + TP_ARGS(inserted, zeroed), + + TP_STRUCT__entry( + __field(u32, inserted) + __field(u32, zeroed) + ), + + TP_fast_assign( + __entry->inserted = inserted; + __entry->zeroed = zeroed; + ), + + TP_printk("failed to insert %u no. of zeroed pages to page_list", + __entry->zeroed - __entry->inserted) +); + +TRACE_EVENT(nvmap_pp_alloc_locked, + TP_PROTO(int force_alloc), + + TP_ARGS(force_alloc), + + TP_STRUCT__entry( + __field(int, force_alloc) + ), + + TP_fast_assign( + __entry->force_alloc = force_alloc; + ), + + TP_printk("allocated one page with force_alloc:%d", __entry->force_alloc) +); + +TRACE_EVENT(nvmap_pp_alloc_lots, + TP_PROTO(u32 alloced, u32 requested), + + TP_ARGS(alloced, requested), + + TP_STRUCT__entry( + __field(u32, alloced) + __field(u32, requested) + ), + + TP_fast_assign( + __entry->alloced = alloced; + __entry->requested = requested; + ), + + TP_printk("requested:%u alloced:%u\n", + __entry->requested, __entry->alloced) +); + +TRACE_EVENT(nvmap_pp_fill_zero_lots, + TP_PROTO(u32 save_to_zero, + u32 to_zero, + u32 ret, + u32 nr + ), + + TP_ARGS(save_to_zero, to_zero, ret, nr), + + TP_STRUCT__entry( + __field(u32, save_to_zero) + __field(u32, to_zero) + __field(u32, ret) + __field(u32, nr) + ), + + TP_fast_assign( + __entry->save_to_zero = save_to_zero; + __entry->to_zero = to_zero; + __entry->ret = ret; + __entry->nr = nr; + ), + + TP_printk("inserted %u pages to zero list, freed %u pages, did not process %u pages", + __entry->to_zero - __entry->save_to_zero, + __entry->ret - (__entry->to_zero - __entry->save_to_zero), + __entry->nr - __entry->ret) +); + +TRACE_EVENT(refcount_get_handle_from_sci_ipc_id, + TP_PROTO(struct nvmap_handle *handle, + struct dma_buf *dmabuf, + int handle_ref, const char *perm + ), + + TP_ARGS(handle, dmabuf, handle_ref, perm), + + TP_STRUCT__entry( + __field(struct nvmap_handle *, handle) + __field(struct dma_buf *, dmabuf) + __field(int, handle_ref) + __string(acc_perm, perm) + ), + + TP_fast_assign( + __entry->handle = handle; + __entry->dmabuf = dmabuf; + __entry->handle_ref = handle_ref; + __assign_str(acc_perm, perm); + ), + + TP_printk("handle=0x%p, dmabuf=0x%p, handle_ref=%d, perm=%s", + __entry->handle, __entry->dmabuf, __entry->handle_ref, + __get_str(acc_perm)) +); + +DECLARE_EVENT_CLASS(nvmap_refcount, + TP_PROTO(struct nvmap_handle *handle, struct dma_buf *dmabuf, \ + int handle_ref, long dmabuf_ref, const char *perm), + + TP_ARGS(handle, dmabuf, handle_ref, dmabuf_ref, perm), + + TP_STRUCT__entry( + __field(struct nvmap_handle *, handle) + __field(struct dma_buf *, dmabuf) + __field(int, handle_ref) + __field(long, dmabuf_ref) + __string(acc_perm, perm) + ), + + TP_fast_assign( + __entry->handle = handle; + __entry->dmabuf = dmabuf; + __entry->handle_ref = handle_ref; + __entry->dmabuf_ref = dmabuf_ref; + __assign_str(acc_perm, perm); + ), + + TP_printk("handle=0x%p, dmabuf=0x%p, handle_ref=%d, dmabuf_ref=%ld, perm=%s", + __entry->handle, __entry->dmabuf, __entry->handle_ref, + __entry->dmabuf_ref, __get_str(acc_perm)) +); + +#define NVMAPREFCOUNTEVENT(ev) DEFINE_EVENT(nvmap_refcount, ev, \ + TP_PROTO(struct nvmap_handle *handle, struct dma_buf *dmabuf, \ + int handle_ref, long dmabuf_ref, const char *perm), \ + TP_ARGS(handle, dmabuf, handle_ref, dmabuf_ref, perm) \ +) + +NVMAPREFCOUNTEVENT(refcount_create_handle); +NVMAPREFCOUNTEVENT(refcount_create_handle_from_va); +NVMAPREFCOUNTEVENT(refcount_create_handle_from_fd); +NVMAPREFCOUNTEVENT(refcount_getfd); +NVMAPREFCOUNTEVENT(refcount_alloc); +NVMAPREFCOUNTEVENT(refcount_get_sci_ipc_id); +NVMAPREFCOUNTEVENT(refcount_create_handle_from_sci_ipc_id); +NVMAPREFCOUNTEVENT(refcount_dup_handle); +NVMAPREFCOUNTEVENT(refcount_free_handle); + +#undef NVMAPREFCOUNTEVENT + +#endif /* _TRACE_NVMAP_H */ + +/* This part must be outside protection */ +#include diff --git a/include/uapi/linux/nvmap.h b/include/uapi/linux/nvmap.h new file mode 100644 index 00000000..e2c6c985 --- /dev/null +++ b/include/uapi/linux/nvmap.h @@ -0,0 +1,366 @@ +/* + * include/uapi/linux/nvmap.h + * + * structure declarations for nvmem and nvmap user-space ioctls + * + * Copyright (c) 2009-2022, NVIDIA CORPORATION. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + */ + +#include +#include + +#ifndef __UAPI_LINUX_NVMAP_H +#define __UAPI_LINUX_NVMAP_H + +/* + * DOC: NvMap Userspace API + * + * create a client by opening /dev/nvmap + * most operations handled via following ioctls + * + */ +enum { + NVMAP_HANDLE_PARAM_SIZE = 1, + NVMAP_HANDLE_PARAM_ALIGNMENT, + NVMAP_HANDLE_PARAM_BASE, + NVMAP_HANDLE_PARAM_HEAP, + NVMAP_HANDLE_PARAM_KIND, + NVMAP_HANDLE_PARAM_COMPR, /* ignored, to be removed */ +}; + +enum { + NVMAP_CACHE_OP_WB = 0, + NVMAP_CACHE_OP_INV, + NVMAP_CACHE_OP_WB_INV, +}; + +enum { + NVMAP_PAGES_UNRESERVE = 0, + NVMAP_PAGES_RESERVE, + NVMAP_INSERT_PAGES_ON_UNRESERVE, + NVMAP_PAGES_PROT_AND_CLEAN, +}; + +#define NVMAP_ELEM_SIZE_U64 (1 << 31) + +struct nvmap_create_handle { + union { + struct { + union { + /* size will be overwritten */ + __u32 size; /* CreateHandle */ + __s32 fd; /* DmaBufFd or FromFd */ + }; + __u32 handle; /* returns nvmap handle */ + }; + struct { + /* one is input parameter, and other is output parameter + * since its a union please note that input parameter + * will be overwritten once ioctl returns + */ + union { + __u64 ivm_id; /* CreateHandle from ivm*/ + __u32 ivm_handle;/* Get ivm_id from handle */ + }; + }; + struct { + union { + /* size64 will be overwritten */ + __u64 size64; /* CreateHandle */ + __u32 handle64; /* returns nvmap handle */ + }; + }; + }; +}; + +struct nvmap_create_handle_from_va { + __u64 va; /* FromVA*/ + __u32 size; /* non-zero for partial memory VMA. zero for end of VMA */ + __u32 flags; /* wb/wc/uc/iwb, tag etc. */ + union { + __u32 handle; /* returns nvmap handle */ + __u64 size64; /* used when size is 0 */ + }; +}; + +struct nvmap_gup_test { + __u64 va; /* FromVA*/ + __u32 handle; /* returns nvmap handle */ + __u32 result; /* result=1 for pass, result=-err for failure */ +}; + +struct nvmap_alloc_handle { + __u32 handle; /* nvmap handle */ + __u32 heap_mask; /* heaps to allocate from */ + __u32 flags; /* wb/wc/uc/iwb etc. */ + __u32 align; /* min alignment necessary */ +}; + +struct nvmap_alloc_ivm_handle { + __u32 handle; /* nvmap handle */ + __u32 heap_mask; /* heaps to allocate from */ + __u32 flags; /* wb/wc/uc/iwb etc. */ + __u32 align; /* min alignment necessary */ + __u32 peer; /* peer with whom handle must be shared. Used + * only for NVMAP_HEAP_CARVEOUT_IVM + */ +}; + +struct nvmap_rw_handle { + __u64 addr; /* user pointer*/ + __u32 handle; /* nvmap handle */ + __u64 offset; /* offset into hmem */ + __u64 elem_size; /* individual atom size */ + __u64 hmem_stride; /* delta in bytes between atoms in hmem */ + __u64 user_stride; /* delta in bytes between atoms in user */ + __u64 count; /* number of atoms to copy */ +}; + +#ifdef __KERNEL__ +#ifdef CONFIG_COMPAT +struct nvmap_rw_handle_32 { + __u32 addr; /* user pointer */ + __u32 handle; /* nvmap handle */ + __u32 offset; /* offset into hmem */ + __u32 elem_size; /* individual atom size */ + __u32 hmem_stride; /* delta in bytes between atoms in hmem */ + __u32 user_stride; /* delta in bytes between atoms in user */ + __u32 count; /* number of atoms to copy */ +}; +#endif /* CONFIG_COMPAT */ +#endif /* __KERNEL__ */ + +struct nvmap_handle_param { + __u32 handle; /* nvmap handle */ + __u32 param; /* size/align/base/heap etc. */ + unsigned long result; /* returns requested info*/ +}; + +#ifdef __KERNEL__ +#ifdef CONFIG_COMPAT +struct nvmap_handle_param_32 { + __u32 handle; /* nvmap handle */ + __u32 param; /* size/align/base/heap etc. */ + __u32 result; /* returns requested info*/ +}; +#endif /* CONFIG_COMPAT */ +#endif /* __KERNEL__ */ + +struct nvmap_cache_op { + unsigned long addr; /* user pointer*/ + __u32 handle; /* nvmap handle */ + __u32 len; /* bytes to flush */ + __s32 op; /* wb/wb_inv/inv */ +}; + +struct nvmap_cache_op_64 { + unsigned long addr; /* user pointer*/ + __u32 handle; /* nvmap handle */ + __u64 len; /* bytes to flush */ + __s32 op; /* wb/wb_inv/inv */ +}; + +#ifdef __KERNEL__ +#ifdef CONFIG_COMPAT +struct nvmap_cache_op_32 { + __u32 addr; /* user pointer*/ + __u32 handle; /* nvmap handle */ + __u32 len; /* bytes to flush */ + __s32 op; /* wb/wb_inv/inv */ +}; +#endif /* CONFIG_COMPAT */ +#endif /* __KERNEL__ */ + +struct nvmap_cache_op_list { + __u64 handles; /* Ptr to u32 type array, holding handles */ + __u64 offsets; /* Ptr to u32 type array, holding offsets + * into handle mem */ + __u64 sizes; /* Ptr to u32 type array, holindg sizes of memory + * regions within each handle */ + __u32 nr; /* Number of handles */ + __s32 op; /* wb/wb_inv/inv */ +}; + +struct nvmap_debugfs_handles_header { + __u8 version; +}; + +struct nvmap_debugfs_handles_entry { + __u64 base; + __u64 size; + __u32 flags; + __u32 share_count; + __u64 mapped_size; +}; + +struct nvmap_set_tag_label { + __u32 tag; + __u32 len; /* in: label length + out: number of characters copied */ + __u64 addr; /* in: pointer to label or NULL to remove */ +}; + +struct nvmap_available_heaps { + __u64 heaps; /* heaps bitmask */ +}; + +struct nvmap_heap_size { + __u32 heap; + __u64 size; +}; + +struct nvmap_sciipc_map { + __u64 auth_token; /* AuthToken */ + __u32 flags; /* Exporter permission flags */ + __u32 sci_ipc_id; /* FromImportId */ + __u32 handle; /* Nvmap handle */ +}; + +struct nvmap_handle_parameters { + __u8 contig; + __u32 import_id; + __u32 handle; + __u32 heap_number; + __u32 access_flags; + __u64 heap; + __u64 align; + __u64 coherency; + __u64 size; + __u64 offset; +}; + +/** + * Struct used while querying heap parameters + */ +struct nvmap_query_heap_params { + __u32 heap_mask; + __u32 flags; + __u8 contig; + __u64 total; + __u64 free; + __u64 largest_free_block; +}; + +/** + * Struct used while duplicating memory handle + */ +struct nvmap_duplicate_handle { + __u32 handle; + __u32 access_flags; + __u32 dup_handle; +}; + +#define NVMAP_IOC_MAGIC 'N' + +/* Creates a new memory handle. On input, the argument is the size of the new + * handle; on return, the argument is the name of the new handle + */ +#define NVMAP_IOC_CREATE _IOWR(NVMAP_IOC_MAGIC, 0, struct nvmap_create_handle) +#define NVMAP_IOC_CREATE_64 \ + _IOWR(NVMAP_IOC_MAGIC, 1, struct nvmap_create_handle) +#define NVMAP_IOC_FROM_ID _IOWR(NVMAP_IOC_MAGIC, 2, struct nvmap_create_handle) + +/* Actually allocates memory for the specified handle */ +#define NVMAP_IOC_ALLOC _IOW(NVMAP_IOC_MAGIC, 3, struct nvmap_alloc_handle) + +/* Frees a memory handle, unpinning any pinned pages and unmapping any mappings + */ +#define NVMAP_IOC_FREE _IO(NVMAP_IOC_MAGIC, 4) + +/* Reads/writes data (possibly strided) from a user-provided buffer into the + * hmem at the specified offset */ +#define NVMAP_IOC_WRITE _IOW(NVMAP_IOC_MAGIC, 6, struct nvmap_rw_handle) +#define NVMAP_IOC_READ _IOW(NVMAP_IOC_MAGIC, 7, struct nvmap_rw_handle) +#ifdef __KERNEL__ +#ifdef CONFIG_COMPAT +#define NVMAP_IOC_WRITE_32 _IOW(NVMAP_IOC_MAGIC, 6, struct nvmap_rw_handle_32) +#define NVMAP_IOC_READ_32 _IOW(NVMAP_IOC_MAGIC, 7, struct nvmap_rw_handle_32) +#endif /* CONFIG_COMPAT */ +#endif /* __KERNEL__ */ + +#define NVMAP_IOC_PARAM _IOWR(NVMAP_IOC_MAGIC, 8, struct nvmap_handle_param) +#ifdef __KERNEL__ +#ifdef CONFIG_COMPAT +#define NVMAP_IOC_PARAM_32 _IOWR(NVMAP_IOC_MAGIC, 8, struct nvmap_handle_param_32) +#endif /* CONFIG_COMPAT */ +#endif /* __KERNEL__ */ + +#define NVMAP_IOC_CACHE _IOW(NVMAP_IOC_MAGIC, 12, struct nvmap_cache_op) +#define NVMAP_IOC_CACHE_64 _IOW(NVMAP_IOC_MAGIC, 12, struct nvmap_cache_op_64) +#ifdef __KERNEL__ +#ifdef CONFIG_COMPAT +#define NVMAP_IOC_CACHE_32 _IOW(NVMAP_IOC_MAGIC, 12, struct nvmap_cache_op_32) +#endif /* CONFIG_COMPAT */ +#endif /* __KERNEL__ */ + +/* Returns a global ID usable to allow a remote process to create a handle + * reference to the same handle */ +#define NVMAP_IOC_GET_ID _IOWR(NVMAP_IOC_MAGIC, 13, struct nvmap_create_handle) + +/* Returns a file id that allows a remote process to create a handle + * reference to the same handle */ +#define NVMAP_IOC_GET_FD _IOWR(NVMAP_IOC_MAGIC, 15, struct nvmap_create_handle) + +/* Create a new memory handle from file id passed */ +#define NVMAP_IOC_FROM_FD _IOWR(NVMAP_IOC_MAGIC, 16, struct nvmap_create_handle) + +/* Perform cache maintenance on a list of handles. */ +#define NVMAP_IOC_CACHE_LIST _IOW(NVMAP_IOC_MAGIC, 17, \ + struct nvmap_cache_op_list) + +#define NVMAP_IOC_FROM_IVC_ID _IOWR(NVMAP_IOC_MAGIC, 19, struct nvmap_create_handle) +#define NVMAP_IOC_GET_IVC_ID _IOWR(NVMAP_IOC_MAGIC, 20, struct nvmap_create_handle) +#define NVMAP_IOC_GET_IVM_HEAPS _IOR(NVMAP_IOC_MAGIC, 21, unsigned int) + +/* Create a new memory handle from VA passed */ +#define NVMAP_IOC_FROM_VA _IOWR(NVMAP_IOC_MAGIC, 22, struct nvmap_create_handle_from_va) + +#define NVMAP_IOC_GUP_TEST _IOWR(NVMAP_IOC_MAGIC, 23, struct nvmap_gup_test) + +/* Define a label for allocation tag */ +#define NVMAP_IOC_SET_TAG_LABEL _IOW(NVMAP_IOC_MAGIC, 24, struct nvmap_set_tag_label) + +#define NVMAP_IOC_GET_AVAILABLE_HEAPS \ + _IOR(NVMAP_IOC_MAGIC, 25, struct nvmap_available_heaps) + +#define NVMAP_IOC_GET_HEAP_SIZE \ + _IOR(NVMAP_IOC_MAGIC, 26, struct nvmap_heap_size) + +#define NVMAP_IOC_PARAMETERS \ + _IOR(NVMAP_IOC_MAGIC, 27, struct nvmap_handle_parameters) + +/* START of T124 IOCTLS */ +/* Actually allocates memory from IVM heaps */ +#define NVMAP_IOC_ALLOC_IVM _IOW(NVMAP_IOC_MAGIC, 101, struct nvmap_alloc_ivm_handle) + +/* Allocate seperate memory for VPR */ +#define NVMAP_IOC_VPR_FLOOR_SIZE _IOW(NVMAP_IOC_MAGIC, 102, __u32) + +/* Get SCI_IPC_ID tied up with nvmap_handle */ +#define NVMAP_IOC_GET_SCIIPCID _IOR(NVMAP_IOC_MAGIC, 103, \ + struct nvmap_sciipc_map) + +/* Get Nvmap handle from SCI_IPC_ID */ +#define NVMAP_IOC_HANDLE_FROM_SCIIPCID _IOR(NVMAP_IOC_MAGIC, 104, \ + struct nvmap_sciipc_map) + +/* Get heap parameters such as total and frre size */ +#define NVMAP_IOC_QUERY_HEAP_PARAMS _IOR(NVMAP_IOC_MAGIC, 105, \ + struct nvmap_query_heap_params) + +/* Duplicate NvRmMemHandle with same/reduced permission */ +#define NVMAP_IOC_DUP_HANDLE _IOWR(NVMAP_IOC_MAGIC, 106, \ + struct nvmap_duplicate_handle) + +#define NVMAP_IOC_MAXNR (_IOC_NR(NVMAP_IOC_DUP_HANDLE)) + +#endif /* __UAPI_LINUX_NVMAP_H */