nvmap: Copy drivers and headers from kernel/nvidia

Copy the driver and header sources of the nvmap to
kernel/nvidia-oot from kernel/nvidia as part of
removing the dependency of kernel/nvidia for OOT drivers.

The latest (few) git history of the files copied are
	b7a355916 video: tegra: nvmap: Fix type casting issue
	2128c5433 video: tegra: nvmap: Fix type casting issues
	0cd082559 video: tegra: nvmap: Change peer vm id data type
	4bd7ece67 tegra: nvmap: mark ivm carveout pages occupied
	e86f3630a video: tegra: nvmap: Fix type casting issue
	c43a23e58 video: tegra: nvmap: Fix type casting issue
	ca1dda22e video: tegra: nvmap: Fix type casting issue
	1f567abfe video: tegra: nvmap: Fix wrap up condition
	29db4d31c video: tegra: nvmap: Remove unnecessary debugfs
	fe72f1413 video: tegra: nvmap: Remove get_drv_data() call
	3b0fc79e7 video: tegra: nvmap: Fix coverity defect
	3cc0ce41b video: tegra: nvmap: Fix coverity defect
	6da39e966 video: tegra: nvmap: Fix WARN_ON condition
	a16351ff1 video: tegra: nvmap: Remove dead code
	9993f2d2d video: tegra: nvmap: Update print level
	6066a2077 video: tegra: nvmap: Remove nvmap_debug_lru_allocations_show
	3cdf2b7ba video: tegra: nvmap: Add kernel version check
	716ded4fc video: tegra: nvmap: Initialize the return value
	9b6c1b4ab video: tegra: nvmap: Correct debugfs code
	33e70118b video: tegra: nvmap: Fix Cert-C error handling bug
	7b960ed79 video: tegra: nvmap: Fix Cert-C error handling bug
	945dc1471 video: tegra: nvmap: Fix Cert-C error handling bug
	31e572de2 video: tegra: nvmap: Fix Cert-C error handling bug
	1f25cbf68 video: tegra: nvmap: Fix Cert-C error handling bug
	fa5428107 video: tegra: nvmap: Remove nvmap_handle_get_from_fd
	df73f2208 video: tegra: nvmap: Protect kmap/kunmap code
	9842e7c6a video: tegra: nvmap: Remove t19x dma_buf map/unmap
	06dff1a8d video: tegra: nvmap: Remove unnecessary export symbols
	6f097f86b video: tegra: nvmap: Fix Cert-C error handling bug
	f14171608 video: tegra: nvmap: load nvmap for T23x compatible platforms
	266812814 video: tegra: nvmap: Get rid of NVMAP_CONFIG_KSTABLE_KERNEL
	1b38c0887 nvmap: Don't use NV_BUILD_KERNEL_OPTIONS
	0ab8dc032 video: tegra: nvmap: Reintroduce NVMAP_CONFIG_VPR_RESIZE
	cc8db9797 driver: platform: tegra: Separate out vpr code
	28955d95c video/tegra: nvmap: Enable build as OOT module
	876d1fbb8 video: tegra: nvmap: Remove IS_ENABLED check
	5ea30867a nvmap: Add support to build as module from OOT kernel
	a71ad020e video: tegra: nvmap: Protect tegra_vpr args under config
	e70061cc1 video: tegra: nvmap: Do not export cvnas_dev
	d2a26ff36 video: tegra: nvmap: Include missing header
	692e4f682 video: tegra: nvmap: Update page coloring algo
	2b9dbb911 video: tegra: nvmap: Check for return value
	de8de12b6 video: tegra: nvmap: Enable legacy init support
	65d478158 video: tegra: nvmap: Remove dependency of cvnas
	38bdd6f05 video: tegra: nvmap: Make nvmap as loadable module
	9668e410b video: tegra: nvmap: Enable handle as ID
	11c6cbd23 tegra: nvmap: Fix build for Linux v5.18
	fbd95c3ab linux: nvmap: change ivm_handle to u32
	eb1e2c302 video: tegra: nvmap: Fix NVSCIIPC support
	022689b29 tegra: nvmap: return error if handle as ID enabled but id is fd
	19e5106ed video: tegra: nvmap: Don't treat ivm as reserved mem carveouts

Bug 4038415

Change-Id: I7108aec3b8532fe79c9423c2835744b1213719e8
Signed-off-by: Laxman Dewangan <ldewangan@nvidia.com>

drivers/video/tegra/nvmap/Makefile

@@ -1,9 +1,47 @@
-# SPDX-License-Identifier: GPL-2.0
+GCOV_PROFILE := y
-# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
 
-# NOTE: Do not change or add anything in this makefile.
+subdir-ccflags-y := -Werror
-# The source code and makefile rules are copied from the
+
-# kernel/nvidia/drivers/video/tegra/nvmap. This file is
+include $(srctree.nvidia)/drivers/video/tegra/nvmap/Makefile.memory.configs
-# just place-holder for empty makefile to avoid any build
+
-# issue when copy is not done from command line and building
+ccflags-y += -I$(srctree.nvidia)/include/
-# the tree independent of source copy.
+
+ifeq ($(NVMAP_CONFIG), y)
+# Build NvMap only when NVMAP_CONFIG is set to y
+nvmap-y := nvmap_core.o \
+	nvmap_alloc.o \
+	nvmap_cache.o \
+	nvmap_dev.o \
+	nvmap_dmabuf.o \
+	nvmap_fault.o \
+	nvmap_handle.o \
+	nvmap_heap.o \
+	nvmap_ioctl.o \
+	nvmap_init.o \
+	nvmap_tag.o \
+	nvmap_mm.o \
+	nvmap_stats.o \
+	nvmap_carveout.o \
+	nvmap_kasan_wrapper.o
+nvmap-$(NVMAP_CONFIG_HANDLE_AS_ID) += nvmap_id_array.o
+
+nvmap-$(NVMAP_CONFIG_SCIIPC) += nvmap_sci_ipc.o
+
+ifeq ($(NVMAP_CONFIG_PAGE_POOLS), y)
+nvmap-y += nvmap_pp.o
+endif #NVMAP_CONFIG_PAGE_POOLS
+
+ifneq ($(filter y,$(CONFIG_ARCH_TEGRA_19x_SOC) $(CONFIG_ARCH_TEGRA_194_SOC)),)
+nvmap-y += nvmap_init_t19x.o
+endif #CONFIG_ARCH_TEGRA_19x_SOC or CONFIG_ARCH_TEGRA_194_SOC
+
+KASAN_SANITIZE_nvmap_kasan_wrapper.o := n
+
+ifeq ($(NVMAP_CONFIG_LOADABLE_MODULE), y)
+nvmap-y += nvmap_cache_maint.o
+obj-m += nvmap.o
+else
+# Not a loadable module
+obj-y += nvmap.o
+endif #NVMAP_CONFIG_LOADABLE_MODULE
+endif #NVMAP_CONFIG

drivers/video/tegra/nvmap/Makefile.memory.configs

@@ -0,0 +1,221 @@
+# Copyright (c) 2021-2022, NVIDIA CORPORATION. All rights reserved.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms and conditions of the GNU General Public License,
+# version 2, as published by the Free Software Foundation.
+#
+# This program is distributed in the hope it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+
+# This file consists of 4 sections
+# Section 1: This section is for doing prerequisite check.
+#            It checkes whether the prereq configs are enabled or not.
+#            If not, return error.
+#
+# Section 2: This section declare all configs with the default values
+#            just similar to Kconfig
+#
+# Section 3: This section consists of checks for kernel versions and
+#            actual values of these configs for corresponding kernel
+#            version. This is the place where we can enable/disable or
+#            set values to configs as per kernel version.
+#
+# Section 4: This section creates ccflags based upon the values specified
+#            in the section 2. These ccflags would be actually used in the
+#            source files. This section also takes care of the dependencies
+#            between the configs.
+#
+
+################################################################################
+ifeq ($(CONFIG_ARCH_TEGRA), y)
+# Section 1
+# For dma_buf support CONFIG_DMA_SHARED_BUFFER needs be enabled
+ifneq ($(CONFIG_DMA_SHARED_BUFFER),y)
+$(error CONFIG_DMA_SHARED_BUFFER is not set)
+endif
+
+################################################################################
+# Section 2
+# This config is used to include the memory management driver for the
+# Tegra GPU, multimedia and display subsystems
+NVMAP_CONFIG := y
+
+# Config to reduce the alloction overhead, which is significant
+# for uncached, writecombine and inner cacheable memories as it
+# involves changing page attributes during every allocation per page
+# and flushing cache. Alloc time is reduced by allcoating the pages
+# ahead and keeping them aside. The reserved pages would be released
+# when system is low on memory and acquired back during release of
+# memory.
+NVMAP_CONFIG_PAGE_POOLS := y
+
+# Config to include some debugging info in the page pools. This
+# adds a bit of unnecessary overhead so only enable this is you
+# suspect there is an issue with the nvmap page pools.
+NVMAP_CONFIG_PAGE_POOL_DEBUG := n
+
+# Config for page pool size in pages
+NVMAP_CONFIG_PAGE_POOL_SIZE := 0x0
+
+# Config to enable page coloring
+# Page coloring rearranges the pages allocated based on the color
+# of the page. It can improve memory access performance.
+# The coloring option enable can optionally overallocate a portion of
+# reqeusted allcoation size to improve the probabilty of better
+# page coloring. If unsure, say Y.
+NVMAP_CONFIG_COLOR_PAGES := y
+
+# Config for FD number to start allocation from
+# NvMap handles are represented with FD's in the user processes.
+# To avoid Linux FD usage limitations, NvMap allocates FD starting
+# from this number.
+NVMAP_CONFIG_FD_START := 0x400
+
+# Config for enabling deferred FD recycle
+# A released nvmap handle would release memory and FD. This FD
+# can be reused immediately for subsequent nvmap allocation req in
+# the same process. Any buggy code in client process that continues to
+# use FD of released allocation would continue to use new allocation
+# and can lead to undesired consequences, which can be hard to debug.
+# Enabling this option would defer recycling FD for longer time and
+# allows debugging incorrect FD references by clients by returning errors
+# for the accesses that occur after handle/FD release.
+NVMAP_CONFIG_DEFER_FD_RECYCLE := n
+
+# Config for FD number to start free FD recycle
+# Once last allocated FD reaches this number, allocation of subsequent
+# FD's start from NVMAP_START_FD.
+NVMAP_CONFIG_DEFER_FD_RECYCLE_MAX_FD := 0x8000
+
+# Config for enabling nvmap mapping with SciIpc secure buffer sharing
+# Enable nvmap mapping with SciIpc secure buffer sharing.
+# Supports nvmap ioctls to get Unique SciIpcId and attach
+# it with nvmap_handle.
+# Suppports getting nvmap_handle from SciIpcId passed via ioctl.
+NVMAP_CONFIG_SCIIPC := n
+
+# Config for enabling NvMap as OOT module
+NVMAP_CONFIG_LOADABLE_MODULE := n
+
+# Config for enabling PROCRANK functionality
+NVMAP_CONFIG_PROCRANK := y
+
+# Config for enabling VPR resize functionality
+NVMAP_CONFIG_VPR_RESIZE := n
+
+# Config for enabling few debugfs which would impact the NvMap performance
+# There are few debugfs which would impact NvMap performance.
+# Disable this when perf regression is observed.
+NVMAP_CONFIG_DEBUG_MAPS := n
+
+# This is fallback option to support handle as FD
+# To support handle as ID, set this to n
+# This config is useful to debug issue if its due to handle as ID or FD
+NVMAP_CONFIG_HANDLE_AS_FD := n
+
+# Config for kstable/OOT kernel
+# This is useful when any kstable/OOT specific checks are needed
+NVMAP_CONFIG_UPSTREAM_KERNEL := n
+################################################################################
+# Section 3
+# Enable/Disable configs based upon the kernel version
+# Specify the values which are different from the default values
+
+ifdef CONFIG_TEGRA_VPR
+# For 4.9
+NVMAP_CONFIG_VPR_RESIZE := y
+else
+# For 5.10+
+NVMAP_CONFIG_LOADABLE_MODULE := y
+NVMAP_CONFIG_PROCRANK := n
+ifneq ($(NVMAP_CONFIG_HANDLE_AS_FD),y)
+NVMAP_CONFIG_HANDLE_AS_ID := y
+NVMAP_CONFIG_FD_START := 0x0
+endif
+NVMAP_CONFIG_SCIIPC := y
+# For OOT build
+ifeq ($(CONFIG_TEGRA_OOT_MODULE),m)
+NVMAP_CONFIG_UPSTREAM_KERNEL := y
+NVMAP_CONFIG_COLOR_PAGES := n
+endif
+endif
+################################################################################
+# Section 4
+# This section creates ccflags based upon the values specified
+# in the section 2. These ccflags would be actually used in the
+# source files.
+
+ifeq ($(NVMAP_CONFIG),y)
+# All other flags make sense only when NVMAP_CONFIG is enabled
+ccflags-y += -DNVMAP_CONFIG
+
+ifeq ($(NVMAP_CONFIG_PAGE_POOLS),y)
+ccflags-y += -DNVMAP_CONFIG_PAGE_POOLS
+
+# NVMAP_CONFIG_PAGE_POOL_DEBUG depends upon NVMAP_CONFIG_PAGE_POOLS
+ifeq ($(NVMAP_CONFIG_PAGE_POOL_DEBUG),y)
+ccflags-y += -DNVMAP_CONFIG_PAGE_POOL_DEBUG
+endif #NVMAP_CONFIG_PAGE_POOL_DEBUG
+
+# NVMAP_CONFIG_PAGE_POOL_SIZE depends upon NVMAP_CONFIG_PAGE_POOLS
+ifdef NVMAP_CONFIG_PAGE_POOL_SIZE
+ccflags-y += -DNVMAP_CONFIG_PAGE_POOL_SIZE=${NVMAP_CONFIG_PAGE_POOL_SIZE}
+endif #NVMAP_CONFIG_PAGE_POOL_SIZE
+endif #NVMAP_CONFIG_PAGE_POOLS
+
+# NVMAP_CONFIG_COLOR_PAGES depends upon CONFIG_ARM64_4K_PAGES
+ifeq ($(CONFIG_ARM64_4K_PAGES),y)
+ifeq ($(NVMAP_CONFIG_COLOR_PAGES),y)
+ccflags-y += -DNVMAP_CONFIG_COLOR_PAGES
+endif #CONFIG_ARM64_4K_PAGES
+endif #NVMAP_CONFIG_COLOR_PAGES
+
+ifdef NVMAP_CONFIG_FD_START
+ccflags-y += -DNVMAP_CONFIG_FD_START=${NVMAP_CONFIG_FD_START}
+endif #NVMAP_CONFIG_FD_START
+
+ifeq ($(NVMAP_CONFIG_DEFER_FD_RECYCLE),y)
+ccflags-y += -DNVMAP_CONFIG_DEFER_FD_RECYCLE
+# NVMAP_CONFIG_DEFER_FD_RECYCLE_MAX_FD depends upon CONFIG_NVMAP_DEFER_FD_RECYCLE
+ifdef NVMAP_CONFIG_DEFER_FD_RECYCLE_MAX_FD
+ccflags-y += -DNVMAP_CONFIG_DEFER_FD_RECYCLE_MAX_FD=${NVMAP_CONFIG_DEFER_FD_RECYCLE_MAX_FD}
+endif #NVMAP_CONFIG_DEFER_FD_RECYCLE_MAX_FD
+endif #NVMAP_CONFIG_DEFER_FD_RECYCLE
+
+ifeq ($(NVMAP_CONFIG_SCIIPC),y)
+ccflags-y += -DNVMAP_CONFIG_SCIIPC
+endif #NVMAP_CONFIG_SCIIPC
+
+ifeq ($(NVMAP_CONFIG_VPR_RESIZE),y)
+ccflags-y += -DNVMAP_CONFIG_VPR_RESIZE
+endif #NVMAP_CONFIG_VPR_RESIZE
+
+ifeq ($(NVMAP_CONFIG_LOADABLE_MODULE),y)
+ccflags-y += -DNVMAP_LOADABLE_MODULE
+endif #NVMAP_CONFIG_LOADABLE_MODULE
+
+ifeq ($(NVMAP_CONFIG_UPSTREAM_KERNEL),y)
+ccflags-y += -DNVMAP_UPSTREAM_KERNEL
+endif #NVMAP_CONFIG_UPSTREAM_KERNEL
+
+ifeq ($(NVMAP_CONFIG_PROCRANK),y)
+ccflags-y += -DNVMAP_CONFIG_PROCRANK
+endif #NVMAP_CONFIG_PROCRANK
+
+ifeq ($(NVMAP_CONFIG_DEBUG_MAPS),y)
+ccflags-y += -DNVMAP_CONFIG_DEBUG_MAPS
+endif #NVMAP_CONFIG_DEBUG_MAPS
+
+ifeq ($(NVMAP_CONFIG_HANDLE_AS_ID),y)
+ccflags-y += -DNVMAP_CONFIG_HANDLE_AS_ID
+endif #NVMAP_CONFIG_HANDLE_AS_ID
+
+ifeq ($(CONFIG_TEGRA_CVNAS),y)
+ccflags-y += -DCVNAS_BUILTIN
+endif #CONFIG_TEGRA_CVNAS
+
+endif #NVMAP_CONFIG
+endif #CONFIG_ARCH_TEGRA

drivers/video/tegra/nvmap/include/linux/nvmap_exports.h

@@ -0,0 +1,23 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+#ifndef __NVMAP_EXPORTS_H
+#define __NVMAP_EXPORTS_H
+
+void *nvmap_dma_alloc_attrs(struct device *dev, size_t size,
+			    dma_addr_t *dma_handle, gfp_t flag,
+			    unsigned long attrs);
+void nvmap_dma_free_attrs(struct device *dev, size_t size, void *cpu_addr,
+			  dma_addr_t dma_handle, unsigned long attrs);
+extern struct device tegra_vpr_dev;
+#endif /* __NVMAP_EXPORTS_H */

drivers/video/tegra/nvmap/nvmap_cache.c

@@ -0,0 +1,431 @@
+/*
+ * drivers/video/tegra/nvmap/nvmap_cache.c
+ *
+ * Copyright (c) 2011-2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#define pr_fmt(fmt)	"nvmap: %s() " fmt, __func__
+
+#include <linux/io.h>
+#include <linux/debugfs.h>
+#include <linux/of.h>
+#include <linux/version.h>
+#if KERNEL_VERSION(4, 15, 0) > LINUX_VERSION_CODE
+#include <soc/tegra/chip-id.h>
+#else
+#include <soc/tegra/fuse.h>
+#endif
+
+#include <linux/sys_soc.h>
+#ifdef NVMAP_LOADABLE_MODULE
+__weak struct arm64_ftr_reg arm64_ftr_reg_ctrel0;
+#endif /*NVMAP_LOADABLE_MODULE */
+
+#include <trace/events/nvmap.h>
+
+#include "nvmap_priv.h"
+
+/*
+ * FIXME:
+ *
+ *   __clean_dcache_page() is only available on ARM64 (well, we haven't
+ *   implemented it on ARMv7).
+ */
+void nvmap_clean_cache_page(struct page *page)
+{
+	__clean_dcache_area_poc(page_address(page), PAGE_SIZE);
+}
+
+void nvmap_clean_cache(struct page **pages, int numpages)
+{
+	int i;
+
+	/* Not technically a flush but that's what nvmap knows about. */
+	nvmap_stats_inc(NS_CFLUSH_DONE, numpages << PAGE_SHIFT);
+	trace_nvmap_cache_flush(numpages << PAGE_SHIFT,
+		nvmap_stats_read(NS_ALLOC),
+		nvmap_stats_read(NS_CFLUSH_RQ),
+		nvmap_stats_read(NS_CFLUSH_DONE));
+
+	for (i = 0; i < numpages; i++)
+		nvmap_clean_cache_page(pages[i]);
+}
+
+void inner_cache_maint(unsigned int op, void *vaddr, size_t size)
+{
+	if (op == NVMAP_CACHE_OP_WB_INV)
+		__dma_flush_area(vaddr, size);
+	else if (op == NVMAP_CACHE_OP_INV)
+		__dma_map_area(vaddr, size, DMA_FROM_DEVICE);
+	else
+		__dma_map_area(vaddr, size, DMA_TO_DEVICE);
+}
+
+static void heap_page_cache_maint(
+	struct nvmap_handle *h, unsigned long start, unsigned long end,
+	unsigned int op, bool inner, bool outer, bool clean_only_dirty)
+{
+	/* Don't perform cache maint for RO mapped buffers */
+	if (h->from_va && h->is_ro)
+		return;
+
+	if (h->userflags & NVMAP_HANDLE_CACHE_SYNC) {
+		/*
+		 * zap user VA->PA mappings so that any access to the pages
+		 * will result in a fault and can be marked dirty
+		 */
+		nvmap_handle_mkclean(h, start, end-start);
+		nvmap_zap_handle(h, start, end - start);
+	}
+
+	if (inner) {
+		if (!h->vaddr) {
+			if (__nvmap_mmap(h))
+				__nvmap_munmap(h, h->vaddr);
+			else
+				goto per_page_cache_maint;
+		}
+		/* Fast inner cache maintenance using single mapping */
+		inner_cache_maint(op, h->vaddr + start, end - start);
+		if (!outer)
+			return;
+		/* Skip per-page inner maintenance in loop below */
+		inner = false;
+
+	}
+per_page_cache_maint:
+
+	while (start < end) {
+		struct page *page;
+		phys_addr_t paddr;
+		unsigned long next;
+		unsigned long off;
+		size_t size;
+		int ret;
+
+		page = nvmap_to_page(h->pgalloc.pages[start >> PAGE_SHIFT]);
+		next = min(((start + PAGE_SIZE) & PAGE_MASK), end);
+		off = start & ~PAGE_MASK;
+		size = next - start;
+		paddr = page_to_phys(page) + off;
+
+		ret = nvmap_cache_maint_phys_range(op, paddr, paddr + size,
+				inner, outer);
+		WARN_ON(ret != 0);
+		start = next;
+	}
+}
+
+struct cache_maint_op {
+	phys_addr_t start;
+	phys_addr_t end;
+	unsigned int op;
+	struct nvmap_handle *h;
+	bool inner;
+	bool outer;
+	bool clean_only_dirty;
+};
+
+int nvmap_cache_maint_phys_range(unsigned int op, phys_addr_t pstart,
+		phys_addr_t pend, int inner, int outer)
+{
+	void __iomem *io_addr;
+	phys_addr_t loop;
+
+	if (!inner)
+		goto do_outer;
+
+	loop = pstart;
+	while (loop < pend) {
+		phys_addr_t next = (loop + PAGE_SIZE) & PAGE_MASK;
+		void *base;
+		next = min(next, pend);
+		io_addr = __ioremap(loop, PAGE_SIZE, PG_PROT_KERNEL);
+		if (io_addr == NULL)
+			return -ENOMEM;
+		base = (__force void *)io_addr + (loop & ~PAGE_MASK);
+		inner_cache_maint(op, base, next - loop);
+		iounmap(io_addr);
+		loop = next;
+	}
+
+do_outer:
+	return 0;
+}
+
+static int do_cache_maint(struct cache_maint_op *cache_work)
+{
+	phys_addr_t pstart = cache_work->start;
+	phys_addr_t pend = cache_work->end;
+	int err = 0;
+	struct nvmap_handle *h = cache_work->h;
+	unsigned int op = cache_work->op;
+
+	if (!h || !h->alloc)
+		return -EFAULT;
+
+	wmb();
+	if (h->flags == NVMAP_HANDLE_UNCACHEABLE ||
+	    h->flags == NVMAP_HANDLE_WRITE_COMBINE || pstart == pend)
+		goto out;
+
+	trace_nvmap_cache_maint(h->owner, h, pstart, pend, op, pend - pstart);
+	if (pstart > h->size || pend > h->size) {
+		pr_warn("cache maintenance outside handle\n");
+		err = -EINVAL;
+		goto out;
+	}
+
+	if (h->heap_pgalloc) {
+		heap_page_cache_maint(h, pstart, pend, op, true,
+			(h->flags == NVMAP_HANDLE_INNER_CACHEABLE) ?
+			false : true, cache_work->clean_only_dirty);
+		goto out;
+	}
+
+	pstart += h->carveout->base;
+	pend += h->carveout->base;
+
+	err = nvmap_cache_maint_phys_range(op, pstart, pend, true,
+			h->flags != NVMAP_HANDLE_INNER_CACHEABLE);
+
+out:
+	if (!err) {
+		nvmap_stats_inc(NS_CFLUSH_DONE, pend - pstart);
+	}
+
+	trace_nvmap_cache_flush(pend - pstart,
+		nvmap_stats_read(NS_ALLOC),
+		nvmap_stats_read(NS_CFLUSH_RQ),
+		nvmap_stats_read(NS_CFLUSH_DONE));
+
+	return 0;
+}
+
+static void nvmap_handle_get_cacheability(struct nvmap_handle *h,
+		bool *inner, bool *outer)
+{
+	*inner = h->flags == NVMAP_HANDLE_CACHEABLE ||
+		 h->flags == NVMAP_HANDLE_INNER_CACHEABLE;
+	*outer = h->flags == NVMAP_HANDLE_CACHEABLE;
+}
+
+int __nvmap_do_cache_maint(struct nvmap_client *client,
+			struct nvmap_handle *h,
+			unsigned long start, unsigned long end,
+			unsigned int op, bool clean_only_dirty)
+{
+	int err;
+	struct cache_maint_op cache_op;
+
+	h = nvmap_handle_get(h);
+	if (!h)
+		return -EFAULT;
+
+	if ((start >= h->size) || (end > h->size)) {
+		pr_debug("%s start: %ld end: %ld h->size: %zu\n", __func__,
+				start, end, h->size);
+		nvmap_handle_put(h);
+		return -EFAULT;
+	}
+
+	if (!(h->heap_type & nvmap_dev->cpu_access_mask)) {
+		pr_debug("%s heap_type %u access_mask 0x%x\n", __func__,
+				h->heap_type, nvmap_dev->cpu_access_mask);
+		nvmap_handle_put(h);
+		return -EPERM;
+	}
+
+	nvmap_kmaps_inc(h);
+	if (op == NVMAP_CACHE_OP_INV)
+		op = NVMAP_CACHE_OP_WB_INV;
+
+	/* clean only dirty is applicable only for Write Back operation */
+	if (op != NVMAP_CACHE_OP_WB)
+		clean_only_dirty = false;
+
+	cache_op.h = h;
+	cache_op.start = start ? start : 0;
+	cache_op.end = end ? end : h->size;
+	cache_op.op = op;
+	nvmap_handle_get_cacheability(h, &cache_op.inner, &cache_op.outer);
+	cache_op.clean_only_dirty = clean_only_dirty;
+
+	nvmap_stats_inc(NS_CFLUSH_RQ, end - start);
+	err = do_cache_maint(&cache_op);
+	nvmap_kmaps_dec(h);
+	nvmap_handle_put(h);
+	return err;
+}
+
+int __nvmap_cache_maint(struct nvmap_client *client,
+			       struct nvmap_cache_op_64 *op)
+{
+	struct vm_area_struct *vma;
+	struct nvmap_vma_priv *priv;
+	struct nvmap_handle *handle;
+	unsigned long start;
+	unsigned long end;
+	int err = 0;
+
+	if (!op->addr || op->op < NVMAP_CACHE_OP_WB ||
+	    op->op > NVMAP_CACHE_OP_WB_INV)
+		return -EINVAL;
+
+	handle = nvmap_handle_get_from_id(client, op->handle);
+	if (IS_ERR_OR_NULL(handle))
+		return -EINVAL;
+
+	nvmap_acquire_mmap_read_lock(current->mm);
+
+	vma = find_vma(current->active_mm, (unsigned long)op->addr);
+	if (!vma || !is_nvmap_vma(vma) ||
+	    (ulong)op->addr < vma->vm_start ||
+	    (ulong)op->addr >= vma->vm_end ||
+	    op->len > vma->vm_end - (ulong)op->addr) {
+		err = -EADDRNOTAVAIL;
+		goto out;
+	}
+
+	priv = (struct nvmap_vma_priv *)vma->vm_private_data;
+
+	if (priv->handle != handle) {
+		err = -EFAULT;
+		goto out;
+	}
+
+	start = (unsigned long)op->addr - vma->vm_start +
+		(vma->vm_pgoff << PAGE_SHIFT);
+	end = start + op->len;
+
+	err = __nvmap_do_cache_maint(client, priv->handle, start, end, op->op,
+				     false);
+out:
+	nvmap_release_mmap_read_lock(current->mm);
+	nvmap_handle_put(handle);
+	return err;
+}
+
+/*
+ * Perform cache op on the list of memory regions within passed handles.
+ * A memory region within handle[i] is identified by offsets[i], sizes[i]
+ *
+ * sizes[i] == 0  is a special case which causes handle wide operation,
+ * this is done by replacing offsets[i] = 0, sizes[i] = handles[i]->size.
+ * So, the input arrays sizes, offsets  are not guaranteed to be read-only
+ *
+ * This will optimze the op if it can.
+ * In the case that all the handles together are larger than the inner cache
+ * maint threshold it is possible to just do an entire inner cache flush.
+ *
+ * NOTE: this omits outer cache operations which is fine for ARM64
+ */
+static int __nvmap_do_cache_maint_list(struct nvmap_handle **handles,
+				u64 *offsets, u64 *sizes, int op, u32 nr_ops,
+				bool is_32)
+{
+	u32 i;
+	u64 total = 0;
+	u64 thresh = ~0;
+
+	WARN(!IS_ENABLED(CONFIG_ARM64),
+		"cache list operation may not function properly");
+
+	for (i = 0; i < nr_ops; i++) {
+		bool inner, outer;
+		u32 *sizes_32 = (u32 *)sizes;
+		u64 size = is_32 ? sizes_32[i] : sizes[i];
+
+		nvmap_handle_get_cacheability(handles[i], &inner, &outer);
+
+		if (!inner && !outer)
+			continue;
+
+		if ((op == NVMAP_CACHE_OP_WB) && nvmap_handle_track_dirty(handles[i]))
+			total += atomic_read(&handles[i]->pgalloc.ndirty);
+		else
+			total += size ? size : handles[i]->size;
+	}
+
+	if (!total)
+		return 0;
+
+	/* Full flush in the case the passed list is bigger than our
+	 * threshold. */
+	if (total >= thresh) {
+		for (i = 0; i < nr_ops; i++) {
+			if (handles[i]->userflags &
+			    NVMAP_HANDLE_CACHE_SYNC) {
+				nvmap_handle_mkclean(handles[i], 0,
+						     handles[i]->size);
+				nvmap_zap_handle(handles[i], 0,
+						 handles[i]->size);
+			}
+		}
+
+		nvmap_stats_inc(NS_CFLUSH_RQ, total);
+		nvmap_stats_inc(NS_CFLUSH_DONE, thresh);
+		trace_nvmap_cache_flush(total,
+					nvmap_stats_read(NS_ALLOC),
+					nvmap_stats_read(NS_CFLUSH_RQ),
+					nvmap_stats_read(NS_CFLUSH_DONE));
+	} else {
+		for (i = 0; i < nr_ops; i++) {
+			u32 *offs_32 = (u32 *)offsets, *sizes_32 = (u32 *)sizes;
+			u64 size = is_32 ? sizes_32[i] : sizes[i];
+			u64 offset = is_32 ? offs_32[i] : offsets[i];
+			int err;
+
+			size = size ?: handles[i]->size;
+			offset = offset ?: 0;
+			err = __nvmap_do_cache_maint(handles[i]->owner,
+						     handles[i], offset,
+						     offset + size,
+						     op, false);
+			if (err) {
+				pr_err("cache maint per handle failed [%d]\n",
+						err);
+				return err;
+			}
+		}
+	}
+
+	return 0;
+}
+
+#if (LINUX_VERSION_CODE > KERNEL_VERSION(4, 9, 0))
+static const struct soc_device_attribute tegra194_soc = {
+	.soc_id = "TEGRA194",
+};
+
+static const struct soc_device_attribute tegra234_soc = {
+	.soc_id = "TEGRA234",
+};
+#endif
+inline int nvmap_do_cache_maint_list(struct nvmap_handle **handles,
+				u64 *offsets, u64 *sizes, int op, u32 nr_ops,
+				bool is_32)
+{
+	/*
+	 * As io-coherency is enabled by default from T194 onwards,
+	 * Don't do cache maint from CPU side. The HW, SCF will do.
+	 */
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0))
+	if (!(tegra_get_chip_id() == TEGRA194))
+#else
+	if (!soc_device_match(&tegra194_soc) &&
+		!soc_device_match(&tegra234_soc))
+#endif
+		return __nvmap_do_cache_maint_list(handles,
+				offsets, sizes, op, nr_ops, is_32);
+	return 0;
+}

drivers/video/tegra/nvmap/nvmap_cache_maint.S

@@ -0,0 +1,236 @@
+/*
+ * Copyright (c) 2021-2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Cache maintenance
+ *
+ * Derived from Linux kernel source file arch/arm64/mm/cache.S
+ * Copyright (C) 2001 Deep Blue Solutions Ltd.
+ * Copyright (C) 2012 ARM Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ */
+
+#include <linux/errno.h>
+#include <linux/linkage.h>
+#include <linux/init.h>
+#include <linux/version.h>
+#include <asm/assembler.h>
+#include <asm/cpufeature.h>
+#include <asm/alternative.h>
+#include <asm/asm-uaccess.h>
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 18, 0)
+#define NVMAP_SYM_FUNC_START(func)	SYM_FUNC_START(__ip_##func)
+#define NVMAP_SYM_FUNC_END(func)	SYM_FUNC_END(__ip_##func)
+#define NVMAP_SYM_FUNC_ALIAS(func)	SYM_FUNC_ALIAS(func, __ip_##func)
+#else
+#define NVMAP_SYM_FUNC_START(func)	SYM_FUNC_START_PI(func)
+#define NVMAP_SYM_FUNC_END(func)	SYM_FUNC_END_PI(func)
+#define NVMAP_SYM_FUNC_ALIAS(func)
+#endif
+
+/*
+ *	invalidate_icache_range(start,end)
+ *
+ *	Ensure that the I cache is invalid within specified region.
+ *
+ *	- start   - virtual start address of region
+ *	- end     - virtual end address of region
+ */
+SYM_FUNC_START(invalidate_icache_range)
+alternative_if ARM64_HAS_CACHE_DIC
+	mov	x0, xzr
+	isb
+	ret
+alternative_else_nop_endif
+
+	uaccess_ttbr0_enable x2, x3, x4
+
+	invalidate_icache_by_line x0, x1, x2, x3, 2f
+	mov	x0, xzr
+1:
+	uaccess_ttbr0_disable x1, x2
+	ret
+2:
+	mov	x0, #-EFAULT
+	b	1b
+SYM_FUNC_END(invalidate_icache_range)
+
+/*
+ *	__flush_dcache_area(kaddr, size)
+ *
+ *	Ensure that any D-cache lines for the interval [kaddr, kaddr+size)
+ *	are cleaned and invalidated to the PoC.
+ *
+ *	- kaddr   - kernel address
+ *	- size    - size in question
+ */
+NVMAP_SYM_FUNC_START(__flush_dcache_area)
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 14, 0)
+	add	x1, x0, x1
+#endif
+	dcache_by_line_op civac, sy, x0, x1, x2, x3
+	ret
+NVMAP_SYM_FUNC_END(__flush_dcache_area)
+NVMAP_SYM_FUNC_ALIAS(__flush_dcache_area)
+
+/*
+ *	__clean_dcache_area_pou(kaddr, size)
+ *
+ * 	Ensure that any D-cache lines for the interval [kaddr, kaddr+size)
+ * 	are cleaned to the PoU.
+ *
+ *	- kaddr   - kernel address
+ *	- size    - size in question
+ */
+SYM_FUNC_START(__clean_dcache_area_pou)
+alternative_if ARM64_HAS_CACHE_IDC
+	dsb	ishst
+	ret
+alternative_else_nop_endif
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 14, 0)
+	add	x1, x0, x1
+#endif
+	dcache_by_line_op cvau, ish, x0, x1, x2, x3
+	ret
+SYM_FUNC_END(__clean_dcache_area_pou)
+
+/*
+ *	__inval_dcache_area(kaddr, size)
+ *
+ * 	Ensure that any D-cache lines for the interval [kaddr, kaddr+size)
+ * 	are invalidated. Any partial lines at the ends of the interval are
+ *	also cleaned to PoC to prevent data loss.
+ *
+ *	- kaddr   - kernel address
+ *	- size    - size in question
+ */
+SYM_FUNC_START_LOCAL(__dma_inv_area)
+NVMAP_SYM_FUNC_START(__inval_dcache_area)
+	/* FALLTHROUGH */
+
+/*
+ *	__dma_inv_area(start, size)
+ *	- start   - virtual start address of region
+ *	- size    - size in question
+ */
+	add	x1, x1, x0
+	dcache_line_size x2, x3
+	sub	x3, x2, #1
+	tst	x1, x3				// end cache line aligned?
+	bic	x1, x1, x3
+	b.eq	1f
+	dc	civac, x1			// clean & invalidate D / U line
+1:	tst	x0, x3				// start cache line aligned?
+	bic	x0, x0, x3
+	b.eq	2f
+	dc	civac, x0			// clean & invalidate D / U line
+	b	3f
+2:	dc	ivac, x0			// invalidate D / U line
+3:	add	x0, x0, x2
+	cmp	x0, x1
+	b.lo	2b
+	dsb	sy
+	ret
+NVMAP_SYM_FUNC_END(__inval_dcache_area)
+NVMAP_SYM_FUNC_ALIAS(__inval_dcache_area)
+SYM_FUNC_END(__dma_inv_area)
+
+/*
+ *	__clean_dcache_area_poc(kaddr, size)
+ *
+ * 	Ensure that any D-cache lines for the interval [kaddr, kaddr+size)
+ * 	are cleaned to the PoC.
+ *
+ *	- kaddr   - kernel address
+ *	- size    - size in question
+ */
+SYM_FUNC_START_LOCAL(__dma_clean_area)
+NVMAP_SYM_FUNC_START(__clean_dcache_area_poc)
+	/* FALLTHROUGH */
+
+/*
+ *	__dma_clean_area(start, size)
+ *	- start   - virtual start address of region
+ *	- size    - size in question
+ */
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 14, 0)
+	add	x1, x0, x1
+#endif
+	dcache_by_line_op cvac, sy, x0, x1, x2, x3
+	ret
+NVMAP_SYM_FUNC_END(__clean_dcache_area_poc)
+NVMAP_SYM_FUNC_ALIAS(__clean_dcache_area_poc)
+SYM_FUNC_END(__dma_clean_area)
+
+/*
+ *	__clean_dcache_area_pop(kaddr, size)
+ *
+ * 	Ensure that any D-cache lines for the interval [kaddr, kaddr+size)
+ * 	are cleaned to the PoP.
+ *
+ *	- kaddr   - kernel address
+ *	- size    - size in question
+ */
+NVMAP_SYM_FUNC_START(__clean_dcache_area_pop)
+	alternative_if_not ARM64_HAS_DCPOP
+	b	__clean_dcache_area_poc
+	alternative_else_nop_endif
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 14, 0)
+	add	x1, x0, x1
+#endif
+	dcache_by_line_op cvap, sy, x0, x1, x2, x3
+	ret
+NVMAP_SYM_FUNC_END(__clean_dcache_area_pop)
+NVMAP_SYM_FUNC_ALIAS(__clean_dcache_area_pop)
+
+/*
+ *	__dma_flush_area(start, size)
+ *
+ *	clean & invalidate D / U line
+ *
+ *	- start   - virtual start address of region
+ *	- size    - size in question
+ */
+NVMAP_SYM_FUNC_START(__dma_flush_area)
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 14, 0)
+	add	x1, x0, x1
+#endif
+	dcache_by_line_op civac, sy, x0, x1, x2, x3
+	ret
+NVMAP_SYM_FUNC_END(__dma_flush_area)
+NVMAP_SYM_FUNC_ALIAS(__dma_flush_area)
+
+/*
+ *	__dma_map_area(start, size, dir)
+ *	- start	- kernel virtual start address
+ *	- size	- size of region
+ *	- dir	- DMA direction
+ */
+NVMAP_SYM_FUNC_START(__dma_map_area)
+	cmp	w2, #DMA_FROM_DEVICE
+	b.eq	__dma_inv_area
+	b	__dma_clean_area
+NVMAP_SYM_FUNC_END(__dma_map_area)
+NVMAP_SYM_FUNC_ALIAS(__dma_map_area)
+
+/*
+ *	__dma_unmap_area(start, size, dir)
+ *	- start	- kernel virtual start address
+ *	- size	- size of region
+ *	- dir	- DMA direction
+ */
+NVMAP_SYM_FUNC_START(__dma_unmap_area)
+	cmp	w2, #DMA_TO_DEVICE
+	b.ne	__dma_inv_area
+	ret
+NVMAP_SYM_FUNC_END(__dma_unmap_area)
+NVMAP_SYM_FUNC_ALIAS(__dma_unmap_area)

drivers/video/tegra/nvmap/nvmap_carveout.c

@@ -0,0 +1,270 @@
+/*
+ * drivers/video/tegra/nvmap/nvmap_dev.c
+ *
+ * Interface with nvmap carveouts
+ *
+ * Copyright (c) 2011-2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#include <linux/debugfs.h>
+
+#include "nvmap_priv.h"
+
+extern struct nvmap_device *nvmap_dev;
+
+extern const struct file_operations debug_clients_fops;
+extern const struct file_operations debug_allocations_fops;
+extern const struct file_operations debug_all_allocations_fops;
+extern const struct file_operations debug_orphan_handles_fops;
+extern const struct file_operations debug_maps_fops;
+#ifdef NVMAP_CONFIG_DEBUG_MAPS
+extern const struct file_operations debug_device_list_fops;
+#endif /* NVMAP_CONFIG_DEBUG_MAPS */
+
+int nvmap_create_carveout(const struct nvmap_platform_carveout *co)
+{
+	int i, err = 0;
+	struct nvmap_carveout_node *node;
+
+	mutex_lock(&nvmap_dev->carveout_lock);
+	if (!nvmap_dev->heaps) {
+		int nr_heaps;
+
+		nvmap_dev->nr_carveouts = 0;
+		if (nvmap_dev->plat)
+			nr_heaps = nvmap_dev->plat->nr_carveouts + 1;
+		else
+			nr_heaps = 1;
+		nvmap_dev->heaps = kzalloc(sizeof(struct nvmap_carveout_node) *
+				     nr_heaps, GFP_KERNEL);
+		if (!nvmap_dev->heaps) {
+			err = -ENOMEM;
+			pr_err("couldn't allocate carveout memory\n");
+			goto out;
+		}
+		nvmap_dev->nr_heaps = nr_heaps;
+	} else if (nvmap_dev->nr_carveouts >= nvmap_dev->nr_heaps) {
+		node = krealloc(nvmap_dev->heaps,
+				sizeof(*node) * (nvmap_dev->nr_carveouts + 1),
+				GFP_KERNEL);
+		if (!node) {
+			err = -ENOMEM;
+			pr_err("nvmap heap array resize failed\n");
+			goto out;
+		}
+		nvmap_dev->heaps = node;
+		nvmap_dev->nr_heaps = nvmap_dev->nr_carveouts + 1;
+	}
+
+	for (i = 0; i < nvmap_dev->nr_heaps; i++)
+		if ((co->usage_mask != NVMAP_HEAP_CARVEOUT_IVM) &&
+		    (nvmap_dev->heaps[i].heap_bit & co->usage_mask)) {
+			pr_err("carveout %s already exists\n", co->name);
+			err = -EEXIST;
+			goto out;
+		}
+
+	node = &nvmap_dev->heaps[nvmap_dev->nr_carveouts];
+
+	node->base = round_up(co->base, PAGE_SIZE);
+	node->size = round_down(co->size -
+				(node->base - co->base), PAGE_SIZE);
+	if (!co->size)
+		goto out;
+
+	node->carveout = nvmap_heap_create(
+			nvmap_dev->dev_user.this_device, co,
+			node->base, node->size, node);
+
+	if (!node->carveout) {
+		err = -ENOMEM;
+		pr_err("couldn't create %s\n", co->name);
+		goto out;
+	}
+	node->index = nvmap_dev->nr_carveouts;
+	nvmap_dev->nr_carveouts++;
+	node->heap_bit = co->usage_mask;
+
+	if (!IS_ERR_OR_NULL(nvmap_dev->debug_root)) {
+		struct dentry *heap_root =
+			debugfs_create_dir(co->name, nvmap_dev->debug_root);
+		if (!IS_ERR_OR_NULL(heap_root)) {
+			debugfs_create_file("clients", S_IRUGO,
+				heap_root,
+				(void *)(uintptr_t)node->heap_bit,
+				&debug_clients_fops);
+			debugfs_create_file("allocations", S_IRUGO,
+				heap_root,
+				(void *)(uintptr_t)node->heap_bit,
+				&debug_allocations_fops);
+			debugfs_create_file("all_allocations", S_IRUGO,
+				heap_root,
+				(void *)(uintptr_t)node->heap_bit,
+				&debug_all_allocations_fops);
+			debugfs_create_file("orphan_handles", S_IRUGO,
+				heap_root,
+				(void *)(uintptr_t)node->heap_bit,
+				&debug_orphan_handles_fops);
+			debugfs_create_file("maps", S_IRUGO,
+				heap_root,
+				(void *)(uintptr_t)node->heap_bit,
+				&debug_maps_fops);
+			debugfs_create_bool("no_cpu_access", S_IRUGO,
+				heap_root, (bool *)&co->no_cpu_access);
+#ifdef NVMAP_CONFIG_DEBUG_MAPS
+			debugfs_create_file("device_list", S_IRUGO,
+				heap_root,
+				(void *)(uintptr_t)node->heap_bit,
+				&debug_device_list_fops);
+#endif /* NVMAP_CONFIG_DEBUG_MAPS */
+			nvmap_heap_debugfs_init(heap_root,
+						node->carveout);
+		}
+	}
+out:
+	mutex_unlock(&nvmap_dev->carveout_lock);
+	return err;
+}
+
+#ifdef NVMAP_CONFIG_DEBUG_MAPS
+struct nvmap_device_list *nvmap_is_device_present(char *device_name, u32 heap_type)
+{
+	struct rb_node *node = NULL;
+	int i;
+
+	if (heap_type == NVMAP_HEAP_IOVMM) {
+		node = nvmap_dev->device_names.rb_node;
+	} else {
+		for (i = 0; i < nvmap_dev->nr_carveouts; i++) {
+			if ((heap_type & nvmap_dev->heaps[i].heap_bit) &&
+					nvmap_dev->heaps[i].carveout) {
+				node = nvmap_dev->heaps[i].carveout->device_names.rb_node;
+				break;
+			}
+		}
+	}
+	while (node) {
+		struct nvmap_device_list *dl = container_of(node,
+						struct nvmap_device_list, node);
+		if (strcmp(dl->device_name, device_name) > 0)
+			node = node->rb_left;
+		else if (strcmp(dl->device_name, device_name) < 0)
+			node = node->rb_right;
+		else
+			return dl;
+	}
+	return NULL;
+}
+
+void nvmap_add_device_name(char *device_name, u64 dma_mask, u32 heap_type)
+{
+	struct rb_root *root = NULL;
+	struct rb_node **new = NULL, *parent = NULL;
+	struct nvmap_device_list *dl = NULL;
+	int i;
+
+	if (heap_type == NVMAP_HEAP_IOVMM) {
+		root = &nvmap_dev->device_names;
+	} else {
+		for (i = 0; i < nvmap_dev->nr_carveouts; i++) {
+			if ((heap_type & nvmap_dev->heaps[i].heap_bit) &&
+				nvmap_dev->heaps[i].carveout) {
+				root = &nvmap_dev->heaps[i].carveout->device_names;
+				break;
+			}
+		}
+	}
+	if (root) {
+		new = &(root->rb_node);
+		while (*new) {
+			dl = container_of(*new, struct nvmap_device_list, node);
+			parent = *new;
+			if (strcmp(dl->device_name, device_name) > 0)
+				new = &((*new)->rb_left);
+			else if (strcmp(dl->device_name, device_name) < 0)
+				new = &((*new)->rb_right);
+		}
+		dl = kzalloc(sizeof(*dl), GFP_KERNEL);
+		if (!dl)
+			return;
+		dl->device_name = kzalloc(strlen(device_name) + 1, GFP_KERNEL);
+		if (!dl->device_name)
+			return;
+		strcpy(dl->device_name, device_name);
+		dl->dma_mask = dma_mask;
+		rb_link_node(&dl->node, parent, new);
+		rb_insert_color(&dl->node, root);
+	}
+}
+
+void nvmap_remove_device_name(char *device_name, u32 heap_type)
+{
+	struct nvmap_device_list *dl = NULL;
+	int i;
+
+	dl = nvmap_is_device_present(device_name, heap_type);
+	if (dl) {
+		if (heap_type == NVMAP_HEAP_IOVMM) {
+			rb_erase(&dl->node,
+				&nvmap_dev->device_names);
+			kfree(dl->device_name);
+			kfree(dl);
+			return;
+		}
+		for (i = 0; i < nvmap_dev->nr_carveouts; i++) {
+			if ((heap_type & nvmap_dev->heaps[i].heap_bit) &&
+				nvmap_dev->heaps[i].carveout) {
+				rb_erase(&dl->node,
+					&nvmap_dev->heaps[i].carveout->device_names);
+				kfree(dl->device_name);
+				kfree(dl);
+				return;
+			}
+		}
+	}
+}
+#endif /* NVMAP_CONFIG_DEBUG_MAPS */
+
+static
+struct nvmap_heap_block *do_nvmap_carveout_alloc(struct nvmap_client *client,
+					      struct nvmap_handle *handle,
+					      unsigned long type,
+					      phys_addr_t *start)
+{
+	struct nvmap_carveout_node *co_heap;
+	struct nvmap_device *dev = nvmap_dev;
+	int i;
+
+	for (i = 0; i < dev->nr_carveouts; i++) {
+		struct nvmap_heap_block *block;
+		co_heap = &dev->heaps[i];
+
+		if (!(co_heap->heap_bit & type))
+			continue;
+
+		if (type & NVMAP_HEAP_CARVEOUT_IVM)
+			handle->size = ALIGN(handle->size, NVMAP_IVM_ALIGNMENT);
+
+		block = nvmap_heap_alloc(co_heap->carveout, handle, start);
+		if (block)
+			return block;
+	}
+	return NULL;
+}
+
+struct nvmap_heap_block *nvmap_carveout_alloc(struct nvmap_client *client,
+					      struct nvmap_handle *handle,
+					      unsigned long type,
+					      phys_addr_t *start)
+{
+	return do_nvmap_carveout_alloc(client, handle, type, start);
+}

drivers/video/tegra/nvmap/nvmap_core.c

@@ -0,0 +1,355 @@
+/*
+ * drivers/video/tegra/nvmap/nvmap_core.c
+ *
+ * Memory manager for Tegra GPU
+ *
+ * Copyright (c) 2009-2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#define pr_fmt(fmt)	"nvmap: %s() " fmt, __func__
+
+#include <linux/err.h>
+#include <linux/highmem.h>
+#include <linux/io.h>
+#include <linux/rbtree.h>
+#include <linux/vmalloc.h>
+#include <linux/wait.h>
+#include <linux/slab.h>
+#include <linux/export.h>
+
+#include <asm/pgtable.h>
+#include <asm/tlbflush.h>
+
+#include <linux/nvmap.h>
+#include <trace/events/nvmap.h>
+
+#include "nvmap_priv.h"
+
+static phys_addr_t handle_phys(struct nvmap_handle *h)
+{
+	if (h->heap_pgalloc)
+		BUG();
+	return h->carveout->base;
+}
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0)
+void *__nvmap_kmap(struct nvmap_handle *h, unsigned int pagenum)
+{
+	phys_addr_t paddr;
+	unsigned long kaddr;
+	void __iomem *addr;
+	pgprot_t prot;
+
+	if (!virt_addr_valid(h))
+		return NULL;
+
+	h = nvmap_handle_get(h);
+	if (!h)
+		return NULL;
+	/*
+	 * If the handle is RO and virtual mapping is requested in
+	 * kernel address space, return error.
+	 */
+	if (h->from_va && h->is_ro)
+		goto put_handle;
+
+	if (!h->alloc)
+		goto put_handle;
+
+	if (!(h->heap_type & nvmap_dev->cpu_access_mask))
+		goto put_handle;
+
+	nvmap_kmaps_inc(h);
+	if (pagenum >= h->size >> PAGE_SHIFT)
+		goto out;
+
+	if (h->vaddr) {
+		kaddr = (unsigned long)h->vaddr + pagenum * PAGE_SIZE;
+	} else {
+		prot = nvmap_pgprot(h, PG_PROT_KERNEL);
+		if (h->heap_pgalloc)
+			paddr = page_to_phys(nvmap_to_page(
+						h->pgalloc.pages[pagenum]));
+		else
+			paddr = h->carveout->base + pagenum * PAGE_SIZE;
+
+		addr = __ioremap(paddr, PAGE_SIZE, prot);
+		if (addr == NULL)
+			goto out;
+		kaddr = (unsigned long)addr;
+	}
+	return (void *)kaddr;
+out:
+	nvmap_kmaps_dec(h);
+put_handle:
+	nvmap_handle_put(h);
+	return NULL;
+}
+
+void __nvmap_kunmap(struct nvmap_handle *h, unsigned int pagenum,
+		  void *addr)
+{
+	phys_addr_t paddr;
+
+	if (!h || !h->alloc ||
+	    WARN_ON(!virt_addr_valid(h)) ||
+	    WARN_ON(!addr) ||
+	    !(h->heap_type & nvmap_dev->cpu_access_mask))
+		return;
+
+	if (WARN_ON(pagenum >= h->size >> PAGE_SHIFT))
+		return;
+
+	if (h->vaddr && (h->vaddr == (addr - pagenum * PAGE_SIZE)))
+		goto out;
+
+	if (h->heap_pgalloc)
+		paddr = page_to_phys(nvmap_to_page(h->pgalloc.pages[pagenum]));
+	else
+		paddr = h->carveout->base + pagenum * PAGE_SIZE;
+
+	if (h->flags != NVMAP_HANDLE_UNCACHEABLE &&
+	    h->flags != NVMAP_HANDLE_WRITE_COMBINE) {
+		__dma_flush_area(addr, PAGE_SIZE);
+		outer_flush_range(paddr, paddr + PAGE_SIZE); /* FIXME */
+	}
+	iounmap((void __iomem *)addr);
+out:
+	nvmap_kmaps_dec(h);
+	nvmap_handle_put(h);
+}
+#endif
+
+void *__nvmap_mmap(struct nvmap_handle *h)
+{
+	pgprot_t prot;
+	void *vaddr;
+	unsigned long adj_size;
+	struct page **pages = NULL;
+	int i = 0;
+
+	if (!virt_addr_valid(h))
+		return NULL;
+
+	h = nvmap_handle_get(h);
+	if (!h)
+		return NULL;
+	/*
+	 * If the handle is RO and virtual mapping is requested in
+	 * kernel address space, return error.
+	 */
+	if (h->from_va && h->is_ro)
+		goto put_handle;
+
+
+	if (!h->alloc)
+		goto put_handle;
+
+	if (!(h->heap_type & nvmap_dev->cpu_access_mask))
+		goto put_handle;
+
+	if (h->vaddr)
+		return h->vaddr;
+
+	nvmap_kmaps_inc(h);
+	prot = nvmap_pgprot(h, PG_PROT_KERNEL);
+
+	if (h->heap_pgalloc) {
+		pages = nvmap_pages(h->pgalloc.pages, h->size >> PAGE_SHIFT);
+		if (!pages)
+			goto out;
+
+		vaddr = vmap(pages, h->size >> PAGE_SHIFT, VM_MAP, prot);
+		if (!vaddr && !h->vaddr)
+			goto out;
+		nvmap_altfree(pages, (h->size >> PAGE_SHIFT) * sizeof(*pages));
+
+		if (vaddr && atomic_long_cmpxchg((atomic_long_t *)&h->vaddr, 0, (long)vaddr)) {
+			nvmap_kmaps_dec(h);
+			vunmap(vaddr);
+		}
+		return h->vaddr;
+	}
+
+	/* carveout - explicitly map the pfns into a vmalloc area */
+	adj_size = h->carveout->base & ~PAGE_MASK;
+	adj_size += h->size;
+	adj_size = PAGE_ALIGN(adj_size);
+
+	if (pfn_valid(__phys_to_pfn(h->carveout->base & PAGE_MASK))) {
+		unsigned long pfn;
+		struct page *page;
+		int nr_pages;
+
+		pfn = ((h->carveout->base) >> PAGE_SHIFT);
+		page = pfn_to_page(pfn);
+		nr_pages = h->size >> PAGE_SHIFT;
+
+		pages = vmalloc(nr_pages * sizeof(*pages));
+		if (!pages)
+			goto out;
+
+		for (i = 0; i < nr_pages; i++)
+			pages[i] = nth_page(page, i);
+
+		vaddr = vmap(pages, nr_pages, VM_MAP, prot);
+	} else {
+		vaddr = (__force void *)__ioremap(h->carveout->base, adj_size,
+			 prot);
+	}
+	if (vaddr == NULL)
+		goto out;
+
+	if (vaddr && atomic_long_cmpxchg((atomic_long_t *)&h->vaddr,
+						 0, (long)vaddr)) {
+		vaddr -= (h->carveout->base & ~PAGE_MASK);
+		/*
+		 * iounmap calls vunmap for vmalloced address, hence
+		 * takes care of vmap/__ioremap freeing part.
+		 */
+		iounmap((void __iomem *)vaddr);
+		nvmap_kmaps_dec(h);
+	}
+
+	if (pages)
+		vfree(pages);
+
+	/* leave the handle ref count incremented by 1, so that
+	 * the handle will not be freed while the kernel mapping exists.
+	 * nvmap_handle_put will be called by unmapping this address */
+	return h->vaddr;
+out:
+	if (pages)
+		vfree(pages);
+	nvmap_kmaps_dec(h);
+put_handle:
+	nvmap_handle_put(h);
+	return NULL;
+}
+
+void __nvmap_munmap(struct nvmap_handle *h, void *addr)
+{
+	if (!h || !h->alloc ||
+	    WARN_ON(!virt_addr_valid(h)) ||
+	    WARN_ON(!addr) ||
+	    !(h->heap_type & nvmap_dev->cpu_access_mask))
+		return;
+
+	nvmap_handle_put(h);
+}
+
+/*
+ * NOTE: this does not ensure the continued existence of the underlying
+ * dma_buf. If you want ensure the existence of the dma_buf you must get an
+ * nvmap_handle_ref as that is what tracks the dma_buf refs.
+ */
+struct nvmap_handle *nvmap_handle_get(struct nvmap_handle *h)
+{
+	int cnt;
+
+	if (WARN_ON(!virt_addr_valid(h))) {
+		pr_err("%s: invalid handle\n", current->group_leader->comm);
+		return NULL;
+	}
+
+	cnt = atomic_inc_return(&h->ref);
+	NVMAP_TAG_TRACE(trace_nvmap_handle_get, h, cnt);
+
+	if (unlikely(cnt <= 1)) {
+		pr_err("%s: %s attempt to get a freed handle\n",
+			__func__, current->group_leader->comm);
+		atomic_dec(&h->ref);
+		return NULL;
+	}
+
+	return h;
+}
+
+void nvmap_handle_put(struct nvmap_handle *h)
+{
+	int cnt;
+
+	if (WARN_ON(!virt_addr_valid(h)))
+		return;
+	cnt = atomic_dec_return(&h->ref);
+	NVMAP_TAG_TRACE(trace_nvmap_handle_put, h, cnt);
+
+	if (WARN_ON(cnt < 0)) {
+		pr_err("%s: %s put to negative references\n",
+			__func__, current->comm);
+	} else if (cnt == 0)
+		_nvmap_handle_free(h);
+}
+
+struct sg_table *__nvmap_sg_table(struct nvmap_client *client,
+		struct nvmap_handle *h)
+{
+	struct sg_table *sgt = NULL;
+	int err, npages;
+	struct page **pages;
+
+	if (!virt_addr_valid(h))
+		return ERR_PTR(-EINVAL);
+
+	h = nvmap_handle_get(h);
+	if (!h)
+		return ERR_PTR(-EINVAL);
+
+	if (!h->alloc) {
+		err = -EINVAL;
+		goto put_handle;
+	}
+
+	npages = PAGE_ALIGN(h->size) >> PAGE_SHIFT;
+	sgt = kzalloc(sizeof(*sgt), GFP_KERNEL);
+	if (!sgt) {
+		err = -ENOMEM;
+		goto err;
+	}
+
+	if (!h->heap_pgalloc) {
+		phys_addr_t paddr = handle_phys(h);
+		struct page *page = phys_to_page(paddr);
+
+		err = sg_alloc_table(sgt, 1, GFP_KERNEL);
+		if (err)
+			goto err;
+
+		sg_set_page(sgt->sgl, page, h->size, offset_in_page(paddr));
+	} else {
+		pages = nvmap_pages(h->pgalloc.pages, npages);
+		if (!pages) {
+			err = -ENOMEM;
+			goto err;
+		}
+		err = sg_alloc_table_from_pages(sgt, pages,
+				npages, 0, h->size, GFP_KERNEL);
+		nvmap_altfree(pages, npages * sizeof(*pages));
+		if (err)
+			goto err;
+	}
+	nvmap_handle_put(h);
+	return sgt;
+
+err:
+	kfree(sgt);
+put_handle:
+	nvmap_handle_put(h);
+	return ERR_PTR(err);
+}
+
+void __nvmap_free_sg_table(struct nvmap_client *client,
+		struct nvmap_handle *h, struct sg_table *sgt)
+{
+	sg_free_table(sgt);
+	kfree(sgt);
+}

drivers/video/tegra/nvmap/nvmap_dmabuf.c

@@ -0,0 +1,666 @@
+/*
+ * dma_buf exporter for nvmap
+ *
+ * Copyright (c) 2012-2022, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#define pr_fmt(fmt)	"nvmap: %s() " fmt, __func__
+
+#include <linux/list.h>
+#include <linux/slab.h>
+#include <linux/uaccess.h>
+#include <linux/export.h>
+#include <linux/nvmap.h>
+#include <linux/dma-buf.h>
+#include <linux/spinlock.h>
+#include <linux/mutex.h>
+#include <linux/atomic.h>
+#include <linux/debugfs.h>
+#include <linux/seq_file.h>
+#include <linux/stringify.h>
+#include <linux/of.h>
+#include <linux/version.h>
+#include <linux/iommu.h>
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 18, 0)
+#include <linux/iosys-map.h>
+#endif
+
+#include <trace/events/nvmap.h>
+
+#include "nvmap_priv.h"
+#include "nvmap_ioctl.h"
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0)
+#define NVMAP_DMABUF_ATTACH  nvmap_dmabuf_attach
+#else
+#define NVMAP_DMABUF_ATTACH  __nvmap_dmabuf_attach
+#endif
+
+static int __nvmap_dmabuf_attach(struct dma_buf *dmabuf, struct device *dev,
+			       struct dma_buf_attachment *attach)
+{
+	struct nvmap_handle_info *info = dmabuf->priv;
+
+	trace_nvmap_dmabuf_attach(dmabuf, dev);
+
+	dev_dbg(dev, "%s() 0x%p\n", __func__, info->handle);
+	return 0;
+}
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0)
+static int nvmap_dmabuf_attach(struct dma_buf *dmabuf,
+			       struct dma_buf_attachment *attach)
+{
+	return __nvmap_dmabuf_attach(dmabuf, attach->dev, attach);
+}
+#endif
+
+static void nvmap_dmabuf_detach(struct dma_buf *dmabuf,
+				struct dma_buf_attachment *attach)
+{
+	struct nvmap_handle_info *info = dmabuf->priv;
+
+	trace_nvmap_dmabuf_detach(dmabuf, attach->dev);
+
+	dev_dbg(attach->dev, "%s() 0x%p\n", __func__, info->handle);
+}
+
+static inline bool access_vpr_phys(struct device *dev)
+{
+	if (!iommu_get_domain_for_dev(dev))
+		return true;
+
+	/*
+	 * Assumes gpu nodes always have DT entry, this is valid as device
+	 * specifying access-vpr-phys will do so through its DT entry.
+	 */
+	if (!dev->of_node)
+		return false;
+
+	return !!of_find_property(dev->of_node, "access-vpr-phys", NULL);
+}
+
+static struct sg_table *nvmap_dmabuf_map_dma_buf(struct dma_buf_attachment *attach,
+						  enum dma_data_direction dir)
+{
+	struct nvmap_handle_info *info = attach->dmabuf->priv;
+	int ents = 0;
+	struct sg_table *sgt;
+#ifdef NVMAP_CONFIG_DEBUG_MAPS
+	char *device_name = NULL;
+	u32 heap_type;
+	u64 dma_mask;
+#endif /* NVMAP_CONFIG_DEBUG_MAPS */
+	DEFINE_DMA_ATTRS(attrs);
+
+	trace_nvmap_dmabuf_map_dma_buf(attach->dmabuf, attach->dev);
+
+	/*
+	 * If the exported buffer is foreign buffer(alloc_from_va) and
+	 * has RO access, don't map it in device space.
+	 * Return error as no access.
+	 */
+	if (info->handle->from_va && info->handle->is_ro &&
+		(dir != DMA_TO_DEVICE))
+		return ERR_PTR(-EACCES);
+
+	nvmap_lru_reset(info->handle);
+	mutex_lock(&info->maps_lock);
+
+	atomic_inc(&info->handle->pin);
+
+	sgt = __nvmap_sg_table(NULL, info->handle);
+	if (IS_ERR(sgt)) {
+		atomic_dec(&info->handle->pin);
+		mutex_unlock(&info->maps_lock);
+		return sgt;
+	}
+
+	if (!info->handle->alloc) {
+		goto err_map;
+	} else if (!(nvmap_dev->dynamic_dma_map_mask &
+			info->handle->heap_type)) {
+		sg_dma_address(sgt->sgl) = info->handle->carveout->base;
+	} else if (info->handle->heap_type == NVMAP_HEAP_CARVEOUT_VPR &&
+			access_vpr_phys(attach->dev)) {
+		sg_dma_address(sgt->sgl) = 0;
+	} else {
+		dma_set_attr(DMA_ATTR_SKIP_CPU_SYNC, __DMA_ATTR(attrs));
+		ents = dma_map_sg_attrs(attach->dev, sgt->sgl,
+					sgt->nents, dir, __DMA_ATTR(attrs));
+		if (ents <= 0)
+			goto err_map;
+	}
+
+	attach->priv = sgt;
+
+#ifdef NVMAP_CONFIG_DEBUG_MAPS
+	/* Insert device name into the carveout's device name rb tree */
+	heap_type = info->handle->heap_type;
+	device_name = (char *)dev_name(attach->dev);
+	dma_mask = *(attach->dev->dma_mask);
+	if (device_name && !nvmap_is_device_present(device_name, heap_type)) {
+		/* If the device name is not already present in the tree, then only add */
+		nvmap_add_device_name(device_name, dma_mask, heap_type);
+	}
+#endif /* NVMAP_CONFIG_DEBUG_MAPS */
+	mutex_unlock(&info->maps_lock);
+	return sgt;
+
+err_map:
+	__nvmap_free_sg_table(NULL, info->handle, sgt);
+	atomic_dec(&info->handle->pin);
+	mutex_unlock(&info->maps_lock);
+	return ERR_PTR(-ENOMEM);
+}
+
+static void nvmap_dmabuf_unmap_dma_buf(struct dma_buf_attachment *attach,
+				       struct sg_table *sgt,
+				       enum dma_data_direction dir)
+{
+	struct nvmap_handle_info *info = attach->dmabuf->priv;
+#ifdef NVMAP_CONFIG_DEBUG_MAPS
+	char *device_name = NULL;
+	u32 heap_type = 0;
+#endif /* NVMAP_CONFIG_DEBUG_MAPS */
+
+	trace_nvmap_dmabuf_unmap_dma_buf(attach->dmabuf, attach->dev);
+
+	mutex_lock(&info->maps_lock);
+	if (!atomic_add_unless(&info->handle->pin, -1, 0)) {
+		mutex_unlock(&info->maps_lock);
+		WARN(1, "Unpinning handle that has yet to be pinned!\n");
+		return;
+	}
+
+	if (!(nvmap_dev->dynamic_dma_map_mask & info->handle->heap_type)) {
+		sg_dma_address(sgt->sgl) = 0;
+	} else if (info->handle->heap_type == NVMAP_HEAP_CARVEOUT_VPR &&
+			access_vpr_phys(attach->dev)) {
+		sg_dma_address(sgt->sgl) = 0;
+	} else {
+		dma_unmap_sg_attrs(attach->dev,
+				   sgt->sgl, sgt->nents,
+				   dir, DMA_ATTR_SKIP_CPU_SYNC);
+	}
+	__nvmap_free_sg_table(NULL, info->handle, sgt);
+
+#ifdef NVMAP_CONFIG_DEBUG_MAPS
+	/* Remove the device name from the list of carveout accessing devices */
+	heap_type = info->handle->heap_type;
+	device_name = (char *)dev_name(attach->dev);
+	if (device_name)
+		nvmap_remove_device_name(device_name, heap_type);
+#endif /* NVMAP_CONFIG_DEBUG_MAPS */
+	mutex_unlock(&info->maps_lock);
+}
+
+static void nvmap_dmabuf_release(struct dma_buf *dmabuf)
+{
+	struct nvmap_handle_info *info = dmabuf->priv;
+
+	trace_nvmap_dmabuf_release(info->handle->owner ?
+				   info->handle->owner->name : "unknown",
+				   info->handle,
+				   dmabuf);
+
+	mutex_lock(&info->handle->lock);
+	if (info->is_ro) {
+		BUG_ON(dmabuf != info->handle->dmabuf_ro);
+		info->handle->dmabuf_ro = NULL;
+	} else {
+		BUG_ON(dmabuf != info->handle->dmabuf);
+		info->handle->dmabuf = NULL;
+	}
+	mutex_unlock(&info->handle->lock);
+
+	nvmap_handle_put(info->handle);
+	kfree(info);
+}
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0)
+static int __nvmap_dmabuf_end_cpu_access(struct dma_buf *dmabuf,
+				       enum dma_data_direction dir)
+{
+	struct nvmap_handle_info *info = dmabuf->priv;
+
+	trace_nvmap_dmabuf_end_cpu_access(dmabuf, 0, dmabuf->size);
+	return __nvmap_do_cache_maint(NULL, info->handle,
+				   0, dmabuf->size,
+				   NVMAP_CACHE_OP_WB, false);
+}
+
+static int __nvmap_dmabuf_begin_cpu_access(struct dma_buf *dmabuf,
+					  enum dma_data_direction dir)
+{
+	struct nvmap_handle_info *info = dmabuf->priv;
+
+	trace_nvmap_dmabuf_begin_cpu_access(dmabuf, 0, dmabuf->size);
+	return __nvmap_do_cache_maint(NULL, info->handle, 0, dmabuf->size,
+				      NVMAP_CACHE_OP_WB_INV, false);
+}
+#define NVMAP_DMABUF_BEGIN_CPU_ACCESS           __nvmap_dmabuf_begin_cpu_access
+#define NVMAP_DMABUF_END_CPU_ACCESS 		__nvmap_dmabuf_end_cpu_access
+#else
+static int nvmap_dmabuf_begin_cpu_access(struct dma_buf *dmabuf,
+					  size_t start, size_t len,
+					  enum dma_data_direction dir)
+{
+	struct nvmap_handle_info *info = dmabuf->priv;
+
+	trace_nvmap_dmabuf_begin_cpu_access(dmabuf, start, len);
+	return __nvmap_do_cache_maint(NULL, info->handle, start, start + len,
+				      NVMAP_CACHE_OP_WB_INV, false);
+}
+
+static void nvmap_dmabuf_end_cpu_access(struct dma_buf *dmabuf,
+				       size_t start, size_t len,
+				       enum dma_data_direction dir)
+{
+	struct nvmap_handle_info *info = dmabuf->priv;
+
+	trace_nvmap_dmabuf_end_cpu_access(dmabuf, start, len);
+	__nvmap_do_cache_maint(NULL, info->handle,
+				   start, start + len,
+				   NVMAP_CACHE_OP_WB, false);
+}
+#define NVMAP_DMABUF_BEGIN_CPU_ACCESS           nvmap_dmabuf_begin_cpu_access
+#define NVMAP_DMABUF_END_CPU_ACCESS 		nvmap_dmabuf_end_cpu_access
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0)
+static void *nvmap_dmabuf_kmap(struct dma_buf *dmabuf, unsigned long page_num)
+{
+	struct nvmap_handle_info *info = dmabuf->priv;
+
+	trace_nvmap_dmabuf_kmap(dmabuf);
+	return __nvmap_kmap(info->handle, page_num);
+}
+
+static void nvmap_dmabuf_kunmap(struct dma_buf *dmabuf,
+		unsigned long page_num, void *addr)
+{
+	struct nvmap_handle_info *info = dmabuf->priv;
+
+	trace_nvmap_dmabuf_kunmap(dmabuf);
+	__nvmap_kunmap(info->handle, page_num, addr);
+}
+
+static void *nvmap_dmabuf_kmap_atomic(struct dma_buf *dmabuf,
+				      unsigned long page_num)
+{
+	WARN(1, "%s() can't be called from atomic\n", __func__);
+	return NULL;
+}
+#endif
+
+int __nvmap_map(struct nvmap_handle *h, struct vm_area_struct *vma)
+{
+	struct nvmap_vma_priv *priv;
+
+	h = nvmap_handle_get(h);
+	if (!h)
+		return -EINVAL;
+
+	if (!(h->heap_type & nvmap_dev->cpu_access_mask)) {
+		nvmap_handle_put(h);
+		return -EPERM;
+	}
+
+	/*
+	 * If the handle is RO and RW mapping is requested, then
+	 * return error.
+	 */
+	if (h->from_va && h->is_ro && (vma->vm_flags & VM_WRITE)) {
+		nvmap_handle_put(h);
+		return -EPERM;
+	}
+	/*
+	 * Don't allow mmap on VPR memory as it would be mapped
+	 * as device memory. User space shouldn't be accessing
+	 * device memory.
+	 */
+	if (h->heap_type == NVMAP_HEAP_CARVEOUT_VPR)  {
+		nvmap_handle_put(h);
+		return -EPERM;
+	}
+
+	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
+	if (!priv) {
+		nvmap_handle_put(h);
+		return -ENOMEM;
+	}
+	priv->handle = h;
+
+	vma->vm_flags |= VM_SHARED | VM_DONTEXPAND |
+			  VM_DONTDUMP | VM_DONTCOPY |
+			  (h->heap_pgalloc ? 0 : VM_PFNMAP);
+	vma->vm_ops = &nvmap_vma_ops;
+	BUG_ON(vma->vm_private_data != NULL);
+	vma->vm_private_data = priv;
+	vma->vm_page_prot = nvmap_pgprot(h, vma->vm_page_prot);
+	nvmap_vma_open(vma);
+	return 0;
+}
+
+static int nvmap_dmabuf_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma)
+{
+	struct nvmap_handle_info *info = dmabuf->priv;
+
+	trace_nvmap_dmabuf_mmap(dmabuf);
+
+	return __nvmap_map(info->handle, vma);
+}
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0)
+static void *nvmap_dmabuf_vmap(struct dma_buf *dmabuf)
+{
+	struct nvmap_handle_info *info = dmabuf->priv;
+
+	trace_nvmap_dmabuf_vmap(dmabuf);
+
+	/* Don't allow vmap on RO buffers */
+	if (info->is_ro)
+		return ERR_PTR(-EPERM);
+
+	return __nvmap_mmap(info->handle);
+}
+
+static void nvmap_dmabuf_vunmap(struct dma_buf *dmabuf, void *vaddr)
+{
+	struct nvmap_handle_info *info = dmabuf->priv;
+
+	trace_nvmap_dmabuf_vunmap(dmabuf);
+	__nvmap_munmap(info->handle, vaddr);
+}
+#else
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 18, 0)
+static int nvmap_dmabuf_vmap(struct dma_buf *dmabuf, struct iosys_map *map)
+#else
+static int nvmap_dmabuf_vmap(struct dma_buf *dmabuf, struct dma_buf_map *map)
+#endif
+{
+	struct nvmap_handle_info *info = dmabuf->priv;
+	void *res;
+	int ret = 0;
+
+	trace_nvmap_dmabuf_vmap(dmabuf);
+
+	/* Don't allow vmap on RO buffers */
+	if (info->is_ro)
+		return -EPERM;
+
+	res = __nvmap_mmap(info->handle);
+	if (res != NULL) {
+		map->vaddr = res;
+		map->is_iomem = false;
+	}
+	else {
+		ret = -ENOMEM;
+	}
+	return ret;
+}
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 18, 0)
+static void nvmap_dmabuf_vunmap(struct dma_buf *dmabuf, struct iosys_map *map)
+#else
+static void nvmap_dmabuf_vunmap(struct dma_buf *dmabuf, struct dma_buf_map *map)
+#endif
+{
+       struct nvmap_handle_info *info = dmabuf->priv;
+
+       trace_nvmap_dmabuf_vunmap(dmabuf);
+       __nvmap_munmap(info->handle, info->handle->vaddr);
+}
+#endif
+
+static struct dma_buf_ops nvmap_dma_buf_ops = {
+	.attach		= NVMAP_DMABUF_ATTACH,
+	.detach		= nvmap_dmabuf_detach,
+	.map_dma_buf	= nvmap_dmabuf_map_dma_buf,
+	.unmap_dma_buf	= nvmap_dmabuf_unmap_dma_buf,
+	.release	= nvmap_dmabuf_release,
+	.begin_cpu_access = NVMAP_DMABUF_BEGIN_CPU_ACCESS,
+	.end_cpu_access = NVMAP_DMABUF_END_CPU_ACCESS,
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0)
+	.kmap_atomic	= nvmap_dmabuf_kmap_atomic,
+	.kmap		= nvmap_dmabuf_kmap,
+	.kunmap		= nvmap_dmabuf_kunmap,
+#elif LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0)
+	.map_atomic	= nvmap_dmabuf_kmap_atomic,
+	.map		= nvmap_dmabuf_kmap,
+	.unmap		= nvmap_dmabuf_kunmap,
+#endif
+	.mmap		= nvmap_dmabuf_mmap,
+	.vmap		= nvmap_dmabuf_vmap,
+	.vunmap		= nvmap_dmabuf_vunmap,
+#if LINUX_VERSION_CODE > KERNEL_VERSION(5, 4, 0)
+	.cache_sgt_mapping = true,
+#endif
+
+};
+
+static char dmabuf_name[] = "nvmap_dmabuf";
+
+bool dmabuf_is_nvmap(struct dma_buf *dmabuf)
+{
+	return dmabuf->exp_name == dmabuf_name;
+}
+
+static struct dma_buf *__dma_buf_export(struct nvmap_handle_info *info,
+					size_t size, bool ro_buf)
+{
+	DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
+
+	exp_info.priv = info;
+	exp_info.ops = &nvmap_dma_buf_ops;
+	exp_info.size = size;
+
+	if (ro_buf) {
+		exp_info.flags = O_RDONLY;
+	} else {
+		exp_info.flags = O_RDWR;
+	}
+
+#ifndef NVMAP_UPSTREAM_KERNEL
+	/* Disable defer unmap feature only for kstable */
+	exp_info.exp_flags = DMABUF_CAN_DEFER_UNMAP |
+				DMABUF_SKIP_CACHE_SYNC;
+#endif /* !NVMAP_UPSTREAM_KERNEL */
+	exp_info.exp_name = dmabuf_name;
+
+	return dma_buf_export(&exp_info);
+}
+
+/*
+ * Make a dmabuf object for an nvmap handle.
+ */
+struct dma_buf *__nvmap_make_dmabuf(struct nvmap_client *client,
+				    struct nvmap_handle *handle, bool ro_buf)
+{
+	int err;
+	struct dma_buf *dmabuf;
+	struct nvmap_handle_info *info;
+
+	info = kzalloc(sizeof(*info), GFP_KERNEL);
+	if (!info) {
+		err = -ENOMEM;
+		goto err_nomem;
+	}
+	info->handle = handle;
+	info->is_ro = ro_buf;
+	INIT_LIST_HEAD(&info->maps);
+	mutex_init(&info->maps_lock);
+
+	dmabuf = __dma_buf_export(info, handle->size, ro_buf);
+	if (IS_ERR(dmabuf)) {
+		err = PTR_ERR(dmabuf);
+		goto err_export;
+	}
+	nvmap_handle_get(handle);
+
+	trace_nvmap_make_dmabuf(client->name, handle, dmabuf);
+	return dmabuf;
+
+err_export:
+	kfree(info);
+err_nomem:
+	return ERR_PTR(err);
+}
+
+int __nvmap_dmabuf_fd(struct nvmap_client *client,
+		      struct dma_buf *dmabuf, int flags)
+{
+#if !defined(NVMAP_CONFIG_HANDLE_AS_ID) && !defined(NVMAP_LOADABLE_MODULE)
+	int start_fd = NVMAP_CONFIG_FD_START;
+#endif
+	int ret;
+
+#ifdef NVMAP_CONFIG_DEFER_FD_RECYCLE
+	if (client->next_fd < NVMAP_CONFIG_FD_START)
+		client->next_fd = NVMAP_CONFIG_FD_START;
+	start_fd = client->next_fd++;
+	if (client->next_fd >= NVMAP_CONFIG_DEFER_FD_RECYCLE_MAX_FD)
+		client->next_fd = NVMAP_CONFIG_FD_START;
+#endif
+	if (!dmabuf || !dmabuf->file)
+		return -EINVAL;
+	/* Allocate fd from start_fd(>=1024) onwards to overcome
+	 * __FD_SETSIZE limitation issue for select(),
+	 * pselect() syscalls.
+	 */
+#if defined(NVMAP_LOADABLE_MODULE) || defined(NVMAP_CONFIG_HANDLE_AS_ID)
+	ret = get_unused_fd_flags(flags);
+#else
+	ret =  __alloc_fd(current->files, start_fd, sysctl_nr_open, flags);
+#endif
+	if (ret == -EMFILE)
+		pr_err_ratelimited("NvMap: FD limit is crossed for uid %d\n",
+				   from_kuid(current_user_ns(), current_uid()));
+	return ret;
+}
+
+static struct dma_buf *__nvmap_dmabuf_export(struct nvmap_client *client,
+				 struct nvmap_handle *handle, bool is_ro)
+{
+	struct dma_buf *buf;
+
+	handle = nvmap_handle_get(handle);
+	if (!handle)
+		return ERR_PTR(-EINVAL);
+	if (is_ro)
+		buf = handle->dmabuf_ro;
+	else
+		buf = handle->dmabuf;
+
+	if (WARN(!buf, "Attempting to get a freed dma_buf!\n")) {
+		nvmap_handle_put(handle);
+		return NULL;
+	}
+
+	get_dma_buf(buf);
+
+	/*
+	 * Don't want to take out refs on the handle here.
+	 */
+	nvmap_handle_put(handle);
+
+	return buf;
+}
+
+int nvmap_get_dmabuf_fd(struct nvmap_client *client, struct nvmap_handle *h,
+			bool is_ro)
+{
+	int fd;
+	struct dma_buf *dmabuf;
+
+	dmabuf = __nvmap_dmabuf_export(client, h, is_ro);
+	if (IS_ERR(dmabuf))
+		return PTR_ERR(dmabuf);
+
+	fd = __nvmap_dmabuf_fd(client, dmabuf, O_CLOEXEC);
+	if (fd < 0)
+		dma_buf_put(dmabuf);
+	return fd;
+}
+
+/*
+ * Returns the nvmap handle ID associated with the passed dma_buf's fd. This
+ * does not affect the ref count of the dma_buf.
+ * NOTE: Callers of this utility function must invoke nvmap_handle_put after
+ * using the returned nvmap_handle. Call to nvmap_handle_get is required in
+ * this utility function to avoid race conditions in code where nvmap_handle
+ * returned by this function is freed concurrently while the caller is still
+ * using it.
+ */
+struct nvmap_handle *nvmap_handle_get_from_dmabuf_fd(
+					struct nvmap_client *client, int fd)
+{
+	struct nvmap_handle *handle = ERR_PTR(-EINVAL);
+	struct dma_buf *dmabuf;
+	struct nvmap_handle_info *info;
+
+	dmabuf = dma_buf_get(fd);
+	if (IS_ERR(dmabuf))
+		return ERR_CAST(dmabuf);
+	if (dmabuf_is_nvmap(dmabuf)) {
+		info = dmabuf->priv;
+		handle = info->handle;
+		if (!nvmap_handle_get(handle))
+			handle = ERR_PTR(-EINVAL);
+	}
+	dma_buf_put(dmabuf);
+	return handle;
+}
+
+bool is_nvmap_dmabuf_fd_ro(int fd)
+{
+	struct dma_buf *dmabuf;
+	struct nvmap_handle_info *info = NULL;
+
+	dmabuf = dma_buf_get(fd);
+	if (IS_ERR(dmabuf)) {
+		return false;
+	}
+	if (dmabuf_is_nvmap(dmabuf)) {
+		info = dmabuf->priv;
+	}
+	dma_buf_put(dmabuf);
+
+	return (info != NULL) ? info->is_ro : false;
+}
+
+/*
+ * Duplicates a generic dma_buf fd. nvmap dma_buf fd has to be duplicated
+ * using existing code paths to preserve memory accounting behavior, so this
+ * function returns -EINVAL on dma_buf fds created by nvmap.
+ */
+int nvmap_dmabuf_duplicate_gen_fd(struct nvmap_client *client,
+		struct dma_buf *dmabuf)
+{
+	int ret = 0;
+
+	if (dmabuf_is_nvmap(dmabuf)) {
+		ret = -EINVAL;
+		goto error;
+	}
+
+	ret = __nvmap_dmabuf_fd(client, dmabuf, O_CLOEXEC);
+	if (ret < 0)
+		goto error;
+
+	return ret;
+
+error:
+	dma_buf_put(dmabuf);
+	return ret;
+}

drivers/video/tegra/nvmap/nvmap_fault.c

@@ -0,0 +1,261 @@
+/*
+ * drivers/video/tegra/nvmap/nvmap_fault.c
+ *
+ * Copyright (c) 2011-2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#define pr_fmt(fmt)	"%s: " fmt, __func__
+
+#include <trace/events/nvmap.h>
+#include <linux/highmem.h>
+
+#include "nvmap_priv.h"
+
+static void nvmap_vma_close(struct vm_area_struct *vma);
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0)
+#define __atomic_add_unless atomic_fetch_add_unless
+static vm_fault_t nvmap_vma_fault(struct vm_fault *vmf);
+#elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)
+static int nvmap_vma_fault(struct vm_fault *vmf);
+#else
+static int nvmap_vma_fault(struct vm_area_struct *vma, struct vm_fault *vmf);
+#endif
+
+struct vm_operations_struct nvmap_vma_ops = {
+	.open		= nvmap_vma_open,
+	.close		= nvmap_vma_close,
+	.fault		= nvmap_vma_fault,
+};
+
+int is_nvmap_vma(struct vm_area_struct *vma)
+{
+	return vma->vm_ops == &nvmap_vma_ops;
+}
+
+/* to ensure that the backing store for the VMA isn't freed while a fork'd
+ * reference still exists, nvmap_vma_open increments the reference count on
+ * the handle, and nvmap_vma_close decrements it. alternatively, we could
+ * disallow copying of the vma, or behave like pmem and zap the pages. FIXME.
+*/
+void nvmap_vma_open(struct vm_area_struct *vma)
+{
+	struct nvmap_vma_priv *priv;
+	struct nvmap_handle *h;
+	struct nvmap_vma_list *vma_list, *tmp;
+	struct list_head *tmp_head = NULL;
+	pid_t current_pid = task_tgid_nr(current);
+	bool vma_pos_found = false;
+	size_t nr_page, i;
+	int vma_open_count;
+
+	priv = vma->vm_private_data;
+	BUG_ON(!priv);
+	BUG_ON(!priv->handle);
+
+	h = priv->handle;
+	nvmap_umaps_inc(h);
+
+	mutex_lock(&h->lock);
+	vma_open_count = atomic_inc_return(&priv->count);
+	if (vma_open_count == 1 && h->heap_pgalloc) {
+		nr_page = h->size >> PAGE_SHIFT;
+		for (i = 0; i < nr_page; i++) {
+			struct page *page = nvmap_to_page(h->pgalloc.pages[i]);
+			/* This is necessry to avoid page being accounted
+			 * under NR_FILE_MAPPED. This way NR_FILE_MAPPED would
+			 * be fully accounted under NR_FILE_PAGES. This allows
+			 * Android low mem killer detect low memory condition
+			 * precisely.
+			 * This has a side effect of inaccurate pss accounting
+			 * for NvMap memory mapped into user space. Android
+			 * procrank and NvMap Procrank both would have same
+			 * issue. Subtracting NvMap_Procrank pss from
+			 * procrank pss would give non-NvMap pss held by process
+			 * and adding NvMap memory used by process represents
+			 * entire memroy consumption by the process.
+			 */
+			atomic_inc(&page->_mapcount);
+		}
+	}
+	mutex_unlock(&h->lock);
+
+	vma_list = kmalloc(sizeof(*vma_list), GFP_KERNEL);
+	if (vma_list) {
+		mutex_lock(&h->lock);
+		tmp_head = &h->vmas;
+
+		/* insert vma into handle's vmas list in the increasing order of
+		 * handle offsets
+		 */
+		list_for_each_entry(tmp, &h->vmas, list) {
+			/* if vma exists in list, just increment refcount */
+			if (tmp->vma == vma) {
+				atomic_inc(&tmp->ref);
+				kfree(vma_list);
+				goto unlock;
+			}
+
+			if (!vma_pos_found && (current_pid == tmp->pid)) {
+				if (vma->vm_pgoff < tmp->vma->vm_pgoff) {
+					tmp_head = &tmp->list;
+					vma_pos_found = true;
+				} else {
+					tmp_head = tmp->list.next;
+				}
+			}
+		}
+
+		vma_list->vma = vma;
+		vma_list->pid = current_pid;
+		vma_list->save_vm_flags = vma->vm_flags;
+		atomic_set(&vma_list->ref, 1);
+		list_add_tail(&vma_list->list, tmp_head);
+unlock:
+		mutex_unlock(&h->lock);
+	} else {
+		WARN(1, "vma not tracked");
+	}
+}
+
+static void nvmap_vma_close(struct vm_area_struct *vma)
+{
+	struct nvmap_vma_priv *priv = vma->vm_private_data;
+	struct nvmap_vma_list *vma_list;
+	struct nvmap_handle *h;
+	bool vma_found = false;
+	size_t nr_page, i;
+
+	if (!priv)
+		return;
+
+	BUG_ON(!priv->handle);
+
+	h = priv->handle;
+	nr_page = h->size >> PAGE_SHIFT;
+
+	mutex_lock(&h->lock);
+	list_for_each_entry(vma_list, &h->vmas, list) {
+		if (vma_list->vma != vma)
+			continue;
+		if (atomic_dec_return(&vma_list->ref) == 0) {
+			list_del(&vma_list->list);
+			kfree(vma_list);
+		}
+		vma_found = true;
+		break;
+	}
+	BUG_ON(!vma_found);
+	nvmap_umaps_dec(h);
+
+	if (__atomic_add_unless(&priv->count, -1, 0) == 1) {
+		if (h->heap_pgalloc) {
+			for (i = 0; i < nr_page; i++) {
+				struct page *page;
+				page = nvmap_to_page(h->pgalloc.pages[i]);
+				atomic_dec(&page->_mapcount);
+			}
+		}
+		mutex_unlock(&h->lock);
+		if (priv->handle)
+			nvmap_handle_put(priv->handle);
+		vma->vm_private_data = NULL;
+		kfree(priv);
+	} else {
+		mutex_unlock(&h->lock);
+	}
+}
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0)
+static vm_fault_t nvmap_vma_fault(struct vm_fault *vmf)
+#define vm_insert_pfn vmf_insert_pfn
+#elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)
+static int nvmap_vma_fault(struct vm_fault *vmf)
+#else
+static int nvmap_vma_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
+#endif
+{
+	struct page *page;
+	struct nvmap_vma_priv *priv;
+	unsigned long offs;
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)
+	struct vm_area_struct *vma = vmf->vma;
+	unsigned long vmf_address = vmf->address;
+#else
+	void __user *vmf_address = vmf->virtual_address;
+#endif
+
+	offs = (unsigned long)(vmf_address - vma->vm_start);
+	priv = vma->vm_private_data;
+	if (!priv || !priv->handle || !priv->handle->alloc)
+		return VM_FAULT_SIGBUS;
+
+	offs += priv->offs;
+	/* if the VMA was split for some reason, vm_pgoff will be the VMA's
+	 * offset from the original VMA */
+	offs += (vma->vm_pgoff << PAGE_SHIFT);
+
+	if (offs >= priv->handle->size)
+		return VM_FAULT_SIGBUS;
+
+	if (!priv->handle->heap_pgalloc) {
+		unsigned long pfn;
+		BUG_ON(priv->handle->carveout->base & ~PAGE_MASK);
+		pfn = ((priv->handle->carveout->base + offs) >> PAGE_SHIFT);
+		if (!pfn_valid(pfn)) {
+			vm_insert_pfn(vma,
+				(unsigned long)vmf_address, pfn);
+			return VM_FAULT_NOPAGE;
+		}
+		/* CMA memory would get here */
+		page = pfn_to_page(pfn);
+	} else {
+		void *kaddr;
+
+		offs >>= PAGE_SHIFT;
+		if (atomic_read(&priv->handle->pgalloc.reserved))
+			return VM_FAULT_SIGBUS;
+		page = nvmap_to_page(priv->handle->pgalloc.pages[offs]);
+
+		if (PageAnon(page) && (vma->vm_flags & VM_SHARED))
+			return VM_FAULT_SIGSEGV;
+
+		if (!nvmap_handle_track_dirty(priv->handle))
+			goto finish;
+
+		mutex_lock(&priv->handle->lock);
+		if (nvmap_page_dirty(priv->handle->pgalloc.pages[offs])) {
+			mutex_unlock(&priv->handle->lock);
+			goto finish;
+		}
+
+		/* inner cache maint */
+		kaddr  = kmap(page);
+		BUG_ON(!kaddr);
+		inner_cache_maint(NVMAP_CACHE_OP_WB_INV, kaddr, PAGE_SIZE);
+		kunmap(page);
+
+		if (priv->handle->flags & NVMAP_HANDLE_INNER_CACHEABLE)
+			goto make_dirty;
+
+make_dirty:
+		nvmap_page_mkdirty(&priv->handle->pgalloc.pages[offs]);
+		atomic_inc(&priv->handle->pgalloc.ndirty);
+		mutex_unlock(&priv->handle->lock);
+	}
+
+finish:
+	if (page)
+		get_page(page);
+	vmf->page = page;
+	return (page) ? 0 : VM_FAULT_SIGBUS;
+}

drivers/video/tegra/nvmap/nvmap_handle.c

@@ -0,0 +1,511 @@
+/*
+ * drivers/video/tegra/nvmap/nvmap_handle.c
+ *
+ * Handle allocation and freeing routines for nvmap
+ *
+ * Copyright (c) 2009-2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#define pr_fmt(fmt)	"%s: " fmt, __func__
+
+#include <linux/err.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/list.h>
+#include <linux/mm.h>
+#include <linux/rbtree.h>
+#include <linux/dma-buf.h>
+#include <linux/moduleparam.h>
+#include <linux/nvmap.h>
+#include <linux/version.h>
+#if KERNEL_VERSION(4, 15, 0) > LINUX_VERSION_CODE
+#include <soc/tegra/chip-id.h>
+#else
+#include <soc/tegra/fuse.h>
+#endif
+
+#include <asm/pgtable.h>
+
+#include <trace/events/nvmap.h>
+
+#include "nvmap_priv.h"
+#include "nvmap_ioctl.h"
+
+/*
+ * Verifies that the passed ID is a valid handle ID. Then the passed client's
+ * reference to the handle is returned.
+ *
+ * Note: to call this function make sure you own the client ref lock.
+ */
+struct nvmap_handle_ref *__nvmap_validate_locked(struct nvmap_client *c,
+						 struct nvmap_handle *h,
+						 bool is_ro)
+{
+	struct rb_node *n = c->handle_refs.rb_node;
+
+	while (n) {
+		struct nvmap_handle_ref *ref;
+		ref = rb_entry(n, struct nvmap_handle_ref, node);
+		if ((ref->handle == h) && (ref->is_ro == is_ro))
+			return ref;
+		else if ((uintptr_t)h > (uintptr_t)ref->handle)
+			n = n->rb_right;
+		else
+			n = n->rb_left;
+	}
+
+	return NULL;
+}
+/* adds a newly-created handle to the device master tree */
+void nvmap_handle_add(struct nvmap_device *dev, struct nvmap_handle *h)
+{
+	struct rb_node **p;
+	struct rb_node *parent = NULL;
+
+	spin_lock(&dev->handle_lock);
+	p = &dev->handles.rb_node;
+	while (*p) {
+		struct nvmap_handle *b;
+
+		parent = *p;
+		b = rb_entry(parent, struct nvmap_handle, node);
+		if (h > b)
+			p = &parent->rb_right;
+		else
+			p = &parent->rb_left;
+	}
+	rb_link_node(&h->node, parent, p);
+	rb_insert_color(&h->node, &dev->handles);
+	nvmap_lru_add(h);
+	spin_unlock(&dev->handle_lock);
+}
+
+/* remove a handle from the device's tree of all handles; called
+ * when freeing handles. */
+int nvmap_handle_remove(struct nvmap_device *dev, struct nvmap_handle *h)
+{
+	spin_lock(&dev->handle_lock);
+
+	/* re-test inside the spinlock if the handle really has no clients;
+	 * only remove the handle if it is unreferenced */
+	if (atomic_add_return(0, &h->ref) > 0) {
+		spin_unlock(&dev->handle_lock);
+		return -EBUSY;
+	}
+	smp_rmb();
+	BUG_ON(atomic_read(&h->ref) < 0);
+	BUG_ON(atomic_read(&h->pin) != 0);
+
+	nvmap_lru_del(h);
+	rb_erase(&h->node, &dev->handles);
+
+	spin_unlock(&dev->handle_lock);
+	return 0;
+}
+
+/* Validates that a handle is in the device master tree and that the
+ * client has permission to access it. */
+struct nvmap_handle *nvmap_validate_get(struct nvmap_handle *id)
+{
+	struct nvmap_handle *h = NULL;
+	struct rb_node *n;
+
+	spin_lock(&nvmap_dev->handle_lock);
+
+	n = nvmap_dev->handles.rb_node;
+
+	while (n) {
+		h = rb_entry(n, struct nvmap_handle, node);
+		if (h == id) {
+			h = nvmap_handle_get(h);
+			spin_unlock(&nvmap_dev->handle_lock);
+			return h;
+		}
+		if (id > h)
+			n = n->rb_right;
+		else
+			n = n->rb_left;
+	}
+	spin_unlock(&nvmap_dev->handle_lock);
+	return NULL;
+}
+
+static void add_handle_ref(struct nvmap_client *client,
+			   struct nvmap_handle_ref *ref)
+{
+	struct rb_node **p, *parent = NULL;
+
+	nvmap_ref_lock(client);
+	p = &client->handle_refs.rb_node;
+	while (*p) {
+		struct nvmap_handle_ref *node;
+		parent = *p;
+		node = rb_entry(parent, struct nvmap_handle_ref, node);
+		if (ref->handle > node->handle)
+			p = &parent->rb_right;
+		else
+			p = &parent->rb_left;
+	}
+	rb_link_node(&ref->node, parent, p);
+	rb_insert_color(&ref->node, &client->handle_refs);
+	client->handle_count++;
+	if (client->handle_count > nvmap_max_handle_count)
+		nvmap_max_handle_count = client->handle_count;
+	atomic_inc(&ref->handle->share_count);
+	nvmap_ref_unlock(client);
+}
+
+struct nvmap_handle_ref *nvmap_create_handle_from_va(struct nvmap_client *client,
+						     ulong vaddr, size_t size,
+						     u32 flags)
+{
+	struct vm_area_struct *vma;
+	struct nvmap_handle_ref *ref;
+	vm_flags_t vm_flags;
+	struct mm_struct *mm = current->mm;
+
+	/* don't allow non-page aligned addresses. */
+	if (vaddr & ~PAGE_MASK)
+		return ERR_PTR(-EINVAL);
+
+	nvmap_acquire_mmap_read_lock(mm);
+	vma = find_vma(mm, vaddr);
+	if (unlikely(!vma)) {
+		nvmap_release_mmap_read_lock(mm);
+		return ERR_PTR(-EINVAL);
+	}
+
+	if (!size)
+		size = vma->vm_end - vaddr;
+
+	/* Don't allow exuberantly large sizes. */
+	if (!is_nvmap_memory_available(size, NVMAP_HEAP_IOVMM)) {
+		pr_debug("Cannot allocate %zu bytes.\n", size);
+		nvmap_release_mmap_read_lock(mm);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	vm_flags = vma->vm_flags;
+	nvmap_release_mmap_read_lock(mm);
+
+	/*
+	 * If buffer is malloc/mprotect as RO but alloc flag is not passed
+	 * as RO, don't create handle.
+	 */
+	if (!(vm_flags & VM_WRITE) && !(flags & NVMAP_HANDLE_RO))
+		return ERR_PTR(-EINVAL);
+
+	ref = nvmap_create_handle(client, size, flags & NVMAP_HANDLE_RO);
+	if (!IS_ERR(ref))
+		ref->handle->orig_size = size;
+
+	return ref;
+}
+
+struct nvmap_handle_ref *nvmap_create_handle(struct nvmap_client *client,
+					     size_t size, bool ro_buf)
+{
+	void *err = ERR_PTR(-ENOMEM);
+	struct nvmap_handle *h;
+	struct nvmap_handle_ref *ref = NULL;
+	struct dma_buf *dmabuf;
+
+	if (!client)
+		return ERR_PTR(-EINVAL);
+
+	if (!size)
+		return ERR_PTR(-EINVAL);
+
+	h = kzalloc(sizeof(*h), GFP_KERNEL);
+	if (!h)
+		return ERR_PTR(-ENOMEM);
+
+	ref = kzalloc(sizeof(*ref), GFP_KERNEL);
+	if (!ref)
+		goto ref_alloc_fail;
+
+	atomic_set(&h->ref, 1);
+	atomic_set(&h->pin, 0);
+	h->owner = client;
+	BUG_ON(!h->owner);
+	h->orig_size = size;
+	h->size = PAGE_ALIGN(size);
+	h->flags = NVMAP_HANDLE_WRITE_COMBINE;
+	h->peer = NVMAP_IVM_INVALID_PEER;
+	mutex_init(&h->lock);
+	INIT_LIST_HEAD(&h->vmas);
+	INIT_LIST_HEAD(&h->lru);
+	INIT_LIST_HEAD(&h->dmabuf_priv);
+
+	/*
+	 * This takes out 1 ref on the dambuf. This corresponds to the
+	 * handle_ref that gets automatically made by nvmap_create_handle().
+	 */
+	dmabuf = __nvmap_make_dmabuf(client, h, ro_buf);
+	if (IS_ERR(dmabuf)) {
+		err = dmabuf;
+		goto make_dmabuf_fail;
+	}
+	if (!ro_buf)
+		h->dmabuf = dmabuf;
+	else
+		h->dmabuf_ro = dmabuf;
+
+	nvmap_handle_add(nvmap_dev, h);
+
+	/*
+	 * Major assumption here: the dma_buf object that the handle contains
+	 * is created with a ref count of 1.
+	 */
+	atomic_set(&ref->dupes, 1);
+	ref->handle = h;
+	add_handle_ref(client, ref);
+	if (ro_buf)
+		ref->is_ro = true;
+	else
+		ref->is_ro = false;
+	trace_nvmap_create_handle(client, client->name, h, size, ref);
+	return ref;
+
+make_dmabuf_fail:
+	kfree(ref);
+ref_alloc_fail:
+	kfree(h);
+	return err;
+}
+
+struct nvmap_handle_ref *nvmap_try_duplicate_by_ivmid(
+		struct nvmap_client *client, u64 ivm_id,
+		struct nvmap_heap_block **block)
+{
+	struct nvmap_handle *h = NULL;
+	struct nvmap_handle_ref *ref = NULL;
+	struct rb_node *n;
+
+	spin_lock(&nvmap_dev->handle_lock);
+
+	n = nvmap_dev->handles.rb_node;
+	for (n = rb_first(&nvmap_dev->handles); n; n = rb_next(n)) {
+		h = rb_entry(n, struct nvmap_handle, node);
+		if (h->ivm_id == ivm_id) {
+			BUG_ON(!virt_addr_valid(h));
+			/* get handle's ref only if non-zero */
+			if (atomic_inc_not_zero(&h->ref) == 0) {
+				*block = h->carveout;
+				/* strip handle's block and fail duplication */
+				h->carveout = NULL;
+				break;
+			}
+			spin_unlock(&nvmap_dev->handle_lock);
+			goto found;
+		}
+	}
+
+	spin_unlock(&nvmap_dev->handle_lock);
+	/* handle is either freed or being freed, don't duplicate it */
+	goto finish;
+
+	/*
+	 * From this point, handle and its buffer are valid and won't be
+	 * freed as a reference is taken on it. The dmabuf can still be
+	 * freed anytime till reference is taken on it below.
+	 */
+found:
+	mutex_lock(&h->lock);
+	/*
+	 * Save this block. If dmabuf's reference is not held in time,
+	 * this can be reused to avoid the delay to free the buffer
+	 * in this old handle and allocate it for a new handle from
+	 * the ivm allocation ioctl.
+	 */
+	*block = h->carveout;
+	if (!h->dmabuf)
+		goto fail;
+	BUG_ON(!h->dmabuf->file);
+	/* This is same as get_dma_buf() if file->f_count was non-zero */
+	if (atomic_long_inc_not_zero(&h->dmabuf->file->f_count) == 0)
+		goto fail;
+	mutex_unlock(&h->lock);
+
+	/* h->dmabuf can't be NULL anymore. Duplicate the handle. */
+	ref = nvmap_duplicate_handle(client, h, true, false);
+	/* put the extra ref taken using get_dma_buf. */
+	dma_buf_put(h->dmabuf);
+finish:
+	return ref;
+fail:
+	/* free handle but not its buffer */
+	h->carveout = NULL;
+	mutex_unlock(&h->lock);
+	nvmap_handle_put(h);
+	return NULL;
+}
+
+struct nvmap_handle_ref *nvmap_duplicate_handle(struct nvmap_client *client,
+					struct nvmap_handle *h, bool skip_val,
+					bool is_ro)
+{
+	struct nvmap_handle_ref *ref = NULL;
+
+	BUG_ON(!client);
+
+	if (!skip_val)
+		/* on success, the reference count for the handle should be
+		 * incremented, so the success paths will not call
+		 * nvmap_handle_put */
+		h = nvmap_validate_get(h);
+
+	if (!h) {
+		pr_debug("%s duplicate handle failed\n",
+			    current->group_leader->comm);
+		return ERR_PTR(-EPERM);
+	}
+
+	if (!h->alloc) {
+		pr_err("%s duplicating unallocated handle\n",
+			current->group_leader->comm);
+		nvmap_handle_put(h);
+		return ERR_PTR(-EINVAL);
+	}
+
+	nvmap_ref_lock(client);
+	ref = __nvmap_validate_locked(client, h, is_ro);
+
+	if (ref) {
+		atomic_inc(&ref->dupes);
+		nvmap_ref_unlock(client);
+		goto out;
+	}
+
+	nvmap_ref_unlock(client);
+
+	ref = kzalloc(sizeof(*ref), GFP_KERNEL);
+	if (!ref) {
+		nvmap_handle_put(h);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	atomic_set(&ref->dupes, 1);
+	ref->handle = h;
+	add_handle_ref(client, ref);
+
+	if (is_ro) {
+		ref->is_ro = true;
+		get_dma_buf(h->dmabuf_ro);
+	} else {
+		ref->is_ro = false;
+		get_dma_buf(h->dmabuf);
+	}
+
+out:
+	NVMAP_TAG_TRACE(trace_nvmap_duplicate_handle,
+		NVMAP_TP_ARGS_CHR(client, h, ref));
+	return ref;
+}
+
+struct nvmap_handle_ref *nvmap_create_handle_from_id(
+			struct nvmap_client *client, u32 id)
+{
+	struct nvmap_handle *handle;
+	struct nvmap_handle_ref *ref;
+
+	if (WARN_ON(!client))
+		return ERR_PTR(-EINVAL);
+
+	if (is_nvmap_id_ro(client, id))
+		return nvmap_dup_handle_ro(client, id);
+
+	handle = nvmap_handle_get_from_id(client, id);
+	if (IS_ERR_OR_NULL(handle)) {
+		/* fd might be dmabuf fd received from parent process.
+		 * Its entry is not made in id_array.
+		 */
+		handle = nvmap_handle_get_from_dmabuf_fd(client, id);
+		if (IS_ERR(handle))
+			return ERR_CAST(handle);
+	}
+
+	ref = nvmap_duplicate_handle(client, handle, false, false);
+	nvmap_handle_put(handle);
+	return ref;
+}
+
+struct nvmap_handle_ref *nvmap_create_handle_from_fd(
+			struct nvmap_client *client, int fd)
+{
+	struct nvmap_handle *handle;
+	struct nvmap_handle_ref *ref;
+	bool is_ro;
+
+	if (WARN_ON(!client))
+		return ERR_PTR(-EINVAL);
+
+	handle = nvmap_handle_get_from_dmabuf_fd(client, fd);
+	if (IS_ERR(handle))
+		return ERR_CAST(handle);
+
+	is_ro = is_nvmap_dmabuf_fd_ro(fd);
+	if (is_ro)
+		ref = nvmap_duplicate_handle(client, handle, false, true);
+	else
+		ref = nvmap_duplicate_handle(client, handle, false, false);
+
+	nvmap_handle_put(handle);
+	return ref;
+}
+
+struct nvmap_handle_ref *nvmap_dup_handle_ro(struct nvmap_client *client,
+					     int id)
+{
+	struct nvmap_handle *h;
+	struct nvmap_handle_ref *ref = NULL;
+	bool dmabuf_created = false;
+
+	if (!client)
+		return ERR_PTR(-EINVAL);
+
+	h = nvmap_handle_get_from_id(client, id);
+	if (IS_ERR_OR_NULL(h)) {
+		/* fd might be dmabuf fd received from parent process.
+		 * Its entry is not made in id_array.
+		 */
+		h = nvmap_handle_get_from_dmabuf_fd(client, id);
+		if (IS_ERR(h))
+			return ERR_CAST(h);
+	}
+
+	if (h->dmabuf_ro == NULL) {
+		h->dmabuf_ro = __nvmap_make_dmabuf(client, h, true);
+		if (IS_ERR(h->dmabuf_ro)) {
+			nvmap_handle_put(h);
+			return ERR_CAST(h->dmabuf_ro);
+		}
+		dmabuf_created = true;
+	}
+
+	ref = nvmap_duplicate_handle(client, h, false, true);
+	if (!ref) {
+		nvmap_handle_put(h);
+		return ref;
+	}
+	/*
+	 * When new dmabuf created (only RO dmabuf is getting created in this function)
+	 * it's counter is incremented one extra time in nvmap_duplicate_handle. Hence
+	 * decrement it by one.
+	 */
+	if (dmabuf_created)
+		dma_buf_put(h->dmabuf_ro);
+
+	nvmap_handle_put(h);
+
+	return ref;
+}

drivers/video/tegra/nvmap/nvmap_heap.c

@@ -0,0 +1,607 @@
+/*
+ * drivers/video/tegra/nvmap/nvmap_heap.c
+ *
+ * GPU heap allocator.
+ *
+ * Copyright (c) 2011-2022, NVIDIA Corporation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#define pr_fmt(fmt)	"%s: " fmt, __func__
+
+#include <linux/debugfs.h>
+#include <linux/device.h>
+#include <linux/kernel.h>
+#include <linux/list.h>
+#include <linux/mm.h>
+#include <linux/mutex.h>
+#include <linux/slab.h>
+#include <linux/err.h>
+#include <linux/bug.h>
+#include <linux/stat.h>
+#include <linux/sizes.h>
+#include <linux/io.h>
+#include <linux/version.h>
+#include <linux/limits.h>
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)
+#include <linux/sched/clock.h>
+#endif
+
+#include <linux/nvmap.h>
+#include <linux/dma-mapping.h>
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0)
+#include <linux/dma-map-ops.h>
+#else
+#include <linux/dma-contiguous.h>
+#endif
+
+#include "nvmap_priv.h"
+#include "nvmap_heap.h"
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0)
+#include "include/linux/nvmap_exports.h"
+#endif
+
+/*
+ * "carveouts" are platform-defined regions of physically contiguous memory
+ * which are not managed by the OS. A platform may specify multiple carveouts,
+ * for either small special-purpose memory regions or reserved regions of main
+ * system memory.
+ *
+ * The carveout allocator returns allocations which are physically contiguous.
+ */
+
+static struct kmem_cache *heap_block_cache;
+
+struct list_block {
+	struct nvmap_heap_block block;
+	struct list_head all_list;
+	unsigned int mem_prot;
+	phys_addr_t orig_addr;
+	size_t size;
+	size_t align;
+	struct nvmap_heap *heap;
+	struct list_head free_list;
+};
+
+struct device *dma_dev_from_handle(unsigned long type)
+{
+	int i;
+	struct nvmap_carveout_node *co_heap;
+
+	for (i = 0; i < nvmap_dev->nr_carveouts; i++) {
+		co_heap = &nvmap_dev->heaps[i];
+
+		if (!(co_heap->heap_bit & type))
+			continue;
+
+		return co_heap->carveout->dma_dev;
+	}
+	return ERR_PTR(-ENODEV);
+}
+
+int nvmap_query_heap_peer(struct nvmap_heap *heap, unsigned int *peer)
+{
+	if (!heap || !heap->is_ivm)
+		return -EINVAL;
+	*peer = heap->peer;
+	return 0;
+}
+
+size_t nvmap_query_heap_size(struct nvmap_heap *heap)
+{
+	if (!heap)
+		return 0;
+
+	return heap->len;
+}
+
+void nvmap_heap_debugfs_init(struct dentry *heap_root, struct nvmap_heap *heap)
+{
+	if (sizeof(heap->base) == sizeof(u64))
+		debugfs_create_x64("base", S_IRUGO,
+			heap_root, (u64 *)&heap->base);
+	else
+		debugfs_create_x32("base", S_IRUGO,
+			heap_root, (u32 *)&heap->base);
+	if (sizeof(heap->len) == sizeof(u64))
+		debugfs_create_x64("size", S_IRUGO,
+			heap_root, (u64 *)&heap->len);
+	else
+		debugfs_create_x32("size", S_IRUGO,
+			heap_root, (u32 *)&heap->len);
+	if (sizeof(heap->free_size) == sizeof(u64))
+		debugfs_create_x64("free_size", S_IRUGO,
+			heap_root, (u64 *)&heap->free_size);
+	else
+		debugfs_create_x32("free_size", S_IRUGO,
+			heap_root, (u32 *)&heap->free_size);
+}
+
+static phys_addr_t nvmap_alloc_mem(struct nvmap_heap *h, size_t len,
+				   phys_addr_t *start)
+{
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0)
+	phys_addr_t pa = DMA_ERROR_CODE;
+#else
+	phys_addr_t pa = DMA_MAPPING_ERROR;
+#endif
+	struct device *dev = h->dma_dev;
+
+	if (len > UINT_MAX) {
+		dev_err(dev, "%s: %d alloc size is out of range\n",
+			__func__, __LINE__);
+			return DMA_ERROR_CODE;
+	}
+
+#ifdef CONFIG_TEGRA_VIRTUALIZATION
+	if (start && h->is_ivm) {
+		void *ret;
+		pa = h->base + (*start);
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0)
+		ret = dma_mark_declared_memory_occupied(dev, pa, len,
+					DMA_ATTR_ALLOC_EXACT_SIZE);
+#else
+		ret = nvmap_dma_mark_declared_memory_occupied(dev, pa, len);
+#endif
+		if (IS_ERR(ret)) {
+			dev_err(dev, "Failed to reserve (%pa) len(%zu)\n",
+					&pa, len);
+			return DMA_ERROR_CODE;
+		}
+	} else
+#endif
+	{
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0)
+		(void)dma_alloc_attrs(dev, len, &pa,
+				GFP_KERNEL, DMA_ATTR_ALLOC_EXACT_SIZE);
+#else
+		(void)nvmap_dma_alloc_attrs(dev, len, &pa,
+				GFP_KERNEL, DMA_ATTR_ALLOC_EXACT_SIZE);
+#endif
+		if (!dma_mapping_error(dev, pa)) {
+#ifdef NVMAP_CONFIG_VPR_RESIZE
+			int ret;
+
+			dev_dbg(dev, "Allocated addr (%pa) len(%zu)\n",
+					&pa, len);
+			if (!dma_is_coherent_dev(dev) && h->cma_dev) {
+				ret = nvmap_cache_maint_phys_range(
+					NVMAP_CACHE_OP_WB, pa, pa + len,
+					true, true);
+				if (!ret)
+					return pa;
+
+				dev_err(dev, "cache WB on (%pa, %zu) failed\n",
+					&pa, len);
+			}
+#endif
+			dev_dbg(dev, "Allocated addr (%pa) len(%zu)\n",
+					&pa, len);
+		}
+	}
+
+	return pa;
+}
+
+static void nvmap_free_mem(struct nvmap_heap *h, phys_addr_t base,
+				size_t len)
+{
+	struct device *dev = h->dma_dev;
+
+	dev_dbg(dev, "Free base (%pa) size (%zu)\n", &base, len);
+
+	if (len > UINT_MAX) {
+		dev_err(dev, "%s: %d freeing length out of range\n",
+			__func__, __LINE__);
+			return;
+	}
+
+#ifdef CONFIG_TEGRA_VIRTUALIZATION
+	if (h->is_ivm && !h->can_alloc) {
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0)
+		dma_mark_declared_memory_unoccupied(dev, base, len,
+						    DMA_ATTR_ALLOC_EXACT_SIZE);
+#else
+		nvmap_dma_mark_declared_memory_unoccupied(dev, base, len);
+#endif
+	} else
+#endif
+	{
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0)
+		dma_free_attrs(dev, len,
+			        (void *)(uintptr_t)base,
+			        (dma_addr_t)base, DMA_ATTR_ALLOC_EXACT_SIZE);
+#else
+		nvmap_dma_free_attrs(dev, len,
+				     (void *)(uintptr_t)base,
+				     (dma_addr_t)base,
+				     DMA_ATTR_ALLOC_EXACT_SIZE);
+#endif
+	}
+}
+
+/*
+ * base_max limits position of allocated chunk in memory.
+ * if base_max is 0 then there is no such limitation.
+ */
+static struct nvmap_heap_block *do_heap_alloc(struct nvmap_heap *heap,
+					      size_t len, size_t align,
+					      unsigned int mem_prot,
+					      phys_addr_t base_max,
+					      phys_addr_t *start)
+{
+	struct list_block *heap_block = NULL;
+	dma_addr_t dev_base;
+	struct device *dev = heap->dma_dev;
+
+	/* since pages are only mappable with one cache attribute,
+	 * and most allocations from carveout heaps are DMA coherent
+	 * (i.e., non-cacheable), round cacheable allocations up to
+	 * a page boundary to ensure that the physical pages will
+	 * only be mapped one way. */
+	if (mem_prot == NVMAP_HANDLE_CACHEABLE ||
+	    mem_prot == NVMAP_HANDLE_INNER_CACHEABLE) {
+		align = max_t(size_t, align, PAGE_SIZE);
+		len = PAGE_ALIGN(len);
+	}
+
+	if (heap->is_ivm)
+		align = max_t(size_t, align, NVMAP_IVM_ALIGNMENT);
+
+	heap_block = kmem_cache_zalloc(heap_block_cache, GFP_KERNEL);
+	if (!heap_block) {
+		dev_err(dev, "%s: failed to alloc heap block %s\n",
+			__func__, dev_name(dev));
+		goto fail_heap_block_alloc;
+	}
+
+	dev_base = nvmap_alloc_mem(heap, len, start);
+	if (dma_mapping_error(dev, dev_base)) {
+		dev_err(dev, "failed to alloc mem of size (%zu)\n",
+			len);
+#ifdef NVMAP_CONFIG_VPR_RESIZE
+		if (dma_is_coherent_dev(dev)) {
+			struct dma_coherent_stats stats;
+
+			dma_get_coherent_stats(dev, &stats);
+			dev_err(dev, "used:%zu,curr_size:%zu max:%zu\n",
+				stats.used, stats.size, stats.max);
+		}
+#endif
+		goto fail_dma_alloc;
+	}
+
+	heap_block->block.base = dev_base;
+	heap_block->orig_addr = dev_base;
+	heap_block->size = len;
+
+	list_add_tail(&heap_block->all_list, &heap->all_list);
+	heap_block->heap = heap;
+	heap->free_size -= len;
+	heap_block->mem_prot = mem_prot;
+	heap_block->align = align;
+	return &heap_block->block;
+
+fail_dma_alloc:
+	kmem_cache_free(heap_block_cache, heap_block);
+fail_heap_block_alloc:
+	return NULL;
+}
+
+static void do_heap_free(struct nvmap_heap_block *block)
+{
+	struct list_block *b = container_of(block, struct list_block, block);
+	struct nvmap_heap *heap = b->heap;
+
+	list_del(&b->all_list);
+
+	heap->free_size += b->size;
+	nvmap_free_mem(heap, block->base, b->size);
+	kmem_cache_free(heap_block_cache, b);
+}
+
+/* nvmap_heap_alloc: allocates a block of memory of len bytes, aligned to
+ * align bytes. */
+struct nvmap_heap_block *nvmap_heap_alloc(struct nvmap_heap *h,
+					  struct nvmap_handle *handle,
+					  phys_addr_t *start)
+{
+	struct nvmap_heap_block *b;
+	size_t len        = handle->size;
+	size_t align      = handle->align;
+	unsigned int prot = handle->flags;
+
+	mutex_lock(&h->lock);
+
+	if (h->is_ivm) { /* Is IVM carveout? */
+		/* Check if this correct IVM heap */
+		if (handle->peer != h->peer) {
+			mutex_unlock(&h->lock);
+			return NULL;
+		} else {
+			if (h->can_alloc && start) {
+				/* If this partition does actual allocation, it
+				 * should not specify start_offset.
+				 */
+				mutex_unlock(&h->lock);
+				return NULL;
+			} else if (!h->can_alloc && !start) {
+				/* If this partition does not do actual
+				 * allocation, it should specify start_offset.
+				 */
+				mutex_unlock(&h->lock);
+				return NULL;
+			}
+		}
+	}
+
+	/*
+	 * If this HEAP has pm_ops defined and powering on the
+	 * RAM attached with the HEAP returns error, don't
+	 * allocate from the heap and return NULL.
+	 */
+	if (h->pm_ops.busy) {
+		if (h->pm_ops.busy() < 0) {
+			pr_err("Unable to power on the heap device\n");
+			mutex_unlock(&h->lock);
+			return NULL;
+		}
+	}
+
+	align = max_t(size_t, align, L1_CACHE_BYTES);
+	b = do_heap_alloc(h, len, align, prot, 0, start);
+	if (b) {
+		b->handle = handle;
+		handle->carveout = b;
+		/* Generate IVM for partition that can alloc */
+		if (h->is_ivm && h->can_alloc) {
+			unsigned int offs = (b->base - h->base);
+
+			BUG_ON(offs & (NVMAP_IVM_ALIGNMENT - 1));
+			BUG_ON((offs >> ffs(NVMAP_IVM_ALIGNMENT)) &
+				~((1 << NVMAP_IVM_OFFSET_WIDTH) - 1));
+			BUG_ON(h->vm_id & ~(NVMAP_IVM_IVMID_MASK));
+			/* So, page alignment is sufficient check.
+			 */
+			BUG_ON(len & ~(PAGE_MASK));
+
+			/* Copy offset from IVM mem pool in nvmap handle.
+			 * The offset will be exported via ioctl.
+			 */
+			handle->offs = offs;
+
+			handle->ivm_id = ((u64)h->vm_id << NVMAP_IVM_IVMID_SHIFT);
+			handle->ivm_id |= (((offs >> (ffs(NVMAP_IVM_ALIGNMENT) - 1)) &
+					 ((1ULL << NVMAP_IVM_OFFSET_WIDTH) - 1)) <<
+					  NVMAP_IVM_OFFSET_SHIFT);
+			handle->ivm_id |= (len >> PAGE_SHIFT);
+		}
+	}
+	mutex_unlock(&h->lock);
+	return b;
+}
+
+struct nvmap_heap *nvmap_block_to_heap(struct nvmap_heap_block *b)
+{
+	struct list_block *lb;
+	lb = container_of(b, struct list_block, block);
+	return lb->heap;
+}
+
+/* nvmap_heap_free: frees block b*/
+void nvmap_heap_free(struct nvmap_heap_block *b)
+{
+	struct nvmap_heap *h;
+	struct list_block *lb;
+
+	if (!b)
+		return;
+
+	h = nvmap_block_to_heap(b);
+	mutex_lock(&h->lock);
+
+	lb = container_of(b, struct list_block, block);
+	nvmap_flush_heap_block(NULL, b, lb->size, lb->mem_prot);
+	do_heap_free(b);
+	/*
+	 * If this HEAP has pm_ops defined and powering off the
+	 * RAM attached with the HEAP returns error, raise warning.
+	 */
+	if (h->pm_ops.idle) {
+		if (h->pm_ops.idle() < 0)
+			WARN_ON(1);
+	}
+
+	mutex_unlock(&h->lock);
+}
+
+/* nvmap_heap_create: create a heap object of len bytes, starting from
+ * address base.
+ */
+struct nvmap_heap *nvmap_heap_create(struct device *parent,
+				     const struct nvmap_platform_carveout *co,
+				     phys_addr_t base, size_t len, void *arg)
+{
+	struct nvmap_heap *h;
+
+	h = kzalloc(sizeof(*h), GFP_KERNEL);
+	if (!h) {
+		dev_err(parent, "%s: out of memory\n", __func__);
+		return NULL;
+	}
+
+	h->dma_dev = co->dma_dev;
+	if (co->cma_dev) {
+#ifdef CONFIG_DMA_CMA
+#ifdef NVMAP_CONFIG_VPR_RESIZE
+		struct dma_contiguous_stats stats;
+
+		if (dma_get_contiguous_stats(co->cma_dev, &stats))
+			goto fail;
+
+		base = stats.base;
+		len = stats.size;
+		h->cma_dev = co->cma_dev;
+#endif
+#else
+		dev_err(parent, "invalid resize config for carveout %s\n",
+				co->name);
+		goto fail;
+#endif
+	} else if (!co->init_done) {
+		int err;
+
+		/* declare Non-CMA heap */
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0)
+		err = dma_declare_coherent_memory(h->dma_dev, 0, base, len,
+				DMA_MEMORY_NOMAP);
+#else
+		err = nvmap_dma_declare_coherent_memory(h->dma_dev, 0, base, len,
+				DMA_MEMORY_NOMAP);
+#endif
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)
+		if (!err) {
+#else
+		if (err & DMA_MEMORY_NOMAP) {
+#endif
+			dev_info(parent,
+				"%s :dma coherent mem declare %pa,%zu\n",
+				co->name, &base, len);
+		} else {
+			dev_err(parent,
+				"%s: dma coherent declare fail %pa,%zu\n",
+				co->name, &base, len);
+			goto fail;
+		}
+	}
+
+	dev_set_name(h->dma_dev, "%s", co->name);
+	dma_set_coherent_mask(h->dma_dev, DMA_BIT_MASK(64));
+	h->name = co->name;
+	h->arg = arg;
+	h->base = base;
+	h->can_alloc = !!co->can_alloc;
+	h->is_ivm = co->is_ivm;
+	h->len = len;
+	h->free_size = len;
+	h->peer = co->peer;
+	h->vm_id = co->vmid;
+	if (co->pm_ops.busy)
+		h->pm_ops.busy = co->pm_ops.busy;
+
+	if (co->pm_ops.idle)
+		h->pm_ops.idle = co->pm_ops.idle;
+
+	INIT_LIST_HEAD(&h->all_list);
+	mutex_init(&h->lock);
+#ifdef NVMAP_CONFIG_DEBUG_MAPS
+	h->device_names = RB_ROOT;
+#endif /* NVMAP_CONFIG_DEBUG_MAPS */
+	if (!co->no_cpu_access && co->usage_mask != NVMAP_HEAP_CARVEOUT_VPR
+		&& nvmap_cache_maint_phys_range(NVMAP_CACHE_OP_WB_INV,
+				base, base + len, true, true)) {
+		dev_err(parent, "cache flush failed\n");
+		goto fail;
+	}
+	wmb();
+
+	if (co->disable_dynamic_dma_map)
+		nvmap_dev->dynamic_dma_map_mask &= ~co->usage_mask;
+
+	if (co->no_cpu_access)
+		nvmap_dev->cpu_access_mask &= ~co->usage_mask;
+
+	dev_info(parent, "created heap %s base 0x%px size (%zuKiB)\n",
+		co->name, (void *)(uintptr_t)base, len/1024);
+	return h;
+fail:
+	kfree(h);
+	return NULL;
+}
+
+/* nvmap_heap_destroy: frees all resources in heap */
+void nvmap_heap_destroy(struct nvmap_heap *heap)
+{
+	WARN_ON(!list_empty(&heap->all_list));
+	while (!list_empty(&heap->all_list)) {
+		struct list_block *l;
+		l = list_first_entry(&heap->all_list, struct list_block,
+				     all_list);
+		list_del(&l->all_list);
+		kmem_cache_free(heap_block_cache, l);
+	}
+	kfree(heap);
+}
+
+int nvmap_heap_init(void)
+{
+	ulong start_time = sched_clock();
+
+	heap_block_cache = KMEM_CACHE(list_block, 0);
+	if (!heap_block_cache) {
+		pr_err("%s: unable to create heap block cache\n", __func__);
+		return -ENOMEM;
+	}
+	pr_info("%s: created heap block cache\n", __func__);
+	nvmap_init_time += sched_clock() - start_time;
+	return 0;
+}
+
+void nvmap_heap_deinit(void)
+{
+	if (heap_block_cache)
+		kmem_cache_destroy(heap_block_cache);
+
+	heap_block_cache = NULL;
+}
+
+/*
+ * This routine is used to flush the carveout memory from cache.
+ * Why cache flush is needed for carveout? Consider the case, where a piece of
+ * carveout is allocated as cached and released. After this, if the same memory is
+ * allocated for uncached request and the memory is not flushed out from cache.
+ * In this case, the client might pass this to H/W engine and it could start modify
+ * the memory. As this was cached earlier, it might have some portion of it in cache.
+ * During cpu request to read/write other memory, the cached portion of this memory
+ * might get flushed back to main memory and would cause corruptions, if it happens
+ * after H/W writes data to memory.
+ *
+ * But flushing out the memory blindly on each carveout allocation is redundant.
+ *
+ * In order to optimize the carveout buffer cache flushes, the following
+ * strategy is used.
+ *
+ * The whole Carveout is flushed out from cache during its initialization.
+ * During allocation, carveout buffers are not flused from cache.
+ * During deallocation, carveout buffers are flushed, if they were allocated as cached.
+ * if they were allocated as uncached/writecombined, no cache flush is needed.
+ * Just draining store buffers is enough.
+ */
+int nvmap_flush_heap_block(struct nvmap_client *client,
+	struct nvmap_heap_block *block, size_t len, unsigned int prot)
+{
+	phys_addr_t phys = block->base;
+	phys_addr_t end = block->base + len;
+	int ret = 0;
+
+	if (prot == NVMAP_HANDLE_UNCACHEABLE || prot == NVMAP_HANDLE_WRITE_COMBINE)
+		goto out;
+
+	ret = nvmap_cache_maint_phys_range(NVMAP_CACHE_OP_WB_INV, phys, end,
+				true, prot != NVMAP_HANDLE_INNER_CACHEABLE);
+	if (ret)
+		goto out;
+out:
+	wmb();
+	return ret;
+}

drivers/video/tegra/nvmap/nvmap_heap.h

@@ -0,0 +1,79 @@
+/*
+ * drivers/video/tegra/nvmap/nvmap_heap.h
+ *
+ * GPU heap allocator.
+ *
+ * Copyright (c) 2010-2022, NVIDIA Corporation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#ifndef __NVMAP_HEAP_H
+#define __NVMAP_HEAP_H
+
+struct device;
+struct nvmap_heap;
+struct nvmap_client;
+
+struct nvmap_heap_block {
+	phys_addr_t	base;
+	unsigned int	type;
+	struct nvmap_handle *handle;
+};
+
+struct nvmap_heap {
+	struct list_head all_list;
+	struct mutex lock;
+	const char *name;
+	void *arg;
+	/* heap base */
+	phys_addr_t base;
+	/* heap size */
+	size_t len;
+	size_t free_size;
+	struct device *cma_dev;
+	struct device *dma_dev;
+	bool is_ivm;
+	bool can_alloc; /* Used only if is_ivm == true */
+	unsigned int peer; /* Used only if is_ivm == true */
+	unsigned int vm_id; /* Used only if is_ivm == true */
+	struct nvmap_pm_ops pm_ops;
+#ifdef NVMAP_CONFIG_DEBUG_MAPS
+	struct rb_root device_names;
+#endif /* NVMAP_CONFIG_DEBUG_MAPS */
+};
+
+struct nvmap_heap *nvmap_heap_create(struct device *parent,
+				     const struct nvmap_platform_carveout *co,
+				     phys_addr_t base, size_t len, void *arg);
+
+void nvmap_heap_destroy(struct nvmap_heap *heap);
+
+struct nvmap_heap_block *nvmap_heap_alloc(struct nvmap_heap *heap,
+					  struct nvmap_handle *handle,
+					  phys_addr_t *start);
+
+struct nvmap_heap *nvmap_block_to_heap(struct nvmap_heap_block *b);
+
+void nvmap_heap_free(struct nvmap_heap_block *block);
+
+int __init nvmap_heap_init(void);
+
+void nvmap_heap_deinit(void);
+
+int nvmap_flush_heap_block(struct nvmap_client *client,
+	struct nvmap_heap_block *block, size_t len, unsigned int prot);
+
+void nvmap_heap_debugfs_init(struct dentry *heap_root, struct nvmap_heap *heap);
+
+int nvmap_query_heap_peer(struct nvmap_heap *heap, unsigned int *peer);
+size_t nvmap_query_heap_size(struct nvmap_heap *heap);
+
+#endif

drivers/video/tegra/nvmap/nvmap_id_array.c

@@ -0,0 +1,62 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2021-2022 NVIDIA Corporation
+ */
+
+#include <linux/xarray.h>
+#include <linux/dma-buf.h>
+#include "nvmap_priv.h"
+
+#define XA_START (U32_MAX / 2)
+/*
+ * Initialize xarray mapping
+ */
+void nvmap_id_array_init(struct xarray *id_arr)
+{
+	xa_init_flags(id_arr, XA_FLAGS_ALLOC1);
+}
+
+/*
+ * Remove id to dma_buf mapping
+ */
+void nvmap_id_array_exit(struct xarray *id_arr)
+{
+	xa_destroy(id_arr);
+}
+
+/*
+ * Create mapping between the id(NvRmMemHandle) and dma_buf
+ */
+int nvmap_id_array_id_alloc(struct xarray *id_arr, u32 *id, struct dma_buf *dmabuf)
+{
+	if (!id_arr || !dmabuf)
+		return -EINVAL;
+
+	return xa_alloc(id_arr, id, dmabuf,
+		       XA_LIMIT(XA_START, U32_MAX), GFP_KERNEL);
+}
+
+/*
+ * Clear mapping between the id(NvRmMemHandle) and dma_buf
+ */
+struct dma_buf *nvmap_id_array_id_release(struct xarray *id_arr, u32 id)
+{
+	if (!id_arr || !id)
+		return NULL;
+
+	return xa_erase(id_arr, id);
+}
+
+/*
+ * Return dma_buf from the id.
+ */
+struct dma_buf *nvmap_id_array_get_dmabuf_from_id(struct xarray *id_arr, u32 id)
+{
+	struct dma_buf *dmabuf;
+
+	dmabuf = xa_load(id_arr, id);
+	if (!IS_ERR_OR_NULL(dmabuf))
+		get_dma_buf(dmabuf);
+
+	return dmabuf;
+}

drivers/video/tegra/nvmap/nvmap_init.c

@@ -0,0 +1,979 @@
+/*
+ * Copyright (c) 2014-2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * Portions derived from Linux kernel source file kernel/dma/coherent.c
+ */
+
+#define pr_fmt(fmt) "%s: " fmt, __func__
+
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_fdt.h>
+#include <linux/of_platform.h>
+#include <linux/nvmap.h>
+#include <linux/version.h>
+#include <linux/kmemleak.h>
+#include <linux/io.h>
+
+#if defined(NVMAP_LOADABLE_MODULE)
+#include <linux/nvmap_t19x.h>
+#endif
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)
+#include <linux/sched/clock.h>
+#endif
+
+#include <linux/cma.h>
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0)
+#include <linux/dma-map-ops.h>
+#else
+#include <linux/dma-contiguous.h>
+#include <asm/dma-contiguous.h>
+#endif
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0)
+#include "include/linux/nvmap_exports.h"
+#endif
+
+#include "nvmap_priv.h"
+
+#ifdef CONFIG_TEGRA_VIRTUALIZATION
+#include <linux/tegra-ivc.h>
+#include <soc/tegra/virt/syscalls.h>
+#endif
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0)
+#ifdef CONFIG_ARM_DMA_IOMMU_ALIGNMENT
+#define DMA_BUF_ALIGNMENT CONFIG_ARM_DMA_IOMMU_ALIGNMENT
+#else
+#define DMA_BUF_ALIGNMENT 8
+#endif
+#endif /* LINUX_VERSION_CODE */
+
+#ifndef NVMAP_UPSTREAM_KERNEL
+#ifndef NVMAP_CONFIG_VPR_RESIZE
+extern phys_addr_t tegra_vpr_start;
+extern phys_addr_t tegra_vpr_size;
+extern bool tegra_vpr_resize;
+#endif /* NVMAP_CONFIG_VPR_RESIZE */
+#endif /* !NVMAP_UPSTREAM_KERNEL */
+
+struct device __weak tegra_generic_dev;
+
+struct device __weak tegra_vpr_dev;
+EXPORT_SYMBOL(tegra_vpr_dev);
+
+struct device __weak tegra_generic_cma_dev;
+struct device __weak tegra_vpr_cma_dev;
+
+#ifdef NVMAP_LOADABLE_MODULE
+static struct platform_device *pdev;
+#endif /* NVMAP_LOADABLE_MODULE */
+
+#ifdef NVMAP_CONFIG_VPR_RESIZE
+struct dma_resize_notifier_ops __weak vpr_dev_ops;
+
+static struct dma_declare_info generic_dma_info = {
+        .name = "generic",
+        .size = 0,
+        .notifier.ops = NULL,
+};
+
+static struct dma_declare_info vpr_dma_info = {
+	.name = "vpr",
+	.size = SZ_32M,
+	.notifier.ops = &vpr_dev_ops,
+};
+#endif
+
+__weak const struct of_device_id nvmap_of_ids[] = {
+        { .compatible = "nvidia,carveouts" },
+        { .compatible = "nvidia,carveouts-t18x" },
+        { }
+};
+
+static struct nvmap_platform_carveout nvmap_carveouts[] = {
+	[0] = {
+		.name		= "generic-0",
+		.usage_mask	= NVMAP_HEAP_CARVEOUT_GENERIC,
+		.base		= 0,
+		.size		= 0,
+		.dma_dev	= &tegra_generic_dev,
+		.cma_dev	= &tegra_generic_cma_dev,
+#ifdef NVMAP_CONFIG_VPR_RESIZE
+		.dma_info	= &generic_dma_info,
+#endif
+	},
+	[1] = {
+		.name		= "vpr",
+		.usage_mask	= NVMAP_HEAP_CARVEOUT_VPR,
+		.base		= 0,
+		.size		= 0,
+		.dma_dev	= &tegra_vpr_dev,
+		.cma_dev	= &tegra_vpr_cma_dev,
+#ifdef NVMAP_CONFIG_VPR_RESIZE
+		.dma_info	= &vpr_dma_info,
+#endif
+		.enable_static_dma_map = true,
+	},
+	[2] = {
+		.name		= "vidmem",
+		.usage_mask	= NVMAP_HEAP_CARVEOUT_VIDMEM,
+		.base		= 0,
+		.size		= 0,
+		.disable_dynamic_dma_map = true,
+		.no_cpu_access = true,
+	},
+	[3] = {
+		.name		= "fsi",
+		.usage_mask	= NVMAP_HEAP_CARVEOUT_FSI,
+		.base		= 0,
+		.size		= 0,
+	},
+	/* Need uninitialized entries for IVM carveouts */
+	[4] = {
+		.name		= NULL,
+		.usage_mask	= NVMAP_HEAP_CARVEOUT_IVM,
+	},
+	[5] = {
+		.name		= NULL,
+		.usage_mask	= NVMAP_HEAP_CARVEOUT_IVM,
+	},
+	[6] = {
+		.name		= NULL,
+		.usage_mask	= NVMAP_HEAP_CARVEOUT_IVM,
+	},
+	[7] = {
+		.name		= NULL,
+		.usage_mask	= NVMAP_HEAP_CARVEOUT_IVM,
+	},
+};
+
+static struct nvmap_platform_data nvmap_data = {
+	.carveouts	= nvmap_carveouts,
+	.nr_carveouts	= 4,
+};
+
+static struct nvmap_platform_carveout *nvmap_get_carveout_pdata(const char *name)
+{
+	struct nvmap_platform_carveout *co;
+	for (co = nvmap_carveouts;
+	     co < nvmap_carveouts + ARRAY_SIZE(nvmap_carveouts); co++) {
+		int i = min_t(int, strcspn(name, "_"), strcspn(name, "-"));
+		/* handle IVM carveouts */
+		if ((co->usage_mask == NVMAP_HEAP_CARVEOUT_IVM) &&  !co->name)
+			goto found;
+
+		if (strncmp(co->name, name, i))
+			continue;
+found:
+		co->dma_dev = co->dma_dev ? co->dma_dev : &co->dev;
+		return co;
+	}
+	pr_err("not enough space for all nvmap carveouts\n");
+	return NULL;
+}
+
+int nvmap_register_vidmem_carveout(struct device *dma_dev,
+				phys_addr_t base, size_t size)
+{
+	struct nvmap_platform_carveout *vidmem_co;
+
+	if (!base || !size || (base != PAGE_ALIGN(base)) ||
+	    (size != PAGE_ALIGN(size)))
+		return -EINVAL;
+
+	vidmem_co = nvmap_get_carveout_pdata("vidmem");
+	if (!vidmem_co)
+		return -ENODEV;
+
+	if (vidmem_co->base || vidmem_co->size)
+		return -EEXIST;
+
+	vidmem_co->base = base;
+	vidmem_co->size = size;
+	if (dma_dev)
+		vidmem_co->dma_dev = dma_dev;
+	return nvmap_create_carveout(vidmem_co);
+}
+EXPORT_SYMBOL(nvmap_register_vidmem_carveout);
+
+#ifdef CONFIG_TEGRA_VIRTUALIZATION
+static int __init nvmap_populate_ivm_carveout(struct device *dev)
+{
+	char *name;
+	const __be32 *prop;
+	int ret = 0;
+	struct nvmap_platform_carveout *co;
+	struct of_phandle_iterator it;
+	struct tegra_hv_ivm_cookie *ivm;
+	unsigned long long id;
+	unsigned int guestid;
+
+	if (!of_phandle_iterator_init(&it, dev->of_node, "memory-region", NULL, 0)) {
+		while (!of_phandle_iterator_next(&it) && it.node) {
+			if (of_device_is_available(it.node) &&
+			    of_device_is_compatible(it.node, "nvidia,ivm_carveout") > 0) {
+				co = nvmap_get_carveout_pdata("nvidia,ivm_carveout");
+				if (!co) {
+					ret = -ENOMEM;
+					goto err;
+				}
+
+				if (hyp_read_gid(&guestid)) {
+					pr_err("failed to read gid\n");
+					ret = -EINVAL;
+					goto err;
+				}
+
+				prop = of_get_property(it.node, "ivm", NULL);
+				if (!prop) {
+					pr_err("failed to read ivm property\n");
+					ret = -EINVAL;
+					goto err;
+				}
+
+				id = of_read_number(prop + 1, 1);
+				if (id > UINT_MAX) {
+					ret = -EINVAL;
+					goto err;
+				}
+				ivm = tegra_hv_mempool_reserve(id);
+				if (IS_ERR_OR_NULL(ivm)) {
+					pr_err("failed to reserve IVM memory pool %llu\n", id);
+					ret = -ENOMEM;
+					goto err;
+				}
+				/* XXX: Are these the available fields from IVM cookie? */
+				co->base     = (phys_addr_t)ivm->ipa;
+				co->peer     = ivm->peer_vmid;
+				co->size     = ivm->size;
+				co->vmid     = guestid;
+
+				if (!co->base || !co->size) {
+					ret = -EINVAL;
+					goto fail;
+				}
+
+				/* See if this VM can allocate (or just create handle from ID)
+				 * generated by peer partition
+				 */
+				prop = of_get_property(it.node, "alloc", NULL);
+				if (!prop) {
+					pr_err("failed to read alloc property\n");
+					ret = -EINVAL;
+					goto fail;
+				}
+
+				name = kzalloc(32, GFP_KERNEL);
+				if (!name) {
+					ret = -ENOMEM;
+					goto fail;
+				}
+
+				co->can_alloc = of_read_number(prop, 1);
+				co->is_ivm    = true;
+				sprintf(name, "ivm%02u%02u%02d", co->vmid, co->peer, co->can_alloc);
+				pr_info("IVM carveout IPA:%p, size=%zu, peer vmid=%u, name=%s\n",
+					(void *)(uintptr_t)co->base, co->size, co->peer, name);
+				co->name      = name;
+				nvmap_data.nr_carveouts++;
+
+			}
+		}
+	}
+	return 0;
+fail:
+	co->base     = 0;
+	co->peer     = 0;
+	co->size     = 0;
+	co->vmid     = 0;
+err:
+	return ret;
+
+}
+#endif /* CONFIG_TEGRA_VIRTUALIZATION */
+
+/*
+ * This requires proper kernel arguments to have been passed.
+ */
+#ifndef NVMAP_UPSTREAM_KERNEL
+static int __nvmap_init_legacy(struct device *dev)
+{
+#ifndef NVMAP_CONFIG_VPR_RESIZE
+	/* VPR */
+	if (!nvmap_carveouts[1].base) {
+		nvmap_carveouts[1].base = tegra_vpr_start;
+		nvmap_carveouts[1].size = tegra_vpr_size;
+		nvmap_carveouts[1].cma_dev = NULL;
+	}
+#endif /* NVMAP_CONFIG_VPR_RESIZE */
+
+	return 0;
+}
+#endif /* !NVMAP_UPSTREAM_KERNEL */
+
+static int __nvmap_init_dt(struct platform_device *pdev)
+{
+	if (!of_match_device(nvmap_of_ids, &pdev->dev)) {
+		pr_err("Missing DT entry!\n");
+		return -EINVAL;
+	}
+
+#ifndef NVMAP_UPSTREAM_KERNEL
+	/* For VM_2 we need carveout. So, enabling it here */
+	__nvmap_init_legacy(&pdev->dev);
+#endif /* !NVMAP_UPSTREAM_KERNEL */
+
+	pdev->dev.platform_data = &nvmap_data;
+
+	return 0;
+}
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0)
+static inline struct page **nvmap_kvzalloc_pages(u32 count)
+{
+	if (count * sizeof(struct page *) <= PAGE_SIZE)
+		return kzalloc(count * sizeof(struct page *), GFP_KERNEL);
+	else
+		return vzalloc(count * sizeof(struct page *));
+}
+
+static void *__nvmap_dma_alloc_from_coherent(struct device *dev,
+					     struct dma_coherent_mem_replica *mem,
+					     size_t size,
+					     dma_addr_t *dma_handle,
+					     unsigned long attrs,
+					     unsigned long start)
+{
+	int order = get_order(size);
+	unsigned long flags;
+	unsigned int count, i = 0, j = 0;
+	unsigned int alloc_size;
+	unsigned long align, pageno;
+	void *addr = NULL;
+	struct page **pages = NULL;
+	int do_memset = 0;
+	int *bitmap_nos = NULL;
+
+	if (dma_get_attr(DMA_ATTR_ALLOC_EXACT_SIZE, attrs))
+		count = PAGE_ALIGN(size) >> PAGE_SHIFT;
+	else
+		count = 1 << order;
+
+	if (!count)
+		return NULL;
+
+	bitmap_nos = vzalloc(count * sizeof(int));
+	if (!bitmap_nos) {
+		dev_err(dev, "failed to allocate memory\n");
+		return NULL;
+	}
+	if ((mem->flags & DMA_MEMORY_NOMAP) &&
+	    dma_get_attr(DMA_ATTR_ALLOC_SINGLE_PAGES, attrs)) {
+		alloc_size = 1;
+		pages = nvmap_kvzalloc_pages(count);
+		if (!pages)
+			return NULL;
+	} else {
+		alloc_size = count;
+	}
+
+	spin_lock_irqsave(&mem->spinlock, flags);
+
+	if (unlikely(size > (mem->size << PAGE_SHIFT)))
+		goto err;
+
+	if ((mem->flags & DMA_MEMORY_NOMAP) &&
+	    dma_get_attr(DMA_ATTR_ALLOC_SINGLE_PAGES, attrs)) {
+		align = 0;
+	} else  {
+		if (order > DMA_BUF_ALIGNMENT)
+			align = (1 << DMA_BUF_ALIGNMENT) - 1;
+		else
+			align = (1 << order) - 1;
+	}
+
+	while (count) {
+		pageno = bitmap_find_next_zero_area(mem->bitmap, mem->size,
+						    start, alloc_size, align);
+
+		if (pageno >= mem->size)
+			goto err;
+
+		count -= alloc_size;
+		if (pages)
+			pages[i++] = pfn_to_page(mem->pfn_base + pageno);
+		bitmap_set(mem->bitmap, pageno, alloc_size);
+		bitmap_nos[j++] = pageno;
+	}
+
+	/*
+	 * Memory was found in the coherent area.
+	 */
+	*dma_handle = mem->device_base + (pageno << PAGE_SHIFT);
+	if (!(mem->flags & DMA_MEMORY_NOMAP)) {
+		addr = mem->virt_base + (pageno << PAGE_SHIFT);
+		do_memset = 1;
+	} else if (dma_get_attr(DMA_ATTR_ALLOC_SINGLE_PAGES, attrs)) {
+		addr = pages;
+	}
+
+	spin_unlock_irqrestore(&mem->spinlock, flags);
+
+	if (do_memset)
+		memset(addr, 0, size);
+
+	kvfree(bitmap_nos);
+	return addr;
+err:
+	while (j--)
+		bitmap_clear(mem->bitmap, bitmap_nos[j], alloc_size);
+
+	spin_unlock_irqrestore(&mem->spinlock, flags);
+	kvfree(pages);
+	kvfree(bitmap_nos);
+	return NULL;
+}
+
+void *nvmap_dma_alloc_attrs(struct device *dev, size_t size,
+			    dma_addr_t *dma_handle,
+			    gfp_t flag, unsigned long attrs)
+{
+	struct dma_coherent_mem_replica *mem;
+
+	if (!dev || !dev->dma_mem)
+		return NULL;
+
+	WARN_ON_ONCE(!dev->coherent_dma_mask);
+
+	mem = (struct dma_coherent_mem_replica *)(dev->dma_mem);
+
+	return __nvmap_dma_alloc_from_coherent(dev, mem, size, dma_handle,
+						   attrs, 0);
+}
+EXPORT_SYMBOL(nvmap_dma_alloc_attrs);
+
+void nvmap_dma_free_attrs(struct device *dev, size_t size, void *cpu_addr,
+			  dma_addr_t dma_handle, unsigned long attrs)
+{
+	void *mem_addr;
+	unsigned long flags;
+	unsigned int pageno;
+	struct dma_coherent_mem_replica *mem;
+
+	if (!dev || !dev->dma_mem)
+		return;
+
+	mem = (struct dma_coherent_mem_replica *)(dev->dma_mem);
+	if ((mem->flags & DMA_MEMORY_NOMAP) &&
+	    dma_get_attr(DMA_ATTR_ALLOC_SINGLE_PAGES, attrs)) {
+		struct page **pages = cpu_addr;
+		int i;
+
+		spin_lock_irqsave(&mem->spinlock, flags);
+		for (i = 0; i < (size >> PAGE_SHIFT); i++) {
+			pageno = page_to_pfn(pages[i]) - mem->pfn_base;
+			if (WARN_ONCE(pageno > mem->size,
+				      "invalid pageno:%d\n", pageno))
+				continue;
+			bitmap_clear(mem->bitmap, pageno, 1);
+		}
+		spin_unlock_irqrestore(&mem->spinlock, flags);
+		kvfree(pages);
+		return;
+	}
+
+	if (mem->flags & DMA_MEMORY_NOMAP)
+		mem_addr =  (void *)(uintptr_t)mem->device_base;
+	else
+		mem_addr =  mem->virt_base;
+
+	if (mem && cpu_addr >= mem_addr &&
+	    cpu_addr - mem_addr < mem->size << PAGE_SHIFT) {
+		unsigned int page = (cpu_addr - mem_addr) >> PAGE_SHIFT;
+		unsigned long flags;
+		unsigned int count;
+
+		if (DMA_ATTR_ALLOC_EXACT_SIZE & attrs)
+			count = PAGE_ALIGN(size) >> PAGE_SHIFT;
+		else
+			count = 1 << get_order(size);
+
+		spin_lock_irqsave(&mem->spinlock, flags);
+		bitmap_clear(mem->bitmap, page, count);
+		spin_unlock_irqrestore(&mem->spinlock, flags);
+	}
+}
+EXPORT_SYMBOL(nvmap_dma_free_attrs);
+
+void *nvmap_dma_mark_declared_memory_occupied(struct device *dev,
+					dma_addr_t device_addr, size_t size)
+{
+	struct dma_coherent_mem_replica *mem;
+	unsigned long flags;
+	int pos, err;
+
+	if (!dev || !dev->dma_mem)
+		return ERR_PTR(-EINVAL);
+
+	mem = (struct dma_coherent_mem_replica *)(dev->dma_mem);
+
+	size += device_addr & ~PAGE_MASK;
+
+	spin_lock_irqsave(&mem->spinlock, flags);
+	pos = PFN_DOWN(device_addr - mem->device_base);
+	err = bitmap_allocate_region(mem->bitmap, pos, get_order(size));
+	spin_unlock_irqrestore(&mem->spinlock, flags);
+
+	if (err != 0)
+		return ERR_PTR(err);
+	return mem->virt_base + (pos << PAGE_SHIFT);
+}
+
+void nvmap_dma_mark_declared_memory_unoccupied(struct device *dev,
+					 dma_addr_t device_addr, size_t size)
+{
+	struct dma_coherent_mem_replica *mem;
+	unsigned long flags;
+	int pos;
+
+	if (!dev || !dev->dma_mem)
+		return;
+
+	mem = (struct dma_coherent_mem_replica *)(dev->dma_mem);
+
+	size += device_addr & ~PAGE_MASK;
+
+	spin_lock_irqsave(&mem->spinlock, flags);
+	pos = PFN_DOWN(device_addr - mem->device_base);
+	bitmap_release_region(mem->bitmap, pos, get_order(size));
+	spin_unlock_irqrestore(&mem->spinlock, flags);
+}
+
+static void nvmap_dma_release_coherent_memory(struct dma_coherent_mem_replica *mem)
+{
+	if (!mem)
+		return;
+	if (!(mem->flags & DMA_MEMORY_NOMAP))
+		memunmap(mem->virt_base);
+	kfree(mem->bitmap);
+	kfree(mem);
+}
+
+static int nvmap_dma_assign_coherent_memory(struct device *dev,
+				      struct dma_coherent_mem_replica *mem)
+{
+	if (!dev)
+		return -ENODEV;
+
+	if (dev->dma_mem)
+		return -EBUSY;
+
+	dev->dma_mem = (struct dma_coherent_mem *)mem;
+	return 0;
+}
+
+static int nvmap_dma_init_coherent_memory(
+	phys_addr_t phys_addr, dma_addr_t device_addr, size_t size, int flags,
+	struct dma_coherent_mem_replica **mem)
+{
+	struct dma_coherent_mem_replica *dma_mem = NULL;
+	void *mem_base = NULL;
+	int pages = size >> PAGE_SHIFT;
+	int bitmap_size = BITS_TO_LONGS(pages) * sizeof(long);
+	int ret;
+
+	if (!size)
+		return -EINVAL;
+
+	if (!(flags & DMA_MEMORY_NOMAP)) {
+		mem_base = memremap(phys_addr, size, MEMREMAP_WC);
+		if (!mem_base)
+			return -EINVAL;
+	}
+
+	dma_mem = kzalloc(sizeof(struct dma_coherent_mem_replica), GFP_KERNEL);
+	if (!dma_mem) {
+		ret = -ENOMEM;
+		goto err_memunmap;
+	}
+
+	dma_mem->bitmap = kzalloc(bitmap_size, GFP_KERNEL);
+	if (!dma_mem->bitmap) {
+		ret = -ENOMEM;
+		goto err_free_dma_mem;
+	}
+
+	dma_mem->virt_base = mem_base;
+	dma_mem->device_base = device_addr;
+	dma_mem->pfn_base = PFN_DOWN(phys_addr);
+	dma_mem->size = pages;
+	dma_mem->flags = flags;
+	spin_lock_init(&dma_mem->spinlock);
+
+	*mem = dma_mem;
+	return 0;
+
+err_free_dma_mem:
+	kfree(dma_mem);
+
+err_memunmap:
+	memunmap(mem_base);
+	return ret;
+}
+
+int nvmap_dma_declare_coherent_memory(struct device *dev, phys_addr_t phys_addr,
+                        dma_addr_t device_addr, size_t size, int flags)
+{
+	struct dma_coherent_mem_replica *mem;
+	int ret;
+
+	ret = nvmap_dma_init_coherent_memory(phys_addr, device_addr, size, flags, &mem);
+	if (ret)
+		return ret;
+
+	ret = nvmap_dma_assign_coherent_memory(dev, mem);
+	if (ret)
+		nvmap_dma_release_coherent_memory(mem);
+	return ret;
+}
+#endif /* LINUX_VERSION_CODE */
+
+static int __init nvmap_co_device_init(struct reserved_mem *rmem,
+					struct device *dev)
+{
+	struct nvmap_platform_carveout *co = rmem->priv;
+	int err = 0;
+
+	if (!co)
+		return -ENODEV;
+
+	/* if co size is 0, => co is not present. So, skip init. */
+	if (!co->size)
+		return 0;
+
+	if (!co->cma_dev) {
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0)
+		err = dma_declare_coherent_memory(co->dma_dev, 0,
+				co->base, co->size,
+				DMA_MEMORY_NOMAP);
+#else
+		err = nvmap_dma_declare_coherent_memory(co->dma_dev, 0,
+				co->base, co->size,
+				DMA_MEMORY_NOMAP);
+#endif
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)
+		if (!err) {
+#else
+		if (err & DMA_MEMORY_NOMAP) {
+#endif
+			dev_info(dev,
+				 "%s :dma coherent mem declare %pa,%zu\n",
+				 co->name, &co->base, co->size);
+			co->init_done = true;
+			err = 0;
+		} else
+			dev_err(dev,
+				"%s :dma coherent mem declare fail %pa,%zu,err:%d\n",
+				co->name, &co->base, co->size, err);
+	} else {
+#ifdef NVMAP_CONFIG_VPR_RESIZE
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0)
+		/*
+		 * When vpr memory is reserved, kmemleak tries to scan vpr
+		 * memory for pointers. vpr memory should not be accessed
+		 * from cpu so avoid scanning it. When vpr memory is removed,
+		 * the memblock_remove() API ensures that kmemleak won't scan
+		 * a removed block.
+		 */
+		if (!strncmp(co->name, "vpr", 3))
+			kmemleak_no_scan(__va(co->base));
+#endif
+
+		co->dma_info->cma_dev = co->cma_dev;
+		err = dma_declare_coherent_resizable_cma_memory(
+				co->dma_dev, co->dma_info);
+		if (err)
+			dev_err(dev, "%s coherent memory declaration failed\n",
+				     co->name);
+		else
+#endif
+			co->init_done = true;
+	}
+	return err;
+}
+
+static void nvmap_co_device_release(struct reserved_mem *rmem,struct device *dev)
+{
+	struct nvmap_platform_carveout *co = rmem->priv;
+
+	if (!co)
+		return;
+
+	if (co->usage_mask == NVMAP_HEAP_CARVEOUT_IVM)
+		kfree(co->name);
+}
+
+static const struct reserved_mem_ops nvmap_co_ops = {
+	.device_init	= nvmap_co_device_init,
+	.device_release	= nvmap_co_device_release,
+};
+
+#ifndef NVMAP_LOADABLE_MODULE
+int __init nvmap_co_setup(struct reserved_mem *rmem)
+{
+	struct nvmap_platform_carveout *co;
+	int ret = 0;
+#ifdef NVMAP_CONFIG_VPR_RESIZE
+	struct cma *cma;
+#endif
+	ulong start = sched_clock();
+
+	co = nvmap_get_carveout_pdata(rmem->name);
+	if (!co)
+		return ret;
+
+	rmem->ops = &nvmap_co_ops;
+	rmem->priv = co;
+
+	co->base = rmem->base;
+	co->size = rmem->size;
+
+#ifdef NVMAP_CONFIG_VPR_RESIZE
+	if (!of_get_flat_dt_prop(rmem->fdt_node, "reusable", NULL) ||
+	    of_get_flat_dt_prop(rmem->fdt_node, "no-map", NULL))
+		goto skip_cma;
+
+	WARN_ON(!rmem->base);
+	if (dev_get_cma_area(co->cma_dev)) {
+		pr_info("cma area initialed in legacy way already\n");
+		goto finish;
+	}
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)
+	ret = cma_init_reserved_mem(rmem->base, rmem->size, 0,
+					rmem->name, &cma);
+#else
+	ret = cma_init_reserved_mem(rmem->base, rmem->size, 0, &cma);
+#endif
+	if (ret) {
+		pr_info("cma_init_reserved_mem fails for %s\n", rmem->name);
+		goto finish;
+	}
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0)
+	dma_contiguous_early_fixup_vpr(rmem->base, rmem->size);
+	if (co->cma_dev)
+		co->cma_dev->cma_area = cma;
+#else
+	dma_contiguous_early_fixup(rmem->base, rmem->size);
+	dev_set_cma_area(co->cma_dev, cma);
+#endif
+	pr_debug("tegra-carveouts carveout=%s %pa@%pa\n",
+		 rmem->name, &rmem->size, &rmem->base);
+	goto finish;
+
+skip_cma:
+#endif
+	co->cma_dev = NULL;
+#ifdef NVMAP_CONFIG_VPR_RESIZE
+finish:
+#endif
+	nvmap_init_time += sched_clock() - start;
+	return ret;
+}
+EXPORT_SYMBOL(nvmap_co_setup);
+#else
+int __init nvmap_co_setup(struct reserved_mem *rmem)
+{
+	struct nvmap_platform_carveout *co;
+	ulong start = sched_clock();
+	int ret = 0;
+
+	co = nvmap_get_carveout_pdata(rmem->name);
+	if (!co)
+		return ret;
+
+	rmem->ops = &nvmap_co_ops;
+	rmem->priv = co;
+
+	co->base = rmem->base;
+	co->size = rmem->size;
+	co->cma_dev = NULL;
+	nvmap_init_time += sched_clock() - start;
+	return ret;
+}
+#endif /* !NVMAP_LOADABLE_MODULE */
+
+RESERVEDMEM_OF_DECLARE(nvmap_co, "nvidia,generic_carveout", nvmap_co_setup);
+#ifndef NVMAP_LOADABLE_MODULE
+RESERVEDMEM_OF_DECLARE(nvmap_vpr_co, "nvidia,vpr-carveout", nvmap_co_setup);
+RESERVEDMEM_OF_DECLARE(nvmap_fsi_co, "nvidia,fsi-carveout", nvmap_co_setup);
+#endif /* !NVMAP_LOADABLE_MODULE */
+
+/*
+ * Fills in the platform data either from the device tree or with the
+ * legacy path.
+ */
+int __init nvmap_init(struct platform_device *pdev)
+{
+	int err;
+	struct reserved_mem rmem;
+#ifdef NVMAP_LOADABLE_MODULE
+	struct reserved_mem *rmem2;
+	struct device_node *np = pdev->dev.of_node;
+	struct of_phandle_iterator it;
+	const char *compp;
+
+	if (!of_phandle_iterator_init(&it, np, "memory-region", NULL, 0)) {
+		while (!of_phandle_iterator_next(&it) && it.node) {
+			if (of_device_is_available(it.node) &&
+			    !of_device_is_compatible(it.node, "nvidia,ivm_carveout")) {
+				rmem2 = of_reserved_mem_lookup(it.node);
+				if (!rmem2) {
+					if (!of_property_read_string(it.node, "compatible", &compp))
+						pr_err("unable to acquire memory-region: %s\n",
+							compp);
+					return -EINVAL;
+				}
+				nvmap_co_setup(rmem2);
+			}
+		}
+	}
+#endif /* NVMAP_LOADABLE_MODULE */
+
+	if (pdev->dev.of_node) {
+		err = __nvmap_init_dt(pdev);
+		if (err)
+			return err;
+	}
+
+	err = of_reserved_mem_device_init(&pdev->dev);
+	if (err)
+		pr_debug("reserved_mem_device_init fails, try legacy init\n");
+
+	/* try legacy init */
+	if (!nvmap_carveouts[0].init_done) {
+		rmem.priv = &nvmap_carveouts[0];
+		err = nvmap_co_device_init(&rmem, &pdev->dev);
+		if (err)
+			goto end;
+	}
+
+	if (!nvmap_carveouts[1].init_done) {
+		rmem.priv = &nvmap_carveouts[1];
+		err = nvmap_co_device_init(&rmem, &pdev->dev);
+		if (err)
+			goto end;
+	}
+
+#ifdef CONFIG_TEGRA_VIRTUALIZATION
+	err = nvmap_populate_ivm_carveout(&pdev->dev);
+#endif /* CONFIG_TEGRA_VIRTUALIZATION */
+
+end:
+	return err;
+}
+
+#ifdef NVMAP_LOADABLE_MODULE
+static bool nvmap_is_carveout_node_present(void)
+{
+	struct device_node *np;
+
+	np = of_find_node_by_name(NULL, "tegra-carveouts");
+	if (of_device_is_available(np)) {
+		of_node_put(np);
+		return true;
+	}
+	of_node_put(np);
+	return false;
+}
+#endif /* NVMAP_LOADABLE_MODULE */
+
+static struct platform_driver __refdata nvmap_driver = {
+	.probe		= nvmap_probe,
+	.remove		= nvmap_remove,
+
+	.driver = {
+		.name	= "tegra-carveouts",
+		.owner	= THIS_MODULE,
+#ifndef NVMAP_LOADABLE_MODULE
+		.of_match_table = nvmap_of_ids,
+#endif /* !NVMAP_LOADABLE_MODULE */
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
+		.suppress_bind_attrs = true,
+	},
+};
+
+static int __init nvmap_init_driver(void)
+{
+	int e = 0;
+
+	e = nvmap_heap_init();
+	if (e)
+		goto fail;
+
+#ifdef NVMAP_LOADABLE_MODULE
+	if (!(of_machine_is_compatible("nvidia,tegra186") ||
+	    of_machine_is_compatible("nvidia,tegra194") ||
+	    of_machine_is_compatible("nvidia,tegra234") ||
+	    of_machine_is_compatible("nvidia,tegra239") ||
+	    of_machine_is_compatible("nvidia,tegra23x") ||
+	    of_machine_is_compatible("nvidia,tegra232"))) {
+		nvmap_heap_deinit();
+		return -ENODEV;
+	}
+#endif /* NVMAP_LOADABLE_MODULE */
+	e = platform_driver_register(&nvmap_driver);
+	if (e) {
+		nvmap_heap_deinit();
+		goto fail;
+	}
+
+#ifdef NVMAP_LOADABLE_MODULE
+	if (!nvmap_is_carveout_node_present()) {
+		pdev = platform_device_register_simple("tegra-carveouts", -1, NULL, 0);
+		if (IS_ERR(pdev)) {
+			platform_driver_unregister(&nvmap_driver);
+			nvmap_heap_deinit();
+			return PTR_ERR(pdev);
+		}
+	}
+#endif /* NVMAP_LOADABLE_MODULE */
+
+fail:
+	return e;
+}
+
+#ifdef NVMAP_LOADABLE_MODULE
+module_init(nvmap_init_driver);
+#else
+fs_initcall(nvmap_init_driver);
+#endif /* NVMAP_LOADABLE_MODULE */
+
+static void __exit nvmap_exit_driver(void)
+{
+#ifdef NVMAP_LOADABLE_MODULE
+	if (!nvmap_is_carveout_node_present())
+		platform_device_unregister(pdev);
+#endif /* NVMAP_LOADABLE_MODULE */
+	platform_driver_unregister(&nvmap_driver);
+	nvmap_heap_deinit();
+	nvmap_dev = NULL;
+}
+module_exit(nvmap_exit_driver);
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 16, 0)
+MODULE_IMPORT_NS(DMA_BUF);
+#endif
+MODULE_DESCRIPTION("NvMap: Nvidia Tegra Memory Management Driver");
+MODULE_AUTHOR("Puneet Saxena <puneets@nvidia.com>");
+MODULE_LICENSE("GPL v2");

drivers/video/tegra/nvmap/nvmap_init_t19x.c

@@ -0,0 +1,62 @@
+/*
+ * drivers/video/tegra/nvmap/nvmap_init_t19x.c
+ *
+ * Copyright (c) 2016-2022, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#define pr_fmt(fmt)	"nvmap: %s() " fmt, __func__
+
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_reserved_mem.h>
+#include <linux/platform_device.h>
+#include <linux/nvmap_t19x.h>
+#include <linux/kobject.h>
+#include <linux/debugfs.h>
+#include <linux/sysfs.h>
+#include <linux/io.h>
+
+#include "nvmap_priv.h"
+
+bool nvmap_version_t19x;
+
+const struct of_device_id nvmap_of_ids[] = {
+	{ .compatible = "nvidia,carveouts" },
+	{ .compatible = "nvidia,carveouts-t18x" },
+	{ .compatible = "nvidia,carveouts-t19x" },
+	{ }
+};
+
+int nvmap_register_cvsram_carveout(struct device *dma_dev,
+		phys_addr_t base, size_t size, int (*busy)(void),
+		int (*idle)(void))
+{
+	static struct nvmap_platform_carveout cvsram = {
+		.name = "cvsram",
+		.usage_mask = NVMAP_HEAP_CARVEOUT_CVSRAM,
+		.disable_dynamic_dma_map = true,
+		.no_cpu_access = true,
+	};
+
+	cvsram.pm_ops.busy = busy;
+	cvsram.pm_ops.idle = idle;
+
+	if (!base || !size || (base != PAGE_ALIGN(base)) ||
+	    (size != PAGE_ALIGN(size)))
+		return -EINVAL;
+	cvsram.base = base;
+	cvsram.size = size;
+
+	cvsram.dma_dev = &cvsram.dev;
+	return nvmap_create_carveout(&cvsram);
+}
+EXPORT_SYMBOL(nvmap_register_cvsram_carveout);

drivers/video/tegra/nvmap/nvmap_ioctl.h

@@ -0,0 +1,79 @@
+/*
+ * drivers/video/tegra/nvmap/nvmap_ioctl.h
+ *
+ * ioctl declarations for nvmap
+ *
+ * Copyright (c) 2010-2019, NVIDIA CORPORATION. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#ifndef __VIDEO_TEGRA_NVMAP_IOCTL_H
+#define __VIDEO_TEGRA_NVMAP_IOCTL_H
+
+#include <linux/nvmap.h>
+
+int nvmap_ioctl_pinop(struct file *filp, bool is_pin, void __user *arg,
+	bool is32);
+
+int nvmap_ioctl_getid(struct file *filp, void __user *arg);
+
+int nvmap_ioctl_get_ivcid(struct file *filp, void __user *arg);
+
+int nvmap_ioctl_getfd(struct file *filp, void __user *arg);
+
+int nvmap_ioctl_alloc(struct file *filp, void __user *arg);
+
+int nvmap_ioctl_alloc_kind(struct file *filp, void __user *arg);
+
+int nvmap_ioctl_alloc_ivm(struct file *filp, void __user *arg);
+
+int nvmap_ioctl_vpr_floor_size(struct file *filp, void __user *arg);
+
+int nvmap_ioctl_free(struct file *filp, unsigned long arg);
+
+int nvmap_ioctl_create(struct file *filp, unsigned int cmd, void __user *arg);
+
+int nvmap_ioctl_create_from_va(struct file *filp, void __user *arg);
+
+int nvmap_ioctl_create_from_ivc(struct file *filp, void __user *arg);
+
+int nvmap_ioctl_get_ivc_heap(struct file *filp, void __user *arg);
+
+int nvmap_map_into_caller_ptr(struct file *filp, void __user *arg, bool is32);
+
+int nvmap_ioctl_cache_maint(struct file *filp, void __user *arg, int size);
+
+int nvmap_ioctl_rw_handle(struct file *filp, int is_read, void __user *arg,
+	size_t op_size);
+
+int nvmap_ioctl_cache_maint_list(struct file *filp, void __user *arg);
+
+int nvmap_ioctl_gup_test(struct file *filp, void __user *arg);
+
+int nvmap_ioctl_set_tag_label(struct file *filp, void __user *arg);
+
+int nvmap_ioctl_get_available_heaps(struct file *filp, void __user *arg);
+
+int nvmap_ioctl_get_heap_size(struct file *filp, void __user *arg);
+void kasan_memcpy_toio(void __iomem *to, const void *from,
+			size_t count);
+
+int nvmap_ioctl_get_handle_parameters(struct file *filp, void __user *arg);
+
+int nvmap_ioctl_get_sci_ipc_id(struct file *filp, void __user *arg);
+
+int nvmap_ioctl_handle_from_sci_ipc_id(struct file *filp, void __user *arg);
+
+int nvmap_ioctl_query_heap_params(struct file *filp, void __user *arg);
+
+int nvmap_ioctl_dup_handle(struct file *filp, void __user *arg);
+
+#endif	/*  __VIDEO_TEGRA_NVMAP_IOCTL_H */

drivers/video/tegra/nvmap/nvmap_kasan_wrapper.c

@@ -0,0 +1,46 @@
+/*
+ * drivers/video/tegra/nvmap/nvmap_kasan_wrapper.c
+ *
+ * place to add wrapper function to drop kasan scan
+ *
+ * Copyright (c) 2018-2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+#include <linux/export.h>
+#include <linux/types.h>
+#include <linux/io.h>
+#include "nvmap_ioctl.h"
+
+void kasan_memcpy_toio(void __iomem *to,
+			const void *from, size_t count)
+{
+	while (count && (!IS_ALIGNED((unsigned long)to, 8) ||
+			 !IS_ALIGNED((unsigned long)from, 8))) {
+		__raw_writeb(*(u8 *)from, to);
+		from++;
+		to++;
+		count--;
+	}
+
+	while (count >= 8) {
+		__raw_writeq(*(u64 *)from, to);
+		from += 8;
+		to += 8;
+		count -= 8;
+	}
+
+	while (count) {
+		__raw_writeb(*(u8 *)from, to);
+		from++;
+		to++;
+		count--;
+	}
+}

drivers/video/tegra/nvmap/nvmap_mm.c

@@ -0,0 +1,84 @@
+/*
+ * drivers/video/tegra/nvmap/nvmap_mm.c
+ *
+ * Some MM related functionality specific to nvmap.
+ *
+ * Copyright (c) 2013-2021, NVIDIA CORPORATION. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#include <trace/events/nvmap.h>
+#include <linux/version.h>
+
+#include <asm/pgtable.h>
+
+#include "nvmap_priv.h"
+
+
+#ifndef NVMAP_LOADABLE_MODULE
+void nvmap_zap_handle(struct nvmap_handle *handle, u64 offset, u64 size)
+{
+	struct list_head *vmas;
+	struct nvmap_vma_list *vma_list;
+	struct vm_area_struct *vma;
+
+	if (!handle->heap_pgalloc)
+		return;
+
+	/* if no dirty page is present, no need to zap */
+	if (nvmap_handle_track_dirty(handle) && !atomic_read(&handle->pgalloc.ndirty))
+		return;
+
+	if (!size) {
+		offset = 0;
+		size = handle->size;
+	}
+
+	size = PAGE_ALIGN((offset & ~PAGE_MASK) + size);
+
+	mutex_lock(&handle->lock);
+	vmas = &handle->vmas;
+	list_for_each_entry(vma_list, vmas, list) {
+		struct nvmap_vma_priv *priv;
+		size_t vm_size = size;
+
+		vma = vma_list->vma;
+		priv = vma->vm_private_data;
+		if ((offset + size) > (vma->vm_end - vma->vm_start))
+			vm_size = vma->vm_end - vma->vm_start - offset;
+
+		if (priv->offs || vma->vm_pgoff)
+			/* vma mapping starts in the middle of handle memory.
+			 * zapping needs special care. zap entire range for now.
+			 * FIXME: optimze zapping.
+			 */
+			zap_page_range(vma, vma->vm_start,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)
+				vma->vm_end - vma->vm_start);
+#else
+				vma->vm_end - vma->vm_start, NULL);
+#endif
+		else
+			zap_page_range(vma, vma->vm_start + offset,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)
+				vm_size);
+#else
+				vm_size, NULL);
+#endif
+	}
+	mutex_unlock(&handle->lock);
+}
+#else
+void nvmap_zap_handle(struct nvmap_handle *handle, u64 offset, u64 size)
+{
+	pr_debug("%s is not supported!\n", __func__);
+}
+#endif /* !NVMAP_LOADABLE_MODULE */

drivers/video/tegra/nvmap/nvmap_pp.c

@@ -0,0 +1,774 @@
+/*
+ * drivers/video/tegra/nvmap/nvmap_pp.c
+ *
+ * Manage page pools to speed up page allocation.
+ *
+ * Copyright (c) 2009-2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#define pr_fmt(fmt) "%s: " fmt, __func__
+
+#include <linux/kernel.h>
+#include <linux/vmalloc.h>
+#include <linux/moduleparam.h>
+#include <linux/nodemask.h>
+#include <linux/shrinker.h>
+#include <linux/kthread.h>
+#include <linux/debugfs.h>
+#include <linux/freezer.h>
+#include <linux/highmem.h>
+#include <linux/version.h>
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)
+#include <linux/sched/clock.h>
+#include <uapi/linux/sched/types.h>
+#endif
+
+#include <trace/events/nvmap.h>
+
+#include "nvmap_priv.h"
+
+#define NVMAP_TEST_PAGE_POOL_SHRINKER     1
+#define PENDING_PAGES_SIZE                (SZ_1M / PAGE_SIZE)
+
+static bool enable_pp = 1;
+static u32 pool_size;
+
+static struct task_struct *background_allocator;
+static DECLARE_WAIT_QUEUE_HEAD(nvmap_bg_wait);
+
+#ifdef NVMAP_CONFIG_PAGE_POOL_DEBUG
+static inline void __pp_dbg_var_add(u64 *dbg_var, u32 nr)
+{
+	*dbg_var += nr;
+}
+#else
+#define __pp_dbg_var_add(dbg_var, nr)
+#endif
+
+#define pp_alloc_add(pool, nr) __pp_dbg_var_add(&(pool)->allocs, nr)
+#define pp_fill_add(pool, nr)  __pp_dbg_var_add(&(pool)->fills, nr)
+#define pp_hit_add(pool, nr)   __pp_dbg_var_add(&(pool)->hits, nr)
+#define pp_miss_add(pool, nr)  __pp_dbg_var_add(&(pool)->misses, nr)
+
+static int __nvmap_page_pool_fill_lots_locked(struct nvmap_page_pool *pool,
+				       struct page **pages, u32 nr);
+
+static inline struct page *get_zero_list_page(struct nvmap_page_pool *pool)
+{
+	struct page *page;
+
+	trace_get_zero_list_page(pool->to_zero);
+
+	if (list_empty(&pool->zero_list))
+		return NULL;
+
+	page = list_first_entry(&pool->zero_list, struct page, lru);
+	list_del(&page->lru);
+
+	pool->to_zero--;
+
+	return page;
+}
+
+static inline struct page *get_page_list_page(struct nvmap_page_pool *pool)
+{
+	struct page *page;
+
+	trace_get_page_list_page(pool->count);
+
+	if (list_empty(&pool->page_list))
+		return NULL;
+
+	page = list_first_entry(&pool->page_list, struct page, lru);
+	list_del(&page->lru);
+
+	pool->count--;
+
+	return page;
+}
+
+#ifdef CONFIG_ARM64_4K_PAGES
+static inline struct page *get_page_list_page_bp(struct nvmap_page_pool *pool)
+{
+	struct page *page;
+
+	if (list_empty(&pool->page_list_bp))
+		return NULL;
+
+	page = list_first_entry(&pool->page_list_bp, struct page, lru);
+	list_del(&page->lru);
+
+	pool->count -= pool->pages_per_big_pg;
+	pool->big_page_count -= pool->pages_per_big_pg;
+
+	return page;
+}
+#endif /* CONFIG_ARM64_4K_PAGES */
+
+static inline bool nvmap_bg_should_run(struct nvmap_page_pool *pool)
+{
+	return !list_empty(&pool->zero_list);
+}
+
+static void nvmap_pp_zero_pages(struct page **pages, int nr)
+{
+	int i;
+
+	for (i = 0; i < nr; i++) {
+		clear_highpage(pages[i]);
+		nvmap_clean_cache_page(pages[i]);
+	}
+
+	trace_nvmap_pp_zero_pages(nr);
+}
+
+static void nvmap_pp_do_background_zero_pages(struct nvmap_page_pool *pool)
+{
+	int i;
+	struct page *page;
+	int ret;
+	/*
+	 * Statically declared array of pages to be zeroed in a batch,
+	 * local to this thread but too big for the stack.
+	 */
+	static struct page *pending_zero_pages[PENDING_PAGES_SIZE];
+
+	rt_mutex_lock(&pool->lock);
+	for (i = 0; i < PENDING_PAGES_SIZE; i++) {
+		page = get_zero_list_page(pool);
+		if (page == NULL)
+			break;
+		pending_zero_pages[i] = page;
+		pool->under_zero++;
+	}
+	rt_mutex_unlock(&pool->lock);
+
+	nvmap_pp_zero_pages(pending_zero_pages, i);
+
+	rt_mutex_lock(&pool->lock);
+	ret = __nvmap_page_pool_fill_lots_locked(pool, pending_zero_pages, i);
+	pool->under_zero -= i;
+	rt_mutex_unlock(&pool->lock);
+
+	trace_nvmap_pp_do_background_zero_pages(ret, i);
+
+	for (; ret < i; ret++)
+		__free_page(pending_zero_pages[ret]);
+}
+
+/*
+ * This thread fills the page pools with zeroed pages. We avoid releasing the
+ * pages directly back into the page pools since we would then have to zero
+ * them ourselves. Instead it is easier to just reallocate zeroed pages. This
+ * happens in the background so that the overhead of allocating zeroed pages is
+ * not directly seen by userspace. Of course if the page pools are empty user
+ * space will suffer.
+ */
+static int nvmap_background_zero_thread(void *arg)
+{
+	struct nvmap_page_pool *pool = &nvmap_dev->pool;
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0)
+	struct sched_param param = { .sched_priority = 0 };
+#endif
+
+	pr_info("PP zeroing thread starting.\n");
+
+	set_freezable();
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0)
+	sched_setscheduler(current, SCHED_NORMAL, &param);
+#else
+	sched_set_normal(current, MAX_NICE);
+#endif
+
+	while (!kthread_should_stop()) {
+		while (nvmap_bg_should_run(pool))
+			nvmap_pp_do_background_zero_pages(pool);
+
+		wait_event_freezable(nvmap_bg_wait,
+				nvmap_bg_should_run(pool) ||
+				kthread_should_stop());
+	}
+
+	return 0;
+}
+
+#ifdef NVMAP_CONFIG_PAGE_POOL_DEBUG
+static void nvmap_pgcount(struct page *page, bool incr)
+{
+	page_ref_add(page, incr ? 1 : -1);
+}
+#endif /* NVMAP_CONFIG_PAGE_POOL_DEBUG */
+
+/*
+ * Free the passed number of pages from the page pool. This happens regardless
+ * of whether the page pools are enabled. This lets one disable the page pools
+ * and then free all the memory therein.
+ *
+ * FIXME: Pages in pending_zero_pages[] can still be unreleased.
+ */
+static ulong nvmap_page_pool_free_pages_locked(struct nvmap_page_pool *pool,
+						      ulong nr_pages)
+{
+	struct page *page;
+	bool use_page_list = false;
+#ifdef CONFIG_ARM64_4K_PAGES
+	bool use_page_list_bp = false;
+	int i;
+#endif /* CONFIG_ARM64_4K_PAGES */
+
+	pr_debug("req to release pages=%ld\n", nr_pages);
+
+	while (nr_pages) {
+
+#ifdef CONFIG_ARM64_4K_PAGES
+		if (use_page_list_bp)
+			page = get_page_list_page_bp(pool);
+		else if (use_page_list)
+#else
+		if (use_page_list)
+#endif /* CONFIG_ARM64_4K_PAGES */
+			page = get_page_list_page(pool);
+		else
+			page = get_zero_list_page(pool);
+
+		if (!page) {
+			if (!use_page_list) {
+				use_page_list = true;
+				continue;
+			}
+#ifdef CONFIG_ARM64_4K_PAGES
+			else if (!use_page_list_bp) {
+				use_page_list_bp = true;
+				continue;
+			}
+#endif /* CONFIG_ARM64_4K_PAGES */
+			break;
+		}
+
+#ifdef CONFIG_ARM64_4K_PAGES
+		if (use_page_list_bp) {
+			for (i = 0; i < pool->pages_per_big_pg; i++)
+				__free_page(nth_page(page, i));
+			pr_debug("released %d pages\n", pool->pages_per_big_pg);
+			if (nr_pages > pool->pages_per_big_pg)
+				nr_pages -= pool->pages_per_big_pg;
+			else
+				nr_pages = 0;
+		} else {
+#endif /* CONFIG_ARM64_4K_PAGES */
+			__free_page(page);
+			nr_pages--;
+			pr_debug("released 1 page\n");
+#ifdef CONFIG_ARM64_4K_PAGES
+		}
+#endif /* CONFIG_ARM64_4K_PAGES */
+	}
+
+	pr_debug("remaining pages to release=%ld\n", nr_pages);
+	return nr_pages;
+}
+
+/*
+ * Alloc a bunch of pages from the page pool. This will alloc as many as it can
+ * and return the number of pages allocated. Pages are placed into the passed
+ * array in a linear fashion starting from index 0.
+ */
+int nvmap_page_pool_alloc_lots(struct nvmap_page_pool *pool,
+				struct page **pages, u32 nr)
+{
+	u32 ind = 0;
+	u32 non_zero_idx;
+	u32 non_zero_cnt = 0;
+
+	if (!enable_pp || !nr)
+		return 0;
+
+	rt_mutex_lock(&pool->lock);
+
+	while (ind < nr) {
+		struct page *page = NULL;
+
+		if (!non_zero_cnt)
+			page = get_page_list_page(pool);
+
+		if (!page) {
+			page = get_zero_list_page(pool);
+			if (!page)
+				break;
+			if (!non_zero_cnt)
+				non_zero_idx = ind;
+			non_zero_cnt++;
+		}
+
+		pages[ind++] = page;
+#ifdef NVMAP_CONFIG_PAGE_POOL_DEBUG
+		nvmap_pgcount(page, false);
+		BUG_ON(page_count(page) != 1);
+#endif /* NVMAP_CONFIG_PAGE_POOL_DEBUG */
+	}
+
+	rt_mutex_unlock(&pool->lock);
+
+	/* Zero non-zeroed pages, if any */
+	if (non_zero_cnt)
+		nvmap_pp_zero_pages(&pages[non_zero_idx], non_zero_cnt);
+
+	pp_alloc_add(pool, ind);
+	pp_hit_add(pool, ind);
+	pp_miss_add(pool, nr - ind);
+
+	trace_nvmap_pp_alloc_lots(ind, nr);
+
+	return ind;
+}
+
+#ifdef CONFIG_ARM64_4K_PAGES
+int nvmap_page_pool_alloc_lots_bp(struct nvmap_page_pool *pool,
+				struct page **pages, u32 nr)
+{
+	u32 ind = 0, nr_pages = nr;
+	struct page *page;
+
+	if (!enable_pp || pool->pages_per_big_pg <= 1 ||
+	    nr_pages < pool->pages_per_big_pg)
+		return 0;
+
+	rt_mutex_lock(&pool->lock);
+
+	while (nr_pages - ind >= pool->pages_per_big_pg) {
+		int i;
+
+		page = get_page_list_page_bp(pool);
+		if (!page)
+			break;
+
+		for (i = 0; i < pool->pages_per_big_pg; i++)
+			pages[ind + i] = nth_page(page, i);
+
+		ind += pool->pages_per_big_pg;
+	}
+
+	rt_mutex_unlock(&pool->lock);
+	return ind;
+}
+
+static bool nvmap_is_big_page(struct nvmap_page_pool *pool,
+			      struct page **pages, int idx, int nr)
+{
+	int i;
+	struct page *page = pages[idx];
+
+	if (pool->pages_per_big_pg <= 1)
+		return false;
+
+	if (nr - idx < pool->pages_per_big_pg)
+		return false;
+
+	/* Allow coalescing pages at big page boundary only */
+	if (page_to_phys(page) & (pool->big_pg_sz - 1))
+		return false;
+
+	for (i = 1; i < pool->pages_per_big_pg; i++)
+		if (pages[idx + i] != nth_page(page, i))
+			break;
+
+	return i == pool->pages_per_big_pg ? true: false;
+}
+#endif /* CONFIG_ARM64_4K_PAGES */
+
+/*
+ * Fill a bunch of pages into the page pool. This will fill as many as it can
+ * and return the number of pages filled. Pages are used from the start of the
+ * passed page pointer array in a linear fashion.
+ *
+ * You must lock the page pool before using this.
+ */
+static int __nvmap_page_pool_fill_lots_locked(struct nvmap_page_pool *pool,
+				       struct page **pages, u32 nr)
+{
+	int real_nr;
+	int pages_to_fill;
+	int ind = 0;
+
+	if (!enable_pp)
+		return 0;
+
+	BUG_ON(pool->count > pool->max);
+	real_nr = min_t(u32, pool->max - pool->count, nr);
+	pages_to_fill = real_nr;
+	if (real_nr == 0)
+		return 0;
+
+	while (real_nr > 0) {
+#ifdef NVMAP_CONFIG_PAGE_POOL_DEBUG
+		nvmap_pgcount(pages[ind], true);
+		BUG_ON(page_count(pages[ind]) != 2);
+#endif /* NVMAP_CONFIG_PAGE_POOL_DEBUG */
+
+#ifdef CONFIG_ARM64_4K_PAGES
+		if (nvmap_is_big_page(pool, pages, ind, pages_to_fill)) {
+			list_add_tail(&pages[ind]->lru, &pool->page_list_bp);
+			ind += pool->pages_per_big_pg;
+			real_nr -= pool->pages_per_big_pg;
+			pool->big_page_count += pool->pages_per_big_pg;
+		} else {
+#endif /* CONFIG_ARM64_4K_PAGES */
+			list_add_tail(&pages[ind++]->lru, &pool->page_list);
+			real_nr--;
+#ifdef CONFIG_ARM64_4K_PAGES
+		}
+#endif /* CONFIG_ARM64_4K_PAGES */
+	}
+
+	pool->count += ind;
+	BUG_ON(pool->count > pool->max);
+	pp_fill_add(pool, ind);
+
+	return ind;
+}
+
+u32 nvmap_page_pool_fill_lots(struct nvmap_page_pool *pool,
+				       struct page **pages, u32 nr)
+{
+	u32 ret = 0;
+	u32 i;
+	u32 save_to_zero;
+
+	rt_mutex_lock(&pool->lock);
+
+	save_to_zero = pool->to_zero;
+
+	ret = min(nr, pool->max - pool->count - pool->to_zero - pool->under_zero);
+
+	for (i = 0; i < ret; i++) {
+		/* If page has additonal referecnces, Don't add it into
+		 * page pool. get_user_pages() on mmap'ed nvmap handle can
+		 * hold a refcount on the page. These pages can't be
+		 * reused till the additional refs are dropped.
+		 */
+		if (page_count(pages[i]) > 1) {
+			__free_page(pages[i]);
+		} else {
+			list_add_tail(&pages[i]->lru, &pool->zero_list);
+			pool->to_zero++;
+		}
+	}
+
+	if (pool->to_zero)
+		wake_up_interruptible(&nvmap_bg_wait);
+	ret = i;
+
+	trace_nvmap_pp_fill_zero_lots(save_to_zero, pool->to_zero,
+			ret, nr);
+
+	rt_mutex_unlock(&pool->lock);
+
+	return ret;
+}
+
+ulong nvmap_page_pool_get_unused_pages(void)
+{
+	int total = 0;
+
+	if (!nvmap_dev)
+		return 0;
+
+	total = nvmap_dev->pool.count + nvmap_dev->pool.to_zero;
+
+	return total;
+}
+
+/*
+ * Remove and free to the system all the pages currently in the page
+ * pool. This operation will happen even if the page pools are disabled.
+ */
+int nvmap_page_pool_clear(void)
+{
+	struct nvmap_page_pool *pool = &nvmap_dev->pool;
+
+	rt_mutex_lock(&pool->lock);
+
+	(void)nvmap_page_pool_free_pages_locked(pool, pool->count + pool->to_zero);
+
+	/* For some reason, if an error occured... */
+	if (!list_empty(&pool->page_list) || !list_empty(&pool->zero_list)) {
+		rt_mutex_unlock(&pool->lock);
+		return -ENOMEM;
+	}
+
+	rt_mutex_unlock(&pool->lock);
+
+	return 0;
+}
+
+/*
+ * Resizes the page pool to the passed size. If the passed size is 0 then
+ * all associated resources are released back to the system. This operation
+ * will only occur if the page pools are enabled.
+ */
+static void nvmap_page_pool_resize(struct nvmap_page_pool *pool, u32 size)
+{
+	u64 curr;
+
+	rt_mutex_lock(&pool->lock);
+
+	curr = nvmap_page_pool_get_unused_pages();
+	if (curr > size)
+		(void)nvmap_page_pool_free_pages_locked(pool, curr - size);
+
+	pr_debug("page pool resized to %d from %d pages\n", size, pool->max);
+	pool->max = size;
+
+	rt_mutex_unlock(&pool->lock);
+}
+
+static unsigned long nvmap_page_pool_count_objects(struct shrinker *shrinker,
+						   struct shrink_control *sc)
+{
+	return nvmap_page_pool_get_unused_pages();
+}
+
+static unsigned long nvmap_page_pool_scan_objects(struct shrinker *shrinker,
+						  struct shrink_control *sc)
+{
+	unsigned long remaining;
+
+	pr_debug("sh_pages=%lu", sc->nr_to_scan);
+
+	rt_mutex_lock(&nvmap_dev->pool.lock);
+	remaining = nvmap_page_pool_free_pages_locked(
+			&nvmap_dev->pool, sc->nr_to_scan);
+	rt_mutex_unlock(&nvmap_dev->pool.lock);
+
+	return (remaining == sc->nr_to_scan) ? \
+			   SHRINK_STOP : (sc->nr_to_scan - remaining);
+}
+
+static struct shrinker nvmap_page_pool_shrinker = {
+	.count_objects = nvmap_page_pool_count_objects,
+	.scan_objects = nvmap_page_pool_scan_objects,
+	.seeks = 1,
+};
+
+static void shrink_page_pools(unsigned long *total_pages, unsigned long *available_pages)
+{
+	struct shrink_control sc;
+
+	if (*total_pages == 0) {
+		sc.gfp_mask = GFP_KERNEL;
+		sc.nr_to_scan = 0;
+		*total_pages = nvmap_page_pool_count_objects(NULL, &sc);
+	}
+	sc.nr_to_scan = *total_pages;
+	nvmap_page_pool_scan_objects(NULL, &sc);
+	*available_pages = nvmap_page_pool_count_objects(NULL, &sc);
+}
+
+#if NVMAP_TEST_PAGE_POOL_SHRINKER
+static int shrink_pp;
+static int shrink_set(const char *arg, const struct kernel_param *kp)
+{
+	int cpu = smp_processor_id();
+	unsigned long long t1, t2;
+	unsigned long total_pages, available_pages;
+
+	param_set_int(arg, kp);
+
+	if (shrink_pp) {
+		total_pages = shrink_pp;
+		t1 = cpu_clock(cpu);
+		shrink_page_pools(&total_pages, &available_pages);
+		t2 = cpu_clock(cpu);
+		pr_debug("shrink page pools: time=%lldns, "
+			"total_pages_released=%lu, free_pages_available=%lu",
+			t2-t1, total_pages, available_pages);
+	}
+	return 0;
+}
+
+static int shrink_get(char *buff, const struct kernel_param *kp)
+{
+	return param_get_int(buff, kp);
+}
+
+static struct kernel_param_ops shrink_ops = {
+	.get = shrink_get,
+	.set = shrink_set,
+};
+
+module_param_cb(shrink_page_pools, &shrink_ops, &shrink_pp, 0644);
+#endif
+
+static int enable_pp_set(const char *arg, const struct kernel_param *kp)
+{
+	int ret;
+
+	ret = param_set_bool(arg, kp);
+	if (ret)
+		return ret;
+
+	if (!enable_pp)
+		nvmap_page_pool_clear();
+
+	return 0;
+}
+
+static int enable_pp_get(char *buff, const struct kernel_param *kp)
+{
+	return param_get_bool(buff, kp);
+}
+
+static struct kernel_param_ops enable_pp_ops = {
+	.get = enable_pp_get,
+	.set = enable_pp_set,
+};
+
+module_param_cb(enable_page_pools, &enable_pp_ops, &enable_pp, 0644);
+
+static int pool_size_set(const char *arg, const struct kernel_param *kp)
+{
+	int ret = param_set_uint(arg, kp);
+
+	if (!ret && (pool_size != nvmap_dev->pool.max))
+		nvmap_page_pool_resize(&nvmap_dev->pool, pool_size);
+
+	return ret;
+}
+
+static int pool_size_get(char *buff, const struct kernel_param *kp)
+{
+	return param_get_int(buff, kp);
+}
+
+static struct kernel_param_ops pool_size_ops = {
+	.get = pool_size_get,
+	.set = pool_size_set,
+};
+
+module_param_cb(pool_size, &pool_size_ops, &pool_size, 0644);
+
+int nvmap_page_pool_debugfs_init(struct dentry *nvmap_root)
+{
+	struct dentry *pp_root;
+
+	if (!nvmap_root)
+		return -ENODEV;
+
+	pp_root = debugfs_create_dir("pagepool", nvmap_root);
+	if (!pp_root)
+		return -ENODEV;
+
+	debugfs_create_u32("page_pool_available_pages",
+			   S_IRUGO, pp_root,
+			   &nvmap_dev->pool.count);
+	debugfs_create_u32("page_pool_pages_to_zero",
+			   S_IRUGO, pp_root,
+			   &nvmap_dev->pool.to_zero);
+#ifdef CONFIG_ARM64_4K_PAGES
+	debugfs_create_u32("page_pool_available_big_pages",
+			   S_IRUGO, pp_root,
+			   &nvmap_dev->pool.big_page_count);
+	debugfs_create_u32("page_pool_big_page_size",
+			   S_IRUGO, pp_root,
+			   &nvmap_dev->pool.big_pg_sz);
+	debugfs_create_u64("total_big_page_allocs",
+			   S_IRUGO, pp_root,
+			   &nvmap_big_page_allocs);
+#endif /* CONFIG_ARM64_4K_PAGES */
+	debugfs_create_u64("total_page_allocs",
+			   S_IRUGO, pp_root,
+			   &nvmap_total_page_allocs);
+
+#ifdef NVMAP_CONFIG_PAGE_POOL_DEBUG
+	debugfs_create_u64("page_pool_allocs",
+			   S_IRUGO, pp_root,
+			   &nvmap_dev->pool.allocs);
+	debugfs_create_u64("page_pool_fills",
+			   S_IRUGO, pp_root,
+			   &nvmap_dev->pool.fills);
+	debugfs_create_u64("page_pool_hits",
+			   S_IRUGO, pp_root,
+			   &nvmap_dev->pool.hits);
+	debugfs_create_u64("page_pool_misses",
+			   S_IRUGO, pp_root,
+			   &nvmap_dev->pool.misses);
+#endif
+
+	return 0;
+}
+
+int nvmap_page_pool_init(struct nvmap_device *dev)
+{
+	struct sysinfo info;
+	struct nvmap_page_pool *pool = &dev->pool;
+
+	memset(pool, 0x0, sizeof(*pool));
+	rt_mutex_init(&pool->lock);
+	INIT_LIST_HEAD(&pool->page_list);
+	INIT_LIST_HEAD(&pool->zero_list);
+#ifdef CONFIG_ARM64_4K_PAGES
+	INIT_LIST_HEAD(&pool->page_list_bp);
+
+	pool->big_pg_sz = NVMAP_PP_BIG_PAGE_SIZE;
+	pool->pages_per_big_pg = NVMAP_PP_BIG_PAGE_SIZE >> PAGE_SHIFT;
+#endif /* CONFIG_ARM64_4K_PAGES */
+
+	si_meminfo(&info);
+	pr_info("Total RAM pages: %lu\n", info.totalram);
+
+	if (!NVMAP_CONFIG_PAGE_POOL_SIZE)
+		/* The ratio is pool pages per 1K ram pages.
+		 * So, the >> 10 */
+		pool->max = (info.totalram * NVMAP_PP_POOL_SIZE) >> 10;
+	else
+		pool->max = NVMAP_CONFIG_PAGE_POOL_SIZE;
+
+	if (pool->max >= info.totalram)
+		goto fail;
+	pool_size = pool->max;
+
+	pr_info("nvmap page pool size: %u pages (%u MB)\n", pool->max,
+		(pool->max * info.mem_unit) >> 20);
+
+	background_allocator = kthread_run(nvmap_background_zero_thread,
+					    NULL, "nvmap-bz");
+	if (IS_ERR(background_allocator))
+		goto fail;
+
+	register_shrinker(&nvmap_page_pool_shrinker);
+
+	return 0;
+fail:
+	nvmap_page_pool_fini(dev);
+	return -ENOMEM;
+}
+
+int nvmap_page_pool_fini(struct nvmap_device *dev)
+{
+	struct nvmap_page_pool *pool = &dev->pool;
+
+	/*
+	 * if background allocator is not initialzed or not
+	 * properly initialized, then shrinker is also not
+	 * registered
+	 */
+	if (!IS_ERR_OR_NULL(background_allocator)) {
+		unregister_shrinker(&nvmap_page_pool_shrinker);
+		kthread_stop(background_allocator);
+		background_allocator = NULL;
+	}
+
+	WARN_ON(!list_empty(&pool->page_list));
+
+	return 0;
+}

drivers/video/tegra/nvmap/nvmap_priv.h

@@ -0,0 +1,904 @@
+/*
+ * drivers/video/tegra/nvmap/nvmap.h
+ *
+ * GPU memory management driver for Tegra
+ *
+ * Copyright (c) 2009-2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#ifndef __VIDEO_TEGRA_NVMAP_NVMAP_H
+#define __VIDEO_TEGRA_NVMAP_NVMAP_H
+
+#include <linux/list.h>
+#include <linux/mm.h>
+#include <linux/mutex.h>
+#include <linux/rtmutex.h>
+#include <linux/rbtree.h>
+#include <linux/sched.h>
+#include <linux/wait.h>
+#include <linux/atomic.h>
+#include <linux/dma-buf.h>
+#include <linux/syscalls.h>
+#include <linux/mm.h>
+#include <linux/miscdevice.h>
+#include <linux/nvmap.h>
+#include <linux/vmalloc.h>
+#include <linux/slab.h>
+#include <linux/version.h>
+
+#include <linux/workqueue.h>
+#include <linux/dma-mapping.h>
+#include <linux/dma-direction.h>
+#include <linux/platform_device.h>
+#include <linux/of.h>
+#include <linux/of_reserved_mem.h>
+
+#include <asm/cacheflush.h>
+#ifndef CONFIG_ARM64
+#include <asm/outercache.h>
+#endif
+#include "nvmap_heap.h"
+#include "nvmap_stats.h"
+
+#include <linux/fdtable.h>
+
+#define DMA_ERROR_CODE	(~(dma_addr_t)0)
+
+#define __DMA_ATTR(attrs) attrs
+#define DEFINE_DMA_ATTRS(attrs) unsigned long attrs = 0
+
+/**
+ * dma_set_attr - set a specific attribute
+ * @attr: attribute to set
+ * @attrs: struct dma_attrs (may be NULL)
+ */
+#define dma_set_attr(attr, attrs) (attrs |= attr)
+
+/**
+ * dma_get_attr - check for a specific attribute
+ * @attr: attribute to set
+ * @attrs: struct dma_attrs (may be NULL)
+ */
+#define dma_get_attr(attr, attrs) (attrs & attr)
+
+#define NVMAP_TAG_LABEL_MAXLEN	(63 - sizeof(struct nvmap_tag_entry))
+
+#define NVMAP_TP_ARGS_H(handle)					      	      \
+	handle,								      \
+	atomic_read(&handle->share_count),				      \
+	handle->heap_type == NVMAP_HEAP_IOVMM ? 0 : 			      \
+			(handle->carveout ? handle->carveout->base : 0),      \
+	handle->size,							      \
+	(handle->userflags & 0xFFFF),                                         \
+	(handle->userflags >> 16),					      \
+	__nvmap_tag_name(nvmap_dev, handle->userflags >> 16)
+
+#define NVMAP_TP_ARGS_CHR(client, handle, ref)			      	      \
+	client,                                                               \
+	client ? nvmap_client_pid((struct nvmap_client *)client) : 0,         \
+	(ref) ? atomic_read(&((struct nvmap_handle_ref *)ref)->dupes) : 1,    \
+	NVMAP_TP_ARGS_H(handle)
+
+#define NVMAP_TAG_TRACE(x, ...) 			\
+do {                                                    \
+	if (x##_enabled()) {                            \
+		mutex_lock(&nvmap_dev->tags_lock);      \
+		x(__VA_ARGS__);                         \
+		mutex_unlock(&nvmap_dev->tags_lock);    \
+	}                                               \
+} while (0)
+
+#define GFP_NVMAP       (GFP_KERNEL | __GFP_HIGHMEM | __GFP_NOWARN)
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0)
+
+/*
+ * DMA_ATTR_ALLOC_EXACT_SIZE: This tells the DMA-mapping
+ * subsystem to allocate the exact number of pages
+ */
+#define DMA_ATTR_ALLOC_EXACT_SIZE	(DMA_ATTR_PRIVILEGED << 2)
+
+#define DMA_MEMORY_NOMAP		0x02
+
+#endif /* LINUX_VERSION_CODE */
+
+#ifdef NVMAP_LOADABLE_MODULE
+
+#ifdef NVMAP_UPSTREAM_KERNEL
+/*
+ * DMA_ATTR_READ_ONLY: for DMA memory allocations, attempt to map
+ * memory as read-only for the device. CPU access will still be
+ * read-write. This corresponds to the direction being DMA_TO_DEVICE
+ * instead of DMA_BIDIRECTIONAL.
+ */
+#define DMA_ATTR_READ_ONLY	(DMA_ATTR_PRIVILEGED << 12)
+
+/* DMA_ATTR_WRITE_ONLY: This tells the DMA-mapping subsystem
+ * to map as write-only
+ */
+#define DMA_ATTR_WRITE_ONLY	(DMA_ATTR_PRIVILEGED << 13)
+
+#endif /* NVMAP_UPSTREAM_KERNEL */
+#endif /* NVMAP_LOADABLE_MODULE */
+
+#define DMA_ALLOC_FREE_ATTR	DMA_ATTR_ALLOC_SINGLE_PAGES
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0)
+#define ACCESS_OK(type, addr, size)	access_ok(type, addr, size)
+#define SYS_CLOSE(arg)	sys_close(arg)
+#elif LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0)
+#define ACCESS_OK(type, addr, size)    access_ok(addr, size)
+#define SYS_CLOSE(arg)	ksys_close(arg)
+#else
+#define ACCESS_OK(type, addr, size)    access_ok(addr, size)
+#define SYS_CLOSE(arg) close_fd(arg)
+#endif
+
+struct page;
+struct nvmap_device;
+
+void _nvmap_handle_free(struct nvmap_handle *h);
+/* holds max number of handles allocted per process at any time */
+extern u32 nvmap_max_handle_count;
+extern u64 nvmap_big_page_allocs;
+extern u64 nvmap_total_page_allocs;
+
+extern bool nvmap_convert_iovmm_to_carveout;
+extern bool nvmap_convert_carveout_to_iovmm;
+
+extern struct vm_operations_struct nvmap_vma_ops;
+
+#ifdef CONFIG_ARM64
+#define PG_PROT_KERNEL PAGE_KERNEL
+#define FLUSH_DCACHE_AREA __flush_dcache_area
+#define outer_flush_range(s, e)
+#define outer_inv_range(s, e)
+#define outer_clean_range(s, e)
+#define outer_flush_all()
+#define outer_clean_all()
+extern void __clean_dcache_page(struct page *);
+extern void __clean_dcache_area_poc(void *addr, size_t len);
+#else
+#define PG_PROT_KERNEL pgprot_kernel
+#define FLUSH_DCACHE_AREA __cpuc_flush_dcache_area
+extern void __flush_dcache_page(struct address_space *, struct page *);
+#endif
+
+struct nvmap_vma_list {
+	struct list_head list;
+	struct vm_area_struct *vma;
+	unsigned long save_vm_flags;
+	pid_t pid;
+	atomic_t ref;
+};
+
+struct nvmap_carveout_node {
+	unsigned int		heap_bit;
+	struct nvmap_heap	*carveout;
+	int			index;
+	phys_addr_t		base;
+	size_t			size;
+};
+
+/* handles allocated using shared system memory (either IOVMM- or high-order
+ * page allocations */
+struct nvmap_pgalloc {
+	struct page **pages;
+	bool contig;			/* contiguous system memory */
+	atomic_t reserved;
+	atomic_t ndirty;	/* count number of dirty pages */
+};
+
+#ifdef NVMAP_CONFIG_DEBUG_MAPS
+struct nvmap_device_list {
+	struct rb_node node;
+	u64 dma_mask;
+	char *device_name;
+};
+#endif /* NVMAP_CONFIG_DEBUG_MAPS */
+
+/* bit 31-29: IVM peer
+ * bit 28-16: offset (aligned to 32K)
+ * bit 15-00: len (aligned to page_size)
+ */
+#define NVMAP_IVM_LENGTH_SHIFT (0)
+#define NVMAP_IVM_LENGTH_WIDTH (16)
+#define NVMAP_IVM_LENGTH_MASK  ((1 << NVMAP_IVM_LENGTH_WIDTH) - 1)
+#define NVMAP_IVM_OFFSET_SHIFT (NVMAP_IVM_LENGTH_SHIFT + NVMAP_IVM_LENGTH_WIDTH)
+#define NVMAP_IVM_OFFSET_WIDTH (13)
+#define NVMAP_IVM_OFFSET_MASK  ((1 << NVMAP_IVM_OFFSET_WIDTH) - 1)
+#define NVMAP_IVM_IVMID_SHIFT  (NVMAP_IVM_OFFSET_SHIFT + NVMAP_IVM_OFFSET_WIDTH)
+#define NVMAP_IVM_IVMID_WIDTH  (3)
+#define NVMAP_IVM_IVMID_MASK   ((1 << NVMAP_IVM_IVMID_WIDTH) - 1)
+#define NVMAP_IVM_ALIGNMENT    (SZ_32K)
+
+struct nvmap_handle_dmabuf_priv {
+	void *priv;
+	struct device *dev;
+	void (*priv_release)(void *priv);
+	struct list_head list;
+};
+
+struct nvmap_handle {
+	struct rb_node node;	/* entry on global handle tree */
+	atomic_t ref;		/* reference count (i.e., # of duplications) */
+	atomic_t pin;		/* pin count */
+	u32 flags;		/* caching flags */
+	size_t size;		/* padded (as-allocated) size */
+	size_t orig_size;	/* original (as-requested) size */
+	size_t align;
+	struct nvmap_client *owner;
+	struct dma_buf *dmabuf;
+	struct dma_buf *dmabuf_ro;
+	union {
+		struct nvmap_pgalloc pgalloc;
+		struct nvmap_heap_block *carveout;
+	};
+	bool heap_pgalloc;	/* handle is page allocated (sysmem / iovmm) */
+	bool alloc;		/* handle has memory allocated */
+	bool from_va;		/* handle memory is from VA */
+	u32 heap_type;		/* handle heap is allocated from */
+	u32 userflags;		/* flags passed from userspace */
+	void *vaddr;		/* mapping used inside kernel */
+	struct list_head vmas;	/* list of all user vma's */
+	atomic_t umap_count;	/* number of outstanding maps from user */
+	atomic_t kmap_count;	/* number of outstanding map from kernel */
+	atomic_t share_count;	/* number of processes sharing the handle */
+	struct list_head lru;	/* list head to track the lru */
+	struct mutex lock;
+	struct list_head dmabuf_priv;
+	u64 ivm_id;
+	unsigned int peer;	/* Peer VM number */
+	int offs;		/* Offset in IVM mem pool */
+	/*
+	 * To be set only in handle created from VA case if the handle is
+	 * read-only.
+	 */
+	bool is_ro;
+};
+
+struct nvmap_handle_info {
+	struct nvmap_handle *handle;
+	struct list_head maps;
+	struct mutex maps_lock;
+	bool is_ro;
+};
+
+struct nvmap_tag_entry {
+	struct rb_node node;
+	atomic_t ref;		/* reference count (i.e., # of duplications) */
+	u32 tag;
+};
+
+/* handle_ref objects are client-local references to an nvmap_handle;
+ * they are distinct objects so that handles can be unpinned and
+ * unreferenced the correct number of times when a client abnormally
+ * terminates */
+struct nvmap_handle_ref {
+	struct nvmap_handle *handle;
+	struct rb_node	node;
+	atomic_t	dupes;	/* number of times to free on file close */
+	bool is_ro;
+};
+
+#if defined(NVMAP_CONFIG_PAGE_POOLS)
+/*
+ * This is the default ratio defining pool size. It can be thought of as pool
+ * size in either MB per GB or KB per MB. That means the max this number can
+ * be is 1024 (all physical memory - not a very good idea) or 0 (no page pool
+ * at all).
+ */
+#define NVMAP_PP_POOL_SIZE               (128)
+
+#ifdef CONFIG_ARM64_4K_PAGES
+#define NVMAP_PP_BIG_PAGE_SIZE           (0x10000)
+#endif /* CONFIG_ARM64_4K_PAGES */
+struct nvmap_page_pool {
+	struct rt_mutex lock;
+	u32 count;      /* Number of pages in the page & dirty list. */
+	u32 max;        /* Max no. of pages in all lists. */
+	u32 to_zero;    /* Number of pages on the zero list */
+	u32 under_zero; /* Number of pages getting zeroed */
+#ifdef CONFIG_ARM64_4K_PAGES
+	u32 big_pg_sz;  /* big page size supported(64k, etc.) */
+	u32 big_page_count;   /* Number of zeroed big pages avaialble */
+	u32 pages_per_big_pg; /* Number of pages in big page */
+#endif /* CONFIG_ARM64_4K_PAGES */
+	struct list_head page_list;
+	struct list_head zero_list;
+#ifdef CONFIG_ARM64_4K_PAGES
+	struct list_head page_list_bp;
+#endif /* CONFIG_ARM64_4K_PAGES */
+
+#ifdef NVMAP_CONFIG_PAGE_POOL_DEBUG
+	u64 allocs;
+	u64 fills;
+	u64 hits;
+	u64 misses;
+#endif
+};
+
+int nvmap_page_pool_init(struct nvmap_device *dev);
+int nvmap_page_pool_fini(struct nvmap_device *dev);
+struct page *nvmap_page_pool_alloc(struct nvmap_page_pool *pool);
+int nvmap_page_pool_alloc_lots(struct nvmap_page_pool *pool,
+					struct page **pages, u32 nr);
+#ifdef CONFIG_ARM64_4K_PAGES
+int nvmap_page_pool_alloc_lots_bp(struct nvmap_page_pool *pool,
+					struct page **pages, u32 nr);
+#endif /* CONFIG_ARM64_4K_PAGES */
+u32 nvmap_page_pool_fill_lots(struct nvmap_page_pool *pool,
+				       struct page **pages, u32 nr);
+int nvmap_page_pool_clear(void);
+int nvmap_page_pool_debugfs_init(struct dentry *nvmap_root);
+#endif
+
+#define NVMAP_IVM_INVALID_PEER		(-1)
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0)
+#ifndef NVMAP_CONFIG_HANDLE_AS_ID
+struct xarray {};
+#endif /* !NVMAP_CONFIG_HANDLE_AS_ID */
+#endif /* KERNEL_VERSION < 5.10 */
+
+struct nvmap_client {
+	const char			*name;
+	struct rb_root			handle_refs;
+	struct mutex			ref_lock;
+	bool				kernel_client;
+	atomic_t			count;
+	struct task_struct		*task;
+	struct list_head		list;
+	u32				handle_count;
+	u32				next_fd;
+	int				warned;
+	int				tag_warned;
+	struct xarray			id_array;
+	struct xarray			*ida;
+};
+
+struct nvmap_vma_priv {
+	struct nvmap_handle *handle;
+	size_t		offs;
+	atomic_t	count;	/* number of processes cloning the VMA */
+};
+
+struct nvmap_device {
+	struct rb_root	handles;
+	spinlock_t	handle_lock;
+	struct miscdevice dev_user;
+	struct nvmap_carveout_node *heaps;
+	int nr_heaps;
+	int nr_carveouts;
+#ifdef NVMAP_CONFIG_PAGE_POOLS
+	struct nvmap_page_pool pool;
+#endif
+	struct list_head clients;
+	struct rb_root pids;
+	struct mutex	clients_lock;
+	struct list_head lru_handles;
+	spinlock_t	lru_lock;
+	struct dentry *handles_by_pid;
+	struct dentry *debug_root;
+	struct nvmap_platform_data *plat;
+	struct rb_root	tags;
+	struct mutex	tags_lock;
+	struct mutex carveout_lock; /* needed to serialize carveout creation */
+	u32 dynamic_dma_map_mask;
+	u32 cpu_access_mask;
+#ifdef NVMAP_CONFIG_DEBUG_MAPS
+	struct rb_root device_names;
+#endif /* NVMAP_CONFIG_DEBUG_MAPS */
+};
+
+
+extern struct nvmap_device *nvmap_dev;
+extern ulong nvmap_init_time;
+
+static inline void nvmap_ref_lock(struct nvmap_client *priv)
+{
+	mutex_lock(&priv->ref_lock);
+}
+
+static inline void nvmap_ref_unlock(struct nvmap_client *priv)
+{
+	mutex_unlock(&priv->ref_lock);
+}
+
+static inline void nvmap_acquire_mmap_read_lock(struct mm_struct *mm)
+{
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0)
+	down_read(&mm->mmap_sem);
+#else
+	down_read(&mm->mmap_lock);
+#endif
+}
+
+static inline void nvmap_release_mmap_read_lock(struct mm_struct *mm)
+{
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0)
+	up_read(&mm->mmap_sem);
+#else
+	up_read(&mm->mmap_lock);
+#endif
+}
+
+static inline pgprot_t nvmap_pgprot(struct nvmap_handle *h, pgprot_t prot)
+{
+	if (h->flags == NVMAP_HANDLE_UNCACHEABLE) {
+#ifdef CONFIG_ARM64
+		if (h->heap_type != NVMAP_HEAP_CARVEOUT_VPR &&
+		    h->owner && !h->owner->warned) {
+			char task_comm[TASK_COMM_LEN];
+			h->owner->warned = 1;
+			get_task_comm(task_comm, h->owner->task);
+			pr_err("PID %d: %s: TAG: 0x%04x WARNING: "
+				"NVMAP_HANDLE_WRITE_COMBINE "
+				"should be used in place of "
+				"NVMAP_HANDLE_UNCACHEABLE on ARM64\n",
+				h->owner->task->pid, task_comm,
+				h->userflags >> 16);
+		}
+#endif
+		return pgprot_noncached(prot);
+	}
+	else if (h->flags == NVMAP_HANDLE_WRITE_COMBINE)
+		return pgprot_writecombine(prot);
+	return prot;
+}
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)
+struct dma_coherent_mem_replica {
+	void		*virt_base;
+	dma_addr_t	device_base;
+	unsigned long	pfn_base;
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0)
+	size_t		size;
+#else
+	int		size;
+#endif
+	int		flags;
+	unsigned long	*bitmap;
+	spinlock_t	spinlock;
+	bool		use_dev_dma_pfn_offset;
+};
+
+int nvmap_dma_declare_coherent_memory(struct device *dev, phys_addr_t phys_addr,
+			dma_addr_t device_addr, size_t size, int flags);
+#endif
+int nvmap_probe(struct platform_device *pdev);
+int nvmap_remove(struct platform_device *pdev);
+int nvmap_init(struct platform_device *pdev);
+
+int nvmap_create_carveout(const struct nvmap_platform_carveout *co);
+int nvmap_co_setup(struct reserved_mem *rmem);
+
+struct device *dma_dev_from_handle(unsigned long type);
+struct nvmap_heap_block *nvmap_carveout_alloc(struct nvmap_client *dev,
+					      struct nvmap_handle *handle,
+					      unsigned long type,
+					      phys_addr_t *start);
+
+struct nvmap_carveout_node;
+
+struct nvmap_handle *nvmap_handle_get(struct nvmap_handle *h);
+void nvmap_handle_put(struct nvmap_handle *h);
+
+struct nvmap_handle_ref *__nvmap_validate_locked(struct nvmap_client *priv,
+						 struct nvmap_handle *h,
+						 bool is_ro);
+
+struct nvmap_handle *nvmap_validate_get(struct nvmap_handle *h);
+
+struct nvmap_handle_ref *nvmap_create_handle(struct nvmap_client *client,
+					     size_t size, bool ro_buf);
+
+struct nvmap_handle_ref *nvmap_create_handle_from_va(struct nvmap_client *client,
+						     ulong addr, size_t size,
+						     unsigned int access_flags);
+
+struct nvmap_handle_ref *nvmap_dup_handle_ro(struct nvmap_client *client,
+					int fd);
+
+bool is_nvmap_dmabuf_fd_ro(int fd);
+
+bool is_nvmap_id_ro(struct nvmap_client *client, int id);
+
+struct nvmap_handle_ref *nvmap_duplicate_handle(struct nvmap_client *client,
+					struct nvmap_handle *h, bool skip_val,
+					bool is_ro);
+
+struct nvmap_handle_ref *nvmap_try_duplicate_by_ivmid(
+			struct nvmap_client *client, u64 ivm_id,
+			struct nvmap_heap_block **block);
+
+struct nvmap_handle_ref *nvmap_create_handle_from_id(
+			struct nvmap_client *client, u32 id);
+
+struct nvmap_handle_ref *nvmap_create_handle_from_fd(
+			struct nvmap_client *client, int fd);
+
+void inner_cache_maint(unsigned int op, void *vaddr, size_t size);
+void outer_cache_maint(unsigned int op, phys_addr_t paddr, size_t size);
+
+int nvmap_alloc_handle(struct nvmap_client *client,
+		       struct nvmap_handle *h, unsigned int heap_mask,
+		       size_t align, u8 kind,
+		       unsigned int flags, unsigned int peer);
+
+int nvmap_alloc_handle_from_va(struct nvmap_client *client,
+			       struct nvmap_handle *h,
+			       ulong addr,
+			       unsigned int flags);
+
+void nvmap_free_handle(struct nvmap_client *c, struct nvmap_handle *h, bool is_ro);
+
+void nvmap_free_handle_from_fd(struct nvmap_client *c, int fd);
+
+int nvmap_handle_remove(struct nvmap_device *dev, struct nvmap_handle *h);
+
+void nvmap_handle_add(struct nvmap_device *dev, struct nvmap_handle *h);
+
+int is_nvmap_vma(struct vm_area_struct *vma);
+
+int nvmap_get_dmabuf_fd(struct nvmap_client *client, struct nvmap_handle *h,
+			bool is_ro);
+struct nvmap_handle *nvmap_handle_get_from_dmabuf_fd(
+				struct nvmap_client *client, int fd);
+int nvmap_dmabuf_duplicate_gen_fd(struct nvmap_client *client,
+		struct dma_buf *dmabuf);
+
+int nvmap_get_handle_param(struct nvmap_client *client,
+		struct nvmap_handle_ref *ref, u32 param, u64 *result);
+
+struct nvmap_handle *nvmap_handle_get_from_fd(int fd);
+
+/* MM definitions. */
+extern void v7_flush_kern_cache_all(void);
+extern void v7_clean_kern_cache_all(void *);
+
+void nvmap_clean_cache(struct page **pages, int numpages);
+void nvmap_clean_cache_page(struct page *page);
+void nvmap_flush_cache(struct page **pages, int numpages);
+int nvmap_cache_maint_phys_range(unsigned int op, phys_addr_t pstart,
+		phys_addr_t pend, int inner, int outer);
+
+int nvmap_do_cache_maint_list(struct nvmap_handle **handles, u64 *offsets,
+			      u64 *sizes, int op, u32 nr_ops, bool is_32);
+int __nvmap_cache_maint(struct nvmap_client *client,
+			       struct nvmap_cache_op_64 *op);
+
+/* Internal API to support dmabuf */
+struct dma_buf *__nvmap_make_dmabuf(struct nvmap_client *client,
+				    struct nvmap_handle *handle, bool ro_buf);
+struct sg_table *__nvmap_sg_table(struct nvmap_client *client,
+				  struct nvmap_handle *h);
+void __nvmap_free_sg_table(struct nvmap_client *client,
+			   struct nvmap_handle *h, struct sg_table *sgt);
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0)
+void *__nvmap_kmap(struct nvmap_handle *h, unsigned int pagenum);
+void __nvmap_kunmap(struct nvmap_handle *h, unsigned int pagenum, void *addr);
+#endif
+void *__nvmap_mmap(struct nvmap_handle *h);
+void __nvmap_munmap(struct nvmap_handle *h, void *addr);
+int __nvmap_map(struct nvmap_handle *h, struct vm_area_struct *vma);
+int __nvmap_do_cache_maint(struct nvmap_client *client, struct nvmap_handle *h,
+			   unsigned long start, unsigned long end,
+			   unsigned int op, bool clean_only_dirty);
+struct nvmap_client *__nvmap_create_client(struct nvmap_device *dev,
+					   const char *name);
+int __nvmap_dmabuf_fd(struct nvmap_client *client,
+		      struct dma_buf *dmabuf, int flags);
+
+int nvmap_dmabuf_stash_init(void);
+
+void *nvmap_altalloc(size_t len);
+void nvmap_altfree(void *ptr, size_t len);
+
+static inline struct page *nvmap_to_page(struct page *page)
+{
+	return (struct page *)((unsigned long)page & ~3UL);
+}
+
+static inline bool nvmap_page_dirty(struct page *page)
+{
+	return (unsigned long)page & 1UL;
+}
+
+static inline bool nvmap_page_mkdirty(struct page **page)
+{
+	if (nvmap_page_dirty(*page))
+		return false;
+	*page = (struct page *)((unsigned long)*page | 1UL);
+	return true;
+}
+
+static inline bool nvmap_page_mkclean(struct page **page)
+{
+	if (!nvmap_page_dirty(*page))
+		return false;
+	*page = (struct page *)((unsigned long)*page & ~1UL);
+	return true;
+}
+
+/*
+ * FIXME: assume user space requests for reserve operations
+ * are page aligned
+ */
+static inline int nvmap_handle_mk(struct nvmap_handle *h,
+				  u32 offset, u32 size,
+				  bool (*fn)(struct page **),
+				  bool locked)
+{
+	int i, nchanged = 0;
+	u32 start_page = offset >> PAGE_SHIFT;
+	u32 end_page = PAGE_ALIGN(offset + size) >> PAGE_SHIFT;
+
+	if (!locked)
+		mutex_lock(&h->lock);
+	if (h->heap_pgalloc &&
+		(offset < h->size) &&
+		(size <= h->size) &&
+		(offset <= (h->size - size))) {
+		for (i = start_page; i < end_page; i++)
+			nchanged += fn(&h->pgalloc.pages[i]) ? 1 : 0;
+	}
+	if (!locked)
+		mutex_unlock(&h->lock);
+	return nchanged;
+}
+
+static inline void nvmap_handle_mkclean(struct nvmap_handle *h,
+					u32 offset, u32 size)
+{
+	int nchanged;
+
+	if (h->heap_pgalloc && !atomic_read(&h->pgalloc.ndirty))
+		return;
+	if (size == 0)
+		size = h->size;
+
+	nchanged = nvmap_handle_mk(h, offset, size, nvmap_page_mkclean, false);
+	if (h->heap_pgalloc)
+		atomic_sub(nchanged, &h->pgalloc.ndirty);
+}
+
+static inline void _nvmap_handle_mkdirty(struct nvmap_handle *h,
+					u32 offset, u32 size)
+{
+	int nchanged;
+
+	if (h->heap_pgalloc &&
+		(atomic_read(&h->pgalloc.ndirty) == (h->size >> PAGE_SHIFT)))
+		return;
+
+	nchanged = nvmap_handle_mk(h, offset, size, nvmap_page_mkdirty, true);
+	if (h->heap_pgalloc)
+		atomic_add(nchanged, &h->pgalloc.ndirty);
+}
+
+static inline struct page **nvmap_pages(struct page **pg_pages, u32 nr_pages)
+{
+	struct page **pages;
+	int i;
+
+	pages = nvmap_altalloc(sizeof(*pages) * nr_pages);
+	if (!pages)
+		return NULL;
+
+	for (i = 0; i < nr_pages; i++)
+		pages[i] = nvmap_to_page(pg_pages[i]);
+
+	return pages;
+}
+
+void nvmap_zap_handle(struct nvmap_handle *handle, u64 offset, u64 size);
+
+void nvmap_vma_open(struct vm_area_struct *vma);
+
+int nvmap_reserve_pages(struct nvmap_handle **handles, u64 *offsets,
+			u64 *sizes, u32 nr, u32 op, bool is_32);
+
+static inline void nvmap_kmaps_inc(struct nvmap_handle *h)
+{
+	mutex_lock(&h->lock);
+	atomic_inc(&h->kmap_count);
+	mutex_unlock(&h->lock);
+}
+
+static inline void nvmap_kmaps_inc_no_lock(struct nvmap_handle *h)
+{
+	atomic_inc(&h->kmap_count);
+}
+
+static inline void nvmap_kmaps_dec(struct nvmap_handle *h)
+{
+	atomic_dec(&h->kmap_count);
+}
+
+static inline void nvmap_umaps_inc(struct nvmap_handle *h)
+{
+	mutex_lock(&h->lock);
+	atomic_inc(&h->umap_count);
+	mutex_unlock(&h->lock);
+}
+
+static inline void nvmap_umaps_dec(struct nvmap_handle *h)
+{
+	atomic_dec(&h->umap_count);
+}
+
+static inline void nvmap_lru_add(struct nvmap_handle *h)
+{
+	spin_lock(&nvmap_dev->lru_lock);
+	BUG_ON(!list_empty(&h->lru));
+	list_add_tail(&h->lru, &nvmap_dev->lru_handles);
+	spin_unlock(&nvmap_dev->lru_lock);
+}
+
+static inline void nvmap_lru_del(struct nvmap_handle *h)
+{
+	spin_lock(&nvmap_dev->lru_lock);
+	list_del(&h->lru);
+	INIT_LIST_HEAD(&h->lru);
+	spin_unlock(&nvmap_dev->lru_lock);
+}
+
+static inline void nvmap_lru_reset(struct nvmap_handle *h)
+{
+	spin_lock(&nvmap_dev->lru_lock);
+	BUG_ON(list_empty(&h->lru));
+	list_del(&h->lru);
+	list_add_tail(&h->lru, &nvmap_dev->lru_handles);
+	spin_unlock(&nvmap_dev->lru_lock);
+}
+
+static inline bool nvmap_handle_track_dirty(struct nvmap_handle *h)
+{
+	if (!h->heap_pgalloc)
+		return false;
+
+	return h->userflags & (NVMAP_HANDLE_CACHE_SYNC |
+			       NVMAP_HANDLE_CACHE_SYNC_AT_RESERVE);
+}
+
+struct nvmap_tag_entry *nvmap_search_tag_entry(struct rb_root *root, u32 tag);
+
+int nvmap_define_tag(struct nvmap_device *dev, u32 tag,
+	const char __user *name, u32 len);
+
+int nvmap_remove_tag(struct nvmap_device *dev, u32 tag);
+
+/* must hold tag_lock */
+static inline char *__nvmap_tag_name(struct nvmap_device *dev, u32 tag)
+{
+	struct nvmap_tag_entry *entry;
+
+	entry = nvmap_search_tag_entry(&dev->tags, tag);
+	return entry ? (char *)(entry + 1) : "";
+}
+static inline pid_t nvmap_client_pid(struct nvmap_client *client)
+{
+	return client->task ? client->task->pid : 0;
+}
+
+/* must be called with mmap_sem held for read or write */
+static inline int nvmap_get_user_pages(ulong vaddr,
+				size_t nr_page, struct page **pages,
+				bool is_user_flags, u32 user_foll_flags)
+{
+	u32 foll_flags = FOLL_FORCE;
+	struct vm_area_struct *vma;
+	vm_flags_t vm_flags;
+	long user_pages = 0;
+	int ret = 0;
+
+	vma = find_vma(current->mm, vaddr);
+	if (vma) {
+		if (is_user_flags) {
+			foll_flags |= user_foll_flags;
+		} else {
+			vm_flags = vma->vm_flags;
+			/*
+			 * If the vaddr points to writable page then only
+			 * pass FOLL_WRITE flag
+			 */
+			if (vm_flags & VM_WRITE)
+				foll_flags |= FOLL_WRITE;
+		}
+		pr_debug("vaddr %lu is_user_flags %d user_foll_flags %x foll_flags %x.\n",
+			vaddr, is_user_flags?1:0, user_foll_flags, foll_flags);
+		user_pages = get_user_pages(vaddr & PAGE_MASK, nr_page,
+					    foll_flags, pages, NULL);
+	}
+	if (user_pages != nr_page) {
+		ret = user_pages < 0 ? user_pages : -ENOMEM;
+		pr_err("get_user_pages requested/got: %zu/%ld]\n", nr_page,
+				user_pages);
+		while (--user_pages >= 0)
+			put_page(pages[user_pages]);
+	}
+	return ret;
+}
+
+#define device_node_from_iter(iter) \
+	iter.node
+
+extern struct of_device_id __nvmapcache_of_table;
+
+#define NVMAP_CACHE_OF_DECLARE(compat, fn) \
+	_OF_DECLARE(nvmapcache, nvmapcache_of, compat, fn, \
+			nvmap_setup_chip_cache_fn)
+
+#ifdef NVMAP_CONFIG_SCIIPC
+int nvmap_sci_ipc_init(void);
+void nvmap_sci_ipc_exit(void);
+#else
+__weak int nvmap_sci_ipc_init(void)
+{
+	return 0;
+}
+__weak void nvmap_sci_ipc_exit(void)
+{
+}
+#endif
+
+#ifdef NVMAP_CONFIG_HANDLE_AS_ID
+void nvmap_id_array_init(struct xarray *xarr);
+void nvmap_id_array_exit(struct xarray *xarr);
+struct dma_buf *nvmap_id_array_get_dmabuf_from_id(struct xarray *xarr, u32 id);
+int nvmap_id_array_id_alloc(struct xarray *xarr, u32 *id, struct dma_buf *dmabuf);
+struct dma_buf *nvmap_id_array_id_release(struct xarray *xarr, u32 id);
+#else
+static inline void nvmap_id_array_init(struct xarray *xarr)
+{
+
+}
+
+static inline void nvmap_id_array_exit(struct xarray *xarr)
+{
+
+}
+
+static inline struct dma_buf *nvmap_id_array_get_dmabuf_from_id(struct xarray *xarr, u32 id)
+{
+	return NULL;
+}
+
+static inline int nvmap_id_array_id_alloc(struct xarray *xarr, u32 *id, struct dma_buf *dmabuf)
+{
+	return 0;
+}
+
+static inline struct dma_buf *nvmap_id_array_id_release(struct xarray *xarr, u32 id)
+{
+	return NULL;
+}
+#endif
+void *nvmap_dmabuf_get_drv_data(struct dma_buf *dmabuf,
+		struct device *dev);
+bool is_nvmap_memory_available(size_t size, uint32_t heap);
+int system_heap_free_mem(unsigned long *mem_val);
+
+#ifdef NVMAP_CONFIG_DEBUG_MAPS
+struct nvmap_device_list *nvmap_is_device_present(char *device_name, u32 heap_type);
+void nvmap_add_device_name(char *device_name, u64 dma_mask, u32 heap_type);
+void nvmap_remove_device_name(char *device_name, u32 heap_type);
+#endif /* NVMAP_CONFIG_DEBUG_MAPS */
+
+bool dmabuf_is_nvmap(struct dma_buf *dmabuf);
+struct nvmap_handle *nvmap_handle_get_from_id(struct nvmap_client *client,
+		u32 id);
+void *nvmap_dma_mark_declared_memory_occupied(struct device *dev,
+					dma_addr_t device_addr, size_t size);
+void nvmap_dma_mark_declared_memory_unoccupied(struct device *dev,
+					 dma_addr_t device_addr, size_t size);
+#endif /* __VIDEO_TEGRA_NVMAP_NVMAP_H */

drivers/video/tegra/nvmap/nvmap_sci_ipc.c

@@ -0,0 +1,365 @@
+/*
+ * drivers/video/tegra/nvmap/nvmap_sci_ipc.c
+ *
+ * mapping between nvmap_hnadle and sci_ipc entery
+ *
+ * Copyright (c) 2019-2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#define pr_fmt(fmt)	"%s: " fmt, __func__
+
+#include <linux/slab.h>
+#include <linux/nvmap.h>
+#include <linux/rbtree.h>
+#include <linux/list.h>
+#include <linux/mman.h>
+
+#include <linux/nvscierror.h>
+#include <linux/nvsciipc_interface.h>
+
+#include <trace/events/nvmap.h>
+#include "nvmap_priv.h"
+#include "nvmap_sci_ipc.h"
+
+struct nvmap_sci_ipc {
+	struct rb_root entries;
+	struct mutex mlock;
+	struct list_head free_sid_list;
+};
+
+struct free_sid_node {
+	struct list_head list;
+	u32 sid;
+};
+
+/* An rb-tree root node for holding sci_ipc_id of clients */
+struct nvmap_sci_ipc_entry {
+	struct rb_node entry;
+	struct nvmap_client *client;
+	struct nvmap_handle *handle;
+	u32 sci_ipc_id;
+	u64 peer_vuid;
+	u32 flags;
+	u32 refcount;
+};
+
+static struct nvmap_sci_ipc *nvmapsciipc;
+
+int nvmap_validate_sci_ipc_params(struct nvmap_client *client,
+			NvSciIpcEndpointAuthToken auth_token,
+			NvSciIpcEndpointVuid *pr_vuid,
+			NvSciIpcEndpointVuid *lu_vuid)
+{
+	NvSciError err = NvSciError_Success;
+	NvSciIpcTopoId pr_topoid;
+	int ret = 0;
+
+	err = NvSciIpcEndpointValidateAuthTokenLinuxCurrent(auth_token,
+			lu_vuid);
+	if (err != NvSciError_Success) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	err = NvSciIpcEndpointMapVuid(*lu_vuid, &pr_topoid, pr_vuid);
+	if (err != NvSciError_Success) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+out:
+	return ret;
+}
+
+static u32 nvmap_unique_sci_ipc_id(void)
+{
+	static atomic_t unq_id = { 0 };
+	u32 id;
+
+	if (!list_empty(&nvmapsciipc->free_sid_list)) {
+		struct free_sid_node *fnode = list_first_entry(
+			&nvmapsciipc->free_sid_list,
+			typeof(*fnode),
+			list);
+
+		id = fnode->sid;
+		list_del(&fnode->list);
+		kfree(fnode);
+		goto ret_id;
+	}
+
+	id = atomic_add_return(2, &unq_id);
+
+ret_id:
+	WARN_ON(id == 0);
+	return id;
+}
+
+static struct nvmap_sci_ipc_entry *nvmap_search_sci_ipc_entry(
+	struct rb_root *root,
+	struct nvmap_handle *h,
+	u32 flags,
+	NvSciIpcEndpointVuid peer_vuid)
+{
+	struct rb_node *node;  /* top of the tree */
+	struct nvmap_sci_ipc_entry *entry;
+
+	for (node = rb_first(root); node; node = rb_next(node)) {
+		entry = rb_entry(node, struct nvmap_sci_ipc_entry, entry);
+
+		if (entry && entry->handle == h
+			&& entry->flags == flags
+			&& entry->peer_vuid == peer_vuid)
+			return entry;
+	}
+	return NULL;
+}
+
+static void nvmap_insert_sci_ipc_entry(struct rb_root *root,
+		struct nvmap_sci_ipc_entry *new)
+{
+	struct nvmap_sci_ipc_entry *entry;
+	struct rb_node *parent = NULL;
+	u32 sid = new->sci_ipc_id;
+	struct rb_node **link;
+
+	link = &root->rb_node;
+	/* Go to the bottom of the tree */
+	while (*link) {
+		parent = *link;
+		entry = rb_entry(parent, struct nvmap_sci_ipc_entry, entry);
+
+		if (entry->sci_ipc_id > sid)
+			link = &parent->rb_left;
+		else
+			link = &parent->rb_right;
+	}
+
+	/* Put the new node there */
+	rb_link_node(&new->entry, parent, link);
+	rb_insert_color(&new->entry, root);
+}
+
+int nvmap_create_sci_ipc_id(struct nvmap_client *client,
+				struct nvmap_handle *h,
+				u32 flags,
+				u32 *sci_ipc_id,
+				NvSciIpcEndpointVuid peer_vuid,
+				bool is_ro)
+{
+	struct nvmap_sci_ipc_entry *new_entry;
+	struct nvmap_sci_ipc_entry *entry;
+	int ret = -EINVAL;
+	u32 id;
+
+	mutex_lock(&nvmapsciipc->mlock);
+
+	entry = nvmap_search_sci_ipc_entry(&nvmapsciipc->entries,
+			h, flags, peer_vuid);
+	if (entry) {
+		entry->refcount++;
+		*sci_ipc_id = entry->sci_ipc_id;
+		pr_debug("%d: matched Sci_Ipc_Id:%u\n", __LINE__, *sci_ipc_id);
+		ret = 0;
+		goto unlock;
+	} else {
+		new_entry = kzalloc(sizeof(*new_entry), GFP_KERNEL);
+		if (!new_entry) {
+			ret = -ENOMEM;
+			goto unlock;
+		}
+		id = nvmap_unique_sci_ipc_id();
+		*sci_ipc_id = id;
+		new_entry->sci_ipc_id = id;
+		new_entry->client = client;
+		new_entry->handle = h;
+		new_entry->peer_vuid = peer_vuid;
+		new_entry->flags = flags;
+		new_entry->refcount = 1;
+
+		pr_debug("%d: New Sci_ipc_id %d entry->pr_vuid: %llu entry->flags: %u new_entry->handle:0x%p\n",
+			__LINE__, new_entry->sci_ipc_id, new_entry->peer_vuid,
+			new_entry->flags, new_entry->handle);
+
+		nvmap_insert_sci_ipc_entry(&nvmapsciipc->entries, new_entry);
+		ret = 0;
+	}
+unlock:
+	mutex_unlock(&nvmapsciipc->mlock);
+	if (!ret)
+		(void)nvmap_handle_get(h);
+
+	return ret;
+}
+
+static struct nvmap_sci_ipc_entry *nvmap_find_entry_for_id(struct rb_root *es,
+					 u32 id)
+{
+	struct nvmap_sci_ipc_entry *e = NULL;
+	struct rb_node *n;
+
+	for (n = rb_first(es); n; n = rb_next(n)) {
+		e = rb_entry(n, struct nvmap_sci_ipc_entry, entry);
+		if (e && e->sci_ipc_id == id)
+			goto found;
+	}
+found:
+	return e;
+}
+
+int nvmap_get_handle_from_sci_ipc_id(struct nvmap_client *client, u32 flags,
+		u32 sci_ipc_id, NvSciIpcEndpointVuid localu_vuid, u32 *handle)
+{
+	bool is_ro = (flags == PROT_READ) ? true : false, dmabuf_created = false;
+	struct nvmap_handle_ref *ref = NULL;
+	struct nvmap_sci_ipc_entry *entry;
+	struct dma_buf *dmabuf = NULL;
+	struct nvmap_handle *h;
+	int ret = 0;
+	int fd;
+
+	mutex_lock(&nvmapsciipc->mlock);
+
+	pr_debug("%d: Sci_Ipc_Id %d local_vuid: %llu flags: %u\n",
+		__LINE__, sci_ipc_id, localu_vuid, flags);
+
+	entry = nvmap_find_entry_for_id(&nvmapsciipc->entries, sci_ipc_id);
+	if ((entry == NULL) || (entry->handle == NULL) ||
+		(entry->peer_vuid != localu_vuid) || (entry->flags != flags)) {
+
+		pr_debug("%d: No matching Sci_Ipc_Id %d found\n",
+		__LINE__, sci_ipc_id);
+
+		ret = -EINVAL;
+		goto unlock;
+	}
+
+	h = entry->handle;
+
+	if (is_ro && h->dmabuf_ro == NULL) {
+		h->dmabuf_ro = __nvmap_make_dmabuf(client, h, true);
+		if (IS_ERR(h->dmabuf_ro)) {
+			ret = PTR_ERR(h->dmabuf_ro);
+			goto unlock;
+		}
+		dmabuf_created = true;
+	}
+
+	ref = nvmap_duplicate_handle(client, h, false, is_ro);
+	if (!ref) {
+		ret = -EINVAL;
+		goto unlock;
+	}
+	nvmap_handle_put(h);
+	/*
+	 * When new dmabuf created (only RO dmabuf is getting created in this function)
+	 * it's counter is incremented one extra time in nvmap_duplicate_handle. Hence
+	 * decrement it by one.
+	 */
+	if (dmabuf_created)
+		dma_buf_put(h->dmabuf_ro);
+	if (!IS_ERR(ref)) {
+		u32 id = 0;
+
+		dmabuf = is_ro ? h->dmabuf_ro : h->dmabuf;
+		if (client->ida) {
+			if (nvmap_id_array_id_alloc(client->ida, &id, dmabuf) < 0) {
+				if (dmabuf)
+					dma_buf_put(dmabuf);
+				nvmap_free_handle(client, h, is_ro);
+				ret = -ENOMEM;
+				goto unlock;
+			}
+			if (!id)
+				*handle = 0;
+			else
+				*handle = id;
+		} else {
+			fd = nvmap_get_dmabuf_fd(client, h, is_ro);
+			if (IS_ERR_VALUE((uintptr_t)fd)) {
+				if (dmabuf)
+					dma_buf_put(dmabuf);
+				nvmap_free_handle(client, h, is_ro);
+				ret = -EINVAL;
+				goto unlock;
+			}
+			*handle = fd;
+			fd_install(fd, dmabuf->file);
+		}
+	}
+	entry->refcount--;
+	if (entry->refcount == 0U) {
+		struct free_sid_node *free_node;
+
+		rb_erase(&entry->entry, &nvmapsciipc->entries);
+		free_node = kzalloc(sizeof(*free_node), GFP_KERNEL);
+		if (free_node == NULL) {
+			ret = -ENOMEM;
+			kfree(entry);
+			goto unlock;
+		}
+		free_node->sid = entry->sci_ipc_id;
+		list_add_tail(&free_node->list, &nvmapsciipc->free_sid_list);
+		kfree(entry);
+	}
+unlock:
+	mutex_unlock(&nvmapsciipc->mlock);
+
+	if (!ret) {
+		if (!client->ida)
+			trace_refcount_create_handle_from_sci_ipc_id(h, dmabuf,
+				atomic_read(&h->ref),
+				atomic_long_read(&dmabuf->file->f_count),
+				is_ro ? "RO" : "RW");
+		else
+			trace_refcount_get_handle_from_sci_ipc_id(h, dmabuf,
+				atomic_read(&h->ref),
+				is_ro ? "RO" : "RW");
+	}
+
+	return ret;
+}
+
+int nvmap_sci_ipc_init(void)
+{
+	nvmapsciipc = kzalloc(sizeof(*nvmapsciipc), GFP_KERNEL);
+	if (!nvmapsciipc)
+		return -ENOMEM;
+	nvmapsciipc->entries = RB_ROOT;
+	INIT_LIST_HEAD(&nvmapsciipc->free_sid_list);
+	mutex_init(&nvmapsciipc->mlock);
+
+	return 0;
+}
+
+void nvmap_sci_ipc_exit(void)
+{
+	struct nvmap_sci_ipc_entry *e;
+	struct free_sid_node *fnode, *temp;
+	struct rb_node *n;
+
+	mutex_lock(&nvmapsciipc->mlock);
+	while ((n = rb_first(&nvmapsciipc->entries))) {
+		e = rb_entry(n, struct nvmap_sci_ipc_entry, entry);
+		rb_erase(&e->entry, &nvmapsciipc->entries);
+		kfree(e);
+	}
+
+	list_for_each_entry_safe(fnode, temp, &nvmapsciipc->free_sid_list, list) {
+		list_del(&fnode->list);
+		kfree(fnode);
+	}
+
+	mutex_unlock(&nvmapsciipc->mlock);
+	kfree(nvmapsciipc);
+	nvmapsciipc = NULL;
+}

drivers/video/tegra/nvmap/nvmap_sci_ipc.h

@@ -0,0 +1,37 @@
+/*
+ * drivers/video/tegra/nvmap/nvmap_sci_ipc.c
+ *
+ * mapping between nvmap_hnadle and sci_ipc entery
+ *
+ * Copyright (c) 2019-2021, NVIDIA CORPORATION. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+#ifndef __VIDEO_TEGRA_NVMAP_SCI_IPC_H
+#define __VIDEO_TEGRA_NVMAP_SCI_IPC_H
+
+int nvmap_validate_sci_ipc_params(struct nvmap_client *client,
+			NvSciIpcEndpointAuthToken auth_token,
+			NvSciIpcEndpointVuid *pr_vuid,
+			NvSciIpcEndpointVuid *localusr_vuid);
+
+int nvmap_create_sci_ipc_id(struct nvmap_client *client,
+				struct nvmap_handle *h,
+				u32 flags,
+				u32 *sci_ipc_id,
+				NvSciIpcEndpointVuid pr_vuid,
+				bool is_ro);
+
+int nvmap_get_handle_from_sci_ipc_id(struct nvmap_client *client,
+				u32 flags,
+				u32 sci_ipc_id,
+				NvSciIpcEndpointVuid localusr_vuid,
+				u32 *h);
+#endif /*  __VIDEO_TEGRA_NVMAP_SCI_IPC_H */

drivers/video/tegra/nvmap/nvmap_stats.c

@@ -0,0 +1,106 @@
+/*
+ * drivers/video/tegra/nvmap/nvmap_stats.c
+ *
+ * Nvmap Stats keeping
+ *
+ * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#include <linux/debugfs.h>
+
+#include "nvmap_priv.h"
+
+struct nvmap_stats nvmap_stats;
+
+static int nvmap_stats_reset(void *data, u64 val)
+{
+	int i;
+
+	if (val) {
+		atomic64_set(&nvmap_stats.collect, 0);
+		for (i = 0; i < NS_NUM; i++) {
+			if (i == NS_TOTAL)
+				continue;
+			atomic64_set(&nvmap_stats.stats[i], 0);
+		}
+	}
+	return 0;
+}
+
+static int nvmap_stats_get(void *data, u64 *val)
+{
+	atomic64_t *ptr = data;
+
+	*val = atomic64_read(ptr);
+	return 0;
+}
+
+static int nvmap_stats_set(void *data, u64 val)
+{
+	atomic64_t *ptr = data;
+
+	atomic64_set(ptr, val);
+	return 0;
+}
+
+DEFINE_SIMPLE_ATTRIBUTE(reset_stats_fops, NULL, nvmap_stats_reset, "%llu\n");
+DEFINE_SIMPLE_ATTRIBUTE(stats_fops, nvmap_stats_get, nvmap_stats_set, "%llu\n");
+
+void nvmap_stats_init(struct dentry *nvmap_debug_root)
+{
+	struct dentry *stats_root;
+
+#define CREATE_DF(x, y) \
+	debugfs_create_file(#x, S_IRUGO, stats_root, &y, &stats_fops);
+
+	stats_root = debugfs_create_dir("stats", nvmap_debug_root);
+	if (!IS_ERR_OR_NULL(stats_root)) {
+		CREATE_DF(alloc, nvmap_stats.stats[NS_ALLOC]);
+		CREATE_DF(release, nvmap_stats.stats[NS_RELEASE]);
+		CREATE_DF(ualloc, nvmap_stats.stats[NS_UALLOC]);
+		CREATE_DF(urelease, nvmap_stats.stats[NS_URELEASE]);
+		CREATE_DF(kalloc, nvmap_stats.stats[NS_KALLOC]);
+		CREATE_DF(krelease, nvmap_stats.stats[NS_KRELEASE]);
+		CREATE_DF(cflush_rq, nvmap_stats.stats[NS_CFLUSH_RQ]);
+		CREATE_DF(cflush_done, nvmap_stats.stats[NS_CFLUSH_DONE]);
+		CREATE_DF(ucflush_rq, nvmap_stats.stats[NS_UCFLUSH_RQ]);
+		CREATE_DF(ucflush_done, nvmap_stats.stats[NS_UCFLUSH_DONE]);
+		CREATE_DF(kcflush_rq, nvmap_stats.stats[NS_KCFLUSH_RQ]);
+		CREATE_DF(kcflush_done, nvmap_stats.stats[NS_KCFLUSH_DONE]);
+		CREATE_DF(total_memory, nvmap_stats.stats[NS_TOTAL]);
+
+		debugfs_create_file("collect", S_IRUGO | S_IWUSR,
+			stats_root, &nvmap_stats.collect, &stats_fops);
+		debugfs_create_file("reset", S_IWUSR,
+			stats_root, NULL, &reset_stats_fops);
+	}
+
+#undef CREATE_DF
+}
+
+void nvmap_stats_inc(enum nvmap_stats_t stat, size_t size)
+{
+	if (atomic64_read(&nvmap_stats.collect) || stat == NS_TOTAL)
+		atomic64_add(size, &nvmap_stats.stats[stat]);
+}
+
+void nvmap_stats_dec(enum nvmap_stats_t stat, size_t size)
+{
+	if (atomic64_read(&nvmap_stats.collect) || stat == NS_TOTAL)
+		atomic64_sub(size, &nvmap_stats.stats[stat]);
+}
+
+u64 nvmap_stats_read(enum nvmap_stats_t stat)
+{
+	return atomic64_read(&nvmap_stats.stats[stat]);
+}
+

drivers/video/tegra/nvmap/nvmap_stats.h

@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#ifndef __VIDEO_TEGRA_NVMAP_STATS_H
+#define __VIDEO_TEGRA_NVMAP_STATS_H
+
+enum nvmap_stats_t {
+	NS_ALLOC = 0,
+	NS_RELEASE,
+	NS_UALLOC,
+	NS_URELEASE,
+	NS_KALLOC,
+	NS_KRELEASE,
+	NS_CFLUSH_RQ,
+	NS_CFLUSH_DONE,
+	NS_UCFLUSH_RQ,
+	NS_UCFLUSH_DONE,
+	NS_KCFLUSH_RQ,
+	NS_KCFLUSH_DONE,
+	NS_TOTAL,
+	NS_NUM,
+};
+
+struct nvmap_stats {
+	atomic64_t stats[NS_NUM];
+	atomic64_t collect;
+};
+
+extern struct nvmap_stats nvmap_stats;
+
+void nvmap_stats_init(struct dentry *nvmap_debug_root);
+void nvmap_stats_inc(enum nvmap_stats_t, size_t size);
+void nvmap_stats_dec(enum nvmap_stats_t, size_t size);
+u64 nvmap_stats_read(enum nvmap_stats_t);
+#endif /* __VIDEO_TEGRA_NVMAP_STATS_H */

drivers/video/tegra/nvmap/nvmap_tag.c

@@ -0,0 +1,113 @@
+/*
+ * drivers/video/tegra/nvmap/nvmap_tag.c
+ *
+ * Allocation tag routines for nvmap
+ *
+ * Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#define pr_fmt(fmt)	"%s: " fmt, __func__
+
+#include <linux/moduleparam.h>
+
+#include <trace/events/nvmap.h>
+
+#include "nvmap_priv.h"
+
+struct nvmap_tag_entry *nvmap_search_tag_entry(struct rb_root *root, u32 tag)
+{
+	struct rb_node *node = root->rb_node;  /* top of the tree */
+	struct nvmap_tag_entry *entry;
+
+	while (node) {
+		entry = rb_entry(node, struct nvmap_tag_entry, node);
+
+		if (entry->tag > tag)
+			node = node->rb_left;
+		else if (entry->tag < tag)
+			node = node->rb_right;
+		else
+			return entry;  /* Found it */
+	}
+	return NULL;
+}
+
+
+static void nvmap_insert_tag_entry(struct rb_root *root,
+		struct nvmap_tag_entry *new)
+{
+	struct rb_node **link = &root->rb_node;
+	struct rb_node *parent = NULL;
+	struct nvmap_tag_entry *entry;
+	u32 tag = new->tag;
+
+	/* Go to the bottom of the tree */
+	while (*link) {
+	    parent = *link;
+	    entry = rb_entry(parent, struct nvmap_tag_entry, node);
+
+	    if (entry->tag > tag)
+		link = &parent->rb_left;
+	    else
+		link = &parent->rb_right;
+	}
+
+	/* Put the new node there */
+	rb_link_node(&new->node, parent, link);
+	rb_insert_color(&new->node, root);
+}
+
+
+int nvmap_define_tag(struct nvmap_device *dev, u32 tag,
+		const char __user *name, u32 len)
+{
+	struct nvmap_tag_entry *new;
+	struct nvmap_tag_entry *old;
+
+	new = kzalloc(sizeof(struct nvmap_tag_entry) + len + 1, GFP_KERNEL);
+	if (!new)
+		return -ENOMEM;
+
+	if (copy_from_user(new + 1, name, len)) {
+		kfree(new);
+		return -EFAULT;
+	}
+
+	new->tag = tag;
+
+	mutex_lock(&dev->tags_lock);
+	old = nvmap_search_tag_entry(&dev->tags, tag);
+	if (old) {
+		rb_replace_node(&old->node, &new->node, &dev->tags);
+		kfree(old);
+	} else {
+		nvmap_insert_tag_entry(&dev->tags, new);
+	}
+	mutex_unlock(&dev->tags_lock);
+
+	return 0;
+}
+
+int nvmap_remove_tag(struct nvmap_device *dev, u32 tag)
+{
+	struct nvmap_tag_entry *old;
+
+	mutex_lock(&dev->tags_lock);
+	old = nvmap_search_tag_entry(&dev->tags, tag);
+	if (old){
+		rb_erase(&old->node, &dev->tags);
+		kfree(old);
+	}
+	mutex_unlock(&dev->tags_lock);
+
+	return 0;
+}

include/linux/nvmap.h

@@ -0,0 +1,113 @@
+/*
+ * include/linux/nvmap.h
+ *
+ * structure declarations for nvmem and nvmap user-space ioctls
+ *
+ * Copyright (c) 2009-2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#ifndef _LINUX_NVMAP_H
+#define _LINUX_NVMAP_H
+
+#include <linux/rbtree.h>
+#include <linux/file.h>
+#include <linux/dma-buf.h>
+#include <linux/device.h>
+#include <linux/version.h>
+#include <uapi/linux/nvmap.h>
+
+#define NVMAP_HEAP_IOVMM   (1ul<<30)
+
+/* common carveout heaps */
+#define NVMAP_HEAP_CARVEOUT_VPR     (1ul<<28)
+#define NVMAP_HEAP_CARVEOUT_TSEC    (1ul<<27)
+#define NVMAP_HEAP_CARVEOUT_VIDMEM  (1ul<<26)
+#define NVMAP_HEAP_CARVEOUT_FSI   (1ul<<2)
+#define NVMAP_HEAP_CARVEOUT_IVM     (1ul<<1)
+#define NVMAP_HEAP_CARVEOUT_GENERIC (1ul<<0)
+
+#define NVMAP_HEAP_CARVEOUT_MASK    (NVMAP_HEAP_IOVMM - 1)
+
+/* allocation flags */
+#define NVMAP_HANDLE_UNCACHEABLE     (0x0ul << 0)
+#define NVMAP_HANDLE_WRITE_COMBINE   (0x1ul << 0)
+#define NVMAP_HANDLE_INNER_CACHEABLE (0x2ul << 0)
+#define NVMAP_HANDLE_CACHEABLE       (0x3ul << 0)
+#define NVMAP_HANDLE_CACHE_FLAG      (0x3ul << 0)
+
+#define NVMAP_HANDLE_SECURE          (0x1ul << 2)
+#define NVMAP_HANDLE_KIND_SPECIFIED  (0x1ul << 3)
+#define NVMAP_HANDLE_COMPR_SPECIFIED (0x1ul << 4)
+#define NVMAP_HANDLE_ZEROED_PAGES    (0x1ul << 5)
+#define NVMAP_HANDLE_PHYS_CONTIG     (0x1ul << 6)
+#define NVMAP_HANDLE_CACHE_SYNC      (0x1ul << 7)
+#define NVMAP_HANDLE_CACHE_SYNC_AT_RESERVE      (0x1ul << 8)
+#define NVMAP_HANDLE_RO	             (0x1ul << 9)
+
+#ifdef NVMAP_CONFIG_PAGE_POOLS
+ulong nvmap_page_pool_get_unused_pages(void);
+#else
+static inline ulong nvmap_page_pool_get_unused_pages(void)
+{
+	return 0;
+}
+#endif
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 15, 0)
+ulong nvmap_iovmm_get_used_pages(void);
+#endif
+
+int nvmap_register_vidmem_carveout(struct device *dma_dev,
+		phys_addr_t base, size_t size);
+
+/*
+ * A heap can be mapped to memory other than DRAM.
+ * The HW, controls the memory, can be power gated/ungated
+ * based upon the clients using the memory.
+ * if no client/alloc happens from the memory, the HW needs
+ * to be power gated. Similarly it should power ungated if
+ * alloc happens from the memory.
+ * int (*busy)(void) - trigger runtime power ungate
+ * int (*idle)(void) - trigger runtime power gate
+ */
+struct nvmap_pm_ops {
+	int (*busy)(void);
+	int (*idle)(void);
+};
+
+struct nvmap_platform_carveout {
+	const char *name;
+	unsigned int usage_mask;
+	phys_addr_t base;
+	size_t size;
+	struct device *cma_dev;
+	bool resize;
+	struct device *dma_dev;
+	struct device dev;
+	struct dma_declare_info *dma_info;
+	bool is_ivm;
+	unsigned int peer;
+	unsigned int vmid;
+	int can_alloc;
+	bool enable_static_dma_map;
+	bool disable_dynamic_dma_map;
+	bool no_cpu_access; /* carveout can't be accessed from cpu at all */
+	bool init_done;	/* FIXME: remove once all caveouts use reserved-memory */
+	struct nvmap_pm_ops pm_ops;
+};
+
+struct nvmap_platform_data {
+	const struct nvmap_platform_carveout *carveouts;
+	unsigned int nr_carveouts;
+};
+
+#endif /* _LINUX_NVMAP_H */

include/linux/nvmap_t19x.h

@@ -0,0 +1,38 @@
+/*
+ * include/linux/nvmap_t19x.h
+ *
+ * structure declarations for nvmem and nvmap user-space ioctls
+ *
+ * Copyright (c) 2009-2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#include <linux/types.h>
+#include <linux/bitmap.h>
+#include <linux/device.h>
+#include <linux/spinlock.h>
+
+#ifndef _LINUX_NVMAP_T19x_H
+#define _LINUX_NVMAP_T19x_H
+
+#define NVMAP_HEAP_CARVEOUT_CVSRAM  (1ul<<25)
+
+int nvmap_register_cvsram_carveout(struct device *dma_dev,
+		phys_addr_t base, size_t size,
+		int (*pmops_busy)(void), int (*pmops_idle)(void));
+
+struct nvmap_handle_t19x {
+	atomic_t nc_pin; /* no. of pins from non io coherent devices */
+};
+
+extern bool nvmap_version_t19x;
+
+#endif /* _LINUX_NVMAP_T19x_H */

include/trace/events/nvmap.h

@@ -0,0 +1,870 @@
+/*
+ * include/trace/events/nvmap.h
+ *
+ * NvMap event logging to ftrace.
+ *
+ * Copyright (c) 2012-2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM nvmap
+
+#if !defined(_TRACE_NVMAP_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_NVMAP_H
+
+#include <linux/nvmap.h>
+#include <linux/dma-buf.h>
+#include <linux/types.h>
+#include <linux/tracepoint.h>
+
+struct nvmap_handle;
+struct nvmap_handle_ref;
+struct nvmap_client;
+
+DECLARE_EVENT_CLASS(nvmap,
+	TP_PROTO(struct nvmap_client *client, const char *name),
+	TP_ARGS(client, name),
+	TP_STRUCT__entry(
+		__field(struct nvmap_client *, client)
+		__string(sname, name)
+	),
+	TP_fast_assign(
+		__entry->client = client;
+		__assign_str(sname, name)
+	),
+	TP_printk("client=%p, name=%s",
+		__entry->client, __get_str(sname))
+);
+
+DEFINE_EVENT(nvmap, nvmap_open,
+	TP_PROTO(struct nvmap_client *client, const char *name),
+	TP_ARGS(client, name)
+);
+
+DEFINE_EVENT(nvmap, nvmap_release,
+	TP_PROTO(struct nvmap_client *client, const char *name),
+	TP_ARGS(client, name)
+);
+
+TRACE_EVENT(nvmap_create_handle,
+	TP_PROTO(struct nvmap_client *client,
+		 const char *name,
+		 struct nvmap_handle *h,
+		 u32 size,
+		 struct nvmap_handle_ref *ref
+	),
+
+	TP_ARGS(client, name, h, size, ref),
+
+	TP_STRUCT__entry(
+		__field(struct nvmap_client *, client)
+		__string(sname, name)
+		__field(struct nvmap_handle *, h)
+		__field(u32, size)
+		__field(struct nvmap_handle_ref *, ref)
+	),
+
+	TP_fast_assign(
+		__entry->client = client;
+		__assign_str(sname, name)
+		__entry->h = h;
+		__entry->size = size;
+		__entry->ref = ref;
+	),
+
+	TP_printk("client=%p, name=%s, handle=%p, size=%d, ref=%p",
+		__entry->client, __get_str(sname),
+		__entry->h, __entry->size, __entry->ref)
+);
+
+TRACE_EVENT(nvmap_alloc_handle,
+	TP_PROTO(struct nvmap_client *client,
+		 struct nvmap_handle *handle,
+		 size_t size,
+		 u32 heap_mask,
+		 u32 align,
+		 u32 flags,
+		 u64 total,
+		 u64 alloc
+	),
+
+	TP_ARGS(client, handle, size, heap_mask, align, flags, total, alloc),
+
+	TP_STRUCT__entry(
+		__field(struct nvmap_client *, client)
+		__field(struct nvmap_handle *, handle)
+		__field(size_t, size)
+		__field(u32, heap_mask)
+		__field(u32, align)
+		__field(u32, flags)
+		__field(u64, total)
+		__field(u64, alloc)
+	),
+
+	TP_fast_assign(
+		__entry->client = client;
+		__entry->handle = handle;
+		__entry->size = size;
+		__entry->heap_mask = heap_mask;
+		__entry->align = align;
+		__entry->flags = flags;
+		__entry->total = total;
+		__entry->alloc = alloc;
+	),
+
+	TP_printk("client=%p, id=0x%p, size=%zu, heap_mask=0x%x, align=%d, flags=0x%x, total=%llu, alloc=%llu",
+		__entry->client, __entry->handle, __entry->size,
+		__entry->heap_mask, __entry->align, __entry->flags,
+		(unsigned long long)__entry->total,
+		(unsigned long long)__entry->alloc)
+);
+
+
+DECLARE_EVENT_CLASS(nvmap_handle_summary,
+	TP_PROTO(struct nvmap_client *client, pid_t pid, u32 dupes,
+		 struct nvmap_handle *handle, u32 share, u64 base,
+		 size_t size, u32 flags, u32 tag, const char *tag_name),
+	TP_ARGS(client, pid, dupes, handle, share, base, size, flags, tag, tag_name),
+
+	TP_STRUCT__entry(
+		__field(struct nvmap_client *, client)
+		__field(pid_t, pid)
+		__field(u32, dupes)
+		__field(struct nvmap_handle *, handle)
+		__field(u32, share)
+		__field(u64, base)
+		__field(size_t, size)
+		__field(u32, flags)
+		__field(u32, tag)
+		__string(tag_name, tag_name)
+	),
+
+	TP_fast_assign(
+		__entry->client = client;
+		__entry->pid = pid;
+		__entry->dupes = dupes;
+		__entry->handle = handle;
+		__entry->share = share;
+		__entry->base = base;
+		__entry->size = size;
+		__entry->flags = flags;
+		__entry->tag = tag;
+		__assign_str(tag_name, tag_name)
+	),
+
+	TP_printk("client=0x%p pid=%d dupes=%u handle=0x%p share=%u base=%llx size=%zu flags=0x%x tag=0x%x %s",
+		__entry->client,
+		__entry->pid,
+		__entry->dupes,
+		__entry->handle,
+		__entry->share,
+		__entry->base,
+		__entry->size,
+		__entry->flags,
+		__entry->tag,
+		__get_str(tag_name)
+	)
+);
+
+DEFINE_EVENT(nvmap_handle_summary,
+	nvmap_alloc_handle_done,
+	TP_PROTO(struct nvmap_client *client, pid_t pid, u32 dupes,
+		 struct nvmap_handle *handle, u32 share, u64 base,
+		 size_t size, u32 flags, u32 tag, const char *tag_name),
+	TP_ARGS(client, pid, dupes, handle, share, base, size, flags, tag, tag_name)
+);
+
+
+DEFINE_EVENT(nvmap_handle_summary,
+	nvmap_duplicate_handle,
+	TP_PROTO(struct nvmap_client *client, pid_t pid, u32 dupes,
+		 struct nvmap_handle *handle, u32 share, u64 base,
+		 size_t size, u32 flags, u32 tag, const char *tag_name),
+	TP_ARGS(client, pid, dupes, handle, share, base, size, flags, tag, tag_name)
+);
+
+DEFINE_EVENT(nvmap_handle_summary,
+	nvmap_free_handle,
+	TP_PROTO(struct nvmap_client *client, pid_t pid, u32 dupes,
+		 struct nvmap_handle *handle, u32 share, u64 base,
+		 size_t size, u32 flags, u32 tag, const char *tag_name),
+	TP_ARGS(client, pid, dupes, handle, share, base, size, flags, tag, tag_name)
+);
+
+DEFINE_EVENT(nvmap_handle_summary,
+	nvmap_destroy_handle,
+	TP_PROTO(struct nvmap_client *client, pid_t pid, u32 dupes,
+		 struct nvmap_handle *handle, u32 share, u64 base,
+		 size_t size, u32 flags, u32 tag, const char *tag_name),
+	TP_ARGS(client, pid, dupes, handle, share, base, size, flags, tag, tag_name)
+);
+
+TRACE_EVENT(nvmap_cache_maint,
+	TP_PROTO(struct nvmap_client *client,
+		 struct nvmap_handle *h,
+		 ulong start,
+		 ulong end,
+		 u32 op,
+		 size_t size
+	),
+
+	TP_ARGS(client, h, start, end, op, size),
+
+	TP_STRUCT__entry(
+		__field(struct nvmap_client *, client)
+		__field(struct nvmap_handle *, h)
+		__field(ulong, start)
+		__field(ulong, end)
+		__field(u32, op)
+		__field(size_t, size)
+	),
+
+	TP_fast_assign(
+		__entry->client = client;
+		__entry->h = h;
+		__entry->start = start;
+		__entry->end = end;
+		__entry->op = op;
+		__entry->size = size;
+	),
+
+	TP_printk("client=%p, h=%p, start=0x%lx, end=0x%lx, op=%d, size=%zu",
+		__entry->client, __entry->h, __entry->start,
+		__entry->end, __entry->op, __entry->size)
+);
+
+TRACE_EVENT(nvmap_cache_flush,
+	TP_PROTO(size_t size,
+		 u64 alloc_rq,
+		 u64 total_rq,
+		 u64 total_done
+	),
+
+	TP_ARGS(size, alloc_rq, total_rq, total_done),
+
+	TP_STRUCT__entry(
+		__field(size_t, size)
+		__field(u64, alloc_rq)
+		__field(u64, total_rq)
+		__field(u64, total_done)
+	),
+
+	TP_fast_assign(
+		__entry->size = size;
+		__entry->alloc_rq = alloc_rq;
+		__entry->total_rq = total_rq;
+		__entry->total_done = total_done;
+	),
+
+	TP_printk("size=%zu, alloc_rq=%llu, total_rq=%llu, total_done=%llu",
+		__entry->size,
+		(unsigned long long)__entry->alloc_rq,
+		(unsigned long long)__entry->total_rq,
+		(unsigned long long)__entry->total_done)
+);
+
+TRACE_EVENT(nvmap_map_into_caller_ptr,
+	TP_PROTO(struct nvmap_client *client,
+		 struct nvmap_handle *h,
+		 u32 offset,
+		 u32 length,
+		 u32 flags
+	),
+
+	TP_ARGS(client, h, offset, length, flags),
+
+	TP_STRUCT__entry(
+		__field(struct nvmap_client *, client)
+		__field(struct nvmap_handle *, h)
+		__field(u32, offset)
+		__field(u32, length)
+		__field(u32, flags)
+	),
+
+	TP_fast_assign(
+		__entry->client = client;
+		__entry->h = h;
+		__entry->offset = offset;
+		__entry->length = length;
+		__entry->flags = flags;
+	),
+
+	TP_printk("client=%p, h=%p, offset=%d, length=%d, flags=0x%x",
+		__entry->client, __entry->h, __entry->offset,
+		__entry->length, __entry->flags)
+);
+
+TRACE_EVENT(nvmap_ioctl_rw_handle,
+	TP_PROTO(struct nvmap_client *client,
+		 struct nvmap_handle *h,
+		 u32 is_read,
+		 u32 offset,
+		 unsigned long addr,
+		 u32 mem_stride,
+		 u32 user_stride,
+		 u32 elem_size,
+		 u32 count
+	),
+
+	TP_ARGS(client, h, is_read, offset, addr, mem_stride,
+		user_stride, elem_size, count),
+
+	TP_STRUCT__entry(
+		__field(struct nvmap_client *, client)
+		__field(struct nvmap_handle *, h)
+		__field(u32, is_read)
+		__field(u32, offset)
+		__field(unsigned long, addr)
+		__field(u32, mem_stride)
+		__field(u32, user_stride)
+		__field(u32, elem_size)
+		__field(u32, count)
+	),
+
+	TP_fast_assign(
+		__entry->client = client;
+		__entry->h = h;
+		__entry->is_read = is_read;
+		__entry->offset = offset;
+		__entry->addr = addr;
+		__entry->mem_stride = mem_stride;
+		__entry->user_stride = user_stride;
+		__entry->elem_size = elem_size;
+		__entry->count = count;
+	),
+
+	TP_printk("client=%p, h=%p, is_read=%d, offset=%d, addr=0x%lx,"
+		"mem_stride=%d, user_stride=%d, elem_size=%d, count=%d",
+		__entry->client, __entry->h, __entry->is_read, __entry->offset,
+		__entry->addr, __entry->mem_stride, __entry->user_stride,
+		__entry->elem_size, __entry->count)
+);
+
+TRACE_EVENT(nvmap_ioctl_pinop,
+	TP_PROTO(struct nvmap_client *client,
+		 u32 is_pin,
+		 u32 count,
+		 struct nvmap_handle **ids
+	),
+
+	TP_ARGS(client, is_pin, count, ids),
+
+	TP_STRUCT__entry(
+		__field(struct nvmap_client *, client)
+		__field(u32, is_pin)
+		__field(u32, count)
+		__field(struct nvmap_handle **, ids)
+		__dynamic_array(struct nvmap_handle *, ids, count)
+	),
+
+	TP_fast_assign(
+		__entry->client = client;
+		__entry->is_pin = is_pin;
+		__entry->count = count;
+		__entry->ids = ids;
+		memcpy(__get_dynamic_array(ids), ids,
+		    sizeof(struct nvmap_handle *) * count);
+	),
+
+	TP_printk("client=%p, is_pin=%d, count=%d, ids=[%s]",
+		__entry->client, __entry->is_pin, __entry->count,
+		__print_hex(__get_dynamic_array(ids), __entry->ids ?
+			    sizeof(struct nvmap_handle *) * __entry->count : 0))
+);
+
+DECLARE_EVENT_CLASS(handle_get_put,
+	TP_PROTO(struct nvmap_handle *handle, u32 ref_count),
+
+	TP_ARGS(handle, ref_count),
+
+	TP_STRUCT__entry(
+		__field(struct nvmap_handle *, handle)
+		__field(u32, ref_count)
+	),
+
+	TP_fast_assign(
+		__entry->handle = handle;
+		__entry->ref_count = ref_count;
+	),
+
+	TP_printk("ref=%u handle=%p",
+		__entry->ref_count,
+		__entry->handle
+	)
+);
+
+DEFINE_EVENT(handle_get_put, nvmap_handle_get,
+	TP_PROTO(struct nvmap_handle *handle, u32 ref_count),
+	TP_ARGS(handle, ref_count)
+);
+
+DEFINE_EVENT(handle_get_put, nvmap_handle_put,
+	TP_PROTO(struct nvmap_handle *handle, u32 ref_count),
+	TP_ARGS(handle, ref_count)
+);
+
+DECLARE_EVENT_CLASS(pin_unpin,
+	TP_PROTO(struct nvmap_client *client,
+		 const char *name,
+		 struct nvmap_handle *h,
+		 u32 pin_count
+	),
+
+	TP_ARGS(client, name, h, pin_count),
+
+	TP_STRUCT__entry(
+		__field(struct nvmap_client *, client)
+		__string(sname, name)
+		__field(struct nvmap_handle *, h)
+		__field(u32, pin_count)
+	),
+
+	TP_fast_assign(
+		__entry->client = client;
+		__assign_str(sname, name)
+		__entry->h = h;
+		__entry->pin_count = pin_count;
+	),
+
+	TP_printk("client=%p, name=%s, h=%p, pin_count=%d",
+		__entry->client, __get_str(sname),
+		__entry->h, __entry->pin_count)
+);
+
+DEFINE_EVENT(pin_unpin, nvmap_pin,
+	TP_PROTO(struct nvmap_client *client,
+		 const char *name,
+		 struct nvmap_handle *h,
+		 u32 pin_count
+	),
+	TP_ARGS(client, name, h, pin_count)
+);
+
+DEFINE_EVENT(pin_unpin, nvmap_unpin,
+	TP_PROTO(struct nvmap_client *client,
+		 const char *name,
+		 struct nvmap_handle *h,
+		 u32 pin_count
+	),
+	TP_ARGS(client, name, h, pin_count)
+);
+
+DEFINE_EVENT(pin_unpin, handle_unpin_error,
+	TP_PROTO(struct nvmap_client *client,
+		 const char *name,
+		 struct nvmap_handle *h,
+		 u32 pin_count
+	),
+	TP_ARGS(client, name, h, pin_count)
+);
+
+/*
+ * Nvmap dmabuf events
+ */
+DECLARE_EVENT_CLASS(nvmap_dmabuf_2,
+	TP_PROTO(struct dma_buf *dbuf,
+		 struct device *dev
+	),
+
+	TP_ARGS(dbuf, dev),
+
+	TP_STRUCT__entry(
+		__field(struct dma_buf *, dbuf)
+		__string(name, dev_name(dev))
+	),
+
+	TP_fast_assign(
+		__entry->dbuf = dbuf;
+		__assign_str(name, dev_name(dev));
+	),
+
+	TP_printk("dmabuf=%p, device=%s",
+		__entry->dbuf, __get_str(name)
+	)
+);
+
+DECLARE_EVENT_CLASS(nvmap_dmabuf_1,
+	TP_PROTO(struct dma_buf *dbuf),
+
+	TP_ARGS(dbuf),
+
+	TP_STRUCT__entry(
+		__field(struct dma_buf *, dbuf)
+	),
+
+	TP_fast_assign(
+		__entry->dbuf = dbuf;
+	),
+
+	TP_printk("dmabuf=%p",
+		__entry->dbuf
+	)
+);
+
+DEFINE_EVENT(nvmap_dmabuf_2, nvmap_dmabuf_attach,
+	TP_PROTO(struct dma_buf *dbuf,
+		 struct device *dev
+	),
+	TP_ARGS(dbuf, dev)
+);
+
+DEFINE_EVENT(nvmap_dmabuf_2, nvmap_dmabuf_detach,
+	TP_PROTO(struct dma_buf *dbuf,
+		 struct device *dev
+	),
+	TP_ARGS(dbuf, dev)
+);
+
+DEFINE_EVENT(nvmap_dmabuf_2, nvmap_dmabuf_map_dma_buf,
+	TP_PROTO(struct dma_buf *dbuf,
+		 struct device *dev
+	),
+	TP_ARGS(dbuf, dev)
+);
+
+DEFINE_EVENT(nvmap_dmabuf_2, nvmap_dmabuf_unmap_dma_buf,
+	TP_PROTO(struct dma_buf *dbuf,
+		 struct device *dev
+	),
+	TP_ARGS(dbuf, dev)
+);
+
+DEFINE_EVENT(nvmap_dmabuf_1, nvmap_dmabuf_mmap,
+	TP_PROTO(struct dma_buf *dbuf),
+	TP_ARGS(dbuf)
+);
+
+DEFINE_EVENT(nvmap_dmabuf_1, nvmap_dmabuf_vmap,
+	TP_PROTO(struct dma_buf *dbuf),
+	TP_ARGS(dbuf)
+);
+
+DEFINE_EVENT(nvmap_dmabuf_1, nvmap_dmabuf_vunmap,
+	TP_PROTO(struct dma_buf *dbuf),
+	TP_ARGS(dbuf)
+);
+
+DEFINE_EVENT(nvmap_dmabuf_1, nvmap_dmabuf_kmap,
+	TP_PROTO(struct dma_buf *dbuf),
+	TP_ARGS(dbuf)
+);
+
+DEFINE_EVENT(nvmap_dmabuf_1, nvmap_dmabuf_kunmap,
+	TP_PROTO(struct dma_buf *dbuf),
+	TP_ARGS(dbuf)
+);
+
+DECLARE_EVENT_CLASS(nvmap_dmabuf_cpu_access,
+	TP_PROTO(struct dma_buf *dbuf,
+		 size_t start,
+		 size_t len),
+
+	TP_ARGS(dbuf, start, len),
+
+	TP_STRUCT__entry(
+		__field(struct dma_buf *, dbuf)
+		__field(size_t, start)
+		__field(size_t, len)
+	),
+
+	TP_fast_assign(
+		__entry->dbuf = dbuf;
+		__entry->start = start;
+		__entry->len = len;
+	),
+
+	TP_printk("dmabuf=%p, start=%zd len=%zd",
+		  __entry->dbuf, __entry->start, __entry->len
+	)
+);
+
+DEFINE_EVENT(nvmap_dmabuf_cpu_access, nvmap_dmabuf_begin_cpu_access,
+	TP_PROTO(struct dma_buf *dbuf,
+		 size_t start,
+		 size_t len),
+	TP_ARGS(dbuf, start, len)
+);
+
+DEFINE_EVENT(nvmap_dmabuf_cpu_access, nvmap_dmabuf_end_cpu_access,
+	TP_PROTO(struct dma_buf *dbuf,
+		 size_t start,
+		 size_t len),
+	TP_ARGS(dbuf, start, len)
+);
+
+DECLARE_EVENT_CLASS(nvmap_dmabuf_make_release,
+	TP_PROTO(const char *cli,
+		 struct nvmap_handle *h,
+		 struct dma_buf *dbuf
+	),
+
+	TP_ARGS(cli, h, dbuf),
+
+	TP_STRUCT__entry(
+		__field(const char *, cli)
+		__field(struct nvmap_handle *, h)
+		__field(struct dma_buf *, dbuf)
+	),
+
+	TP_fast_assign(
+		__entry->cli = cli;
+		__entry->h = h;
+		__entry->dbuf = dbuf;
+	),
+
+	TP_printk("cli=%s handle=%p dmabuf=%p",
+		  __entry->cli, __entry->h, __entry->dbuf
+	)
+);
+
+DEFINE_EVENT(nvmap_dmabuf_make_release, nvmap_make_dmabuf,
+	TP_PROTO(const char *cli,
+		 struct nvmap_handle *h,
+		 struct dma_buf *dbuf
+	),
+	TP_ARGS(cli, h, dbuf)
+);
+
+DEFINE_EVENT(nvmap_dmabuf_make_release, nvmap_dmabuf_release,
+	TP_PROTO(const char *cli,
+		 struct nvmap_handle *h,
+		 struct dma_buf *dbuf
+	),
+	TP_ARGS(cli, h, dbuf)
+);
+
+TRACE_EVENT(pp_clean_cache,
+	TP_PROTO(u32 dirty_pages,
+		 size_t cache_maint_th,
+		 int cache_maint_by_set_ways
+	),
+
+	TP_ARGS(dirty_pages, cache_maint_th, cache_maint_by_set_ways),
+
+	TP_STRUCT__entry(
+		__field(u32, dirty_pages)
+		__field(size_t, cache_maint_th)
+		__field(int, cache_maint_by_set_ways)
+	),
+
+	TP_fast_assign(
+		__entry->dirty_pages = dirty_pages;
+		__entry->cache_maint_th = cache_maint_th >> PAGE_SHIFT;
+		__entry->cache_maint_by_set_ways = cache_maint_by_set_ways;
+	),
+
+	TP_printk("dirty_pages=%u, cache_maint_th=%zu, cache_maint_by_set_ways=%d",
+		__entry->dirty_pages, __entry->cache_maint_th,
+		__entry->cache_maint_by_set_ways)
+);
+
+DECLARE_EVENT_CLASS(nvmap_get_list_page,
+	TP_PROTO(u32 count),
+
+	TP_ARGS(count),
+
+	TP_STRUCT__entry(
+		__field(u32, count)
+	),
+
+	TP_fast_assign(
+		__entry->count = count;
+	),
+
+	TP_printk("pages left in list=%u", __entry->count)
+);
+
+DEFINE_EVENT(nvmap_get_list_page, get_zero_list_page,
+	TP_PROTO(u32 count),
+	TP_ARGS(count)
+);
+
+DEFINE_EVENT(nvmap_get_list_page, get_page_list_page,
+	TP_PROTO(u32 count),
+	TP_ARGS(count)
+);
+
+TRACE_EVENT(nvmap_pp_zero_pages,
+	TP_PROTO(u32 count),
+
+	TP_ARGS(count),
+
+	TP_STRUCT__entry(
+		__field(u32, count)
+	),
+
+	TP_fast_assign(
+		__entry->count = count;
+	),
+
+	TP_printk("no. of pages zeroed=%u", __entry->count)
+);
+
+TRACE_EVENT(nvmap_pp_do_background_zero_pages,
+	TP_PROTO(u32 inserted, u32 zeroed),
+
+	TP_ARGS(inserted, zeroed),
+
+	TP_STRUCT__entry(
+		__field(u32, inserted)
+		__field(u32, zeroed)
+	),
+
+	TP_fast_assign(
+		__entry->inserted = inserted;
+		__entry->zeroed = zeroed;
+	),
+
+	TP_printk("failed to insert %u no. of zeroed pages to page_list",
+		__entry->zeroed - __entry->inserted)
+);
+
+TRACE_EVENT(nvmap_pp_alloc_locked,
+	TP_PROTO(int force_alloc),
+
+	TP_ARGS(force_alloc),
+
+	TP_STRUCT__entry(
+		__field(int, force_alloc)
+	),
+
+	TP_fast_assign(
+		__entry->force_alloc = force_alloc;
+	),
+
+	TP_printk("allocated one page with force_alloc:%d", __entry->force_alloc)
+);
+
+TRACE_EVENT(nvmap_pp_alloc_lots,
+	TP_PROTO(u32 alloced, u32 requested),
+
+	TP_ARGS(alloced, requested),
+
+	TP_STRUCT__entry(
+		__field(u32, alloced)
+		__field(u32, requested)
+	),
+
+	TP_fast_assign(
+		__entry->alloced = alloced;
+		__entry->requested = requested;
+	),
+
+	TP_printk("requested:%u alloced:%u\n",
+		__entry->requested, __entry->alloced)
+);
+
+TRACE_EVENT(nvmap_pp_fill_zero_lots,
+	TP_PROTO(u32 save_to_zero,
+		 u32 to_zero,
+		 u32 ret,
+		 u32 nr
+	),
+
+	TP_ARGS(save_to_zero, to_zero, ret, nr),
+
+	TP_STRUCT__entry(
+		__field(u32, save_to_zero)
+		__field(u32, to_zero)
+		__field(u32, ret)
+		__field(u32, nr)
+	),
+
+	TP_fast_assign(
+		__entry->save_to_zero = save_to_zero;
+		__entry->to_zero = to_zero;
+		__entry->ret = ret;
+		__entry->nr = nr;
+	),
+
+	TP_printk("inserted %u pages to zero list, freed %u pages, did not process %u pages",
+		__entry->to_zero - __entry->save_to_zero,
+		__entry->ret - (__entry->to_zero - __entry->save_to_zero),
+		__entry->nr - __entry->ret)
+);
+
+TRACE_EVENT(refcount_get_handle_from_sci_ipc_id,
+	TP_PROTO(struct nvmap_handle *handle,
+		 struct dma_buf *dmabuf,
+		 int handle_ref, const char *perm
+	),
+
+	TP_ARGS(handle, dmabuf, handle_ref, perm),
+
+	TP_STRUCT__entry(
+		__field(struct nvmap_handle *, handle)
+		__field(struct dma_buf *, dmabuf)
+		__field(int, handle_ref)
+		__string(acc_perm, perm)
+	),
+
+	TP_fast_assign(
+		__entry->handle = handle;
+		__entry->dmabuf = dmabuf;
+		__entry->handle_ref = handle_ref;
+		__assign_str(acc_perm, perm);
+	),
+
+	TP_printk("handle=0x%p, dmabuf=0x%p, handle_ref=%d, perm=%s",
+		   __entry->handle, __entry->dmabuf, __entry->handle_ref,
+		   __get_str(acc_perm))
+);
+
+DECLARE_EVENT_CLASS(nvmap_refcount,
+	TP_PROTO(struct nvmap_handle *handle, struct dma_buf *dmabuf, \
+		 int handle_ref,  long dmabuf_ref, const char *perm),
+
+	TP_ARGS(handle, dmabuf, handle_ref, dmabuf_ref, perm),
+
+	TP_STRUCT__entry(
+		__field(struct nvmap_handle *, handle)
+		__field(struct dma_buf *, dmabuf)
+		__field(int, handle_ref)
+		__field(long, dmabuf_ref)
+		__string(acc_perm, perm)
+	),
+
+	TP_fast_assign(
+		__entry->handle = handle;
+		__entry->dmabuf = dmabuf;
+		__entry->handle_ref = handle_ref;
+		__entry->dmabuf_ref = dmabuf_ref;
+		__assign_str(acc_perm, perm);
+	),
+
+	TP_printk("handle=0x%p, dmabuf=0x%p, handle_ref=%d, dmabuf_ref=%ld, perm=%s",
+		   __entry->handle, __entry->dmabuf, __entry->handle_ref,
+		   __entry->dmabuf_ref, __get_str(acc_perm))
+);
+
+#define NVMAPREFCOUNTEVENT(ev) DEFINE_EVENT(nvmap_refcount, ev, \
+	TP_PROTO(struct nvmap_handle *handle, struct dma_buf *dmabuf, \
+		 int handle_ref,  long dmabuf_ref, const char *perm), \
+	TP_ARGS(handle, dmabuf, handle_ref, dmabuf_ref, perm) \
+)
+
+NVMAPREFCOUNTEVENT(refcount_create_handle);
+NVMAPREFCOUNTEVENT(refcount_create_handle_from_va);
+NVMAPREFCOUNTEVENT(refcount_create_handle_from_fd);
+NVMAPREFCOUNTEVENT(refcount_getfd);
+NVMAPREFCOUNTEVENT(refcount_alloc);
+NVMAPREFCOUNTEVENT(refcount_get_sci_ipc_id);
+NVMAPREFCOUNTEVENT(refcount_create_handle_from_sci_ipc_id);
+NVMAPREFCOUNTEVENT(refcount_dup_handle);
+NVMAPREFCOUNTEVENT(refcount_free_handle);
+
+#undef NVMAPREFCOUNTEVENT
+
+#endif /* _TRACE_NVMAP_H */
+
+/* This part must be outside protection */
+#include <trace/define_trace.h>

include/uapi/linux/nvmap.h

@@ -0,0 +1,366 @@
+/*
+ * include/uapi/linux/nvmap.h
+ *
+ * structure declarations for nvmem and nvmap user-space ioctls
+ *
+ * Copyright (c) 2009-2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#include <linux/ioctl.h>
+#include <linux/types.h>
+
+#ifndef __UAPI_LINUX_NVMAP_H
+#define __UAPI_LINUX_NVMAP_H
+
+/*
+ * DOC: NvMap Userspace API
+ *
+ * create a client by opening /dev/nvmap
+ * most operations handled via following ioctls
+ *
+ */
+enum {
+	NVMAP_HANDLE_PARAM_SIZE = 1,
+	NVMAP_HANDLE_PARAM_ALIGNMENT,
+	NVMAP_HANDLE_PARAM_BASE,
+	NVMAP_HANDLE_PARAM_HEAP,
+	NVMAP_HANDLE_PARAM_KIND,
+	NVMAP_HANDLE_PARAM_COMPR, /* ignored, to be removed */
+};
+
+enum {
+	NVMAP_CACHE_OP_WB = 0,
+	NVMAP_CACHE_OP_INV,
+	NVMAP_CACHE_OP_WB_INV,
+};
+
+enum {
+	NVMAP_PAGES_UNRESERVE = 0,
+	NVMAP_PAGES_RESERVE,
+	NVMAP_INSERT_PAGES_ON_UNRESERVE,
+	NVMAP_PAGES_PROT_AND_CLEAN,
+};
+
+#define NVMAP_ELEM_SIZE_U64 (1 << 31)
+
+struct nvmap_create_handle {
+	union {
+		struct {
+			union {
+				/* size will be overwritten */
+				__u32 size;	/* CreateHandle */
+				__s32 fd;	/* DmaBufFd or FromFd */
+			};
+			__u32 handle;		/* returns nvmap handle */
+		};
+		struct {
+			/* one is input parameter, and other is output parameter
+			 * since its a union please note that input parameter
+			 * will be overwritten once ioctl returns
+			 */
+			union {
+				__u64 ivm_id;	 /* CreateHandle from ivm*/
+				__u32 ivm_handle;/* Get ivm_id from handle */
+			};
+		};
+		struct {
+			union {
+				/* size64 will be overwritten */
+				__u64 size64; /* CreateHandle */
+				__u32 handle64; /* returns nvmap handle */
+			};
+		};
+	};
+};
+
+struct nvmap_create_handle_from_va {
+	__u64 va;		/* FromVA*/
+	__u32 size;		/* non-zero for partial memory VMA. zero for end of VMA */
+	__u32 flags;		/* wb/wc/uc/iwb, tag etc. */
+	union {
+		__u32 handle;		/* returns nvmap handle */
+		__u64 size64;		/* used when size is 0 */
+	};
+};
+
+struct nvmap_gup_test {
+	__u64 va;		/* FromVA*/
+	__u32 handle;		/* returns nvmap handle */
+	__u32 result;		/* result=1 for pass, result=-err for failure */
+};
+
+struct nvmap_alloc_handle {
+	__u32 handle;		/* nvmap handle */
+	__u32 heap_mask;	/* heaps to allocate from */
+	__u32 flags;		/* wb/wc/uc/iwb etc. */
+	__u32 align;		/* min alignment necessary */
+};
+
+struct nvmap_alloc_ivm_handle {
+	__u32 handle;		/* nvmap handle */
+	__u32 heap_mask;	/* heaps to allocate from */
+	__u32 flags;		/* wb/wc/uc/iwb etc. */
+	__u32 align;		/* min alignment necessary */
+	__u32 peer;		/* peer with whom handle must be shared. Used
+				 *  only for NVMAP_HEAP_CARVEOUT_IVM
+				 */
+};
+
+struct nvmap_rw_handle {
+	__u64 addr;		/* user pointer*/
+	__u32 handle;		/* nvmap handle */
+	__u64 offset;		/* offset into hmem */
+	__u64 elem_size;	/* individual atom size */
+	__u64 hmem_stride;	/* delta in bytes between atoms in hmem */
+	__u64 user_stride;	/* delta in bytes between atoms in user */
+	__u64 count;		/* number of atoms to copy */
+};
+
+#ifdef __KERNEL__
+#ifdef CONFIG_COMPAT
+struct nvmap_rw_handle_32 {
+	__u32 addr;		/* user pointer */
+	__u32 handle;		/* nvmap handle */
+	__u32 offset;		/* offset into hmem */
+	__u32 elem_size;	/* individual atom size */
+	__u32 hmem_stride;	/* delta in bytes between atoms in hmem */
+	__u32 user_stride;	/* delta in bytes between atoms in user */
+	__u32 count;		/* number of atoms to copy */
+};
+#endif /* CONFIG_COMPAT */
+#endif /* __KERNEL__ */
+
+struct nvmap_handle_param {
+	__u32 handle;		/* nvmap handle */
+	__u32 param;		/* size/align/base/heap etc. */
+	unsigned long result;	/* returns requested info*/
+};
+
+#ifdef __KERNEL__
+#ifdef CONFIG_COMPAT
+struct nvmap_handle_param_32 {
+	__u32 handle;		/* nvmap handle */
+	__u32 param;		/* size/align/base/heap etc. */
+	__u32 result;		/* returns requested info*/
+};
+#endif /* CONFIG_COMPAT */
+#endif /* __KERNEL__ */
+
+struct nvmap_cache_op {
+	unsigned long addr;	/* user pointer*/
+	__u32 handle;		/* nvmap handle */
+	__u32 len;		/* bytes to flush */
+	__s32 op;		/* wb/wb_inv/inv */
+};
+
+struct nvmap_cache_op_64 {
+	unsigned long addr;	/* user pointer*/
+	__u32 handle;		/* nvmap handle */
+	__u64 len;		/* bytes to flush */
+	__s32 op;		/* wb/wb_inv/inv */
+};
+
+#ifdef __KERNEL__
+#ifdef CONFIG_COMPAT
+struct nvmap_cache_op_32 {
+	__u32 addr;		/* user pointer*/
+	__u32 handle;		/* nvmap handle */
+	__u32 len;		/* bytes to flush */
+	__s32 op;		/* wb/wb_inv/inv */
+};
+#endif /* CONFIG_COMPAT */
+#endif /* __KERNEL__ */
+
+struct nvmap_cache_op_list {
+	__u64 handles;		/* Ptr to u32 type array, holding handles */
+	__u64 offsets;		/* Ptr to u32 type array, holding offsets
+				 * into handle mem */
+	__u64 sizes;		/* Ptr to u32 type array, holindg sizes of memory
+				 * regions within each handle */
+	__u32 nr;		/* Number of handles */
+	__s32 op;		/* wb/wb_inv/inv */
+};
+
+struct nvmap_debugfs_handles_header {
+	__u8 version;
+};
+
+struct nvmap_debugfs_handles_entry {
+	__u64 base;
+	__u64 size;
+	__u32 flags;
+	__u32 share_count;
+	__u64 mapped_size;
+};
+
+struct nvmap_set_tag_label {
+	__u32 tag;
+	__u32 len;		/* in: label length
+				   out: number of characters copied */
+	__u64 addr;		/* in: pointer to label or NULL to remove */
+};
+
+struct nvmap_available_heaps {
+	__u64 heaps;		/* heaps bitmask */
+};
+
+struct nvmap_heap_size {
+	__u32 heap;
+	__u64 size;
+};
+
+struct nvmap_sciipc_map {
+	__u64 auth_token;    /* AuthToken */
+	__u32 flags;       /* Exporter permission flags */
+	__u32 sci_ipc_id;  /* FromImportId */
+	__u32 handle;      /* Nvmap handle */
+};
+
+struct nvmap_handle_parameters {
+    __u8 contig;
+    __u32 import_id;
+    __u32 handle;
+    __u32 heap_number;
+    __u32 access_flags;
+    __u64 heap;
+    __u64 align;
+    __u64 coherency;
+    __u64 size;
+    __u64 offset;
+};
+
+/**
+ * Struct used while querying heap parameters
+ */
+struct nvmap_query_heap_params {
+	__u32 heap_mask;
+	__u32 flags;
+	__u8 contig;
+	__u64 total;
+	__u64 free;
+	__u64 largest_free_block;
+};
+
+/**
+ * Struct used while duplicating memory handle
+ */
+struct nvmap_duplicate_handle {
+	__u32 handle;
+	__u32 access_flags;
+	__u32 dup_handle;
+};
+
+#define NVMAP_IOC_MAGIC 'N'
+
+/* Creates a new memory handle. On input, the argument is the size of the new
+ * handle; on return, the argument is the name of the new handle
+ */
+#define NVMAP_IOC_CREATE  _IOWR(NVMAP_IOC_MAGIC, 0, struct nvmap_create_handle)
+#define NVMAP_IOC_CREATE_64 \
+	_IOWR(NVMAP_IOC_MAGIC, 1, struct nvmap_create_handle)
+#define NVMAP_IOC_FROM_ID _IOWR(NVMAP_IOC_MAGIC, 2, struct nvmap_create_handle)
+
+/* Actually allocates memory for the specified handle */
+#define NVMAP_IOC_ALLOC    _IOW(NVMAP_IOC_MAGIC, 3, struct nvmap_alloc_handle)
+
+/* Frees a memory handle, unpinning any pinned pages and unmapping any mappings
+ */
+#define NVMAP_IOC_FREE       _IO(NVMAP_IOC_MAGIC, 4)
+
+/* Reads/writes data (possibly strided) from a user-provided buffer into the
+ * hmem at the specified offset */
+#define NVMAP_IOC_WRITE      _IOW(NVMAP_IOC_MAGIC, 6, struct nvmap_rw_handle)
+#define NVMAP_IOC_READ       _IOW(NVMAP_IOC_MAGIC, 7, struct nvmap_rw_handle)
+#ifdef __KERNEL__
+#ifdef CONFIG_COMPAT
+#define NVMAP_IOC_WRITE_32   _IOW(NVMAP_IOC_MAGIC, 6, struct nvmap_rw_handle_32)
+#define NVMAP_IOC_READ_32    _IOW(NVMAP_IOC_MAGIC, 7, struct nvmap_rw_handle_32)
+#endif /* CONFIG_COMPAT */
+#endif /* __KERNEL__ */
+
+#define NVMAP_IOC_PARAM _IOWR(NVMAP_IOC_MAGIC, 8, struct nvmap_handle_param)
+#ifdef __KERNEL__
+#ifdef CONFIG_COMPAT
+#define NVMAP_IOC_PARAM_32 _IOWR(NVMAP_IOC_MAGIC, 8, struct nvmap_handle_param_32)
+#endif /* CONFIG_COMPAT */
+#endif /* __KERNEL__ */
+
+#define NVMAP_IOC_CACHE      _IOW(NVMAP_IOC_MAGIC, 12, struct nvmap_cache_op)
+#define NVMAP_IOC_CACHE_64   _IOW(NVMAP_IOC_MAGIC, 12, struct nvmap_cache_op_64)
+#ifdef __KERNEL__
+#ifdef CONFIG_COMPAT
+#define NVMAP_IOC_CACHE_32  _IOW(NVMAP_IOC_MAGIC, 12, struct nvmap_cache_op_32)
+#endif /* CONFIG_COMPAT */
+#endif /* __KERNEL__ */
+
+/* Returns a global ID usable to allow a remote process to create a handle
+ * reference to the same handle */
+#define NVMAP_IOC_GET_ID  _IOWR(NVMAP_IOC_MAGIC, 13, struct nvmap_create_handle)
+
+/* Returns a file id that allows a remote process to create a handle
+ * reference to the same handle */
+#define NVMAP_IOC_GET_FD  _IOWR(NVMAP_IOC_MAGIC, 15, struct nvmap_create_handle)
+
+/* Create a new memory handle from file id passed */
+#define NVMAP_IOC_FROM_FD _IOWR(NVMAP_IOC_MAGIC, 16, struct nvmap_create_handle)
+
+/* Perform cache maintenance on a list of handles. */
+#define NVMAP_IOC_CACHE_LIST _IOW(NVMAP_IOC_MAGIC, 17,	\
+				  struct nvmap_cache_op_list)
+
+#define NVMAP_IOC_FROM_IVC_ID _IOWR(NVMAP_IOC_MAGIC, 19, struct nvmap_create_handle)
+#define NVMAP_IOC_GET_IVC_ID _IOWR(NVMAP_IOC_MAGIC, 20, struct nvmap_create_handle)
+#define NVMAP_IOC_GET_IVM_HEAPS _IOR(NVMAP_IOC_MAGIC, 21, unsigned int)
+
+/* Create a new memory handle from VA passed */
+#define NVMAP_IOC_FROM_VA _IOWR(NVMAP_IOC_MAGIC, 22, struct nvmap_create_handle_from_va)
+
+#define NVMAP_IOC_GUP_TEST _IOWR(NVMAP_IOC_MAGIC, 23, struct nvmap_gup_test)
+
+/* Define a label for allocation tag */
+#define NVMAP_IOC_SET_TAG_LABEL	_IOW(NVMAP_IOC_MAGIC, 24, struct nvmap_set_tag_label)
+
+#define NVMAP_IOC_GET_AVAILABLE_HEAPS \
+	_IOR(NVMAP_IOC_MAGIC, 25, struct nvmap_available_heaps)
+
+#define NVMAP_IOC_GET_HEAP_SIZE \
+	_IOR(NVMAP_IOC_MAGIC, 26, struct nvmap_heap_size)
+
+#define NVMAP_IOC_PARAMETERS \
+	_IOR(NVMAP_IOC_MAGIC, 27, struct nvmap_handle_parameters)
+
+/* START of T124 IOCTLS */
+/* Actually allocates memory from IVM heaps */
+#define NVMAP_IOC_ALLOC_IVM _IOW(NVMAP_IOC_MAGIC, 101, struct nvmap_alloc_ivm_handle)
+
+/* Allocate seperate memory for VPR */
+#define NVMAP_IOC_VPR_FLOOR_SIZE _IOW(NVMAP_IOC_MAGIC, 102, __u32)
+
+/* Get SCI_IPC_ID tied up with nvmap_handle */
+#define NVMAP_IOC_GET_SCIIPCID _IOR(NVMAP_IOC_MAGIC, 103, \
+		struct nvmap_sciipc_map)
+
+/* Get Nvmap handle from SCI_IPC_ID */
+#define NVMAP_IOC_HANDLE_FROM_SCIIPCID _IOR(NVMAP_IOC_MAGIC, 104, \
+		struct nvmap_sciipc_map)
+
+/* Get heap parameters such as total and frre size */
+#define NVMAP_IOC_QUERY_HEAP_PARAMS _IOR(NVMAP_IOC_MAGIC, 105, \
+		struct nvmap_query_heap_params)
+
+/* Duplicate NvRmMemHandle with same/reduced permission */
+#define NVMAP_IOC_DUP_HANDLE _IOWR(NVMAP_IOC_MAGIC, 106, \
+		struct nvmap_duplicate_handle)
+
+#define NVMAP_IOC_MAXNR (_IOC_NR(NVMAP_IOC_DUP_HANDLE))
+
+#endif /* __UAPI_LINUX_NVMAP_H */