nv-tegra-mirror/linux-nv-oot
1
0
Fork 0
mirror of git://nv-tegra.nvidia.com/linux-nv-oot.git synced 2025-12-22 09:11:26 +03:00
Code Issues Packages Projects Releases Wiki Activity

nvmap: Copy drivers and headers from kernel/nvidia

Browse Source 
Copy the driver and header sources of the nvmap to
kernel/nvidia-oot from kernel/nvidia as part of
removing the dependency of kernel/nvidia for OOT drivers.

The latest (few) git history of the files copied are
	b7a355916 video: tegra: nvmap: Fix type casting issue
	2128c5433 video: tegra: nvmap: Fix type casting issues
	0cd082559 video: tegra: nvmap: Change peer vm id data type
	4bd7ece67 tegra: nvmap: mark ivm carveout pages occupied
	e86f3630a video: tegra: nvmap: Fix type casting issue
	c43a23e58 video: tegra: nvmap: Fix type casting issue
	ca1dda22e video: tegra: nvmap: Fix type casting issue
	1f567abfe video: tegra: nvmap: Fix wrap up condition
	29db4d31c video: tegra: nvmap: Remove unnecessary debugfs
	fe72f1413 video: tegra: nvmap: Remove get_drv_data() call
	3b0fc79e7 video: tegra: nvmap: Fix coverity defect
	3cc0ce41b video: tegra: nvmap: Fix coverity defect
	6da39e966 video: tegra: nvmap: Fix WARN_ON condition
	a16351ff1 video: tegra: nvmap: Remove dead code
	9993f2d2d video: tegra: nvmap: Update print level
	6066a2077 video: tegra: nvmap: Remove nvmap_debug_lru_allocations_show
	3cdf2b7ba video: tegra: nvmap: Add kernel version check
	716ded4fc video: tegra: nvmap: Initialize the return value
	9b6c1b4ab video: tegra: nvmap: Correct debugfs code
	33e70118b video: tegra: nvmap: Fix Cert-C error handling bug
	7b960ed79 video: tegra: nvmap: Fix Cert-C error handling bug
	945dc1471 video: tegra: nvmap: Fix Cert-C error handling bug
	31e572de2 video: tegra: nvmap: Fix Cert-C error handling bug
	1f25cbf68 video: tegra: nvmap: Fix Cert-C error handling bug
	fa5428107 video: tegra: nvmap: Remove nvmap_handle_get_from_fd
	df73f2208 video: tegra: nvmap: Protect kmap/kunmap code
	9842e7c6a video: tegra: nvmap: Remove t19x dma_buf map/unmap
	06dff1a8d video: tegra: nvmap: Remove unnecessary export symbols
	6f097f86b video: tegra: nvmap: Fix Cert-C error handling bug
	f14171608 video: tegra: nvmap: load nvmap for T23x compatible platforms
	266812814 video: tegra: nvmap: Get rid of NVMAP_CONFIG_KSTABLE_KERNEL
	1b38c0887 nvmap: Don't use NV_BUILD_KERNEL_OPTIONS
	0ab8dc032 video: tegra: nvmap: Reintroduce NVMAP_CONFIG_VPR_RESIZE
	cc8db9797 driver: platform: tegra: Separate out vpr code
	28955d95c video/tegra: nvmap: Enable build as OOT module
	876d1fbb8 video: tegra: nvmap: Remove IS_ENABLED check
	5ea30867a nvmap: Add support to build as module from OOT kernel
	a71ad020e video: tegra: nvmap: Protect tegra_vpr args under config
	e70061cc1 video: tegra: nvmap: Do not export cvnas_dev
	d2a26ff36 video: tegra: nvmap: Include missing header
	692e4f682 video: tegra: nvmap: Update page coloring algo
	2b9dbb911 video: tegra: nvmap: Check for return value
	de8de12b6 video: tegra: nvmap: Enable legacy init support
	65d478158 video: tegra: nvmap: Remove dependency of cvnas
	38bdd6f05 video: tegra: nvmap: Make nvmap as loadable module
	9668e410b video: tegra: nvmap: Enable handle as ID
	11c6cbd23 tegra: nvmap: Fix build for Linux v5.18
	fbd95c3ab linux: nvmap: change ivm_handle to u32
	eb1e2c302 video: tegra: nvmap: Fix NVSCIIPC support
	022689b29 tegra: nvmap: return error if handle as ID enabled but id is fd
	19e5106ed video: tegra: nvmap: Don't treat ivm as reserved mem carveouts

Bug 4038415

Change-Id: I7108aec3b8532fe79c9423c2835744b1213719e8
Signed-off-by: Laxman Dewangan <ldewangan@nvidia.com>
This commit is contained in:
Laxman Dewangan
2023-04-11 05:47:21 +00:00
parent 4f365f1336
commit 4cf8c80669
32 changed files with 12890 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
drivers/video/tegra/nvmap/Makefile
View File

@@ -1,9 +1,47 @@
# SPDX-License-Identifier: GPL-2.0
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
GCOV_PROFILE := y
# NOTE: Do not change or add anything in this makefile.
# The source code and makefile rules are copied from the
# kernel/nvidia/drivers/video/tegra/nvmap. This file is
# just place-holder for empty makefile to avoid any build
# issue when copy is not done from command line and building
# the tree independent of source copy.
subdir-ccflags-y := -Werror
include $(srctree.nvidia)/drivers/video/tegra/nvmap/Makefile.memory.configs
ccflags-y += -I$(srctree.nvidia)/include/
ifeq ($(NVMAP_CONFIG), y)
# Build NvMap only when NVMAP_CONFIG is set to y
nvmap-y := nvmap_core.o \
nvmap_alloc.o \
nvmap_cache.o \
nvmap_dev.o \
nvmap_dmabuf.o \
nvmap_fault.o \
nvmap_handle.o \
nvmap_heap.o \
nvmap_ioctl.o \
nvmap_init.o \
nvmap_tag.o \
nvmap_mm.o \
nvmap_stats.o \
nvmap_carveout.o \
nvmap_kasan_wrapper.o
nvmap-$(NVMAP_CONFIG_HANDLE_AS_ID) += nvmap_id_array.o
nvmap-$(NVMAP_CONFIG_SCIIPC) += nvmap_sci_ipc.o
ifeq ($(NVMAP_CONFIG_PAGE_POOLS), y)
nvmap-y += nvmap_pp.o
endif #NVMAP_CONFIG_PAGE_POOLS
ifneq ($(filter y,$(CONFIG_ARCH_TEGRA_19x_SOC) $(CONFIG_ARCH_TEGRA_194_SOC)),)
nvmap-y += nvmap_init_t19x.o
endif #CONFIG_ARCH_TEGRA_19x_SOC or CONFIG_ARCH_TEGRA_194_SOC
KASAN_SANITIZE_nvmap_kasan_wrapper.o := n
ifeq ($(NVMAP_CONFIG_LOADABLE_MODULE), y)
nvmap-y += nvmap_cache_maint.o
obj-m += nvmap.o
else
# Not a loadable module
obj-y += nvmap.o
endif #NVMAP_CONFIG_LOADABLE_MODULE
endif #NVMAP_CONFIG

221
drivers/video/tegra/nvmap/Makefile.memory.configs Normal file
View File

@@ -0,0 +1,221 @@
# Copyright (c) 2021-2022, NVIDIA CORPORATION. All rights reserved.
#
# This program is free software; you can redistribute it and/or modify it
# under the terms and conditions of the GNU General Public License,
# version 2, as published by the Free Software Foundation.
#
# This program is distributed in the hope it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details.
#
# This file consists of 4 sections
# Section 1: This section is for doing prerequisite check.
# It checkes whether the prereq configs are enabled or not.
# If not, return error.
#
# Section 2: This section declare all configs with the default values
# just similar to Kconfig
#
# Section 3: This section consists of checks for kernel versions and
# actual values of these configs for corresponding kernel
# version. This is the place where we can enable/disable or
# set values to configs as per kernel version.
#
# Section 4: This section creates ccflags based upon the values specified
# in the section 2. These ccflags would be actually used in the
# source files. This section also takes care of the dependencies
# between the configs.
#
################################################################################
ifeq ($(CONFIG_ARCH_TEGRA), y)
# Section 1
# For dma_buf support CONFIG_DMA_SHARED_BUFFER needs be enabled
ifneq ($(CONFIG_DMA_SHARED_BUFFER),y)
$(error CONFIG_DMA_SHARED_BUFFER is not set)
endif
################################################################################
# Section 2
# This config is used to include the memory management driver for the
# Tegra GPU, multimedia and display subsystems
NVMAP_CONFIG := y
# Config to reduce the alloction overhead, which is significant
# for uncached, writecombine and inner cacheable memories as it
# involves changing page attributes during every allocation per page
# and flushing cache. Alloc time is reduced by allcoating the pages
# ahead and keeping them aside. The reserved pages would be released
# when system is low on memory and acquired back during release of
# memory.
NVMAP_CONFIG_PAGE_POOLS := y
# Config to include some debugging info in the page pools. This
# adds a bit of unnecessary overhead so only enable this is you
# suspect there is an issue with the nvmap page pools.
NVMAP_CONFIG_PAGE_POOL_DEBUG := n
# Config for page pool size in pages
NVMAP_CONFIG_PAGE_POOL_SIZE := 0x0
# Config to enable page coloring
# Page coloring rearranges the pages allocated based on the color
# of the page. It can improve memory access performance.
# The coloring option enable can optionally overallocate a portion of
# reqeusted allcoation size to improve the probabilty of better
# page coloring. If unsure, say Y.
NVMAP_CONFIG_COLOR_PAGES := y
# Config for FD number to start allocation from
# NvMap handles are represented with FD's in the user processes.
# To avoid Linux FD usage limitations, NvMap allocates FD starting
# from this number.
NVMAP_CONFIG_FD_START := 0x400
# Config for enabling deferred FD recycle
# A released nvmap handle would release memory and FD. This FD
# can be reused immediately for subsequent nvmap allocation req in
# the same process. Any buggy code in client process that continues to
# use FD of released allocation would continue to use new allocation
# and can lead to undesired consequences, which can be hard to debug.
# Enabling this option would defer recycling FD for longer time and
# allows debugging incorrect FD references by clients by returning errors
# for the accesses that occur after handle/FD release.
NVMAP_CONFIG_DEFER_FD_RECYCLE := n
# Config for FD number to start free FD recycle
# Once last allocated FD reaches this number, allocation of subsequent
# FD's start from NVMAP_START_FD.
NVMAP_CONFIG_DEFER_FD_RECYCLE_MAX_FD := 0x8000
# Config for enabling nvmap mapping with SciIpc secure buffer sharing
# Enable nvmap mapping with SciIpc secure buffer sharing.
# Supports nvmap ioctls to get Unique SciIpcId and attach
# it with nvmap_handle.
# Suppports getting nvmap_handle from SciIpcId passed via ioctl.
NVMAP_CONFIG_SCIIPC := n
# Config for enabling NvMap as OOT module
NVMAP_CONFIG_LOADABLE_MODULE := n
# Config for enabling PROCRANK functionality
NVMAP_CONFIG_PROCRANK := y
# Config for enabling VPR resize functionality
NVMAP_CONFIG_VPR_RESIZE := n
# Config for enabling few debugfs which would impact the NvMap performance
# There are few debugfs which would impact NvMap performance.
# Disable this when perf regression is observed.
NVMAP_CONFIG_DEBUG_MAPS := n
# This is fallback option to support handle as FD
# To support handle as ID, set this to n
# This config is useful to debug issue if its due to handle as ID or FD
NVMAP_CONFIG_HANDLE_AS_FD := n
# Config for kstable/OOT kernel
# This is useful when any kstable/OOT specific checks are needed
NVMAP_CONFIG_UPSTREAM_KERNEL := n
################################################################################
# Section 3
# Enable/Disable configs based upon the kernel version
# Specify the values which are different from the default values
ifdef CONFIG_TEGRA_VPR
# For 4.9
NVMAP_CONFIG_VPR_RESIZE := y
else
# For 5.10+
NVMAP_CONFIG_LOADABLE_MODULE := y
NVMAP_CONFIG_PROCRANK := n
ifneq ($(NVMAP_CONFIG_HANDLE_AS_FD),y)
NVMAP_CONFIG_HANDLE_AS_ID := y
NVMAP_CONFIG_FD_START := 0x0
endif
NVMAP_CONFIG_SCIIPC := y
# For OOT build
ifeq ($(CONFIG_TEGRA_OOT_MODULE),m)
NVMAP_CONFIG_UPSTREAM_KERNEL := y
NVMAP_CONFIG_COLOR_PAGES := n
endif
endif
################################################################################
# Section 4
# This section creates ccflags based upon the values specified
# in the section 2. These ccflags would be actually used in the
# source files.
ifeq ($(NVMAP_CONFIG),y)
# All other flags make sense only when NVMAP_CONFIG is enabled
ccflags-y += -DNVMAP_CONFIG
ifeq ($(NVMAP_CONFIG_PAGE_POOLS),y)
ccflags-y += -DNVMAP_CONFIG_PAGE_POOLS
# NVMAP_CONFIG_PAGE_POOL_DEBUG depends upon NVMAP_CONFIG_PAGE_POOLS
ifeq ($(NVMAP_CONFIG_PAGE_POOL_DEBUG),y)
ccflags-y += -DNVMAP_CONFIG_PAGE_POOL_DEBUG
endif #NVMAP_CONFIG_PAGE_POOL_DEBUG
# NVMAP_CONFIG_PAGE_POOL_SIZE depends upon NVMAP_CONFIG_PAGE_POOLS
ifdef NVMAP_CONFIG_PAGE_POOL_SIZE
ccflags-y += -DNVMAP_CONFIG_PAGE_POOL_SIZE=${NVMAP_CONFIG_PAGE_POOL_SIZE}
endif #NVMAP_CONFIG_PAGE_POOL_SIZE
endif #NVMAP_CONFIG_PAGE_POOLS
# NVMAP_CONFIG_COLOR_PAGES depends upon CONFIG_ARM64_4K_PAGES
ifeq ($(CONFIG_ARM64_4K_PAGES),y)
ifeq ($(NVMAP_CONFIG_COLOR_PAGES),y)
ccflags-y += -DNVMAP_CONFIG_COLOR_PAGES
endif #CONFIG_ARM64_4K_PAGES
endif #NVMAP_CONFIG_COLOR_PAGES
ifdef NVMAP_CONFIG_FD_START
ccflags-y += -DNVMAP_CONFIG_FD_START=${NVMAP_CONFIG_FD_START}
endif #NVMAP_CONFIG_FD_START
ifeq ($(NVMAP_CONFIG_DEFER_FD_RECYCLE),y)
ccflags-y += -DNVMAP_CONFIG_DEFER_FD_RECYCLE
# NVMAP_CONFIG_DEFER_FD_RECYCLE_MAX_FD depends upon CONFIG_NVMAP_DEFER_FD_RECYCLE
ifdef NVMAP_CONFIG_DEFER_FD_RECYCLE_MAX_FD
ccflags-y += -DNVMAP_CONFIG_DEFER_FD_RECYCLE_MAX_FD=${NVMAP_CONFIG_DEFER_FD_RECYCLE_MAX_FD}
endif #NVMAP_CONFIG_DEFER_FD_RECYCLE_MAX_FD
endif #NVMAP_CONFIG_DEFER_FD_RECYCLE
ifeq ($(NVMAP_CONFIG_SCIIPC),y)
ccflags-y += -DNVMAP_CONFIG_SCIIPC
endif #NVMAP_CONFIG_SCIIPC
ifeq ($(NVMAP_CONFIG_VPR_RESIZE),y)
ccflags-y += -DNVMAP_CONFIG_VPR_RESIZE
endif #NVMAP_CONFIG_VPR_RESIZE
ifeq ($(NVMAP_CONFIG_LOADABLE_MODULE),y)
ccflags-y += -DNVMAP_LOADABLE_MODULE
endif #NVMAP_CONFIG_LOADABLE_MODULE
ifeq ($(NVMAP_CONFIG_UPSTREAM_KERNEL),y)
ccflags-y += -DNVMAP_UPSTREAM_KERNEL
endif #NVMAP_CONFIG_UPSTREAM_KERNEL
ifeq ($(NVMAP_CONFIG_PROCRANK),y)
ccflags-y += -DNVMAP_CONFIG_PROCRANK
endif #NVMAP_CONFIG_PROCRANK
ifeq ($(NVMAP_CONFIG_DEBUG_MAPS),y)
ccflags-y += -DNVMAP_CONFIG_DEBUG_MAPS
endif #NVMAP_CONFIG_DEBUG_MAPS
ifeq ($(NVMAP_CONFIG_HANDLE_AS_ID),y)
ccflags-y += -DNVMAP_CONFIG_HANDLE_AS_ID
endif #NVMAP_CONFIG_HANDLE_AS_ID
ifeq ($(CONFIG_TEGRA_CVNAS),y)
ccflags-y += -DCVNAS_BUILTIN
endif #CONFIG_TEGRA_CVNAS
endif #NVMAP_CONFIG
endif #CONFIG_ARCH_TEGRA

23
drivers/video/tegra/nvmap/include/linux/nvmap_exports.h Normal file
View File

@@ -0,0 +1,23 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*/
#ifndef __NVMAP_EXPORTS_H
#define __NVMAP_EXPORTS_H
void *nvmap_dma_alloc_attrs(struct device *dev, size_t size,
dma_addr_t *dma_handle, gfp_t flag,
unsigned long attrs);
void nvmap_dma_free_attrs(struct device *dev, size_t size, void *cpu_addr,
dma_addr_t dma_handle, unsigned long attrs);
extern struct device tegra_vpr_dev;
#endif /* __NVMAP_EXPORTS_H */

1150
drivers/video/tegra/nvmap/nvmap_alloc.c Normal file
View File

File diff suppressed because it is too large Load Diff

431
drivers/video/tegra/nvmap/nvmap_cache.c Normal file
View File

@@ -0,0 +1,431 @@
/*
* drivers/video/tegra/nvmap/nvmap_cache.c
*
* Copyright (c) 2011-2022, NVIDIA CORPORATION. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*/
#define pr_fmt(fmt) "nvmap: %s() " fmt, __func__
#include <linux/io.h>
#include <linux/debugfs.h>
#include <linux/of.h>
#include <linux/version.h>
#if KERNEL_VERSION(4, 15, 0) > LINUX_VERSION_CODE
#include <soc/tegra/chip-id.h>
#else
#include <soc/tegra/fuse.h>
#endif
#include <linux/sys_soc.h>
#ifdef NVMAP_LOADABLE_MODULE
__weak struct arm64_ftr_reg arm64_ftr_reg_ctrel0;
#endif /*NVMAP_LOADABLE_MODULE */
#include <trace/events/nvmap.h>
#include "nvmap_priv.h"
/*
* FIXME:
*
* __clean_dcache_page() is only available on ARM64 (well, we haven't
* implemented it on ARMv7).
*/
void nvmap_clean_cache_page(struct page *page)
{
__clean_dcache_area_poc(page_address(page), PAGE_SIZE);
}
void nvmap_clean_cache(struct page **pages, int numpages)
{
int i;
/* Not technically a flush but that's what nvmap knows about. */
nvmap_stats_inc(NS_CFLUSH_DONE, numpages << PAGE_SHIFT);
trace_nvmap_cache_flush(numpages << PAGE_SHIFT,
nvmap_stats_read(NS_ALLOC),
nvmap_stats_read(NS_CFLUSH_RQ),
nvmap_stats_read(NS_CFLUSH_DONE));
for (i = 0; i < numpages; i++)
nvmap_clean_cache_page(pages[i]);
}
void inner_cache_maint(unsigned int op, void *vaddr, size_t size)
{
if (op == NVMAP_CACHE_OP_WB_INV)
__dma_flush_area(vaddr, size);
else if (op == NVMAP_CACHE_OP_INV)
__dma_map_area(vaddr, size, DMA_FROM_DEVICE);
else
__dma_map_area(vaddr, size, DMA_TO_DEVICE);
}
static void heap_page_cache_maint(
struct nvmap_handle *h, unsigned long start, unsigned long end,
unsigned int op, bool inner, bool outer, bool clean_only_dirty)
{
/* Don't perform cache maint for RO mapped buffers */
if (h->from_va && h->is_ro)
return;
if (h->userflags & NVMAP_HANDLE_CACHE_SYNC) {
/*
* zap user VA->PA mappings so that any access to the pages
* will result in a fault and can be marked dirty
*/
nvmap_handle_mkclean(h, start, end-start);
nvmap_zap_handle(h, start, end - start);
}
if (inner) {
if (!h->vaddr) {
if (__nvmap_mmap(h))
__nvmap_munmap(h, h->vaddr);
else
goto per_page_cache_maint;
}
/* Fast inner cache maintenance using single mapping */
inner_cache_maint(op, h->vaddr + start, end - start);
if (!outer)
return;
/* Skip per-page inner maintenance in loop below */
inner = false;
}
per_page_cache_maint:
while (start < end) {
struct page *page;
phys_addr_t paddr;
unsigned long next;
unsigned long off;
size_t size;
int ret;
page = nvmap_to_page(h->pgalloc.pages[start >> PAGE_SHIFT]);
next = min(((start + PAGE_SIZE) & PAGE_MASK), end);
off = start & ~PAGE_MASK;
size = next - start;
paddr = page_to_phys(page) + off;
ret = nvmap_cache_maint_phys_range(op, paddr, paddr + size,
inner, outer);
WARN_ON(ret != 0);
start = next;
}
}
struct cache_maint_op {
phys_addr_t start;
phys_addr_t end;
unsigned int op;
struct nvmap_handle *h;
bool inner;
bool outer;
bool clean_only_dirty;
};
int nvmap_cache_maint_phys_range(unsigned int op, phys_addr_t pstart,
phys_addr_t pend, int inner, int outer)
{
void __iomem *io_addr;
phys_addr_t loop;
if (!inner)
goto do_outer;
loop = pstart;
while (loop < pend) {
phys_addr_t next = (loop + PAGE_SIZE) & PAGE_MASK;
void *base;
next = min(next, pend);
io_addr = __ioremap(loop, PAGE_SIZE, PG_PROT_KERNEL);
if (io_addr == NULL)
return -ENOMEM;
base = (__force void *)io_addr + (loop & ~PAGE_MASK);
inner_cache_maint(op, base, next - loop);
iounmap(io_addr);
loop = next;
}
do_outer:
return 0;
}
static int do_cache_maint(struct cache_maint_op *cache_work)
{
phys_addr_t pstart = cache_work->start;
phys_addr_t pend = cache_work->end;
int err = 0;
struct nvmap_handle *h = cache_work->h;
unsigned int op = cache_work->op;
if (!h || !h->alloc)
return -EFAULT;
wmb();
if (h->flags == NVMAP_HANDLE_UNCACHEABLE ||
h->flags == NVMAP_HANDLE_WRITE_COMBINE || pstart == pend)
goto out;
trace_nvmap_cache_maint(h->owner, h, pstart, pend, op, pend - pstart);
if (pstart > h->size || pend > h->size) {
pr_warn("cache maintenance outside handle\n");
err = -EINVAL;
goto out;
}
if (h->heap_pgalloc) {
heap_page_cache_maint(h, pstart, pend, op, true,
(h->flags == NVMAP_HANDLE_INNER_CACHEABLE) ?
false : true, cache_work->clean_only_dirty);
goto out;
}
pstart += h->carveout->base;
pend += h->carveout->base;
err = nvmap_cache_maint_phys_range(op, pstart, pend, true,
h->flags != NVMAP_HANDLE_INNER_CACHEABLE);
out:
if (!err) {
nvmap_stats_inc(NS_CFLUSH_DONE, pend - pstart);
}
trace_nvmap_cache_flush(pend - pstart,
nvmap_stats_read(NS_ALLOC),
nvmap_stats_read(NS_CFLUSH_RQ),
nvmap_stats_read(NS_CFLUSH_DONE));
return 0;
}
static void nvmap_handle_get_cacheability(struct nvmap_handle *h,
bool *inner, bool *outer)
{
*inner = h->flags == NVMAP_HANDLE_CACHEABLE ||
h->flags == NVMAP_HANDLE_INNER_CACHEABLE;
*outer = h->flags == NVMAP_HANDLE_CACHEABLE;
}
int __nvmap_do_cache_maint(struct nvmap_client *client,
struct nvmap_handle *h,
unsigned long start, unsigned long end,
unsigned int op, bool clean_only_dirty)
{
int err;
struct cache_maint_op cache_op;
h = nvmap_handle_get(h);
if (!h)
return -EFAULT;
if ((start >= h->size) || (end > h->size)) {
pr_debug("%s start: %ld end: %ld h->size: %zu\n", __func__,
start, end, h->size);
nvmap_handle_put(h);
return -EFAULT;
}
if (!(h->heap_type & nvmap_dev->cpu_access_mask)) {
pr_debug("%s heap_type %u access_mask 0x%x\n", __func__,
h->heap_type, nvmap_dev->cpu_access_mask);
nvmap_handle_put(h);
return -EPERM;
}
nvmap_kmaps_inc(h);
if (op == NVMAP_CACHE_OP_INV)
op = NVMAP_CACHE_OP_WB_INV;
/* clean only dirty is applicable only for Write Back operation */
if (op != NVMAP_CACHE_OP_WB)
clean_only_dirty = false;
cache_op.h = h;
cache_op.start = start ? start : 0;
cache_op.end = end ? end : h->size;
cache_op.op = op;
nvmap_handle_get_cacheability(h, &cache_op.inner, &cache_op.outer);
cache_op.clean_only_dirty = clean_only_dirty;
nvmap_stats_inc(NS_CFLUSH_RQ, end - start);
err = do_cache_maint(&cache_op);
nvmap_kmaps_dec(h);
nvmap_handle_put(h);
return err;
}
int __nvmap_cache_maint(struct nvmap_client *client,
struct nvmap_cache_op_64 *op)
{
struct vm_area_struct *vma;
struct nvmap_vma_priv *priv;
struct nvmap_handle *handle;
unsigned long start;
unsigned long end;
int err = 0;
if (!op->addr || op->op < NVMAP_CACHE_OP_WB ||
op->op > NVMAP_CACHE_OP_WB_INV)
return -EINVAL;
handle = nvmap_handle_get_from_id(client, op->handle);
if (IS_ERR_OR_NULL(handle))
return -EINVAL;
nvmap_acquire_mmap_read_lock(current->mm);
vma = find_vma(current->active_mm, (unsigned long)op->addr);
if (!vma || !is_nvmap_vma(vma) ||
(ulong)op->addr < vma->vm_start ||
(ulong)op->addr >= vma->vm_end ||
op->len > vma->vm_end - (ulong)op->addr) {
err = -EADDRNOTAVAIL;
goto out;
}
priv = (struct nvmap_vma_priv *)vma->vm_private_data;
if (priv->handle != handle) {
err = -EFAULT;
goto out;
}
start = (unsigned long)op->addr - vma->vm_start +
(vma->vm_pgoff << PAGE_SHIFT);
end = start + op->len;
err = __nvmap_do_cache_maint(client, priv->handle, start, end, op->op,
false);
out:
nvmap_release_mmap_read_lock(current->mm);
nvmap_handle_put(handle);
return err;
}
/*
* Perform cache op on the list of memory regions within passed handles.
* A memory region within handle[i] is identified by offsets[i], sizes[i]
*
* sizes[i] == 0 is a special case which causes handle wide operation,
* this is done by replacing offsets[i] = 0, sizes[i] = handles[i]->size.
* So, the input arrays sizes, offsets are not guaranteed to be read-only
*
* This will optimze the op if it can.
* In the case that all the handles together are larger than the inner cache
* maint threshold it is possible to just do an entire inner cache flush.
*
* NOTE: this omits outer cache operations which is fine for ARM64
*/
static int __nvmap_do_cache_maint_list(struct nvmap_handle **handles,
u64 *offsets, u64 *sizes, int op, u32 nr_ops,
bool is_32)
{
u32 i;
u64 total = 0;
u64 thresh = ~0;
WARN(!IS_ENABLED(CONFIG_ARM64),
"cache list operation may not function properly");
for (i = 0; i < nr_ops; i++) {
bool inner, outer;
u32 *sizes_32 = (u32 *)sizes;
u64 size = is_32 ? sizes_32[i] : sizes[i];
nvmap_handle_get_cacheability(handles[i], &inner, &outer);
if (!inner && !outer)
continue;
if ((op == NVMAP_CACHE_OP_WB) && nvmap_handle_track_dirty(handles[i]))
total += atomic_read(&handles[i]->pgalloc.ndirty);
else
total += size ? size : handles[i]->size;
}
if (!total)
return 0;
/* Full flush in the case the passed list is bigger than our
* threshold. */
if (total >= thresh) {
for (i = 0; i < nr_ops; i++) {
if (handles[i]->userflags &
NVMAP_HANDLE_CACHE_SYNC) {
nvmap_handle_mkclean(handles[i], 0,
handles[i]->size);
nvmap_zap_handle(handles[i], 0,
handles[i]->size);
}
}
nvmap_stats_inc(NS_CFLUSH_RQ, total);
nvmap_stats_inc(NS_CFLUSH_DONE, thresh);
trace_nvmap_cache_flush(total,
nvmap_stats_read(NS_ALLOC),
nvmap_stats_read(NS_CFLUSH_RQ),
nvmap_stats_read(NS_CFLUSH_DONE));
} else {
for (i = 0; i < nr_ops; i++) {
u32 *offs_32 = (u32 *)offsets, *sizes_32 = (u32 *)sizes;
u64 size = is_32 ? sizes_32[i] : sizes[i];
u64 offset = is_32 ? offs_32[i] : offsets[i];
int err;
size = size ?: handles[i]->size;
offset = offset ?: 0;
err = __nvmap_do_cache_maint(handles[i]->owner,
handles[i], offset,
offset + size,
op, false);
if (err) {
pr_err("cache maint per handle failed [%d]\n",
err);
return err;
}
}
}
return 0;
}
#if (LINUX_VERSION_CODE > KERNEL_VERSION(4, 9, 0))
static const struct soc_device_attribute tegra194_soc = {
.soc_id = "TEGRA194",
};
static const struct soc_device_attribute tegra234_soc = {
.soc_id = "TEGRA234",
};
#endif
inline int nvmap_do_cache_maint_list(struct nvmap_handle **handles,
u64 *offsets, u64 *sizes, int op, u32 nr_ops,
bool is_32)
{
/*
* As io-coherency is enabled by default from T194 onwards,
* Don't do cache maint from CPU side. The HW, SCF will do.
*/
#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0))
if (!(tegra_get_chip_id() == TEGRA194))
#else
if (!soc_device_match(&tegra194_soc) &&
!soc_device_match(&tegra234_soc))
#endif
return __nvmap_do_cache_maint_list(handles,
offsets, sizes, op, nr_ops, is_32);
return 0;
}

236
drivers/video/tegra/nvmap/nvmap_cache_maint.S Normal file
View File

@@ -0,0 +1,236 @@
/*
* Copyright (c) 2021-2022, NVIDIA CORPORATION. All rights reserved.
*
* Cache maintenance
*
* Derived from Linux kernel source file arch/arm64/mm/cache.S
* Copyright (C) 2001 Deep Blue Solutions Ltd.
* Copyright (C) 2012 ARM Ltd.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
*/
#include <linux/errno.h>
#include <linux/linkage.h>
#include <linux/init.h>
#include <linux/version.h>
#include <asm/assembler.h>
#include <asm/cpufeature.h>
#include <asm/alternative.h>
#include <asm/asm-uaccess.h>
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 18, 0)
#define NVMAP_SYM_FUNC_START(func) SYM_FUNC_START(__ip_##func)
#define NVMAP_SYM_FUNC_END(func) SYM_FUNC_END(__ip_##func)
#define NVMAP_SYM_FUNC_ALIAS(func) SYM_FUNC_ALIAS(func, __ip_##func)
#else
#define NVMAP_SYM_FUNC_START(func) SYM_FUNC_START_PI(func)
#define NVMAP_SYM_FUNC_END(func) SYM_FUNC_END_PI(func)
#define NVMAP_SYM_FUNC_ALIAS(func)
#endif
/*
* invalidate_icache_range(start,end)
*
* Ensure that the I cache is invalid within specified region.
*
* - start - virtual start address of region
* - end - virtual end address of region
*/
SYM_FUNC_START(invalidate_icache_range)
alternative_if ARM64_HAS_CACHE_DIC
mov x0, xzr
isb
ret
alternative_else_nop_endif
uaccess_ttbr0_enable x2, x3, x4
invalidate_icache_by_line x0, x1, x2, x3, 2f
mov x0, xzr
1:
uaccess_ttbr0_disable x1, x2
ret
2:
mov x0, #-EFAULT
b 1b
SYM_FUNC_END(invalidate_icache_range)
/*
* __flush_dcache_area(kaddr, size)
*
* Ensure that any D-cache lines for the interval [kaddr, kaddr+size)
* are cleaned and invalidated to the PoC.
*
* - kaddr - kernel address
* - size - size in question
*/
NVMAP_SYM_FUNC_START(__flush_dcache_area)
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 14, 0)
add x1, x0, x1
#endif
dcache_by_line_op civac, sy, x0, x1, x2, x3
ret
NVMAP_SYM_FUNC_END(__flush_dcache_area)
NVMAP_SYM_FUNC_ALIAS(__flush_dcache_area)
/*
* __clean_dcache_area_pou(kaddr, size)
*
* Ensure that any D-cache lines for the interval [kaddr, kaddr+size)
* are cleaned to the PoU.
*
* - kaddr - kernel address
* - size - size in question
*/
SYM_FUNC_START(__clean_dcache_area_pou)
alternative_if ARM64_HAS_CACHE_IDC
dsb ishst
ret
alternative_else_nop_endif
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 14, 0)
add x1, x0, x1
#endif
dcache_by_line_op cvau, ish, x0, x1, x2, x3
ret
SYM_FUNC_END(__clean_dcache_area_pou)
/*
* __inval_dcache_area(kaddr, size)
*
* Ensure that any D-cache lines for the interval [kaddr, kaddr+size)
* are invalidated. Any partial lines at the ends of the interval are
* also cleaned to PoC to prevent data loss.
*
* - kaddr - kernel address
* - size - size in question
*/
SYM_FUNC_START_LOCAL(__dma_inv_area)
NVMAP_SYM_FUNC_START(__inval_dcache_area)
/* FALLTHROUGH */
/*
* __dma_inv_area(start, size)
* - start - virtual start address of region
* - size - size in question
*/
add x1, x1, x0
dcache_line_size x2, x3
sub x3, x2, #1
tst x1, x3 // end cache line aligned?
bic x1, x1, x3
b.eq 1f
dc civac, x1 // clean & invalidate D / U line
1: tst x0, x3 // start cache line aligned?
bic x0, x0, x3
b.eq 2f
dc civac, x0 // clean & invalidate D / U line
b 3f
2: dc ivac, x0 // invalidate D / U line
3: add x0, x0, x2
cmp x0, x1
b.lo 2b
dsb sy
ret
NVMAP_SYM_FUNC_END(__inval_dcache_area)
NVMAP_SYM_FUNC_ALIAS(__inval_dcache_area)
SYM_FUNC_END(__dma_inv_area)
/*
* __clean_dcache_area_poc(kaddr, size)
*
* Ensure that any D-cache lines for the interval [kaddr, kaddr+size)
* are cleaned to the PoC.
*
* - kaddr - kernel address
* - size - size in question
*/
SYM_FUNC_START_LOCAL(__dma_clean_area)
NVMAP_SYM_FUNC_START(__clean_dcache_area_poc)
/* FALLTHROUGH */
/*
* __dma_clean_area(start, size)
* - start - virtual start address of region
* - size - size in question
*/
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 14, 0)
add x1, x0, x1
#endif
dcache_by_line_op cvac, sy, x0, x1, x2, x3
ret
NVMAP_SYM_FUNC_END(__clean_dcache_area_poc)
NVMAP_SYM_FUNC_ALIAS(__clean_dcache_area_poc)
SYM_FUNC_END(__dma_clean_area)
/*
* __clean_dcache_area_pop(kaddr, size)
*
* Ensure that any D-cache lines for the interval [kaddr, kaddr+size)
* are cleaned to the PoP.
*
* - kaddr - kernel address
* - size - size in question
*/
NVMAP_SYM_FUNC_START(__clean_dcache_area_pop)
alternative_if_not ARM64_HAS_DCPOP
b __clean_dcache_area_poc
alternative_else_nop_endif
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 14, 0)
add x1, x0, x1
#endif
dcache_by_line_op cvap, sy, x0, x1, x2, x3
ret
NVMAP_SYM_FUNC_END(__clean_dcache_area_pop)
NVMAP_SYM_FUNC_ALIAS(__clean_dcache_area_pop)
/*
* __dma_flush_area(start, size)
*
* clean & invalidate D / U line
*
* - start - virtual start address of region
* - size - size in question
*/
NVMAP_SYM_FUNC_START(__dma_flush_area)
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 14, 0)
add x1, x0, x1
#endif
dcache_by_line_op civac, sy, x0, x1, x2, x3
ret
NVMAP_SYM_FUNC_END(__dma_flush_area)
NVMAP_SYM_FUNC_ALIAS(__dma_flush_area)
/*
* __dma_map_area(start, size, dir)
* - start - kernel virtual start address
* - size - size of region
* - dir - DMA direction
*/
NVMAP_SYM_FUNC_START(__dma_map_area)
cmp w2, #DMA_FROM_DEVICE
b.eq __dma_inv_area
b __dma_clean_area
NVMAP_SYM_FUNC_END(__dma_map_area)
NVMAP_SYM_FUNC_ALIAS(__dma_map_area)
/*
* __dma_unmap_area(start, size, dir)
* - start - kernel virtual start address
* - size - size of region
* - dir - DMA direction
*/
NVMAP_SYM_FUNC_START(__dma_unmap_area)
cmp w2, #DMA_TO_DEVICE
b.ne __dma_inv_area
ret
NVMAP_SYM_FUNC_END(__dma_unmap_area)
NVMAP_SYM_FUNC_ALIAS(__dma_unmap_area)

270
drivers/video/tegra/nvmap/nvmap_carveout.c Normal file
View File

@@ -0,0 +1,270 @@
/*
* drivers/video/tegra/nvmap/nvmap_dev.c
*
* Interface with nvmap carveouts
*
* Copyright (c) 2011-2022, NVIDIA CORPORATION. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*/
#include <linux/debugfs.h>
#include "nvmap_priv.h"
extern struct nvmap_device *nvmap_dev;
extern const struct file_operations debug_clients_fops;
extern const struct file_operations debug_allocations_fops;
extern const struct file_operations debug_all_allocations_fops;
extern const struct file_operations debug_orphan_handles_fops;
extern const struct file_operations debug_maps_fops;
#ifdef NVMAP_CONFIG_DEBUG_MAPS
extern const struct file_operations debug_device_list_fops;
#endif /* NVMAP_CONFIG_DEBUG_MAPS */
int nvmap_create_carveout(const struct nvmap_platform_carveout *co)
{
int i, err = 0;
struct nvmap_carveout_node *node;
mutex_lock(&nvmap_dev->carveout_lock);
if (!nvmap_dev->heaps) {
int nr_heaps;
nvmap_dev->nr_carveouts = 0;
if (nvmap_dev->plat)
nr_heaps = nvmap_dev->plat->nr_carveouts + 1;
else
nr_heaps = 1;
nvmap_dev->heaps = kzalloc(sizeof(struct nvmap_carveout_node) *
nr_heaps, GFP_KERNEL);
if (!nvmap_dev->heaps) {
err = -ENOMEM;
pr_err("couldn't allocate carveout memory\n");
goto out;
}
nvmap_dev->nr_heaps = nr_heaps;
} else if (nvmap_dev->nr_carveouts >= nvmap_dev->nr_heaps) {
node = krealloc(nvmap_dev->heaps,
sizeof(*node) * (nvmap_dev->nr_carveouts + 1),
GFP_KERNEL);
if (!node) {
err = -ENOMEM;
pr_err("nvmap heap array resize failed\n");
goto out;
}
nvmap_dev->heaps = node;
nvmap_dev->nr_heaps = nvmap_dev->nr_carveouts + 1;
}
for (i = 0; i < nvmap_dev->nr_heaps; i++)
if ((co->usage_mask != NVMAP_HEAP_CARVEOUT_IVM) &&
(nvmap_dev->heaps[i].heap_bit & co->usage_mask)) {
pr_err("carveout %s already exists\n", co->name);
err = -EEXIST;
goto out;
}
node = &nvmap_dev->heaps[nvmap_dev->nr_carveouts];
node->base = round_up(co->base, PAGE_SIZE);
node->size = round_down(co->size -
(node->base - co->base), PAGE_SIZE);
if (!co->size)
goto out;
node->carveout = nvmap_heap_create(
nvmap_dev->dev_user.this_device, co,
node->base, node->size, node);
if (!node->carveout) {
err = -ENOMEM;
pr_err("couldn't create %s\n", co->name);
goto out;
}
node->index = nvmap_dev->nr_carveouts;
nvmap_dev->nr_carveouts++;
node->heap_bit = co->usage_mask;
if (!IS_ERR_OR_NULL(nvmap_dev->debug_root)) {
struct dentry *heap_root =
debugfs_create_dir(co->name, nvmap_dev->debug_root);
if (!IS_ERR_OR_NULL(heap_root)) {
debugfs_create_file("clients", S_IRUGO,
heap_root,
(void *)(uintptr_t)node->heap_bit,
&debug_clients_fops);
debugfs_create_file("allocations", S_IRUGO,
heap_root,
(void *)(uintptr_t)node->heap_bit,
&debug_allocations_fops);
debugfs_create_file("all_allocations", S_IRUGO,
heap_root,
(void *)(uintptr_t)node->heap_bit,
&debug_all_allocations_fops);
debugfs_create_file("orphan_handles", S_IRUGO,
heap_root,
(void *)(uintptr_t)node->heap_bit,
&debug_orphan_handles_fops);
debugfs_create_file("maps", S_IRUGO,
heap_root,
(void *)(uintptr_t)node->heap_bit,
&debug_maps_fops);
debugfs_create_bool("no_cpu_access", S_IRUGO,
heap_root, (bool *)&co->no_cpu_access);
#ifdef NVMAP_CONFIG_DEBUG_MAPS
debugfs_create_file("device_list", S_IRUGO,
heap_root,
(void *)(uintptr_t)node->heap_bit,
&debug_device_list_fops);
#endif /* NVMAP_CONFIG_DEBUG_MAPS */
nvmap_heap_debugfs_init(heap_root,
node->carveout);
}
}
out:
mutex_unlock(&nvmap_dev->carveout_lock);
return err;
}
#ifdef NVMAP_CONFIG_DEBUG_MAPS
struct nvmap_device_list *nvmap_is_device_present(char *device_name, u32 heap_type)
{
struct rb_node *node = NULL;
int i;
if (heap_type == NVMAP_HEAP_IOVMM) {
node = nvmap_dev->device_names.rb_node;
} else {
for (i = 0; i < nvmap_dev->nr_carveouts; i++) {
if ((heap_type & nvmap_dev->heaps[i].heap_bit) &&
nvmap_dev->heaps[i].carveout) {
node = nvmap_dev->heaps[i].carveout->device_names.rb_node;
break;
}
}
}
while (node) {
struct nvmap_device_list *dl = container_of(node,
struct nvmap_device_list, node);
if (strcmp(dl->device_name, device_name) > 0)
node = node->rb_left;
else if (strcmp(dl->device_name, device_name) < 0)
node = node->rb_right;
else
return dl;
}
return NULL;
}
void nvmap_add_device_name(char *device_name, u64 dma_mask, u32 heap_type)
{
struct rb_root *root = NULL;
struct rb_node **new = NULL, *parent = NULL;
struct nvmap_device_list *dl = NULL;
int i;
if (heap_type == NVMAP_HEAP_IOVMM) {
root = &nvmap_dev->device_names;
} else {
for (i = 0; i < nvmap_dev->nr_carveouts; i++) {
if ((heap_type & nvmap_dev->heaps[i].heap_bit) &&
nvmap_dev->heaps[i].carveout) {
root = &nvmap_dev->heaps[i].carveout->device_names;
break;
}
}
}
if (root) {
new = &(root->rb_node);
while (*new) {
dl = container_of(*new, struct nvmap_device_list, node);
parent = *new;
if (strcmp(dl->device_name, device_name) > 0)
new = &((*new)->rb_left);
else if (strcmp(dl->device_name, device_name) < 0)
new = &((*new)->rb_right);
}
dl = kzalloc(sizeof(*dl), GFP_KERNEL);
if (!dl)
return;
dl->device_name = kzalloc(strlen(device_name) + 1, GFP_KERNEL);
if (!dl->device_name)
return;
strcpy(dl->device_name, device_name);
dl->dma_mask = dma_mask;
rb_link_node(&dl->node, parent, new);
rb_insert_color(&dl->node, root);
}
}
void nvmap_remove_device_name(char *device_name, u32 heap_type)
{
struct nvmap_device_list *dl = NULL;
int i;
dl = nvmap_is_device_present(device_name, heap_type);
if (dl) {
if (heap_type == NVMAP_HEAP_IOVMM) {
rb_erase(&dl->node,
&nvmap_dev->device_names);
kfree(dl->device_name);
kfree(dl);
return;
}
for (i = 0; i < nvmap_dev->nr_carveouts; i++) {
if ((heap_type & nvmap_dev->heaps[i].heap_bit) &&
nvmap_dev->heaps[i].carveout) {
rb_erase(&dl->node,
&nvmap_dev->heaps[i].carveout->device_names);
kfree(dl->device_name);
kfree(dl);
return;
}
}
}
}
#endif /* NVMAP_CONFIG_DEBUG_MAPS */
static
struct nvmap_heap_block *do_nvmap_carveout_alloc(struct nvmap_client *client,
struct nvmap_handle *handle,
unsigned long type,
phys_addr_t *start)
{
struct nvmap_carveout_node *co_heap;
struct nvmap_device *dev = nvmap_dev;
int i;
for (i = 0; i < dev->nr_carveouts; i++) {
struct nvmap_heap_block *block;
co_heap = &dev->heaps[i];
if (!(co_heap->heap_bit & type))
continue;
if (type & NVMAP_HEAP_CARVEOUT_IVM)
handle->size = ALIGN(handle->size, NVMAP_IVM_ALIGNMENT);
block = nvmap_heap_alloc(co_heap->carveout, handle, start);
if (block)
return block;
}
return NULL;
}
struct nvmap_heap_block *nvmap_carveout_alloc(struct nvmap_client *client,
struct nvmap_handle *handle,
unsigned long type,
phys_addr_t *start)
{
return do_nvmap_carveout_alloc(client, handle, type, start);
}

355
drivers/video/tegra/nvmap/nvmap_core.c Normal file
View File

@@ -0,0 +1,355 @@
/*
* drivers/video/tegra/nvmap/nvmap_core.c
*
* Memory manager for Tegra GPU
*
* Copyright (c) 2009-2022, NVIDIA CORPORATION. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*/
#define pr_fmt(fmt) "nvmap: %s() " fmt, __func__
#include <linux/err.h>
#include <linux/highmem.h>
#include <linux/io.h>
#include <linux/rbtree.h>
#include <linux/vmalloc.h>
#include <linux/wait.h>
#include <linux/slab.h>
#include <linux/export.h>
#include <asm/pgtable.h>
#include <asm/tlbflush.h>
#include <linux/nvmap.h>
#include <trace/events/nvmap.h>
#include "nvmap_priv.h"
static phys_addr_t handle_phys(struct nvmap_handle *h)
{
if (h->heap_pgalloc)
BUG();
return h->carveout->base;
}
#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0)
void *__nvmap_kmap(struct nvmap_handle *h, unsigned int pagenum)
{
phys_addr_t paddr;
unsigned long kaddr;
void __iomem *addr;
pgprot_t prot;
if (!virt_addr_valid(h))
return NULL;
h = nvmap_handle_get(h);
if (!h)
return NULL;
/*
* If the handle is RO and virtual mapping is requested in
* kernel address space, return error.
*/
if (h->from_va && h->is_ro)
goto put_handle;
if (!h->alloc)
goto put_handle;
if (!(h->heap_type & nvmap_dev->cpu_access_mask))
goto put_handle;
nvmap_kmaps_inc(h);
if (pagenum >= h->size >> PAGE_SHIFT)
goto out;
if (h->vaddr) {
kaddr = (unsigned long)h->vaddr + pagenum * PAGE_SIZE;
} else {
prot = nvmap_pgprot(h, PG_PROT_KERNEL);
if (h->heap_pgalloc)
paddr = page_to_phys(nvmap_to_page(
h->pgalloc.pages[pagenum]));
else
paddr = h->carveout->base + pagenum * PAGE_SIZE;
addr = __ioremap(paddr, PAGE_SIZE, prot);
if (addr == NULL)
goto out;
kaddr = (unsigned long)addr;
}
return (void *)kaddr;
out:
nvmap_kmaps_dec(h);
put_handle:
nvmap_handle_put(h);
return NULL;
}
void __nvmap_kunmap(struct nvmap_handle *h, unsigned int pagenum,
void *addr)
{
phys_addr_t paddr;
if (!h || !h->alloc ||
WARN_ON(!virt_addr_valid(h)) ||
WARN_ON(!addr) ||
!(h->heap_type & nvmap_dev->cpu_access_mask))
return;
if (WARN_ON(pagenum >= h->size >> PAGE_SHIFT))
return;
if (h->vaddr && (h->vaddr == (addr - pagenum * PAGE_SIZE)))
goto out;
if (h->heap_pgalloc)
paddr = page_to_phys(nvmap_to_page(h->pgalloc.pages[pagenum]));
else
paddr = h->carveout->base + pagenum * PAGE_SIZE;
if (h->flags != NVMAP_HANDLE_UNCACHEABLE &&
h->flags != NVMAP_HANDLE_WRITE_COMBINE) {
__dma_flush_area(addr, PAGE_SIZE);
outer_flush_range(paddr, paddr + PAGE_SIZE); /* FIXME */
}
iounmap((void __iomem *)addr);
out:
nvmap_kmaps_dec(h);
nvmap_handle_put(h);
}
#endif
void *__nvmap_mmap(struct nvmap_handle *h)
{
pgprot_t prot;
void *vaddr;
unsigned long adj_size;
struct page **pages = NULL;
int i = 0;
if (!virt_addr_valid(h))
return NULL;
h = nvmap_handle_get(h);
if (!h)
return NULL;
/*
* If the handle is RO and virtual mapping is requested in
* kernel address space, return error.
*/
if (h->from_va && h->is_ro)
goto put_handle;
if (!h->alloc)
goto put_handle;
if (!(h->heap_type & nvmap_dev->cpu_access_mask))
goto put_handle;
if (h->vaddr)
return h->vaddr;
nvmap_kmaps_inc(h);
prot = nvmap_pgprot(h, PG_PROT_KERNEL);
if (h->heap_pgalloc) {
pages = nvmap_pages(h->pgalloc.pages, h->size >> PAGE_SHIFT);
if (!pages)
goto out;
vaddr = vmap(pages, h->size >> PAGE_SHIFT, VM_MAP, prot);
if (!vaddr && !h->vaddr)
goto out;
nvmap_altfree(pages, (h->size >> PAGE_SHIFT) * sizeof(*pages));
if (vaddr && atomic_long_cmpxchg((atomic_long_t *)&h->vaddr, 0, (long)vaddr)) {
nvmap_kmaps_dec(h);
vunmap(vaddr);
}
return h->vaddr;
}
/* carveout - explicitly map the pfns into a vmalloc area */
adj_size = h->carveout->base & ~PAGE_MASK;
adj_size += h->size;
adj_size = PAGE_ALIGN(adj_size);
if (pfn_valid(__phys_to_pfn(h->carveout->base & PAGE_MASK))) {
unsigned long pfn;
struct page *page;
int nr_pages;
pfn = ((h->carveout->base) >> PAGE_SHIFT);
page = pfn_to_page(pfn);
nr_pages = h->size >> PAGE_SHIFT;
pages = vmalloc(nr_pages * sizeof(*pages));
if (!pages)
goto out;
for (i = 0; i < nr_pages; i++)
pages[i] = nth_page(page, i);
vaddr = vmap(pages, nr_pages, VM_MAP, prot);
} else {
vaddr = (__force void *)__ioremap(h->carveout->base, adj_size,
prot);
}
if (vaddr == NULL)
goto out;
if (vaddr && atomic_long_cmpxchg((atomic_long_t *)&h->vaddr,
0, (long)vaddr)) {
vaddr -= (h->carveout->base & ~PAGE_MASK);
/*
* iounmap calls vunmap for vmalloced address, hence
* takes care of vmap/__ioremap freeing part.
*/
iounmap((void __iomem *)vaddr);
nvmap_kmaps_dec(h);
}
if (pages)
vfree(pages);
/* leave the handle ref count incremented by 1, so that
* the handle will not be freed while the kernel mapping exists.
* nvmap_handle_put will be called by unmapping this address */
return h->vaddr;
out:
if (pages)
vfree(pages);
nvmap_kmaps_dec(h);
put_handle:
nvmap_handle_put(h);
return NULL;
}
void __nvmap_munmap(struct nvmap_handle *h, void *addr)
{
if (!h || !h->alloc ||
WARN_ON(!virt_addr_valid(h)) ||
WARN_ON(!addr) ||
!(h->heap_type & nvmap_dev->cpu_access_mask))
return;
nvmap_handle_put(h);
}
/*
* NOTE: this does not ensure the continued existence of the underlying
* dma_buf. If you want ensure the existence of the dma_buf you must get an
* nvmap_handle_ref as that is what tracks the dma_buf refs.
*/
struct nvmap_handle *nvmap_handle_get(struct nvmap_handle *h)
{
int cnt;
if (WARN_ON(!virt_addr_valid(h))) {
pr_err("%s: invalid handle\n", current->group_leader->comm);
return NULL;
}
cnt = atomic_inc_return(&h->ref);
NVMAP_TAG_TRACE(trace_nvmap_handle_get, h, cnt);
if (unlikely(cnt <= 1)) {
pr_err("%s: %s attempt to get a freed handle\n",
__func__, current->group_leader->comm);
atomic_dec(&h->ref);
return NULL;
}
return h;
}
void nvmap_handle_put(struct nvmap_handle *h)
{
int cnt;
if (WARN_ON(!virt_addr_valid(h)))
return;
cnt = atomic_dec_return(&h->ref);
NVMAP_TAG_TRACE(trace_nvmap_handle_put, h, cnt);
if (WARN_ON(cnt < 0)) {
pr_err("%s: %s put to negative references\n",
__func__, current->comm);
} else if (cnt == 0)
_nvmap_handle_free(h);
}
struct sg_table *__nvmap_sg_table(struct nvmap_client *client,
struct nvmap_handle *h)
{
struct sg_table *sgt = NULL;
int err, npages;
struct page **pages;
if (!virt_addr_valid(h))
return ERR_PTR(-EINVAL);
h = nvmap_handle_get(h);
if (!h)
return ERR_PTR(-EINVAL);
if (!h->alloc) {
err = -EINVAL;
goto put_handle;
}
npages = PAGE_ALIGN(h->size) >> PAGE_SHIFT;
sgt = kzalloc(sizeof(*sgt), GFP_KERNEL);
if (!sgt) {
err = -ENOMEM;
goto err;
}
if (!h->heap_pgalloc) {
phys_addr_t paddr = handle_phys(h);
struct page *page = phys_to_page(paddr);
err = sg_alloc_table(sgt, 1, GFP_KERNEL);
if (err)
goto err;
sg_set_page(sgt->sgl, page, h->size, offset_in_page(paddr));
} else {
pages = nvmap_pages(h->pgalloc.pages, npages);
if (!pages) {
err = -ENOMEM;
goto err;
}
err = sg_alloc_table_from_pages(sgt, pages,
npages, 0, h->size, GFP_KERNEL);
nvmap_altfree(pages, npages * sizeof(*pages));
if (err)
goto err;
}
nvmap_handle_put(h);
return sgt;
err:
kfree(sgt);
put_handle:
nvmap_handle_put(h);
return ERR_PTR(err);
}
void __nvmap_free_sg_table(struct nvmap_client *client,
struct nvmap_handle *h, struct sg_table *sgt)
{
sg_free_table(sgt);
kfree(sgt);
}

1557
drivers/video/tegra/nvmap/nvmap_dev.c Normal file
View File

File diff suppressed because it is too large Load Diff

666
drivers/video/tegra/nvmap/nvmap_dmabuf.c Normal file
View File

@@ -0,0 +1,666 @@
/*
* dma_buf exporter for nvmap
*
* Copyright (c) 2012-2022, NVIDIA CORPORATION. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*/
#define pr_fmt(fmt) "nvmap: %s() " fmt, __func__
#include <linux/list.h>
#include <linux/slab.h>
#include <linux/uaccess.h>
#include <linux/export.h>
#include <linux/nvmap.h>
#include <linux/dma-buf.h>
#include <linux/spinlock.h>
#include <linux/mutex.h>
#include <linux/atomic.h>
#include <linux/debugfs.h>
#include <linux/seq_file.h>
#include <linux/stringify.h>
#include <linux/of.h>
#include <linux/version.h>
#include <linux/iommu.h>
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 18, 0)
#include <linux/iosys-map.h>
#endif
#include <trace/events/nvmap.h>
#include "nvmap_priv.h"
#include "nvmap_ioctl.h"
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0)
#define NVMAP_DMABUF_ATTACH nvmap_dmabuf_attach
#else
#define NVMAP_DMABUF_ATTACH __nvmap_dmabuf_attach
#endif
static int __nvmap_dmabuf_attach(struct dma_buf *dmabuf, struct device *dev,
struct dma_buf_attachment *attach)
{
struct nvmap_handle_info *info = dmabuf->priv;
trace_nvmap_dmabuf_attach(dmabuf, dev);
dev_dbg(dev, "%s() 0x%p\n", __func__, info->handle);
return 0;
}
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0)
static int nvmap_dmabuf_attach(struct dma_buf *dmabuf,
struct dma_buf_attachment *attach)
{
return __nvmap_dmabuf_attach(dmabuf, attach->dev, attach);
}
#endif
static void nvmap_dmabuf_detach(struct dma_buf *dmabuf,
struct dma_buf_attachment *attach)
{
struct nvmap_handle_info *info = dmabuf->priv;
trace_nvmap_dmabuf_detach(dmabuf, attach->dev);
dev_dbg(attach->dev, "%s() 0x%p\n", __func__, info->handle);
}
static inline bool access_vpr_phys(struct device *dev)
{
if (!iommu_get_domain_for_dev(dev))
return true;
/*
* Assumes gpu nodes always have DT entry, this is valid as device
* specifying access-vpr-phys will do so through its DT entry.
*/
if (!dev->of_node)
return false;
return !!of_find_property(dev->of_node, "access-vpr-phys", NULL);
}
static struct sg_table *nvmap_dmabuf_map_dma_buf(struct dma_buf_attachment *attach,
enum dma_data_direction dir)
{
struct nvmap_handle_info *info = attach->dmabuf->priv;
int ents = 0;
struct sg_table *sgt;
#ifdef NVMAP_CONFIG_DEBUG_MAPS
char *device_name = NULL;
u32 heap_type;
u64 dma_mask;
#endif /* NVMAP_CONFIG_DEBUG_MAPS */
DEFINE_DMA_ATTRS(attrs);
trace_nvmap_dmabuf_map_dma_buf(attach->dmabuf, attach->dev);
/*
* If the exported buffer is foreign buffer(alloc_from_va) and
* has RO access, don't map it in device space.
* Return error as no access.
*/
if (info->handle->from_va && info->handle->is_ro &&
(dir != DMA_TO_DEVICE))
return ERR_PTR(-EACCES);
nvmap_lru_reset(info->handle);
mutex_lock(&info->maps_lock);
atomic_inc(&info->handle->pin);
sgt = __nvmap_sg_table(NULL, info->handle);
if (IS_ERR(sgt)) {
atomic_dec(&info->handle->pin);
mutex_unlock(&info->maps_lock);
return sgt;
}
if (!info->handle->alloc) {
goto err_map;
} else if (!(nvmap_dev->dynamic_dma_map_mask &
info->handle->heap_type)) {
sg_dma_address(sgt->sgl) = info->handle->carveout->base;
} else if (info->handle->heap_type == NVMAP_HEAP_CARVEOUT_VPR &&
access_vpr_phys(attach->dev)) {
sg_dma_address(sgt->sgl) = 0;
} else {
dma_set_attr(DMA_ATTR_SKIP_CPU_SYNC, __DMA_ATTR(attrs));
ents = dma_map_sg_attrs(attach->dev, sgt->sgl,
sgt->nents, dir, __DMA_ATTR(attrs));
if (ents <= 0)
goto err_map;
}
attach->priv = sgt;
#ifdef NVMAP_CONFIG_DEBUG_MAPS
/* Insert device name into the carveout's device name rb tree */
heap_type = info->handle->heap_type;
device_name = (char *)dev_name(attach->dev);
dma_mask = *(attach->dev->dma_mask);
if (device_name && !nvmap_is_device_present(device_name, heap_type)) {
/* If the device name is not already present in the tree, then only add */
nvmap_add_device_name(device_name, dma_mask, heap_type);
}
#endif /* NVMAP_CONFIG_DEBUG_MAPS */
mutex_unlock(&info->maps_lock);
return sgt;
err_map:
__nvmap_free_sg_table(NULL, info->handle, sgt);
atomic_dec(&info->handle->pin);
mutex_unlock(&info->maps_lock);
return ERR_PTR(-ENOMEM);
}
static void nvmap_dmabuf_unmap_dma_buf(struct dma_buf_attachment *attach,
struct sg_table *sgt,
enum dma_data_direction dir)
{
struct nvmap_handle_info *info = attach->dmabuf->priv;
#ifdef NVMAP_CONFIG_DEBUG_MAPS
char *device_name = NULL;
u32 heap_type = 0;
#endif /* NVMAP_CONFIG_DEBUG_MAPS */
trace_nvmap_dmabuf_unmap_dma_buf(attach->dmabuf, attach->dev);
mutex_lock(&info->maps_lock);
if (!atomic_add_unless(&info->handle->pin, -1, 0)) {
mutex_unlock(&info->maps_lock);
WARN(1, "Unpinning handle that has yet to be pinned!\n");
return;
}
if (!(nvmap_dev->dynamic_dma_map_mask & info->handle->heap_type)) {
sg_dma_address(sgt->sgl) = 0;
} else if (info->handle->heap_type == NVMAP_HEAP_CARVEOUT_VPR &&
access_vpr_phys(attach->dev)) {
sg_dma_address(sgt->sgl) = 0;
} else {
dma_unmap_sg_attrs(attach->dev,
sgt->sgl, sgt->nents,
dir, DMA_ATTR_SKIP_CPU_SYNC);
}
__nvmap_free_sg_table(NULL, info->handle, sgt);
#ifdef NVMAP_CONFIG_DEBUG_MAPS
/* Remove the device name from the list of carveout accessing devices */
heap_type = info->handle->heap_type;
device_name = (char *)dev_name(attach->dev);
if (device_name)
nvmap_remove_device_name(device_name, heap_type);
#endif /* NVMAP_CONFIG_DEBUG_MAPS */
mutex_unlock(&info->maps_lock);
}
static void nvmap_dmabuf_release(struct dma_buf *dmabuf)
{
struct nvmap_handle_info *info = dmabuf->priv;
trace_nvmap_dmabuf_release(info->handle->owner ?
info->handle->owner->name : "unknown",
info->handle,
dmabuf);
mutex_lock(&info->handle->lock);
if (info->is_ro) {
BUG_ON(dmabuf != info->handle->dmabuf_ro);
info->handle->dmabuf_ro = NULL;
} else {
BUG_ON(dmabuf != info->handle->dmabuf);
info->handle->dmabuf = NULL;
}
mutex_unlock(&info->handle->lock);
nvmap_handle_put(info->handle);
kfree(info);
}
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0)
static int __nvmap_dmabuf_end_cpu_access(struct dma_buf *dmabuf,
enum dma_data_direction dir)
{
struct nvmap_handle_info *info = dmabuf->priv;
trace_nvmap_dmabuf_end_cpu_access(dmabuf, 0, dmabuf->size);
return __nvmap_do_cache_maint(NULL, info->handle,
0, dmabuf->size,
NVMAP_CACHE_OP_WB, false);
}
static int __nvmap_dmabuf_begin_cpu_access(struct dma_buf *dmabuf,
enum dma_data_direction dir)
{
struct nvmap_handle_info *info = dmabuf->priv;
trace_nvmap_dmabuf_begin_cpu_access(dmabuf, 0, dmabuf->size);
return __nvmap_do_cache_maint(NULL, info->handle, 0, dmabuf->size,
NVMAP_CACHE_OP_WB_INV, false);
}
#define NVMAP_DMABUF_BEGIN_CPU_ACCESS __nvmap_dmabuf_begin_cpu_access
#define NVMAP_DMABUF_END_CPU_ACCESS __nvmap_dmabuf_end_cpu_access
#else
static int nvmap_dmabuf_begin_cpu_access(struct dma_buf *dmabuf,
size_t start, size_t len,
enum dma_data_direction dir)
{
struct nvmap_handle_info *info = dmabuf->priv;
trace_nvmap_dmabuf_begin_cpu_access(dmabuf, start, len);
return __nvmap_do_cache_maint(NULL, info->handle, start, start + len,
NVMAP_CACHE_OP_WB_INV, false);
}
static void nvmap_dmabuf_end_cpu_access(struct dma_buf *dmabuf,
size_t start, size_t len,
enum dma_data_direction dir)
{
struct nvmap_handle_info *info = dmabuf->priv;
trace_nvmap_dmabuf_end_cpu_access(dmabuf, start, len);
__nvmap_do_cache_maint(NULL, info->handle,
start, start + len,
NVMAP_CACHE_OP_WB, false);
}
#define NVMAP_DMABUF_BEGIN_CPU_ACCESS nvmap_dmabuf_begin_cpu_access
#define NVMAP_DMABUF_END_CPU_ACCESS nvmap_dmabuf_end_cpu_access
#endif
#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0)
static void *nvmap_dmabuf_kmap(struct dma_buf *dmabuf, unsigned long page_num)
{
struct nvmap_handle_info *info = dmabuf->priv;
trace_nvmap_dmabuf_kmap(dmabuf);
return __nvmap_kmap(info->handle, page_num);
}
static void nvmap_dmabuf_kunmap(struct dma_buf *dmabuf,
unsigned long page_num, void *addr)
{
struct nvmap_handle_info *info = dmabuf->priv;
trace_nvmap_dmabuf_kunmap(dmabuf);
__nvmap_kunmap(info->handle, page_num, addr);
}
static void *nvmap_dmabuf_kmap_atomic(struct dma_buf *dmabuf,
unsigned long page_num)
{
WARN(1, "%s() can't be called from atomic\n", __func__);
return NULL;
}
#endif
int __nvmap_map(struct nvmap_handle *h, struct vm_area_struct *vma)
{
struct nvmap_vma_priv *priv;
h = nvmap_handle_get(h);
if (!h)
return -EINVAL;
if (!(h->heap_type & nvmap_dev->cpu_access_mask)) {
nvmap_handle_put(h);
return -EPERM;
}
/*
* If the handle is RO and RW mapping is requested, then
* return error.
*/
if (h->from_va && h->is_ro && (vma->vm_flags & VM_WRITE)) {
nvmap_handle_put(h);
return -EPERM;
}
/*
* Don't allow mmap on VPR memory as it would be mapped
* as device memory. User space shouldn't be accessing
* device memory.
*/
if (h->heap_type == NVMAP_HEAP_CARVEOUT_VPR) {
nvmap_handle_put(h);
return -EPERM;
}
priv = kzalloc(sizeof(*priv), GFP_KERNEL);
if (!priv) {
nvmap_handle_put(h);
return -ENOMEM;
}
priv->handle = h;
vma->vm_flags |= VM_SHARED | VM_DONTEXPAND |
VM_DONTDUMP | VM_DONTCOPY |
(h->heap_pgalloc ? 0 : VM_PFNMAP);
vma->vm_ops = &nvmap_vma_ops;
BUG_ON(vma->vm_private_data != NULL);
vma->vm_private_data = priv;
vma->vm_page_prot = nvmap_pgprot(h, vma->vm_page_prot);
nvmap_vma_open(vma);
return 0;
}
static int nvmap_dmabuf_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma)
{
struct nvmap_handle_info *info = dmabuf->priv;
trace_nvmap_dmabuf_mmap(dmabuf);
return __nvmap_map(info->handle, vma);
}
#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0)
static void *nvmap_dmabuf_vmap(struct dma_buf *dmabuf)
{
struct nvmap_handle_info *info = dmabuf->priv;
trace_nvmap_dmabuf_vmap(dmabuf);
/* Don't allow vmap on RO buffers */
if (info->is_ro)
return ERR_PTR(-EPERM);
return __nvmap_mmap(info->handle);
}
static void nvmap_dmabuf_vunmap(struct dma_buf *dmabuf, void *vaddr)
{
struct nvmap_handle_info *info = dmabuf->priv;
trace_nvmap_dmabuf_vunmap(dmabuf);
__nvmap_munmap(info->handle, vaddr);
}
#else
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 18, 0)
static int nvmap_dmabuf_vmap(struct dma_buf *dmabuf, struct iosys_map *map)
#else
static int nvmap_dmabuf_vmap(struct dma_buf *dmabuf, struct dma_buf_map *map)
#endif
{
struct nvmap_handle_info *info = dmabuf->priv;
void *res;
int ret = 0;
trace_nvmap_dmabuf_vmap(dmabuf);
/* Don't allow vmap on RO buffers */
if (info->is_ro)
return -EPERM;
res = __nvmap_mmap(info->handle);
if (res != NULL) {
map->vaddr = res;
map->is_iomem = false;
}
else {
ret = -ENOMEM;
}
return ret;
}
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 18, 0)
static void nvmap_dmabuf_vunmap(struct dma_buf *dmabuf, struct iosys_map *map)
#else
static void nvmap_dmabuf_vunmap(struct dma_buf *dmabuf, struct dma_buf_map *map)
#endif
{
struct nvmap_handle_info *info = dmabuf->priv;
trace_nvmap_dmabuf_vunmap(dmabuf);
__nvmap_munmap(info->handle, info->handle->vaddr);
}
#endif
static struct dma_buf_ops nvmap_dma_buf_ops = {
.attach = NVMAP_DMABUF_ATTACH,
.detach = nvmap_dmabuf_detach,
.map_dma_buf = nvmap_dmabuf_map_dma_buf,
.unmap_dma_buf = nvmap_dmabuf_unmap_dma_buf,
.release = nvmap_dmabuf_release,
.begin_cpu_access = NVMAP_DMABUF_BEGIN_CPU_ACCESS,
.end_cpu_access = NVMAP_DMABUF_END_CPU_ACCESS,
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0)
.kmap_atomic = nvmap_dmabuf_kmap_atomic,
.kmap = nvmap_dmabuf_kmap,
.kunmap = nvmap_dmabuf_kunmap,
#elif LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0)
.map_atomic = nvmap_dmabuf_kmap_atomic,
.map = nvmap_dmabuf_kmap,
.unmap = nvmap_dmabuf_kunmap,
#endif
.mmap = nvmap_dmabuf_mmap,
.vmap = nvmap_dmabuf_vmap,
.vunmap = nvmap_dmabuf_vunmap,
#if LINUX_VERSION_CODE > KERNEL_VERSION(5, 4, 0)
.cache_sgt_mapping = true,
#endif
};
static char dmabuf_name[] = "nvmap_dmabuf";
bool dmabuf_is_nvmap(struct dma_buf *dmabuf)
{
return dmabuf->exp_name == dmabuf_name;
}
static struct dma_buf *__dma_buf_export(struct nvmap_handle_info *info,
size_t size, bool ro_buf)
{
DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
exp_info.priv = info;
exp_info.ops = &nvmap_dma_buf_ops;
exp_info.size = size;
if (ro_buf) {
exp_info.flags = O_RDONLY;
} else {
exp_info.flags = O_RDWR;
}
#ifndef NVMAP_UPSTREAM_KERNEL
/* Disable defer unmap feature only for kstable */
exp_info.exp_flags = DMABUF_CAN_DEFER_UNMAP |
DMABUF_SKIP_CACHE_SYNC;
#endif /* !NVMAP_UPSTREAM_KERNEL */
exp_info.exp_name = dmabuf_name;
return dma_buf_export(&exp_info);
}
/*
* Make a dmabuf object for an nvmap handle.
*/
struct dma_buf *__nvmap_make_dmabuf(struct nvmap_client *client,
struct nvmap_handle *handle, bool ro_buf)
{
int err;
struct dma_buf *dmabuf;
struct nvmap_handle_info *info;
info = kzalloc(sizeof(*info), GFP_KERNEL);
if (!info) {
err = -ENOMEM;
goto err_nomem;
}
info->handle = handle;
info->is_ro = ro_buf;
INIT_LIST_HEAD(&info->maps);
mutex_init(&info->maps_lock);
dmabuf = __dma_buf_export(info, handle->size, ro_buf);
if (IS_ERR(dmabuf)) {
err = PTR_ERR(dmabuf);
goto err_export;
}
nvmap_handle_get(handle);
trace_nvmap_make_dmabuf(client->name, handle, dmabuf);
return dmabuf;
err_export:
kfree(info);
err_nomem:
return ERR_PTR(err);
}
int __nvmap_dmabuf_fd(struct nvmap_client *client,
struct dma_buf *dmabuf, int flags)
{
#if !defined(NVMAP_CONFIG_HANDLE_AS_ID) && !defined(NVMAP_LOADABLE_MODULE)
int start_fd = NVMAP_CONFIG_FD_START;
#endif
int ret;
#ifdef NVMAP_CONFIG_DEFER_FD_RECYCLE
if (client->next_fd < NVMAP_CONFIG_FD_START)
client->next_fd = NVMAP_CONFIG_FD_START;
start_fd = client->next_fd++;
if (client->next_fd >= NVMAP_CONFIG_DEFER_FD_RECYCLE_MAX_FD)
client->next_fd = NVMAP_CONFIG_FD_START;
#endif
if (!dmabuf || !dmabuf->file)
return -EINVAL;
/* Allocate fd from start_fd(>=1024) onwards to overcome
* __FD_SETSIZE limitation issue for select(),
* pselect() syscalls.
*/
#if defined(NVMAP_LOADABLE_MODULE) || defined(NVMAP_CONFIG_HANDLE_AS_ID)
ret = get_unused_fd_flags(flags);
#else
ret = __alloc_fd(current->files, start_fd, sysctl_nr_open, flags);
#endif
if (ret == -EMFILE)
pr_err_ratelimited("NvMap: FD limit is crossed for uid %d\n",
from_kuid(current_user_ns(), current_uid()));
return ret;
}
static struct dma_buf *__nvmap_dmabuf_export(struct nvmap_client *client,
struct nvmap_handle *handle, bool is_ro)
{
struct dma_buf *buf;
handle = nvmap_handle_get(handle);
if (!handle)
return ERR_PTR(-EINVAL);
if (is_ro)
buf = handle->dmabuf_ro;
else
buf = handle->dmabuf;
if (WARN(!buf, "Attempting to get a freed dma_buf!\n")) {
nvmap_handle_put(handle);
return NULL;
}
get_dma_buf(buf);
/*
* Don't want to take out refs on the handle here.
*/
nvmap_handle_put(handle);
return buf;
}
int nvmap_get_dmabuf_fd(struct nvmap_client *client, struct nvmap_handle *h,
bool is_ro)
{
int fd;
struct dma_buf *dmabuf;
dmabuf = __nvmap_dmabuf_export(client, h, is_ro);
if (IS_ERR(dmabuf))
return PTR_ERR(dmabuf);
fd = __nvmap_dmabuf_fd(client, dmabuf, O_CLOEXEC);
if (fd < 0)
dma_buf_put(dmabuf);
return fd;
}
/*
* Returns the nvmap handle ID associated with the passed dma_buf's fd. This
* does not affect the ref count of the dma_buf.
* NOTE: Callers of this utility function must invoke nvmap_handle_put after
* using the returned nvmap_handle. Call to nvmap_handle_get is required in
* this utility function to avoid race conditions in code where nvmap_handle
* returned by this function is freed concurrently while the caller is still
* using it.
*/
struct nvmap_handle *nvmap_handle_get_from_dmabuf_fd(
struct nvmap_client *client, int fd)
{
struct nvmap_handle *handle = ERR_PTR(-EINVAL);
struct dma_buf *dmabuf;
struct nvmap_handle_info *info;
dmabuf = dma_buf_get(fd);
if (IS_ERR(dmabuf))
return ERR_CAST(dmabuf);
if (dmabuf_is_nvmap(dmabuf)) {
info = dmabuf->priv;
handle = info->handle;
if (!nvmap_handle_get(handle))
handle = ERR_PTR(-EINVAL);
}
dma_buf_put(dmabuf);
return handle;
}
bool is_nvmap_dmabuf_fd_ro(int fd)
{
struct dma_buf *dmabuf;
struct nvmap_handle_info *info = NULL;
dmabuf = dma_buf_get(fd);
if (IS_ERR(dmabuf)) {
return false;
}
if (dmabuf_is_nvmap(dmabuf)) {
info = dmabuf->priv;
}
dma_buf_put(dmabuf);
return (info != NULL) ? info->is_ro : false;
}
/*
* Duplicates a generic dma_buf fd. nvmap dma_buf fd has to be duplicated
* using existing code paths to preserve memory accounting behavior, so this
* function returns -EINVAL on dma_buf fds created by nvmap.
*/
int nvmap_dmabuf_duplicate_gen_fd(struct nvmap_client *client,
struct dma_buf *dmabuf)
{
int ret = 0;
if (dmabuf_is_nvmap(dmabuf)) {
ret = -EINVAL;
goto error;
}
ret = __nvmap_dmabuf_fd(client, dmabuf, O_CLOEXEC);
if (ret < 0)
goto error;
return ret;
error:
dma_buf_put(dmabuf);
return ret;
}

261
drivers/video/tegra/nvmap/nvmap_fault.c Normal file
View File

@@ -0,0 +1,261 @@
/*
* drivers/video/tegra/nvmap/nvmap_fault.c
*
* Copyright (c) 2011-2022, NVIDIA CORPORATION. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*/
#define pr_fmt(fmt) "%s: " fmt, __func__
#include <trace/events/nvmap.h>
#include <linux/highmem.h>
#include "nvmap_priv.h"
static void nvmap_vma_close(struct vm_area_struct *vma);
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0)
#define __atomic_add_unless atomic_fetch_add_unless
static vm_fault_t nvmap_vma_fault(struct vm_fault *vmf);
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)
static int nvmap_vma_fault(struct vm_fault *vmf);
#else
static int nvmap_vma_fault(struct vm_area_struct *vma, struct vm_fault *vmf);
#endif
struct vm_operations_struct nvmap_vma_ops = {
.open = nvmap_vma_open,
.close = nvmap_vma_close,
.fault = nvmap_vma_fault,
};
int is_nvmap_vma(struct vm_area_struct *vma)
{
return vma->vm_ops == &nvmap_vma_ops;
}
/* to ensure that the backing store for the VMA isn't freed while a fork'd
* reference still exists, nvmap_vma_open increments the reference count on
* the handle, and nvmap_vma_close decrements it. alternatively, we could
* disallow copying of the vma, or behave like pmem and zap the pages. FIXME.
*/
void nvmap_vma_open(struct vm_area_struct *vma)
{
struct nvmap_vma_priv *priv;
struct nvmap_handle *h;
struct nvmap_vma_list *vma_list, *tmp;
struct list_head *tmp_head = NULL;
pid_t current_pid = task_tgid_nr(current);
bool vma_pos_found = false;
size_t nr_page, i;
int vma_open_count;
priv = vma->vm_private_data;
BUG_ON(!priv);
BUG_ON(!priv->handle);
h = priv->handle;
nvmap_umaps_inc(h);
mutex_lock(&h->lock);
vma_open_count = atomic_inc_return(&priv->count);
if (vma_open_count == 1 && h->heap_pgalloc) {
nr_page = h->size >> PAGE_SHIFT;
for (i = 0; i < nr_page; i++) {
struct page *page = nvmap_to_page(h->pgalloc.pages[i]);
/* This is necessry to avoid page being accounted
* under NR_FILE_MAPPED. This way NR_FILE_MAPPED would
* be fully accounted under NR_FILE_PAGES. This allows
* Android low mem killer detect low memory condition
* precisely.
* This has a side effect of inaccurate pss accounting
* for NvMap memory mapped into user space. Android
* procrank and NvMap Procrank both would have same
* issue. Subtracting NvMap_Procrank pss from
* procrank pss would give non-NvMap pss held by process
* and adding NvMap memory used by process represents
* entire memroy consumption by the process.
*/
atomic_inc(&page->_mapcount);
}
}
mutex_unlock(&h->lock);
vma_list = kmalloc(sizeof(*vma_list), GFP_KERNEL);
if (vma_list) {
mutex_lock(&h->lock);
tmp_head = &h->vmas;
/* insert vma into handle's vmas list in the increasing order of
* handle offsets
*/
list_for_each_entry(tmp, &h->vmas, list) {
/* if vma exists in list, just increment refcount */
if (tmp->vma == vma) {
atomic_inc(&tmp->ref);
kfree(vma_list);
goto unlock;
}
if (!vma_pos_found && (current_pid == tmp->pid)) {
if (vma->vm_pgoff < tmp->vma->vm_pgoff) {
tmp_head = &tmp->list;
vma_pos_found = true;
} else {
tmp_head = tmp->list.next;
}
}
}
vma_list->vma = vma;
vma_list->pid = current_pid;
vma_list->save_vm_flags = vma->vm_flags;
atomic_set(&vma_list->ref, 1);
list_add_tail(&vma_list->list, tmp_head);
unlock:
mutex_unlock(&h->lock);
} else {
WARN(1, "vma not tracked");
}
}
static void nvmap_vma_close(struct vm_area_struct *vma)
{
struct nvmap_vma_priv *priv = vma->vm_private_data;
struct nvmap_vma_list *vma_list;
struct nvmap_handle *h;
bool vma_found = false;
size_t nr_page, i;
if (!priv)
return;
BUG_ON(!priv->handle);
h = priv->handle;
nr_page = h->size >> PAGE_SHIFT;
mutex_lock(&h->lock);
list_for_each_entry(vma_list, &h->vmas, list) {
if (vma_list->vma != vma)
continue;
if (atomic_dec_return(&vma_list->ref) == 0) {
list_del(&vma_list->list);
kfree(vma_list);
}
vma_found = true;
break;
}
BUG_ON(!vma_found);
nvmap_umaps_dec(h);
if (__atomic_add_unless(&priv->count, -1, 0) == 1) {
if (h->heap_pgalloc) {
for (i = 0; i < nr_page; i++) {
struct page *page;
page = nvmap_to_page(h->pgalloc.pages[i]);
atomic_dec(&page->_mapcount);
}
}
mutex_unlock(&h->lock);
if (priv->handle)
nvmap_handle_put(priv->handle);
vma->vm_private_data = NULL;
kfree(priv);
} else {
mutex_unlock(&h->lock);
}
}
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0)
static vm_fault_t nvmap_vma_fault(struct vm_fault *vmf)
#define vm_insert_pfn vmf_insert_pfn
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)
static int nvmap_vma_fault(struct vm_fault *vmf)
#else
static int nvmap_vma_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
#endif
{
struct page *page;
struct nvmap_vma_priv *priv;
unsigned long offs;
#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)
struct vm_area_struct *vma = vmf->vma;
unsigned long vmf_address = vmf->address;
#else
void __user *vmf_address = vmf->virtual_address;
#endif
offs = (unsigned long)(vmf_address - vma->vm_start);
priv = vma->vm_private_data;
if (!priv || !priv->handle || !priv->handle->alloc)
return VM_FAULT_SIGBUS;
offs += priv->offs;
/* if the VMA was split for some reason, vm_pgoff will be the VMA's
* offset from the original VMA */
offs += (vma->vm_pgoff << PAGE_SHIFT);
if (offs >= priv->handle->size)
return VM_FAULT_SIGBUS;
if (!priv->handle->heap_pgalloc) {
unsigned long pfn;
BUG_ON(priv->handle->carveout->base & ~PAGE_MASK);
pfn = ((priv->handle->carveout->base + offs) >> PAGE_SHIFT);
if (!pfn_valid(pfn)) {
vm_insert_pfn(vma,
(unsigned long)vmf_address, pfn);
return VM_FAULT_NOPAGE;
}
/* CMA memory would get here */
page = pfn_to_page(pfn);
} else {
void *kaddr;
offs >>= PAGE_SHIFT;
if (atomic_read(&priv->handle->pgalloc.reserved))
return VM_FAULT_SIGBUS;
page = nvmap_to_page(priv->handle->pgalloc.pages[offs]);
if (PageAnon(page) && (vma->vm_flags & VM_SHARED))
return VM_FAULT_SIGSEGV;
if (!nvmap_handle_track_dirty(priv->handle))
goto finish;
mutex_lock(&priv->handle->lock);
if (nvmap_page_dirty(priv->handle->pgalloc.pages[offs])) {
mutex_unlock(&priv->handle->lock);
goto finish;
}
/* inner cache maint */
kaddr = kmap(page);
BUG_ON(!kaddr);
inner_cache_maint(NVMAP_CACHE_OP_WB_INV, kaddr, PAGE_SIZE);
kunmap(page);
if (priv->handle->flags & NVMAP_HANDLE_INNER_CACHEABLE)
goto make_dirty;
make_dirty:
nvmap_page_mkdirty(&priv->handle->pgalloc.pages[offs]);
atomic_inc(&priv->handle->pgalloc.ndirty);
mutex_unlock(&priv->handle->lock);
}
finish:
if (page)
get_page(page);
vmf->page = page;
return (page) ? 0 : VM_FAULT_SIGBUS;
}

511
drivers/video/tegra/nvmap/nvmap_handle.c Normal file
View File

@@ -0,0 +1,511 @@
/*
* drivers/video/tegra/nvmap/nvmap_handle.c
*
* Handle allocation and freeing routines for nvmap
*
* Copyright (c) 2009-2022, NVIDIA CORPORATION. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*/
#define pr_fmt(fmt) "%s: " fmt, __func__
#include <linux/err.h>
#include <linux/io.h>
#include <linux/kernel.h>
#include <linux/list.h>
#include <linux/mm.h>
#include <linux/rbtree.h>
#include <linux/dma-buf.h>
#include <linux/moduleparam.h>
#include <linux/nvmap.h>
#include <linux/version.h>
#if KERNEL_VERSION(4, 15, 0) > LINUX_VERSION_CODE
#include <soc/tegra/chip-id.h>
#else
#include <soc/tegra/fuse.h>
#endif
#include <asm/pgtable.h>
#include <trace/events/nvmap.h>
#include "nvmap_priv.h"
#include "nvmap_ioctl.h"
/*
* Verifies that the passed ID is a valid handle ID. Then the passed client's
* reference to the handle is returned.
*
* Note: to call this function make sure you own the client ref lock.
*/
struct nvmap_handle_ref *__nvmap_validate_locked(struct nvmap_client *c,
struct nvmap_handle *h,
bool is_ro)
{
struct rb_node *n = c->handle_refs.rb_node;
while (n) {
struct nvmap_handle_ref *ref;
ref = rb_entry(n, struct nvmap_handle_ref, node);
if ((ref->handle == h) && (ref->is_ro == is_ro))
return ref;
else if ((uintptr_t)h > (uintptr_t)ref->handle)
n = n->rb_right;
else
n = n->rb_left;
}
return NULL;
}
/* adds a newly-created handle to the device master tree */
void nvmap_handle_add(struct nvmap_device *dev, struct nvmap_handle *h)
{
struct rb_node **p;
struct rb_node *parent = NULL;
spin_lock(&dev->handle_lock);
p = &dev->handles.rb_node;
while (*p) {
struct nvmap_handle *b;
parent = *p;
b = rb_entry(parent, struct nvmap_handle, node);
if (h > b)
p = &parent->rb_right;
else
p = &parent->rb_left;
}
rb_link_node(&h->node, parent, p);
rb_insert_color(&h->node, &dev->handles);
nvmap_lru_add(h);
spin_unlock(&dev->handle_lock);
}
/* remove a handle from the device's tree of all handles; called
* when freeing handles. */
int nvmap_handle_remove(struct nvmap_device *dev, struct nvmap_handle *h)
{
spin_lock(&dev->handle_lock);
/* re-test inside the spinlock if the handle really has no clients;
* only remove the handle if it is unreferenced */
if (atomic_add_return(0, &h->ref) > 0) {
spin_unlock(&dev->handle_lock);
return -EBUSY;
}
smp_rmb();
BUG_ON(atomic_read(&h->ref) < 0);
BUG_ON(atomic_read(&h->pin) != 0);
nvmap_lru_del(h);
rb_erase(&h->node, &dev->handles);
spin_unlock(&dev->handle_lock);
return 0;
}
/* Validates that a handle is in the device master tree and that the
* client has permission to access it. */
struct nvmap_handle *nvmap_validate_get(struct nvmap_handle *id)
{
struct nvmap_handle *h = NULL;
struct rb_node *n;
spin_lock(&nvmap_dev->handle_lock);
n = nvmap_dev->handles.rb_node;
while (n) {
h = rb_entry(n, struct nvmap_handle, node);
if (h == id) {
h = nvmap_handle_get(h);
spin_unlock(&nvmap_dev->handle_lock);
return h;
}
if (id > h)
n = n->rb_right;
else
n = n->rb_left;
}
spin_unlock(&nvmap_dev->handle_lock);
return NULL;
}
static void add_handle_ref(struct nvmap_client *client,
struct nvmap_handle_ref *ref)
{
struct rb_node **p, *parent = NULL;
nvmap_ref_lock(client);
p = &client->handle_refs.rb_node;
while (*p) {
struct nvmap_handle_ref *node;
parent = *p;
node = rb_entry(parent, struct nvmap_handle_ref, node);
if (ref->handle > node->handle)
p = &parent->rb_right;
else
p = &parent->rb_left;
}
rb_link_node(&ref->node, parent, p);
rb_insert_color(&ref->node, &client->handle_refs);
client->handle_count++;
if (client->handle_count > nvmap_max_handle_count)
nvmap_max_handle_count = client->handle_count;
atomic_inc(&ref->handle->share_count);
nvmap_ref_unlock(client);
}
struct nvmap_handle_ref *nvmap_create_handle_from_va(struct nvmap_client *client,
ulong vaddr, size_t size,
u32 flags)
{
struct vm_area_struct *vma;
struct nvmap_handle_ref *ref;
vm_flags_t vm_flags;
struct mm_struct *mm = current->mm;
/* don't allow non-page aligned addresses. */
if (vaddr & ~PAGE_MASK)
return ERR_PTR(-EINVAL);
nvmap_acquire_mmap_read_lock(mm);
vma = find_vma(mm, vaddr);
if (unlikely(!vma)) {
nvmap_release_mmap_read_lock(mm);
return ERR_PTR(-EINVAL);
}
if (!size)
size = vma->vm_end - vaddr;
/* Don't allow exuberantly large sizes. */
if (!is_nvmap_memory_available(size, NVMAP_HEAP_IOVMM)) {
pr_debug("Cannot allocate %zu bytes.\n", size);
nvmap_release_mmap_read_lock(mm);
return ERR_PTR(-ENOMEM);
}
vm_flags = vma->vm_flags;
nvmap_release_mmap_read_lock(mm);
/*
* If buffer is malloc/mprotect as RO but alloc flag is not passed
* as RO, don't create handle.
*/
if (!(vm_flags & VM_WRITE) && !(flags & NVMAP_HANDLE_RO))
return ERR_PTR(-EINVAL);
ref = nvmap_create_handle(client, size, flags & NVMAP_HANDLE_RO);
if (!IS_ERR(ref))
ref->handle->orig_size = size;
return ref;
}
struct nvmap_handle_ref *nvmap_create_handle(struct nvmap_client *client,
size_t size, bool ro_buf)
{
void *err = ERR_PTR(-ENOMEM);
struct nvmap_handle *h;
struct nvmap_handle_ref *ref = NULL;
struct dma_buf *dmabuf;
if (!client)
return ERR_PTR(-EINVAL);
if (!size)
return ERR_PTR(-EINVAL);
h = kzalloc(sizeof(*h), GFP_KERNEL);
if (!h)
return ERR_PTR(-ENOMEM);
ref = kzalloc(sizeof(*ref), GFP_KERNEL);
if (!ref)
goto ref_alloc_fail;
atomic_set(&h->ref, 1);
atomic_set(&h->pin, 0);
h->owner = client;
BUG_ON(!h->owner);
h->orig_size = size;
h->size = PAGE_ALIGN(size);
h->flags = NVMAP_HANDLE_WRITE_COMBINE;
h->peer = NVMAP_IVM_INVALID_PEER;
mutex_init(&h->lock);
INIT_LIST_HEAD(&h->vmas);
INIT_LIST_HEAD(&h->lru);
INIT_LIST_HEAD(&h->dmabuf_priv);
/*
* This takes out 1 ref on the dambuf. This corresponds to the
* handle_ref that gets automatically made by nvmap_create_handle().
*/
dmabuf = __nvmap_make_dmabuf(client, h, ro_buf);
if (IS_ERR(dmabuf)) {
err = dmabuf;
goto make_dmabuf_fail;
}
if (!ro_buf)
h->dmabuf = dmabuf;
else
h->dmabuf_ro = dmabuf;
nvmap_handle_add(nvmap_dev, h);
/*
* Major assumption here: the dma_buf object that the handle contains
* is created with a ref count of 1.
*/
atomic_set(&ref->dupes, 1);
ref->handle = h;
add_handle_ref(client, ref);
if (ro_buf)
ref->is_ro = true;
else
ref->is_ro = false;
trace_nvmap_create_handle(client, client->name, h, size, ref);
return ref;
make_dmabuf_fail:
kfree(ref);
ref_alloc_fail:
kfree(h);
return err;
}
struct nvmap_handle_ref *nvmap_try_duplicate_by_ivmid(
struct nvmap_client *client, u64 ivm_id,
struct nvmap_heap_block **block)
{
struct nvmap_handle *h = NULL;
struct nvmap_handle_ref *ref = NULL;
struct rb_node *n;
spin_lock(&nvmap_dev->handle_lock);
n = nvmap_dev->handles.rb_node;
for (n = rb_first(&nvmap_dev->handles); n; n = rb_next(n)) {
h = rb_entry(n, struct nvmap_handle, node);
if (h->ivm_id == ivm_id) {
BUG_ON(!virt_addr_valid(h));
/* get handle's ref only if non-zero */
if (atomic_inc_not_zero(&h->ref) == 0) {
*block = h->carveout;
/* strip handle's block and fail duplication */
h->carveout = NULL;
break;
}
spin_unlock(&nvmap_dev->handle_lock);
goto found;
}
}
spin_unlock(&nvmap_dev->handle_lock);
/* handle is either freed or being freed, don't duplicate it */
goto finish;
/*
* From this point, handle and its buffer are valid and won't be
* freed as a reference is taken on it. The dmabuf can still be
* freed anytime till reference is taken on it below.
*/
found:
mutex_lock(&h->lock);
/*
* Save this block. If dmabuf's reference is not held in time,
* this can be reused to avoid the delay to free the buffer
* in this old handle and allocate it for a new handle from
* the ivm allocation ioctl.
*/
*block = h->carveout;
if (!h->dmabuf)
goto fail;
BUG_ON(!h->dmabuf->file);
/* This is same as get_dma_buf() if file->f_count was non-zero */
if (atomic_long_inc_not_zero(&h->dmabuf->file->f_count) == 0)
goto fail;
mutex_unlock(&h->lock);
/* h->dmabuf can't be NULL anymore. Duplicate the handle. */
ref = nvmap_duplicate_handle(client, h, true, false);
/* put the extra ref taken using get_dma_buf. */
dma_buf_put(h->dmabuf);
finish:
return ref;
fail:
/* free handle but not its buffer */
h->carveout = NULL;
mutex_unlock(&h->lock);
nvmap_handle_put(h);
return NULL;
}
struct nvmap_handle_ref *nvmap_duplicate_handle(struct nvmap_client *client,
struct nvmap_handle *h, bool skip_val,
bool is_ro)
{
struct nvmap_handle_ref *ref = NULL;
BUG_ON(!client);
if (!skip_val)
/* on success, the reference count for the handle should be
* incremented, so the success paths will not call
* nvmap_handle_put */
h = nvmap_validate_get(h);
if (!h) {
pr_debug("%s duplicate handle failed\n",
current->group_leader->comm);
return ERR_PTR(-EPERM);
}
if (!h->alloc) {
pr_err("%s duplicating unallocated handle\n",
current->group_leader->comm);
nvmap_handle_put(h);
return ERR_PTR(-EINVAL);
}
nvmap_ref_lock(client);
ref = __nvmap_validate_locked(client, h, is_ro);
if (ref) {
atomic_inc(&ref->dupes);
nvmap_ref_unlock(client);
goto out;
}
nvmap_ref_unlock(client);
ref = kzalloc(sizeof(*ref), GFP_KERNEL);
if (!ref) {
nvmap_handle_put(h);
return ERR_PTR(-ENOMEM);
}
atomic_set(&ref->dupes, 1);
ref->handle = h;
add_handle_ref(client, ref);
if (is_ro) {
ref->is_ro = true;
get_dma_buf(h->dmabuf_ro);
} else {
ref->is_ro = false;
get_dma_buf(h->dmabuf);
}
out:
NVMAP_TAG_TRACE(trace_nvmap_duplicate_handle,
NVMAP_TP_ARGS_CHR(client, h, ref));
return ref;
}
struct nvmap_handle_ref *nvmap_create_handle_from_id(
struct nvmap_client *client, u32 id)
{
struct nvmap_handle *handle;
struct nvmap_handle_ref *ref;
if (WARN_ON(!client))
return ERR_PTR(-EINVAL);
if (is_nvmap_id_ro(client, id))
return nvmap_dup_handle_ro(client, id);
handle = nvmap_handle_get_from_id(client, id);
if (IS_ERR_OR_NULL(handle)) {
/* fd might be dmabuf fd received from parent process.
* Its entry is not made in id_array.
*/
handle = nvmap_handle_get_from_dmabuf_fd(client, id);
if (IS_ERR(handle))
return ERR_CAST(handle);
}
ref = nvmap_duplicate_handle(client, handle, false, false);
nvmap_handle_put(handle);
return ref;
}
struct nvmap_handle_ref *nvmap_create_handle_from_fd(
struct nvmap_client *client, int fd)
{
struct nvmap_handle *handle;
struct nvmap_handle_ref *ref;
bool is_ro;
if (WARN_ON(!client))
return ERR_PTR(-EINVAL);
handle = nvmap_handle_get_from_dmabuf_fd(client, fd);
if (IS_ERR(handle))
return ERR_CAST(handle);
is_ro = is_nvmap_dmabuf_fd_ro(fd);
if (is_ro)
ref = nvmap_duplicate_handle(client, handle, false, true);
else
ref = nvmap_duplicate_handle(client, handle, false, false);
nvmap_handle_put(handle);
return ref;
}
struct nvmap_handle_ref *nvmap_dup_handle_ro(struct nvmap_client *client,
int id)
{
struct nvmap_handle *h;
struct nvmap_handle_ref *ref = NULL;
bool dmabuf_created = false;
if (!client)
return ERR_PTR(-EINVAL);
h = nvmap_handle_get_from_id(client, id);
if (IS_ERR_OR_NULL(h)) {
/* fd might be dmabuf fd received from parent process.
* Its entry is not made in id_array.
*/
h = nvmap_handle_get_from_dmabuf_fd(client, id);
if (IS_ERR(h))
return ERR_CAST(h);
}
if (h->dmabuf_ro == NULL) {
h->dmabuf_ro = __nvmap_make_dmabuf(client, h, true);
if (IS_ERR(h->dmabuf_ro)) {
nvmap_handle_put(h);
return ERR_CAST(h->dmabuf_ro);
}
dmabuf_created = true;
}
ref = nvmap_duplicate_handle(client, h, false, true);
if (!ref) {
nvmap_handle_put(h);
return ref;
}
/*
* When new dmabuf created (only RO dmabuf is getting created in this function)
* it's counter is incremented one extra time in nvmap_duplicate_handle. Hence
* decrement it by one.
*/
if (dmabuf_created)
dma_buf_put(h->dmabuf_ro);
nvmap_handle_put(h);
return ref;
}

607
drivers/video/tegra/nvmap/nvmap_heap.c Normal file
View File

@@ -0,0 +1,607 @@
/*
* drivers/video/tegra/nvmap/nvmap_heap.c
*
* GPU heap allocator.
*
* Copyright (c) 2011-2022, NVIDIA Corporation. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*/
#define pr_fmt(fmt) "%s: " fmt, __func__
#include <linux/debugfs.h>
#include <linux/device.h>
#include <linux/kernel.h>
#include <linux/list.h>
#include <linux/mm.h>
#include <linux/mutex.h>
#include <linux/slab.h>
#include <linux/err.h>
#include <linux/bug.h>
#include <linux/stat.h>
#include <linux/sizes.h>
#include <linux/io.h>
#include <linux/version.h>
#include <linux/limits.h>
#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)
#include <linux/sched/clock.h>
#endif
#include <linux/nvmap.h>
#include <linux/dma-mapping.h>
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0)
#include <linux/dma-map-ops.h>
#else
#include <linux/dma-contiguous.h>
#endif
#include "nvmap_priv.h"
#include "nvmap_heap.h"
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0)
#include "include/linux/nvmap_exports.h"
#endif
/*
* "carveouts" are platform-defined regions of physically contiguous memory
* which are not managed by the OS. A platform may specify multiple carveouts,
* for either small special-purpose memory regions or reserved regions of main
* system memory.
*
* The carveout allocator returns allocations which are physically contiguous.
*/
static struct kmem_cache *heap_block_cache;
struct list_block {
struct nvmap_heap_block block;
struct list_head all_list;
unsigned int mem_prot;
phys_addr_t orig_addr;
size_t size;
size_t align;
struct nvmap_heap *heap;
struct list_head free_list;
};
struct device *dma_dev_from_handle(unsigned long type)
{
int i;
struct nvmap_carveout_node *co_heap;
for (i = 0; i < nvmap_dev->nr_carveouts; i++) {
co_heap = &nvmap_dev->heaps[i];
if (!(co_heap->heap_bit & type))
continue;
return co_heap->carveout->dma_dev;
}
return ERR_PTR(-ENODEV);
}
int nvmap_query_heap_peer(struct nvmap_heap *heap, unsigned int *peer)
{
if (!heap || !heap->is_ivm)
return -EINVAL;
*peer = heap->peer;
return 0;
}
size_t nvmap_query_heap_size(struct nvmap_heap *heap)
{
if (!heap)
return 0;
return heap->len;
}
void nvmap_heap_debugfs_init(struct dentry *heap_root, struct nvmap_heap *heap)
{
if (sizeof(heap->base) == sizeof(u64))
debugfs_create_x64("base", S_IRUGO,
heap_root, (u64 *)&heap->base);
else
debugfs_create_x32("base", S_IRUGO,
heap_root, (u32 *)&heap->base);
if (sizeof(heap->len) == sizeof(u64))
debugfs_create_x64("size", S_IRUGO,
heap_root, (u64 *)&heap->len);
else
debugfs_create_x32("size", S_IRUGO,
heap_root, (u32 *)&heap->len);
if (sizeof(heap->free_size) == sizeof(u64))
debugfs_create_x64("free_size", S_IRUGO,
heap_root, (u64 *)&heap->free_size);
else
debugfs_create_x32("free_size", S_IRUGO,
heap_root, (u32 *)&heap->free_size);
}
static phys_addr_t nvmap_alloc_mem(struct nvmap_heap *h, size_t len,
phys_addr_t *start)
{
#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0)
phys_addr_t pa = DMA_ERROR_CODE;
#else
phys_addr_t pa = DMA_MAPPING_ERROR;
#endif
struct device *dev = h->dma_dev;
if (len > UINT_MAX) {
dev_err(dev, "%s: %d alloc size is out of range\n",
__func__, __LINE__);
return DMA_ERROR_CODE;
}
#ifdef CONFIG_TEGRA_VIRTUALIZATION
if (start && h->is_ivm) {
void *ret;
pa = h->base + (*start);
#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0)
ret = dma_mark_declared_memory_occupied(dev, pa, len,
DMA_ATTR_ALLOC_EXACT_SIZE);
#else
ret = nvmap_dma_mark_declared_memory_occupied(dev, pa, len);
#endif
if (IS_ERR(ret)) {
dev_err(dev, "Failed to reserve (%pa) len(%zu)\n",
&pa, len);
return DMA_ERROR_CODE;
}
} else
#endif
{
#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0)
(void)dma_alloc_attrs(dev, len, &pa,
GFP_KERNEL, DMA_ATTR_ALLOC_EXACT_SIZE);
#else
(void)nvmap_dma_alloc_attrs(dev, len, &pa,
GFP_KERNEL, DMA_ATTR_ALLOC_EXACT_SIZE);
#endif
if (!dma_mapping_error(dev, pa)) {
#ifdef NVMAP_CONFIG_VPR_RESIZE
int ret;
dev_dbg(dev, "Allocated addr (%pa) len(%zu)\n",
&pa, len);
if (!dma_is_coherent_dev(dev) && h->cma_dev) {
ret = nvmap_cache_maint_phys_range(
NVMAP_CACHE_OP_WB, pa, pa + len,
true, true);
if (!ret)
return pa;
dev_err(dev, "cache WB on (%pa, %zu) failed\n",
&pa, len);
}
#endif
dev_dbg(dev, "Allocated addr (%pa) len(%zu)\n",
&pa, len);
}
}
return pa;
}
static void nvmap_free_mem(struct nvmap_heap *h, phys_addr_t base,
size_t len)
{
struct device *dev = h->dma_dev;
dev_dbg(dev, "Free base (%pa) size (%zu)\n", &base, len);
if (len > UINT_MAX) {
dev_err(dev, "%s: %d freeing length out of range\n",
__func__, __LINE__);
return;
}
#ifdef CONFIG_TEGRA_VIRTUALIZATION
if (h->is_ivm && !h->can_alloc) {
#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0)
dma_mark_declared_memory_unoccupied(dev, base, len,
DMA_ATTR_ALLOC_EXACT_SIZE);
#else
nvmap_dma_mark_declared_memory_unoccupied(dev, base, len);
#endif
} else
#endif
{
#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0)
dma_free_attrs(dev, len,
(void *)(uintptr_t)base,
(dma_addr_t)base, DMA_ATTR_ALLOC_EXACT_SIZE);
#else
nvmap_dma_free_attrs(dev, len,
(void *)(uintptr_t)base,
(dma_addr_t)base,
DMA_ATTR_ALLOC_EXACT_SIZE);
#endif
}
}
/*
* base_max limits position of allocated chunk in memory.
* if base_max is 0 then there is no such limitation.
*/
static struct nvmap_heap_block *do_heap_alloc(struct nvmap_heap *heap,
size_t len, size_t align,
unsigned int mem_prot,
phys_addr_t base_max,
phys_addr_t *start)
{
struct list_block *heap_block = NULL;
dma_addr_t dev_base;
struct device *dev = heap->dma_dev;
/* since pages are only mappable with one cache attribute,
* and most allocations from carveout heaps are DMA coherent
* (i.e., non-cacheable), round cacheable allocations up to
* a page boundary to ensure that the physical pages will
* only be mapped one way. */
if (mem_prot == NVMAP_HANDLE_CACHEABLE ||
mem_prot == NVMAP_HANDLE_INNER_CACHEABLE) {
align = max_t(size_t, align, PAGE_SIZE);
len = PAGE_ALIGN(len);
}
if (heap->is_ivm)
align = max_t(size_t, align, NVMAP_IVM_ALIGNMENT);
heap_block = kmem_cache_zalloc(heap_block_cache, GFP_KERNEL);
if (!heap_block) {
dev_err(dev, "%s: failed to alloc heap block %s\n",
__func__, dev_name(dev));
goto fail_heap_block_alloc;
}
dev_base = nvmap_alloc_mem(heap, len, start);
if (dma_mapping_error(dev, dev_base)) {
dev_err(dev, "failed to alloc mem of size (%zu)\n",
len);
#ifdef NVMAP_CONFIG_VPR_RESIZE
if (dma_is_coherent_dev(dev)) {
struct dma_coherent_stats stats;
dma_get_coherent_stats(dev, &stats);
dev_err(dev, "used:%zu,curr_size:%zu max:%zu\n",
stats.used, stats.size, stats.max);
}
#endif
goto fail_dma_alloc;
}
heap_block->block.base = dev_base;
heap_block->orig_addr = dev_base;
heap_block->size = len;
list_add_tail(&heap_block->all_list, &heap->all_list);
heap_block->heap = heap;
heap->free_size -= len;
heap_block->mem_prot = mem_prot;
heap_block->align = align;
return &heap_block->block;
fail_dma_alloc:
kmem_cache_free(heap_block_cache, heap_block);
fail_heap_block_alloc:
return NULL;
}
static void do_heap_free(struct nvmap_heap_block *block)
{
struct list_block *b = container_of(block, struct list_block, block);
struct nvmap_heap *heap = b->heap;
list_del(&b->all_list);
heap->free_size += b->size;
nvmap_free_mem(heap, block->base, b->size);
kmem_cache_free(heap_block_cache, b);
}
/* nvmap_heap_alloc: allocates a block of memory of len bytes, aligned to
* align bytes. */
struct nvmap_heap_block *nvmap_heap_alloc(struct nvmap_heap *h,
struct nvmap_handle *handle,
phys_addr_t *start)
{
struct nvmap_heap_block *b;
size_t len = handle->size;
size_t align = handle->align;
unsigned int prot = handle->flags;
mutex_lock(&h->lock);
if (h->is_ivm) { /* Is IVM carveout? */
/* Check if this correct IVM heap */
if (handle->peer != h->peer) {
mutex_unlock(&h->lock);
return NULL;
} else {
if (h->can_alloc && start) {
/* If this partition does actual allocation, it
* should not specify start_offset.
*/
mutex_unlock(&h->lock);
return NULL;
} else if (!h->can_alloc && !start) {
/* If this partition does not do actual
* allocation, it should specify start_offset.
*/
mutex_unlock(&h->lock);
return NULL;
}
}
}
/*
* If this HEAP has pm_ops defined and powering on the
* RAM attached with the HEAP returns error, don't
* allocate from the heap and return NULL.
*/
if (h->pm_ops.busy) {
if (h->pm_ops.busy() < 0) {
pr_err("Unable to power on the heap device\n");
mutex_unlock(&h->lock);
return NULL;
}
}
align = max_t(size_t, align, L1_CACHE_BYTES);
b = do_heap_alloc(h, len, align, prot, 0, start);
if (b) {
b->handle = handle;
handle->carveout = b;
/* Generate IVM for partition that can alloc */
if (h->is_ivm && h->can_alloc) {
unsigned int offs = (b->base - h->base);
BUG_ON(offs & (NVMAP_IVM_ALIGNMENT - 1));
BUG_ON((offs >> ffs(NVMAP_IVM_ALIGNMENT)) &
~((1 << NVMAP_IVM_OFFSET_WIDTH) - 1));
BUG_ON(h->vm_id & ~(NVMAP_IVM_IVMID_MASK));
/* So, page alignment is sufficient check.
*/
BUG_ON(len & ~(PAGE_MASK));
/* Copy offset from IVM mem pool in nvmap handle.
* The offset will be exported via ioctl.
*/
handle->offs = offs;
handle->ivm_id = ((u64)h->vm_id << NVMAP_IVM_IVMID_SHIFT);
handle->ivm_id |= (((offs >> (ffs(NVMAP_IVM_ALIGNMENT) - 1)) &
((1ULL << NVMAP_IVM_OFFSET_WIDTH) - 1)) <<
NVMAP_IVM_OFFSET_SHIFT);
handle->ivm_id |= (len >> PAGE_SHIFT);
}
}
mutex_unlock(&h->lock);
return b;
}
struct nvmap_heap *nvmap_block_to_heap(struct nvmap_heap_block *b)
{
struct list_block *lb;
lb = container_of(b, struct list_block, block);
return lb->heap;
}
/* nvmap_heap_free: frees block b*/
void nvmap_heap_free(struct nvmap_heap_block *b)
{
struct nvmap_heap *h;
struct list_block *lb;
if (!b)
return;
h = nvmap_block_to_heap(b);
mutex_lock(&h->lock);
lb = container_of(b, struct list_block, block);
nvmap_flush_heap_block(NULL, b, lb->size, lb->mem_prot);
do_heap_free(b);
/*
* If this HEAP has pm_ops defined and powering off the
* RAM attached with the HEAP returns error, raise warning.
*/
if (h->pm_ops.idle) {
if (h->pm_ops.idle() < 0)
WARN_ON(1);
}
mutex_unlock(&h->lock);
}
/* nvmap_heap_create: create a heap object of len bytes, starting from
* address base.
*/
struct nvmap_heap *nvmap_heap_create(struct device *parent,
const struct nvmap_platform_carveout *co,
phys_addr_t base, size_t len, void *arg)
{
struct nvmap_heap *h;
h = kzalloc(sizeof(*h), GFP_KERNEL);
if (!h) {
dev_err(parent, "%s: out of memory\n", __func__);
return NULL;
}
h->dma_dev = co->dma_dev;
if (co->cma_dev) {
#ifdef CONFIG_DMA_CMA
#ifdef NVMAP_CONFIG_VPR_RESIZE
struct dma_contiguous_stats stats;
if (dma_get_contiguous_stats(co->cma_dev, &stats))
goto fail;
base = stats.base;
len = stats.size;
h->cma_dev = co->cma_dev;
#endif
#else
dev_err(parent, "invalid resize config for carveout %s\n",
co->name);
goto fail;
#endif
} else if (!co->init_done) {
int err;
/* declare Non-CMA heap */
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0)
err = dma_declare_coherent_memory(h->dma_dev, 0, base, len,
DMA_MEMORY_NOMAP);
#else
err = nvmap_dma_declare_coherent_memory(h->dma_dev, 0, base, len,
DMA_MEMORY_NOMAP);
#endif
#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)
if (!err) {
#else
if (err & DMA_MEMORY_NOMAP) {
#endif
dev_info(parent,
"%s :dma coherent mem declare %pa,%zu\n",
co->name, &base, len);
} else {
dev_err(parent,
"%s: dma coherent declare fail %pa,%zu\n",
co->name, &base, len);
goto fail;
}
}
dev_set_name(h->dma_dev, "%s", co->name);
dma_set_coherent_mask(h->dma_dev, DMA_BIT_MASK(64));
h->name = co->name;
h->arg = arg;
h->base = base;
h->can_alloc = !!co->can_alloc;
h->is_ivm = co->is_ivm;
h->len = len;
h->free_size = len;
h->peer = co->peer;
h->vm_id = co->vmid;
if (co->pm_ops.busy)
h->pm_ops.busy = co->pm_ops.busy;
if (co->pm_ops.idle)
h->pm_ops.idle = co->pm_ops.idle;
INIT_LIST_HEAD(&h->all_list);
mutex_init(&h->lock);
#ifdef NVMAP_CONFIG_DEBUG_MAPS
h->device_names = RB_ROOT;
#endif /* NVMAP_CONFIG_DEBUG_MAPS */
if (!co->no_cpu_access && co->usage_mask != NVMAP_HEAP_CARVEOUT_VPR
&& nvmap_cache_maint_phys_range(NVMAP_CACHE_OP_WB_INV,
base, base + len, true, true)) {
dev_err(parent, "cache flush failed\n");
goto fail;
}
wmb();
if (co->disable_dynamic_dma_map)
nvmap_dev->dynamic_dma_map_mask &= ~co->usage_mask;
if (co->no_cpu_access)
nvmap_dev->cpu_access_mask &= ~co->usage_mask;
dev_info(parent, "created heap %s base 0x%px size (%zuKiB)\n",
co->name, (void *)(uintptr_t)base, len/1024);
return h;
fail:
kfree(h);
return NULL;
}
/* nvmap_heap_destroy: frees all resources in heap */
void nvmap_heap_destroy(struct nvmap_heap *heap)
{
WARN_ON(!list_empty(&heap->all_list));
while (!list_empty(&heap->all_list)) {
struct list_block *l;
l = list_first_entry(&heap->all_list, struct list_block,
all_list);
list_del(&l->all_list);
kmem_cache_free(heap_block_cache, l);
}
kfree(heap);
}
int nvmap_heap_init(void)
{
ulong start_time = sched_clock();
heap_block_cache = KMEM_CACHE(list_block, 0);
if (!heap_block_cache) {
pr_err("%s: unable to create heap block cache\n", __func__);
return -ENOMEM;
}
pr_info("%s: created heap block cache\n", __func__);
nvmap_init_time += sched_clock() - start_time;
return 0;
}
void nvmap_heap_deinit(void)
{
if (heap_block_cache)
kmem_cache_destroy(heap_block_cache);
heap_block_cache = NULL;
}
/*
* This routine is used to flush the carveout memory from cache.
* Why cache flush is needed for carveout? Consider the case, where a piece of
* carveout is allocated as cached and released. After this, if the same memory is
* allocated for uncached request and the memory is not flushed out from cache.
* In this case, the client might pass this to H/W engine and it could start modify
* the memory. As this was cached earlier, it might have some portion of it in cache.
* During cpu request to read/write other memory, the cached portion of this memory
* might get flushed back to main memory and would cause corruptions, if it happens
* after H/W writes data to memory.
*
* But flushing out the memory blindly on each carveout allocation is redundant.
*
* In order to optimize the carveout buffer cache flushes, the following
* strategy is used.
*
* The whole Carveout is flushed out from cache during its initialization.
* During allocation, carveout buffers are not flused from cache.
* During deallocation, carveout buffers are flushed, if they were allocated as cached.
* if they were allocated as uncached/writecombined, no cache flush is needed.
* Just draining store buffers is enough.
*/
int nvmap_flush_heap_block(struct nvmap_client *client,
struct nvmap_heap_block *block, size_t len, unsigned int prot)
{
phys_addr_t phys = block->base;
phys_addr_t end = block->base + len;
int ret = 0;
if (prot == NVMAP_HANDLE_UNCACHEABLE || prot == NVMAP_HANDLE_WRITE_COMBINE)
goto out;
ret = nvmap_cache_maint_phys_range(NVMAP_CACHE_OP_WB_INV, phys, end,
true, prot != NVMAP_HANDLE_INNER_CACHEABLE);
if (ret)
goto out;
out:
wmb();
return ret;
}

79
drivers/video/tegra/nvmap/nvmap_heap.h Normal file
View File

@@ -0,0 +1,79 @@
/*
* drivers/video/tegra/nvmap/nvmap_heap.h
*
* GPU heap allocator.
*
* Copyright (c) 2010-2022, NVIDIA Corporation. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*/
#ifndef __NVMAP_HEAP_H
#define __NVMAP_HEAP_H
struct device;
struct nvmap_heap;
struct nvmap_client;
struct nvmap_heap_block {
phys_addr_t base;
unsigned int type;
struct nvmap_handle *handle;
};
struct nvmap_heap {
struct list_head all_list;
struct mutex lock;
const char *name;
void *arg;
/* heap base */
phys_addr_t base;
/* heap size */
size_t len;
size_t free_size;
struct device *cma_dev;
struct device *dma_dev;
bool is_ivm;
bool can_alloc; /* Used only if is_ivm == true */
unsigned int peer; /* Used only if is_ivm == true */
unsigned int vm_id; /* Used only if is_ivm == true */
struct nvmap_pm_ops pm_ops;
#ifdef NVMAP_CONFIG_DEBUG_MAPS
struct rb_root device_names;
#endif /* NVMAP_CONFIG_DEBUG_MAPS */
};
struct nvmap_heap *nvmap_heap_create(struct device *parent,
const struct nvmap_platform_carveout *co,
phys_addr_t base, size_t len, void *arg);
void nvmap_heap_destroy(struct nvmap_heap *heap);
struct nvmap_heap_block *nvmap_heap_alloc(struct nvmap_heap *heap,
struct nvmap_handle *handle,
phys_addr_t *start);
struct nvmap_heap *nvmap_block_to_heap(struct nvmap_heap_block *b);
void nvmap_heap_free(struct nvmap_heap_block *block);
int __init nvmap_heap_init(void);
void nvmap_heap_deinit(void);
int nvmap_flush_heap_block(struct nvmap_client *client,
struct nvmap_heap_block *block, size_t len, unsigned int prot);
void nvmap_heap_debugfs_init(struct dentry *heap_root, struct nvmap_heap *heap);
int nvmap_query_heap_peer(struct nvmap_heap *heap, unsigned int *peer);
size_t nvmap_query_heap_size(struct nvmap_heap *heap);
#endif

62
drivers/video/tegra/nvmap/nvmap_id_array.c Normal file
View File

@@ -0,0 +1,62 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (c) 2021-2022 NVIDIA Corporation
*/
#include <linux/xarray.h>
#include <linux/dma-buf.h>
#include "nvmap_priv.h"
#define XA_START (U32_MAX / 2)
/*
* Initialize xarray mapping
*/
void nvmap_id_array_init(struct xarray *id_arr)
{
xa_init_flags(id_arr, XA_FLAGS_ALLOC1);
}
/*
* Remove id to dma_buf mapping
*/
void nvmap_id_array_exit(struct xarray *id_arr)
{
xa_destroy(id_arr);
}
/*
* Create mapping between the id(NvRmMemHandle) and dma_buf
*/
int nvmap_id_array_id_alloc(struct xarray *id_arr, u32 *id, struct dma_buf *dmabuf)
{
if (!id_arr || !dmabuf)
return -EINVAL;
return xa_alloc(id_arr, id, dmabuf,
XA_LIMIT(XA_START, U32_MAX), GFP_KERNEL);
}
/*
* Clear mapping between the id(NvRmMemHandle) and dma_buf
*/
struct dma_buf *nvmap_id_array_id_release(struct xarray *id_arr, u32 id)
{
if (!id_arr || !id)
return NULL;
return xa_erase(id_arr, id);
}
/*
* Return dma_buf from the id.
*/
struct dma_buf *nvmap_id_array_get_dmabuf_from_id(struct xarray *id_arr, u32 id)
{
struct dma_buf *dmabuf;
dmabuf = xa_load(id_arr, id);
if (!IS_ERR_OR_NULL(dmabuf))
get_dma_buf(dmabuf);
return dmabuf;
}

979
drivers/video/tegra/nvmap/nvmap_init.c Normal file
View File

@@ -0,0 +1,979 @@
/*
* Copyright (c) 2014-2022, NVIDIA CORPORATION. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* Portions derived from Linux kernel source file kernel/dma/coherent.c
*/
#define pr_fmt(fmt) "%s: " fmt, __func__
#include <linux/module.h>
#include <linux/of.h>
#include <linux/of_fdt.h>
#include <linux/of_platform.h>
#include <linux/nvmap.h>
#include <linux/version.h>
#include <linux/kmemleak.h>
#include <linux/io.h>
#if defined(NVMAP_LOADABLE_MODULE)
#include <linux/nvmap_t19x.h>
#endif
#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)
#include <linux/sched/clock.h>
#endif
#include <linux/cma.h>
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0)
#include <linux/dma-map-ops.h>
#else
#include <linux/dma-contiguous.h>
#include <asm/dma-contiguous.h>
#endif
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0)
#include "include/linux/nvmap_exports.h"
#endif
#include "nvmap_priv.h"
#ifdef CONFIG_TEGRA_VIRTUALIZATION
#include <linux/tegra-ivc.h>
#include <soc/tegra/virt/syscalls.h>
#endif
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0)
#ifdef CONFIG_ARM_DMA_IOMMU_ALIGNMENT
#define DMA_BUF_ALIGNMENT CONFIG_ARM_DMA_IOMMU_ALIGNMENT
#else
#define DMA_BUF_ALIGNMENT 8
#endif
#endif /* LINUX_VERSION_CODE */
#ifndef NVMAP_UPSTREAM_KERNEL
#ifndef NVMAP_CONFIG_VPR_RESIZE
extern phys_addr_t tegra_vpr_start;
extern phys_addr_t tegra_vpr_size;
extern bool tegra_vpr_resize;
#endif /* NVMAP_CONFIG_VPR_RESIZE */
#endif /* !NVMAP_UPSTREAM_KERNEL */
struct device __weak tegra_generic_dev;
struct device __weak tegra_vpr_dev;
EXPORT_SYMBOL(tegra_vpr_dev);
struct device __weak tegra_generic_cma_dev;
struct device __weak tegra_vpr_cma_dev;
#ifdef NVMAP_LOADABLE_MODULE
static struct platform_device *pdev;
#endif /* NVMAP_LOADABLE_MODULE */
#ifdef NVMAP_CONFIG_VPR_RESIZE
struct dma_resize_notifier_ops __weak vpr_dev_ops;
static struct dma_declare_info generic_dma_info = {
.name = "generic",
.size = 0,
.notifier.ops = NULL,
};
static struct dma_declare_info vpr_dma_info = {
.name = "vpr",
.size = SZ_32M,
.notifier.ops = &vpr_dev_ops,
};
#endif
__weak const struct of_device_id nvmap_of_ids[] = {
{ .compatible = "nvidia,carveouts" },
{ .compatible = "nvidia,carveouts-t18x" },
{ }
};
static struct nvmap_platform_carveout nvmap_carveouts[] = {
[0] = {
.name = "generic-0",
.usage_mask = NVMAP_HEAP_CARVEOUT_GENERIC,
.base = 0,
.size = 0,
.dma_dev = &tegra_generic_dev,
.cma_dev = &tegra_generic_cma_dev,
#ifdef NVMAP_CONFIG_VPR_RESIZE
.dma_info = &generic_dma_info,
#endif
},
[1] = {
.name = "vpr",
.usage_mask = NVMAP_HEAP_CARVEOUT_VPR,
.base = 0,
.size = 0,
.dma_dev = &tegra_vpr_dev,
.cma_dev = &tegra_vpr_cma_dev,
#ifdef NVMAP_CONFIG_VPR_RESIZE
.dma_info = &vpr_dma_info,
#endif
.enable_static_dma_map = true,
},
[2] = {
.name = "vidmem",
.usage_mask = NVMAP_HEAP_CARVEOUT_VIDMEM,
.base = 0,
.size = 0,
.disable_dynamic_dma_map = true,
.no_cpu_access = true,
},
[3] = {
.name = "fsi",
.usage_mask = NVMAP_HEAP_CARVEOUT_FSI,
.base = 0,
.size = 0,
},
/* Need uninitialized entries for IVM carveouts */
[4] = {
.name = NULL,
.usage_mask = NVMAP_HEAP_CARVEOUT_IVM,
},
[5] = {
.name = NULL,
.usage_mask = NVMAP_HEAP_CARVEOUT_IVM,
},
[6] = {
.name = NULL,
.usage_mask = NVMAP_HEAP_CARVEOUT_IVM,
},
[7] = {
.name = NULL,
.usage_mask = NVMAP_HEAP_CARVEOUT_IVM,
},
};
static struct nvmap_platform_data nvmap_data = {
.carveouts = nvmap_carveouts,
.nr_carveouts = 4,
};
static struct nvmap_platform_carveout *nvmap_get_carveout_pdata(const char *name)
{
struct nvmap_platform_carveout *co;
for (co = nvmap_carveouts;
co < nvmap_carveouts + ARRAY_SIZE(nvmap_carveouts); co++) {
int i = min_t(int, strcspn(name, "_"), strcspn(name, "-"));
/* handle IVM carveouts */
if ((co->usage_mask == NVMAP_HEAP_CARVEOUT_IVM) && !co->name)
goto found;
if (strncmp(co->name, name, i))
continue;
found:
co->dma_dev = co->dma_dev ? co->dma_dev : &co->dev;
return co;
}
pr_err("not enough space for all nvmap carveouts\n");
return NULL;
}
int nvmap_register_vidmem_carveout(struct device *dma_dev,
phys_addr_t base, size_t size)
{
struct nvmap_platform_carveout *vidmem_co;
if (!base || !size || (base != PAGE_ALIGN(base)) ||
(size != PAGE_ALIGN(size)))
return -EINVAL;
vidmem_co = nvmap_get_carveout_pdata("vidmem");
if (!vidmem_co)
return -ENODEV;
if (vidmem_co->base || vidmem_co->size)
return -EEXIST;
vidmem_co->base = base;
vidmem_co->size = size;
if (dma_dev)
vidmem_co->dma_dev = dma_dev;
return nvmap_create_carveout(vidmem_co);
}
EXPORT_SYMBOL(nvmap_register_vidmem_carveout);
#ifdef CONFIG_TEGRA_VIRTUALIZATION
static int __init nvmap_populate_ivm_carveout(struct device *dev)
{
char *name;
const __be32 *prop;
int ret = 0;
struct nvmap_platform_carveout *co;
struct of_phandle_iterator it;
struct tegra_hv_ivm_cookie *ivm;
unsigned long long id;
unsigned int guestid;
if (!of_phandle_iterator_init(&it, dev->of_node, "memory-region", NULL, 0)) {
while (!of_phandle_iterator_next(&it) && it.node) {
if (of_device_is_available(it.node) &&
of_device_is_compatible(it.node, "nvidia,ivm_carveout") > 0) {
co = nvmap_get_carveout_pdata("nvidia,ivm_carveout");
if (!co) {
ret = -ENOMEM;
goto err;
}
if (hyp_read_gid(&guestid)) {
pr_err("failed to read gid\n");
ret = -EINVAL;
goto err;
}
prop = of_get_property(it.node, "ivm", NULL);
if (!prop) {
pr_err("failed to read ivm property\n");
ret = -EINVAL;
goto err;
}
id = of_read_number(prop + 1, 1);
if (id > UINT_MAX) {
ret = -EINVAL;
goto err;
}
ivm = tegra_hv_mempool_reserve(id);
if (IS_ERR_OR_NULL(ivm)) {
pr_err("failed to reserve IVM memory pool %llu\n", id);
ret = -ENOMEM;
goto err;
}
/* XXX: Are these the available fields from IVM cookie? */
co->base = (phys_addr_t)ivm->ipa;
co->peer = ivm->peer_vmid;
co->size = ivm->size;
co->vmid = guestid;
if (!co->base || !co->size) {
ret = -EINVAL;
goto fail;
}
/* See if this VM can allocate (or just create handle from ID)
* generated by peer partition
*/
prop = of_get_property(it.node, "alloc", NULL);
if (!prop) {
pr_err("failed to read alloc property\n");
ret = -EINVAL;
goto fail;
}
name = kzalloc(32, GFP_KERNEL);
if (!name) {
ret = -ENOMEM;
goto fail;
}
co->can_alloc = of_read_number(prop, 1);
co->is_ivm = true;
sprintf(name, "ivm%02u%02u%02d", co->vmid, co->peer, co->can_alloc);
pr_info("IVM carveout IPA:%p, size=%zu, peer vmid=%u, name=%s\n",
(void *)(uintptr_t)co->base, co->size, co->peer, name);
co->name = name;
nvmap_data.nr_carveouts++;
}
}
}
return 0;
fail:
co->base = 0;
co->peer = 0;
co->size = 0;
co->vmid = 0;
err:
return ret;
}
#endif /* CONFIG_TEGRA_VIRTUALIZATION */
/*
* This requires proper kernel arguments to have been passed.
*/
#ifndef NVMAP_UPSTREAM_KERNEL
static int __nvmap_init_legacy(struct device *dev)
{
#ifndef NVMAP_CONFIG_VPR_RESIZE
/* VPR */
if (!nvmap_carveouts[1].base) {
nvmap_carveouts[1].base = tegra_vpr_start;
nvmap_carveouts[1].size = tegra_vpr_size;
nvmap_carveouts[1].cma_dev = NULL;
}
#endif /* NVMAP_CONFIG_VPR_RESIZE */
return 0;
}
#endif /* !NVMAP_UPSTREAM_KERNEL */
static int __nvmap_init_dt(struct platform_device *pdev)
{
if (!of_match_device(nvmap_of_ids, &pdev->dev)) {
pr_err("Missing DT entry!\n");
return -EINVAL;
}
#ifndef NVMAP_UPSTREAM_KERNEL
/* For VM_2 we need carveout. So, enabling it here */
__nvmap_init_legacy(&pdev->dev);
#endif /* !NVMAP_UPSTREAM_KERNEL */
pdev->dev.platform_data = &nvmap_data;
return 0;
}
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0)
static inline struct page **nvmap_kvzalloc_pages(u32 count)
{
if (count * sizeof(struct page *) <= PAGE_SIZE)
return kzalloc(count * sizeof(struct page *), GFP_KERNEL);
else
return vzalloc(count * sizeof(struct page *));
}
static void *__nvmap_dma_alloc_from_coherent(struct device *dev,
struct dma_coherent_mem_replica *mem,
size_t size,
dma_addr_t *dma_handle,
unsigned long attrs,
unsigned long start)
{
int order = get_order(size);
unsigned long flags;
unsigned int count, i = 0, j = 0;
unsigned int alloc_size;
unsigned long align, pageno;
void *addr = NULL;
struct page **pages = NULL;
int do_memset = 0;
int *bitmap_nos = NULL;
if (dma_get_attr(DMA_ATTR_ALLOC_EXACT_SIZE, attrs))
count = PAGE_ALIGN(size) >> PAGE_SHIFT;
else
count = 1 << order;
if (!count)
return NULL;
bitmap_nos = vzalloc(count * sizeof(int));
if (!bitmap_nos) {
dev_err(dev, "failed to allocate memory\n");
return NULL;
}
if ((mem->flags & DMA_MEMORY_NOMAP) &&
dma_get_attr(DMA_ATTR_ALLOC_SINGLE_PAGES, attrs)) {
alloc_size = 1;
pages = nvmap_kvzalloc_pages(count);
if (!pages)
return NULL;
} else {
alloc_size = count;
}
spin_lock_irqsave(&mem->spinlock, flags);
if (unlikely(size > (mem->size << PAGE_SHIFT)))
goto err;
if ((mem->flags & DMA_MEMORY_NOMAP) &&
dma_get_attr(DMA_ATTR_ALLOC_SINGLE_PAGES, attrs)) {
align = 0;
} else {
if (order > DMA_BUF_ALIGNMENT)
align = (1 << DMA_BUF_ALIGNMENT) - 1;
else
align = (1 << order) - 1;
}
while (count) {
pageno = bitmap_find_next_zero_area(mem->bitmap, mem->size,
start, alloc_size, align);
if (pageno >= mem->size)
goto err;
count -= alloc_size;
if (pages)
pages[i++] = pfn_to_page(mem->pfn_base + pageno);
bitmap_set(mem->bitmap, pageno, alloc_size);
bitmap_nos[j++] = pageno;
}
/*
* Memory was found in the coherent area.
*/
*dma_handle = mem->device_base + (pageno << PAGE_SHIFT);
if (!(mem->flags & DMA_MEMORY_NOMAP)) {
addr = mem->virt_base + (pageno << PAGE_SHIFT);
do_memset = 1;
} else if (dma_get_attr(DMA_ATTR_ALLOC_SINGLE_PAGES, attrs)) {
addr = pages;
}
spin_unlock_irqrestore(&mem->spinlock, flags);
if (do_memset)
memset(addr, 0, size);
kvfree(bitmap_nos);
return addr;
err:
while (j--)
bitmap_clear(mem->bitmap, bitmap_nos[j], alloc_size);
spin_unlock_irqrestore(&mem->spinlock, flags);
kvfree(pages);
kvfree(bitmap_nos);
return NULL;
}
void *nvmap_dma_alloc_attrs(struct device *dev, size_t size,
dma_addr_t *dma_handle,
gfp_t flag, unsigned long attrs)
{
struct dma_coherent_mem_replica *mem;
if (!dev || !dev->dma_mem)
return NULL;
WARN_ON_ONCE(!dev->coherent_dma_mask);
mem = (struct dma_coherent_mem_replica *)(dev->dma_mem);
return __nvmap_dma_alloc_from_coherent(dev, mem, size, dma_handle,
attrs, 0);
}
EXPORT_SYMBOL(nvmap_dma_alloc_attrs);
void nvmap_dma_free_attrs(struct device *dev, size_t size, void *cpu_addr,
dma_addr_t dma_handle, unsigned long attrs)
{
void *mem_addr;
unsigned long flags;
unsigned int pageno;
struct dma_coherent_mem_replica *mem;
if (!dev || !dev->dma_mem)
return;
mem = (struct dma_coherent_mem_replica *)(dev->dma_mem);
if ((mem->flags & DMA_MEMORY_NOMAP) &&
dma_get_attr(DMA_ATTR_ALLOC_SINGLE_PAGES, attrs)) {
struct page **pages = cpu_addr;
int i;
spin_lock_irqsave(&mem->spinlock, flags);
for (i = 0; i < (size >> PAGE_SHIFT); i++) {
pageno = page_to_pfn(pages[i]) - mem->pfn_base;
if (WARN_ONCE(pageno > mem->size,
"invalid pageno:%d\n", pageno))
continue;
bitmap_clear(mem->bitmap, pageno, 1);
}
spin_unlock_irqrestore(&mem->spinlock, flags);
kvfree(pages);
return;
}
if (mem->flags & DMA_MEMORY_NOMAP)
mem_addr = (void *)(uintptr_t)mem->device_base;
else
mem_addr = mem->virt_base;
if (mem && cpu_addr >= mem_addr &&
cpu_addr - mem_addr < mem->size << PAGE_SHIFT) {
unsigned int page = (cpu_addr - mem_addr) >> PAGE_SHIFT;
unsigned long flags;
unsigned int count;
if (DMA_ATTR_ALLOC_EXACT_SIZE & attrs)
count = PAGE_ALIGN(size) >> PAGE_SHIFT;
else
count = 1 << get_order(size);
spin_lock_irqsave(&mem->spinlock, flags);
bitmap_clear(mem->bitmap, page, count);
spin_unlock_irqrestore(&mem->spinlock, flags);
}
}
EXPORT_SYMBOL(nvmap_dma_free_attrs);
void *nvmap_dma_mark_declared_memory_occupied(struct device *dev,
dma_addr_t device_addr, size_t size)
{
struct dma_coherent_mem_replica *mem;
unsigned long flags;
int pos, err;
if (!dev || !dev->dma_mem)
return ERR_PTR(-EINVAL);
mem = (struct dma_coherent_mem_replica *)(dev->dma_mem);
size += device_addr & ~PAGE_MASK;
spin_lock_irqsave(&mem->spinlock, flags);
pos = PFN_DOWN(device_addr - mem->device_base);
err = bitmap_allocate_region(mem->bitmap, pos, get_order(size));
spin_unlock_irqrestore(&mem->spinlock, flags);
if (err != 0)
return ERR_PTR(err);
return mem->virt_base + (pos << PAGE_SHIFT);
}
void nvmap_dma_mark_declared_memory_unoccupied(struct device *dev,
dma_addr_t device_addr, size_t size)
{
struct dma_coherent_mem_replica *mem;
unsigned long flags;
int pos;
if (!dev || !dev->dma_mem)
return;
mem = (struct dma_coherent_mem_replica *)(dev->dma_mem);
size += device_addr & ~PAGE_MASK;
spin_lock_irqsave(&mem->spinlock, flags);
pos = PFN_DOWN(device_addr - mem->device_base);
bitmap_release_region(mem->bitmap, pos, get_order(size));
spin_unlock_irqrestore(&mem->spinlock, flags);
}
static void nvmap_dma_release_coherent_memory(struct dma_coherent_mem_replica *mem)
{
if (!mem)
return;
if (!(mem->flags & DMA_MEMORY_NOMAP))
memunmap(mem->virt_base);
kfree(mem->bitmap);
kfree(mem);
}
static int nvmap_dma_assign_coherent_memory(struct device *dev,
struct dma_coherent_mem_replica *mem)
{
if (!dev)
return -ENODEV;
if (dev->dma_mem)
return -EBUSY;
dev->dma_mem = (struct dma_coherent_mem *)mem;
return 0;
}
static int nvmap_dma_init_coherent_memory(
phys_addr_t phys_addr, dma_addr_t device_addr, size_t size, int flags,
struct dma_coherent_mem_replica **mem)
{
struct dma_coherent_mem_replica *dma_mem = NULL;
void *mem_base = NULL;
int pages = size >> PAGE_SHIFT;
int bitmap_size = BITS_TO_LONGS(pages) * sizeof(long);
int ret;
if (!size)
return -EINVAL;
if (!(flags & DMA_MEMORY_NOMAP)) {
mem_base = memremap(phys_addr, size, MEMREMAP_WC);
if (!mem_base)
return -EINVAL;
}
dma_mem = kzalloc(sizeof(struct dma_coherent_mem_replica), GFP_KERNEL);
if (!dma_mem) {
ret = -ENOMEM;
goto err_memunmap;
}
dma_mem->bitmap = kzalloc(bitmap_size, GFP_KERNEL);
if (!dma_mem->bitmap) {
ret = -ENOMEM;
goto err_free_dma_mem;
}
dma_mem->virt_base = mem_base;
dma_mem->device_base = device_addr;
dma_mem->pfn_base = PFN_DOWN(phys_addr);
dma_mem->size = pages;
dma_mem->flags = flags;
spin_lock_init(&dma_mem->spinlock);
*mem = dma_mem;
return 0;
err_free_dma_mem:
kfree(dma_mem);
err_memunmap:
memunmap(mem_base);
return ret;
}
int nvmap_dma_declare_coherent_memory(struct device *dev, phys_addr_t phys_addr,
dma_addr_t device_addr, size_t size, int flags)
{
struct dma_coherent_mem_replica *mem;
int ret;
ret = nvmap_dma_init_coherent_memory(phys_addr, device_addr, size, flags, &mem);
if (ret)
return ret;
ret = nvmap_dma_assign_coherent_memory(dev, mem);
if (ret)
nvmap_dma_release_coherent_memory(mem);
return ret;
}
#endif /* LINUX_VERSION_CODE */
static int __init nvmap_co_device_init(struct reserved_mem *rmem,
struct device *dev)
{
struct nvmap_platform_carveout *co = rmem->priv;
int err = 0;
if (!co)
return -ENODEV;
/* if co size is 0, => co is not present. So, skip init. */
if (!co->size)
return 0;
if (!co->cma_dev) {
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0)
err = dma_declare_coherent_memory(co->dma_dev, 0,
co->base, co->size,
DMA_MEMORY_NOMAP);
#else
err = nvmap_dma_declare_coherent_memory(co->dma_dev, 0,
co->base, co->size,
DMA_MEMORY_NOMAP);
#endif
#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)
if (!err) {
#else
if (err & DMA_MEMORY_NOMAP) {
#endif
dev_info(dev,
"%s :dma coherent mem declare %pa,%zu\n",
co->name, &co->base, co->size);
co->init_done = true;
err = 0;
} else
dev_err(dev,
"%s :dma coherent mem declare fail %pa,%zu,err:%d\n",
co->name, &co->base, co->size, err);
} else {
#ifdef NVMAP_CONFIG_VPR_RESIZE
#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0)
/*
* When vpr memory is reserved, kmemleak tries to scan vpr
* memory for pointers. vpr memory should not be accessed
* from cpu so avoid scanning it. When vpr memory is removed,
* the memblock_remove() API ensures that kmemleak won't scan
* a removed block.
*/
if (!strncmp(co->name, "vpr", 3))
kmemleak_no_scan(__va(co->base));
#endif
co->dma_info->cma_dev = co->cma_dev;
err = dma_declare_coherent_resizable_cma_memory(
co->dma_dev, co->dma_info);
if (err)
dev_err(dev, "%s coherent memory declaration failed\n",
co->name);
else
#endif
co->init_done = true;
}
return err;
}
static void nvmap_co_device_release(struct reserved_mem *rmem,struct device *dev)
{
struct nvmap_platform_carveout *co = rmem->priv;
if (!co)
return;
if (co->usage_mask == NVMAP_HEAP_CARVEOUT_IVM)
kfree(co->name);
}
static const struct reserved_mem_ops nvmap_co_ops = {
.device_init = nvmap_co_device_init,
.device_release = nvmap_co_device_release,
};
#ifndef NVMAP_LOADABLE_MODULE
int __init nvmap_co_setup(struct reserved_mem *rmem)
{
struct nvmap_platform_carveout *co;
int ret = 0;
#ifdef NVMAP_CONFIG_VPR_RESIZE
struct cma *cma;
#endif
ulong start = sched_clock();
co = nvmap_get_carveout_pdata(rmem->name);
if (!co)
return ret;
rmem->ops = &nvmap_co_ops;
rmem->priv = co;
co->base = rmem->base;
co->size = rmem->size;
#ifdef NVMAP_CONFIG_VPR_RESIZE
if (!of_get_flat_dt_prop(rmem->fdt_node, "reusable", NULL) ||
of_get_flat_dt_prop(rmem->fdt_node, "no-map", NULL))
goto skip_cma;
WARN_ON(!rmem->base);
if (dev_get_cma_area(co->cma_dev)) {
pr_info("cma area initialed in legacy way already\n");
goto finish;
}
#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)
ret = cma_init_reserved_mem(rmem->base, rmem->size, 0,
rmem->name, &cma);
#else
ret = cma_init_reserved_mem(rmem->base, rmem->size, 0, &cma);
#endif
if (ret) {
pr_info("cma_init_reserved_mem fails for %s\n", rmem->name);
goto finish;
}
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0)
dma_contiguous_early_fixup_vpr(rmem->base, rmem->size);
if (co->cma_dev)
co->cma_dev->cma_area = cma;
#else
dma_contiguous_early_fixup(rmem->base, rmem->size);
dev_set_cma_area(co->cma_dev, cma);
#endif
pr_debug("tegra-carveouts carveout=%s %pa@%pa\n",
rmem->name, &rmem->size, &rmem->base);
goto finish;
skip_cma:
#endif
co->cma_dev = NULL;
#ifdef NVMAP_CONFIG_VPR_RESIZE
finish:
#endif
nvmap_init_time += sched_clock() - start;
return ret;
}
EXPORT_SYMBOL(nvmap_co_setup);
#else
int __init nvmap_co_setup(struct reserved_mem *rmem)
{
struct nvmap_platform_carveout *co;
ulong start = sched_clock();
int ret = 0;
co = nvmap_get_carveout_pdata(rmem->name);
if (!co)
return ret;
rmem->ops = &nvmap_co_ops;
rmem->priv = co;
co->base = rmem->base;
co->size = rmem->size;
co->cma_dev = NULL;
nvmap_init_time += sched_clock() - start;
return ret;
}
#endif /* !NVMAP_LOADABLE_MODULE */
RESERVEDMEM_OF_DECLARE(nvmap_co, "nvidia,generic_carveout", nvmap_co_setup);
#ifndef NVMAP_LOADABLE_MODULE
RESERVEDMEM_OF_DECLARE(nvmap_vpr_co, "nvidia,vpr-carveout", nvmap_co_setup);
RESERVEDMEM_OF_DECLARE(nvmap_fsi_co, "nvidia,fsi-carveout", nvmap_co_setup);
#endif /* !NVMAP_LOADABLE_MODULE */
/*
* Fills in the platform data either from the device tree or with the
* legacy path.
*/
int __init nvmap_init(struct platform_device *pdev)
{
int err;
struct reserved_mem rmem;
#ifdef NVMAP_LOADABLE_MODULE
struct reserved_mem *rmem2;
struct device_node *np = pdev->dev.of_node;
struct of_phandle_iterator it;
const char *compp;
if (!of_phandle_iterator_init(&it, np, "memory-region", NULL, 0)) {
while (!of_phandle_iterator_next(&it) && it.node) {
if (of_device_is_available(it.node) &&
!of_device_is_compatible(it.node, "nvidia,ivm_carveout")) {
rmem2 = of_reserved_mem_lookup(it.node);
if (!rmem2) {
if (!of_property_read_string(it.node, "compatible", &compp))
pr_err("unable to acquire memory-region: %s\n",
compp);
return -EINVAL;
}
nvmap_co_setup(rmem2);
}
}
}
#endif /* NVMAP_LOADABLE_MODULE */
if (pdev->dev.of_node) {
err = __nvmap_init_dt(pdev);
if (err)
return err;
}
err = of_reserved_mem_device_init(&pdev->dev);
if (err)
pr_debug("reserved_mem_device_init fails, try legacy init\n");
/* try legacy init */
if (!nvmap_carveouts[0].init_done) {
rmem.priv = &nvmap_carveouts[0];
err = nvmap_co_device_init(&rmem, &pdev->dev);
if (err)
goto end;
}
if (!nvmap_carveouts[1].init_done) {
rmem.priv = &nvmap_carveouts[1];
err = nvmap_co_device_init(&rmem, &pdev->dev);
if (err)
goto end;
}
#ifdef CONFIG_TEGRA_VIRTUALIZATION
err = nvmap_populate_ivm_carveout(&pdev->dev);
#endif /* CONFIG_TEGRA_VIRTUALIZATION */
end:
return err;
}
#ifdef NVMAP_LOADABLE_MODULE
static bool nvmap_is_carveout_node_present(void)
{
struct device_node *np;
np = of_find_node_by_name(NULL, "tegra-carveouts");
if (of_device_is_available(np)) {
of_node_put(np);
return true;
}
of_node_put(np);
return false;
}
#endif /* NVMAP_LOADABLE_MODULE */
static struct platform_driver __refdata nvmap_driver = {
.probe = nvmap_probe,
.remove = nvmap_remove,
.driver = {
.name = "tegra-carveouts",
.owner = THIS_MODULE,
#ifndef NVMAP_LOADABLE_MODULE
.of_match_table = nvmap_of_ids,
#endif /* !NVMAP_LOADABLE_MODULE */
.probe_type = PROBE_PREFER_ASYNCHRONOUS,
.suppress_bind_attrs = true,
},
};
static int __init nvmap_init_driver(void)
{
int e = 0;
e = nvmap_heap_init();
if (e)
goto fail;
#ifdef NVMAP_LOADABLE_MODULE
if (!(of_machine_is_compatible("nvidia,tegra186") ||
of_machine_is_compatible("nvidia,tegra194") ||
of_machine_is_compatible("nvidia,tegra234") ||
of_machine_is_compatible("nvidia,tegra239") ||
of_machine_is_compatible("nvidia,tegra23x") ||
of_machine_is_compatible("nvidia,tegra232"))) {
nvmap_heap_deinit();
return -ENODEV;
}
#endif /* NVMAP_LOADABLE_MODULE */
e = platform_driver_register(&nvmap_driver);
if (e) {
nvmap_heap_deinit();
goto fail;
}
#ifdef NVMAP_LOADABLE_MODULE
if (!nvmap_is_carveout_node_present()) {
pdev = platform_device_register_simple("tegra-carveouts", -1, NULL, 0);
if (IS_ERR(pdev)) {
platform_driver_unregister(&nvmap_driver);
nvmap_heap_deinit();
return PTR_ERR(pdev);
}
}
#endif /* NVMAP_LOADABLE_MODULE */
fail:
return e;
}
#ifdef NVMAP_LOADABLE_MODULE
module_init(nvmap_init_driver);
#else
fs_initcall(nvmap_init_driver);
#endif /* NVMAP_LOADABLE_MODULE */
static void __exit nvmap_exit_driver(void)
{
#ifdef NVMAP_LOADABLE_MODULE
if (!nvmap_is_carveout_node_present())
platform_device_unregister(pdev);
#endif /* NVMAP_LOADABLE_MODULE */
platform_driver_unregister(&nvmap_driver);
nvmap_heap_deinit();
nvmap_dev = NULL;
}
module_exit(nvmap_exit_driver);
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 16, 0)
MODULE_IMPORT_NS(DMA_BUF);
#endif
MODULE_DESCRIPTION("NvMap: Nvidia Tegra Memory Management Driver");
MODULE_AUTHOR("Puneet Saxena <puneets@nvidia.com>");
MODULE_LICENSE("GPL v2");

62
drivers/video/tegra/nvmap/nvmap_init_t19x.c Normal file
View File

@@ -0,0 +1,62 @@
/*
* drivers/video/tegra/nvmap/nvmap_init_t19x.c
*
* Copyright (c) 2016-2022, NVIDIA CORPORATION. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*/
#define pr_fmt(fmt) "nvmap: %s() " fmt, __func__
#include <linux/module.h>
#include <linux/of.h>
#include <linux/of_reserved_mem.h>
#include <linux/platform_device.h>
#include <linux/nvmap_t19x.h>
#include <linux/kobject.h>
#include <linux/debugfs.h>
#include <linux/sysfs.h>
#include <linux/io.h>
#include "nvmap_priv.h"
bool nvmap_version_t19x;
const struct of_device_id nvmap_of_ids[] = {
{ .compatible = "nvidia,carveouts" },
{ .compatible = "nvidia,carveouts-t18x" },
{ .compatible = "nvidia,carveouts-t19x" },
{ }
};
int nvmap_register_cvsram_carveout(struct device *dma_dev,
phys_addr_t base, size_t size, int (*busy)(void),
int (*idle)(void))
{
static struct nvmap_platform_carveout cvsram = {
.name = "cvsram",
.usage_mask = NVMAP_HEAP_CARVEOUT_CVSRAM,
.disable_dynamic_dma_map = true,
.no_cpu_access = true,
};
cvsram.pm_ops.busy = busy;
cvsram.pm_ops.idle = idle;
if (!base || !size || (base != PAGE_ALIGN(base)) ||
(size != PAGE_ALIGN(size)))
return -EINVAL;
cvsram.base = base;
cvsram.size = size;
cvsram.dma_dev = &cvsram.dev;
return nvmap_create_carveout(&cvsram);
}
EXPORT_SYMBOL(nvmap_register_cvsram_carveout);

1434
drivers/video/tegra/nvmap/nvmap_ioctl.c Normal file
View File

File diff suppressed because it is too large Load Diff

79
drivers/video/tegra/nvmap/nvmap_ioctl.h Normal file
View File

@@ -0,0 +1,79 @@
/*
* drivers/video/tegra/nvmap/nvmap_ioctl.h
*
* ioctl declarations for nvmap
*
* Copyright (c) 2010-2019, NVIDIA CORPORATION. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*/
#ifndef __VIDEO_TEGRA_NVMAP_IOCTL_H
#define __VIDEO_TEGRA_NVMAP_IOCTL_H
#include <linux/nvmap.h>
int nvmap_ioctl_pinop(struct file *filp, bool is_pin, void __user *arg,
bool is32);
int nvmap_ioctl_getid(struct file *filp, void __user *arg);
int nvmap_ioctl_get_ivcid(struct file *filp, void __user *arg);
int nvmap_ioctl_getfd(struct file *filp, void __user *arg);
int nvmap_ioctl_alloc(struct file *filp, void __user *arg);
int nvmap_ioctl_alloc_kind(struct file *filp, void __user *arg);
int nvmap_ioctl_alloc_ivm(struct file *filp, void __user *arg);
int nvmap_ioctl_vpr_floor_size(struct file *filp, void __user *arg);
int nvmap_ioctl_free(struct file *filp, unsigned long arg);
int nvmap_ioctl_create(struct file *filp, unsigned int cmd, void __user *arg);
int nvmap_ioctl_create_from_va(struct file *filp, void __user *arg);
int nvmap_ioctl_create_from_ivc(struct file *filp, void __user *arg);
int nvmap_ioctl_get_ivc_heap(struct file *filp, void __user *arg);
int nvmap_map_into_caller_ptr(struct file *filp, void __user *arg, bool is32);
int nvmap_ioctl_cache_maint(struct file *filp, void __user *arg, int size);
int nvmap_ioctl_rw_handle(struct file *filp, int is_read, void __user *arg,
size_t op_size);
int nvmap_ioctl_cache_maint_list(struct file *filp, void __user *arg);
int nvmap_ioctl_gup_test(struct file *filp, void __user *arg);
int nvmap_ioctl_set_tag_label(struct file *filp, void __user *arg);
int nvmap_ioctl_get_available_heaps(struct file *filp, void __user *arg);
int nvmap_ioctl_get_heap_size(struct file *filp, void __user *arg);
void kasan_memcpy_toio(void __iomem *to, const void *from,
size_t count);
int nvmap_ioctl_get_handle_parameters(struct file *filp, void __user *arg);
int nvmap_ioctl_get_sci_ipc_id(struct file *filp, void __user *arg);
int nvmap_ioctl_handle_from_sci_ipc_id(struct file *filp, void __user *arg);
int nvmap_ioctl_query_heap_params(struct file *filp, void __user *arg);
int nvmap_ioctl_dup_handle(struct file *filp, void __user *arg);
#endif /* __VIDEO_TEGRA_NVMAP_IOCTL_H */

46
drivers/video/tegra/nvmap/nvmap_kasan_wrapper.c Normal file
View File

@@ -0,0 +1,46 @@
/*
* drivers/video/tegra/nvmap/nvmap_kasan_wrapper.c
*
* place to add wrapper function to drop kasan scan
*
* Copyright (c) 2018-2022, NVIDIA CORPORATION. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*/
#include <linux/export.h>
#include <linux/types.h>
#include <linux/io.h>
#include "nvmap_ioctl.h"
void kasan_memcpy_toio(void __iomem *to,
const void *from, size_t count)
{
while (count && (!IS_ALIGNED((unsigned long)to, 8) ||
!IS_ALIGNED((unsigned long)from, 8))) {
__raw_writeb(*(u8 *)from, to);
from++;
to++;
count--;
}
while (count >= 8) {
__raw_writeq(*(u64 *)from, to);
from += 8;
to += 8;
count -= 8;
}
while (count) {
__raw_writeb(*(u8 *)from, to);
from++;
to++;
count--;
}
}

84
drivers/video/tegra/nvmap/nvmap_mm.c Normal file
View File

@@ -0,0 +1,84 @@
/*
* drivers/video/tegra/nvmap/nvmap_mm.c
*
* Some MM related functionality specific to nvmap.
*
* Copyright (c) 2013-2021, NVIDIA CORPORATION. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*/
#include <trace/events/nvmap.h>
#include <linux/version.h>
#include <asm/pgtable.h>
#include "nvmap_priv.h"
#ifndef NVMAP_LOADABLE_MODULE
void nvmap_zap_handle(struct nvmap_handle *handle, u64 offset, u64 size)
{
struct list_head *vmas;
struct nvmap_vma_list *vma_list;
struct vm_area_struct *vma;
if (!handle->heap_pgalloc)
return;
/* if no dirty page is present, no need to zap */
if (nvmap_handle_track_dirty(handle) && !atomic_read(&handle->pgalloc.ndirty))
return;
if (!size) {
offset = 0;
size = handle->size;
}
size = PAGE_ALIGN((offset & ~PAGE_MASK) + size);
mutex_lock(&handle->lock);
vmas = &handle->vmas;
list_for_each_entry(vma_list, vmas, list) {
struct nvmap_vma_priv *priv;
size_t vm_size = size;
vma = vma_list->vma;
priv = vma->vm_private_data;
if ((offset + size) > (vma->vm_end - vma->vm_start))
vm_size = vma->vm_end - vma->vm_start - offset;
if (priv->offs || vma->vm_pgoff)
/* vma mapping starts in the middle of handle memory.
* zapping needs special care. zap entire range for now.
* FIXME: optimze zapping.
*/
zap_page_range(vma, vma->vm_start,
#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)
vma->vm_end - vma->vm_start);
#else
vma->vm_end - vma->vm_start, NULL);
#endif
else
zap_page_range(vma, vma->vm_start + offset,
#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)
vm_size);
#else
vm_size, NULL);
#endif
}
mutex_unlock(&handle->lock);
}
#else
void nvmap_zap_handle(struct nvmap_handle *handle, u64 offset, u64 size)
{
pr_debug("%s is not supported!\n", __func__);
}
#endif /* !NVMAP_LOADABLE_MODULE */

774
drivers/video/tegra/nvmap/nvmap_pp.c Normal file
View File

@@ -0,0 +1,774 @@
/*
* drivers/video/tegra/nvmap/nvmap_pp.c
*
* Manage page pools to speed up page allocation.
*
* Copyright (c) 2009-2022, NVIDIA CORPORATION. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*/
#define pr_fmt(fmt) "%s: " fmt, __func__
#include <linux/kernel.h>
#include <linux/vmalloc.h>
#include <linux/moduleparam.h>
#include <linux/nodemask.h>
#include <linux/shrinker.h>
#include <linux/kthread.h>
#include <linux/debugfs.h>
#include <linux/freezer.h>
#include <linux/highmem.h>
#include <linux/version.h>
#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)
#include <linux/sched/clock.h>
#include <uapi/linux/sched/types.h>
#endif
#include <trace/events/nvmap.h>
#include "nvmap_priv.h"
#define NVMAP_TEST_PAGE_POOL_SHRINKER 1
#define PENDING_PAGES_SIZE (SZ_1M / PAGE_SIZE)
static bool enable_pp = 1;
static u32 pool_size;
static struct task_struct *background_allocator;
static DECLARE_WAIT_QUEUE_HEAD(nvmap_bg_wait);
#ifdef NVMAP_CONFIG_PAGE_POOL_DEBUG
static inline void __pp_dbg_var_add(u64 *dbg_var, u32 nr)
{
*dbg_var += nr;
}
#else
#define __pp_dbg_var_add(dbg_var, nr)
#endif
#define pp_alloc_add(pool, nr) __pp_dbg_var_add(&(pool)->allocs, nr)
#define pp_fill_add(pool, nr) __pp_dbg_var_add(&(pool)->fills, nr)
#define pp_hit_add(pool, nr) __pp_dbg_var_add(&(pool)->hits, nr)
#define pp_miss_add(pool, nr) __pp_dbg_var_add(&(pool)->misses, nr)
static int __nvmap_page_pool_fill_lots_locked(struct nvmap_page_pool *pool,
struct page **pages, u32 nr);
static inline struct page *get_zero_list_page(struct nvmap_page_pool *pool)
{
struct page *page;
trace_get_zero_list_page(pool->to_zero);
if (list_empty(&pool->zero_list))
return NULL;
page = list_first_entry(&pool->zero_list, struct page, lru);
list_del(&page->lru);
pool->to_zero--;
return page;
}
static inline struct page *get_page_list_page(struct nvmap_page_pool *pool)
{
struct page *page;
trace_get_page_list_page(pool->count);
if (list_empty(&pool->page_list))
return NULL;
page = list_first_entry(&pool->page_list, struct page, lru);
list_del(&page->lru);
pool->count--;
return page;
}
#ifdef CONFIG_ARM64_4K_PAGES
static inline struct page *get_page_list_page_bp(struct nvmap_page_pool *pool)
{
struct page *page;
if (list_empty(&pool->page_list_bp))
return NULL;
page = list_first_entry(&pool->page_list_bp, struct page, lru);
list_del(&page->lru);
pool->count -= pool->pages_per_big_pg;
pool->big_page_count -= pool->pages_per_big_pg;
return page;
}
#endif /* CONFIG_ARM64_4K_PAGES */
static inline bool nvmap_bg_should_run(struct nvmap_page_pool *pool)
{
return !list_empty(&pool->zero_list);
}
static void nvmap_pp_zero_pages(struct page **pages, int nr)
{
int i;
for (i = 0; i < nr; i++) {
clear_highpage(pages[i]);
nvmap_clean_cache_page(pages[i]);
}
trace_nvmap_pp_zero_pages(nr);
}
static void nvmap_pp_do_background_zero_pages(struct nvmap_page_pool *pool)
{
int i;
struct page *page;
int ret;
/*
* Statically declared array of pages to be zeroed in a batch,
* local to this thread but too big for the stack.
*/
static struct page *pending_zero_pages[PENDING_PAGES_SIZE];
rt_mutex_lock(&pool->lock);
for (i = 0; i < PENDING_PAGES_SIZE; i++) {
page = get_zero_list_page(pool);
if (page == NULL)
break;
pending_zero_pages[i] = page;
pool->under_zero++;
}
rt_mutex_unlock(&pool->lock);
nvmap_pp_zero_pages(pending_zero_pages, i);
rt_mutex_lock(&pool->lock);
ret = __nvmap_page_pool_fill_lots_locked(pool, pending_zero_pages, i);
pool->under_zero -= i;
rt_mutex_unlock(&pool->lock);
trace_nvmap_pp_do_background_zero_pages(ret, i);
for (; ret < i; ret++)
__free_page(pending_zero_pages[ret]);
}
/*
* This thread fills the page pools with zeroed pages. We avoid releasing the
* pages directly back into the page pools since we would then have to zero
* them ourselves. Instead it is easier to just reallocate zeroed pages. This
* happens in the background so that the overhead of allocating zeroed pages is
* not directly seen by userspace. Of course if the page pools are empty user
* space will suffer.
*/
static int nvmap_background_zero_thread(void *arg)
{
struct nvmap_page_pool *pool = &nvmap_dev->pool;
#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0)
struct sched_param param = { .sched_priority = 0 };
#endif
pr_info("PP zeroing thread starting.\n");
set_freezable();
#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0)
sched_setscheduler(current, SCHED_NORMAL, &param);
#else
sched_set_normal(current, MAX_NICE);
#endif
while (!kthread_should_stop()) {
while (nvmap_bg_should_run(pool))
nvmap_pp_do_background_zero_pages(pool);
wait_event_freezable(nvmap_bg_wait,
nvmap_bg_should_run(pool) ||
kthread_should_stop());
}
return 0;
}
#ifdef NVMAP_CONFIG_PAGE_POOL_DEBUG
static void nvmap_pgcount(struct page *page, bool incr)
{
page_ref_add(page, incr ? 1 : -1);
}
#endif /* NVMAP_CONFIG_PAGE_POOL_DEBUG */
/*
* Free the passed number of pages from the page pool. This happens regardless
* of whether the page pools are enabled. This lets one disable the page pools
* and then free all the memory therein.
*
* FIXME: Pages in pending_zero_pages[] can still be unreleased.
*/
static ulong nvmap_page_pool_free_pages_locked(struct nvmap_page_pool *pool,
ulong nr_pages)
{
struct page *page;
bool use_page_list = false;
#ifdef CONFIG_ARM64_4K_PAGES
bool use_page_list_bp = false;
int i;
#endif /* CONFIG_ARM64_4K_PAGES */
pr_debug("req to release pages=%ld\n", nr_pages);
while (nr_pages) {
#ifdef CONFIG_ARM64_4K_PAGES
if (use_page_list_bp)
page = get_page_list_page_bp(pool);
else if (use_page_list)
#else
if (use_page_list)
#endif /* CONFIG_ARM64_4K_PAGES */
page = get_page_list_page(pool);
else
page = get_zero_list_page(pool);
if (!page) {
if (!use_page_list) {
use_page_list = true;
continue;
}
#ifdef CONFIG_ARM64_4K_PAGES
else if (!use_page_list_bp) {
use_page_list_bp = true;
continue;
}
#endif /* CONFIG_ARM64_4K_PAGES */
break;
}
#ifdef CONFIG_ARM64_4K_PAGES
if (use_page_list_bp) {
for (i = 0; i < pool->pages_per_big_pg; i++)
__free_page(nth_page(page, i));
pr_debug("released %d pages\n", pool->pages_per_big_pg);
if (nr_pages > pool->pages_per_big_pg)
nr_pages -= pool->pages_per_big_pg;
else
nr_pages = 0;
} else {
#endif /* CONFIG_ARM64_4K_PAGES */
__free_page(page);
nr_pages--;
pr_debug("released 1 page\n");
#ifdef CONFIG_ARM64_4K_PAGES
}
#endif /* CONFIG_ARM64_4K_PAGES */
}
pr_debug("remaining pages to release=%ld\n", nr_pages);
return nr_pages;
}
/*
* Alloc a bunch of pages from the page pool. This will alloc as many as it can
* and return the number of pages allocated. Pages are placed into the passed
* array in a linear fashion starting from index 0.
*/
int nvmap_page_pool_alloc_lots(struct nvmap_page_pool *pool,
struct page **pages, u32 nr)
{
u32 ind = 0;
u32 non_zero_idx;
u32 non_zero_cnt = 0;
if (!enable_pp || !nr)
return 0;
rt_mutex_lock(&pool->lock);
while (ind < nr) {
struct page *page = NULL;
if (!non_zero_cnt)
page = get_page_list_page(pool);
if (!page) {
page = get_zero_list_page(pool);
if (!page)
break;
if (!non_zero_cnt)
non_zero_idx = ind;
non_zero_cnt++;
}
pages[ind++] = page;
#ifdef NVMAP_CONFIG_PAGE_POOL_DEBUG
nvmap_pgcount(page, false);
BUG_ON(page_count(page) != 1);
#endif /* NVMAP_CONFIG_PAGE_POOL_DEBUG */
}
rt_mutex_unlock(&pool->lock);
/* Zero non-zeroed pages, if any */
if (non_zero_cnt)
nvmap_pp_zero_pages(&pages[non_zero_idx], non_zero_cnt);
pp_alloc_add(pool, ind);
pp_hit_add(pool, ind);
pp_miss_add(pool, nr - ind);
trace_nvmap_pp_alloc_lots(ind, nr);
return ind;
}
#ifdef CONFIG_ARM64_4K_PAGES
int nvmap_page_pool_alloc_lots_bp(struct nvmap_page_pool *pool,
struct page **pages, u32 nr)
{
u32 ind = 0, nr_pages = nr;
struct page *page;
if (!enable_pp || pool->pages_per_big_pg <= 1 ||
nr_pages < pool->pages_per_big_pg)
return 0;
rt_mutex_lock(&pool->lock);
while (nr_pages - ind >= pool->pages_per_big_pg) {
int i;
page = get_page_list_page_bp(pool);
if (!page)
break;
for (i = 0; i < pool->pages_per_big_pg; i++)
pages[ind + i] = nth_page(page, i);
ind += pool->pages_per_big_pg;
}
rt_mutex_unlock(&pool->lock);
return ind;
}
static bool nvmap_is_big_page(struct nvmap_page_pool *pool,
struct page **pages, int idx, int nr)
{
int i;
struct page *page = pages[idx];
if (pool->pages_per_big_pg <= 1)
return false;
if (nr - idx < pool->pages_per_big_pg)
return false;
/* Allow coalescing pages at big page boundary only */
if (page_to_phys(page) & (pool->big_pg_sz - 1))
return false;
for (i = 1; i < pool->pages_per_big_pg; i++)
if (pages[idx + i] != nth_page(page, i))
break;
return i == pool->pages_per_big_pg ? true: false;
}
#endif /* CONFIG_ARM64_4K_PAGES */
/*
* Fill a bunch of pages into the page pool. This will fill as many as it can
* and return the number of pages filled. Pages are used from the start of the
* passed page pointer array in a linear fashion.
*
* You must lock the page pool before using this.
*/
static int __nvmap_page_pool_fill_lots_locked(struct nvmap_page_pool *pool,
struct page **pages, u32 nr)
{
int real_nr;
int pages_to_fill;
int ind = 0;
if (!enable_pp)
return 0;
BUG_ON(pool->count > pool->max);
real_nr = min_t(u32, pool->max - pool->count, nr);
pages_to_fill = real_nr;
if (real_nr == 0)
return 0;
while (real_nr > 0) {
#ifdef NVMAP_CONFIG_PAGE_POOL_DEBUG
nvmap_pgcount(pages[ind], true);
BUG_ON(page_count(pages[ind]) != 2);
#endif /* NVMAP_CONFIG_PAGE_POOL_DEBUG */
#ifdef CONFIG_ARM64_4K_PAGES
if (nvmap_is_big_page(pool, pages, ind, pages_to_fill)) {
list_add_tail(&pages[ind]->lru, &pool->page_list_bp);
ind += pool->pages_per_big_pg;
real_nr -= pool->pages_per_big_pg;
pool->big_page_count += pool->pages_per_big_pg;
} else {
#endif /* CONFIG_ARM64_4K_PAGES */
list_add_tail(&pages[ind++]->lru, &pool->page_list);
real_nr--;
#ifdef CONFIG_ARM64_4K_PAGES
}
#endif /* CONFIG_ARM64_4K_PAGES */
}
pool->count += ind;
BUG_ON(pool->count > pool->max);
pp_fill_add(pool, ind);
return ind;
}
u32 nvmap_page_pool_fill_lots(struct nvmap_page_pool *pool,
struct page **pages, u32 nr)
{
u32 ret = 0;
u32 i;
u32 save_to_zero;
rt_mutex_lock(&pool->lock);
save_to_zero = pool->to_zero;
ret = min(nr, pool->max - pool->count - pool->to_zero - pool->under_zero);
for (i = 0; i < ret; i++) {
/* If page has additonal referecnces, Don't add it into
* page pool. get_user_pages() on mmap'ed nvmap handle can
* hold a refcount on the page. These pages can't be
* reused till the additional refs are dropped.
*/
if (page_count(pages[i]) > 1) {
__free_page(pages[i]);
} else {
list_add_tail(&pages[i]->lru, &pool->zero_list);
pool->to_zero++;
}
}
if (pool->to_zero)
wake_up_interruptible(&nvmap_bg_wait);
ret = i;
trace_nvmap_pp_fill_zero_lots(save_to_zero, pool->to_zero,
ret, nr);
rt_mutex_unlock(&pool->lock);
return ret;
}
ulong nvmap_page_pool_get_unused_pages(void)
{
int total = 0;
if (!nvmap_dev)
return 0;
total = nvmap_dev->pool.count + nvmap_dev->pool.to_zero;
return total;
}
/*
* Remove and free to the system all the pages currently in the page
* pool. This operation will happen even if the page pools are disabled.
*/
int nvmap_page_pool_clear(void)
{
struct nvmap_page_pool *pool = &nvmap_dev->pool;
rt_mutex_lock(&pool->lock);
(void)nvmap_page_pool_free_pages_locked(pool, pool->count + pool->to_zero);
/* For some reason, if an error occured... */
if (!list_empty(&pool->page_list) || !list_empty(&pool->zero_list)) {
rt_mutex_unlock(&pool->lock);
return -ENOMEM;
}
rt_mutex_unlock(&pool->lock);
return 0;
}
/*
* Resizes the page pool to the passed size. If the passed size is 0 then
* all associated resources are released back to the system. This operation
* will only occur if the page pools are enabled.
*/
static void nvmap_page_pool_resize(struct nvmap_page_pool *pool, u32 size)
{
u64 curr;
rt_mutex_lock(&pool->lock);
curr = nvmap_page_pool_get_unused_pages();
if (curr > size)
(void)nvmap_page_pool_free_pages_locked(pool, curr - size);
pr_debug("page pool resized to %d from %d pages\n", size, pool->max);
pool->max = size;
rt_mutex_unlock(&pool->lock);
}
static unsigned long nvmap_page_pool_count_objects(struct shrinker *shrinker,
struct shrink_control *sc)
{
return nvmap_page_pool_get_unused_pages();
}
static unsigned long nvmap_page_pool_scan_objects(struct shrinker *shrinker,
struct shrink_control *sc)
{
unsigned long remaining;
pr_debug("sh_pages=%lu", sc->nr_to_scan);
rt_mutex_lock(&nvmap_dev->pool.lock);
remaining = nvmap_page_pool_free_pages_locked(
&nvmap_dev->pool, sc->nr_to_scan);
rt_mutex_unlock(&nvmap_dev->pool.lock);
return (remaining == sc->nr_to_scan) ? \
SHRINK_STOP : (sc->nr_to_scan - remaining);
}
static struct shrinker nvmap_page_pool_shrinker = {
.count_objects = nvmap_page_pool_count_objects,
.scan_objects = nvmap_page_pool_scan_objects,
.seeks = 1,
};
static void shrink_page_pools(unsigned long *total_pages, unsigned long *available_pages)
{
struct shrink_control sc;
if (*total_pages == 0) {
sc.gfp_mask = GFP_KERNEL;
sc.nr_to_scan = 0;
*total_pages = nvmap_page_pool_count_objects(NULL, &sc);
}
sc.nr_to_scan = *total_pages;
nvmap_page_pool_scan_objects(NULL, &sc);
*available_pages = nvmap_page_pool_count_objects(NULL, &sc);
}
#if NVMAP_TEST_PAGE_POOL_SHRINKER
static int shrink_pp;
static int shrink_set(const char *arg, const struct kernel_param *kp)
{
int cpu = smp_processor_id();
unsigned long long t1, t2;
unsigned long total_pages, available_pages;
param_set_int(arg, kp);
if (shrink_pp) {
total_pages = shrink_pp;
t1 = cpu_clock(cpu);
shrink_page_pools(&total_pages, &available_pages);
t2 = cpu_clock(cpu);
pr_debug("shrink page pools: time=%lldns, "
"total_pages_released=%lu, free_pages_available=%lu",
t2-t1, total_pages, available_pages);
}
return 0;
}
static int shrink_get(char *buff, const struct kernel_param *kp)
{
return param_get_int(buff, kp);
}
static struct kernel_param_ops shrink_ops = {
.get = shrink_get,
.set = shrink_set,
};
module_param_cb(shrink_page_pools, &shrink_ops, &shrink_pp, 0644);
#endif
static int enable_pp_set(const char *arg, const struct kernel_param *kp)
{
int ret;
ret = param_set_bool(arg, kp);
if (ret)
return ret;
if (!enable_pp)
nvmap_page_pool_clear();
return 0;
}
static int enable_pp_get(char *buff, const struct kernel_param *kp)
{
return param_get_bool(buff, kp);
}
static struct kernel_param_ops enable_pp_ops = {
.get = enable_pp_get,
.set = enable_pp_set,
};
module_param_cb(enable_page_pools, &enable_pp_ops, &enable_pp, 0644);
static int pool_size_set(const char *arg, const struct kernel_param *kp)
{
int ret = param_set_uint(arg, kp);
if (!ret && (pool_size != nvmap_dev->pool.max))
nvmap_page_pool_resize(&nvmap_dev->pool, pool_size);
return ret;
}
static int pool_size_get(char *buff, const struct kernel_param *kp)
{
return param_get_int(buff, kp);
}
static struct kernel_param_ops pool_size_ops = {
.get = pool_size_get,
.set = pool_size_set,
};
module_param_cb(pool_size, &pool_size_ops, &pool_size, 0644);
int nvmap_page_pool_debugfs_init(struct dentry *nvmap_root)
{
struct dentry *pp_root;
if (!nvmap_root)
return -ENODEV;
pp_root = debugfs_create_dir("pagepool", nvmap_root);
if (!pp_root)
return -ENODEV;
debugfs_create_u32("page_pool_available_pages",
S_IRUGO, pp_root,
&nvmap_dev->pool.count);
debugfs_create_u32("page_pool_pages_to_zero",
S_IRUGO, pp_root,
&nvmap_dev->pool.to_zero);
#ifdef CONFIG_ARM64_4K_PAGES
debugfs_create_u32("page_pool_available_big_pages",
S_IRUGO, pp_root,
&nvmap_dev->pool.big_page_count);
debugfs_create_u32("page_pool_big_page_size",
S_IRUGO, pp_root,
&nvmap_dev->pool.big_pg_sz);
debugfs_create_u64("total_big_page_allocs",
S_IRUGO, pp_root,
&nvmap_big_page_allocs);
#endif /* CONFIG_ARM64_4K_PAGES */
debugfs_create_u64("total_page_allocs",
S_IRUGO, pp_root,
&nvmap_total_page_allocs);
#ifdef NVMAP_CONFIG_PAGE_POOL_DEBUG
debugfs_create_u64("page_pool_allocs",
S_IRUGO, pp_root,
&nvmap_dev->pool.allocs);
debugfs_create_u64("page_pool_fills",
S_IRUGO, pp_root,
&nvmap_dev->pool.fills);
debugfs_create_u64("page_pool_hits",
S_IRUGO, pp_root,
&nvmap_dev->pool.hits);
debugfs_create_u64("page_pool_misses",
S_IRUGO, pp_root,
&nvmap_dev->pool.misses);
#endif
return 0;
}
int nvmap_page_pool_init(struct nvmap_device *dev)
{
struct sysinfo info;
struct nvmap_page_pool *pool = &dev->pool;
memset(pool, 0x0, sizeof(*pool));
rt_mutex_init(&pool->lock);
INIT_LIST_HEAD(&pool->page_list);
INIT_LIST_HEAD(&pool->zero_list);
#ifdef CONFIG_ARM64_4K_PAGES
INIT_LIST_HEAD(&pool->page_list_bp);
pool->big_pg_sz = NVMAP_PP_BIG_PAGE_SIZE;
pool->pages_per_big_pg = NVMAP_PP_BIG_PAGE_SIZE >> PAGE_SHIFT;
#endif /* CONFIG_ARM64_4K_PAGES */
si_meminfo(&info);
pr_info("Total RAM pages: %lu\n", info.totalram);
if (!NVMAP_CONFIG_PAGE_POOL_SIZE)
/* The ratio is pool pages per 1K ram pages.
* So, the >> 10 */
pool->max = (info.totalram * NVMAP_PP_POOL_SIZE) >> 10;
else
pool->max = NVMAP_CONFIG_PAGE_POOL_SIZE;
if (pool->max >= info.totalram)
goto fail;
pool_size = pool->max;
pr_info("nvmap page pool size: %u pages (%u MB)\n", pool->max,
(pool->max * info.mem_unit) >> 20);
background_allocator = kthread_run(nvmap_background_zero_thread,
NULL, "nvmap-bz");
if (IS_ERR(background_allocator))
goto fail;
register_shrinker(&nvmap_page_pool_shrinker);
return 0;
fail:
nvmap_page_pool_fini(dev);
return -ENOMEM;
}
int nvmap_page_pool_fini(struct nvmap_device *dev)
{
struct nvmap_page_pool *pool = &dev->pool;
/*
* if background allocator is not initialzed or not
* properly initialized, then shrinker is also not
* registered
*/
if (!IS_ERR_OR_NULL(background_allocator)) {
unregister_shrinker(&nvmap_page_pool_shrinker);
kthread_stop(background_allocator);
background_allocator = NULL;
}
WARN_ON(!list_empty(&pool->page_list));
return 0;
}

904
drivers/video/tegra/nvmap/nvmap_priv.h Normal file
View File

@@ -0,0 +1,904 @@
/*
* drivers/video/tegra/nvmap/nvmap.h
*
* GPU memory management driver for Tegra
*
* Copyright (c) 2009-2022, NVIDIA CORPORATION. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*/
#ifndef __VIDEO_TEGRA_NVMAP_NVMAP_H
#define __VIDEO_TEGRA_NVMAP_NVMAP_H
#include <linux/list.h>
#include <linux/mm.h>
#include <linux/mutex.h>
#include <linux/rtmutex.h>
#include <linux/rbtree.h>
#include <linux/sched.h>
#include <linux/wait.h>
#include <linux/atomic.h>
#include <linux/dma-buf.h>
#include <linux/syscalls.h>
#include <linux/mm.h>
#include <linux/miscdevice.h>
#include <linux/nvmap.h>
#include <linux/vmalloc.h>
#include <linux/slab.h>
#include <linux/version.h>
#include <linux/workqueue.h>
#include <linux/dma-mapping.h>
#include <linux/dma-direction.h>
#include <linux/platform_device.h>
#include <linux/of.h>
#include <linux/of_reserved_mem.h>
#include <asm/cacheflush.h>
#ifndef CONFIG_ARM64
#include <asm/outercache.h>
#endif
#include "nvmap_heap.h"
#include "nvmap_stats.h"
#include <linux/fdtable.h>
#define DMA_ERROR_CODE (~(dma_addr_t)0)
#define __DMA_ATTR(attrs) attrs
#define DEFINE_DMA_ATTRS(attrs) unsigned long attrs = 0
/**
* dma_set_attr - set a specific attribute
* @attr: attribute to set
* @attrs: struct dma_attrs (may be NULL)
*/
#define dma_set_attr(attr, attrs) (attrs |= attr)
/**
* dma_get_attr - check for a specific attribute
* @attr: attribute to set
* @attrs: struct dma_attrs (may be NULL)
*/
#define dma_get_attr(attr, attrs) (attrs & attr)
#define NVMAP_TAG_LABEL_MAXLEN (63 - sizeof(struct nvmap_tag_entry))
#define NVMAP_TP_ARGS_H(handle) \
handle, \
atomic_read(&handle->share_count), \
handle->heap_type == NVMAP_HEAP_IOVMM ? 0 : \
(handle->carveout ? handle->carveout->base : 0), \
handle->size, \
(handle->userflags & 0xFFFF), \
(handle->userflags >> 16), \
__nvmap_tag_name(nvmap_dev, handle->userflags >> 16)
#define NVMAP_TP_ARGS_CHR(client, handle, ref) \
client, \
client ? nvmap_client_pid((struct nvmap_client *)client) : 0, \
(ref) ? atomic_read(&((struct nvmap_handle_ref *)ref)->dupes) : 1, \
NVMAP_TP_ARGS_H(handle)
#define NVMAP_TAG_TRACE(x, ...) \
do { \
if (x##_enabled()) { \
mutex_lock(&nvmap_dev->tags_lock); \
x(__VA_ARGS__); \
mutex_unlock(&nvmap_dev->tags_lock); \
} \
} while (0)
#define GFP_NVMAP (GFP_KERNEL | __GFP_HIGHMEM | __GFP_NOWARN)
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0)
/*
* DMA_ATTR_ALLOC_EXACT_SIZE: This tells the DMA-mapping
* subsystem to allocate the exact number of pages
*/
#define DMA_ATTR_ALLOC_EXACT_SIZE (DMA_ATTR_PRIVILEGED << 2)
#define DMA_MEMORY_NOMAP 0x02
#endif /* LINUX_VERSION_CODE */
#ifdef NVMAP_LOADABLE_MODULE
#ifdef NVMAP_UPSTREAM_KERNEL
/*
* DMA_ATTR_READ_ONLY: for DMA memory allocations, attempt to map
* memory as read-only for the device. CPU access will still be
* read-write. This corresponds to the direction being DMA_TO_DEVICE
* instead of DMA_BIDIRECTIONAL.
*/
#define DMA_ATTR_READ_ONLY (DMA_ATTR_PRIVILEGED << 12)
/* DMA_ATTR_WRITE_ONLY: This tells the DMA-mapping subsystem
* to map as write-only
*/
#define DMA_ATTR_WRITE_ONLY (DMA_ATTR_PRIVILEGED << 13)
#endif /* NVMAP_UPSTREAM_KERNEL */
#endif /* NVMAP_LOADABLE_MODULE */
#define DMA_ALLOC_FREE_ATTR DMA_ATTR_ALLOC_SINGLE_PAGES
#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0)
#define ACCESS_OK(type, addr, size) access_ok(type, addr, size)
#define SYS_CLOSE(arg) sys_close(arg)
#elif LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0)
#define ACCESS_OK(type, addr, size) access_ok(addr, size)
#define SYS_CLOSE(arg) ksys_close(arg)
#else
#define ACCESS_OK(type, addr, size) access_ok(addr, size)
#define SYS_CLOSE(arg) close_fd(arg)
#endif
struct page;
struct nvmap_device;
void _nvmap_handle_free(struct nvmap_handle *h);
/* holds max number of handles allocted per process at any time */
extern u32 nvmap_max_handle_count;
extern u64 nvmap_big_page_allocs;
extern u64 nvmap_total_page_allocs;
extern bool nvmap_convert_iovmm_to_carveout;
extern bool nvmap_convert_carveout_to_iovmm;
extern struct vm_operations_struct nvmap_vma_ops;
#ifdef CONFIG_ARM64
#define PG_PROT_KERNEL PAGE_KERNEL
#define FLUSH_DCACHE_AREA __flush_dcache_area
#define outer_flush_range(s, e)
#define outer_inv_range(s, e)
#define outer_clean_range(s, e)
#define outer_flush_all()
#define outer_clean_all()
extern void __clean_dcache_page(struct page *);
extern void __clean_dcache_area_poc(void *addr, size_t len);
#else
#define PG_PROT_KERNEL pgprot_kernel
#define FLUSH_DCACHE_AREA __cpuc_flush_dcache_area
extern void __flush_dcache_page(struct address_space *, struct page *);
#endif
struct nvmap_vma_list {
struct list_head list;
struct vm_area_struct *vma;
unsigned long save_vm_flags;
pid_t pid;
atomic_t ref;
};
struct nvmap_carveout_node {
unsigned int heap_bit;
struct nvmap_heap *carveout;
int index;
phys_addr_t base;
size_t size;
};
/* handles allocated using shared system memory (either IOVMM- or high-order
* page allocations */
struct nvmap_pgalloc {
struct page **pages;
bool contig; /* contiguous system memory */
atomic_t reserved;
atomic_t ndirty; /* count number of dirty pages */
};
#ifdef NVMAP_CONFIG_DEBUG_MAPS
struct nvmap_device_list {
struct rb_node node;
u64 dma_mask;
char *device_name;
};
#endif /* NVMAP_CONFIG_DEBUG_MAPS */
/* bit 31-29: IVM peer
* bit 28-16: offset (aligned to 32K)
* bit 15-00: len (aligned to page_size)
*/
#define NVMAP_IVM_LENGTH_SHIFT (0)
#define NVMAP_IVM_LENGTH_WIDTH (16)
#define NVMAP_IVM_LENGTH_MASK ((1 << NVMAP_IVM_LENGTH_WIDTH) - 1)
#define NVMAP_IVM_OFFSET_SHIFT (NVMAP_IVM_LENGTH_SHIFT + NVMAP_IVM_LENGTH_WIDTH)
#define NVMAP_IVM_OFFSET_WIDTH (13)
#define NVMAP_IVM_OFFSET_MASK ((1 << NVMAP_IVM_OFFSET_WIDTH) - 1)
#define NVMAP_IVM_IVMID_SHIFT (NVMAP_IVM_OFFSET_SHIFT + NVMAP_IVM_OFFSET_WIDTH)
#define NVMAP_IVM_IVMID_WIDTH (3)
#define NVMAP_IVM_IVMID_MASK ((1 << NVMAP_IVM_IVMID_WIDTH) - 1)
#define NVMAP_IVM_ALIGNMENT (SZ_32K)
struct nvmap_handle_dmabuf_priv {
void *priv;
struct device *dev;
void (*priv_release)(void *priv);
struct list_head list;
};
struct nvmap_handle {
struct rb_node node; /* entry on global handle tree */
atomic_t ref; /* reference count (i.e., # of duplications) */
atomic_t pin; /* pin count */
u32 flags; /* caching flags */
size_t size; /* padded (as-allocated) size */
size_t orig_size; /* original (as-requested) size */
size_t align;
struct nvmap_client *owner;
struct dma_buf *dmabuf;
struct dma_buf *dmabuf_ro;
union {
struct nvmap_pgalloc pgalloc;
struct nvmap_heap_block *carveout;
};
bool heap_pgalloc; /* handle is page allocated (sysmem / iovmm) */
bool alloc; /* handle has memory allocated */
bool from_va; /* handle memory is from VA */
u32 heap_type; /* handle heap is allocated from */
u32 userflags; /* flags passed from userspace */
void *vaddr; /* mapping used inside kernel */
struct list_head vmas; /* list of all user vma's */
atomic_t umap_count; /* number of outstanding maps from user */
atomic_t kmap_count; /* number of outstanding map from kernel */
atomic_t share_count; /* number of processes sharing the handle */
struct list_head lru; /* list head to track the lru */
struct mutex lock;
struct list_head dmabuf_priv;
u64 ivm_id;
unsigned int peer; /* Peer VM number */
int offs; /* Offset in IVM mem pool */
/*
* To be set only in handle created from VA case if the handle is
* read-only.
*/
bool is_ro;
};
struct nvmap_handle_info {
struct nvmap_handle *handle;
struct list_head maps;
struct mutex maps_lock;
bool is_ro;
};
struct nvmap_tag_entry {
struct rb_node node;
atomic_t ref; /* reference count (i.e., # of duplications) */
u32 tag;
};
/* handle_ref objects are client-local references to an nvmap_handle;
* they are distinct objects so that handles can be unpinned and
* unreferenced the correct number of times when a client abnormally
* terminates */
struct nvmap_handle_ref {
struct nvmap_handle *handle;
struct rb_node node;
atomic_t dupes; /* number of times to free on file close */
bool is_ro;
};
#if defined(NVMAP_CONFIG_PAGE_POOLS)
/*
* This is the default ratio defining pool size. It can be thought of as pool
* size in either MB per GB or KB per MB. That means the max this number can
* be is 1024 (all physical memory - not a very good idea) or 0 (no page pool
* at all).
*/
#define NVMAP_PP_POOL_SIZE (128)
#ifdef CONFIG_ARM64_4K_PAGES
#define NVMAP_PP_BIG_PAGE_SIZE (0x10000)
#endif /* CONFIG_ARM64_4K_PAGES */
struct nvmap_page_pool {
struct rt_mutex lock;
u32 count; /* Number of pages in the page & dirty list. */
u32 max; /* Max no. of pages in all lists. */
u32 to_zero; /* Number of pages on the zero list */
u32 under_zero; /* Number of pages getting zeroed */
#ifdef CONFIG_ARM64_4K_PAGES
u32 big_pg_sz; /* big page size supported(64k, etc.) */
u32 big_page_count; /* Number of zeroed big pages avaialble */
u32 pages_per_big_pg; /* Number of pages in big page */
#endif /* CONFIG_ARM64_4K_PAGES */
struct list_head page_list;
struct list_head zero_list;
#ifdef CONFIG_ARM64_4K_PAGES
struct list_head page_list_bp;
#endif /* CONFIG_ARM64_4K_PAGES */
#ifdef NVMAP_CONFIG_PAGE_POOL_DEBUG
u64 allocs;
u64 fills;
u64 hits;
u64 misses;
#endif
};
int nvmap_page_pool_init(struct nvmap_device *dev);
int nvmap_page_pool_fini(struct nvmap_device *dev);
struct page *nvmap_page_pool_alloc(struct nvmap_page_pool *pool);
int nvmap_page_pool_alloc_lots(struct nvmap_page_pool *pool,
struct page **pages, u32 nr);
#ifdef CONFIG_ARM64_4K_PAGES
int nvmap_page_pool_alloc_lots_bp(struct nvmap_page_pool *pool,
struct page **pages, u32 nr);
#endif /* CONFIG_ARM64_4K_PAGES */
u32 nvmap_page_pool_fill_lots(struct nvmap_page_pool *pool,
struct page **pages, u32 nr);
int nvmap_page_pool_clear(void);
int nvmap_page_pool_debugfs_init(struct dentry *nvmap_root);
#endif
#define NVMAP_IVM_INVALID_PEER (-1)
#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0)
#ifndef NVMAP_CONFIG_HANDLE_AS_ID
struct xarray {};
#endif /* !NVMAP_CONFIG_HANDLE_AS_ID */
#endif /* KERNEL_VERSION < 5.10 */
struct nvmap_client {
const char *name;
struct rb_root handle_refs;
struct mutex ref_lock;
bool kernel_client;
atomic_t count;
struct task_struct *task;
struct list_head list;
u32 handle_count;
u32 next_fd;
int warned;
int tag_warned;
struct xarray id_array;
struct xarray *ida;
};
struct nvmap_vma_priv {
struct nvmap_handle *handle;
size_t offs;
atomic_t count; /* number of processes cloning the VMA */
};
struct nvmap_device {
struct rb_root handles;
spinlock_t handle_lock;
struct miscdevice dev_user;
struct nvmap_carveout_node *heaps;
int nr_heaps;
int nr_carveouts;
#ifdef NVMAP_CONFIG_PAGE_POOLS
struct nvmap_page_pool pool;
#endif
struct list_head clients;
struct rb_root pids;
struct mutex clients_lock;
struct list_head lru_handles;
spinlock_t lru_lock;
struct dentry *handles_by_pid;
struct dentry *debug_root;
struct nvmap_platform_data *plat;
struct rb_root tags;
struct mutex tags_lock;
struct mutex carveout_lock; /* needed to serialize carveout creation */
u32 dynamic_dma_map_mask;
u32 cpu_access_mask;
#ifdef NVMAP_CONFIG_DEBUG_MAPS
struct rb_root device_names;
#endif /* NVMAP_CONFIG_DEBUG_MAPS */
};
extern struct nvmap_device *nvmap_dev;
extern ulong nvmap_init_time;
static inline void nvmap_ref_lock(struct nvmap_client *priv)
{
mutex_lock(&priv->ref_lock);
}
static inline void nvmap_ref_unlock(struct nvmap_client *priv)
{
mutex_unlock(&priv->ref_lock);
}
static inline void nvmap_acquire_mmap_read_lock(struct mm_struct *mm)
{
#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0)
down_read(&mm->mmap_sem);
#else
down_read(&mm->mmap_lock);
#endif
}
static inline void nvmap_release_mmap_read_lock(struct mm_struct *mm)
{
#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0)
up_read(&mm->mmap_sem);
#else
up_read(&mm->mmap_lock);
#endif
}
static inline pgprot_t nvmap_pgprot(struct nvmap_handle *h, pgprot_t prot)
{
if (h->flags == NVMAP_HANDLE_UNCACHEABLE) {
#ifdef CONFIG_ARM64
if (h->heap_type != NVMAP_HEAP_CARVEOUT_VPR &&
h->owner && !h->owner->warned) {
char task_comm[TASK_COMM_LEN];
h->owner->warned = 1;
get_task_comm(task_comm, h->owner->task);
pr_err("PID %d: %s: TAG: 0x%04x WARNING: "
"NVMAP_HANDLE_WRITE_COMBINE "
"should be used in place of "
"NVMAP_HANDLE_UNCACHEABLE on ARM64\n",
h->owner->task->pid, task_comm,
h->userflags >> 16);
}
#endif
return pgprot_noncached(prot);
}
else if (h->flags == NVMAP_HANDLE_WRITE_COMBINE)
return pgprot_writecombine(prot);
return prot;
}
#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)
struct dma_coherent_mem_replica {
void *virt_base;
dma_addr_t device_base;
unsigned long pfn_base;
#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0)
size_t size;
#else
int size;
#endif
int flags;
unsigned long *bitmap;
spinlock_t spinlock;
bool use_dev_dma_pfn_offset;
};
int nvmap_dma_declare_coherent_memory(struct device *dev, phys_addr_t phys_addr,
dma_addr_t device_addr, size_t size, int flags);
#endif
int nvmap_probe(struct platform_device *pdev);
int nvmap_remove(struct platform_device *pdev);
int nvmap_init(struct platform_device *pdev);
int nvmap_create_carveout(const struct nvmap_platform_carveout *co);
int nvmap_co_setup(struct reserved_mem *rmem);
struct device *dma_dev_from_handle(unsigned long type);
struct nvmap_heap_block *nvmap_carveout_alloc(struct nvmap_client *dev,
struct nvmap_handle *handle,
unsigned long type,
phys_addr_t *start);
struct nvmap_carveout_node;
struct nvmap_handle *nvmap_handle_get(struct nvmap_handle *h);
void nvmap_handle_put(struct nvmap_handle *h);
struct nvmap_handle_ref *__nvmap_validate_locked(struct nvmap_client *priv,
struct nvmap_handle *h,
bool is_ro);
struct nvmap_handle *nvmap_validate_get(struct nvmap_handle *h);
struct nvmap_handle_ref *nvmap_create_handle(struct nvmap_client *client,
size_t size, bool ro_buf);
struct nvmap_handle_ref *nvmap_create_handle_from_va(struct nvmap_client *client,
ulong addr, size_t size,
unsigned int access_flags);
struct nvmap_handle_ref *nvmap_dup_handle_ro(struct nvmap_client *client,
int fd);
bool is_nvmap_dmabuf_fd_ro(int fd);
bool is_nvmap_id_ro(struct nvmap_client *client, int id);
struct nvmap_handle_ref *nvmap_duplicate_handle(struct nvmap_client *client,
struct nvmap_handle *h, bool skip_val,
bool is_ro);
struct nvmap_handle_ref *nvmap_try_duplicate_by_ivmid(
struct nvmap_client *client, u64 ivm_id,
struct nvmap_heap_block **block);
struct nvmap_handle_ref *nvmap_create_handle_from_id(
struct nvmap_client *client, u32 id);
struct nvmap_handle_ref *nvmap_create_handle_from_fd(
struct nvmap_client *client, int fd);
void inner_cache_maint(unsigned int op, void *vaddr, size_t size);
void outer_cache_maint(unsigned int op, phys_addr_t paddr, size_t size);
int nvmap_alloc_handle(struct nvmap_client *client,
struct nvmap_handle *h, unsigned int heap_mask,
size_t align, u8 kind,
unsigned int flags, unsigned int peer);
int nvmap_alloc_handle_from_va(struct nvmap_client *client,
struct nvmap_handle *h,
ulong addr,
unsigned int flags);
void nvmap_free_handle(struct nvmap_client *c, struct nvmap_handle *h, bool is_ro);
void nvmap_free_handle_from_fd(struct nvmap_client *c, int fd);
int nvmap_handle_remove(struct nvmap_device *dev, struct nvmap_handle *h);
void nvmap_handle_add(struct nvmap_device *dev, struct nvmap_handle *h);
int is_nvmap_vma(struct vm_area_struct *vma);
int nvmap_get_dmabuf_fd(struct nvmap_client *client, struct nvmap_handle *h,
bool is_ro);
struct nvmap_handle *nvmap_handle_get_from_dmabuf_fd(
struct nvmap_client *client, int fd);
int nvmap_dmabuf_duplicate_gen_fd(struct nvmap_client *client,
struct dma_buf *dmabuf);
int nvmap_get_handle_param(struct nvmap_client *client,
struct nvmap_handle_ref *ref, u32 param, u64 *result);
struct nvmap_handle *nvmap_handle_get_from_fd(int fd);
/* MM definitions. */
extern void v7_flush_kern_cache_all(void);
extern void v7_clean_kern_cache_all(void *);
void nvmap_clean_cache(struct page **pages, int numpages);
void nvmap_clean_cache_page(struct page *page);
void nvmap_flush_cache(struct page **pages, int numpages);
int nvmap_cache_maint_phys_range(unsigned int op, phys_addr_t pstart,
phys_addr_t pend, int inner, int outer);
int nvmap_do_cache_maint_list(struct nvmap_handle **handles, u64 *offsets,
u64 *sizes, int op, u32 nr_ops, bool is_32);
int __nvmap_cache_maint(struct nvmap_client *client,
struct nvmap_cache_op_64 *op);
/* Internal API to support dmabuf */
struct dma_buf *__nvmap_make_dmabuf(struct nvmap_client *client,
struct nvmap_handle *handle, bool ro_buf);
struct sg_table *__nvmap_sg_table(struct nvmap_client *client,
struct nvmap_handle *h);
void __nvmap_free_sg_table(struct nvmap_client *client,
struct nvmap_handle *h, struct sg_table *sgt);
#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0)
void *__nvmap_kmap(struct nvmap_handle *h, unsigned int pagenum);
void __nvmap_kunmap(struct nvmap_handle *h, unsigned int pagenum, void *addr);
#endif
void *__nvmap_mmap(struct nvmap_handle *h);
void __nvmap_munmap(struct nvmap_handle *h, void *addr);
int __nvmap_map(struct nvmap_handle *h, struct vm_area_struct *vma);
int __nvmap_do_cache_maint(struct nvmap_client *client, struct nvmap_handle *h,
unsigned long start, unsigned long end,
unsigned int op, bool clean_only_dirty);
struct nvmap_client *__nvmap_create_client(struct nvmap_device *dev,
const char *name);
int __nvmap_dmabuf_fd(struct nvmap_client *client,
struct dma_buf *dmabuf, int flags);
int nvmap_dmabuf_stash_init(void);
void *nvmap_altalloc(size_t len);
void nvmap_altfree(void *ptr, size_t len);
static inline struct page *nvmap_to_page(struct page *page)
{
return (struct page *)((unsigned long)page & ~3UL);
}
static inline bool nvmap_page_dirty(struct page *page)
{
return (unsigned long)page & 1UL;
}
static inline bool nvmap_page_mkdirty(struct page **page)
{
if (nvmap_page_dirty(*page))
return false;
*page = (struct page *)((unsigned long)*page | 1UL);
return true;
}
static inline bool nvmap_page_mkclean(struct page **page)
{
if (!nvmap_page_dirty(*page))
return false;
*page = (struct page *)((unsigned long)*page & ~1UL);
return true;
}
/*
* FIXME: assume user space requests for reserve operations
* are page aligned
*/
static inline int nvmap_handle_mk(struct nvmap_handle *h,
u32 offset, u32 size,
bool (*fn)(struct page **),
bool locked)
{
int i, nchanged = 0;
u32 start_page = offset >> PAGE_SHIFT;
u32 end_page = PAGE_ALIGN(offset + size) >> PAGE_SHIFT;
if (!locked)
mutex_lock(&h->lock);
if (h->heap_pgalloc &&
(offset < h->size) &&
(size <= h->size) &&
(offset <= (h->size - size))) {
for (i = start_page; i < end_page; i++)
nchanged += fn(&h->pgalloc.pages[i]) ? 1 : 0;
}
if (!locked)
mutex_unlock(&h->lock);
return nchanged;
}
static inline void nvmap_handle_mkclean(struct nvmap_handle *h,
u32 offset, u32 size)
{
int nchanged;
if (h->heap_pgalloc && !atomic_read(&h->pgalloc.ndirty))
return;
if (size == 0)
size = h->size;
nchanged = nvmap_handle_mk(h, offset, size, nvmap_page_mkclean, false);
if (h->heap_pgalloc)
atomic_sub(nchanged, &h->pgalloc.ndirty);
}
static inline void _nvmap_handle_mkdirty(struct nvmap_handle *h,
u32 offset, u32 size)
{
int nchanged;
if (h->heap_pgalloc &&
(atomic_read(&h->pgalloc.ndirty) == (h->size >> PAGE_SHIFT)))
return;
nchanged = nvmap_handle_mk(h, offset, size, nvmap_page_mkdirty, true);
if (h->heap_pgalloc)
atomic_add(nchanged, &h->pgalloc.ndirty);
}
static inline struct page **nvmap_pages(struct page **pg_pages, u32 nr_pages)
{
struct page **pages;
int i;
pages = nvmap_altalloc(sizeof(*pages) * nr_pages);
if (!pages)
return NULL;
for (i = 0; i < nr_pages; i++)
pages[i] = nvmap_to_page(pg_pages[i]);
return pages;
}
void nvmap_zap_handle(struct nvmap_handle *handle, u64 offset, u64 size);
void nvmap_vma_open(struct vm_area_struct *vma);
int nvmap_reserve_pages(struct nvmap_handle **handles, u64 *offsets,
u64 *sizes, u32 nr, u32 op, bool is_32);
static inline void nvmap_kmaps_inc(struct nvmap_handle *h)
{
mutex_lock(&h->lock);
atomic_inc(&h->kmap_count);
mutex_unlock(&h->lock);
}
static inline void nvmap_kmaps_inc_no_lock(struct nvmap_handle *h)
{
atomic_inc(&h->kmap_count);
}
static inline void nvmap_kmaps_dec(struct nvmap_handle *h)
{
atomic_dec(&h->kmap_count);
}
static inline void nvmap_umaps_inc(struct nvmap_handle *h)
{
mutex_lock(&h->lock);
atomic_inc(&h->umap_count);
mutex_unlock(&h->lock);
}
static inline void nvmap_umaps_dec(struct nvmap_handle *h)
{
atomic_dec(&h->umap_count);
}
static inline void nvmap_lru_add(struct nvmap_handle *h)
{
spin_lock(&nvmap_dev->lru_lock);
BUG_ON(!list_empty(&h->lru));
list_add_tail(&h->lru, &nvmap_dev->lru_handles);
spin_unlock(&nvmap_dev->lru_lock);
}
static inline void nvmap_lru_del(struct nvmap_handle *h)
{
spin_lock(&nvmap_dev->lru_lock);
list_del(&h->lru);
INIT_LIST_HEAD(&h->lru);
spin_unlock(&nvmap_dev->lru_lock);
}
static inline void nvmap_lru_reset(struct nvmap_handle *h)
{
spin_lock(&nvmap_dev->lru_lock);
BUG_ON(list_empty(&h->lru));
list_del(&h->lru);
list_add_tail(&h->lru, &nvmap_dev->lru_handles);
spin_unlock(&nvmap_dev->lru_lock);
}
static inline bool nvmap_handle_track_dirty(struct nvmap_handle *h)
{
if (!h->heap_pgalloc)
return false;
return h->userflags & (NVMAP_HANDLE_CACHE_SYNC |
NVMAP_HANDLE_CACHE_SYNC_AT_RESERVE);
}
struct nvmap_tag_entry *nvmap_search_tag_entry(struct rb_root *root, u32 tag);
int nvmap_define_tag(struct nvmap_device *dev, u32 tag,
const char __user *name, u32 len);
int nvmap_remove_tag(struct nvmap_device *dev, u32 tag);
/* must hold tag_lock */
static inline char *__nvmap_tag_name(struct nvmap_device *dev, u32 tag)
{
struct nvmap_tag_entry *entry;
entry = nvmap_search_tag_entry(&dev->tags, tag);
return entry ? (char *)(entry + 1) : "";
}
static inline pid_t nvmap_client_pid(struct nvmap_client *client)
{
return client->task ? client->task->pid : 0;
}
/* must be called with mmap_sem held for read or write */
static inline int nvmap_get_user_pages(ulong vaddr,
size_t nr_page, struct page **pages,
bool is_user_flags, u32 user_foll_flags)
{
u32 foll_flags = FOLL_FORCE;
struct vm_area_struct *vma;
vm_flags_t vm_flags;
long user_pages = 0;
int ret = 0;
vma = find_vma(current->mm, vaddr);
if (vma) {
if (is_user_flags) {
foll_flags |= user_foll_flags;
} else {
vm_flags = vma->vm_flags;
/*
* If the vaddr points to writable page then only
* pass FOLL_WRITE flag
*/
if (vm_flags & VM_WRITE)
foll_flags |= FOLL_WRITE;
}
pr_debug("vaddr %lu is_user_flags %d user_foll_flags %x foll_flags %x.\n",
vaddr, is_user_flags?1:0, user_foll_flags, foll_flags);
user_pages = get_user_pages(vaddr & PAGE_MASK, nr_page,
foll_flags, pages, NULL);
}
if (user_pages != nr_page) {
ret = user_pages < 0 ? user_pages : -ENOMEM;
pr_err("get_user_pages requested/got: %zu/%ld]\n", nr_page,
user_pages);
while (--user_pages >= 0)
put_page(pages[user_pages]);
}
return ret;
}
#define device_node_from_iter(iter) \
iter.node
extern struct of_device_id __nvmapcache_of_table;
#define NVMAP_CACHE_OF_DECLARE(compat, fn) \
_OF_DECLARE(nvmapcache, nvmapcache_of, compat, fn, \
nvmap_setup_chip_cache_fn)
#ifdef NVMAP_CONFIG_SCIIPC
int nvmap_sci_ipc_init(void);
void nvmap_sci_ipc_exit(void);
#else
__weak int nvmap_sci_ipc_init(void)
{
return 0;
}
__weak void nvmap_sci_ipc_exit(void)
{
}
#endif
#ifdef NVMAP_CONFIG_HANDLE_AS_ID
void nvmap_id_array_init(struct xarray *xarr);
void nvmap_id_array_exit(struct xarray *xarr);
struct dma_buf *nvmap_id_array_get_dmabuf_from_id(struct xarray *xarr, u32 id);
int nvmap_id_array_id_alloc(struct xarray *xarr, u32 *id, struct dma_buf *dmabuf);
struct dma_buf *nvmap_id_array_id_release(struct xarray *xarr, u32 id);
#else
static inline void nvmap_id_array_init(struct xarray *xarr)
{
}
static inline void nvmap_id_array_exit(struct xarray *xarr)
{
}
static inline struct dma_buf *nvmap_id_array_get_dmabuf_from_id(struct xarray *xarr, u32 id)
{
return NULL;
}
static inline int nvmap_id_array_id_alloc(struct xarray *xarr, u32 *id, struct dma_buf *dmabuf)
{
return 0;
}
static inline struct dma_buf *nvmap_id_array_id_release(struct xarray *xarr, u32 id)
{
return NULL;
}
#endif
void *nvmap_dmabuf_get_drv_data(struct dma_buf *dmabuf,
struct device *dev);
bool is_nvmap_memory_available(size_t size, uint32_t heap);
int system_heap_free_mem(unsigned long *mem_val);
#ifdef NVMAP_CONFIG_DEBUG_MAPS
struct nvmap_device_list *nvmap_is_device_present(char *device_name, u32 heap_type);
void nvmap_add_device_name(char *device_name, u64 dma_mask, u32 heap_type);
void nvmap_remove_device_name(char *device_name, u32 heap_type);
#endif /* NVMAP_CONFIG_DEBUG_MAPS */
bool dmabuf_is_nvmap(struct dma_buf *dmabuf);
struct nvmap_handle *nvmap_handle_get_from_id(struct nvmap_client *client,
u32 id);
void *nvmap_dma_mark_declared_memory_occupied(struct device *dev,
dma_addr_t device_addr, size_t size);
void nvmap_dma_mark_declared_memory_unoccupied(struct device *dev,
dma_addr_t device_addr, size_t size);
#endif /* __VIDEO_TEGRA_NVMAP_NVMAP_H */

365
drivers/video/tegra/nvmap/nvmap_sci_ipc.c Normal file
View File

@@ -0,0 +1,365 @@
/*
* drivers/video/tegra/nvmap/nvmap_sci_ipc.c
*
* mapping between nvmap_hnadle and sci_ipc entery
*
* Copyright (c) 2019-2022, NVIDIA CORPORATION. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*/
#define pr_fmt(fmt) "%s: " fmt, __func__
#include <linux/slab.h>
#include <linux/nvmap.h>
#include <linux/rbtree.h>
#include <linux/list.h>
#include <linux/mman.h>
#include <linux/nvscierror.h>
#include <linux/nvsciipc_interface.h>
#include <trace/events/nvmap.h>
#include "nvmap_priv.h"
#include "nvmap_sci_ipc.h"
struct nvmap_sci_ipc {
struct rb_root entries;
struct mutex mlock;
struct list_head free_sid_list;
};
struct free_sid_node {
struct list_head list;
u32 sid;
};
/* An rb-tree root node for holding sci_ipc_id of clients */
struct nvmap_sci_ipc_entry {
struct rb_node entry;
struct nvmap_client *client;
struct nvmap_handle *handle;
u32 sci_ipc_id;
u64 peer_vuid;
u32 flags;
u32 refcount;
};
static struct nvmap_sci_ipc *nvmapsciipc;
int nvmap_validate_sci_ipc_params(struct nvmap_client *client,
NvSciIpcEndpointAuthToken auth_token,
NvSciIpcEndpointVuid *pr_vuid,
NvSciIpcEndpointVuid *lu_vuid)
{
NvSciError err = NvSciError_Success;
NvSciIpcTopoId pr_topoid;
int ret = 0;
err = NvSciIpcEndpointValidateAuthTokenLinuxCurrent(auth_token,
lu_vuid);
if (err != NvSciError_Success) {
ret = -EINVAL;
goto out;
}
err = NvSciIpcEndpointMapVuid(*lu_vuid, &pr_topoid, pr_vuid);
if (err != NvSciError_Success) {
ret = -EINVAL;
goto out;
}
out:
return ret;
}
static u32 nvmap_unique_sci_ipc_id(void)
{
static atomic_t unq_id = { 0 };
u32 id;
if (!list_empty(&nvmapsciipc->free_sid_list)) {
struct free_sid_node *fnode = list_first_entry(
&nvmapsciipc->free_sid_list,
typeof(*fnode),
list);
id = fnode->sid;
list_del(&fnode->list);
kfree(fnode);
goto ret_id;
}
id = atomic_add_return(2, &unq_id);
ret_id:
WARN_ON(id == 0);
return id;
}
static struct nvmap_sci_ipc_entry *nvmap_search_sci_ipc_entry(
struct rb_root *root,
struct nvmap_handle *h,
u32 flags,
NvSciIpcEndpointVuid peer_vuid)
{
struct rb_node *node; /* top of the tree */
struct nvmap_sci_ipc_entry *entry;
for (node = rb_first(root); node; node = rb_next(node)) {
entry = rb_entry(node, struct nvmap_sci_ipc_entry, entry);
if (entry && entry->handle == h
&& entry->flags == flags
&& entry->peer_vuid == peer_vuid)
return entry;
}
return NULL;
}
static void nvmap_insert_sci_ipc_entry(struct rb_root *root,
struct nvmap_sci_ipc_entry *new)
{
struct nvmap_sci_ipc_entry *entry;
struct rb_node *parent = NULL;
u32 sid = new->sci_ipc_id;
struct rb_node **link;
link = &root->rb_node;
/* Go to the bottom of the tree */
while (*link) {
parent = *link;
entry = rb_entry(parent, struct nvmap_sci_ipc_entry, entry);
if (entry->sci_ipc_id > sid)
link = &parent->rb_left;
else
link = &parent->rb_right;
}
/* Put the new node there */
rb_link_node(&new->entry, parent, link);
rb_insert_color(&new->entry, root);
}
int nvmap_create_sci_ipc_id(struct nvmap_client *client,
struct nvmap_handle *h,
u32 flags,
u32 *sci_ipc_id,
NvSciIpcEndpointVuid peer_vuid,
bool is_ro)
{
struct nvmap_sci_ipc_entry *new_entry;
struct nvmap_sci_ipc_entry *entry;
int ret = -EINVAL;
u32 id;
mutex_lock(&nvmapsciipc->mlock);
entry = nvmap_search_sci_ipc_entry(&nvmapsciipc->entries,
h, flags, peer_vuid);
if (entry) {
entry->refcount++;
*sci_ipc_id = entry->sci_ipc_id;
pr_debug("%d: matched Sci_Ipc_Id:%u\n", __LINE__, *sci_ipc_id);
ret = 0;
goto unlock;
} else {
new_entry = kzalloc(sizeof(*new_entry), GFP_KERNEL);
if (!new_entry) {
ret = -ENOMEM;
goto unlock;
}
id = nvmap_unique_sci_ipc_id();
*sci_ipc_id = id;
new_entry->sci_ipc_id = id;
new_entry->client = client;
new_entry->handle = h;
new_entry->peer_vuid = peer_vuid;
new_entry->flags = flags;
new_entry->refcount = 1;
pr_debug("%d: New Sci_ipc_id %d entry->pr_vuid: %llu entry->flags: %u new_entry->handle:0x%p\n",
__LINE__, new_entry->sci_ipc_id, new_entry->peer_vuid,
new_entry->flags, new_entry->handle);
nvmap_insert_sci_ipc_entry(&nvmapsciipc->entries, new_entry);
ret = 0;
}
unlock:
mutex_unlock(&nvmapsciipc->mlock);
if (!ret)
(void)nvmap_handle_get(h);
return ret;
}
static struct nvmap_sci_ipc_entry *nvmap_find_entry_for_id(struct rb_root *es,
u32 id)
{
struct nvmap_sci_ipc_entry *e = NULL;
struct rb_node *n;
for (n = rb_first(es); n; n = rb_next(n)) {
e = rb_entry(n, struct nvmap_sci_ipc_entry, entry);
if (e && e->sci_ipc_id == id)
goto found;
}
found:
return e;
}
int nvmap_get_handle_from_sci_ipc_id(struct nvmap_client *client, u32 flags,
u32 sci_ipc_id, NvSciIpcEndpointVuid localu_vuid, u32 *handle)
{
bool is_ro = (flags == PROT_READ) ? true : false, dmabuf_created = false;
struct nvmap_handle_ref *ref = NULL;
struct nvmap_sci_ipc_entry *entry;
struct dma_buf *dmabuf = NULL;
struct nvmap_handle *h;
int ret = 0;
int fd;
mutex_lock(&nvmapsciipc->mlock);
pr_debug("%d: Sci_Ipc_Id %d local_vuid: %llu flags: %u\n",
__LINE__, sci_ipc_id, localu_vuid, flags);
entry = nvmap_find_entry_for_id(&nvmapsciipc->entries, sci_ipc_id);
if ((entry == NULL) || (entry->handle == NULL) ||
(entry->peer_vuid != localu_vuid) || (entry->flags != flags)) {
pr_debug("%d: No matching Sci_Ipc_Id %d found\n",
__LINE__, sci_ipc_id);
ret = -EINVAL;
goto unlock;
}
h = entry->handle;
if (is_ro && h->dmabuf_ro == NULL) {
h->dmabuf_ro = __nvmap_make_dmabuf(client, h, true);
if (IS_ERR(h->dmabuf_ro)) {
ret = PTR_ERR(h->dmabuf_ro);
goto unlock;
}
dmabuf_created = true;
}
ref = nvmap_duplicate_handle(client, h, false, is_ro);
if (!ref) {
ret = -EINVAL;
goto unlock;
}
nvmap_handle_put(h);
/*
* When new dmabuf created (only RO dmabuf is getting created in this function)
* it's counter is incremented one extra time in nvmap_duplicate_handle. Hence
* decrement it by one.
*/
if (dmabuf_created)
dma_buf_put(h->dmabuf_ro);
if (!IS_ERR(ref)) {
u32 id = 0;
dmabuf = is_ro ? h->dmabuf_ro : h->dmabuf;
if (client->ida) {
if (nvmap_id_array_id_alloc(client->ida, &id, dmabuf) < 0) {
if (dmabuf)
dma_buf_put(dmabuf);
nvmap_free_handle(client, h, is_ro);
ret = -ENOMEM;
goto unlock;
}
if (!id)
*handle = 0;
else
*handle = id;
} else {
fd = nvmap_get_dmabuf_fd(client, h, is_ro);
if (IS_ERR_VALUE((uintptr_t)fd)) {
if (dmabuf)
dma_buf_put(dmabuf);
nvmap_free_handle(client, h, is_ro);
ret = -EINVAL;
goto unlock;
}
*handle = fd;
fd_install(fd, dmabuf->file);
}
}
entry->refcount--;
if (entry->refcount == 0U) {
struct free_sid_node *free_node;
rb_erase(&entry->entry, &nvmapsciipc->entries);
free_node = kzalloc(sizeof(*free_node), GFP_KERNEL);
if (free_node == NULL) {
ret = -ENOMEM;
kfree(entry);
goto unlock;
}
free_node->sid = entry->sci_ipc_id;
list_add_tail(&free_node->list, &nvmapsciipc->free_sid_list);
kfree(entry);
}
unlock:
mutex_unlock(&nvmapsciipc->mlock);
if (!ret) {
if (!client->ida)
trace_refcount_create_handle_from_sci_ipc_id(h, dmabuf,
atomic_read(&h->ref),
atomic_long_read(&dmabuf->file->f_count),
is_ro ? "RO" : "RW");
else
trace_refcount_get_handle_from_sci_ipc_id(h, dmabuf,
atomic_read(&h->ref),
is_ro ? "RO" : "RW");
}
return ret;
}
int nvmap_sci_ipc_init(void)
{
nvmapsciipc = kzalloc(sizeof(*nvmapsciipc), GFP_KERNEL);
if (!nvmapsciipc)
return -ENOMEM;
nvmapsciipc->entries = RB_ROOT;
INIT_LIST_HEAD(&nvmapsciipc->free_sid_list);
mutex_init(&nvmapsciipc->mlock);
return 0;
}
void nvmap_sci_ipc_exit(void)
{
struct nvmap_sci_ipc_entry *e;
struct free_sid_node *fnode, *temp;
struct rb_node *n;
mutex_lock(&nvmapsciipc->mlock);
while ((n = rb_first(&nvmapsciipc->entries))) {
e = rb_entry(n, struct nvmap_sci_ipc_entry, entry);
rb_erase(&e->entry, &nvmapsciipc->entries);
kfree(e);
}
list_for_each_entry_safe(fnode, temp, &nvmapsciipc->free_sid_list, list) {
list_del(&fnode->list);
kfree(fnode);
}
mutex_unlock(&nvmapsciipc->mlock);
kfree(nvmapsciipc);
nvmapsciipc = NULL;
}

37
drivers/video/tegra/nvmap/nvmap_sci_ipc.h Normal file
View File

@@ -0,0 +1,37 @@
/*
* drivers/video/tegra/nvmap/nvmap_sci_ipc.c
*
* mapping between nvmap_hnadle and sci_ipc entery
*
* Copyright (c) 2019-2021, NVIDIA CORPORATION. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*/
#ifndef __VIDEO_TEGRA_NVMAP_SCI_IPC_H
#define __VIDEO_TEGRA_NVMAP_SCI_IPC_H
int nvmap_validate_sci_ipc_params(struct nvmap_client *client,
NvSciIpcEndpointAuthToken auth_token,
NvSciIpcEndpointVuid *pr_vuid,
NvSciIpcEndpointVuid *localusr_vuid);
int nvmap_create_sci_ipc_id(struct nvmap_client *client,
struct nvmap_handle *h,
u32 flags,
u32 *sci_ipc_id,
NvSciIpcEndpointVuid pr_vuid,
bool is_ro);
int nvmap_get_handle_from_sci_ipc_id(struct nvmap_client *client,
u32 flags,
u32 sci_ipc_id,
NvSciIpcEndpointVuid localusr_vuid,
u32 *h);
#endif /* __VIDEO_TEGRA_NVMAP_SCI_IPC_H */

106
drivers/video/tegra/nvmap/nvmap_stats.c Normal file
View File

@@ -0,0 +1,106 @@
/*
* drivers/video/tegra/nvmap/nvmap_stats.c
*
* Nvmap Stats keeping
*
* Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*/
#include <linux/debugfs.h>
#include "nvmap_priv.h"
struct nvmap_stats nvmap_stats;
static int nvmap_stats_reset(void *data, u64 val)
{
int i;
if (val) {
atomic64_set(&nvmap_stats.collect, 0);
for (i = 0; i < NS_NUM; i++) {
if (i == NS_TOTAL)
continue;
atomic64_set(&nvmap_stats.stats[i], 0);
}
}
return 0;
}
static int nvmap_stats_get(void *data, u64 *val)
{
atomic64_t *ptr = data;
*val = atomic64_read(ptr);
return 0;
}
static int nvmap_stats_set(void *data, u64 val)
{
atomic64_t *ptr = data;
atomic64_set(ptr, val);
return 0;
}
DEFINE_SIMPLE_ATTRIBUTE(reset_stats_fops, NULL, nvmap_stats_reset, "%llu\n");
DEFINE_SIMPLE_ATTRIBUTE(stats_fops, nvmap_stats_get, nvmap_stats_set, "%llu\n");
void nvmap_stats_init(struct dentry *nvmap_debug_root)
{
struct dentry *stats_root;
#define CREATE_DF(x, y) \
debugfs_create_file(#x, S_IRUGO, stats_root, &y, &stats_fops);
stats_root = debugfs_create_dir("stats", nvmap_debug_root);
if (!IS_ERR_OR_NULL(stats_root)) {
CREATE_DF(alloc, nvmap_stats.stats[NS_ALLOC]);
CREATE_DF(release, nvmap_stats.stats[NS_RELEASE]);
CREATE_DF(ualloc, nvmap_stats.stats[NS_UALLOC]);
CREATE_DF(urelease, nvmap_stats.stats[NS_URELEASE]);
CREATE_DF(kalloc, nvmap_stats.stats[NS_KALLOC]);
CREATE_DF(krelease, nvmap_stats.stats[NS_KRELEASE]);
CREATE_DF(cflush_rq, nvmap_stats.stats[NS_CFLUSH_RQ]);
CREATE_DF(cflush_done, nvmap_stats.stats[NS_CFLUSH_DONE]);
CREATE_DF(ucflush_rq, nvmap_stats.stats[NS_UCFLUSH_RQ]);
CREATE_DF(ucflush_done, nvmap_stats.stats[NS_UCFLUSH_DONE]);
CREATE_DF(kcflush_rq, nvmap_stats.stats[NS_KCFLUSH_RQ]);
CREATE_DF(kcflush_done, nvmap_stats.stats[NS_KCFLUSH_DONE]);
CREATE_DF(total_memory, nvmap_stats.stats[NS_TOTAL]);
debugfs_create_file("collect", S_IRUGO | S_IWUSR,
stats_root, &nvmap_stats.collect, &stats_fops);
debugfs_create_file("reset", S_IWUSR,
stats_root, NULL, &reset_stats_fops);
}
#undef CREATE_DF
}
void nvmap_stats_inc(enum nvmap_stats_t stat, size_t size)
{
if (atomic64_read(&nvmap_stats.collect) || stat == NS_TOTAL)
atomic64_add(size, &nvmap_stats.stats[stat]);
}
void nvmap_stats_dec(enum nvmap_stats_t stat, size_t size)
{
if (atomic64_read(&nvmap_stats.collect) || stat == NS_TOTAL)
atomic64_sub(size, &nvmap_stats.stats[stat]);
}
u64 nvmap_stats_read(enum nvmap_stats_t stat)
{
return atomic64_read(&nvmap_stats.stats[stat]);
}

45
drivers/video/tegra/nvmap/nvmap_stats.h Normal file
View File

@@ -0,0 +1,45 @@
/*
* Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*/
#ifndef __VIDEO_TEGRA_NVMAP_STATS_H
#define __VIDEO_TEGRA_NVMAP_STATS_H
enum nvmap_stats_t {
NS_ALLOC = 0,
NS_RELEASE,
NS_UALLOC,
NS_URELEASE,
NS_KALLOC,
NS_KRELEASE,
NS_CFLUSH_RQ,
NS_CFLUSH_DONE,
NS_UCFLUSH_RQ,
NS_UCFLUSH_DONE,
NS_KCFLUSH_RQ,
NS_KCFLUSH_DONE,
NS_TOTAL,
NS_NUM,
};
struct nvmap_stats {
atomic64_t stats[NS_NUM];
atomic64_t collect;
};
extern struct nvmap_stats nvmap_stats;
void nvmap_stats_init(struct dentry *nvmap_debug_root);
void nvmap_stats_inc(enum nvmap_stats_t, size_t size);
void nvmap_stats_dec(enum nvmap_stats_t, size_t size);
u64 nvmap_stats_read(enum nvmap_stats_t);
#endif /* __VIDEO_TEGRA_NVMAP_STATS_H */

113
drivers/video/tegra/nvmap/nvmap_tag.c Normal file
View File

@@ -0,0 +1,113 @@
/*
* drivers/video/tegra/nvmap/nvmap_tag.c
*
* Allocation tag routines for nvmap
*
* Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*/
#define pr_fmt(fmt) "%s: " fmt, __func__
#include <linux/moduleparam.h>
#include <trace/events/nvmap.h>
#include "nvmap_priv.h"
struct nvmap_tag_entry *nvmap_search_tag_entry(struct rb_root *root, u32 tag)
{
struct rb_node *node = root->rb_node; /* top of the tree */
struct nvmap_tag_entry *entry;
while (node) {
entry = rb_entry(node, struct nvmap_tag_entry, node);
if (entry->tag > tag)
node = node->rb_left;
else if (entry->tag < tag)
node = node->rb_right;
else
return entry; /* Found it */
}
return NULL;
}
static void nvmap_insert_tag_entry(struct rb_root *root,
struct nvmap_tag_entry *new)
{
struct rb_node **link = &root->rb_node;
struct rb_node *parent = NULL;
struct nvmap_tag_entry *entry;
u32 tag = new->tag;
/* Go to the bottom of the tree */
while (*link) {
parent = *link;
entry = rb_entry(parent, struct nvmap_tag_entry, node);
if (entry->tag > tag)
link = &parent->rb_left;
else
link = &parent->rb_right;
}
/* Put the new node there */
rb_link_node(&new->node, parent, link);
rb_insert_color(&new->node, root);
}
int nvmap_define_tag(struct nvmap_device *dev, u32 tag,
const char __user *name, u32 len)
{
struct nvmap_tag_entry *new;
struct nvmap_tag_entry *old;
new = kzalloc(sizeof(struct nvmap_tag_entry) + len + 1, GFP_KERNEL);
if (!new)
return -ENOMEM;
if (copy_from_user(new + 1, name, len)) {
kfree(new);
return -EFAULT;
}
new->tag = tag;
mutex_lock(&dev->tags_lock);
old = nvmap_search_tag_entry(&dev->tags, tag);
if (old) {
rb_replace_node(&old->node, &new->node, &dev->tags);
kfree(old);
} else {
nvmap_insert_tag_entry(&dev->tags, new);
}
mutex_unlock(&dev->tags_lock);
return 0;
}
int nvmap_remove_tag(struct nvmap_device *dev, u32 tag)
{
struct nvmap_tag_entry *old;
mutex_lock(&dev->tags_lock);
old = nvmap_search_tag_entry(&dev->tags, tag);
if (old){
rb_erase(&old->node, &dev->tags);
kfree(old);
}
mutex_unlock(&dev->tags_lock);
return 0;
}

113
include/linux/nvmap.h Normal file
View File

@@ -0,0 +1,113 @@
/*
* include/linux/nvmap.h
*
* structure declarations for nvmem and nvmap user-space ioctls
*
* Copyright (c) 2009-2022, NVIDIA CORPORATION. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*/
#ifndef _LINUX_NVMAP_H
#define _LINUX_NVMAP_H
#include <linux/rbtree.h>
#include <linux/file.h>
#include <linux/dma-buf.h>
#include <linux/device.h>
#include <linux/version.h>
#include <uapi/linux/nvmap.h>
#define NVMAP_HEAP_IOVMM (1ul<<30)
/* common carveout heaps */
#define NVMAP_HEAP_CARVEOUT_VPR (1ul<<28)
#define NVMAP_HEAP_CARVEOUT_TSEC (1ul<<27)
#define NVMAP_HEAP_CARVEOUT_VIDMEM (1ul<<26)
#define NVMAP_HEAP_CARVEOUT_FSI (1ul<<2)
#define NVMAP_HEAP_CARVEOUT_IVM (1ul<<1)
#define NVMAP_HEAP_CARVEOUT_GENERIC (1ul<<0)
#define NVMAP_HEAP_CARVEOUT_MASK (NVMAP_HEAP_IOVMM - 1)
/* allocation flags */
#define NVMAP_HANDLE_UNCACHEABLE (0x0ul << 0)
#define NVMAP_HANDLE_WRITE_COMBINE (0x1ul << 0)
#define NVMAP_HANDLE_INNER_CACHEABLE (0x2ul << 0)
#define NVMAP_HANDLE_CACHEABLE (0x3ul << 0)
#define NVMAP_HANDLE_CACHE_FLAG (0x3ul << 0)
#define NVMAP_HANDLE_SECURE (0x1ul << 2)
#define NVMAP_HANDLE_KIND_SPECIFIED (0x1ul << 3)
#define NVMAP_HANDLE_COMPR_SPECIFIED (0x1ul << 4)
#define NVMAP_HANDLE_ZEROED_PAGES (0x1ul << 5)
#define NVMAP_HANDLE_PHYS_CONTIG (0x1ul << 6)
#define NVMAP_HANDLE_CACHE_SYNC (0x1ul << 7)
#define NVMAP_HANDLE_CACHE_SYNC_AT_RESERVE (0x1ul << 8)
#define NVMAP_HANDLE_RO (0x1ul << 9)
#ifdef NVMAP_CONFIG_PAGE_POOLS
ulong nvmap_page_pool_get_unused_pages(void);
#else
static inline ulong nvmap_page_pool_get_unused_pages(void)
{
return 0;
}
#endif
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 15, 0)
ulong nvmap_iovmm_get_used_pages(void);
#endif
int nvmap_register_vidmem_carveout(struct device *dma_dev,
phys_addr_t base, size_t size);
/*
* A heap can be mapped to memory other than DRAM.
* The HW, controls the memory, can be power gated/ungated
* based upon the clients using the memory.
* if no client/alloc happens from the memory, the HW needs
* to be power gated. Similarly it should power ungated if
* alloc happens from the memory.
* int (*busy)(void) - trigger runtime power ungate
* int (*idle)(void) - trigger runtime power gate
*/
struct nvmap_pm_ops {
int (*busy)(void);
int (*idle)(void);
};
struct nvmap_platform_carveout {
const char *name;
unsigned int usage_mask;
phys_addr_t base;
size_t size;
struct device *cma_dev;
bool resize;
struct device *dma_dev;
struct device dev;
struct dma_declare_info *dma_info;
bool is_ivm;
unsigned int peer;
unsigned int vmid;
int can_alloc;
bool enable_static_dma_map;
bool disable_dynamic_dma_map;
bool no_cpu_access; /* carveout can't be accessed from cpu at all */
bool init_done; /* FIXME: remove once all caveouts use reserved-memory */
struct nvmap_pm_ops pm_ops;
};
struct nvmap_platform_data {
const struct nvmap_platform_carveout *carveouts;
unsigned int nr_carveouts;
};
#endif /* _LINUX_NVMAP_H */

38
include/linux/nvmap_t19x.h Normal file
View File

@@ -0,0 +1,38 @@
/*
* include/linux/nvmap_t19x.h
*
* structure declarations for nvmem and nvmap user-space ioctls
*
* Copyright (c) 2009-2022, NVIDIA CORPORATION. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*/
#include <linux/types.h>
#include <linux/bitmap.h>
#include <linux/device.h>
#include <linux/spinlock.h>
#ifndef _LINUX_NVMAP_T19x_H
#define _LINUX_NVMAP_T19x_H
#define NVMAP_HEAP_CARVEOUT_CVSRAM (1ul<<25)
int nvmap_register_cvsram_carveout(struct device *dma_dev,
phys_addr_t base, size_t size,
int (*pmops_busy)(void), int (*pmops_idle)(void));
struct nvmap_handle_t19x {
atomic_t nc_pin; /* no. of pins from non io coherent devices */
};
extern bool nvmap_version_t19x;
#endif /* _LINUX_NVMAP_T19x_H */

870
include/trace/events/nvmap.h Normal file
View File

@@ -0,0 +1,870 @@
/*
* include/trace/events/nvmap.h
*
* NvMap event logging to ftrace.
*
* Copyright (c) 2012-2022, NVIDIA CORPORATION. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*/
#undef TRACE_SYSTEM
#define TRACE_SYSTEM nvmap
#if !defined(_TRACE_NVMAP_H) || defined(TRACE_HEADER_MULTI_READ)
#define _TRACE_NVMAP_H
#include <linux/nvmap.h>
#include <linux/dma-buf.h>
#include <linux/types.h>
#include <linux/tracepoint.h>
struct nvmap_handle;
struct nvmap_handle_ref;
struct nvmap_client;
DECLARE_EVENT_CLASS(nvmap,
TP_PROTO(struct nvmap_client *client, const char *name),
TP_ARGS(client, name),
TP_STRUCT__entry(
__field(struct nvmap_client *, client)
__string(sname, name)
),
TP_fast_assign(
__entry->client = client;
__assign_str(sname, name)
),
TP_printk("client=%p, name=%s",
__entry->client, __get_str(sname))
);
DEFINE_EVENT(nvmap, nvmap_open,
TP_PROTO(struct nvmap_client *client, const char *name),
TP_ARGS(client, name)
);
DEFINE_EVENT(nvmap, nvmap_release,
TP_PROTO(struct nvmap_client *client, const char *name),
TP_ARGS(client, name)
);
TRACE_EVENT(nvmap_create_handle,
TP_PROTO(struct nvmap_client *client,
const char *name,
struct nvmap_handle *h,
u32 size,
struct nvmap_handle_ref *ref
),
TP_ARGS(client, name, h, size, ref),
TP_STRUCT__entry(
__field(struct nvmap_client *, client)
__string(sname, name)
__field(struct nvmap_handle *, h)
__field(u32, size)
__field(struct nvmap_handle_ref *, ref)
),
TP_fast_assign(
__entry->client = client;
__assign_str(sname, name)
__entry->h = h;
__entry->size = size;
__entry->ref = ref;
),
TP_printk("client=%p, name=%s, handle=%p, size=%d, ref=%p",
__entry->client, __get_str(sname),
__entry->h, __entry->size, __entry->ref)
);
TRACE_EVENT(nvmap_alloc_handle,
TP_PROTO(struct nvmap_client *client,
struct nvmap_handle *handle,
size_t size,
u32 heap_mask,
u32 align,
u32 flags,
u64 total,
u64 alloc
),
TP_ARGS(client, handle, size, heap_mask, align, flags, total, alloc),
TP_STRUCT__entry(
__field(struct nvmap_client *, client)
__field(struct nvmap_handle *, handle)
__field(size_t, size)
__field(u32, heap_mask)
__field(u32, align)
__field(u32, flags)
__field(u64, total)
__field(u64, alloc)
),
TP_fast_assign(
__entry->client = client;
__entry->handle = handle;
__entry->size = size;
__entry->heap_mask = heap_mask;
__entry->align = align;
__entry->flags = flags;
__entry->total = total;
__entry->alloc = alloc;
),
TP_printk("client=%p, id=0x%p, size=%zu, heap_mask=0x%x, align=%d, flags=0x%x, total=%llu, alloc=%llu",
__entry->client, __entry->handle, __entry->size,
__entry->heap_mask, __entry->align, __entry->flags,
(unsigned long long)__entry->total,
(unsigned long long)__entry->alloc)
);
DECLARE_EVENT_CLASS(nvmap_handle_summary,
TP_PROTO(struct nvmap_client *client, pid_t pid, u32 dupes,
struct nvmap_handle *handle, u32 share, u64 base,
size_t size, u32 flags, u32 tag, const char *tag_name),
TP_ARGS(client, pid, dupes, handle, share, base, size, flags, tag, tag_name),
TP_STRUCT__entry(
__field(struct nvmap_client *, client)
__field(pid_t, pid)
__field(u32, dupes)
__field(struct nvmap_handle *, handle)
__field(u32, share)
__field(u64, base)
__field(size_t, size)
__field(u32, flags)
__field(u32, tag)
__string(tag_name, tag_name)
),
TP_fast_assign(
__entry->client = client;
__entry->pid = pid;
__entry->dupes = dupes;
__entry->handle = handle;
__entry->share = share;
__entry->base = base;
__entry->size = size;
__entry->flags = flags;
__entry->tag = tag;
__assign_str(tag_name, tag_name)
),
TP_printk("client=0x%p pid=%d dupes=%u handle=0x%p share=%u base=%llx size=%zu flags=0x%x tag=0x%x %s",
__entry->client,
__entry->pid,
__entry->dupes,
__entry->handle,
__entry->share,
__entry->base,
__entry->size,
__entry->flags,
__entry->tag,
__get_str(tag_name)
)
);
DEFINE_EVENT(nvmap_handle_summary,
nvmap_alloc_handle_done,
TP_PROTO(struct nvmap_client *client, pid_t pid, u32 dupes,
struct nvmap_handle *handle, u32 share, u64 base,
size_t size, u32 flags, u32 tag, const char *tag_name),
TP_ARGS(client, pid, dupes, handle, share, base, size, flags, tag, tag_name)
);
DEFINE_EVENT(nvmap_handle_summary,
nvmap_duplicate_handle,
TP_PROTO(struct nvmap_client *client, pid_t pid, u32 dupes,
struct nvmap_handle *handle, u32 share, u64 base,
size_t size, u32 flags, u32 tag, const char *tag_name),
TP_ARGS(client, pid, dupes, handle, share, base, size, flags, tag, tag_name)
);
DEFINE_EVENT(nvmap_handle_summary,
nvmap_free_handle,
TP_PROTO(struct nvmap_client *client, pid_t pid, u32 dupes,
struct nvmap_handle *handle, u32 share, u64 base,
size_t size, u32 flags, u32 tag, const char *tag_name),
TP_ARGS(client, pid, dupes, handle, share, base, size, flags, tag, tag_name)
);
DEFINE_EVENT(nvmap_handle_summary,
nvmap_destroy_handle,
TP_PROTO(struct nvmap_client *client, pid_t pid, u32 dupes,
struct nvmap_handle *handle, u32 share, u64 base,
size_t size, u32 flags, u32 tag, const char *tag_name),
TP_ARGS(client, pid, dupes, handle, share, base, size, flags, tag, tag_name)
);
TRACE_EVENT(nvmap_cache_maint,
TP_PROTO(struct nvmap_client *client,
struct nvmap_handle *h,
ulong start,
ulong end,
u32 op,
size_t size
),
TP_ARGS(client, h, start, end, op, size),
TP_STRUCT__entry(
__field(struct nvmap_client *, client)
__field(struct nvmap_handle *, h)
__field(ulong, start)
__field(ulong, end)
__field(u32, op)
__field(size_t, size)
),
TP_fast_assign(
__entry->client = client;
__entry->h = h;
__entry->start = start;
__entry->end = end;
__entry->op = op;
__entry->size = size;
),
TP_printk("client=%p, h=%p, start=0x%lx, end=0x%lx, op=%d, size=%zu",
__entry->client, __entry->h, __entry->start,
__entry->end, __entry->op, __entry->size)
);
TRACE_EVENT(nvmap_cache_flush,
TP_PROTO(size_t size,
u64 alloc_rq,
u64 total_rq,
u64 total_done
),
TP_ARGS(size, alloc_rq, total_rq, total_done),
TP_STRUCT__entry(
__field(size_t, size)
__field(u64, alloc_rq)
__field(u64, total_rq)
__field(u64, total_done)
),
TP_fast_assign(
__entry->size = size;
__entry->alloc_rq = alloc_rq;
__entry->total_rq = total_rq;
__entry->total_done = total_done;
),
TP_printk("size=%zu, alloc_rq=%llu, total_rq=%llu, total_done=%llu",
__entry->size,
(unsigned long long)__entry->alloc_rq,
(unsigned long long)__entry->total_rq,
(unsigned long long)__entry->total_done)
);
TRACE_EVENT(nvmap_map_into_caller_ptr,
TP_PROTO(struct nvmap_client *client,
struct nvmap_handle *h,
u32 offset,
u32 length,
u32 flags
),
TP_ARGS(client, h, offset, length, flags),
TP_STRUCT__entry(
__field(struct nvmap_client *, client)
__field(struct nvmap_handle *, h)
__field(u32, offset)
__field(u32, length)
__field(u32, flags)
),
TP_fast_assign(
__entry->client = client;
__entry->h = h;
__entry->offset = offset;
__entry->length = length;
__entry->flags = flags;
),
TP_printk("client=%p, h=%p, offset=%d, length=%d, flags=0x%x",
__entry->client, __entry->h, __entry->offset,
__entry->length, __entry->flags)
);
TRACE_EVENT(nvmap_ioctl_rw_handle,
TP_PROTO(struct nvmap_client *client,
struct nvmap_handle *h,
u32 is_read,
u32 offset,
unsigned long addr,
u32 mem_stride,
u32 user_stride,
u32 elem_size,
u32 count
),
TP_ARGS(client, h, is_read, offset, addr, mem_stride,
user_stride, elem_size, count),
TP_STRUCT__entry(
__field(struct nvmap_client *, client)
__field(struct nvmap_handle *, h)
__field(u32, is_read)
__field(u32, offset)
__field(unsigned long, addr)
__field(u32, mem_stride)
__field(u32, user_stride)
__field(u32, elem_size)
__field(u32, count)
),
TP_fast_assign(
__entry->client = client;
__entry->h = h;
__entry->is_read = is_read;
__entry->offset = offset;
__entry->addr = addr;
__entry->mem_stride = mem_stride;
__entry->user_stride = user_stride;
__entry->elem_size = elem_size;
__entry->count = count;
),
TP_printk("client=%p, h=%p, is_read=%d, offset=%d, addr=0x%lx,"
"mem_stride=%d, user_stride=%d, elem_size=%d, count=%d",
__entry->client, __entry->h, __entry->is_read, __entry->offset,
__entry->addr, __entry->mem_stride, __entry->user_stride,
__entry->elem_size, __entry->count)
);
TRACE_EVENT(nvmap_ioctl_pinop,
TP_PROTO(struct nvmap_client *client,
u32 is_pin,
u32 count,
struct nvmap_handle **ids
),
TP_ARGS(client, is_pin, count, ids),
TP_STRUCT__entry(
__field(struct nvmap_client *, client)
__field(u32, is_pin)
__field(u32, count)
__field(struct nvmap_handle **, ids)
__dynamic_array(struct nvmap_handle *, ids, count)
),
TP_fast_assign(
__entry->client = client;
__entry->is_pin = is_pin;
__entry->count = count;
__entry->ids = ids;
memcpy(__get_dynamic_array(ids), ids,
sizeof(struct nvmap_handle *) * count);
),
TP_printk("client=%p, is_pin=%d, count=%d, ids=[%s]",
__entry->client, __entry->is_pin, __entry->count,
__print_hex(__get_dynamic_array(ids), __entry->ids ?
sizeof(struct nvmap_handle *) * __entry->count : 0))
);
DECLARE_EVENT_CLASS(handle_get_put,
TP_PROTO(struct nvmap_handle *handle, u32 ref_count),
TP_ARGS(handle, ref_count),
TP_STRUCT__entry(
__field(struct nvmap_handle *, handle)
__field(u32, ref_count)
),
TP_fast_assign(
__entry->handle = handle;
__entry->ref_count = ref_count;
),
TP_printk("ref=%u handle=%p",
__entry->ref_count,
__entry->handle
)
);
DEFINE_EVENT(handle_get_put, nvmap_handle_get,
TP_PROTO(struct nvmap_handle *handle, u32 ref_count),
TP_ARGS(handle, ref_count)
);
DEFINE_EVENT(handle_get_put, nvmap_handle_put,
TP_PROTO(struct nvmap_handle *handle, u32 ref_count),
TP_ARGS(handle, ref_count)
);
DECLARE_EVENT_CLASS(pin_unpin,
TP_PROTO(struct nvmap_client *client,
const char *name,
struct nvmap_handle *h,
u32 pin_count
),
TP_ARGS(client, name, h, pin_count),
TP_STRUCT__entry(
__field(struct nvmap_client *, client)
__string(sname, name)
__field(struct nvmap_handle *, h)
__field(u32, pin_count)
),
TP_fast_assign(
__entry->client = client;
__assign_str(sname, name)
__entry->h = h;
__entry->pin_count = pin_count;
),
TP_printk("client=%p, name=%s, h=%p, pin_count=%d",
__entry->client, __get_str(sname),
__entry->h, __entry->pin_count)
);
DEFINE_EVENT(pin_unpin, nvmap_pin,
TP_PROTO(struct nvmap_client *client,
const char *name,
struct nvmap_handle *h,
u32 pin_count
),
TP_ARGS(client, name, h, pin_count)
);
DEFINE_EVENT(pin_unpin, nvmap_unpin,
TP_PROTO(struct nvmap_client *client,
const char *name,
struct nvmap_handle *h,
u32 pin_count
),
TP_ARGS(client, name, h, pin_count)
);
DEFINE_EVENT(pin_unpin, handle_unpin_error,
TP_PROTO(struct nvmap_client *client,
const char *name,
struct nvmap_handle *h,
u32 pin_count
),
TP_ARGS(client, name, h, pin_count)
);
/*
* Nvmap dmabuf events
*/
DECLARE_EVENT_CLASS(nvmap_dmabuf_2,
TP_PROTO(struct dma_buf *dbuf,
struct device *dev
),
TP_ARGS(dbuf, dev),
TP_STRUCT__entry(
__field(struct dma_buf *, dbuf)
__string(name, dev_name(dev))
),
TP_fast_assign(
__entry->dbuf = dbuf;
__assign_str(name, dev_name(dev));
),
TP_printk("dmabuf=%p, device=%s",
__entry->dbuf, __get_str(name)
)
);
DECLARE_EVENT_CLASS(nvmap_dmabuf_1,
TP_PROTO(struct dma_buf *dbuf),
TP_ARGS(dbuf),
TP_STRUCT__entry(
__field(struct dma_buf *, dbuf)
),
TP_fast_assign(
__entry->dbuf = dbuf;
),
TP_printk("dmabuf=%p",
__entry->dbuf
)
);
DEFINE_EVENT(nvmap_dmabuf_2, nvmap_dmabuf_attach,
TP_PROTO(struct dma_buf *dbuf,
struct device *dev
),
TP_ARGS(dbuf, dev)
);
DEFINE_EVENT(nvmap_dmabuf_2, nvmap_dmabuf_detach,
TP_PROTO(struct dma_buf *dbuf,
struct device *dev
),
TP_ARGS(dbuf, dev)
);
DEFINE_EVENT(nvmap_dmabuf_2, nvmap_dmabuf_map_dma_buf,
TP_PROTO(struct dma_buf *dbuf,
struct device *dev
),
TP_ARGS(dbuf, dev)
);
DEFINE_EVENT(nvmap_dmabuf_2, nvmap_dmabuf_unmap_dma_buf,
TP_PROTO(struct dma_buf *dbuf,
struct device *dev
),
TP_ARGS(dbuf, dev)
);
DEFINE_EVENT(nvmap_dmabuf_1, nvmap_dmabuf_mmap,
TP_PROTO(struct dma_buf *dbuf),
TP_ARGS(dbuf)
);
DEFINE_EVENT(nvmap_dmabuf_1, nvmap_dmabuf_vmap,
TP_PROTO(struct dma_buf *dbuf),
TP_ARGS(dbuf)
);
DEFINE_EVENT(nvmap_dmabuf_1, nvmap_dmabuf_vunmap,
TP_PROTO(struct dma_buf *dbuf),
TP_ARGS(dbuf)
);
DEFINE_EVENT(nvmap_dmabuf_1, nvmap_dmabuf_kmap,
TP_PROTO(struct dma_buf *dbuf),
TP_ARGS(dbuf)
);
DEFINE_EVENT(nvmap_dmabuf_1, nvmap_dmabuf_kunmap,
TP_PROTO(struct dma_buf *dbuf),
TP_ARGS(dbuf)
);
DECLARE_EVENT_CLASS(nvmap_dmabuf_cpu_access,
TP_PROTO(struct dma_buf *dbuf,
size_t start,
size_t len),
TP_ARGS(dbuf, start, len),
TP_STRUCT__entry(
__field(struct dma_buf *, dbuf)
__field(size_t, start)
__field(size_t, len)
),
TP_fast_assign(
__entry->dbuf = dbuf;
__entry->start = start;
__entry->len = len;
),
TP_printk("dmabuf=%p, start=%zd len=%zd",
__entry->dbuf, __entry->start, __entry->len
)
);
DEFINE_EVENT(nvmap_dmabuf_cpu_access, nvmap_dmabuf_begin_cpu_access,
TP_PROTO(struct dma_buf *dbuf,
size_t start,
size_t len),
TP_ARGS(dbuf, start, len)
);
DEFINE_EVENT(nvmap_dmabuf_cpu_access, nvmap_dmabuf_end_cpu_access,
TP_PROTO(struct dma_buf *dbuf,
size_t start,
size_t len),
TP_ARGS(dbuf, start, len)
);
DECLARE_EVENT_CLASS(nvmap_dmabuf_make_release,
TP_PROTO(const char *cli,
struct nvmap_handle *h,
struct dma_buf *dbuf
),
TP_ARGS(cli, h, dbuf),
TP_STRUCT__entry(
__field(const char *, cli)
__field(struct nvmap_handle *, h)
__field(struct dma_buf *, dbuf)
),
TP_fast_assign(
__entry->cli = cli;
__entry->h = h;
__entry->dbuf = dbuf;
),
TP_printk("cli=%s handle=%p dmabuf=%p",
__entry->cli, __entry->h, __entry->dbuf
)
);
DEFINE_EVENT(nvmap_dmabuf_make_release, nvmap_make_dmabuf,
TP_PROTO(const char *cli,
struct nvmap_handle *h,
struct dma_buf *dbuf
),
TP_ARGS(cli, h, dbuf)
);
DEFINE_EVENT(nvmap_dmabuf_make_release, nvmap_dmabuf_release,
TP_PROTO(const char *cli,
struct nvmap_handle *h,
struct dma_buf *dbuf
),
TP_ARGS(cli, h, dbuf)
);
TRACE_EVENT(pp_clean_cache,
TP_PROTO(u32 dirty_pages,
size_t cache_maint_th,
int cache_maint_by_set_ways
),
TP_ARGS(dirty_pages, cache_maint_th, cache_maint_by_set_ways),
TP_STRUCT__entry(
__field(u32, dirty_pages)
__field(size_t, cache_maint_th)
__field(int, cache_maint_by_set_ways)
),
TP_fast_assign(
__entry->dirty_pages = dirty_pages;
__entry->cache_maint_th = cache_maint_th >> PAGE_SHIFT;
__entry->cache_maint_by_set_ways = cache_maint_by_set_ways;
),
TP_printk("dirty_pages=%u, cache_maint_th=%zu, cache_maint_by_set_ways=%d",
__entry->dirty_pages, __entry->cache_maint_th,
__entry->cache_maint_by_set_ways)
);
DECLARE_EVENT_CLASS(nvmap_get_list_page,
TP_PROTO(u32 count),
TP_ARGS(count),
TP_STRUCT__entry(
__field(u32, count)
),
TP_fast_assign(
__entry->count = count;
),
TP_printk("pages left in list=%u", __entry->count)
);
DEFINE_EVENT(nvmap_get_list_page, get_zero_list_page,
TP_PROTO(u32 count),
TP_ARGS(count)
);
DEFINE_EVENT(nvmap_get_list_page, get_page_list_page,
TP_PROTO(u32 count),
TP_ARGS(count)
);
TRACE_EVENT(nvmap_pp_zero_pages,
TP_PROTO(u32 count),
TP_ARGS(count),
TP_STRUCT__entry(
__field(u32, count)
),
TP_fast_assign(
__entry->count = count;
),
TP_printk("no. of pages zeroed=%u", __entry->count)
);
TRACE_EVENT(nvmap_pp_do_background_zero_pages,
TP_PROTO(u32 inserted, u32 zeroed),
TP_ARGS(inserted, zeroed),
TP_STRUCT__entry(
__field(u32, inserted)
__field(u32, zeroed)
),
TP_fast_assign(
__entry->inserted = inserted;
__entry->zeroed = zeroed;
),
TP_printk("failed to insert %u no. of zeroed pages to page_list",
__entry->zeroed - __entry->inserted)
);
TRACE_EVENT(nvmap_pp_alloc_locked,
TP_PROTO(int force_alloc),
TP_ARGS(force_alloc),
TP_STRUCT__entry(
__field(int, force_alloc)
),
TP_fast_assign(
__entry->force_alloc = force_alloc;
),
TP_printk("allocated one page with force_alloc:%d", __entry->force_alloc)
);
TRACE_EVENT(nvmap_pp_alloc_lots,
TP_PROTO(u32 alloced, u32 requested),
TP_ARGS(alloced, requested),
TP_STRUCT__entry(
__field(u32, alloced)
__field(u32, requested)
),
TP_fast_assign(
__entry->alloced = alloced;
__entry->requested = requested;
),
TP_printk("requested:%u alloced:%u\n",
__entry->requested, __entry->alloced)
);
TRACE_EVENT(nvmap_pp_fill_zero_lots,
TP_PROTO(u32 save_to_zero,
u32 to_zero,
u32 ret,
u32 nr
),
TP_ARGS(save_to_zero, to_zero, ret, nr),
TP_STRUCT__entry(
__field(u32, save_to_zero)
__field(u32, to_zero)
__field(u32, ret)
__field(u32, nr)
),
TP_fast_assign(
__entry->save_to_zero = save_to_zero;
__entry->to_zero = to_zero;
__entry->ret = ret;
__entry->nr = nr;
),
TP_printk("inserted %u pages to zero list, freed %u pages, did not process %u pages",
__entry->to_zero - __entry->save_to_zero,
__entry->ret - (__entry->to_zero - __entry->save_to_zero),
__entry->nr - __entry->ret)
);
TRACE_EVENT(refcount_get_handle_from_sci_ipc_id,
TP_PROTO(struct nvmap_handle *handle,
struct dma_buf *dmabuf,
int handle_ref, const char *perm
),
TP_ARGS(handle, dmabuf, handle_ref, perm),
TP_STRUCT__entry(
__field(struct nvmap_handle *, handle)
__field(struct dma_buf *, dmabuf)
__field(int, handle_ref)
__string(acc_perm, perm)
),
TP_fast_assign(
__entry->handle = handle;
__entry->dmabuf = dmabuf;
__entry->handle_ref = handle_ref;
__assign_str(acc_perm, perm);
),
TP_printk("handle=0x%p, dmabuf=0x%p, handle_ref=%d, perm=%s",
__entry->handle, __entry->dmabuf, __entry->handle_ref,
__get_str(acc_perm))
);
DECLARE_EVENT_CLASS(nvmap_refcount,
TP_PROTO(struct nvmap_handle *handle, struct dma_buf *dmabuf, \
int handle_ref, long dmabuf_ref, const char *perm),
TP_ARGS(handle, dmabuf, handle_ref, dmabuf_ref, perm),
TP_STRUCT__entry(
__field(struct nvmap_handle *, handle)
__field(struct dma_buf *, dmabuf)
__field(int, handle_ref)
__field(long, dmabuf_ref)
__string(acc_perm, perm)
),
TP_fast_assign(
__entry->handle = handle;
__entry->dmabuf = dmabuf;
__entry->handle_ref = handle_ref;
__entry->dmabuf_ref = dmabuf_ref;
__assign_str(acc_perm, perm);
),
TP_printk("handle=0x%p, dmabuf=0x%p, handle_ref=%d, dmabuf_ref=%ld, perm=%s",
__entry->handle, __entry->dmabuf, __entry->handle_ref,
__entry->dmabuf_ref, __get_str(acc_perm))
);
#define NVMAPREFCOUNTEVENT(ev) DEFINE_EVENT(nvmap_refcount, ev, \
TP_PROTO(struct nvmap_handle *handle, struct dma_buf *dmabuf, \
int handle_ref, long dmabuf_ref, const char *perm), \
TP_ARGS(handle, dmabuf, handle_ref, dmabuf_ref, perm) \
)
NVMAPREFCOUNTEVENT(refcount_create_handle);
NVMAPREFCOUNTEVENT(refcount_create_handle_from_va);
NVMAPREFCOUNTEVENT(refcount_create_handle_from_fd);
NVMAPREFCOUNTEVENT(refcount_getfd);
NVMAPREFCOUNTEVENT(refcount_alloc);
NVMAPREFCOUNTEVENT(refcount_get_sci_ipc_id);
NVMAPREFCOUNTEVENT(refcount_create_handle_from_sci_ipc_id);
NVMAPREFCOUNTEVENT(refcount_dup_handle);
NVMAPREFCOUNTEVENT(refcount_free_handle);
#undef NVMAPREFCOUNTEVENT
#endif /* _TRACE_NVMAP_H */
/* This part must be outside protection */
#include <trace/define_trace.h>

366
include/uapi/linux/nvmap.h Normal file
View File

@@ -0,0 +1,366 @@
/*
* include/uapi/linux/nvmap.h
*
* structure declarations for nvmem and nvmap user-space ioctls
*
* Copyright (c) 2009-2022, NVIDIA CORPORATION. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*/
#include <linux/ioctl.h>
#include <linux/types.h>
#ifndef __UAPI_LINUX_NVMAP_H
#define __UAPI_LINUX_NVMAP_H
/*
* DOC: NvMap Userspace API
*
* create a client by opening /dev/nvmap
* most operations handled via following ioctls
*
*/
enum {
NVMAP_HANDLE_PARAM_SIZE = 1,
NVMAP_HANDLE_PARAM_ALIGNMENT,
NVMAP_HANDLE_PARAM_BASE,
NVMAP_HANDLE_PARAM_HEAP,
NVMAP_HANDLE_PARAM_KIND,
NVMAP_HANDLE_PARAM_COMPR, /* ignored, to be removed */
};
enum {
NVMAP_CACHE_OP_WB = 0,
NVMAP_CACHE_OP_INV,
NVMAP_CACHE_OP_WB_INV,
};
enum {
NVMAP_PAGES_UNRESERVE = 0,
NVMAP_PAGES_RESERVE,
NVMAP_INSERT_PAGES_ON_UNRESERVE,
NVMAP_PAGES_PROT_AND_CLEAN,
};
#define NVMAP_ELEM_SIZE_U64 (1 << 31)
struct nvmap_create_handle {
union {
struct {
union {
/* size will be overwritten */
__u32 size; /* CreateHandle */
__s32 fd; /* DmaBufFd or FromFd */
};
__u32 handle; /* returns nvmap handle */
};
struct {
/* one is input parameter, and other is output parameter
* since its a union please note that input parameter
* will be overwritten once ioctl returns
*/
union {
__u64 ivm_id; /* CreateHandle from ivm*/
__u32 ivm_handle;/* Get ivm_id from handle */
};
};
struct {
union {
/* size64 will be overwritten */
__u64 size64; /* CreateHandle */
__u32 handle64; /* returns nvmap handle */
};
};
};
};
struct nvmap_create_handle_from_va {
__u64 va; /* FromVA*/
__u32 size; /* non-zero for partial memory VMA. zero for end of VMA */
__u32 flags; /* wb/wc/uc/iwb, tag etc. */
union {
__u32 handle; /* returns nvmap handle */
__u64 size64; /* used when size is 0 */
};
};
struct nvmap_gup_test {
__u64 va; /* FromVA*/
__u32 handle; /* returns nvmap handle */
__u32 result; /* result=1 for pass, result=-err for failure */
};
struct nvmap_alloc_handle {
__u32 handle; /* nvmap handle */
__u32 heap_mask; /* heaps to allocate from */
__u32 flags; /* wb/wc/uc/iwb etc. */
__u32 align; /* min alignment necessary */
};
struct nvmap_alloc_ivm_handle {
__u32 handle; /* nvmap handle */
__u32 heap_mask; /* heaps to allocate from */
__u32 flags; /* wb/wc/uc/iwb etc. */
__u32 align; /* min alignment necessary */
__u32 peer; /* peer with whom handle must be shared. Used
* only for NVMAP_HEAP_CARVEOUT_IVM
*/
};
struct nvmap_rw_handle {
__u64 addr; /* user pointer*/
__u32 handle; /* nvmap handle */
__u64 offset; /* offset into hmem */
__u64 elem_size; /* individual atom size */
__u64 hmem_stride; /* delta in bytes between atoms in hmem */
__u64 user_stride; /* delta in bytes between atoms in user */
__u64 count; /* number of atoms to copy */
};
#ifdef __KERNEL__
#ifdef CONFIG_COMPAT
struct nvmap_rw_handle_32 {
__u32 addr; /* user pointer */
__u32 handle; /* nvmap handle */
__u32 offset; /* offset into hmem */
__u32 elem_size; /* individual atom size */
__u32 hmem_stride; /* delta in bytes between atoms in hmem */
__u32 user_stride; /* delta in bytes between atoms in user */
__u32 count; /* number of atoms to copy */
};
#endif /* CONFIG_COMPAT */
#endif /* __KERNEL__ */
struct nvmap_handle_param {
__u32 handle; /* nvmap handle */
__u32 param; /* size/align/base/heap etc. */
unsigned long result; /* returns requested info*/
};
#ifdef __KERNEL__
#ifdef CONFIG_COMPAT
struct nvmap_handle_param_32 {
__u32 handle; /* nvmap handle */
__u32 param; /* size/align/base/heap etc. */
__u32 result; /* returns requested info*/
};
#endif /* CONFIG_COMPAT */
#endif /* __KERNEL__ */
struct nvmap_cache_op {
unsigned long addr; /* user pointer*/
__u32 handle; /* nvmap handle */
__u32 len; /* bytes to flush */
__s32 op; /* wb/wb_inv/inv */
};
struct nvmap_cache_op_64 {
unsigned long addr; /* user pointer*/
__u32 handle; /* nvmap handle */
__u64 len; /* bytes to flush */
__s32 op; /* wb/wb_inv/inv */
};
#ifdef __KERNEL__
#ifdef CONFIG_COMPAT
struct nvmap_cache_op_32 {
__u32 addr; /* user pointer*/
__u32 handle; /* nvmap handle */
__u32 len; /* bytes to flush */
__s32 op; /* wb/wb_inv/inv */
};
#endif /* CONFIG_COMPAT */
#endif /* __KERNEL__ */
struct nvmap_cache_op_list {
__u64 handles; /* Ptr to u32 type array, holding handles */
__u64 offsets; /* Ptr to u32 type array, holding offsets
* into handle mem */
__u64 sizes; /* Ptr to u32 type array, holindg sizes of memory
* regions within each handle */
__u32 nr; /* Number of handles */
__s32 op; /* wb/wb_inv/inv */
};
struct nvmap_debugfs_handles_header {
__u8 version;
};
struct nvmap_debugfs_handles_entry {
__u64 base;
__u64 size;
__u32 flags;
__u32 share_count;
__u64 mapped_size;
};
struct nvmap_set_tag_label {
__u32 tag;
__u32 len; /* in: label length
out: number of characters copied */
__u64 addr; /* in: pointer to label or NULL to remove */
};
struct nvmap_available_heaps {
__u64 heaps; /* heaps bitmask */
};
struct nvmap_heap_size {
__u32 heap;
__u64 size;
};
struct nvmap_sciipc_map {
__u64 auth_token; /* AuthToken */
__u32 flags; /* Exporter permission flags */
__u32 sci_ipc_id; /* FromImportId */
__u32 handle; /* Nvmap handle */
};
struct nvmap_handle_parameters {
__u8 contig;
__u32 import_id;
__u32 handle;
__u32 heap_number;
__u32 access_flags;
__u64 heap;
__u64 align;
__u64 coherency;
__u64 size;
__u64 offset;
};
/**
* Struct used while querying heap parameters
*/
struct nvmap_query_heap_params {
__u32 heap_mask;
__u32 flags;
__u8 contig;
__u64 total;
__u64 free;
__u64 largest_free_block;
};
/**
* Struct used while duplicating memory handle
*/
struct nvmap_duplicate_handle {
__u32 handle;
__u32 access_flags;
__u32 dup_handle;
};
#define NVMAP_IOC_MAGIC 'N'
/* Creates a new memory handle. On input, the argument is the size of the new
* handle; on return, the argument is the name of the new handle
*/
#define NVMAP_IOC_CREATE _IOWR(NVMAP_IOC_MAGIC, 0, struct nvmap_create_handle)
#define NVMAP_IOC_CREATE_64 \
_IOWR(NVMAP_IOC_MAGIC, 1, struct nvmap_create_handle)
#define NVMAP_IOC_FROM_ID _IOWR(NVMAP_IOC_MAGIC, 2, struct nvmap_create_handle)
/* Actually allocates memory for the specified handle */
#define NVMAP_IOC_ALLOC _IOW(NVMAP_IOC_MAGIC, 3, struct nvmap_alloc_handle)
/* Frees a memory handle, unpinning any pinned pages and unmapping any mappings
*/
#define NVMAP_IOC_FREE _IO(NVMAP_IOC_MAGIC, 4)
/* Reads/writes data (possibly strided) from a user-provided buffer into the
* hmem at the specified offset */
#define NVMAP_IOC_WRITE _IOW(NVMAP_IOC_MAGIC, 6, struct nvmap_rw_handle)
#define NVMAP_IOC_READ _IOW(NVMAP_IOC_MAGIC, 7, struct nvmap_rw_handle)
#ifdef __KERNEL__
#ifdef CONFIG_COMPAT
#define NVMAP_IOC_WRITE_32 _IOW(NVMAP_IOC_MAGIC, 6, struct nvmap_rw_handle_32)
#define NVMAP_IOC_READ_32 _IOW(NVMAP_IOC_MAGIC, 7, struct nvmap_rw_handle_32)
#endif /* CONFIG_COMPAT */
#endif /* __KERNEL__ */
#define NVMAP_IOC_PARAM _IOWR(NVMAP_IOC_MAGIC, 8, struct nvmap_handle_param)
#ifdef __KERNEL__
#ifdef CONFIG_COMPAT
#define NVMAP_IOC_PARAM_32 _IOWR(NVMAP_IOC_MAGIC, 8, struct nvmap_handle_param_32)
#endif /* CONFIG_COMPAT */
#endif /* __KERNEL__ */
#define NVMAP_IOC_CACHE _IOW(NVMAP_IOC_MAGIC, 12, struct nvmap_cache_op)
#define NVMAP_IOC_CACHE_64 _IOW(NVMAP_IOC_MAGIC, 12, struct nvmap_cache_op_64)
#ifdef __KERNEL__
#ifdef CONFIG_COMPAT
#define NVMAP_IOC_CACHE_32 _IOW(NVMAP_IOC_MAGIC, 12, struct nvmap_cache_op_32)
#endif /* CONFIG_COMPAT */
#endif /* __KERNEL__ */
/* Returns a global ID usable to allow a remote process to create a handle
* reference to the same handle */
#define NVMAP_IOC_GET_ID _IOWR(NVMAP_IOC_MAGIC, 13, struct nvmap_create_handle)
/* Returns a file id that allows a remote process to create a handle
* reference to the same handle */
#define NVMAP_IOC_GET_FD _IOWR(NVMAP_IOC_MAGIC, 15, struct nvmap_create_handle)
/* Create a new memory handle from file id passed */
#define NVMAP_IOC_FROM_FD _IOWR(NVMAP_IOC_MAGIC, 16, struct nvmap_create_handle)
/* Perform cache maintenance on a list of handles. */
#define NVMAP_IOC_CACHE_LIST _IOW(NVMAP_IOC_MAGIC, 17, \
struct nvmap_cache_op_list)
#define NVMAP_IOC_FROM_IVC_ID _IOWR(NVMAP_IOC_MAGIC, 19, struct nvmap_create_handle)
#define NVMAP_IOC_GET_IVC_ID _IOWR(NVMAP_IOC_MAGIC, 20, struct nvmap_create_handle)
#define NVMAP_IOC_GET_IVM_HEAPS _IOR(NVMAP_IOC_MAGIC, 21, unsigned int)
/* Create a new memory handle from VA passed */
#define NVMAP_IOC_FROM_VA _IOWR(NVMAP_IOC_MAGIC, 22, struct nvmap_create_handle_from_va)
#define NVMAP_IOC_GUP_TEST _IOWR(NVMAP_IOC_MAGIC, 23, struct nvmap_gup_test)
/* Define a label for allocation tag */
#define NVMAP_IOC_SET_TAG_LABEL _IOW(NVMAP_IOC_MAGIC, 24, struct nvmap_set_tag_label)
#define NVMAP_IOC_GET_AVAILABLE_HEAPS \
_IOR(NVMAP_IOC_MAGIC, 25, struct nvmap_available_heaps)
#define NVMAP_IOC_GET_HEAP_SIZE \
_IOR(NVMAP_IOC_MAGIC, 26, struct nvmap_heap_size)
#define NVMAP_IOC_PARAMETERS \
_IOR(NVMAP_IOC_MAGIC, 27, struct nvmap_handle_parameters)
/* START of T124 IOCTLS */
/* Actually allocates memory from IVM heaps */
#define NVMAP_IOC_ALLOC_IVM _IOW(NVMAP_IOC_MAGIC, 101, struct nvmap_alloc_ivm_handle)
/* Allocate seperate memory for VPR */
#define NVMAP_IOC_VPR_FLOOR_SIZE _IOW(NVMAP_IOC_MAGIC, 102, __u32)
/* Get SCI_IPC_ID tied up with nvmap_handle */
#define NVMAP_IOC_GET_SCIIPCID _IOR(NVMAP_IOC_MAGIC, 103, \
struct nvmap_sciipc_map)
/* Get Nvmap handle from SCI_IPC_ID */
#define NVMAP_IOC_HANDLE_FROM_SCIIPCID _IOR(NVMAP_IOC_MAGIC, 104, \
struct nvmap_sciipc_map)
/* Get heap parameters such as total and frre size */
#define NVMAP_IOC_QUERY_HEAP_PARAMS _IOR(NVMAP_IOC_MAGIC, 105, \
struct nvmap_query_heap_params)
/* Duplicate NvRmMemHandle with same/reduced permission */
#define NVMAP_IOC_DUP_HANDLE _IOWR(NVMAP_IOC_MAGIC, 106, \
struct nvmap_duplicate_handle)
#define NVMAP_IOC_MAXNR (_IOC_NR(NVMAP_IOC_DUP_HANDLE))
#endif /* __UAPI_LINUX_NVMAP_H */