nv-tegra-mirror/linux-nvgpu
1
0
Fork 0
mirror of git://nv-tegra.nvidia.com/linux-nvgpu.git synced 2025-12-22 17:36:20 +03:00
Code Issues Packages Projects Releases Wiki Activity

gpu: nvgpu: add linux REMAP support

Browse Source 
Add REMAP ioctl and accompanying support to the linux nvgpu driver.

REMAP support provides per-page control over sparse VM areas using the
concept of a virtual memory pool.

The REMAP ioctl accepts a list of operations (each a map or unmap) that
modify the VM area pages tracked by the virtual mmemory pool.

Inclusion of REMAP support in the nvgpu build is controlled by the new
CONFIG_NVGPU_REMAP flag.  This flag is enabled by default for linux builds.
A new NVGPU_GPU_FLAGS_SUPPORT_REMAP characteristics flag is added for use
in detecting when REMAP support is available.

When a VM allocation tagged with NVGPU_VM_AREA_ALLOC_SPARSE is made the
base virtual memory pool resources are allocated.  Per-page resources are
later allocated when the NVGPU_AS_IOCTL_REMAP ioctl is issued.  All REMAP
resources are released when the corresponding VM area is freed.

Jira NVGPU-6804

Change-Id: I1f2cdc0c06c1698a62640c1c6fbcb2f9db24a0bc
Signed-off-by: scottl <scottl@nvidia.com>
Reviewed-on: https://git-master.nvidia.com/r/c/linux-nvgpu/+/2542178
Tested-by: mobile promotions <svcmobile_promotions@nvidia.com>
Reviewed-by: mobile promotions <svcmobile_promotions@nvidia.com>
This commit is contained in:
scottl
2021-06-08 23:59:08 -07:00
committed by mobile promotions
parent cecb0666f6
commit 3cd256b344
20 changed files with 1761 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
arch/nvgpu-common.yaml
View File

@@ -675,6 +675,12 @@ mm:
include/nvgpu/vm_area.h ]
deps: [ ]
tags: unit-testable
vm_remap:
safe: no
sources: [ common/mm/vm_remap.c,
include/nvgpu/vm_remap.h ]
deps: [ ]
tags: unit-testable
vm:
safe: yes
sources: [ common/mm/vm.c,

6
arch/nvgpu-linux.yaml
View File

@@ -228,7 +228,8 @@ vgpu:
os/linux/vgpu/vgpu_linux.h ]
vm:
sources: [ os/linux/vm.c ]
sources: [ os/linux/vm.c,
os/linux/vm_remap.c ]
cic:
sources: [ os/linux/cic/cic_stub.c ]
@@ -252,7 +253,8 @@ headers:
include/nvgpu/linux/thread.h,
include/nvgpu/linux/log.h,
include/nvgpu/linux/utils.h,
include/nvgpu/linux/vm.h ]
include/nvgpu/linux/vm.h,
include/nvgpu/linux/vm_remap.h ]
# An extra unit to lump all the unclassified Linux files.
extra:

2
arch/nvgpu-posix.yaml
View File

@@ -22,6 +22,7 @@ all:
os/posix/posix-nvgpu_mem.c,
os/posix/posix-tsg.c,
os/posix/posix-vm.c,
os/posix/posix-vm_remap.c,
os/posix/soc.c,
os/posix/bsearch.c,
os/posix/posix-clk_arb.c,
@@ -51,6 +52,7 @@ headers:
include/nvgpu/posix/probe.h,
include/nvgpu/posix/soc_fuse.h,
include/nvgpu/posix/vm.h,
include/nvgpu/posix/posix-vm_remap.h,
include/nvgpu/posix/posix_vidmem.h,
include/nvgpu/posix/posix-nvhost.h,
include/nvgpu/posix/trace_gk20a.h ]

4
drivers/gpu/nvgpu/Makefile
View File

@@ -650,6 +650,10 @@ nvgpu-$(CONFIG_NVGPU_COMPRESSION) += \
hal/cbc/cbc_gv11b.o \
hal/cbc/cbc_tu104.o \
nvgpu-$(CONFIG_NVGPU_REMAP) += \
os/linux/vm_remap.o \
common/mm/vm_remap.o
# FUSA (Functionally Safe) HAL source files
nvgpu-y += \
hal/mm/mm_gv11b_fusa.o \

6
drivers/gpu/nvgpu/Makefile.linux.configs
View File

@@ -40,6 +40,9 @@ ifeq ($(CONFIG_NVGPU_COMPRESSION),y)
CONFIG_NVGPU_SUPPORT_CDE := y
endif
# Support for remap
CONFIG_NVGPU_REMAP := y
ifeq ($(CONFIG_COMMON_CLK),y)
ifeq ($(CONFIG_PM_DEVFREQ),y)
# Select this entry to enable gk20a scaling
@@ -172,6 +175,9 @@ endif
ifeq ($(CONFIG_NVGPU_COMPRESSION),y)
ccflags-y += -DCONFIG_NVGPU_COMPRESSION
endif
ifeq ($(CONFIG_NVGPU_REMAP),y)
ccflags-y += -DCONFIG_NVGPU_REMAP
endif
ifeq ($(CONFIG_NVGPU_SUPPORT_CDE),y)
ccflags-y += -DCONFIG_NVGPU_SUPPORT_CDE
endif

8
drivers/gpu/nvgpu/Makefile.sources
View File

@@ -60,6 +60,10 @@ ifeq ($(CONFIG_NVGPU_COMPRESSION),1)
srcs += os/posix/posix-comptags.c
endif
ifeq ($(CONFIG_NVGPU_REMAP),1)
srcs += os/posix/posix-vm_remap.c
endif
ifeq ($(CONFIG_NVGPU_LOGGING),1)
srcs += os/posix/log.c
endif
@@ -604,6 +608,10 @@ srcs += common/mm/comptags.c \
hal/cbc/cbc_gv11b.c
endif
ifeq ($(CONFIG_NVGPU_REMAP),1)
srcs += common/mm/vm_remap.c
endif
ifeq ($(CONFIG_NVGPU_NVLINK),1)
srcs += common/vbios/nvlink_bios.c \
common/nvlink/probe.c \

40
drivers/gpu/nvgpu/common/mm/vm_area.c
View File

@@ -25,6 +25,9 @@
#include <nvgpu/barrier.h>
#include <nvgpu/gk20a.h>
#include <nvgpu/static_analysis.h>
#ifdef CONFIG_NVGPU_REMAP
#include <nvgpu/vm_remap.h>
#endif
struct nvgpu_vm_area *nvgpu_vm_area_find(struct vm_gk20a *vm, u64 addr)
{
@@ -200,6 +203,7 @@ int nvgpu_vm_area_alloc(struct vm_gk20a *vm, u64 pages, u32 page_size,
u64 vaddr_start = 0;
u64 our_addr = *addr;
u32 pgsz_idx = GMMU_PAGE_SIZE_SMALL;
int err = 0;
/*
* If we have a fixed address then use the passed address in *addr. This
@@ -226,8 +230,10 @@ int nvgpu_vm_area_alloc(struct vm_gk20a *vm, u64 pages, u32 page_size,
}
vma = vm->vma[pgsz_idx];
if (nvgpu_vm_area_alloc_memory(vma, our_addr, pages,
page_size, flags, &vaddr_start) != 0) {
err = nvgpu_vm_area_alloc_memory(vma, our_addr, pages, page_size,
flags, &vaddr_start);
if (err != 0) {
goto free_vm_area;
}
@@ -238,10 +244,21 @@ int nvgpu_vm_area_alloc(struct vm_gk20a *vm, u64 pages, u32 page_size,
nvgpu_init_list_node(&vm_area->buffer_list_head);
nvgpu_init_list_node(&vm_area->vm_area_list);
#ifdef CONFIG_NVGPU_REMAP
if (((flags & NVGPU_VM_AREA_ALLOC_SPARSE) != 0U) &&
(vm_area->pgsz_idx == GMMU_PAGE_SIZE_BIG)) {
err = nvgpu_vm_remap_vpool_create(vm, vm_area, pages);
if (err != 0) {
goto free_vaddr;
}
}
#endif
nvgpu_mutex_acquire(&vm->update_gmmu_lock);
if (nvgpu_vm_area_alloc_gmmu_map(vm, vm_area, vaddr_start,
pgsz_idx, flags) != 0) {
err = nvgpu_vm_area_alloc_gmmu_map(vm, vm_area, vaddr_start,
pgsz_idx, flags);
if (err != 0) {
nvgpu_mutex_release(&vm->update_gmmu_lock);
goto free_vaddr;
}
@@ -252,10 +269,16 @@ int nvgpu_vm_area_alloc(struct vm_gk20a *vm, u64 pages, u32 page_size,
return 0;
free_vaddr:
#ifdef CONFIG_NVGPU_REMAP
if (vm_area->vpool != NULL) {
nvgpu_vm_remap_vpool_destroy(vm, vm_area);
vm_area->vpool = NULL;
}
#endif
nvgpu_free(vma, vaddr_start);
free_vm_area:
nvgpu_kfree(g, vm_area);
return -ENOMEM;
return err;
}
int nvgpu_vm_area_free(struct vm_gk20a *vm, u64 addr)
@@ -302,6 +325,13 @@ int nvgpu_vm_area_free(struct vm_gk20a *vm, u64 addr)
NULL);
}
#ifdef CONFIG_NVGPU_REMAP
/* clean up any remap resources */
if (vm_area->vpool != NULL) {
nvgpu_vm_remap_vpool_destroy(vm, vm_area);
}
#endif
nvgpu_mutex_release(&vm->update_gmmu_lock);
nvgpu_free(vm->vma[vm_area->pgsz_idx], vm_area->addr);

797
drivers/gpu/nvgpu/common/mm/vm_remap.c Normal file
View File

@@ -0,0 +1,797 @@
/*
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
#include <nvgpu/gk20a.h>
#include <nvgpu/vm.h>
#include <nvgpu/vm_area.h>
#include <nvgpu/vm_remap.h>
#include <nvgpu/comptags.h>
#include <nvgpu/string.h>
/*
* Return page size index of page size for VM areas that can be used with
* remap.
*/
static inline u32 nvgpu_vm_remap_pgsz_idx(struct vm_gk20a *vm)
{
return GMMU_PAGE_SIZE_BIG;
}
/*
* Return page size for VM areas that can be used with remap.
*/
static inline u64 nvgpu_vm_remap_page_size(struct vm_gk20a *vm)
{
return vm->gmmu_page_sizes[GMMU_PAGE_SIZE_BIG];
}
/*
* Return a pointer to the os-specific structure for the specified physical
* memory pool.
*/
static inline struct nvgpu_vm_remap_os_buffer *nvgpu_vm_remap_mpool_handle(
struct nvgpu_vm_remap_mpool *mpool)
{
if (mpool == NULL) {
return NULL;
}
return &mpool->remap_os_buf;
}
/*
* Add a reference to the specified physical memory pool.
*/
static inline struct nvgpu_vm_remap_mpool *nvgpu_vm_remap_mpool_get(
struct nvgpu_vm_remap_mpool *mpool)
{
nvgpu_ref_get(&mpool->ref);
return mpool;
}
/*
* Cleanup physical memory pool resources. This function is called when
* the reference count for the physical memory pool goes to zero.
*/
static void nvgpu_vm_remap_mpool_release(struct nvgpu_ref *ref)
{
struct nvgpu_vm_remap_mpool *mpool;
struct nvgpu_vm_remap_vpool *vpool;
struct vm_gk20a *vm;
struct gk20a *g;
mpool = nvgpu_vm_remap_mpool_from_ref(ref);
vpool = mpool->vpool;
vm = vpool->vm;
g = gk20a_from_vm(vm);
nvgpu_rbtree_unlink(&mpool->node, &vpool->mpools);
/* L2 must be flushed before we destroy any SMMU mappings. */
if (*mpool->l2_flushed == false) {
(void) g->ops.mm.cache.l2_flush(g, true);
*mpool->l2_flushed = true;
}
nvgpu_vm_remap_os_buf_put(vm, &mpool->remap_os_buf);
nvgpu_kfree(g, mpool);
}
/*
* Release a reference to the specified physical memory pool.
*/
static inline void nvgpu_vm_remap_mpool_put(struct vm_gk20a *vm,
struct nvgpu_vm_remap_vpool *vpool,
struct nvgpu_vm_remap_mpool *mpool,
bool *l2_flushed)
{
if (mpool != NULL) {
mpool->l2_flushed = l2_flushed;
nvgpu_ref_put(&mpool->ref, nvgpu_vm_remap_mpool_release);
mpool->l2_flushed = NULL;
}
}
/*
* Insert a physical memory pool into the rbtree of the specified virtual
* memory pool.
*/
static inline struct nvgpu_vm_remap_mpool *nvgpu_vm_remap_mpool_add(
struct vm_gk20a *vm,
struct nvgpu_vm_remap_vpool *vpool,
struct nvgpu_vm_remap_os_buffer *remap_os_buf)
{
struct gk20a *g = gk20a_from_vm(vm);
struct nvgpu_rbtree_node **root = &vpool->mpools;
struct nvgpu_vm_remap_mpool *mpool;
u64 key;
mpool = nvgpu_kzalloc(g, sizeof(*mpool));
if (mpool == NULL) {
return NULL;
}
key = nvgpu_vm_remap_get_handle(remap_os_buf);
mpool->node.key_start = key;
mpool->node.key_end = key;
nvgpu_ref_init(&mpool->ref);
mpool->vpool = vpool;
nvgpu_memcpy((u8 *)&mpool->remap_os_buf, (u8 *)remap_os_buf,
sizeof(mpool->remap_os_buf));
nvgpu_rbtree_insert(&mpool->node, root);
return mpool;
}
/*
* Return a pointer to the physical memory pool for the specified rbtree node.
*/
static inline struct nvgpu_vm_remap_mpool *
nvgpu_vm_remap_mpool_from_tree_entry(struct nvgpu_rbtree_node *node)
{
return (struct nvgpu_vm_remap_mpool *)
((uintptr_t)node - offsetof(struct nvgpu_vm_remap_mpool, node));
};
/*
* Return a pointer to the physical memory pool for the associated physical
* memory buffer.
*/
static inline struct nvgpu_vm_remap_mpool *
nvgpu_vm_remap_mpool_find(struct nvgpu_rbtree_node *root,
struct nvgpu_vm_remap_os_buffer *remap_os_buf)
{
struct nvgpu_rbtree_node *node;
u64 key = nvgpu_vm_remap_get_handle(remap_os_buf);
nvgpu_rbtree_search(key, &node, root);
if (node == NULL) {
return NULL;
}
return nvgpu_vm_remap_mpool_from_tree_entry(node);
}
#ifdef CONFIG_NVGPU_COMPRESSION
/*
* Ensure that compression resources are allocated to the specified
* physical memory buffer.
*/
static inline int nvgpu_vm_remap_ensure_comptags(struct vm_gk20a *vm,
struct nvgpu_vm_remap_os_buffer *remap_os_buf)
{
struct gk20a *g = gk20a_from_vm(vm);
struct gk20a_comptags comptags = { 0 };
struct nvgpu_os_buffer *os_buf = &remap_os_buf->os_buf;
int err = 0;
err = gk20a_alloc_or_get_comptags(g, os_buf,
&g->cbc->comp_tags,
&comptags);
if (err != 0) {
nvgpu_err(g, "cannot alloc comptags: %d", err);
return err;
}
if (comptags.needs_clear) {
nvgpu_assert(g->ops.cbc.ctrl != NULL);
if (gk20a_comptags_start_clear(os_buf)) {
err = g->ops.cbc.ctrl(g, nvgpu_cbc_op_clear,
comptags.offset,
(comptags.offset +
comptags.lines - 1U));
gk20a_comptags_finish_clear(os_buf, err == 0);
}
}
return err;
}
#endif
/*
* Validate that the specified remap operation resides within the target
* virtual memory pool.
*/
static int nvgpu_vm_remap_validate_vpool(struct nvgpu_vm_remap_vpool *vpool,
struct nvgpu_vm_remap_op *op)
{
u64 first_page = op->virt_offset_in_pages;
u64 last_page = op->virt_offset_in_pages + op->num_pages - 1ULL;
if (first_page < vpool->base_offset_in_pages ||
last_page >= vpool->base_offset_in_pages + vpool->num_pages ||
last_page < first_page) {
return -EINVAL;
}
return 0;
}
/*
* Validate an unmap operation.
*/
static int nvgpu_vm_remap_validate_unmap(struct vm_gk20a *vm,
struct nvgpu_vm_remap_vpool *vpool,
struct nvgpu_vm_remap_op *op)
{
return nvgpu_vm_remap_validate_vpool(vpool, op);
}
/*
* Validate a map operation.
*/
static int nvgpu_vm_remap_validate_map(struct vm_gk20a *vm,
struct nvgpu_vm_remap_vpool *vpool,
struct nvgpu_vm_remap_op *op,
struct nvgpu_vm_remap_os_buffer *remap_os_buf)
{
u64 page_size = nvgpu_vm_remap_page_size(vm);
u64 map_offset;
u64 map_size;
u64 os_buf_size;
map_offset = nvgpu_safe_mult_u64(op->mem_offset_in_pages, page_size);
map_size = nvgpu_safe_mult_u64(op->num_pages, page_size);
os_buf_size = nvgpu_os_buf_get_size(&remap_os_buf->os_buf);
if ((map_size > os_buf_size) ||
((os_buf_size - map_size) < map_offset)) {
return -EINVAL;
}
#ifdef CONFIG_NVGPU_COMPRESSION
if (op->compr_kind != NVGPU_KIND_INVALID) {
if (nvgpu_vm_remap_ensure_comptags(vm, remap_os_buf)) {
/* inform caller there are no more compbits */
op->compr_kind = NVGPU_KIND_INVALID;
}
}
#endif
return nvgpu_vm_remap_validate_vpool(vpool, op);
}
/*
* Return a pointer to the virtual pool for the specified remap operation.
* Note that this function must be called with the VM's GMMU update lock
* held.
*/
static struct nvgpu_vm_remap_vpool *nvgpu_vm_remap_get_vpool_locked(
struct vm_gk20a *vm, struct nvgpu_vm_remap_op *op)
{
struct gk20a *g = gk20a_from_vm(vm);
u64 page_size = nvgpu_vm_remap_page_size(vm);
u64 offset = nvgpu_safe_mult_u64(op->virt_offset_in_pages, page_size);
struct nvgpu_vm_area *vm_area = nvgpu_vm_area_find(vm, offset);
if ((vm_area == NULL) || (vm_area->vpool == NULL)) {
return NULL;
}
/* allocate per-page tracking if first time vpool is used */
if ((vm_area->vpool->mpool_by_page == NULL) &&
(vm_area->vpool->num_pages != 0)) {
struct nvgpu_vm_remap_mpool **mp;
mp = nvgpu_kzalloc(g, vm_area->vpool->num_pages *
sizeof(struct nvgpu_vm_remap_mpool *));
if (mp == NULL) {
return NULL;
}
vm_area->vpool->mpool_by_page = mp;
}
return vm_area->vpool;
}
/*
* Update physical memory pool reference counts for the specified virtual
* pool range of pages.
*/
static void nvgpu_vm_remap_update_pool_refcounts(
struct vm_gk20a *vm,
struct nvgpu_vm_remap_vpool *vpool,
u64 first_page,
u64 num_pages,
struct nvgpu_vm_remap_mpool *new_pool,
bool *l2_flushed)
{
u64 pgnum;
if (vpool->num_pages < first_page + num_pages) {
nvgpu_err(gk20a_from_vm(vm), "bad vpool range; update skipped");
return;
}
for (pgnum = first_page; pgnum < first_page + num_pages; pgnum++) {
/*
* Decrement refcount of the physical pool that was previously
* mapped to this page.
*/
nvgpu_vm_remap_mpool_put(vm, vpool,
vpool->mpool_by_page[pgnum], l2_flushed);
/*
* And record the fact that the current page now refers to
* "new_pool".
*/
vpool->mpool_by_page[pgnum] = new_pool;
/*
* Increment refcount of the new physical pool.
*/
if (new_pool != NULL) {
(void) nvgpu_vm_remap_mpool_get(new_pool);
}
}
}
/*
* Return the ctag offset (if applicable) for the specified map operation.
*/
static u64 nvgpu_vm_remap_get_ctag_offset(struct vm_gk20a *vm,
struct nvgpu_vm_remap_op *op,
struct nvgpu_os_buffer *os_buf,
s16 *kind)
{
#ifdef CONFIG_NVGPU_COMPRESSION
struct gk20a *g = gk20a_from_vm(vm);
struct gk20a_comptags comptags;
u64 ctag = 0;
u64 ctag_offset = 0;
u64 page_size = nvgpu_vm_remap_page_size(vm);
u64 phys_offset = nvgpu_safe_mult_u64(op->mem_offset_in_pages,
page_size);
gk20a_get_comptags(os_buf, &comptags);
if (comptags.lines != 0) {
ctag = (u64)comptags.offset;
}
if (op->compr_kind != NVGPU_KIND_INVALID) {
*kind = op->compr_kind;
if (ctag != 0) {
ctag_offset = ctag + (phys_offset >>
ilog2(g->ops.fb.compression_page_size(g)));
} else {
ctag_offset = 0;
}
} else {
*kind = op->incompr_kind;
ctag_offset = 0;
}
return ctag_offset;
#else
return 0;
#endif
}
static u32 nvgpu_vm_remap_get_map_flags(struct nvgpu_vm_remap_op *op)
{
u32 flags = 0;
if ((op->flags & NVGPU_VM_REMAP_OP_FLAGS_CACHEABLE) != 0U) {
flags = NVGPU_VM_MAP_CACHEABLE;
}
return flags;
}
static enum gk20a_mem_rw_flag nvgpu_vm_remap_get_map_rw_flag(
struct nvgpu_vm_remap_op *op)
{
enum gk20a_mem_rw_flag rw_flag = gk20a_mem_flag_none;
if ((op->flags & NVGPU_VM_REMAP_OP_FLAGS_ACCESS_NO_WRITE) != 0U) {
rw_flag = gk20a_mem_flag_read_only;
}
return rw_flag;
}
/*
* Execute remap operations in sequence.
* All remap operations must succeed for this routine to return success.
*/
static int nvgpu_vm_remap_execute_remaps(struct vm_gk20a *vm,
struct nvgpu_vm_remap_vpool *vpool,
struct nvgpu_vm_remap_mpool **mpools,
struct nvgpu_vm_remap_op *ops, u32 *num_ops,
bool *l2_flushed)
{
struct gk20a *g = gk20a_from_vm(vm);
struct nvgpu_vm_remap_op *op;
struct nvgpu_vm_remap_os_buffer *remap_os_buf;
u32 pgsz_idx = nvgpu_vm_remap_pgsz_idx(vm);
u64 page_size = nvgpu_vm_remap_page_size(vm);
u64 map_addr = 0;
u64 phys_offset = 0;
u64 map_size;
u64 ctag_offset = 0;
enum gk20a_mem_rw_flag rw_flag;
u32 flags;
struct nvgpu_vm_remap_os_buffer *curr_remap_os_buf = NULL;
struct vm_gk20a_mapping_batch batch;
s16 curr_kind = 0;
u32 i;
int err = 0;
nvgpu_vm_mapping_batch_start(&batch);
/* Update GPU page tables. */
for (i = 0; i < *num_ops; i++) {
op = &ops[i];
remap_os_buf = nvgpu_vm_remap_mpool_handle(mpools[i]);
map_size = nvgpu_safe_mult_u64(op->num_pages, page_size);
map_addr = nvgpu_safe_mult_u64(op->virt_offset_in_pages,
page_size);
phys_offset = nvgpu_safe_mult_u64(op->mem_offset_in_pages,
page_size);
if (remap_os_buf == NULL) {
/*
* Unmap range.
*/
(void) g->ops.mm.gmmu.unmap(vm,
map_addr,
map_size,
pgsz_idx,
false, /* va_allocated */
gk20a_mem_flag_none,
true, /* sparse */
&batch);
} else {
/*
* If physical pool changed from the previous ops,
* update curr_pool* variables.
*/
if (remap_os_buf != curr_remap_os_buf) {
curr_remap_os_buf = remap_os_buf;
}
ctag_offset = nvgpu_vm_remap_get_ctag_offset(vm, op,
&curr_remap_os_buf->os_buf,
&curr_kind);
flags = nvgpu_vm_remap_get_map_flags(op);
rw_flag = nvgpu_vm_remap_get_map_rw_flag(op);
/*
* Remap range.
*/
map_addr = g->ops.mm.gmmu.map(vm,
map_addr,
remap_os_buf->nv_sgt,
phys_offset,
map_size,
pgsz_idx,
(u8)curr_kind,
ctag_offset,
flags,
rw_flag,
false, /* ctags clear */
true,
false,
&batch,
remap_os_buf->aperture);
if (map_addr == 0ULL) {
nvgpu_err(g, "map addr is zero");
err = -ENOMEM;
break;
}
}
}
/*
* Handle possible error condition by updating num_ops to reflect
* the number of remap ops that actually succeeded.
*/
if (i != *num_ops) {
*num_ops = i;
}
nvgpu_vm_mapping_batch_finish_locked(vm, &batch);
/*
* Release references to the previously mapped pages of all
* remap operations that succeeded.
*/
for (i = 0; i < *num_ops; i++) {
op = &ops[i];
nvgpu_vm_remap_update_pool_refcounts(
vm,
vpool,
nvgpu_safe_sub_u64(op->virt_offset_in_pages,
vpool->base_offset_in_pages),
op->num_pages,
mpools[i],
l2_flushed);
}
return err;
}
static int nvgpu_vm_remap_get_mpool(struct vm_gk20a *vm,
struct nvgpu_vm_remap_vpool *vpool,
struct nvgpu_vm_remap_op *op,
struct nvgpu_vm_remap_mpool **curr_mpool,
u32 *curr_mem_handle)
{
struct nvgpu_vm_remap_os_buffer remap_os_buf = { 0 };
int err = 0;
if (op->mem_handle == *curr_mem_handle) {
/*
* Physical pool didn't change from the previous op, so we
* can skip validation and just use the curr_pool pointer
* again. Just take one extra ref.
*/
*curr_mpool = nvgpu_vm_remap_mpool_get(*curr_mpool);
} else {
/*
* Move to the next memory handle (validate access + acquire
* reference).
*/
err = nvgpu_vm_remap_os_buf_get(vm, op, &remap_os_buf);
if (err != 0) {
goto done;
}
/*
* Make sure the new memory handle is included in (and
* referenced by) the list of memory handles mapped in the
* virtual pool.
*/
*curr_mpool = nvgpu_vm_remap_mpool_find(vpool->mpools,
&remap_os_buf);
if (*curr_mpool != NULL) {
/*
* This memory handle was already mapped to the virtual
* pool, so we don't need to keep the extra reference
* to the nvmap handle.
*/
nvgpu_vm_remap_os_buf_put(vm, &remap_os_buf);
*curr_mpool = nvgpu_vm_remap_mpool_get(*curr_mpool);
} else {
/*
* Add the physical memory to the list of mapped
* handles.
*/
*curr_mpool = nvgpu_vm_remap_mpool_add(vm, vpool,
&remap_os_buf);
if (*curr_mpool == NULL) {
nvgpu_vm_remap_os_buf_put(vm, &remap_os_buf);
err = -ENOMEM;
goto done;
}
}
*curr_mem_handle = op->mem_handle;
}
done:
return err;
}
/*
* Prepare to execute remap operations. Allocate an array to track the
* associated physical memory pool for each specified operation and
* then validate the parameters for each operation. Note that this function
* must be called with the VM's GMMU update lock held.
*/
static int nvgpu_vm_remap_locked(struct vm_gk20a *vm,
struct nvgpu_vm_remap_vpool *vpool,
struct nvgpu_vm_remap_op *ops, u32 *num_ops)
{
struct gk20a *g = gk20a_from_vm(vm);
bool l2_flushed = false;
struct nvgpu_vm_remap_mpool **mpools;
struct nvgpu_vm_remap_mpool *curr_mpool = NULL;
u32 curr_mem_handle = 0U;
u32 num_ops_validated = 0U;
u32 i;
int err = 0;
if (*num_ops == 0U) {
return 0;
}
if (vpool == NULL) {
return -EINVAL;
}
mpools = nvgpu_kzalloc(g, *num_ops * sizeof(mpools[0]));
if (mpools == NULL) {
return -ENOMEM;
}
/* We cache the validated memory handle across ops to avoid
* revalidation in the common case where the physical pool doesn't
* change between ops.
*/
for (i = 0; i < *num_ops; i++) {
struct nvgpu_vm_remap_op *op = &ops[i];
if (op->mem_handle == 0U) {
err = nvgpu_vm_remap_validate_unmap(vm, vpool, op);
if (err != 0) {
nvgpu_err(g, "validate_unmap failed: %d", err);
break;
}
} else {
err = nvgpu_vm_remap_get_mpool(vm, vpool, op,
&curr_mpool, &curr_mem_handle);
if (err != 0) {
nvgpu_err(g, "get_mpool failed: %d", err);
break;
}
/*
* Validate that the mapping request is valid.
* This may demote the kind from compressed to
* uncompressed if we have run out of compbits.
*/
err = nvgpu_vm_remap_validate_map(vm, vpool, op,
nvgpu_vm_remap_mpool_handle(curr_mpool));
if (err != 0) {
nvgpu_err(g, "validate_map failed: %d", err);
nvgpu_vm_remap_mpool_put(vm, vpool,
curr_mpool, &l2_flushed);
break;
}
mpools[i] = curr_mpool;
}
}
num_ops_validated = i;
if (err == 0) {
/*
* The validation stage completed without errors so
* execute all map and unmap operations sequentially.
*/
err = nvgpu_vm_remap_execute_remaps(vm, vpool, mpools,
ops, num_ops, &l2_flushed);
} else {
/*
* Validation failed so set successful num_ops count to zero.
*/
*num_ops = 0;
}
/* Release references acquired during ops validation. */
for (i = 0; i < num_ops_validated; i++) {
nvgpu_vm_remap_mpool_put(vm, vpool,
mpools[i], &l2_flushed);
}
nvgpu_kfree(g, mpools);
return err;
}
/*
* Top-level remap handler. This function is used by the os-specific REMAP
* API handler to execute remap operations.
*/
int nvgpu_vm_remap(struct vm_gk20a *vm,
struct nvgpu_vm_remap_op *ops,
u32 *num_ops)
{
int ret = 0;
struct nvgpu_vm_remap_vpool *vpool;
nvgpu_mutex_acquire(&vm->update_gmmu_lock);
vpool = nvgpu_vm_remap_get_vpool_locked(vm, &ops[0]);
if (vpool != NULL) {
ret = nvgpu_vm_remap_locked(vm, vpool, ops, num_ops);
} else {
*num_ops = 0;
ret = -EINVAL;
}
nvgpu_mutex_release(&vm->update_gmmu_lock);
return ret;
}
/*
* Create a virtual memory pool.
*/
int nvgpu_vm_remap_vpool_create(struct vm_gk20a *vm,
struct nvgpu_vm_area *vm_area,
u64 num_pages)
{
struct gk20a *g = gk20a_from_vm(vm);
struct nvgpu_vm_remap_vpool *vp;
u64 start_page_nr = 0;
if ((num_pages == 0ULL) ||
(vm_area->pgsz_idx != GMMU_PAGE_SIZE_BIG) ||
((vm_area->flags & NVGPU_VM_AREA_ALLOC_SPARSE) == 0U)) {
return -EINVAL;
}
vp = nvgpu_kzalloc(g, sizeof(struct nvgpu_vm_remap_vpool));
if (vp == NULL) {
return -ENOMEM;
}
start_page_nr = vm_area->addr >>
ilog2(vm->gmmu_page_sizes[vm_area->pgsz_idx]);
vp->base_offset_in_pages = start_page_nr;
vp->num_pages = num_pages;
vp->vm = vm;
vm_area->vpool = vp;
return 0;
}
/*
* Destroy a virtual memory pool.
*/
void nvgpu_vm_remap_vpool_destroy(struct vm_gk20a *vm,
struct nvgpu_vm_area *vm_area)
{
struct gk20a *g = gk20a_from_vm(vm);
struct nvgpu_vm_remap_vpool *vpool = vm_area->vpool;
if (vpool == NULL) {
return;
}
if (vpool->mpools != NULL) {
struct nvgpu_vm_remap_op op = { 0 };
u32 num_ops = 1;
int err;
op.virt_offset_in_pages = vpool->base_offset_in_pages;
op.num_pages = vpool->num_pages;
err = nvgpu_vm_remap_locked(vm, vpool, &op, &num_ops);
nvgpu_assert(err == 0);
}
nvgpu_assert(vpool->mpools == NULL);
if (vpool->mpool_by_page != NULL) {
nvgpu_kfree(g, vpool->mpool_by_page);
}
nvgpu_kfree(g, vpool);
vm_area->vpool = NULL;
return;
}

2
drivers/gpu/nvgpu/include/nvgpu/enabled.h
View File

@@ -82,6 +82,8 @@ struct gk20a;
"Support batch mapping"), \
DEFINE_FLAG(NVGPU_SUPPORT_MAPPING_MODIFY, \
"Support mapping modify"), \
DEFINE_FLAG(NVGPU_SUPPORT_REMAP, \
"Support remap"), \
DEFINE_FLAG(NVGPU_USE_COHERENT_SYSMEM, \
"Use coherent aperture for sysmem"), \
DEFINE_FLAG(NVGPU_MM_USE_PHYSICAL_SG, \

54
drivers/gpu/nvgpu/include/nvgpu/linux/vm_remap.h Normal file
View File

@@ -0,0 +1,54 @@
/*
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
#ifndef __COMMON_LINUX_VM_REMAP_H__
#define __COMMON_LINUX_VM_REMAP_H__
#include <nvgpu/vm.h>
#include <nvgpu/types.h>
struct nvgpu_vm_remap_op;
struct nvgpu_as_remap_op;
/* struct tracking os-specific remap memory buffer state */
struct nvgpu_vm_remap_os_buffer {
struct nvgpu_os_buffer os_buf;
struct nvgpu_mapped_buf_priv os_priv;
struct nvgpu_sgt *nv_sgt;
u64 aperture;
};
int nvgpu_vm_remap_translate_as_op(struct vm_gk20a *vm,
struct nvgpu_vm_remap_op *vm_op,
struct nvgpu_as_remap_op *as_op);
void nvgpu_vm_remap_translate_vm_op(struct nvgpu_as_remap_op *as_op,
struct nvgpu_vm_remap_op *vm_op);
u64 nvgpu_vm_remap_get_handle(struct nvgpu_vm_remap_os_buffer *remap_os_buf);
int nvgpu_vm_remap_os_buf_get(struct vm_gk20a *vm,
struct nvgpu_vm_remap_op *op,
struct nvgpu_vm_remap_os_buffer *remap_os_buf);
void nvgpu_vm_remap_os_buf_put(struct vm_gk20a *vm,
struct nvgpu_vm_remap_os_buffer *remap_os_buf);
#endif /* __COMMON_LINUX_VM_REMAP_H__ */

42
drivers/gpu/nvgpu/include/nvgpu/posix/posix-vm_remap.h Normal file
View File

@@ -0,0 +1,42 @@
/*
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
#ifndef NVGPU_POSIX_VM__REMAP_H
#define NVGPU_POSIX_VM_REMAP_H
#include <nvgpu/types.h>
struct nvgpu_vm_remap_os_buffer {
struct nvgpu_os_buffer os_buf;
struct nvgpu_mapped_buf_priv os_priv;
struct nvgpu_sgt *nv_sgt;
u64 aperture;
};
u64 nvgpu_vm_remap_get_handle(struct nvgpu_vm_remap_os_buffer *remap_os_buf);
int nvgpu_vm_remap_os_buf_get(struct vm_gk20a *vm, u32 mem_handle,
struct nvgpu_vm_remap_os_buffer *remap_os_buf);
void nvgpu_vm_remap_os_buf_put(struct vm_gk20a *vm,
struct nvgpu_vm_remap_os_buffer *remap_os_buf);
#endif /* NVGPU_POSIX_VM_REMAP_H */

16
drivers/gpu/nvgpu/include/nvgpu/vm_area.h
View File

@@ -30,6 +30,8 @@ struct vm_gk20a;
struct gk20a_as_share;
struct nvgpu_as_alloc_space_args;
struct nvgpu_as_free_space_args;
struct nvgpu_vm_remap_vpool;
/**
* Carve out virtual address space from virtual memory context.
* This is needed for fixed address mapping.
@@ -66,6 +68,13 @@ struct nvgpu_vm_area {
* @sa NVGPU_VM_AREA_ALLOC_SPARSE
*/
bool sparse;
#ifdef CONFIG_NVGPU_REMAP
/**
* Virtual pool for remap support of sparse VM areas.
*/
struct nvgpu_vm_remap_vpool *vpool;
#endif
};
static inline struct nvgpu_vm_area *
@@ -91,6 +100,13 @@ nvgpu_vm_area_from_vm_area_list(struct nvgpu_list_node *node)
*/
#define NVGPU_VM_AREA_ALLOC_SPARSE BIT32(1)
/**
* Enable REMAP control of the the vmarea. REMAP uses a virtual
* memory pool that provides control over each page in the vmarea.
* Note that REMAP is only permitted with SPARSE vmareas.
*/
#define NVGPU_VM_AREA_ALLOC_REMAP BIT32(2)
/**
* @brief Allocate a virtual memory area with given size and start.
*

278
drivers/gpu/nvgpu/include/nvgpu/vm_remap.h Normal file
View File

@@ -0,0 +1,278 @@
/*
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
#ifndef NVGPU_VM_REMAP_H
#define NVGPU_VM_REMAP_H
#include <nvgpu/vm.h>
#include <nvgpu/types.h>
#ifdef __KERNEL__
#include <nvgpu/linux/vm_remap.h>
#elif defined(__NVGPU_POSIX__)
#include <nvgpu/posix/posix-vm_remap.h>
#endif
/*
* Supported remap operation flags.
*/
#define NVGPU_VM_REMAP_OP_FLAGS_CACHEABLE BIT32(1)
#define NVGPU_VM_REMAP_OP_FLAGS_ACCESS_NO_WRITE BIT32(7)
/**
* This structure describes a single remap operation (either a map or unmap).
*/
struct nvgpu_vm_remap_op {
/**
* When a map operation is specified this field contains any flags
* to use when setting up the mapping. When an unmap operation is
* specified this field must be zero.
*/
u32 flags;
/**
* When a map operation is specified this field can be used to specify
* the compressed kind for the mapping. If the specified value is
* NVGPU_KIND_INVALID then no compression resources are requested and
* the #incompr_kind value is used for the mapping. If a value other
* than NVGPU_KIND_INVALID is specified but there are no compression
* resources available for the mapping then the #incompr_kind value
* is used as a fallback for the mapping. When an unmap operation
* is specified this value must be zero.
*/
s16 compr_kind;
/**
* When a map operation is specified and the #compr_kind field is
* NVGPU_KIND_INVALID then this field specifies the incompressed
* kind to use for the mapping. When an unmap operation is specified
* this value must be zero.
*/
s16 incompr_kind;
/**
* This field is used to distinguish between a map and unmap operation.
* When this field is non-zero then it indicates a map operation with
* the value containing the handle to the physical memory buffer to
* map into the virtual pool. When this field is zero then it
* indicates an unmap operation.
*/
u32 mem_handle;
/**
* Page offset into the memory buffer referenced by #mem_handle from
* which physical memory should be mapped.
*/
u64 mem_offset_in_pages;
/**
* Page offset into the virtual pool at which to start the mapping.
*/
u64 virt_offset_in_pages;
/**
* Number of pages to map or unmap.
*/
u64 num_pages;
};
/*
* Defined by each OS. Allows the common VM code do things to the OS specific
* buffer structures.
*/
struct nvgpu_os_vm_remap_buffer;
/**
* This structure describes a physical memory pool.
* There is one physical memory pool for each physical memory buffer that
* is mapped into the corresponding virtual pool.
*/
struct nvgpu_vm_remap_mpool {
/**
* Red black tree node to the memory pool.
*/
struct nvgpu_rbtree_node node;
/**
* Number of references to this physical memory pool. This
* value increments for each map operation and decrements with
* each unmap operation that references the associated physical
* memory buffer tracked by #remap_os_buf. When the reference
* count goes to zero then the reference to the associated
* physical memory buffer tracked by #remap_os_buf is released.
*/
struct nvgpu_ref ref;
/**
* If non-NULL, the ref put function will check this l2 flag and issue
* a flush if necessary when releasing a mapping.
*/
bool *l2_flushed;
/**
* OS-specific structure that tracks the associated physical memory
* buffer.
*/
struct nvgpu_vm_remap_os_buffer remap_os_buf;
/**
* Pointer to virtual pool into which this physical memory pool
* is mapped.
*/
struct nvgpu_vm_remap_vpool *vpool;
};
static inline struct nvgpu_vm_remap_mpool *
nvgpu_vm_remap_mpool_from_ref(struct nvgpu_ref *ref)
{
return (struct nvgpu_vm_remap_mpool *)
((uintptr_t)ref -
offsetof(struct nvgpu_vm_remap_mpool, ref));
}
/**
* This structure describes a virtual memory pool.
* There is one virtual memory pool for each sparse VM area allocation that
* uses big pages. A virtual memory pool tracks the association between
* each mapped big page in the pool and the corresponding physical memory.
*/
struct nvgpu_vm_remap_vpool {
/**
* Pointer to associated VM.
*/
struct vm_gk20a *vm;
/**
* Tree of physical memory pools that are currently mapped to this
* virtual pool.
*/
struct nvgpu_rbtree_node *mpools;
/**
* Base offset in pages within the associated VM context of the
* virtual memory pool. This value is specified to
* #nvgpu_vm_remap_vpool_create when the associated VM area is
* allocated.
*/
u64 base_offset_in_pages;
/*
* Number of pages mapped into the virtual memory pool.
*/
u64 num_pages;
/**
* Pointer to array of physical memory pool pointers (one per page
* in the virtual memory pool).
*/
struct nvgpu_vm_remap_mpool **mpool_by_page;
};
/**
* @brief Process list of remap operations on a sparse VM area.
*
* @param vm [in] Pointer to VM context.
* @param ops [in] Pointer to list of remap operations.
* @param num_ops [in/out] On input number of entries in #ops list.
* On output number of successful remap
* operations.
*
* This function processes one or more remap operations on the virtual memory
* pool associatd with the target sparse VM area. The operations are first
* validated and then executed in order.
*
* - Acquire the vm.update_gmmu_lock.
* - Get the associated VM area using #nvgpu_vm_find_area.
* - Verify VM area is valid and has a valid virtual pool.
* - Allocate (temporary) storage for a physical memory pool structure
* for each operation in the #ops list.
* - Perform validation of each operation in the #ops list.
* - For unmap operations validate that the specified virtual memory range
* resides entirely within the target virtual memory pool.
* - For map operations validate that the specified physical memory range
* resides entirely within the associated physical memory buffer and that
* the specified virtual memory range resides entirely within the target
* virtual memory pool. Also process compression resource requests including
* fallback handling if compression resources are desired but not available.
* - If validation succeeds then proceed to process the remap operations
* in sequence. Unmap operations remove the specified mapping from the
* GPU page tables (note this sets the sparse for each unmapped page). Map
* operations insert the specified mapping into the GPU page tables.
* - Update physical memory pool reference counts for each operation in the
* #ops list. If a memory pool reference count goes to zero then release
* the reference to the associated physical memory buffer.
* - Release the vm.update_gmmu_lock.
*
* @return Zero if the virtual pool init succeeds.
* Suitable errors, for failures.
* @retval -EINVAL if a value of zero is specified for #num_ops.
* @retval -EINVAL if there is no valid virtual pool for the target VM area.
* @retval -EINVAL if the specified virtual address range does not reside
* entirely within the target VM area.
* @retval -EINVAL if the specified physical address range does not reside
* entirely within the source physical memory buffer.
* @retval -ENOMEM if memory allocation for physical memory pool structures
* fails.
*/
int nvgpu_vm_remap(struct vm_gk20a *vm, struct nvgpu_vm_remap_op *ops,
u32 *num_ops);
/**
* @brief Create a virtual memory pool.
*
* @param vm [in] Pointer to VM context.
* @param vm_area [in] Pointer to pointer VM area.
* @param num_pages [in] Number of pages in virtual memory pool.
*
* - Check that #num_pages is non-zero.
* - Check that VM area is using big pages.
* - Check that VM area is configured as sparse.
* - Allocate memory for internal virtual pool management structures.
* - Initialize virtual pool management structures including storing #vm
* and #num_pages arguments.
* - Store virtual memory pool pointer in #vm_area.
*
* @return Zero if the virtual pool create succeeds.
* Suitable errors, for failures.
* @retval -EINVAL if a value of zero is specified for #num_pages.
* @retval -EINVAL if the VM area is not using big pages.
* @retval -EINVAL if the VM area is not configured as sparse.
* @retval -ENOMEM if memory allocation for internal resources fails.
*/
int nvgpu_vm_remap_vpool_create(struct vm_gk20a *vm,
struct nvgpu_vm_area *vm_area,
u64 num_pages);
/**
* @brief Destroy a virtual memory pool.
*
* @param vm [in] Pointer to VM context.
* @param vm_area [in] Pointer to VM area.
*
* - Release references to physical memory pools (if any).
* - Free memory used to track virtual memory pool state.
*
* @return None.
*/
void nvgpu_vm_remap_vpool_destroy(struct vm_gk20a *vm,
struct nvgpu_vm_area *vm_area);
#endif /* NVGPU_VM_REMAP_H */

86
drivers/gpu/nvgpu/os/linux/ioctl_as.c
View File

@@ -23,6 +23,7 @@
#include <nvgpu/gmmu.h>
#include <nvgpu/mm.h>
#include <nvgpu/vm_area.h>
#include <nvgpu/vm_remap.h>
#include <nvgpu/log2.h>
#include <nvgpu/gk20a.h>
#include <nvgpu/nvgpu_init.h>
@@ -325,6 +326,82 @@ static int nvgpu_as_ioctl_mapping_modify(
args->buffer_size);
}
static int nvgpu_as_ioctl_remap(
struct gk20a_as_share *as_share,
struct nvgpu_as_remap_args *args)
{
struct gk20a *g = gk20a_from_vm(as_share->vm);
struct nvgpu_as_remap_op __user *user_remap_ops = NULL;
struct nvgpu_as_remap_op remap_op;
struct nvgpu_vm_remap_op *nvgpu_vm_remap_ops = NULL;
u32 i;
int err = 0;
nvgpu_log_fn(g, " ");
if (!nvgpu_is_enabled(g, NVGPU_SUPPORT_REMAP)) {
return -ENOTTY;
}
if (args->num_ops == 0) {
return 0;
}
/* allocate buffer for internal representation of remap ops */
nvgpu_vm_remap_ops = nvgpu_kzalloc(g, args->num_ops *
sizeof(struct nvgpu_vm_remap_op));
if (nvgpu_vm_remap_ops == NULL) {
err = -ENOMEM;
goto out;
}
user_remap_ops =
(struct nvgpu_as_remap_op __user *)(uintptr_t)args->ops;
for (i = 0; i < args->num_ops; i++) {
if (copy_from_user(&remap_op, &user_remap_ops[i],
sizeof(remap_op))) {
err = -EFAULT;
goto out;
}
err = nvgpu_vm_remap_translate_as_op(as_share->vm,
&nvgpu_vm_remap_ops[i],
&remap_op);
if (err != 0) {
args->num_ops = 0;
goto out;
}
}
/* execute remap ops */
err = nvgpu_vm_remap(as_share->vm, nvgpu_vm_remap_ops,
&args->num_ops);
if (err != 0) {
goto out;
}
/* update user params */
for (i = 0; i < args->num_ops; i++) {
nvgpu_vm_remap_translate_vm_op(&remap_op,
&nvgpu_vm_remap_ops[i]);
if (copy_to_user(&user_remap_ops[i], &remap_op,
sizeof(remap_op))) {
err = -EFAULT;
args->num_ops = i;
goto out;
}
}
out:
if (nvgpu_vm_remap_ops != NULL) {
nvgpu_kfree(g, nvgpu_vm_remap_ops);
}
return err;
}
int gk20a_as_dev_open(struct inode *inode, struct file *filp)
{
struct gk20a_as_share *as_share;
@@ -371,7 +448,7 @@ long gk20a_as_dev_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
int err = 0;
struct gk20a_as_share *as_share = filp->private_data;
struct gk20a *g = gk20a_from_as(as_share->as);
bool always_copy_to_user = false;
u8 buf[NVGPU_AS_IOCTL_MAX_ARG_SIZE];
nvgpu_log_fn(g, "start %d", _IOC_NR(cmd));
@@ -466,6 +543,11 @@ long gk20a_as_dev_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
err = nvgpu_as_ioctl_mapping_modify(as_share,
(struct nvgpu_as_mapping_modify_args *)buf);
break;
case NVGPU_AS_IOCTL_REMAP:
err = nvgpu_as_ioctl_remap(as_share,
(struct nvgpu_as_remap_args *)buf);
always_copy_to_user = true;
break;
default:
err = -ENOTTY;
break;
@@ -473,7 +555,7 @@ long gk20a_as_dev_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
gk20a_idle(g);
if ((err == 0) && (_IOC_DIR(cmd) & _IOC_READ))
if ((err == 0 || always_copy_to_user) && (_IOC_DIR(cmd) & _IOC_READ))
if (copy_to_user((void __user *)arg, buf, _IOC_SIZE(cmd)))
err = -EFAULT;

2
drivers/gpu/nvgpu/os/linux/ioctl_ctrl.c
View File

@@ -268,6 +268,8 @@ static struct nvgpu_flags_mapping flags_mapping[] = {
NVGPU_SUPPORT_FAULT_RECOVERY},
{NVGPU_GPU_FLAGS_SUPPORT_MAPPING_MODIFY,
NVGPU_SUPPORT_MAPPING_MODIFY},
{NVGPU_GPU_FLAGS_SUPPORT_REMAP,
NVGPU_SUPPORT_REMAP},
{NVGPU_GPU_FLAGS_SUPPORT_COMPRESSION,
NVGPU_SUPPORT_COMPRESSION},
{NVGPU_GPU_FLAGS_SUPPORT_SM_TTU,

1
drivers/gpu/nvgpu/os/linux/module.c
View File

@@ -312,6 +312,7 @@ void gk20a_init_linux_characteristics(struct gk20a *g)
nvgpu_set_enabled(g, NVGPU_SUPPORT_PARTIAL_MAPPINGS, true);
nvgpu_set_enabled(g, NVGPU_SUPPORT_DETERMINISTIC_OPTS, true);
nvgpu_set_enabled(g, NVGPU_SUPPORT_USERSPACE_MANAGED_AS, true);
nvgpu_set_enabled(g, NVGPU_SUPPORT_REMAP, true);
if (!IS_ENABLED(CONFIG_NVGPU_SYNCFD_NONE)) {
nvgpu_set_enabled(g, NVGPU_SUPPORT_SYNC_FENCE_FDS, true);

242
drivers/gpu/nvgpu/os/linux/vm_remap.c Normal file
View File

@@ -0,0 +1,242 @@
/*
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
#include <linux/dma-buf.h>
#include <uapi/linux/nvgpu.h>
#include <nvgpu/gk20a.h>
#include <nvgpu/vm.h>
#include <nvgpu/vm_remap.h>
#include <nvgpu/nvgpu_sgt.h>
#include "os_linux.h"
#include "dmabuf_priv.h"
#define dev_from_vm(vm) dev_from_gk20a(vm->mm->g)
u64 nvgpu_vm_remap_get_handle(struct nvgpu_vm_remap_os_buffer *remap_os_buf)
{
return (u64)(uintptr_t)remap_os_buf->os_priv.dmabuf;
}
int nvgpu_vm_remap_os_buf_get(struct vm_gk20a *vm,
struct nvgpu_vm_remap_op *op,
struct nvgpu_vm_remap_os_buffer *remap_os_buf)
{
struct gk20a *g = gk20a_from_vm(vm);
struct device *dev = dev_from_gk20a(g);
struct dma_buf *dmabuf;
struct sg_table *sgt = NULL;
struct nvgpu_sgt *nv_sgt = NULL;
struct dma_buf_attachment *attachment;
enum nvgpu_aperture aperture;
enum dma_data_direction dmabuf_direction;
int err = 0;
/* get ref to the dmabuf fd */
dmabuf = dma_buf_get(op->mem_handle);
if (IS_ERR(dmabuf)) {
nvgpu_warn(g, "mem_handle 0x%x is not a dmabuf",
op->mem_handle);
return -EINVAL;
}
if (!(dmabuf->file->f_mode & (FMODE_WRITE | FMODE_PWRITE)) &&
!(op->flags & NVGPU_VM_REMAP_OP_FLAGS_ACCESS_NO_WRITE)) {
nvgpu_err(g, "RW access requested for RO mapped buffer");
err = -EINVAL;
goto clean_up;
}
err = gk20a_dmabuf_alloc_drvdata(dmabuf, dev_from_vm(vm));
if (err) {
nvgpu_warn(g, "failed to alloc drvdata");
goto clean_up;
}
if ((op->flags & NVGPU_VM_REMAP_OP_FLAGS_ACCESS_NO_WRITE) != 0) {
dmabuf_direction = DMA_TO_DEVICE;
} else {
dmabuf_direction = DMA_BIDIRECTIONAL;
}
sgt = nvgpu_mm_pin(dev, dmabuf, &attachment, dmabuf_direction);
if (IS_ERR(sgt)) {
nvgpu_warn(g, "failed to pin dma_buf");
err = -ENOMEM;
goto clean_up;
}
aperture = gk20a_dmabuf_aperture(g, dmabuf);
if (aperture == APERTURE_INVALID) {
err = -EINVAL;
goto clean_up;
}
nv_sgt = nvgpu_linux_sgt_create(g, sgt);
if (nv_sgt == NULL) {
nvgpu_warn(g, "failed to create nv_sgt");
err = -ENOMEM;
goto clean_up;
}
remap_os_buf->os_priv.dmabuf = dmabuf;
remap_os_buf->os_priv.attachment = attachment;
remap_os_buf->os_priv.sgt = sgt;
remap_os_buf->os_buf.dmabuf = dmabuf;
remap_os_buf->os_buf.attachment = attachment;
remap_os_buf->os_buf.dev = dev;
remap_os_buf->nv_sgt = nv_sgt;
remap_os_buf->aperture = aperture;
return 0;
clean_up:
if (nv_sgt != NULL) {
nvgpu_sgt_free(g, nv_sgt);
}
if (IS_ERR(sgt)) {
nvgpu_mm_unpin(dev, dmabuf, attachment, sgt);
}
dma_buf_put(dmabuf);
return err;
}
void nvgpu_vm_remap_os_buf_put(struct vm_gk20a *vm,
struct nvgpu_vm_remap_os_buffer *remap_os_buf)
{
struct gk20a *g = gk20a_from_vm(vm);
struct device *dev = dev_from_gk20a(g);
struct gk20a_comptags comptags;
nvgpu_mm_unpin(dev, remap_os_buf->os_priv.dmabuf,
remap_os_buf->os_priv.attachment, remap_os_buf->os_priv.sgt);
gk20a_get_comptags(&remap_os_buf->os_buf, &comptags);
/*
* Flush compression bit cache before releasing the physical
* memory buffer reference.
*/
if (comptags.offset != 0) {
g->ops.cbc.ctrl(g, nvgpu_cbc_op_clean, 0, 0);
g->ops.mm.cache.l2_flush(g, true);
}
nvgpu_sgt_free(g, remap_os_buf->nv_sgt);
dma_buf_put(remap_os_buf->os_priv.dmabuf);
}
static int nvgpu_vm_remap_validate_map_op(struct nvgpu_as_remap_op *op)
{
int err = 0;
u32 valid_flags = (NVGPU_AS_REMAP_OP_FLAGS_CACHEABLE |
NVGPU_AS_REMAP_OP_FLAGS_ACCESS_NO_WRITE);
if ((op->flags & ~valid_flags) != 0) {
err = -EINVAL;
}
return err;
}
static int nvgpu_vm_remap_validate_unmap_op(struct nvgpu_as_remap_op *op)
{
int err = 0;
if ((op->compr_kind != NVGPU_KIND_INVALID) ||
(op->incompr_kind != NVGPU_KIND_INVALID) ||
(op->flags != 0) ||
(op->mem_offset_in_pages != 0)) {
err = -EINVAL;
}
return err;
}
static u32 nvgpu_vm_remap_translate_as_flags(u32 flags)
{
u32 core_flags = 0;
if ((flags & NVGPU_AS_REMAP_OP_FLAGS_CACHEABLE) != 0) {
core_flags |= NVGPU_VM_REMAP_OP_FLAGS_CACHEABLE;
}
if ((flags & NVGPU_AS_REMAP_OP_FLAGS_ACCESS_NO_WRITE) != 0) {
core_flags |= NVGPU_VM_REMAP_OP_FLAGS_ACCESS_NO_WRITE;
}
return core_flags;
}
int nvgpu_vm_remap_translate_as_op(struct vm_gk20a *vm,
struct nvgpu_vm_remap_op *vm_op,
struct nvgpu_as_remap_op *as_op)
{
int err = 0;
u64 page_size;
u64 max_num_pages;
if (as_op->mem_handle == 0) {
err = nvgpu_vm_remap_validate_unmap_op(as_op);
} else {
err = nvgpu_vm_remap_validate_map_op(as_op);
}
if (err != 0)
goto clean_up;
page_size = vm->gmmu_page_sizes[GMMU_PAGE_SIZE_BIG];
max_num_pages = (ULONG_MAX / page_size);
if ((as_op->num_pages == 0) ||
(as_op->num_pages > max_num_pages) ||
(as_op->mem_offset_in_pages > max_num_pages) ||
(as_op->virt_offset_in_pages > max_num_pages)) {
err = -EINVAL;
goto clean_up;
}
vm_op->flags = nvgpu_vm_remap_translate_as_flags(as_op->flags);
vm_op->compr_kind = as_op->compr_kind;
vm_op->incompr_kind = as_op->incompr_kind;
vm_op->mem_handle = as_op->mem_handle;
vm_op->mem_offset_in_pages = as_op->mem_offset_in_pages;
vm_op->virt_offset_in_pages = as_op->virt_offset_in_pages;
vm_op->num_pages = as_op->num_pages;
return 0;
clean_up:
return err;
}
void nvgpu_vm_remap_translate_vm_op(struct nvgpu_as_remap_op *as_op,
struct nvgpu_vm_remap_op *vm_op)
{
as_op->compr_kind = vm_op->compr_kind;
as_op->incompr_kind = vm_op->incompr_kind;
}

40
drivers/gpu/nvgpu/os/posix/posix-vm_remap.c Normal file
View File

@@ -0,0 +1,40 @@
/*
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
#include <nvgpu/vm.h>
#include <nvgpu/vm_remap.h>
u64 nvgpu_vm_remap_get_handle(struct nvgpu_vm_remap_os_buffer *remap_os_buf)
{
return 0;
}
int nvgpu_vm_remap_os_buf_get(struct vm_gk20a *vm, u32 mem_handle,
struct nvgpu_vm_remap_os_buffer *remap_os_buf)
{
return 0;
}
void nvgpu_vm_remap_os_buf_put(struct vm_gk20a *vm,
struct nvgpu_vm_remap_os_buffer *remap_os_buf)
{
return;
}

137
include/uapi/linux/nvgpu-as.h
View File

@@ -431,6 +431,137 @@ struct nvgpu_as_mapping_modify_args {
__u64 map_address; /* in, base virtual address of mapped buffer */
};
/*
* VM remap operation.
*
* The VM remap operation structure represents a single map or unmap operation
* to be executed by the NVGPU_AS_IOCTL_REMAP ioctl.
*
* The format of the structure is as follows:
*
* @flags [IN]
*
* The following remap operation flags are supported:
*
* %NVGPU_AS_REMAP_OP_FLAGS_CACHEABLE
*
* Specify that the associated mapping shall be GPU cachable.
*
* %NVGPU_AS_REMAP_OP_FLAGS_ACCESS_NO_WRITE
*
* Specify that the associated mapping shall be read-only. This flag
* must be set if the physical memory buffer represented by @mem_handle
* is mapped read-only.
*
* This field must be zero for unmap operations.
*
* @compr_kind [IN/OUT]
* @incompr_kind [IN/OUT]
*
* On input these fields specify the compressible and incompressible kinds
* to be used for the mapping. If @compr_kind is not set to NV_KIND_INVALID
* then nvgpu will attempt to allocate compression resources. If
* @compr_kind is set to NV_KIND_INVALID or there are no compression
* resources then nvgpu will attempt to use @incompr_kind. If both
* @compr_kind and @incompr_kind are set to NV_KIND_INVALID then -EINVAL is
* returned. These fields must be set to NV_KIND_INVALID for unmap
* operations. On output these fields return the selected kind. If
* @compr_kind is set to a valid compressible kind but the required
* compression resources are not available then @compr_kind will return
* NV_INVALID_KIND and the @incompr_kind value will be used for the mapping.
*
* @mem_handle [IN]
*
* Specify the memory handle (dmabuf_fd) associated with the physical
* memory buffer to be mapped. This field must be zero for unmap
* operations.
*
* @mem_offset_in_pages [IN]
*
* Specify an offset into the physical buffer associated with mem_handle at
* which to start the mapping. This value is in pages and the page size
* is the big page size in the associated sparse address space. This value
* must be zero for unmap operations.
*
* @virt_offset_in_pages [IN]
*
* Specify the virtual memory start offset of the region to map or unmap.
* This value is in pages and the page size is the big page size in the
* associated sparse address space.
*
* @num_pages [IN]
* Specify the number of pages to map or unmap.
*/
struct nvgpu_as_remap_op {
#define NVGPU_AS_REMAP_OP_FLAGS_CACHEABLE (1 << 2)
#define NVGPU_AS_REMAP_OP_FLAGS_ACCESS_NO_WRITE (1 << 10)
/* in: For map operations, this field specifies the mask of
* NVGPU_AS_REMAP flags to use for the mapping. For unmap operations
* this field must be zero */
__u32 flags;
/* in: For map operations, this field specifies the desired
* compressible kind. For unmap operations this field must be set
* to NV_KIND_INVALID.
* out: For map operations this field returns the actual kind used
* for the mapping. This can be useful for detecting if a compressed
* mapping request was forced to use the fallback incompressible kind
* value because sufficient compression resources are not available. */
__s16 compr_kind;
/* in: For map operations, this field specifies the desired
* incompressible kind. This value will be used as the fallback kind
* if a valid compressible kind value was specified in the compr_kind
* field but sufficient compression resources are not available. For
* unmap operations this field must be set to NV_KIND_INVALID. */
__s16 incompr_kind;
/* in: For map operations, this field specifies the handle (dmabuf_fd)
* for the physical memory buffer to map into the specified virtual
* address range. For unmap operations, this field must be set to
* zero. */
__u32 mem_handle;
/* This field is reserved for padding purposes. */
__s32 reserved;
/* in: For map operations this field specifies the offset (in pages)
* into the physical memory buffer associated with mem_handle from
* from which physical page information should be collected for
* the mapping. For unmap operations this field must be zero. */
__u64 mem_offset_in_pages;
/* in: For both map and unmap operations this field specifies the
* virtual address space start offset in pages for the operation. */
__u64 virt_offset_in_pages;
/* in: For both map and unmap operations this field specifies the
* number of pages to map or unmap. */
__u64 num_pages;
};
/*
* VM remap IOCTL
*
* This ioctl can be used to issue multiple map and/or unmap operations in
* a single request. VM remap operations are only valid on address spaces
* that have been allocated with NVGPU_AS_ALLOC_SPACE_FLAGS_SPARSE.
* Validation of remap operations is performed before any changes are made
* to the associated sparse address space so either all map and/or unmap
* operations are performed or none of them area.
*/
struct nvgpu_as_remap_args {
/* in: This field specifies a pointer into the caller's address space
* containing an array of one or more nvgpu_as_remap_op structures. */
__u64 ops;
/* in/out: On input this field specifies the number of operations in
* the ops array. On output this field returns the successful
* number of remap operations. */
__u32 num_ops;
};
#define NVGPU_AS_IOCTL_BIND_CHANNEL \
_IOWR(NVGPU_AS_IOCTL_MAGIC, 1, struct nvgpu_as_bind_channel_args)
#define NVGPU32_AS_IOCTL_ALLOC_SPACE \
@@ -455,10 +586,12 @@ struct nvgpu_as_mapping_modify_args {
_IOR(NVGPU_AS_IOCTL_MAGIC, 12, struct nvgpu_as_get_sync_ro_map_args)
#define NVGPU_AS_IOCTL_MAPPING_MODIFY \
_IOWR(NVGPU_AS_IOCTL_MAGIC, 13, struct nvgpu_as_mapping_modify_args)
#define NVGPU_AS_IOCTL_REMAP \
_IOWR(NVGPU_AS_IOCTL_MAGIC, 14, struct nvgpu_as_remap_args)
#define NVGPU_AS_IOCTL_LAST \
_IOC_NR(NVGPU_AS_IOCTL_MAPPING_MODIFY)
_IOC_NR(NVGPU_AS_IOCTL_REMAP)
#define NVGPU_AS_IOCTL_MAX_ARG_SIZE \
sizeof(struct nvgpu_as_map_buffer_ex_args)
#endif
#endif /* #define _UAPI__LINUX_NVGPU_AS_H__ */

4
include/uapi/linux/nvgpu-ctrl.h
View File

@@ -158,6 +158,8 @@ struct nvgpu_gpu_zbc_query_table_args {
#define NVGPU_GPU_FLAGS_SUPPORT_FAULT_RECOVERY (1ULL << 33)
/* Mapping modify is enabled */
#define NVGPU_GPU_FLAGS_SUPPORT_MAPPING_MODIFY (1ULL << 34)
/* Remap is enabled */
#define NVGPU_GPU_FLAGS_SUPPORT_REMAP (1ULL << 35)
/* Compression is enabled */
#define NVGPU_GPU_FLAGS_SUPPORT_COMPRESSION (1ULL << 36)
/* SM TTU is enabled */
@@ -1100,4 +1102,4 @@ struct nvgpu_gpu_register_buffer_args {
#define NVGPU_GPU_IOCTL_MAX_ARG_SIZE \
sizeof(struct nvgpu_gpu_get_cpu_time_correlation_info_args)
#endif
#endif /* _UAPI__LINUX_NVGPU_CTRL_H__ */