From 3cd256b34476b8794a22fd16288860e124c79178 Mon Sep 17 00:00:00 2001 From: scottl Date: Tue, 8 Jun 2021 23:59:08 -0700 Subject: [PATCH] gpu: nvgpu: add linux REMAP support Add REMAP ioctl and accompanying support to the linux nvgpu driver. REMAP support provides per-page control over sparse VM areas using the concept of a virtual memory pool. The REMAP ioctl accepts a list of operations (each a map or unmap) that modify the VM area pages tracked by the virtual mmemory pool. Inclusion of REMAP support in the nvgpu build is controlled by the new CONFIG_NVGPU_REMAP flag. This flag is enabled by default for linux builds. A new NVGPU_GPU_FLAGS_SUPPORT_REMAP characteristics flag is added for use in detecting when REMAP support is available. When a VM allocation tagged with NVGPU_VM_AREA_ALLOC_SPARSE is made the base virtual memory pool resources are allocated. Per-page resources are later allocated when the NVGPU_AS_IOCTL_REMAP ioctl is issued. All REMAP resources are released when the corresponding VM area is freed. Jira NVGPU-6804 Change-Id: I1f2cdc0c06c1698a62640c1c6fbcb2f9db24a0bc Signed-off-by: scottl Reviewed-on: https://git-master.nvidia.com/r/c/linux-nvgpu/+/2542178 Tested-by: mobile promotions Reviewed-by: mobile promotions --- arch/nvgpu-common.yaml | 6 + arch/nvgpu-linux.yaml | 6 +- arch/nvgpu-posix.yaml | 2 + drivers/gpu/nvgpu/Makefile | 4 + drivers/gpu/nvgpu/Makefile.linux.configs | 6 + drivers/gpu/nvgpu/Makefile.sources | 8 + drivers/gpu/nvgpu/common/mm/vm_area.c | 40 +- drivers/gpu/nvgpu/common/mm/vm_remap.c | 797 ++++++++++++++++++ drivers/gpu/nvgpu/include/nvgpu/enabled.h | 2 + .../gpu/nvgpu/include/nvgpu/linux/vm_remap.h | 54 ++ .../include/nvgpu/posix/posix-vm_remap.h | 42 + drivers/gpu/nvgpu/include/nvgpu/vm_area.h | 16 + drivers/gpu/nvgpu/include/nvgpu/vm_remap.h | 278 ++++++ drivers/gpu/nvgpu/os/linux/ioctl_as.c | 86 +- drivers/gpu/nvgpu/os/linux/ioctl_ctrl.c | 2 + drivers/gpu/nvgpu/os/linux/module.c | 1 + drivers/gpu/nvgpu/os/linux/vm_remap.c | 242 ++++++ drivers/gpu/nvgpu/os/posix/posix-vm_remap.c | 40 + include/uapi/linux/nvgpu-as.h | 137 ++- include/uapi/linux/nvgpu-ctrl.h | 4 +- 20 files changed, 1761 insertions(+), 12 deletions(-) create mode 100644 drivers/gpu/nvgpu/common/mm/vm_remap.c create mode 100644 drivers/gpu/nvgpu/include/nvgpu/linux/vm_remap.h create mode 100644 drivers/gpu/nvgpu/include/nvgpu/posix/posix-vm_remap.h create mode 100644 drivers/gpu/nvgpu/include/nvgpu/vm_remap.h create mode 100644 drivers/gpu/nvgpu/os/linux/vm_remap.c create mode 100644 drivers/gpu/nvgpu/os/posix/posix-vm_remap.c diff --git a/arch/nvgpu-common.yaml b/arch/nvgpu-common.yaml index f7f816b3a..479fe09c3 100644 --- a/arch/nvgpu-common.yaml +++ b/arch/nvgpu-common.yaml @@ -675,6 +675,12 @@ mm: include/nvgpu/vm_area.h ] deps: [ ] tags: unit-testable + vm_remap: + safe: no + sources: [ common/mm/vm_remap.c, + include/nvgpu/vm_remap.h ] + deps: [ ] + tags: unit-testable vm: safe: yes sources: [ common/mm/vm.c, diff --git a/arch/nvgpu-linux.yaml b/arch/nvgpu-linux.yaml index 74598f9f1..901085941 100644 --- a/arch/nvgpu-linux.yaml +++ b/arch/nvgpu-linux.yaml @@ -228,7 +228,8 @@ vgpu: os/linux/vgpu/vgpu_linux.h ] vm: - sources: [ os/linux/vm.c ] + sources: [ os/linux/vm.c, + os/linux/vm_remap.c ] cic: sources: [ os/linux/cic/cic_stub.c ] @@ -252,7 +253,8 @@ headers: include/nvgpu/linux/thread.h, include/nvgpu/linux/log.h, include/nvgpu/linux/utils.h, - include/nvgpu/linux/vm.h ] + include/nvgpu/linux/vm.h, + include/nvgpu/linux/vm_remap.h ] # An extra unit to lump all the unclassified Linux files. extra: diff --git a/arch/nvgpu-posix.yaml b/arch/nvgpu-posix.yaml index 2755be068..9168eb2a9 100644 --- a/arch/nvgpu-posix.yaml +++ b/arch/nvgpu-posix.yaml @@ -22,6 +22,7 @@ all: os/posix/posix-nvgpu_mem.c, os/posix/posix-tsg.c, os/posix/posix-vm.c, + os/posix/posix-vm_remap.c, os/posix/soc.c, os/posix/bsearch.c, os/posix/posix-clk_arb.c, @@ -51,6 +52,7 @@ headers: include/nvgpu/posix/probe.h, include/nvgpu/posix/soc_fuse.h, include/nvgpu/posix/vm.h, + include/nvgpu/posix/posix-vm_remap.h, include/nvgpu/posix/posix_vidmem.h, include/nvgpu/posix/posix-nvhost.h, include/nvgpu/posix/trace_gk20a.h ] diff --git a/drivers/gpu/nvgpu/Makefile b/drivers/gpu/nvgpu/Makefile index abb5fcae6..59c37a8ef 100644 --- a/drivers/gpu/nvgpu/Makefile +++ b/drivers/gpu/nvgpu/Makefile @@ -650,6 +650,10 @@ nvgpu-$(CONFIG_NVGPU_COMPRESSION) += \ hal/cbc/cbc_gv11b.o \ hal/cbc/cbc_tu104.o \ +nvgpu-$(CONFIG_NVGPU_REMAP) += \ + os/linux/vm_remap.o \ + common/mm/vm_remap.o + # FUSA (Functionally Safe) HAL source files nvgpu-y += \ hal/mm/mm_gv11b_fusa.o \ diff --git a/drivers/gpu/nvgpu/Makefile.linux.configs b/drivers/gpu/nvgpu/Makefile.linux.configs index 576d896a0..be42b79fe 100644 --- a/drivers/gpu/nvgpu/Makefile.linux.configs +++ b/drivers/gpu/nvgpu/Makefile.linux.configs @@ -40,6 +40,9 @@ ifeq ($(CONFIG_NVGPU_COMPRESSION),y) CONFIG_NVGPU_SUPPORT_CDE := y endif +# Support for remap +CONFIG_NVGPU_REMAP := y + ifeq ($(CONFIG_COMMON_CLK),y) ifeq ($(CONFIG_PM_DEVFREQ),y) # Select this entry to enable gk20a scaling @@ -172,6 +175,9 @@ endif ifeq ($(CONFIG_NVGPU_COMPRESSION),y) ccflags-y += -DCONFIG_NVGPU_COMPRESSION endif +ifeq ($(CONFIG_NVGPU_REMAP),y) +ccflags-y += -DCONFIG_NVGPU_REMAP +endif ifeq ($(CONFIG_NVGPU_SUPPORT_CDE),y) ccflags-y += -DCONFIG_NVGPU_SUPPORT_CDE endif diff --git a/drivers/gpu/nvgpu/Makefile.sources b/drivers/gpu/nvgpu/Makefile.sources index 26461f5f2..f3a5d7a8d 100644 --- a/drivers/gpu/nvgpu/Makefile.sources +++ b/drivers/gpu/nvgpu/Makefile.sources @@ -60,6 +60,10 @@ ifeq ($(CONFIG_NVGPU_COMPRESSION),1) srcs += os/posix/posix-comptags.c endif +ifeq ($(CONFIG_NVGPU_REMAP),1) +srcs += os/posix/posix-vm_remap.c +endif + ifeq ($(CONFIG_NVGPU_LOGGING),1) srcs += os/posix/log.c endif @@ -604,6 +608,10 @@ srcs += common/mm/comptags.c \ hal/cbc/cbc_gv11b.c endif +ifeq ($(CONFIG_NVGPU_REMAP),1) +srcs += common/mm/vm_remap.c +endif + ifeq ($(CONFIG_NVGPU_NVLINK),1) srcs += common/vbios/nvlink_bios.c \ common/nvlink/probe.c \ diff --git a/drivers/gpu/nvgpu/common/mm/vm_area.c b/drivers/gpu/nvgpu/common/mm/vm_area.c index 2fb7593e0..34170a4ce 100644 --- a/drivers/gpu/nvgpu/common/mm/vm_area.c +++ b/drivers/gpu/nvgpu/common/mm/vm_area.c @@ -25,6 +25,9 @@ #include #include #include +#ifdef CONFIG_NVGPU_REMAP +#include +#endif struct nvgpu_vm_area *nvgpu_vm_area_find(struct vm_gk20a *vm, u64 addr) { @@ -200,6 +203,7 @@ int nvgpu_vm_area_alloc(struct vm_gk20a *vm, u64 pages, u32 page_size, u64 vaddr_start = 0; u64 our_addr = *addr; u32 pgsz_idx = GMMU_PAGE_SIZE_SMALL; + int err = 0; /* * If we have a fixed address then use the passed address in *addr. This @@ -226,8 +230,10 @@ int nvgpu_vm_area_alloc(struct vm_gk20a *vm, u64 pages, u32 page_size, } vma = vm->vma[pgsz_idx]; - if (nvgpu_vm_area_alloc_memory(vma, our_addr, pages, - page_size, flags, &vaddr_start) != 0) { + + err = nvgpu_vm_area_alloc_memory(vma, our_addr, pages, page_size, + flags, &vaddr_start); + if (err != 0) { goto free_vm_area; } @@ -238,10 +244,21 @@ int nvgpu_vm_area_alloc(struct vm_gk20a *vm, u64 pages, u32 page_size, nvgpu_init_list_node(&vm_area->buffer_list_head); nvgpu_init_list_node(&vm_area->vm_area_list); +#ifdef CONFIG_NVGPU_REMAP + if (((flags & NVGPU_VM_AREA_ALLOC_SPARSE) != 0U) && + (vm_area->pgsz_idx == GMMU_PAGE_SIZE_BIG)) { + err = nvgpu_vm_remap_vpool_create(vm, vm_area, pages); + if (err != 0) { + goto free_vaddr; + } + } +#endif + nvgpu_mutex_acquire(&vm->update_gmmu_lock); - if (nvgpu_vm_area_alloc_gmmu_map(vm, vm_area, vaddr_start, - pgsz_idx, flags) != 0) { + err = nvgpu_vm_area_alloc_gmmu_map(vm, vm_area, vaddr_start, + pgsz_idx, flags); + if (err != 0) { nvgpu_mutex_release(&vm->update_gmmu_lock); goto free_vaddr; } @@ -252,10 +269,16 @@ int nvgpu_vm_area_alloc(struct vm_gk20a *vm, u64 pages, u32 page_size, return 0; free_vaddr: +#ifdef CONFIG_NVGPU_REMAP + if (vm_area->vpool != NULL) { + nvgpu_vm_remap_vpool_destroy(vm, vm_area); + vm_area->vpool = NULL; + } +#endif nvgpu_free(vma, vaddr_start); free_vm_area: nvgpu_kfree(g, vm_area); - return -ENOMEM; + return err; } int nvgpu_vm_area_free(struct vm_gk20a *vm, u64 addr) @@ -302,6 +325,13 @@ int nvgpu_vm_area_free(struct vm_gk20a *vm, u64 addr) NULL); } +#ifdef CONFIG_NVGPU_REMAP + /* clean up any remap resources */ + if (vm_area->vpool != NULL) { + nvgpu_vm_remap_vpool_destroy(vm, vm_area); + } +#endif + nvgpu_mutex_release(&vm->update_gmmu_lock); nvgpu_free(vm->vma[vm_area->pgsz_idx], vm_area->addr); diff --git a/drivers/gpu/nvgpu/common/mm/vm_remap.c b/drivers/gpu/nvgpu/common/mm/vm_remap.c new file mode 100644 index 000000000..48d61dec2 --- /dev/null +++ b/drivers/gpu/nvgpu/common/mm/vm_remap.c @@ -0,0 +1,797 @@ +/* + * Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include + +/* + * Return page size index of page size for VM areas that can be used with + * remap. + */ +static inline u32 nvgpu_vm_remap_pgsz_idx(struct vm_gk20a *vm) +{ + return GMMU_PAGE_SIZE_BIG; +} + +/* + * Return page size for VM areas that can be used with remap. + */ +static inline u64 nvgpu_vm_remap_page_size(struct vm_gk20a *vm) +{ + return vm->gmmu_page_sizes[GMMU_PAGE_SIZE_BIG]; +} + +/* + * Return a pointer to the os-specific structure for the specified physical + * memory pool. + */ +static inline struct nvgpu_vm_remap_os_buffer *nvgpu_vm_remap_mpool_handle( + struct nvgpu_vm_remap_mpool *mpool) +{ + if (mpool == NULL) { + return NULL; + } + + return &mpool->remap_os_buf; +} + +/* + * Add a reference to the specified physical memory pool. + */ +static inline struct nvgpu_vm_remap_mpool *nvgpu_vm_remap_mpool_get( + struct nvgpu_vm_remap_mpool *mpool) +{ + nvgpu_ref_get(&mpool->ref); + + return mpool; +} + +/* + * Cleanup physical memory pool resources. This function is called when + * the reference count for the physical memory pool goes to zero. + */ +static void nvgpu_vm_remap_mpool_release(struct nvgpu_ref *ref) +{ + struct nvgpu_vm_remap_mpool *mpool; + struct nvgpu_vm_remap_vpool *vpool; + struct vm_gk20a *vm; + struct gk20a *g; + + mpool = nvgpu_vm_remap_mpool_from_ref(ref); + vpool = mpool->vpool; + vm = vpool->vm; + g = gk20a_from_vm(vm); + + nvgpu_rbtree_unlink(&mpool->node, &vpool->mpools); + + /* L2 must be flushed before we destroy any SMMU mappings. */ + if (*mpool->l2_flushed == false) { + (void) g->ops.mm.cache.l2_flush(g, true); + *mpool->l2_flushed = true; + } + + nvgpu_vm_remap_os_buf_put(vm, &mpool->remap_os_buf); + nvgpu_kfree(g, mpool); +} + +/* + * Release a reference to the specified physical memory pool. + */ +static inline void nvgpu_vm_remap_mpool_put(struct vm_gk20a *vm, + struct nvgpu_vm_remap_vpool *vpool, + struct nvgpu_vm_remap_mpool *mpool, + bool *l2_flushed) +{ + if (mpool != NULL) { + mpool->l2_flushed = l2_flushed; + nvgpu_ref_put(&mpool->ref, nvgpu_vm_remap_mpool_release); + mpool->l2_flushed = NULL; + } +} + +/* + * Insert a physical memory pool into the rbtree of the specified virtual + * memory pool. + */ +static inline struct nvgpu_vm_remap_mpool *nvgpu_vm_remap_mpool_add( + struct vm_gk20a *vm, + struct nvgpu_vm_remap_vpool *vpool, + struct nvgpu_vm_remap_os_buffer *remap_os_buf) +{ + struct gk20a *g = gk20a_from_vm(vm); + struct nvgpu_rbtree_node **root = &vpool->mpools; + struct nvgpu_vm_remap_mpool *mpool; + u64 key; + + mpool = nvgpu_kzalloc(g, sizeof(*mpool)); + if (mpool == NULL) { + return NULL; + } + + key = nvgpu_vm_remap_get_handle(remap_os_buf); + mpool->node.key_start = key; + mpool->node.key_end = key; + nvgpu_ref_init(&mpool->ref); + mpool->vpool = vpool; + + nvgpu_memcpy((u8 *)&mpool->remap_os_buf, (u8 *)remap_os_buf, + sizeof(mpool->remap_os_buf)); + + nvgpu_rbtree_insert(&mpool->node, root); + + return mpool; +} + +/* + * Return a pointer to the physical memory pool for the specified rbtree node. + */ +static inline struct nvgpu_vm_remap_mpool * +nvgpu_vm_remap_mpool_from_tree_entry(struct nvgpu_rbtree_node *node) +{ + return (struct nvgpu_vm_remap_mpool *) + ((uintptr_t)node - offsetof(struct nvgpu_vm_remap_mpool, node)); +}; + +/* + * Return a pointer to the physical memory pool for the associated physical + * memory buffer. + */ +static inline struct nvgpu_vm_remap_mpool * +nvgpu_vm_remap_mpool_find(struct nvgpu_rbtree_node *root, + struct nvgpu_vm_remap_os_buffer *remap_os_buf) +{ + struct nvgpu_rbtree_node *node; + u64 key = nvgpu_vm_remap_get_handle(remap_os_buf); + + nvgpu_rbtree_search(key, &node, root); + if (node == NULL) { + return NULL; + } + + return nvgpu_vm_remap_mpool_from_tree_entry(node); +} + +#ifdef CONFIG_NVGPU_COMPRESSION +/* + * Ensure that compression resources are allocated to the specified + * physical memory buffer. + */ +static inline int nvgpu_vm_remap_ensure_comptags(struct vm_gk20a *vm, + struct nvgpu_vm_remap_os_buffer *remap_os_buf) +{ + struct gk20a *g = gk20a_from_vm(vm); + struct gk20a_comptags comptags = { 0 }; + struct nvgpu_os_buffer *os_buf = &remap_os_buf->os_buf; + int err = 0; + + err = gk20a_alloc_or_get_comptags(g, os_buf, + &g->cbc->comp_tags, + &comptags); + if (err != 0) { + nvgpu_err(g, "cannot alloc comptags: %d", err); + return err; + } + + if (comptags.needs_clear) { + nvgpu_assert(g->ops.cbc.ctrl != NULL); + if (gk20a_comptags_start_clear(os_buf)) { + err = g->ops.cbc.ctrl(g, nvgpu_cbc_op_clear, + comptags.offset, + (comptags.offset + + comptags.lines - 1U)); + gk20a_comptags_finish_clear(os_buf, err == 0); + } + } + + return err; +} +#endif + +/* + * Validate that the specified remap operation resides within the target + * virtual memory pool. + */ +static int nvgpu_vm_remap_validate_vpool(struct nvgpu_vm_remap_vpool *vpool, + struct nvgpu_vm_remap_op *op) +{ + u64 first_page = op->virt_offset_in_pages; + u64 last_page = op->virt_offset_in_pages + op->num_pages - 1ULL; + + if (first_page < vpool->base_offset_in_pages || + last_page >= vpool->base_offset_in_pages + vpool->num_pages || + last_page < first_page) { + return -EINVAL; + } + + return 0; +} + +/* + * Validate an unmap operation. + */ +static int nvgpu_vm_remap_validate_unmap(struct vm_gk20a *vm, + struct nvgpu_vm_remap_vpool *vpool, + struct nvgpu_vm_remap_op *op) +{ + return nvgpu_vm_remap_validate_vpool(vpool, op); +} + +/* + * Validate a map operation. + */ +static int nvgpu_vm_remap_validate_map(struct vm_gk20a *vm, + struct nvgpu_vm_remap_vpool *vpool, + struct nvgpu_vm_remap_op *op, + struct nvgpu_vm_remap_os_buffer *remap_os_buf) +{ + u64 page_size = nvgpu_vm_remap_page_size(vm); + u64 map_offset; + u64 map_size; + u64 os_buf_size; + + map_offset = nvgpu_safe_mult_u64(op->mem_offset_in_pages, page_size); + map_size = nvgpu_safe_mult_u64(op->num_pages, page_size); + os_buf_size = nvgpu_os_buf_get_size(&remap_os_buf->os_buf); + + if ((map_size > os_buf_size) || + ((os_buf_size - map_size) < map_offset)) { + return -EINVAL; + } + +#ifdef CONFIG_NVGPU_COMPRESSION + if (op->compr_kind != NVGPU_KIND_INVALID) { + if (nvgpu_vm_remap_ensure_comptags(vm, remap_os_buf)) { + /* inform caller there are no more compbits */ + op->compr_kind = NVGPU_KIND_INVALID; + } + } +#endif + + return nvgpu_vm_remap_validate_vpool(vpool, op); +} + +/* + * Return a pointer to the virtual pool for the specified remap operation. + * Note that this function must be called with the VM's GMMU update lock + * held. + */ +static struct nvgpu_vm_remap_vpool *nvgpu_vm_remap_get_vpool_locked( + struct vm_gk20a *vm, struct nvgpu_vm_remap_op *op) +{ + struct gk20a *g = gk20a_from_vm(vm); + u64 page_size = nvgpu_vm_remap_page_size(vm); + u64 offset = nvgpu_safe_mult_u64(op->virt_offset_in_pages, page_size); + struct nvgpu_vm_area *vm_area = nvgpu_vm_area_find(vm, offset); + + if ((vm_area == NULL) || (vm_area->vpool == NULL)) { + return NULL; + } + + /* allocate per-page tracking if first time vpool is used */ + if ((vm_area->vpool->mpool_by_page == NULL) && + (vm_area->vpool->num_pages != 0)) { + struct nvgpu_vm_remap_mpool **mp; + + mp = nvgpu_kzalloc(g, vm_area->vpool->num_pages * + sizeof(struct nvgpu_vm_remap_mpool *)); + if (mp == NULL) { + return NULL; + } + vm_area->vpool->mpool_by_page = mp; + } + + return vm_area->vpool; +} + +/* + * Update physical memory pool reference counts for the specified virtual + * pool range of pages. + */ +static void nvgpu_vm_remap_update_pool_refcounts( + struct vm_gk20a *vm, + struct nvgpu_vm_remap_vpool *vpool, + u64 first_page, + u64 num_pages, + struct nvgpu_vm_remap_mpool *new_pool, + bool *l2_flushed) +{ + u64 pgnum; + + if (vpool->num_pages < first_page + num_pages) { + nvgpu_err(gk20a_from_vm(vm), "bad vpool range; update skipped"); + return; + } + + for (pgnum = first_page; pgnum < first_page + num_pages; pgnum++) { + + /* + * Decrement refcount of the physical pool that was previously + * mapped to this page. + */ + nvgpu_vm_remap_mpool_put(vm, vpool, + vpool->mpool_by_page[pgnum], l2_flushed); + + /* + * And record the fact that the current page now refers to + * "new_pool". + */ + vpool->mpool_by_page[pgnum] = new_pool; + + /* + * Increment refcount of the new physical pool. + */ + if (new_pool != NULL) { + (void) nvgpu_vm_remap_mpool_get(new_pool); + } + } +} + +/* + * Return the ctag offset (if applicable) for the specified map operation. + */ +static u64 nvgpu_vm_remap_get_ctag_offset(struct vm_gk20a *vm, + struct nvgpu_vm_remap_op *op, + struct nvgpu_os_buffer *os_buf, + s16 *kind) +{ +#ifdef CONFIG_NVGPU_COMPRESSION + struct gk20a *g = gk20a_from_vm(vm); + struct gk20a_comptags comptags; + u64 ctag = 0; + u64 ctag_offset = 0; + u64 page_size = nvgpu_vm_remap_page_size(vm); + u64 phys_offset = nvgpu_safe_mult_u64(op->mem_offset_in_pages, + page_size); + + gk20a_get_comptags(os_buf, &comptags); + + if (comptags.lines != 0) { + ctag = (u64)comptags.offset; + } + + if (op->compr_kind != NVGPU_KIND_INVALID) { + *kind = op->compr_kind; + if (ctag != 0) { + ctag_offset = ctag + (phys_offset >> + ilog2(g->ops.fb.compression_page_size(g))); + } else { + ctag_offset = 0; + } + } else { + *kind = op->incompr_kind; + ctag_offset = 0; + } + + return ctag_offset; +#else + return 0; +#endif +} + +static u32 nvgpu_vm_remap_get_map_flags(struct nvgpu_vm_remap_op *op) +{ + u32 flags = 0; + + if ((op->flags & NVGPU_VM_REMAP_OP_FLAGS_CACHEABLE) != 0U) { + flags = NVGPU_VM_MAP_CACHEABLE; + } + + return flags; +} + +static enum gk20a_mem_rw_flag nvgpu_vm_remap_get_map_rw_flag( + struct nvgpu_vm_remap_op *op) +{ + enum gk20a_mem_rw_flag rw_flag = gk20a_mem_flag_none; + + if ((op->flags & NVGPU_VM_REMAP_OP_FLAGS_ACCESS_NO_WRITE) != 0U) { + rw_flag = gk20a_mem_flag_read_only; + } + + return rw_flag; +} + +/* + * Execute remap operations in sequence. + * All remap operations must succeed for this routine to return success. + */ +static int nvgpu_vm_remap_execute_remaps(struct vm_gk20a *vm, + struct nvgpu_vm_remap_vpool *vpool, + struct nvgpu_vm_remap_mpool **mpools, + struct nvgpu_vm_remap_op *ops, u32 *num_ops, + bool *l2_flushed) +{ + struct gk20a *g = gk20a_from_vm(vm); + struct nvgpu_vm_remap_op *op; + struct nvgpu_vm_remap_os_buffer *remap_os_buf; + u32 pgsz_idx = nvgpu_vm_remap_pgsz_idx(vm); + u64 page_size = nvgpu_vm_remap_page_size(vm); + u64 map_addr = 0; + u64 phys_offset = 0; + u64 map_size; + u64 ctag_offset = 0; + enum gk20a_mem_rw_flag rw_flag; + u32 flags; + struct nvgpu_vm_remap_os_buffer *curr_remap_os_buf = NULL; + struct vm_gk20a_mapping_batch batch; + s16 curr_kind = 0; + u32 i; + int err = 0; + + nvgpu_vm_mapping_batch_start(&batch); + + /* Update GPU page tables. */ + for (i = 0; i < *num_ops; i++) { + op = &ops[i]; + remap_os_buf = nvgpu_vm_remap_mpool_handle(mpools[i]); + + map_size = nvgpu_safe_mult_u64(op->num_pages, page_size); + map_addr = nvgpu_safe_mult_u64(op->virt_offset_in_pages, + page_size); + phys_offset = nvgpu_safe_mult_u64(op->mem_offset_in_pages, + page_size); + + if (remap_os_buf == NULL) { + /* + * Unmap range. + */ + (void) g->ops.mm.gmmu.unmap(vm, + map_addr, + map_size, + pgsz_idx, + false, /* va_allocated */ + gk20a_mem_flag_none, + true, /* sparse */ + &batch); + } else { + /* + * If physical pool changed from the previous ops, + * update curr_pool* variables. + */ + if (remap_os_buf != curr_remap_os_buf) { + curr_remap_os_buf = remap_os_buf; + } + + ctag_offset = nvgpu_vm_remap_get_ctag_offset(vm, op, + &curr_remap_os_buf->os_buf, + &curr_kind); + + flags = nvgpu_vm_remap_get_map_flags(op); + rw_flag = nvgpu_vm_remap_get_map_rw_flag(op); + + /* + * Remap range. + */ + map_addr = g->ops.mm.gmmu.map(vm, + map_addr, + remap_os_buf->nv_sgt, + phys_offset, + map_size, + pgsz_idx, + (u8)curr_kind, + ctag_offset, + flags, + rw_flag, + false, /* ctags clear */ + true, + false, + &batch, + remap_os_buf->aperture); + + if (map_addr == 0ULL) { + nvgpu_err(g, "map addr is zero"); + err = -ENOMEM; + break; + } + } + } + + /* + * Handle possible error condition by updating num_ops to reflect + * the number of remap ops that actually succeeded. + */ + if (i != *num_ops) { + *num_ops = i; + } + + nvgpu_vm_mapping_batch_finish_locked(vm, &batch); + + /* + * Release references to the previously mapped pages of all + * remap operations that succeeded. + */ + for (i = 0; i < *num_ops; i++) { + op = &ops[i]; + nvgpu_vm_remap_update_pool_refcounts( + vm, + vpool, + nvgpu_safe_sub_u64(op->virt_offset_in_pages, + vpool->base_offset_in_pages), + op->num_pages, + mpools[i], + l2_flushed); + } + + return err; +} + +static int nvgpu_vm_remap_get_mpool(struct vm_gk20a *vm, + struct nvgpu_vm_remap_vpool *vpool, + struct nvgpu_vm_remap_op *op, + struct nvgpu_vm_remap_mpool **curr_mpool, + u32 *curr_mem_handle) +{ + struct nvgpu_vm_remap_os_buffer remap_os_buf = { 0 }; + int err = 0; + + if (op->mem_handle == *curr_mem_handle) { + /* + * Physical pool didn't change from the previous op, so we + * can skip validation and just use the curr_pool pointer + * again. Just take one extra ref. + */ + *curr_mpool = nvgpu_vm_remap_mpool_get(*curr_mpool); + } else { + /* + * Move to the next memory handle (validate access + acquire + * reference). + */ + err = nvgpu_vm_remap_os_buf_get(vm, op, &remap_os_buf); + if (err != 0) { + goto done; + } + + /* + * Make sure the new memory handle is included in (and + * referenced by) the list of memory handles mapped in the + * virtual pool. + */ + *curr_mpool = nvgpu_vm_remap_mpool_find(vpool->mpools, + &remap_os_buf); + if (*curr_mpool != NULL) { + /* + * This memory handle was already mapped to the virtual + * pool, so we don't need to keep the extra reference + * to the nvmap handle. + */ + nvgpu_vm_remap_os_buf_put(vm, &remap_os_buf); + *curr_mpool = nvgpu_vm_remap_mpool_get(*curr_mpool); + } else { + /* + * Add the physical memory to the list of mapped + * handles. + */ + *curr_mpool = nvgpu_vm_remap_mpool_add(vm, vpool, + &remap_os_buf); + if (*curr_mpool == NULL) { + nvgpu_vm_remap_os_buf_put(vm, &remap_os_buf); + err = -ENOMEM; + goto done; + } + } + + *curr_mem_handle = op->mem_handle; + } + +done: + return err; +} + +/* + * Prepare to execute remap operations. Allocate an array to track the + * associated physical memory pool for each specified operation and + * then validate the parameters for each operation. Note that this function + * must be called with the VM's GMMU update lock held. + */ +static int nvgpu_vm_remap_locked(struct vm_gk20a *vm, + struct nvgpu_vm_remap_vpool *vpool, + struct nvgpu_vm_remap_op *ops, u32 *num_ops) +{ + struct gk20a *g = gk20a_from_vm(vm); + bool l2_flushed = false; + struct nvgpu_vm_remap_mpool **mpools; + struct nvgpu_vm_remap_mpool *curr_mpool = NULL; + u32 curr_mem_handle = 0U; + u32 num_ops_validated = 0U; + u32 i; + int err = 0; + + if (*num_ops == 0U) { + return 0; + } + + if (vpool == NULL) { + return -EINVAL; + } + + mpools = nvgpu_kzalloc(g, *num_ops * sizeof(mpools[0])); + if (mpools == NULL) { + return -ENOMEM; + } + + /* We cache the validated memory handle across ops to avoid + * revalidation in the common case where the physical pool doesn't + * change between ops. + */ + for (i = 0; i < *num_ops; i++) { + struct nvgpu_vm_remap_op *op = &ops[i]; + + if (op->mem_handle == 0U) { + err = nvgpu_vm_remap_validate_unmap(vm, vpool, op); + if (err != 0) { + nvgpu_err(g, "validate_unmap failed: %d", err); + break; + } + } else { + err = nvgpu_vm_remap_get_mpool(vm, vpool, op, + &curr_mpool, &curr_mem_handle); + if (err != 0) { + nvgpu_err(g, "get_mpool failed: %d", err); + break; + } + + /* + * Validate that the mapping request is valid. + * This may demote the kind from compressed to + * uncompressed if we have run out of compbits. + */ + err = nvgpu_vm_remap_validate_map(vm, vpool, op, + nvgpu_vm_remap_mpool_handle(curr_mpool)); + if (err != 0) { + nvgpu_err(g, "validate_map failed: %d", err); + nvgpu_vm_remap_mpool_put(vm, vpool, + curr_mpool, &l2_flushed); + break; + } + + mpools[i] = curr_mpool; + } + } + + num_ops_validated = i; + + if (err == 0) { + /* + * The validation stage completed without errors so + * execute all map and unmap operations sequentially. + */ + err = nvgpu_vm_remap_execute_remaps(vm, vpool, mpools, + ops, num_ops, &l2_flushed); + } else { + /* + * Validation failed so set successful num_ops count to zero. + */ + *num_ops = 0; + } + + /* Release references acquired during ops validation. */ + for (i = 0; i < num_ops_validated; i++) { + nvgpu_vm_remap_mpool_put(vm, vpool, + mpools[i], &l2_flushed); + } + + nvgpu_kfree(g, mpools); + + return err; +} + +/* + * Top-level remap handler. This function is used by the os-specific REMAP + * API handler to execute remap operations. + */ +int nvgpu_vm_remap(struct vm_gk20a *vm, + struct nvgpu_vm_remap_op *ops, + u32 *num_ops) +{ + int ret = 0; + struct nvgpu_vm_remap_vpool *vpool; + + nvgpu_mutex_acquire(&vm->update_gmmu_lock); + + vpool = nvgpu_vm_remap_get_vpool_locked(vm, &ops[0]); + + if (vpool != NULL) { + ret = nvgpu_vm_remap_locked(vm, vpool, ops, num_ops); + } else { + *num_ops = 0; + ret = -EINVAL; + } + + nvgpu_mutex_release(&vm->update_gmmu_lock); + + return ret; +} + +/* + * Create a virtual memory pool. + */ +int nvgpu_vm_remap_vpool_create(struct vm_gk20a *vm, + struct nvgpu_vm_area *vm_area, + u64 num_pages) +{ + struct gk20a *g = gk20a_from_vm(vm); + struct nvgpu_vm_remap_vpool *vp; + u64 start_page_nr = 0; + + if ((num_pages == 0ULL) || + (vm_area->pgsz_idx != GMMU_PAGE_SIZE_BIG) || + ((vm_area->flags & NVGPU_VM_AREA_ALLOC_SPARSE) == 0U)) { + return -EINVAL; + } + + vp = nvgpu_kzalloc(g, sizeof(struct nvgpu_vm_remap_vpool)); + if (vp == NULL) { + return -ENOMEM; + } + + start_page_nr = vm_area->addr >> + ilog2(vm->gmmu_page_sizes[vm_area->pgsz_idx]); + + vp->base_offset_in_pages = start_page_nr; + vp->num_pages = num_pages; + vp->vm = vm; + + vm_area->vpool = vp; + + return 0; +} + +/* + * Destroy a virtual memory pool. + */ +void nvgpu_vm_remap_vpool_destroy(struct vm_gk20a *vm, + struct nvgpu_vm_area *vm_area) +{ + struct gk20a *g = gk20a_from_vm(vm); + struct nvgpu_vm_remap_vpool *vpool = vm_area->vpool; + + if (vpool == NULL) { + return; + } + + if (vpool->mpools != NULL) { + struct nvgpu_vm_remap_op op = { 0 }; + u32 num_ops = 1; + int err; + + op.virt_offset_in_pages = vpool->base_offset_in_pages; + op.num_pages = vpool->num_pages; + + err = nvgpu_vm_remap_locked(vm, vpool, &op, &num_ops); + nvgpu_assert(err == 0); + } + + nvgpu_assert(vpool->mpools == NULL); + + if (vpool->mpool_by_page != NULL) { + nvgpu_kfree(g, vpool->mpool_by_page); + } + nvgpu_kfree(g, vpool); + + vm_area->vpool = NULL; + + return; +} diff --git a/drivers/gpu/nvgpu/include/nvgpu/enabled.h b/drivers/gpu/nvgpu/include/nvgpu/enabled.h index b476d345f..3536d4aa9 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/enabled.h +++ b/drivers/gpu/nvgpu/include/nvgpu/enabled.h @@ -82,6 +82,8 @@ struct gk20a; "Support batch mapping"), \ DEFINE_FLAG(NVGPU_SUPPORT_MAPPING_MODIFY, \ "Support mapping modify"), \ + DEFINE_FLAG(NVGPU_SUPPORT_REMAP, \ + "Support remap"), \ DEFINE_FLAG(NVGPU_USE_COHERENT_SYSMEM, \ "Use coherent aperture for sysmem"), \ DEFINE_FLAG(NVGPU_MM_USE_PHYSICAL_SG, \ diff --git a/drivers/gpu/nvgpu/include/nvgpu/linux/vm_remap.h b/drivers/gpu/nvgpu/include/nvgpu/linux/vm_remap.h new file mode 100644 index 000000000..635ec94f5 --- /dev/null +++ b/drivers/gpu/nvgpu/include/nvgpu/linux/vm_remap.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef __COMMON_LINUX_VM_REMAP_H__ +#define __COMMON_LINUX_VM_REMAP_H__ + +#include +#include + +struct nvgpu_vm_remap_op; +struct nvgpu_as_remap_op; + +/* struct tracking os-specific remap memory buffer state */ +struct nvgpu_vm_remap_os_buffer { + struct nvgpu_os_buffer os_buf; + struct nvgpu_mapped_buf_priv os_priv; + + struct nvgpu_sgt *nv_sgt; + u64 aperture; +}; + +int nvgpu_vm_remap_translate_as_op(struct vm_gk20a *vm, + struct nvgpu_vm_remap_op *vm_op, + struct nvgpu_as_remap_op *as_op); +void nvgpu_vm_remap_translate_vm_op(struct nvgpu_as_remap_op *as_op, + struct nvgpu_vm_remap_op *vm_op); + +u64 nvgpu_vm_remap_get_handle(struct nvgpu_vm_remap_os_buffer *remap_os_buf); +int nvgpu_vm_remap_os_buf_get(struct vm_gk20a *vm, + struct nvgpu_vm_remap_op *op, + struct nvgpu_vm_remap_os_buffer *remap_os_buf); +void nvgpu_vm_remap_os_buf_put(struct vm_gk20a *vm, + struct nvgpu_vm_remap_os_buffer *remap_os_buf); + +#endif /* __COMMON_LINUX_VM_REMAP_H__ */ diff --git a/drivers/gpu/nvgpu/include/nvgpu/posix/posix-vm_remap.h b/drivers/gpu/nvgpu/include/nvgpu/posix/posix-vm_remap.h new file mode 100644 index 000000000..512268e08 --- /dev/null +++ b/drivers/gpu/nvgpu/include/nvgpu/posix/posix-vm_remap.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef NVGPU_POSIX_VM__REMAP_H +#define NVGPU_POSIX_VM_REMAP_H + +#include + +struct nvgpu_vm_remap_os_buffer { + struct nvgpu_os_buffer os_buf; + struct nvgpu_mapped_buf_priv os_priv; + + struct nvgpu_sgt *nv_sgt; + u64 aperture; +}; + +u64 nvgpu_vm_remap_get_handle(struct nvgpu_vm_remap_os_buffer *remap_os_buf); +int nvgpu_vm_remap_os_buf_get(struct vm_gk20a *vm, u32 mem_handle, + struct nvgpu_vm_remap_os_buffer *remap_os_buf); +void nvgpu_vm_remap_os_buf_put(struct vm_gk20a *vm, + struct nvgpu_vm_remap_os_buffer *remap_os_buf); + +#endif /* NVGPU_POSIX_VM_REMAP_H */ diff --git a/drivers/gpu/nvgpu/include/nvgpu/vm_area.h b/drivers/gpu/nvgpu/include/nvgpu/vm_area.h index 15d856f75..3865069cb 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/vm_area.h +++ b/drivers/gpu/nvgpu/include/nvgpu/vm_area.h @@ -30,6 +30,8 @@ struct vm_gk20a; struct gk20a_as_share; struct nvgpu_as_alloc_space_args; struct nvgpu_as_free_space_args; +struct nvgpu_vm_remap_vpool; + /** * Carve out virtual address space from virtual memory context. * This is needed for fixed address mapping. @@ -66,6 +68,13 @@ struct nvgpu_vm_area { * @sa NVGPU_VM_AREA_ALLOC_SPARSE */ bool sparse; + +#ifdef CONFIG_NVGPU_REMAP + /** + * Virtual pool for remap support of sparse VM areas. + */ + struct nvgpu_vm_remap_vpool *vpool; +#endif }; static inline struct nvgpu_vm_area * @@ -91,6 +100,13 @@ nvgpu_vm_area_from_vm_area_list(struct nvgpu_list_node *node) */ #define NVGPU_VM_AREA_ALLOC_SPARSE BIT32(1) +/** + * Enable REMAP control of the the vmarea. REMAP uses a virtual + * memory pool that provides control over each page in the vmarea. + * Note that REMAP is only permitted with SPARSE vmareas. + */ +#define NVGPU_VM_AREA_ALLOC_REMAP BIT32(2) + /** * @brief Allocate a virtual memory area with given size and start. * diff --git a/drivers/gpu/nvgpu/include/nvgpu/vm_remap.h b/drivers/gpu/nvgpu/include/nvgpu/vm_remap.h new file mode 100644 index 000000000..5059b8b75 --- /dev/null +++ b/drivers/gpu/nvgpu/include/nvgpu/vm_remap.h @@ -0,0 +1,278 @@ +/* + * Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef NVGPU_VM_REMAP_H +#define NVGPU_VM_REMAP_H + +#include +#include + +#ifdef __KERNEL__ +#include +#elif defined(__NVGPU_POSIX__) +#include +#endif + +/* + * Supported remap operation flags. + */ +#define NVGPU_VM_REMAP_OP_FLAGS_CACHEABLE BIT32(1) +#define NVGPU_VM_REMAP_OP_FLAGS_ACCESS_NO_WRITE BIT32(7) +/** + * This structure describes a single remap operation (either a map or unmap). + */ +struct nvgpu_vm_remap_op { + /** + * When a map operation is specified this field contains any flags + * to use when setting up the mapping. When an unmap operation is + * specified this field must be zero. + */ + u32 flags; + + /** + * When a map operation is specified this field can be used to specify + * the compressed kind for the mapping. If the specified value is + * NVGPU_KIND_INVALID then no compression resources are requested and + * the #incompr_kind value is used for the mapping. If a value other + * than NVGPU_KIND_INVALID is specified but there are no compression + * resources available for the mapping then the #incompr_kind value + * is used as a fallback for the mapping. When an unmap operation + * is specified this value must be zero. + */ + s16 compr_kind; + + /** + * When a map operation is specified and the #compr_kind field is + * NVGPU_KIND_INVALID then this field specifies the incompressed + * kind to use for the mapping. When an unmap operation is specified + * this value must be zero. + */ + s16 incompr_kind; + + /** + * This field is used to distinguish between a map and unmap operation. + * When this field is non-zero then it indicates a map operation with + * the value containing the handle to the physical memory buffer to + * map into the virtual pool. When this field is zero then it + * indicates an unmap operation. + */ + u32 mem_handle; + + /** + * Page offset into the memory buffer referenced by #mem_handle from + * which physical memory should be mapped. + */ + u64 mem_offset_in_pages; + + /** + * Page offset into the virtual pool at which to start the mapping. + */ + u64 virt_offset_in_pages; + + /** + * Number of pages to map or unmap. + */ + u64 num_pages; +}; + +/* + * Defined by each OS. Allows the common VM code do things to the OS specific + * buffer structures. + */ +struct nvgpu_os_vm_remap_buffer; + +/** + * This structure describes a physical memory pool. + * There is one physical memory pool for each physical memory buffer that + * is mapped into the corresponding virtual pool. + */ +struct nvgpu_vm_remap_mpool { + /** + * Red black tree node to the memory pool. + */ + struct nvgpu_rbtree_node node; + + /** + * Number of references to this physical memory pool. This + * value increments for each map operation and decrements with + * each unmap operation that references the associated physical + * memory buffer tracked by #remap_os_buf. When the reference + * count goes to zero then the reference to the associated + * physical memory buffer tracked by #remap_os_buf is released. + */ + struct nvgpu_ref ref; + + /** + * If non-NULL, the ref put function will check this l2 flag and issue + * a flush if necessary when releasing a mapping. + */ + bool *l2_flushed; + + /** + * OS-specific structure that tracks the associated physical memory + * buffer. + */ + struct nvgpu_vm_remap_os_buffer remap_os_buf; + + /** + * Pointer to virtual pool into which this physical memory pool + * is mapped. + */ + struct nvgpu_vm_remap_vpool *vpool; +}; + +static inline struct nvgpu_vm_remap_mpool * +nvgpu_vm_remap_mpool_from_ref(struct nvgpu_ref *ref) +{ + return (struct nvgpu_vm_remap_mpool *) + ((uintptr_t)ref - + offsetof(struct nvgpu_vm_remap_mpool, ref)); +} + +/** + * This structure describes a virtual memory pool. + * There is one virtual memory pool for each sparse VM area allocation that + * uses big pages. A virtual memory pool tracks the association between + * each mapped big page in the pool and the corresponding physical memory. + */ +struct nvgpu_vm_remap_vpool { + /** + * Pointer to associated VM. + */ + struct vm_gk20a *vm; + + /** + * Tree of physical memory pools that are currently mapped to this + * virtual pool. + */ + struct nvgpu_rbtree_node *mpools; + + /** + * Base offset in pages within the associated VM context of the + * virtual memory pool. This value is specified to + * #nvgpu_vm_remap_vpool_create when the associated VM area is + * allocated. + */ + u64 base_offset_in_pages; + + /* + * Number of pages mapped into the virtual memory pool. + */ + u64 num_pages; + + /** + * Pointer to array of physical memory pool pointers (one per page + * in the virtual memory pool). + */ + struct nvgpu_vm_remap_mpool **mpool_by_page; +}; + +/** + * @brief Process list of remap operations on a sparse VM area. + * + * @param vm [in] Pointer to VM context. + * @param ops [in] Pointer to list of remap operations. + * @param num_ops [in/out] On input number of entries in #ops list. + * On output number of successful remap + * operations. + * + * This function processes one or more remap operations on the virtual memory + * pool associatd with the target sparse VM area. The operations are first + * validated and then executed in order. + * + * - Acquire the vm.update_gmmu_lock. + * - Get the associated VM area using #nvgpu_vm_find_area. + * - Verify VM area is valid and has a valid virtual pool. + * - Allocate (temporary) storage for a physical memory pool structure + * for each operation in the #ops list. + * - Perform validation of each operation in the #ops list. + * - For unmap operations validate that the specified virtual memory range + * resides entirely within the target virtual memory pool. + * - For map operations validate that the specified physical memory range + * resides entirely within the associated physical memory buffer and that + * the specified virtual memory range resides entirely within the target + * virtual memory pool. Also process compression resource requests including + * fallback handling if compression resources are desired but not available. + * - If validation succeeds then proceed to process the remap operations + * in sequence. Unmap operations remove the specified mapping from the + * GPU page tables (note this sets the sparse for each unmapped page). Map + * operations insert the specified mapping into the GPU page tables. + * - Update physical memory pool reference counts for each operation in the + * #ops list. If a memory pool reference count goes to zero then release + * the reference to the associated physical memory buffer. + * - Release the vm.update_gmmu_lock. + * + * @return Zero if the virtual pool init succeeds. + * Suitable errors, for failures. + * @retval -EINVAL if a value of zero is specified for #num_ops. + * @retval -EINVAL if there is no valid virtual pool for the target VM area. + * @retval -EINVAL if the specified virtual address range does not reside + * entirely within the target VM area. + * @retval -EINVAL if the specified physical address range does not reside + * entirely within the source physical memory buffer. + * @retval -ENOMEM if memory allocation for physical memory pool structures + * fails. + */ +int nvgpu_vm_remap(struct vm_gk20a *vm, struct nvgpu_vm_remap_op *ops, + u32 *num_ops); + +/** + * @brief Create a virtual memory pool. + * + * @param vm [in] Pointer to VM context. + * @param vm_area [in] Pointer to pointer VM area. + * @param num_pages [in] Number of pages in virtual memory pool. + * + * - Check that #num_pages is non-zero. + * - Check that VM area is using big pages. + * - Check that VM area is configured as sparse. + * - Allocate memory for internal virtual pool management structures. + * - Initialize virtual pool management structures including storing #vm + * and #num_pages arguments. + * - Store virtual memory pool pointer in #vm_area. + * + * @return Zero if the virtual pool create succeeds. + * Suitable errors, for failures. + * @retval -EINVAL if a value of zero is specified for #num_pages. + * @retval -EINVAL if the VM area is not using big pages. + * @retval -EINVAL if the VM area is not configured as sparse. + * @retval -ENOMEM if memory allocation for internal resources fails. + */ +int nvgpu_vm_remap_vpool_create(struct vm_gk20a *vm, + struct nvgpu_vm_area *vm_area, + u64 num_pages); + +/** + * @brief Destroy a virtual memory pool. + * + * @param vm [in] Pointer to VM context. + * @param vm_area [in] Pointer to VM area. + * + * - Release references to physical memory pools (if any). + * - Free memory used to track virtual memory pool state. + * + * @return None. + */ +void nvgpu_vm_remap_vpool_destroy(struct vm_gk20a *vm, + struct nvgpu_vm_area *vm_area); + +#endif /* NVGPU_VM_REMAP_H */ diff --git a/drivers/gpu/nvgpu/os/linux/ioctl_as.c b/drivers/gpu/nvgpu/os/linux/ioctl_as.c index 981a1d58d..31a309db2 100644 --- a/drivers/gpu/nvgpu/os/linux/ioctl_as.c +++ b/drivers/gpu/nvgpu/os/linux/ioctl_as.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -325,6 +326,82 @@ static int nvgpu_as_ioctl_mapping_modify( args->buffer_size); } +static int nvgpu_as_ioctl_remap( + struct gk20a_as_share *as_share, + struct nvgpu_as_remap_args *args) +{ + struct gk20a *g = gk20a_from_vm(as_share->vm); + struct nvgpu_as_remap_op __user *user_remap_ops = NULL; + struct nvgpu_as_remap_op remap_op; + struct nvgpu_vm_remap_op *nvgpu_vm_remap_ops = NULL; + u32 i; + int err = 0; + + nvgpu_log_fn(g, " "); + + if (!nvgpu_is_enabled(g, NVGPU_SUPPORT_REMAP)) { + return -ENOTTY; + } + + if (args->num_ops == 0) { + return 0; + } + + /* allocate buffer for internal representation of remap ops */ + nvgpu_vm_remap_ops = nvgpu_kzalloc(g, args->num_ops * + sizeof(struct nvgpu_vm_remap_op)); + if (nvgpu_vm_remap_ops == NULL) { + err = -ENOMEM; + goto out; + } + + user_remap_ops = + (struct nvgpu_as_remap_op __user *)(uintptr_t)args->ops; + + for (i = 0; i < args->num_ops; i++) { + if (copy_from_user(&remap_op, &user_remap_ops[i], + sizeof(remap_op))) { + err = -EFAULT; + goto out; + } + + err = nvgpu_vm_remap_translate_as_op(as_share->vm, + &nvgpu_vm_remap_ops[i], + &remap_op); + if (err != 0) { + args->num_ops = 0; + goto out; + } + } + + /* execute remap ops */ + err = nvgpu_vm_remap(as_share->vm, nvgpu_vm_remap_ops, + &args->num_ops); + if (err != 0) { + goto out; + } + + /* update user params */ + for (i = 0; i < args->num_ops; i++) { + nvgpu_vm_remap_translate_vm_op(&remap_op, + &nvgpu_vm_remap_ops[i]); + + if (copy_to_user(&user_remap_ops[i], &remap_op, + sizeof(remap_op))) { + err = -EFAULT; + args->num_ops = i; + goto out; + } + } + +out: + if (nvgpu_vm_remap_ops != NULL) { + nvgpu_kfree(g, nvgpu_vm_remap_ops); + } + + return err; +} + int gk20a_as_dev_open(struct inode *inode, struct file *filp) { struct gk20a_as_share *as_share; @@ -371,7 +448,7 @@ long gk20a_as_dev_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) int err = 0; struct gk20a_as_share *as_share = filp->private_data; struct gk20a *g = gk20a_from_as(as_share->as); - + bool always_copy_to_user = false; u8 buf[NVGPU_AS_IOCTL_MAX_ARG_SIZE]; nvgpu_log_fn(g, "start %d", _IOC_NR(cmd)); @@ -466,6 +543,11 @@ long gk20a_as_dev_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) err = nvgpu_as_ioctl_mapping_modify(as_share, (struct nvgpu_as_mapping_modify_args *)buf); break; + case NVGPU_AS_IOCTL_REMAP: + err = nvgpu_as_ioctl_remap(as_share, + (struct nvgpu_as_remap_args *)buf); + always_copy_to_user = true; + break; default: err = -ENOTTY; break; @@ -473,7 +555,7 @@ long gk20a_as_dev_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) gk20a_idle(g); - if ((err == 0) && (_IOC_DIR(cmd) & _IOC_READ)) + if ((err == 0 || always_copy_to_user) && (_IOC_DIR(cmd) & _IOC_READ)) if (copy_to_user((void __user *)arg, buf, _IOC_SIZE(cmd))) err = -EFAULT; diff --git a/drivers/gpu/nvgpu/os/linux/ioctl_ctrl.c b/drivers/gpu/nvgpu/os/linux/ioctl_ctrl.c index 0ebc36f29..59d8d9e9d 100644 --- a/drivers/gpu/nvgpu/os/linux/ioctl_ctrl.c +++ b/drivers/gpu/nvgpu/os/linux/ioctl_ctrl.c @@ -268,6 +268,8 @@ static struct nvgpu_flags_mapping flags_mapping[] = { NVGPU_SUPPORT_FAULT_RECOVERY}, {NVGPU_GPU_FLAGS_SUPPORT_MAPPING_MODIFY, NVGPU_SUPPORT_MAPPING_MODIFY}, + {NVGPU_GPU_FLAGS_SUPPORT_REMAP, + NVGPU_SUPPORT_REMAP}, {NVGPU_GPU_FLAGS_SUPPORT_COMPRESSION, NVGPU_SUPPORT_COMPRESSION}, {NVGPU_GPU_FLAGS_SUPPORT_SM_TTU, diff --git a/drivers/gpu/nvgpu/os/linux/module.c b/drivers/gpu/nvgpu/os/linux/module.c index ae0e7fabd..f2f313059 100644 --- a/drivers/gpu/nvgpu/os/linux/module.c +++ b/drivers/gpu/nvgpu/os/linux/module.c @@ -312,6 +312,7 @@ void gk20a_init_linux_characteristics(struct gk20a *g) nvgpu_set_enabled(g, NVGPU_SUPPORT_PARTIAL_MAPPINGS, true); nvgpu_set_enabled(g, NVGPU_SUPPORT_DETERMINISTIC_OPTS, true); nvgpu_set_enabled(g, NVGPU_SUPPORT_USERSPACE_MANAGED_AS, true); + nvgpu_set_enabled(g, NVGPU_SUPPORT_REMAP, true); if (!IS_ENABLED(CONFIG_NVGPU_SYNCFD_NONE)) { nvgpu_set_enabled(g, NVGPU_SUPPORT_SYNC_FENCE_FDS, true); diff --git a/drivers/gpu/nvgpu/os/linux/vm_remap.c b/drivers/gpu/nvgpu/os/linux/vm_remap.c new file mode 100644 index 000000000..7925841ef --- /dev/null +++ b/drivers/gpu/nvgpu/os/linux/vm_remap.c @@ -0,0 +1,242 @@ +/* + * Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include + +#include + +#include +#include +#include +#include + +#include "os_linux.h" +#include "dmabuf_priv.h" + +#define dev_from_vm(vm) dev_from_gk20a(vm->mm->g) + +u64 nvgpu_vm_remap_get_handle(struct nvgpu_vm_remap_os_buffer *remap_os_buf) +{ + return (u64)(uintptr_t)remap_os_buf->os_priv.dmabuf; +} + +int nvgpu_vm_remap_os_buf_get(struct vm_gk20a *vm, + struct nvgpu_vm_remap_op *op, + struct nvgpu_vm_remap_os_buffer *remap_os_buf) +{ + struct gk20a *g = gk20a_from_vm(vm); + struct device *dev = dev_from_gk20a(g); + struct dma_buf *dmabuf; + struct sg_table *sgt = NULL; + struct nvgpu_sgt *nv_sgt = NULL; + struct dma_buf_attachment *attachment; + enum nvgpu_aperture aperture; + enum dma_data_direction dmabuf_direction; + int err = 0; + + /* get ref to the dmabuf fd */ + dmabuf = dma_buf_get(op->mem_handle); + if (IS_ERR(dmabuf)) { + nvgpu_warn(g, "mem_handle 0x%x is not a dmabuf", + op->mem_handle); + return -EINVAL; + } + + if (!(dmabuf->file->f_mode & (FMODE_WRITE | FMODE_PWRITE)) && + !(op->flags & NVGPU_VM_REMAP_OP_FLAGS_ACCESS_NO_WRITE)) { + nvgpu_err(g, "RW access requested for RO mapped buffer"); + err = -EINVAL; + goto clean_up; + } + + err = gk20a_dmabuf_alloc_drvdata(dmabuf, dev_from_vm(vm)); + if (err) { + nvgpu_warn(g, "failed to alloc drvdata"); + goto clean_up; + } + + if ((op->flags & NVGPU_VM_REMAP_OP_FLAGS_ACCESS_NO_WRITE) != 0) { + dmabuf_direction = DMA_TO_DEVICE; + } else { + dmabuf_direction = DMA_BIDIRECTIONAL; + } + + sgt = nvgpu_mm_pin(dev, dmabuf, &attachment, dmabuf_direction); + if (IS_ERR(sgt)) { + nvgpu_warn(g, "failed to pin dma_buf"); + err = -ENOMEM; + goto clean_up; + } + + aperture = gk20a_dmabuf_aperture(g, dmabuf); + if (aperture == APERTURE_INVALID) { + err = -EINVAL; + goto clean_up; + } + + nv_sgt = nvgpu_linux_sgt_create(g, sgt); + if (nv_sgt == NULL) { + nvgpu_warn(g, "failed to create nv_sgt"); + err = -ENOMEM; + goto clean_up; + } + + remap_os_buf->os_priv.dmabuf = dmabuf; + remap_os_buf->os_priv.attachment = attachment; + remap_os_buf->os_priv.sgt = sgt; + + remap_os_buf->os_buf.dmabuf = dmabuf; + remap_os_buf->os_buf.attachment = attachment; + remap_os_buf->os_buf.dev = dev; + + remap_os_buf->nv_sgt = nv_sgt; + remap_os_buf->aperture = aperture; + + return 0; + +clean_up: + if (nv_sgt != NULL) { + nvgpu_sgt_free(g, nv_sgt); + } + if (IS_ERR(sgt)) { + nvgpu_mm_unpin(dev, dmabuf, attachment, sgt); + } + dma_buf_put(dmabuf); + + return err; +} + +void nvgpu_vm_remap_os_buf_put(struct vm_gk20a *vm, + struct nvgpu_vm_remap_os_buffer *remap_os_buf) +{ + struct gk20a *g = gk20a_from_vm(vm); + struct device *dev = dev_from_gk20a(g); + struct gk20a_comptags comptags; + + nvgpu_mm_unpin(dev, remap_os_buf->os_priv.dmabuf, + remap_os_buf->os_priv.attachment, remap_os_buf->os_priv.sgt); + + gk20a_get_comptags(&remap_os_buf->os_buf, &comptags); + + /* + * Flush compression bit cache before releasing the physical + * memory buffer reference. + */ + if (comptags.offset != 0) { + g->ops.cbc.ctrl(g, nvgpu_cbc_op_clean, 0, 0); + g->ops.mm.cache.l2_flush(g, true); + } + + nvgpu_sgt_free(g, remap_os_buf->nv_sgt); + + dma_buf_put(remap_os_buf->os_priv.dmabuf); +} + +static int nvgpu_vm_remap_validate_map_op(struct nvgpu_as_remap_op *op) +{ + int err = 0; + u32 valid_flags = (NVGPU_AS_REMAP_OP_FLAGS_CACHEABLE | + NVGPU_AS_REMAP_OP_FLAGS_ACCESS_NO_WRITE); + + if ((op->flags & ~valid_flags) != 0) { + err = -EINVAL; + } + + return err; +} + +static int nvgpu_vm_remap_validate_unmap_op(struct nvgpu_as_remap_op *op) +{ + int err = 0; + + if ((op->compr_kind != NVGPU_KIND_INVALID) || + (op->incompr_kind != NVGPU_KIND_INVALID) || + (op->flags != 0) || + (op->mem_offset_in_pages != 0)) { + err = -EINVAL; + } + + return err; +} + +static u32 nvgpu_vm_remap_translate_as_flags(u32 flags) +{ + u32 core_flags = 0; + + if ((flags & NVGPU_AS_REMAP_OP_FLAGS_CACHEABLE) != 0) { + core_flags |= NVGPU_VM_REMAP_OP_FLAGS_CACHEABLE; + } + if ((flags & NVGPU_AS_REMAP_OP_FLAGS_ACCESS_NO_WRITE) != 0) { + core_flags |= NVGPU_VM_REMAP_OP_FLAGS_ACCESS_NO_WRITE; + } + + return core_flags; +} + +int nvgpu_vm_remap_translate_as_op(struct vm_gk20a *vm, + struct nvgpu_vm_remap_op *vm_op, + struct nvgpu_as_remap_op *as_op) +{ + int err = 0; + u64 page_size; + u64 max_num_pages; + + if (as_op->mem_handle == 0) { + err = nvgpu_vm_remap_validate_unmap_op(as_op); + } else { + err = nvgpu_vm_remap_validate_map_op(as_op); + } + + if (err != 0) + goto clean_up; + + page_size = vm->gmmu_page_sizes[GMMU_PAGE_SIZE_BIG]; + max_num_pages = (ULONG_MAX / page_size); + + if ((as_op->num_pages == 0) || + (as_op->num_pages > max_num_pages) || + (as_op->mem_offset_in_pages > max_num_pages) || + (as_op->virt_offset_in_pages > max_num_pages)) { + err = -EINVAL; + goto clean_up; + } + + vm_op->flags = nvgpu_vm_remap_translate_as_flags(as_op->flags); + vm_op->compr_kind = as_op->compr_kind; + vm_op->incompr_kind = as_op->incompr_kind; + vm_op->mem_handle = as_op->mem_handle; + vm_op->mem_offset_in_pages = as_op->mem_offset_in_pages; + vm_op->virt_offset_in_pages = as_op->virt_offset_in_pages; + vm_op->num_pages = as_op->num_pages; + + return 0; + +clean_up: + return err; +} + +void nvgpu_vm_remap_translate_vm_op(struct nvgpu_as_remap_op *as_op, + struct nvgpu_vm_remap_op *vm_op) +{ + as_op->compr_kind = vm_op->compr_kind; + as_op->incompr_kind = vm_op->incompr_kind; +} diff --git a/drivers/gpu/nvgpu/os/posix/posix-vm_remap.c b/drivers/gpu/nvgpu/os/posix/posix-vm_remap.c new file mode 100644 index 000000000..ab2cb3807 --- /dev/null +++ b/drivers/gpu/nvgpu/os/posix/posix-vm_remap.c @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ +#include +#include + +u64 nvgpu_vm_remap_get_handle(struct nvgpu_vm_remap_os_buffer *remap_os_buf) +{ + return 0; +} + +int nvgpu_vm_remap_os_buf_get(struct vm_gk20a *vm, u32 mem_handle, + struct nvgpu_vm_remap_os_buffer *remap_os_buf) +{ + return 0; +} + +void nvgpu_vm_remap_os_buf_put(struct vm_gk20a *vm, + struct nvgpu_vm_remap_os_buffer *remap_os_buf) +{ + return; +} diff --git a/include/uapi/linux/nvgpu-as.h b/include/uapi/linux/nvgpu-as.h index b349ece92..8b98b86e7 100644 --- a/include/uapi/linux/nvgpu-as.h +++ b/include/uapi/linux/nvgpu-as.h @@ -431,6 +431,137 @@ struct nvgpu_as_mapping_modify_args { __u64 map_address; /* in, base virtual address of mapped buffer */ }; +/* + * VM remap operation. + * + * The VM remap operation structure represents a single map or unmap operation + * to be executed by the NVGPU_AS_IOCTL_REMAP ioctl. + * + * The format of the structure is as follows: + * + * @flags [IN] + * + * The following remap operation flags are supported: + * + * %NVGPU_AS_REMAP_OP_FLAGS_CACHEABLE + * + * Specify that the associated mapping shall be GPU cachable. + * + * %NVGPU_AS_REMAP_OP_FLAGS_ACCESS_NO_WRITE + * + * Specify that the associated mapping shall be read-only. This flag + * must be set if the physical memory buffer represented by @mem_handle + * is mapped read-only. + * + * This field must be zero for unmap operations. + * + * @compr_kind [IN/OUT] + * @incompr_kind [IN/OUT] + * + * On input these fields specify the compressible and incompressible kinds + * to be used for the mapping. If @compr_kind is not set to NV_KIND_INVALID + * then nvgpu will attempt to allocate compression resources. If + * @compr_kind is set to NV_KIND_INVALID or there are no compression + * resources then nvgpu will attempt to use @incompr_kind. If both + * @compr_kind and @incompr_kind are set to NV_KIND_INVALID then -EINVAL is + * returned. These fields must be set to NV_KIND_INVALID for unmap + * operations. On output these fields return the selected kind. If + * @compr_kind is set to a valid compressible kind but the required + * compression resources are not available then @compr_kind will return + * NV_INVALID_KIND and the @incompr_kind value will be used for the mapping. + * + * @mem_handle [IN] + * + * Specify the memory handle (dmabuf_fd) associated with the physical + * memory buffer to be mapped. This field must be zero for unmap + * operations. + * + * @mem_offset_in_pages [IN] + * + * Specify an offset into the physical buffer associated with mem_handle at + * which to start the mapping. This value is in pages and the page size + * is the big page size in the associated sparse address space. This value + * must be zero for unmap operations. + * + * @virt_offset_in_pages [IN] + * + * Specify the virtual memory start offset of the region to map or unmap. + * This value is in pages and the page size is the big page size in the + * associated sparse address space. + * + * @num_pages [IN] + * Specify the number of pages to map or unmap. + */ +struct nvgpu_as_remap_op { +#define NVGPU_AS_REMAP_OP_FLAGS_CACHEABLE (1 << 2) +#define NVGPU_AS_REMAP_OP_FLAGS_ACCESS_NO_WRITE (1 << 10) + + /* in: For map operations, this field specifies the mask of + * NVGPU_AS_REMAP flags to use for the mapping. For unmap operations + * this field must be zero */ + __u32 flags; + + /* in: For map operations, this field specifies the desired + * compressible kind. For unmap operations this field must be set + * to NV_KIND_INVALID. + * out: For map operations this field returns the actual kind used + * for the mapping. This can be useful for detecting if a compressed + * mapping request was forced to use the fallback incompressible kind + * value because sufficient compression resources are not available. */ + __s16 compr_kind; + + /* in: For map operations, this field specifies the desired + * incompressible kind. This value will be used as the fallback kind + * if a valid compressible kind value was specified in the compr_kind + * field but sufficient compression resources are not available. For + * unmap operations this field must be set to NV_KIND_INVALID. */ + __s16 incompr_kind; + + /* in: For map operations, this field specifies the handle (dmabuf_fd) + * for the physical memory buffer to map into the specified virtual + * address range. For unmap operations, this field must be set to + * zero. */ + __u32 mem_handle; + + /* This field is reserved for padding purposes. */ + __s32 reserved; + + /* in: For map operations this field specifies the offset (in pages) + * into the physical memory buffer associated with mem_handle from + * from which physical page information should be collected for + * the mapping. For unmap operations this field must be zero. */ + __u64 mem_offset_in_pages; + + /* in: For both map and unmap operations this field specifies the + * virtual address space start offset in pages for the operation. */ + __u64 virt_offset_in_pages; + + /* in: For both map and unmap operations this field specifies the + * number of pages to map or unmap. */ + __u64 num_pages; +}; + +/* + * VM remap IOCTL + * + * This ioctl can be used to issue multiple map and/or unmap operations in + * a single request. VM remap operations are only valid on address spaces + * that have been allocated with NVGPU_AS_ALLOC_SPACE_FLAGS_SPARSE. + * Validation of remap operations is performed before any changes are made + * to the associated sparse address space so either all map and/or unmap + * operations are performed or none of them area. + */ +struct nvgpu_as_remap_args { + /* in: This field specifies a pointer into the caller's address space + * containing an array of one or more nvgpu_as_remap_op structures. */ + __u64 ops; + + /* in/out: On input this field specifies the number of operations in + * the ops array. On output this field returns the successful + * number of remap operations. */ + __u32 num_ops; +}; + #define NVGPU_AS_IOCTL_BIND_CHANNEL \ _IOWR(NVGPU_AS_IOCTL_MAGIC, 1, struct nvgpu_as_bind_channel_args) #define NVGPU32_AS_IOCTL_ALLOC_SPACE \ @@ -455,10 +586,12 @@ struct nvgpu_as_mapping_modify_args { _IOR(NVGPU_AS_IOCTL_MAGIC, 12, struct nvgpu_as_get_sync_ro_map_args) #define NVGPU_AS_IOCTL_MAPPING_MODIFY \ _IOWR(NVGPU_AS_IOCTL_MAGIC, 13, struct nvgpu_as_mapping_modify_args) +#define NVGPU_AS_IOCTL_REMAP \ + _IOWR(NVGPU_AS_IOCTL_MAGIC, 14, struct nvgpu_as_remap_args) #define NVGPU_AS_IOCTL_LAST \ - _IOC_NR(NVGPU_AS_IOCTL_MAPPING_MODIFY) + _IOC_NR(NVGPU_AS_IOCTL_REMAP) #define NVGPU_AS_IOCTL_MAX_ARG_SIZE \ sizeof(struct nvgpu_as_map_buffer_ex_args) -#endif +#endif /* #define _UAPI__LINUX_NVGPU_AS_H__ */ diff --git a/include/uapi/linux/nvgpu-ctrl.h b/include/uapi/linux/nvgpu-ctrl.h index e6e09389c..9d43b6908 100644 --- a/include/uapi/linux/nvgpu-ctrl.h +++ b/include/uapi/linux/nvgpu-ctrl.h @@ -158,6 +158,8 @@ struct nvgpu_gpu_zbc_query_table_args { #define NVGPU_GPU_FLAGS_SUPPORT_FAULT_RECOVERY (1ULL << 33) /* Mapping modify is enabled */ #define NVGPU_GPU_FLAGS_SUPPORT_MAPPING_MODIFY (1ULL << 34) +/* Remap is enabled */ +#define NVGPU_GPU_FLAGS_SUPPORT_REMAP (1ULL << 35) /* Compression is enabled */ #define NVGPU_GPU_FLAGS_SUPPORT_COMPRESSION (1ULL << 36) /* SM TTU is enabled */ @@ -1100,4 +1102,4 @@ struct nvgpu_gpu_register_buffer_args { #define NVGPU_GPU_IOCTL_MAX_ARG_SIZE \ sizeof(struct nvgpu_gpu_get_cpu_time_correlation_info_args) -#endif +#endif /* _UAPI__LINUX_NVGPU_CTRL_H__ */