nv-tegra-mirror/linux-nvgpu
1
0
Fork 0
mirror of git://nv-tegra.nvidia.com/linux-nvgpu.git synced 2025-12-22 17:36:20 +03:00
Code Issues Packages Projects Releases Wiki Activity

gpu: nvgpu: fix REMAP to support small/big pages

Browse Source 
Initially, REMAP only worked with big pages but in some cases
only small pages are supported where REMAP functionality is
also needed.

This cleans up some page size assumptions. In particular, on a
remap request, the nvgpu_vm_area is found from the passed in VA,
but can only be done from virt_offset_in_pages if we're also
told the page size.

This now occurs from _PAGESIZE_ flags which are required by
both map and unmap operations.

Jira NVGPU-6804

Change-Id: I311980a1b5e0e5e1840bdc1123479350a5c9d469
Signed-off-by: Chris Johnson <cwj@nvidia.com>
Reviewed-on: https://git-master.nvidia.com/r/c/linux-nvgpu/+/2566087
Tested-by: mobile promotions <svcmobile_promotions@nvidia.com>
Reviewed-by: mobile promotions <svcmobile_promotions@nvidia.com>
This commit is contained in:
Chris Johnson
2022-01-16 19:28:26 -08:00
committed by mobile promotions
parent 359e83b45a
commit 14ed75e857
5 changed files with 124 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
drivers/gpu/nvgpu/common/mm/vm_area.c
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2017-2022, NVIDIA CORPORATION. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
@@ -245,8 +245,7 @@ int nvgpu_vm_area_alloc(struct vm_gk20a *vm, u64 pages, u32 page_size,
nvgpu_init_list_node(&vm_area->vm_area_list);
#ifdef CONFIG_NVGPU_REMAP
if (((flags & NVGPU_VM_AREA_ALLOC_SPARSE) != 0U) &&
(vm_area->pgsz_idx == GMMU_PAGE_SIZE_BIG)) {
if ((flags & NVGPU_VM_AREA_ALLOC_SPARSE) != 0U) {
err = nvgpu_vm_remap_vpool_create(vm, vm_area, pages);
if (err != 0) {
goto free_vaddr;

50
drivers/gpu/nvgpu/common/mm/vm_remap.c
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2017-2022, NVIDIA CORPORATION. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
@@ -28,23 +28,6 @@
#include <nvgpu/string.h>
#include <nvgpu/power_features/pg.h>
/*
* Return page size index of page size for VM areas that can be used with
* remap.
*/
static inline u32 nvgpu_vm_remap_pgsz_idx(struct vm_gk20a *vm)
{
return GMMU_PAGE_SIZE_BIG;
}
/*
* Return page size for VM areas that can be used with remap.
*/
static inline u64 nvgpu_vm_remap_page_size(struct vm_gk20a *vm)
{
return vm->gmmu_page_sizes[GMMU_PAGE_SIZE_BIG];
}
/*
* Return a pointer to the os-specific structure for the specified physical
* memory pool.
@@ -184,6 +167,11 @@ static int nvgpu_vm_remap_validate_vpool(struct nvgpu_vm_remap_vpool *vpool,
{
u64 first_page = op->virt_offset_in_pages;
u64 last_page = op->virt_offset_in_pages + op->num_pages - 1ULL;
u64 page_size = nvgpu_vm_remap_page_size(op);
if (page_size == 0) {
return -EINVAL;
}
if (first_page < vpool->base_offset_in_pages ||
last_page >= vpool->base_offset_in_pages + vpool->num_pages ||
@@ -212,7 +200,7 @@ static int nvgpu_vm_remap_validate_map(struct vm_gk20a *vm,
struct nvgpu_vm_remap_op *op,
struct nvgpu_vm_remap_os_buffer *remap_os_buf)
{
u64 page_size = nvgpu_vm_remap_page_size(vm);
u64 page_size = nvgpu_vm_remap_page_size(op);
u64 map_offset;
u64 map_size;
u64 os_buf_size;
@@ -278,11 +266,19 @@ static struct nvgpu_vm_remap_vpool *nvgpu_vm_remap_get_vpool_locked(
struct vm_gk20a *vm, struct nvgpu_vm_remap_op *op)
{
struct gk20a *g = gk20a_from_vm(vm);
u64 page_size = nvgpu_vm_remap_page_size(vm);
u64 offset = nvgpu_safe_mult_u64(op->virt_offset_in_pages, page_size);
struct nvgpu_vm_area *vm_area = nvgpu_vm_area_find(vm, offset);
u64 page_size = nvgpu_vm_remap_page_size(op);
u64 offset;
struct nvgpu_vm_area *vm_area;
if ((vm_area == NULL) || (vm_area->vpool == NULL)) {
if (page_size == 0) {
return NULL;
}
offset = nvgpu_safe_mult_u64(op->virt_offset_in_pages, page_size);
vm_area = nvgpu_vm_area_find(vm, offset);
if ((vm_area == NULL) || (vm_area->vpool == NULL) ||
(vm->gmmu_page_sizes[vm_area->pgsz_idx] != page_size)) {
return NULL;
}
@@ -358,7 +354,7 @@ static u64 nvgpu_vm_remap_get_ctag_offset(struct vm_gk20a *vm,
struct gk20a_comptags comptags;
u64 ctag = 0;
u64 ctag_offset = 0;
u64 page_size = nvgpu_vm_remap_page_size(vm);
u64 page_size = nvgpu_vm_remap_page_size(op);
u64 phys_offset = nvgpu_safe_mult_u64(op->mem_offset_in_pages,
page_size);
@@ -423,8 +419,8 @@ static int nvgpu_vm_remap_execute_remaps(struct vm_gk20a *vm,
struct gk20a *g = gk20a_from_vm(vm);
struct nvgpu_vm_remap_op *op;
struct nvgpu_vm_remap_os_buffer *remap_os_buf;
u32 pgsz_idx = nvgpu_vm_remap_pgsz_idx(vm);
u64 page_size = nvgpu_vm_remap_page_size(vm);
u32 pgsz_idx = vpool->vm_area->pgsz_idx;
u64 page_size = vm->gmmu_page_sizes[pgsz_idx];
u64 map_addr = 0;
u64 phys_offset = 0;
u64 map_size;
@@ -733,7 +729,6 @@ int nvgpu_vm_remap_vpool_create(struct vm_gk20a *vm,
u64 start_page_nr = 0;
if ((num_pages == 0ULL) ||
(vm_area->pgsz_idx != GMMU_PAGE_SIZE_BIG) ||
((vm_area->flags & NVGPU_VM_AREA_ALLOC_SPARSE) == 0U)) {
return -EINVAL;
}
@@ -750,6 +745,7 @@ int nvgpu_vm_remap_vpool_create(struct vm_gk20a *vm,
vp->num_pages = num_pages;
vp->vm = vm;
vp->vm_area = vm_area;
vm_area->vpool = vp;
return 0;

44
drivers/gpu/nvgpu/include/nvgpu/vm_remap.h
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2017-2022, NVIDIA CORPORATION. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
@@ -37,14 +37,17 @@
*/
#define NVGPU_VM_REMAP_OP_FLAGS_CACHEABLE BIT32(1)
#define NVGPU_VM_REMAP_OP_FLAGS_ACCESS_NO_WRITE BIT32(7)
#define NVGPU_VM_REMAP_OP_FLAGS_PAGESIZE_4K BIT32(12)
#define NVGPU_VM_REMAP_OP_FLAGS_PAGESIZE_64K BIT32(13)
#define NVGPU_VM_REMAP_OP_FLAGS_PAGESIZE_128K BIT32(14)
/**
* This structure describes a single remap operation (either a map or unmap).
*/
struct nvgpu_vm_remap_op {
/**
* When a map operation is specified this field contains any flags
* to use when setting up the mapping. When an unmap operation is
* specified this field must be zero.
* When a map/unmap operation is specified this field contains flags
* needed to determine the page size used to generate the map/unmap
* mem and virt offsets and/or flags used when setting up the mapping.
*/
u32 flags;
@@ -56,7 +59,7 @@ struct nvgpu_vm_remap_op {
* than NVGPU_KIND_INVALID is specified but there are no compression
* resources available for the mapping then the #incompr_kind value
* is used as a fallback for the mapping. When an unmap operation
* is specified this value must be zero.
* is specified this value must be zero.
*/
s16 compr_kind;
@@ -123,7 +126,7 @@ struct nvgpu_vm_remap_mpool {
/**
* If non-NULL, the ref put function will check this l2 flag and issue
* a flush if necessary when releasing a mapping.
* a flush if necessary when releasing a mapping.
*/
bool *l2_flushed;
@@ -148,11 +151,27 @@ nvgpu_vm_remap_mpool_from_ref(struct nvgpu_ref *ref)
offsetof(struct nvgpu_vm_remap_mpool, ref));
}
static inline u64 nvgpu_vm_remap_page_size(struct nvgpu_vm_remap_op *op)
{
u64 pagesize = 0;
/* validate_map/unmap_op ensures a single pagesize flag */
if (op->flags & NVGPU_VM_REMAP_OP_FLAGS_PAGESIZE_4K)
pagesize = SZ_4K;
if (op->flags & NVGPU_VM_REMAP_OP_FLAGS_PAGESIZE_64K)
pagesize = SZ_64K;
if (op->flags & NVGPU_VM_REMAP_OP_FLAGS_PAGESIZE_128K)
pagesize = SZ_128K;
nvgpu_assert(pagesize);
return pagesize;
}
/**
* This structure describes a virtual memory pool.
* There is one virtual memory pool for each sparse VM area allocation that
* uses big pages. A virtual memory pool tracks the association between
* each mapped big page in the pool and the corresponding physical memory.
* There is one virtual memory pool for each sparse VM area allocation.
* A virtual memory pool tracks the association between each mapped page
* in the pool and the corresponding physical memory.
*/
struct nvgpu_vm_remap_vpool {
/**
@@ -160,6 +179,11 @@ struct nvgpu_vm_remap_vpool {
*/
struct vm_gk20a *vm;
/**
* Pointer to associated VM area.
*/
struct nvgpu_vm_area *vm_area;
/**
* Tree of physical memory pools that are currently mapped to this
* virtual pool.
@@ -243,7 +267,6 @@ int nvgpu_vm_remap(struct vm_gk20a *vm, struct nvgpu_vm_remap_op *ops,
* @param num_pages [in] Number of pages in virtual memory pool.
*
* - Check that #num_pages is non-zero.
* - Check that VM area is using big pages.
* - Check that VM area is configured as sparse.
* - Allocate memory for internal virtual pool management structures.
* - Initialize virtual pool management structures including storing #vm
@@ -253,7 +276,6 @@ int nvgpu_vm_remap(struct vm_gk20a *vm, struct nvgpu_vm_remap_op *ops,
* @return Zero if the virtual pool create succeeds.
* Suitable errors, for failures.
* @retval -EINVAL if a value of zero is specified for #num_pages.
* @retval -EINVAL if the VM area is not using big pages.
* @retval -EINVAL if the VM area is not configured as sparse.
* @retval -ENOMEM if memory allocation for internal resources fails.
*/

57
drivers/gpu/nvgpu/os/linux/vm_remap.c
View File

@@ -1,5 +1,5 @@
/*
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2017-2022, NVIDIA CORPORATION. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
@@ -157,23 +157,50 @@ void nvgpu_vm_remap_os_buf_put(struct vm_gk20a *vm,
static int nvgpu_vm_remap_validate_map_op(struct nvgpu_as_remap_op *op)
{
int err = 0;
u32 valid_flags = (NVGPU_AS_REMAP_OP_FLAGS_CACHEABLE |
const u32 pagesize_flags = (NVGPU_AS_REMAP_OP_FLAGS_PAGESIZE_4K |
NVGPU_AS_REMAP_OP_FLAGS_PAGESIZE_64K |
NVGPU_AS_REMAP_OP_FLAGS_PAGESIZE_128K);
const u32 valid_flags = (pagesize_flags |
NVGPU_AS_REMAP_OP_FLAGS_CACHEABLE |
NVGPU_AS_REMAP_OP_FLAGS_ACCESS_NO_WRITE);
const u32 pagesize = op->flags & pagesize_flags;
if ((op->flags & ~valid_flags) != 0) {
err = -EINVAL;
}
/* must be set and to a single pagesize */
if ((pagesize != NVGPU_AS_REMAP_OP_FLAGS_PAGESIZE_4K) &&
(pagesize != NVGPU_AS_REMAP_OP_FLAGS_PAGESIZE_64K) &&
(pagesize != NVGPU_AS_REMAP_OP_FLAGS_PAGESIZE_128K)) {
err = -EINVAL;
}
return err;
}
static int nvgpu_vm_remap_validate_unmap_op(struct nvgpu_as_remap_op *op)
{
int err = 0;
const u32 pagesize_flags = (NVGPU_AS_REMAP_OP_FLAGS_PAGESIZE_4K |
NVGPU_AS_REMAP_OP_FLAGS_PAGESIZE_64K |
NVGPU_AS_REMAP_OP_FLAGS_PAGESIZE_128K);
const u32 valid_flags = pagesize_flags;
const u32 pagesize = op->flags & pagesize_flags;
if ((op->flags & ~valid_flags) != 0) {
err = -EINVAL;
}
/* must be set and to a single pagesize */
if ((pagesize != NVGPU_AS_REMAP_OP_FLAGS_PAGESIZE_4K) &&
(pagesize != NVGPU_AS_REMAP_OP_FLAGS_PAGESIZE_64K) &&
(pagesize != NVGPU_AS_REMAP_OP_FLAGS_PAGESIZE_128K)) {
err = -EINVAL;
}
if ((op->compr_kind != NVGPU_KIND_INVALID) ||
(op->incompr_kind != NVGPU_KIND_INVALID) ||
(op->flags != 0) ||
(op->mem_offset_in_pages != 0)) {
err = -EINVAL;
}
@@ -191,7 +218,15 @@ static u32 nvgpu_vm_remap_translate_as_flags(u32 flags)
if ((flags & NVGPU_AS_REMAP_OP_FLAGS_ACCESS_NO_WRITE) != 0) {
core_flags |= NVGPU_VM_REMAP_OP_FLAGS_ACCESS_NO_WRITE;
}
if ((flags & NVGPU_AS_REMAP_OP_FLAGS_PAGESIZE_4K) != 0) {
core_flags |= NVGPU_VM_REMAP_OP_FLAGS_PAGESIZE_4K;
}
if ((flags & NVGPU_AS_REMAP_OP_FLAGS_PAGESIZE_64K) != 0) {
core_flags |= NVGPU_VM_REMAP_OP_FLAGS_PAGESIZE_64K;
}
if ((flags & NVGPU_AS_REMAP_OP_FLAGS_PAGESIZE_128K) != 0) {
core_flags |= NVGPU_VM_REMAP_OP_FLAGS_PAGESIZE_128K;
}
return core_flags;
}
@@ -201,7 +236,6 @@ int nvgpu_vm_remap_translate_as_op(struct vm_gk20a *vm,
{
int err = 0;
u64 page_size;
u64 max_num_pages;
if (as_op->mem_handle == 0) {
err = nvgpu_vm_remap_validate_unmap_op(as_op);
@@ -212,18 +246,17 @@ int nvgpu_vm_remap_translate_as_op(struct vm_gk20a *vm,
if (err != 0)
goto clean_up;
page_size = vm->gmmu_page_sizes[GMMU_PAGE_SIZE_BIG];
max_num_pages = (ULONG_MAX / page_size);
vm_op->flags = nvgpu_vm_remap_translate_as_flags(as_op->flags);
page_size = nvgpu_vm_remap_page_size(vm_op);
if ((as_op->num_pages == 0) ||
(as_op->num_pages > max_num_pages) ||
(as_op->mem_offset_in_pages > max_num_pages) ||
(as_op->virt_offset_in_pages > max_num_pages)) {
if ((as_op->num_pages == 0) || (page_size == 0) ||
(as_op->num_pages > (vm->va_limit / page_size)) ||
(as_op->mem_offset_in_pages > (vm->va_limit / page_size)) ||
(as_op->virt_offset_in_pages > (vm->va_limit / page_size))) {
err = -EINVAL;
goto clean_up;
}
vm_op->flags = nvgpu_vm_remap_translate_as_flags(as_op->flags);
vm_op->compr_kind = as_op->compr_kind;
vm_op->incompr_kind = as_op->incompr_kind;
vm_op->mem_handle = as_op->mem_handle;

34
include/uapi/linux/nvgpu-as.h
View File

@@ -454,7 +454,14 @@ struct nvgpu_as_mapping_modify_args {
* must be set if the physical memory buffer represented by @mem_handle
* is mapped read-only.
*
* This field must be zero for unmap operations.
* %NVGPU_AS_REMAP_OP_FLAGS_PAGESIZE_4K
* %NVGPU_AS_REMAP_OP_FLAGS_PAGESIZE_64K
* %NVGPU_AS_REMAP_OP_FLAGS_PAGESIZE_128K
*
* One, and only one, of these flags must be set for both map/unmap
* ops and indicates the assumed page size of the mem_offset_in_pages
* and virt_offset_in_pages. This value is also verified against the
* page size of the address space.
*
* @compr_kind [IN/OUT]
* @incompr_kind [IN/OUT]
@@ -479,16 +486,14 @@ struct nvgpu_as_mapping_modify_args {
*
* @mem_offset_in_pages [IN]
*
* Specify an offset into the physical buffer associated with mem_handle at
* which to start the mapping. This value is in pages and the page size
* is the big page size in the associated sparse address space. This value
* must be zero for unmap operations.
* Specify an offset (in pages) into the physical buffer associated with
* mem_handle at which to start the mapping. This value must be zero for
* unmap operations.
*
* @virt_offset_in_pages [IN]
*
* Specify the virtual memory start offset of the region to map or unmap.
* This value is in pages and the page size is the big page size in the
* associated sparse address space.
* Specify the virtual memory start offset (in pages) of the region to map
* or unmap.
*
* @num_pages [IN]
* Specify the number of pages to map or unmap.
@@ -496,14 +501,17 @@ struct nvgpu_as_mapping_modify_args {
struct nvgpu_as_remap_op {
#define NVGPU_AS_REMAP_OP_FLAGS_CACHEABLE (1 << 2)
#define NVGPU_AS_REMAP_OP_FLAGS_ACCESS_NO_WRITE (1 << 10)
#define NVGPU_AS_REMAP_OP_FLAGS_PAGESIZE_4K (1 << 15)
#define NVGPU_AS_REMAP_OP_FLAGS_PAGESIZE_64K (1 << 16)
#define NVGPU_AS_REMAP_OP_FLAGS_PAGESIZE_128K (1 << 17)
/* in: For map operations, this field specifies the mask of
* NVGPU_AS_REMAP flags to use for the mapping. For unmap operations
* this field must be zero */
/* in: For map and unmap (one and only one) of the _PAGESIZE_ flags is
* required to interpret the mem_offset_in_pages and virt_offset_in_pages
* correctly. The other flags are used only with map operations. */
__u32 flags;
/* in: For map operations, this field specifies the desired
* compressible kind. For unmap operations this field must be set
* compressible kind. For unmap operations this field must be set
* to NV_KIND_INVALID.
* out: For map operations this field returns the actual kind used
* for the mapping. This can be useful for detecting if a compressed
@@ -550,7 +558,7 @@ struct nvgpu_as_remap_op {
* that have been allocated with NVGPU_AS_ALLOC_SPACE_FLAGS_SPARSE.
* Validation of remap operations is performed before any changes are made
* to the associated sparse address space so either all map and/or unmap
* operations are performed or none of them area.
* operations are performed or none of them are.
*/
struct nvgpu_as_remap_args {
/* in: This field specifies a pointer into the caller's address space