diff --git a/drivers/gpu/nvgpu/common/mm/vm_area.c b/drivers/gpu/nvgpu/common/mm/vm_area.c index 34170a4ce..842d1ca44 100644 --- a/drivers/gpu/nvgpu/common/mm/vm_area.c +++ b/drivers/gpu/nvgpu/common/mm/vm_area.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2017-2022, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -245,8 +245,7 @@ int nvgpu_vm_area_alloc(struct vm_gk20a *vm, u64 pages, u32 page_size, nvgpu_init_list_node(&vm_area->vm_area_list); #ifdef CONFIG_NVGPU_REMAP - if (((flags & NVGPU_VM_AREA_ALLOC_SPARSE) != 0U) && - (vm_area->pgsz_idx == GMMU_PAGE_SIZE_BIG)) { + if ((flags & NVGPU_VM_AREA_ALLOC_SPARSE) != 0U) { err = nvgpu_vm_remap_vpool_create(vm, vm_area, pages); if (err != 0) { goto free_vaddr; diff --git a/drivers/gpu/nvgpu/common/mm/vm_remap.c b/drivers/gpu/nvgpu/common/mm/vm_remap.c index 8807f1a63..051ed321f 100644 --- a/drivers/gpu/nvgpu/common/mm/vm_remap.c +++ b/drivers/gpu/nvgpu/common/mm/vm_remap.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2017-2022, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -28,23 +28,6 @@ #include #include -/* - * Return page size index of page size for VM areas that can be used with - * remap. - */ -static inline u32 nvgpu_vm_remap_pgsz_idx(struct vm_gk20a *vm) -{ - return GMMU_PAGE_SIZE_BIG; -} - -/* - * Return page size for VM areas that can be used with remap. - */ -static inline u64 nvgpu_vm_remap_page_size(struct vm_gk20a *vm) -{ - return vm->gmmu_page_sizes[GMMU_PAGE_SIZE_BIG]; -} - /* * Return a pointer to the os-specific structure for the specified physical * memory pool. @@ -184,6 +167,11 @@ static int nvgpu_vm_remap_validate_vpool(struct nvgpu_vm_remap_vpool *vpool, { u64 first_page = op->virt_offset_in_pages; u64 last_page = op->virt_offset_in_pages + op->num_pages - 1ULL; + u64 page_size = nvgpu_vm_remap_page_size(op); + + if (page_size == 0) { + return -EINVAL; + } if (first_page < vpool->base_offset_in_pages || last_page >= vpool->base_offset_in_pages + vpool->num_pages || @@ -212,7 +200,7 @@ static int nvgpu_vm_remap_validate_map(struct vm_gk20a *vm, struct nvgpu_vm_remap_op *op, struct nvgpu_vm_remap_os_buffer *remap_os_buf) { - u64 page_size = nvgpu_vm_remap_page_size(vm); + u64 page_size = nvgpu_vm_remap_page_size(op); u64 map_offset; u64 map_size; u64 os_buf_size; @@ -278,11 +266,19 @@ static struct nvgpu_vm_remap_vpool *nvgpu_vm_remap_get_vpool_locked( struct vm_gk20a *vm, struct nvgpu_vm_remap_op *op) { struct gk20a *g = gk20a_from_vm(vm); - u64 page_size = nvgpu_vm_remap_page_size(vm); - u64 offset = nvgpu_safe_mult_u64(op->virt_offset_in_pages, page_size); - struct nvgpu_vm_area *vm_area = nvgpu_vm_area_find(vm, offset); + u64 page_size = nvgpu_vm_remap_page_size(op); + u64 offset; + struct nvgpu_vm_area *vm_area; - if ((vm_area == NULL) || (vm_area->vpool == NULL)) { + if (page_size == 0) { + return NULL; + } + + offset = nvgpu_safe_mult_u64(op->virt_offset_in_pages, page_size); + vm_area = nvgpu_vm_area_find(vm, offset); + + if ((vm_area == NULL) || (vm_area->vpool == NULL) || + (vm->gmmu_page_sizes[vm_area->pgsz_idx] != page_size)) { return NULL; } @@ -358,7 +354,7 @@ static u64 nvgpu_vm_remap_get_ctag_offset(struct vm_gk20a *vm, struct gk20a_comptags comptags; u64 ctag = 0; u64 ctag_offset = 0; - u64 page_size = nvgpu_vm_remap_page_size(vm); + u64 page_size = nvgpu_vm_remap_page_size(op); u64 phys_offset = nvgpu_safe_mult_u64(op->mem_offset_in_pages, page_size); @@ -423,8 +419,8 @@ static int nvgpu_vm_remap_execute_remaps(struct vm_gk20a *vm, struct gk20a *g = gk20a_from_vm(vm); struct nvgpu_vm_remap_op *op; struct nvgpu_vm_remap_os_buffer *remap_os_buf; - u32 pgsz_idx = nvgpu_vm_remap_pgsz_idx(vm); - u64 page_size = nvgpu_vm_remap_page_size(vm); + u32 pgsz_idx = vpool->vm_area->pgsz_idx; + u64 page_size = vm->gmmu_page_sizes[pgsz_idx]; u64 map_addr = 0; u64 phys_offset = 0; u64 map_size; @@ -733,7 +729,6 @@ int nvgpu_vm_remap_vpool_create(struct vm_gk20a *vm, u64 start_page_nr = 0; if ((num_pages == 0ULL) || - (vm_area->pgsz_idx != GMMU_PAGE_SIZE_BIG) || ((vm_area->flags & NVGPU_VM_AREA_ALLOC_SPARSE) == 0U)) { return -EINVAL; } @@ -750,6 +745,7 @@ int nvgpu_vm_remap_vpool_create(struct vm_gk20a *vm, vp->num_pages = num_pages; vp->vm = vm; + vp->vm_area = vm_area; vm_area->vpool = vp; return 0; diff --git a/drivers/gpu/nvgpu/include/nvgpu/vm_remap.h b/drivers/gpu/nvgpu/include/nvgpu/vm_remap.h index 5059b8b75..767ce41b1 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/vm_remap.h +++ b/drivers/gpu/nvgpu/include/nvgpu/vm_remap.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2017-2022, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -37,14 +37,17 @@ */ #define NVGPU_VM_REMAP_OP_FLAGS_CACHEABLE BIT32(1) #define NVGPU_VM_REMAP_OP_FLAGS_ACCESS_NO_WRITE BIT32(7) +#define NVGPU_VM_REMAP_OP_FLAGS_PAGESIZE_4K BIT32(12) +#define NVGPU_VM_REMAP_OP_FLAGS_PAGESIZE_64K BIT32(13) +#define NVGPU_VM_REMAP_OP_FLAGS_PAGESIZE_128K BIT32(14) /** * This structure describes a single remap operation (either a map or unmap). */ struct nvgpu_vm_remap_op { /** - * When a map operation is specified this field contains any flags - * to use when setting up the mapping. When an unmap operation is - * specified this field must be zero. + * When a map/unmap operation is specified this field contains flags + * needed to determine the page size used to generate the map/unmap + * mem and virt offsets and/or flags used when setting up the mapping. */ u32 flags; @@ -56,7 +59,7 @@ struct nvgpu_vm_remap_op { * than NVGPU_KIND_INVALID is specified but there are no compression * resources available for the mapping then the #incompr_kind value * is used as a fallback for the mapping. When an unmap operation - * is specified this value must be zero. + * is specified this value must be zero. */ s16 compr_kind; @@ -123,7 +126,7 @@ struct nvgpu_vm_remap_mpool { /** * If non-NULL, the ref put function will check this l2 flag and issue - * a flush if necessary when releasing a mapping. + * a flush if necessary when releasing a mapping. */ bool *l2_flushed; @@ -148,11 +151,27 @@ nvgpu_vm_remap_mpool_from_ref(struct nvgpu_ref *ref) offsetof(struct nvgpu_vm_remap_mpool, ref)); } +static inline u64 nvgpu_vm_remap_page_size(struct nvgpu_vm_remap_op *op) +{ + u64 pagesize = 0; + + /* validate_map/unmap_op ensures a single pagesize flag */ + if (op->flags & NVGPU_VM_REMAP_OP_FLAGS_PAGESIZE_4K) + pagesize = SZ_4K; + if (op->flags & NVGPU_VM_REMAP_OP_FLAGS_PAGESIZE_64K) + pagesize = SZ_64K; + if (op->flags & NVGPU_VM_REMAP_OP_FLAGS_PAGESIZE_128K) + pagesize = SZ_128K; + + nvgpu_assert(pagesize); + return pagesize; +} + /** * This structure describes a virtual memory pool. - * There is one virtual memory pool for each sparse VM area allocation that - * uses big pages. A virtual memory pool tracks the association between - * each mapped big page in the pool and the corresponding physical memory. + * There is one virtual memory pool for each sparse VM area allocation. + * A virtual memory pool tracks the association between each mapped page + * in the pool and the corresponding physical memory. */ struct nvgpu_vm_remap_vpool { /** @@ -160,6 +179,11 @@ struct nvgpu_vm_remap_vpool { */ struct vm_gk20a *vm; + /** + * Pointer to associated VM area. + */ + struct nvgpu_vm_area *vm_area; + /** * Tree of physical memory pools that are currently mapped to this * virtual pool. @@ -243,7 +267,6 @@ int nvgpu_vm_remap(struct vm_gk20a *vm, struct nvgpu_vm_remap_op *ops, * @param num_pages [in] Number of pages in virtual memory pool. * * - Check that #num_pages is non-zero. - * - Check that VM area is using big pages. * - Check that VM area is configured as sparse. * - Allocate memory for internal virtual pool management structures. * - Initialize virtual pool management structures including storing #vm @@ -253,7 +276,6 @@ int nvgpu_vm_remap(struct vm_gk20a *vm, struct nvgpu_vm_remap_op *ops, * @return Zero if the virtual pool create succeeds. * Suitable errors, for failures. * @retval -EINVAL if a value of zero is specified for #num_pages. - * @retval -EINVAL if the VM area is not using big pages. * @retval -EINVAL if the VM area is not configured as sparse. * @retval -ENOMEM if memory allocation for internal resources fails. */ diff --git a/drivers/gpu/nvgpu/os/linux/vm_remap.c b/drivers/gpu/nvgpu/os/linux/vm_remap.c index 6e3ff1f0b..e0808653e 100644 --- a/drivers/gpu/nvgpu/os/linux/vm_remap.c +++ b/drivers/gpu/nvgpu/os/linux/vm_remap.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2017-2022, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -157,23 +157,50 @@ void nvgpu_vm_remap_os_buf_put(struct vm_gk20a *vm, static int nvgpu_vm_remap_validate_map_op(struct nvgpu_as_remap_op *op) { int err = 0; - u32 valid_flags = (NVGPU_AS_REMAP_OP_FLAGS_CACHEABLE | + const u32 pagesize_flags = (NVGPU_AS_REMAP_OP_FLAGS_PAGESIZE_4K | + NVGPU_AS_REMAP_OP_FLAGS_PAGESIZE_64K | + NVGPU_AS_REMAP_OP_FLAGS_PAGESIZE_128K); + const u32 valid_flags = (pagesize_flags | + NVGPU_AS_REMAP_OP_FLAGS_CACHEABLE | NVGPU_AS_REMAP_OP_FLAGS_ACCESS_NO_WRITE); + const u32 pagesize = op->flags & pagesize_flags; if ((op->flags & ~valid_flags) != 0) { err = -EINVAL; } + /* must be set and to a single pagesize */ + if ((pagesize != NVGPU_AS_REMAP_OP_FLAGS_PAGESIZE_4K) && + (pagesize != NVGPU_AS_REMAP_OP_FLAGS_PAGESIZE_64K) && + (pagesize != NVGPU_AS_REMAP_OP_FLAGS_PAGESIZE_128K)) { + err = -EINVAL; + } + return err; } static int nvgpu_vm_remap_validate_unmap_op(struct nvgpu_as_remap_op *op) { int err = 0; + const u32 pagesize_flags = (NVGPU_AS_REMAP_OP_FLAGS_PAGESIZE_4K | + NVGPU_AS_REMAP_OP_FLAGS_PAGESIZE_64K | + NVGPU_AS_REMAP_OP_FLAGS_PAGESIZE_128K); + const u32 valid_flags = pagesize_flags; + const u32 pagesize = op->flags & pagesize_flags; + + if ((op->flags & ~valid_flags) != 0) { + err = -EINVAL; + } + + /* must be set and to a single pagesize */ + if ((pagesize != NVGPU_AS_REMAP_OP_FLAGS_PAGESIZE_4K) && + (pagesize != NVGPU_AS_REMAP_OP_FLAGS_PAGESIZE_64K) && + (pagesize != NVGPU_AS_REMAP_OP_FLAGS_PAGESIZE_128K)) { + err = -EINVAL; + } if ((op->compr_kind != NVGPU_KIND_INVALID) || (op->incompr_kind != NVGPU_KIND_INVALID) || - (op->flags != 0) || (op->mem_offset_in_pages != 0)) { err = -EINVAL; } @@ -191,7 +218,15 @@ static u32 nvgpu_vm_remap_translate_as_flags(u32 flags) if ((flags & NVGPU_AS_REMAP_OP_FLAGS_ACCESS_NO_WRITE) != 0) { core_flags |= NVGPU_VM_REMAP_OP_FLAGS_ACCESS_NO_WRITE; } - + if ((flags & NVGPU_AS_REMAP_OP_FLAGS_PAGESIZE_4K) != 0) { + core_flags |= NVGPU_VM_REMAP_OP_FLAGS_PAGESIZE_4K; + } + if ((flags & NVGPU_AS_REMAP_OP_FLAGS_PAGESIZE_64K) != 0) { + core_flags |= NVGPU_VM_REMAP_OP_FLAGS_PAGESIZE_64K; + } + if ((flags & NVGPU_AS_REMAP_OP_FLAGS_PAGESIZE_128K) != 0) { + core_flags |= NVGPU_VM_REMAP_OP_FLAGS_PAGESIZE_128K; + } return core_flags; } @@ -201,7 +236,6 @@ int nvgpu_vm_remap_translate_as_op(struct vm_gk20a *vm, { int err = 0; u64 page_size; - u64 max_num_pages; if (as_op->mem_handle == 0) { err = nvgpu_vm_remap_validate_unmap_op(as_op); @@ -212,18 +246,17 @@ int nvgpu_vm_remap_translate_as_op(struct vm_gk20a *vm, if (err != 0) goto clean_up; - page_size = vm->gmmu_page_sizes[GMMU_PAGE_SIZE_BIG]; - max_num_pages = (ULONG_MAX / page_size); + vm_op->flags = nvgpu_vm_remap_translate_as_flags(as_op->flags); + page_size = nvgpu_vm_remap_page_size(vm_op); - if ((as_op->num_pages == 0) || - (as_op->num_pages > max_num_pages) || - (as_op->mem_offset_in_pages > max_num_pages) || - (as_op->virt_offset_in_pages > max_num_pages)) { + if ((as_op->num_pages == 0) || (page_size == 0) || + (as_op->num_pages > (vm->va_limit / page_size)) || + (as_op->mem_offset_in_pages > (vm->va_limit / page_size)) || + (as_op->virt_offset_in_pages > (vm->va_limit / page_size))) { err = -EINVAL; goto clean_up; } - vm_op->flags = nvgpu_vm_remap_translate_as_flags(as_op->flags); vm_op->compr_kind = as_op->compr_kind; vm_op->incompr_kind = as_op->incompr_kind; vm_op->mem_handle = as_op->mem_handle; diff --git a/include/uapi/linux/nvgpu-as.h b/include/uapi/linux/nvgpu-as.h index 26f8615bd..0830aa573 100644 --- a/include/uapi/linux/nvgpu-as.h +++ b/include/uapi/linux/nvgpu-as.h @@ -454,7 +454,14 @@ struct nvgpu_as_mapping_modify_args { * must be set if the physical memory buffer represented by @mem_handle * is mapped read-only. * - * This field must be zero for unmap operations. + * %NVGPU_AS_REMAP_OP_FLAGS_PAGESIZE_4K + * %NVGPU_AS_REMAP_OP_FLAGS_PAGESIZE_64K + * %NVGPU_AS_REMAP_OP_FLAGS_PAGESIZE_128K + * + * One, and only one, of these flags must be set for both map/unmap + * ops and indicates the assumed page size of the mem_offset_in_pages + * and virt_offset_in_pages. This value is also verified against the + * page size of the address space. * * @compr_kind [IN/OUT] * @incompr_kind [IN/OUT] @@ -479,16 +486,14 @@ struct nvgpu_as_mapping_modify_args { * * @mem_offset_in_pages [IN] * - * Specify an offset into the physical buffer associated with mem_handle at - * which to start the mapping. This value is in pages and the page size - * is the big page size in the associated sparse address space. This value - * must be zero for unmap operations. + * Specify an offset (in pages) into the physical buffer associated with + * mem_handle at which to start the mapping. This value must be zero for + * unmap operations. * * @virt_offset_in_pages [IN] * - * Specify the virtual memory start offset of the region to map or unmap. - * This value is in pages and the page size is the big page size in the - * associated sparse address space. + * Specify the virtual memory start offset (in pages) of the region to map + * or unmap. * * @num_pages [IN] * Specify the number of pages to map or unmap. @@ -496,14 +501,17 @@ struct nvgpu_as_mapping_modify_args { struct nvgpu_as_remap_op { #define NVGPU_AS_REMAP_OP_FLAGS_CACHEABLE (1 << 2) #define NVGPU_AS_REMAP_OP_FLAGS_ACCESS_NO_WRITE (1 << 10) +#define NVGPU_AS_REMAP_OP_FLAGS_PAGESIZE_4K (1 << 15) +#define NVGPU_AS_REMAP_OP_FLAGS_PAGESIZE_64K (1 << 16) +#define NVGPU_AS_REMAP_OP_FLAGS_PAGESIZE_128K (1 << 17) - /* in: For map operations, this field specifies the mask of - * NVGPU_AS_REMAP flags to use for the mapping. For unmap operations - * this field must be zero */ + /* in: For map and unmap (one and only one) of the _PAGESIZE_ flags is + * required to interpret the mem_offset_in_pages and virt_offset_in_pages + * correctly. The other flags are used only with map operations. */ __u32 flags; /* in: For map operations, this field specifies the desired - * compressible kind. For unmap operations this field must be set + * compressible kind. For unmap operations this field must be set * to NV_KIND_INVALID. * out: For map operations this field returns the actual kind used * for the mapping. This can be useful for detecting if a compressed @@ -550,7 +558,7 @@ struct nvgpu_as_remap_op { * that have been allocated with NVGPU_AS_ALLOC_SPACE_FLAGS_SPARSE. * Validation of remap operations is performed before any changes are made * to the associated sparse address space so either all map and/or unmap - * operations are performed or none of them area. + * operations are performed or none of them are. */ struct nvgpu_as_remap_args { /* in: This field specifies a pointer into the caller's address space