diff --git a/arch/nvgpu-common.yaml b/arch/nvgpu-common.yaml index 2672abcb4..e0e3d8cf5 100644 --- a/arch/nvgpu-common.yaml +++ b/arch/nvgpu-common.yaml @@ -527,6 +527,7 @@ mm: page_table: safe: yes sources: [ common/mm/gmmu/page_table.c, + common/mm/gmmu/pte.c, include/nvgpu/gmmu.h ] deps: [ nvgpu.interface.kmem ] tags: M4, unit-testable diff --git a/drivers/gpu/nvgpu/Makefile b/drivers/gpu/nvgpu/Makefile index 01de0c2f0..31ad13b61 100644 --- a/drivers/gpu/nvgpu/Makefile +++ b/drivers/gpu/nvgpu/Makefile @@ -493,6 +493,7 @@ nvgpu-y += \ common/mm/allocators/lockless_allocator.o \ common/mm/gmmu/page_table.o \ common/mm/gmmu/pd_cache.o \ + common/mm/gmmu/pte.o \ common/mm/as.o \ common/mm/vm.o \ common/mm/vm_area.o \ diff --git a/drivers/gpu/nvgpu/Makefile.sources b/drivers/gpu/nvgpu/Makefile.sources index 75ce9e521..47864c74b 100644 --- a/drivers/gpu/nvgpu/Makefile.sources +++ b/drivers/gpu/nvgpu/Makefile.sources @@ -99,6 +99,7 @@ srcs += common/utils/enabled.c \ common/mm/allocators/buddy_allocator.c \ common/mm/gmmu/page_table.c \ common/mm/gmmu/pd_cache.c \ + common/mm/gmmu/pte.c \ common/mm/as.c \ common/mm/vm.c \ common/mm/vm_area.c \ diff --git a/drivers/gpu/nvgpu/common/mm/gmmu/pte.c b/drivers/gpu/nvgpu/common/mm/gmmu/pte.c new file mode 100644 index 000000000..9af8dc512 --- /dev/null +++ b/drivers/gpu/nvgpu/common/mm/gmmu/pte.c @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include +#include + +u32 nvgpu_gmmu_default_big_page_size(void) +{ + return U32(SZ_64K); +} + +/* + * MSS NVLINK HW settings are in force_snoop mode. + * This will force all the GPU mappings to be coherent. + * By default the mem aperture is set to sysmem_non_coherent and will use L2 + * atomics. + * Change target pte aperture to sysmem_coherent if mem attribute requests for + * platform atomics to use rmw atomic capability. + * + */ +u32 nvgpu_gmmu_aperture_mask(struct gk20a *g, + enum nvgpu_aperture mem_ap, + bool platform_atomic_attr, + u32 sysmem_mask, + u32 sysmem_coh_mask, + u32 vidmem_mask) +{ + if (nvgpu_is_enabled(g, NVGPU_SUPPORT_PLATFORM_ATOMIC) && + platform_atomic_attr) { + mem_ap = APERTURE_SYSMEM_COH; + } + + return nvgpu_aperture_mask_raw(g, mem_ap, + sysmem_mask, + sysmem_coh_mask, + vidmem_mask); +} + +static char *map_attrs_to_str(char *dest, struct nvgpu_gmmu_attrs *attrs) +{ + dest[0] = attrs->cacheable ? 'C' : '-'; + dest[1] = attrs->sparse ? 'S' : '-'; + dest[2] = attrs->priv ? 'P' : '-'; + dest[3] = attrs->valid ? 'V' : '-'; + dest[4] = attrs->platform_atomic ? 'A' : '-'; + dest[5] = '\0'; + + return dest; +} + +void nvgpu_pte_dbg_print(struct gk20a *g, + struct nvgpu_gmmu_attrs *attrs, + const char *vm_name, u32 pd_idx, u32 mmu_level_entry_size, + u64 virt_addr, u64 phys_addr, u32 page_size, u32 *pte_w) +{ + char attrs_str[6]; + char ctag_str[32] = "\0"; + const char *aperture_str = nvgpu_aperture_str(attrs->aperture); + const char *perm_str = nvgpu_gmmu_perm_str(attrs->rw_flag); +#ifdef CONFIG_NVGPU_COMPRESSION + u32 ctag = nvgpu_safe_cast_u64_to_u32(attrs->ctag / + g->ops.fb.compression_page_size(g)); + (void)strcpy(ctag_str, "ctag=0x"); + (void)nvgpu_strnadd_u32(ctag_str, ctag, (u32)strlen(ctag_str), 10U); +#endif + (void)map_attrs_to_str(attrs_str, attrs); + pte_dbg(g, attrs, + "vm=%s " + "PTE: i=%-4u size=%-2u | " + "GPU %#-12llx phys %#-12llx " + "pgsz: %3dkb perm=%-2s kind=%#02x APT=%-6s %-5s " + "%s " + "[0x%08x, 0x%08x]", + vm_name, + pd_idx, mmu_level_entry_size, + virt_addr, phys_addr, + page_size >> 10, + perm_str, + attrs->kind_v, + aperture_str, + attrs_str, + ctag_str, + pte_w[1], pte_w[0]); +} diff --git a/drivers/gpu/nvgpu/hal/init/hal_gm20b.c b/drivers/gpu/nvgpu/hal/init/hal_gm20b.c index 5103e5e80..2061dcd7c 100644 --- a/drivers/gpu/nvgpu/hal/init/hal_gm20b.c +++ b/drivers/gpu/nvgpu/hal/init/hal_gm20b.c @@ -869,7 +869,7 @@ static const struct gpu_ops gm20b_ops = { .unmap = nvgpu_gmmu_unmap_locked, .get_big_page_sizes = gm20b_mm_get_big_page_sizes, .get_default_big_page_size = - gm20b_mm_get_default_big_page_size, + nvgpu_gmmu_default_big_page_size, .get_iommu_bit = gk20a_mm_get_iommu_bit, .gpu_phys_addr = gm20b_gpu_phys_addr, } diff --git a/drivers/gpu/nvgpu/hal/init/hal_gp10b.c b/drivers/gpu/nvgpu/hal/init/hal_gp10b.c index e284391f2..1d0243da6 100644 --- a/drivers/gpu/nvgpu/hal/init/hal_gp10b.c +++ b/drivers/gpu/nvgpu/hal/init/hal_gp10b.c @@ -976,7 +976,7 @@ static const struct gpu_ops gp10b_ops = { .unmap = nvgpu_gmmu_unmap_locked, .get_big_page_sizes = gm20b_mm_get_big_page_sizes, .get_default_big_page_size = - gp10b_mm_get_default_big_page_size, + nvgpu_gmmu_default_big_page_size, .get_iommu_bit = gp10b_mm_get_iommu_bit, .gpu_phys_addr = gm20b_gpu_phys_addr, } diff --git a/drivers/gpu/nvgpu/hal/init/hal_gv11b.c b/drivers/gpu/nvgpu/hal/init/hal_gv11b.c index 5ddfde04e..d813f1d02 100644 --- a/drivers/gpu/nvgpu/hal/init/hal_gv11b.c +++ b/drivers/gpu/nvgpu/hal/init/hal_gv11b.c @@ -1198,7 +1198,7 @@ NVGPU_COV_WHITELIST_BLOCK_END(NVGPU_MISRA(Rule, 8_7)) .unmap = nvgpu_gmmu_unmap_locked, .get_big_page_sizes = gm20b_mm_get_big_page_sizes, .get_default_big_page_size = - gp10b_mm_get_default_big_page_size, + nvgpu_gmmu_default_big_page_size, .get_iommu_bit = gp10b_mm_get_iommu_bit, .gpu_phys_addr = gv11b_gpu_phys_addr, } diff --git a/drivers/gpu/nvgpu/hal/init/hal_tu104.c b/drivers/gpu/nvgpu/hal/init/hal_tu104.c index 2b66defd3..19738d735 100644 --- a/drivers/gpu/nvgpu/hal/init/hal_tu104.c +++ b/drivers/gpu/nvgpu/hal/init/hal_tu104.c @@ -1223,7 +1223,8 @@ static const struct gpu_ops tu104_ops = { .unmap = nvgpu_gmmu_unmap_locked, .get_big_page_sizes = gm20b_mm_get_big_page_sizes, .get_default_big_page_size = - gp10b_mm_get_default_big_page_size, + nvgpu_gmmu_default_big_page_size, + .get_iommu_bit = gp10b_mm_get_iommu_bit, .gpu_phys_addr = gv11b_gpu_phys_addr, } }, diff --git a/drivers/gpu/nvgpu/hal/mm/gmmu/gmmu_gm20b.c b/drivers/gpu/nvgpu/hal/mm/gmmu/gmmu_gm20b.c index b0de407b3..34cdf0469 100644 --- a/drivers/gpu/nvgpu/hal/mm/gmmu/gmmu_gm20b.c +++ b/drivers/gpu/nvgpu/hal/mm/gmmu/gmmu_gm20b.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2019-2020, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -27,11 +27,6 @@ #include "gmmu_gm20b.h" -u32 gm20b_mm_get_default_big_page_size(void) -{ - return U32(SZ_64K); -} - u64 gm20b_gpu_phys_addr(struct gk20a *g, struct nvgpu_gmmu_attrs *attrs, u64 phys) { diff --git a/drivers/gpu/nvgpu/hal/mm/gmmu/gmmu_gm20b.h b/drivers/gpu/nvgpu/hal/mm/gmmu/gmmu_gm20b.h index c8027138b..08c60c135 100644 --- a/drivers/gpu/nvgpu/hal/mm/gmmu/gmmu_gm20b.h +++ b/drivers/gpu/nvgpu/hal/mm/gmmu/gmmu_gm20b.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2019-2020, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -30,7 +30,6 @@ struct nvgpu_gmmu_attrs; u32 gm20b_mm_get_big_page_sizes(void); #ifdef CONFIG_NVGPU_HAL_NON_FUSA -u32 gm20b_mm_get_default_big_page_size(void); u64 gm20b_gpu_phys_addr(struct gk20a *g, struct nvgpu_gmmu_attrs *attrs, u64 phys); #endif diff --git a/drivers/gpu/nvgpu/hal/mm/gmmu/gmmu_gp10b.h b/drivers/gpu/nvgpu/hal/mm/gmmu/gmmu_gp10b.h index adab99bbd..42dae79b9 100644 --- a/drivers/gpu/nvgpu/hal/mm/gmmu/gmmu_gp10b.h +++ b/drivers/gpu/nvgpu/hal/mm/gmmu/gmmu_gp10b.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2019-2020, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -28,7 +28,6 @@ struct gk20a; struct gk20a_mmu_level; -u32 gp10b_mm_get_default_big_page_size(void); u32 gp10b_mm_get_iommu_bit(struct gk20a *g); const struct gk20a_mmu_level *gp10b_mm_get_mmu_levels( struct gk20a *g, u64 big_page_size); diff --git a/drivers/gpu/nvgpu/hal/mm/gmmu/gmmu_gp10b_fusa.c b/drivers/gpu/nvgpu/hal/mm/gmmu/gmmu_gp10b_fusa.c index 21ef474e7..55b443f0d 100644 --- a/drivers/gpu/nvgpu/hal/mm/gmmu/gmmu_gp10b_fusa.c +++ b/drivers/gpu/nvgpu/hal/mm/gmmu/gmmu_gp10b_fusa.c @@ -30,40 +30,16 @@ #include "gmmu_gk20a.h" #include "gmmu_gp10b.h" -u32 gp10b_mm_get_default_big_page_size(void) -{ - return nvgpu_safe_cast_u64_to_u32(SZ_64K); -} +/* + * Compression support is provided for 64GB memory. + * 36 bits (0 to 35) are required for addressing compression memory. + * Use 36th bit to describe l3_alloc or iommu bit. + */ +#define GP10B_MM_IOMMU_BIT 36U u32 gp10b_mm_get_iommu_bit(struct gk20a *g) { - return 36; -} - -/* - * For GV11B and TU104 MSS NVLINK HW settings are in force_snoop mode. - * This will force all the GPU mappings to be coherent. - * By default the mem aperture sets as sysmem_non_coherent and will use L2 mode. - * Change target pte aperture to sysmem_coherent if mem attribute requests for - * platform atomics to use rmw atomic capability. - * - */ -static u32 gmmu_aperture_mask(struct gk20a *g, - enum nvgpu_aperture mem_ap, - bool platform_atomic_attr, - u32 sysmem_mask, - u32 sysmem_coh_mask, - u32 vidmem_mask) -{ - if (nvgpu_is_enabled(g, NVGPU_SUPPORT_PLATFORM_ATOMIC) && - platform_atomic_attr) { - mem_ap = APERTURE_SYSMEM_COH; - } - - return nvgpu_aperture_mask_raw(g, mem_ap, - sysmem_mask, - sysmem_coh_mask, - vidmem_mask); + return GP10B_MM_IOMMU_BIT; } static void update_gmmu_pde3_locked(struct vm_gk20a *vm, @@ -188,7 +164,7 @@ static void update_pte(struct vm_gk20a *vm, u32 pte_addr = (attrs->aperture == APERTURE_SYSMEM) ? gmmu_new_pte_address_sys_f(u64_lo32(phys_shifted)) : gmmu_new_pte_address_vid_f(u64_lo32(phys_shifted)); - u32 pte_tgt = gmmu_aperture_mask(g, + u32 pte_tgt = nvgpu_gmmu_aperture_mask(g, attrs->aperture, attrs->platform_atomic, gmmu_new_pte_aperture_sys_mem_ncoh_f(), @@ -235,52 +211,6 @@ static void update_pte_sparse(u32 *pte_w) pte_w[0] |= gmmu_new_pte_vol_true_f(); } -static char *map_attrs_to_str(char *dest, struct nvgpu_gmmu_attrs *attrs) -{ - dest[0] = attrs->cacheable ? 'C' : '-'; - dest[1] = attrs->sparse ? 'S' : '-'; - dest[2] = attrs->priv ? 'P' : '-'; - dest[3] = attrs->valid ? 'V' : '-'; - dest[4] = attrs->platform_atomic ? 'A' : '-'; - - return dest; -} - -static void pte_dbg_print(struct gk20a *g, - struct nvgpu_gmmu_attrs *attrs, - const char *vm_name, u32 pd_idx, u32 mmu_level_entry_size, - u64 virt_addr, u64 phys_addr, u32 page_size, u32 *pte_w) -{ - char attrs_str[5]; - char ctag_str[32] = "\0"; - const char *aperture_str = nvgpu_aperture_str(attrs->aperture); - const char *perm_str = nvgpu_gmmu_perm_str(attrs->rw_flag); -#ifdef CONFIG_NVGPU_COMPRESSION - u32 ctag = nvgpu_safe_cast_u64_to_u32(attrs->ctag / - g->ops.fb.compression_page_size(g)); - (void)strcpy(ctag_str, "ctag=0x"); - (void)nvgpu_strnadd_u32(ctag_str, ctag, (u32)strlen(ctag_str), 10U); -#endif - (void)map_attrs_to_str(attrs_str, attrs); - pte_dbg(g, attrs, - "vm=%s " - "PTE: i=%-4u size=%-2u | " - "GPU %#-12llx phys %#-12llx " - "pgsz: %3dkb perm=%-2s kind=%#02x APT=%-6s %-5s " - "%s " - "[0x%08x, 0x%08x]", - vm_name, - pd_idx, mmu_level_entry_size, - virt_addr, phys_addr, - page_size >> 10, - perm_str, - attrs->kind_v, - aperture_str, - attrs_str, - ctag_str, - pte_w[1], pte_w[0]); -} - static void update_gmmu_pte_locked(struct vm_gk20a *vm, const struct gk20a_mmu_level *l, struct nvgpu_gmmu_pd *pd, @@ -302,8 +232,8 @@ static void update_gmmu_pte_locked(struct vm_gk20a *vm, } } - pte_dbg_print(g, attrs, vm->name, pd_idx, l->entry_size, virt_addr, - phys_addr, page_size, pte_w); + nvgpu_pte_dbg_print(g, attrs, vm->name, pd_idx, l->entry_size, + virt_addr, phys_addr, page_size, pte_w); nvgpu_pd_write(g, pd, (size_t)nvgpu_safe_add_u32(pd_offset, 0U), pte_w[0]); diff --git a/drivers/gpu/nvgpu/hal/mm/gmmu/gmmu_gv11b_fusa.c b/drivers/gpu/nvgpu/hal/mm/gmmu/gmmu_gv11b_fusa.c index 53875754b..c5832ad9b 100644 --- a/drivers/gpu/nvgpu/hal/mm/gmmu/gmmu_gv11b_fusa.c +++ b/drivers/gpu/nvgpu/hal/mm/gmmu/gmmu_gv11b_fusa.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2019-2020, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -25,8 +25,6 @@ #include "gmmu_gv11b.h" -#define NVGPU_L3_ALLOC_BIT BIT64(36) - /* * On Volta the GPU determines whether to do L3 allocation for a mapping by * checking bit 36 of the phsyical address. So if a mapping should allocte lines @@ -36,7 +34,7 @@ u64 gv11b_gpu_phys_addr(struct gk20a *g, struct nvgpu_gmmu_attrs *attrs, u64 phys) { if ((attrs != NULL) && attrs->l3_alloc) { - return phys | NVGPU_L3_ALLOC_BIT; + return phys | BIT64(g->ops.mm.gmmu.get_iommu_bit(g)); } return phys; diff --git a/drivers/gpu/nvgpu/hal/vgpu/init/vgpu_hal_gp10b.c b/drivers/gpu/nvgpu/hal/vgpu/init/vgpu_hal_gp10b.c index e768c54ea..189bfe774 100644 --- a/drivers/gpu/nvgpu/hal/vgpu/init/vgpu_hal_gp10b.c +++ b/drivers/gpu/nvgpu/hal/vgpu/init/vgpu_hal_gp10b.c @@ -664,7 +664,7 @@ static const struct gpu_ops vgpu_gp10b_ops = { .unmap = vgpu_locked_gmmu_unmap, .get_big_page_sizes = gm20b_mm_get_big_page_sizes, .get_default_big_page_size = - gp10b_mm_get_default_big_page_size, + nvgpu_gmmu_default_big_page_size, .gpu_phys_addr = gm20b_gpu_phys_addr, .get_iommu_bit = gk20a_mm_get_iommu_bit, .get_mmu_levels = gp10b_mm_get_mmu_levels, diff --git a/drivers/gpu/nvgpu/hal/vgpu/init/vgpu_hal_gv11b.c b/drivers/gpu/nvgpu/hal/vgpu/init/vgpu_hal_gv11b.c index 76045135f..149c33dd0 100644 --- a/drivers/gpu/nvgpu/hal/vgpu/init/vgpu_hal_gv11b.c +++ b/drivers/gpu/nvgpu/hal/vgpu/init/vgpu_hal_gv11b.c @@ -794,7 +794,7 @@ static const struct gpu_ops vgpu_gv11b_ops = { .unmap = vgpu_locked_gmmu_unmap, .get_big_page_sizes = gm20b_mm_get_big_page_sizes, .get_default_big_page_size = - gp10b_mm_get_default_big_page_size, + nvgpu_gmmu_default_big_page_size, .gpu_phys_addr = gm20b_gpu_phys_addr, .get_iommu_bit = gk20a_mm_get_iommu_bit, .get_mmu_levels = gp10b_mm_get_mmu_levels, diff --git a/drivers/gpu/nvgpu/include/nvgpu/gmmu.h b/drivers/gpu/nvgpu/include/nvgpu/gmmu.h index 4c888a234..ce2eb9cf0 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/gmmu.h +++ b/drivers/gpu/nvgpu/include/nvgpu/gmmu.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -41,6 +41,7 @@ * GMMU. */ +struct gk20a; struct vm_gk20a; struct nvgpu_mem; struct nvgpu_gmmu_pd; @@ -537,4 +538,26 @@ void nvgpu_gmmu_unmap_locked(struct vm_gk20a *vm, NVGPU_COV_WHITELIST(false_positive, NVGPU_MISRA(Rule, 14_4), "Bug 2623654") \ } while (false) +/** + * @brief Function to get the default big page size in bytes. + * + * Default big page size: + * - Big page size is same for all GPU families. + * + * @return Default big page size + */ +u32 nvgpu_gmmu_default_big_page_size(void); + +u32 nvgpu_gmmu_aperture_mask(struct gk20a *g, + enum nvgpu_aperture mem_ap, + bool platform_atomic_attr, + u32 sysmem_mask, + u32 sysmem_coh_mask, + u32 vidmem_mask); + +void nvgpu_pte_dbg_print(struct gk20a *g, + struct nvgpu_gmmu_attrs *attrs, + const char *vm_name, u32 pd_idx, u32 mmu_level_entry_size, + u64 virt_addr, u64 phys_addr, u32 page_size, u32 *pte_w); + #endif /* NVGPU_GMMU_H */ diff --git a/libs/dgpu/libnvgpu-drv-dgpu_safe.export b/libs/dgpu/libnvgpu-drv-dgpu_safe.export index 1bd51bed1..55d5bbf26 100644 --- a/libs/dgpu/libnvgpu-drv-dgpu_safe.export +++ b/libs/dgpu/libnvgpu-drv-dgpu_safe.export @@ -87,7 +87,6 @@ gm20b_bus_bar1_bind gp10b_bus_bar2_bind gp10b_ce_nonstall_isr gp10b_get_max_page_table_levels -gp10b_mm_get_default_big_page_size gp10b_mm_get_iommu_bit gp10b_mm_get_mmu_levels gp10b_mm_init_bar2_vm @@ -413,6 +412,7 @@ nvgpu_free_gr_ctx_struct nvgpu_get nvgpu_get_gpfifo_entry_size nvgpu_get_pte +nvgpu_gmmu_default_big_page_size nvgpu_gmmu_init_page_table nvgpu_gmmu_map nvgpu_gmmu_map_locked diff --git a/libs/igpu/libnvgpu-drv-igpu_safe.export b/libs/igpu/libnvgpu-drv-igpu_safe.export index 65c3e32b8..6542d134b 100644 --- a/libs/igpu/libnvgpu-drv-igpu_safe.export +++ b/libs/igpu/libnvgpu-drv-igpu_safe.export @@ -87,7 +87,6 @@ gm20b_bus_bar1_bind gp10b_bus_bar2_bind gp10b_ce_nonstall_isr gp10b_get_max_page_table_levels -gp10b_mm_get_default_big_page_size gp10b_mm_get_iommu_bit gp10b_mm_get_mmu_levels gp10b_mm_init_bar2_vm @@ -421,6 +420,7 @@ nvgpu_free_gr_ctx_struct nvgpu_get nvgpu_get_gpfifo_entry_size nvgpu_get_pte +nvgpu_gmmu_default_big_page_size nvgpu_gmmu_init_page_table nvgpu_gmmu_map nvgpu_gmmu_map_locked diff --git a/userspace/units/mm/allocators/buddy_allocator/buddy_allocator.c b/userspace/units/mm/allocators/buddy_allocator/buddy_allocator.c index 8c729c155..4977b13f2 100644 --- a/userspace/units/mm/allocators/buddy_allocator/buddy_allocator.c +++ b/userspace/units/mm/allocators/buddy_allocator/buddy_allocator.c @@ -73,7 +73,7 @@ static struct vm_gk20a *init_vm_env(struct unit_module *m, struct gk20a *g, /* Minimum HALs for vm_init */ g->ops.mm.gmmu.get_default_big_page_size = - gp10b_mm_get_default_big_page_size; + nvgpu_gmmu_default_big_page_size; g->ops.mm.gmmu.get_mmu_levels = gp10b_mm_get_mmu_levels; g->ops.mm.gmmu.get_max_page_table_levels = gp10b_get_max_page_table_levels; diff --git a/userspace/units/mm/as/as.c b/userspace/units/mm/as/as.c index f1112a457..48bec4940 100644 --- a/userspace/units/mm/as/as.c +++ b/userspace/units/mm/as/as.c @@ -133,12 +133,13 @@ int test_init_mm(struct unit_module *m, struct gk20a *g, void *args) p->mm_is_iommuable = true; g->ops.mm.gmmu.get_default_big_page_size = - gp10b_mm_get_default_big_page_size; + nvgpu_gmmu_default_big_page_size; g->ops.mm.gmmu.get_mmu_levels = gp10b_mm_get_mmu_levels; g->ops.mm.gmmu.get_max_page_table_levels = gp10b_get_max_page_table_levels; g->ops.mm.init_inst_block = gv11b_mm_init_inst_block; g->ops.mm.gmmu.map = nvgpu_gmmu_map_locked; g->ops.mm.gmmu.unmap = nvgpu_gmmu_unmap_locked; + g->ops.mm.gmmu.get_iommu_bit = gp10b_mm_get_iommu_bit; g->ops.mm.gmmu.gpu_phys_addr = gv11b_gpu_phys_addr; g->ops.mm.is_bar1_supported = gv11b_mm_is_bar1_supported; g->ops.mm.cache.l2_flush = gv11b_mm_l2_flush; diff --git a/userspace/units/mm/dma/dma.c b/userspace/units/mm/dma/dma.c index 4b659caa7..6ebb1b02e 100644 --- a/userspace/units/mm/dma/dma.c +++ b/userspace/units/mm/dma/dma.c @@ -158,12 +158,13 @@ static int init_mm(struct unit_module *m, struct gk20a *g) } g->ops.mm.gmmu.get_default_big_page_size = - gp10b_mm_get_default_big_page_size; + nvgpu_gmmu_default_big_page_size; g->ops.mm.gmmu.get_mmu_levels = gp10b_mm_get_mmu_levels; g->ops.mm.gmmu.get_max_page_table_levels = gp10b_get_max_page_table_levels; g->ops.mm.init_inst_block = gv11b_mm_init_inst_block; g->ops.mm.gmmu.map = nvgpu_gmmu_map_locked; g->ops.mm.gmmu.unmap = nvgpu_gmmu_unmap_locked; + g->ops.mm.gmmu.get_iommu_bit = gp10b_mm_get_iommu_bit; g->ops.mm.gmmu.gpu_phys_addr = gv11b_gpu_phys_addr; g->ops.mm.is_bar1_supported = gv11b_mm_is_bar1_supported; g->ops.mm.cache.l2_flush = gv11b_mm_l2_flush; diff --git a/userspace/units/mm/gmmu/page_table/page_table.c b/userspace/units/mm/gmmu/page_table/page_table.c index 84f8e21e4..cea6f25bf 100644 --- a/userspace/units/mm/gmmu/page_table/page_table.c +++ b/userspace/units/mm/gmmu/page_table/page_table.c @@ -317,12 +317,13 @@ static int init_mm(struct unit_module *m, struct gk20a *g) p->mm_is_iommuable = true; g->ops.mm.gmmu.get_default_big_page_size = - gp10b_mm_get_default_big_page_size; + nvgpu_gmmu_default_big_page_size; g->ops.mm.gmmu.get_mmu_levels = gp10b_mm_get_mmu_levels; g->ops.mm.gmmu.get_max_page_table_levels = gp10b_get_max_page_table_levels; g->ops.mm.init_inst_block = gv11b_mm_init_inst_block; g->ops.mm.gmmu.map = nvgpu_gmmu_map_locked; g->ops.mm.gmmu.unmap = nvgpu_gmmu_unmap_locked; + g->ops.mm.gmmu.get_iommu_bit = gp10b_mm_get_iommu_bit; g->ops.mm.gmmu.gpu_phys_addr = gv11b_gpu_phys_addr; g->ops.mm.is_bar1_supported = gv11b_mm_is_bar1_supported; g->ops.mm.cache.l2_flush = gv11b_mm_l2_flush; diff --git a/userspace/units/mm/hal/cache/flush_gk20a_fusa/flush-gk20a-fusa.c b/userspace/units/mm/hal/cache/flush_gk20a_fusa/flush-gk20a-fusa.c index 2b05821b4..ce4c196ab 100644 --- a/userspace/units/mm/hal/cache/flush_gk20a_fusa/flush-gk20a-fusa.c +++ b/userspace/units/mm/hal/cache/flush_gk20a_fusa/flush-gk20a-fusa.c @@ -128,7 +128,7 @@ static int init_mm(struct unit_module *m, struct gk20a *g) /* Minimum HALs for page_table */ g->ops.mm.gmmu.get_default_big_page_size = - gp10b_mm_get_default_big_page_size; + nvgpu_gmmu_default_big_page_size; g->ops.mm.init_inst_block = gv11b_mm_init_inst_block; g->ops.mm.gmmu.get_mmu_levels = gp10b_mm_get_mmu_levels; g->ops.ramin.init_pdb = gp10b_ramin_init_pdb; diff --git a/userspace/units/mm/hal/cache/flush_gv11b_fusa/flush-gv11b-fusa.c b/userspace/units/mm/hal/cache/flush_gv11b_fusa/flush-gv11b-fusa.c index aed29b4dc..677592ccc 100644 --- a/userspace/units/mm/hal/cache/flush_gv11b_fusa/flush-gv11b-fusa.c +++ b/userspace/units/mm/hal/cache/flush_gv11b_fusa/flush-gv11b-fusa.c @@ -117,7 +117,7 @@ static int init_mm(struct unit_module *m, struct gk20a *g) /* Minimum HALs for page_table */ g->ops.mm.gmmu.get_default_big_page_size = - gp10b_mm_get_default_big_page_size; + nvgpu_gmmu_default_big_page_size; g->ops.mm.init_inst_block = gv11b_mm_init_inst_block; g->ops.mm.gmmu.get_mmu_levels = gp10b_mm_get_mmu_levels; g->ops.ramin.init_pdb = gp10b_ramin_init_pdb; diff --git a/userspace/units/mm/hal/gmmu/gmmu_gp10b_fusa/gmmu-gp10b-fusa.c b/userspace/units/mm/hal/gmmu/gmmu_gp10b_fusa/gmmu-gp10b-fusa.c index f79332dff..285aec7c5 100644 --- a/userspace/units/mm/hal/gmmu/gmmu_gp10b_fusa/gmmu-gp10b-fusa.c +++ b/userspace/units/mm/hal/gmmu/gmmu_gp10b_fusa/gmmu-gp10b-fusa.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2018-2020, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -46,7 +46,7 @@ int test_gp10b_mm_get_default_big_page_size(struct unit_module *m, u32 ret_pgsz; int ret = UNIT_FAIL; - ret_pgsz = gp10b_mm_get_default_big_page_size(); + ret_pgsz = nvgpu_gmmu_default_big_page_size(); unit_assert(ret_pgsz == U32(SZ_64K), goto done); ret = UNIT_SUCCESS; diff --git a/userspace/units/mm/hal/gmmu/gmmu_gp10b_fusa/gmmu-gp10b-fusa.h b/userspace/units/mm/hal/gmmu/gmmu_gp10b_fusa/gmmu-gp10b-fusa.h index ee32e12fd..0415b7bba 100644 --- a/userspace/units/mm/hal/gmmu/gmmu_gp10b_fusa/gmmu-gp10b-fusa.h +++ b/userspace/units/mm/hal/gmmu/gmmu_gp10b_fusa/gmmu-gp10b-fusa.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2019-2020, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -40,7 +40,7 @@ struct unit_module; * Test Type: Feature * * Targets: gops_mm.gops_mm_gmmu.get_default_big_page_size, - * gp10b_mm_get_default_big_page_size + * nvgpu_gmmu_default_big_page_size * * Input: None * diff --git a/userspace/units/mm/hal/gmmu/gmmu_gv11b_fusa/gmmu-gv11b-fusa.c b/userspace/units/mm/hal/gmmu/gmmu_gv11b_fusa/gmmu-gv11b-fusa.c index 96c5a9f9b..903f616d2 100644 --- a/userspace/units/mm/hal/gmmu/gmmu_gv11b_fusa/gmmu-gv11b-fusa.c +++ b/userspace/units/mm/hal/gmmu/gmmu_gv11b_fusa/gmmu-gv11b-fusa.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2018-2020, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -29,6 +29,7 @@ #include #include +#include "hal/mm/gmmu/gmmu_gp10b.h" #include "hal/mm/gmmu/gmmu_gv11b.h" #include "gmmu-gv11b-fusa.h" @@ -37,11 +38,9 @@ #define F_GV11B_GPU_PHYS_ADDR_L3_ALLOC_FALSE 1 #define F_GV11B_GPU_PHYS_ADDR_L3_ALLOC_TRUE 2 -/* defined in gmmu_gv11b_fusa.c */ -#define NVGPU_L3_ALLOC_BIT BIT64(36) - int test_gv11b_gpu_phys_addr(struct unit_module *m, struct gk20a *g, void *args) { + struct gpu_ops gops = g->ops; struct nvgpu_gmmu_attrs attrs = {0}; struct nvgpu_gmmu_attrs *attrs_ptr; u64 phys = BIT(10); @@ -49,6 +48,8 @@ int test_gv11b_gpu_phys_addr(struct unit_module *m, struct gk20a *g, void *args) u64 branch = (u64)args; int ret = UNIT_FAIL; + g->ops.mm.gmmu.get_iommu_bit = gp10b_mm_get_iommu_bit; + attrs_ptr = branch == F_GV11B_GPU_PHYS_ADDR_GMMU_ATTRS_NULL ? NULL : &attrs; @@ -58,7 +59,9 @@ int test_gv11b_gpu_phys_addr(struct unit_module *m, struct gk20a *g, void *args) ret_phys = gv11b_gpu_phys_addr(g, attrs_ptr, phys); if (branch == F_GV11B_GPU_PHYS_ADDR_L3_ALLOC_TRUE) { - unit_assert(ret_phys == (phys | NVGPU_L3_ALLOC_BIT), goto done); + unit_assert(ret_phys == (phys | + BIT64(g->ops.mm.gmmu.get_iommu_bit(g))), + goto done); } else { unit_assert(ret_phys == phys, goto done); } @@ -71,6 +74,7 @@ done: __func__); } + g->ops = gops; return ret; } diff --git a/userspace/units/mm/hal/gp10b_fusa/mm-gp10b-fusa.c b/userspace/units/mm/hal/gp10b_fusa/mm-gp10b-fusa.c index a81ec86bf..9c0406e7f 100644 --- a/userspace/units/mm/hal/gp10b_fusa/mm-gp10b-fusa.c +++ b/userspace/units/mm/hal/gp10b_fusa/mm-gp10b-fusa.c @@ -105,7 +105,7 @@ static int init_mm(struct unit_module *m, struct gk20a *g) /* Minimum HALs for page_table */ g->ops.mm.gmmu.get_default_big_page_size = - gp10b_mm_get_default_big_page_size; + nvgpu_gmmu_default_big_page_size; g->ops.mm.init_inst_block = gv11b_mm_init_inst_block; g->ops.mm.gmmu.get_mmu_levels = gp10b_mm_get_mmu_levels; g->ops.ramin.init_pdb = gp10b_ramin_init_pdb; diff --git a/userspace/units/mm/hal/gv11b_fusa/mm-gv11b-fusa.c b/userspace/units/mm/hal/gv11b_fusa/mm-gv11b-fusa.c index 64dba4e68..aea3081ae 100644 --- a/userspace/units/mm/hal/gv11b_fusa/mm-gv11b-fusa.c +++ b/userspace/units/mm/hal/gv11b_fusa/mm-gv11b-fusa.c @@ -101,7 +101,7 @@ static int init_mm(struct unit_module *m, struct gk20a *g) /* Minimum HALs for page_table */ g->ops.mm.gmmu.get_default_big_page_size = - gp10b_mm_get_default_big_page_size; + nvgpu_gmmu_default_big_page_size; g->ops.mm.gmmu.get_mmu_levels = gp10b_mm_get_mmu_levels; g->ops.ramin.init_pdb = gp10b_ramin_init_pdb; g->ops.ramin.alloc_size = gk20a_ramin_alloc_size; diff --git a/userspace/units/mm/hal/mmu_fault/gv11b_fusa/mmu-fault-gv11b-fusa.c b/userspace/units/mm/hal/mmu_fault/gv11b_fusa/mmu-fault-gv11b-fusa.c index 6922133e7..6a7d74eba 100644 --- a/userspace/units/mm/hal/mmu_fault/gv11b_fusa/mmu-fault-gv11b-fusa.c +++ b/userspace/units/mm/hal/mmu_fault/gv11b_fusa/mmu-fault-gv11b-fusa.c @@ -131,7 +131,7 @@ static int init_mm(struct unit_module *m, struct gk20a *g) /* Minimum HALs for page_table */ g->ops.mm.gmmu.get_default_big_page_size = - gp10b_mm_get_default_big_page_size; + nvgpu_gmmu_default_big_page_size; g->ops.mm.init_inst_block = gv11b_mm_init_inst_block; g->ops.mm.gmmu.get_mmu_levels = gp10b_mm_get_mmu_levels; g->ops.ramin.init_pdb = gp10b_ramin_init_pdb; @@ -143,6 +143,7 @@ static int init_mm(struct unit_module *m, struct gk20a *g) g->ops.channel.count = stub_channel_count; g->ops.mm.gmmu.map = nvgpu_gmmu_map_locked; g->ops.mm.gmmu.unmap = nvgpu_gmmu_unmap_locked; + g->ops.mm.gmmu.get_iommu_bit = gp10b_mm_get_iommu_bit; g->ops.mm.gmmu.gpu_phys_addr = gv11b_gpu_phys_addr; g->ops.mm.cache.l2_flush = stub_mm_l2_flush; g->ops.fb.tlb_invalidate = gm20b_fb_tlb_invalidate; diff --git a/userspace/units/mm/mm/mm.c b/userspace/units/mm/mm/mm.c index a5b09adf7..e8c6e7cc8 100644 --- a/userspace/units/mm/mm/mm.c +++ b/userspace/units/mm/mm/mm.c @@ -458,12 +458,13 @@ int test_mm_init_hal(struct unit_module *m, struct gk20a *g, void *args) mc_gp10b_intr_nonstall_unit_config; g->ops.mm.gmmu.get_default_big_page_size = - gp10b_mm_get_default_big_page_size; + nvgpu_gmmu_default_big_page_size; g->ops.mm.gmmu.get_mmu_levels = gp10b_mm_get_mmu_levels; g->ops.mm.gmmu.get_max_page_table_levels = gp10b_get_max_page_table_levels; g->ops.mm.init_inst_block = gv11b_mm_init_inst_block; g->ops.mm.gmmu.map = nvgpu_gmmu_map_locked; g->ops.mm.gmmu.unmap = nvgpu_gmmu_unmap_locked; + g->ops.mm.gmmu.get_iommu_bit = gp10b_mm_get_iommu_bit; g->ops.mm.gmmu.gpu_phys_addr = gv11b_gpu_phys_addr; g->ops.mm.is_bar1_supported = gv11b_mm_is_bar1_supported; g->ops.mm.cache.l2_flush = gv11b_mm_l2_flush; diff --git a/userspace/units/mm/page_table_faults/page_table_faults.c b/userspace/units/mm/page_table_faults/page_table_faults.c index 35edc43f6..2e5e0d223 100644 --- a/userspace/units/mm/page_table_faults/page_table_faults.c +++ b/userspace/units/mm/page_table_faults/page_table_faults.c @@ -128,12 +128,13 @@ static int init_mm(struct unit_module *m, struct gk20a *g) /* Minimum HALs for page_table */ g->ops.mm.gmmu.get_default_big_page_size = - gp10b_mm_get_default_big_page_size; + nvgpu_gmmu_default_big_page_size; g->ops.mm.gmmu.get_mmu_levels = gp10b_mm_get_mmu_levels; g->ops.mm.gmmu.get_max_page_table_levels = gp10b_get_max_page_table_levels; g->ops.mm.init_inst_block = gv11b_mm_init_inst_block; g->ops.mm.gmmu.map = nvgpu_gmmu_map_locked; g->ops.mm.gmmu.unmap = nvgpu_gmmu_unmap_locked; + g->ops.mm.gmmu.get_iommu_bit = gp10b_mm_get_iommu_bit; g->ops.mm.gmmu.gpu_phys_addr = gv11b_gpu_phys_addr; #ifdef CONFIG_NVGPU_COMPRESSION g->ops.fb.compression_page_size = gp10b_fb_compression_page_size; diff --git a/userspace/units/mm/page_table_faults/page_table_faults.h b/userspace/units/mm/page_table_faults/page_table_faults.h index d52801f72..51cf932c5 100644 --- a/userspace/units/mm/page_table_faults/page_table_faults.h +++ b/userspace/units/mm/page_table_faults/page_table_faults.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2019-2020, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -121,7 +121,7 @@ int test_page_faults_disable_hw(struct unit_module *m, struct gk20a *g, * Test Type: Feature * * Targets: gops_mm.gops_mm_gmmu.get_default_big_page_size, - * gp10b_mm_get_default_big_page_size, nvgpu_alloc_inst_block, + * nvgpu_gmmu_default_big_page_size, nvgpu_alloc_inst_block, * gops_mm.init_inst_block, gv11b_mm_init_inst_block * * Input: test_page_faults_init diff --git a/userspace/units/mm/vm/vm.c b/userspace/units/mm/vm/vm.c index f6c804956..b2997791a 100644 --- a/userspace/units/mm/vm/vm.c +++ b/userspace/units/mm/vm/vm.c @@ -183,11 +183,12 @@ static int init_test_env(struct unit_module *m, struct gk20a *g) g->ops.fb.tlb_invalidate = gm20b_fb_tlb_invalidate; g->ops.mm.gmmu.get_default_big_page_size = - gp10b_mm_get_default_big_page_size; + nvgpu_gmmu_default_big_page_size; g->ops.mm.gmmu.get_mmu_levels = gp10b_mm_get_mmu_levels; g->ops.mm.gmmu.get_max_page_table_levels = gp10b_get_max_page_table_levels; g->ops.mm.gmmu.map = nvgpu_gmmu_map_locked; g->ops.mm.gmmu.unmap = nvgpu_gmmu_unmap_locked; + g->ops.mm.gmmu.get_iommu_bit = gp10b_mm_get_iommu_bit; g->ops.mm.gmmu.gpu_phys_addr = gv11b_gpu_phys_addr; g->ops.mm.cache.l2_flush = gv11b_mm_l2_flush; g->ops.mm.cache.fb_flush = gk20a_mm_fb_flush;