nv-tegra-mirror/linux-nvgpu
1
0
Fork 0
mirror of git://nv-tegra.nvidia.com/linux-nvgpu.git synced 2025-12-22 17:36:20 +03:00
Code Issues Packages Projects Releases Wiki Activity

gpu: nvgpu: common: mm: Fix MISRA 15.6 violations

Browse Source 
MISRA Rule-15.6 requires that all if-else blocks be enclosed in braces,
including single statement blocks. Fix errors due to single statement
if blocks without braces, introducing the braces.

JIRA NVGPU-671

Change-Id: Ieeecf719dca9acc1a116d2893637bf770caf4f5b
Signed-off-by: Srirangan <smadhavan@nvidia.com>
Reviewed-on: https://git-master.nvidia.com/r/1794241
GVS: Gerrit_Virtual_Submit
Reviewed-by: Adeel Raza <araza@nvidia.com>
Reviewed-by: Alex Waterman <alexw@nvidia.com>
Reviewed-by: mobile promotions <svcmobile_promotions@nvidia.com>
Tested-by: mobile promotions <svcmobile_promotions@nvidia.com>
This commit is contained in:
Srirangan
2018-08-14 10:57:15 +05:30
committed by mobile promotions
parent 553fdf3534
commit 70c20bb75b
6 changed files with 348 additions and 183 deletions
Download Patch File Download Diff File Expand all files Collapse all files

166
drivers/gpu/nvgpu/common/mm/buddy_allocator.c
View File

@@ -79,10 +79,12 @@ static void balloc_compute_max_order(struct nvgpu_buddy_allocator *a)
return; return;
} }
if (a->max_order > true_max_order) if (a->max_order > true_max_order) {
a->max_order = true_max_order; a->max_order = true_max_order;
if (a->max_order > GPU_BALLOC_MAX_ORDER) }
if (a->max_order > GPU_BALLOC_MAX_ORDER) {
a->max_order = GPU_BALLOC_MAX_ORDER; a->max_order = GPU_BALLOC_MAX_ORDER;
}
} }
/* /*
@@ -108,8 +110,9 @@ static struct nvgpu_buddy *balloc_new_buddy(struct nvgpu_buddy_allocator *a,
struct nvgpu_buddy *new_buddy; struct nvgpu_buddy *new_buddy;
new_buddy = nvgpu_kmem_cache_alloc(a->buddy_cache); new_buddy = nvgpu_kmem_cache_alloc(a->buddy_cache);
if (!new_buddy) if (!new_buddy) {
return NULL; return NULL;
}
memset(new_buddy, 0, sizeof(struct nvgpu_buddy)); memset(new_buddy, 0, sizeof(struct nvgpu_buddy));
@@ -139,10 +142,11 @@ static void __balloc_buddy_list_add(struct nvgpu_buddy_allocator *a,
* without cycling through the entire list. * without cycling through the entire list.
*/ */
if (a->flags & GPU_ALLOC_GVA_SPACE && if (a->flags & GPU_ALLOC_GVA_SPACE &&
b->pte_size == gmmu_page_size_big) b->pte_size == gmmu_page_size_big) {
nvgpu_list_add_tail(&b->buddy_entry, list); nvgpu_list_add_tail(&b->buddy_entry, list);
else } else {
nvgpu_list_add(&b->buddy_entry, list); nvgpu_list_add(&b->buddy_entry, list);
}
buddy_set_in_list(b); buddy_set_in_list(b);
} }
@@ -181,8 +185,9 @@ static void balloc_blist_rem(struct nvgpu_buddy_allocator *a,
static u64 balloc_get_order(struct nvgpu_buddy_allocator *a, u64 len) static u64 balloc_get_order(struct nvgpu_buddy_allocator *a, u64 len)
{ {
if (len == 0) if (len == 0) {
return 0; return 0;
}
len--; len--;
len >>= a->blk_shift; len >>= a->blk_shift;
@@ -195,10 +200,11 @@ static u64 __balloc_max_order_in(struct nvgpu_buddy_allocator *a,
{ {
u64 size = (end - start) >> a->blk_shift; u64 size = (end - start) >> a->blk_shift;
if (size > 0) if (size > 0) {
return min_t(u64, ilog2(size), a->max_order); return min_t(u64, ilog2(size), a->max_order);
else } else {
return GPU_BALLOC_MAX_ORDER; return GPU_BALLOC_MAX_ORDER;
}
} }
/* /*
@@ -222,8 +228,9 @@ static int balloc_init_lists(struct nvgpu_buddy_allocator *a)
order = __balloc_max_order_in(a, bstart, bend); order = __balloc_max_order_in(a, bstart, bend);
buddy = balloc_new_buddy(a, NULL, bstart, order); buddy = balloc_new_buddy(a, NULL, bstart, order);
if (!buddy) if (!buddy) {
goto cleanup; goto cleanup;
}
balloc_blist_add(a, buddy); balloc_blist_add(a, buddy);
bstart += balloc_order_to_len(a, order); bstart += balloc_order_to_len(a, order);
@@ -340,17 +347,20 @@ static void balloc_coalesce(struct nvgpu_buddy_allocator *a,
{ {
struct nvgpu_buddy *parent; struct nvgpu_buddy *parent;
if (buddy_is_alloced(b) || buddy_is_split(b)) if (buddy_is_alloced(b) || buddy_is_split(b)) {
return; return;
}
/* /*
* If both our buddy and I are both not allocated and not split then * If both our buddy and I are both not allocated and not split then
* we can coalesce ourselves. * we can coalesce ourselves.
*/ */
if (!b->buddy) if (!b->buddy) {
return; return;
if (buddy_is_alloced(b->buddy) || buddy_is_split(b->buddy)) }
if (buddy_is_alloced(b->buddy) || buddy_is_split(b->buddy)) {
return; return;
}
parent = b->parent; parent = b->parent;
@@ -383,8 +393,9 @@ static int balloc_split_buddy(struct nvgpu_buddy_allocator *a,
u64 half; u64 half;
left = balloc_new_buddy(a, b, b->start, b->order - 1); left = balloc_new_buddy(a, b, b->start, b->order - 1);
if (!left) if (!left) {
return -ENOMEM; return -ENOMEM;
}
half = (b->end - b->start) / 2; half = (b->end - b->start) / 2;
@@ -449,8 +460,9 @@ static struct nvgpu_buddy *balloc_free_buddy(struct nvgpu_buddy_allocator *a,
struct nvgpu_buddy *bud; struct nvgpu_buddy *bud;
nvgpu_rbtree_search(addr, &node, a->alloced_buddies); nvgpu_rbtree_search(addr, &node, a->alloced_buddies);
if (!node) if (!node) {
return NULL; return NULL;
}
bud = nvgpu_buddy_from_rbtree_node(node); bud = nvgpu_buddy_from_rbtree_node(node);
@@ -470,21 +482,24 @@ static struct nvgpu_buddy *__balloc_find_buddy(struct nvgpu_buddy_allocator *a,
struct nvgpu_buddy *bud; struct nvgpu_buddy *bud;
if (order > a->max_order || if (order > a->max_order ||
nvgpu_list_empty(balloc_get_order_list(a, order))) nvgpu_list_empty(balloc_get_order_list(a, order))) {
return NULL; return NULL;
}
if (a->flags & GPU_ALLOC_GVA_SPACE && if (a->flags & GPU_ALLOC_GVA_SPACE &&
pte_size == gmmu_page_size_big) pte_size == gmmu_page_size_big) {
bud = nvgpu_list_last_entry(balloc_get_order_list(a, order), bud = nvgpu_list_last_entry(balloc_get_order_list(a, order),
nvgpu_buddy, buddy_entry); nvgpu_buddy, buddy_entry);
else } else {
bud = nvgpu_list_first_entry(balloc_get_order_list(a, order), bud = nvgpu_list_first_entry(balloc_get_order_list(a, order),
nvgpu_buddy, buddy_entry); nvgpu_buddy, buddy_entry);
}
if (pte_size != BALLOC_PTE_SIZE_ANY && if (pte_size != BALLOC_PTE_SIZE_ANY &&
pte_size != bud->pte_size && pte_size != bud->pte_size &&
bud->pte_size != BALLOC_PTE_SIZE_ANY) bud->pte_size != BALLOC_PTE_SIZE_ANY) {
return NULL; return NULL;
}
return bud; return bud;
} }
@@ -511,12 +526,14 @@ static u64 __balloc_do_alloc(struct nvgpu_buddy_allocator *a,
} }
/* Out of memory! */ /* Out of memory! */
if (!bud) if (!bud) {
return 0; return 0;
}
while (bud->order != order) { while (bud->order != order) {
if (balloc_split_buddy(a, bud, pte_size)) if (balloc_split_buddy(a, bud, pte_size)) {
return 0; /* No mem... */ return 0; /* No mem... */
}
bud = bud->left; bud = bud->left;
} }
@@ -540,19 +557,22 @@ static int balloc_is_range_free(struct nvgpu_buddy_allocator *a,
struct nvgpu_buddy *bud; struct nvgpu_buddy *bud;
nvgpu_rbtree_enum_start(0, &node, a->alloced_buddies); nvgpu_rbtree_enum_start(0, &node, a->alloced_buddies);
if (!node) if (!node) {
return 1; /* No allocs yet. */ return 1; /* No allocs yet. */
}
bud = nvgpu_buddy_from_rbtree_node(node); bud = nvgpu_buddy_from_rbtree_node(node);
while (bud->start < end) { while (bud->start < end) {
if ((bud->start > base && bud->start < end) || if ((bud->start > base && bud->start < end) ||
(bud->end > base && bud->end < end)) (bud->end > base && bud->end < end)) {
return 0; return 0;
}
nvgpu_rbtree_enum_next(&node, node); nvgpu_rbtree_enum_next(&node, node);
if (!node) if (!node) {
break; break;
}
bud = nvgpu_buddy_from_rbtree_node(node); bud = nvgpu_buddy_from_rbtree_node(node);
} }
@@ -581,8 +601,9 @@ static struct nvgpu_fixed_alloc *balloc_free_fixed(
struct nvgpu_rbtree_node *node = NULL; struct nvgpu_rbtree_node *node = NULL;
nvgpu_rbtree_search(addr, &node, a->fixed_allocs); nvgpu_rbtree_search(addr, &node, a->fixed_allocs);
if (!node) if (!node) {
return NULL; return NULL;
}
falloc = nvgpu_fixed_alloc_from_rbtree_node(node); falloc = nvgpu_fixed_alloc_from_rbtree_node(node);
@@ -657,8 +678,9 @@ static struct nvgpu_buddy *__balloc_make_fixed_buddy(
} }
} }
if (found) if (found) {
break; break;
}
__balloc_get_parent_range(a, cur_base, cur_order, __balloc_get_parent_range(a, cur_base, cur_order,
&cur_base, &cur_order); &cur_base, &cur_order);
@@ -679,10 +701,11 @@ static struct nvgpu_buddy *__balloc_make_fixed_buddy(
return NULL; return NULL;
} }
if (base < bud->right->start) if (base < bud->right->start) {
bud = bud->left; bud = bud->left;
else } else {
bud = bud->right; bud = bud->right;
}
} }
@@ -697,12 +720,13 @@ static u64 __balloc_do_alloc_fixed(struct nvgpu_buddy_allocator *a,
u64 align_order; u64 align_order;
shifted_base = balloc_base_shift(a, base); shifted_base = balloc_base_shift(a, base);
if (shifted_base == 0) if (shifted_base == 0) {
align_order = __fls(len >> a->blk_shift); align_order = __fls(len >> a->blk_shift);
else } else {
align_order = min_t(u64, align_order = min_t(u64,
__ffs(shifted_base >> a->blk_shift), __ffs(shifted_base >> a->blk_shift),
__fls(len >> a->blk_shift)); __fls(len >> a->blk_shift));
}
if (align_order > a->max_order) { if (align_order > a->max_order) {
alloc_dbg(balloc_owner(a), alloc_dbg(balloc_owner(a),
@@ -741,10 +765,11 @@ static u64 __balloc_do_alloc_fixed(struct nvgpu_buddy_allocator *a,
align_order = __ffs(inc_base >> a->blk_shift); align_order = __ffs(inc_base >> a->blk_shift);
/* If we don't have much left - trim down align_order. */ /* If we don't have much left - trim down align_order. */
if (balloc_order_to_len(a, align_order) > remaining) if (balloc_order_to_len(a, align_order) > remaining) {
align_order = __balloc_max_order_in(a, inc_base, align_order = __balloc_max_order_in(a, inc_base,
inc_base + remaining); inc_base + remaining);
} }
}
return base; return base;
@@ -805,10 +830,11 @@ static u64 nvgpu_buddy_balloc(struct nvgpu_allocator *__a, u64 len)
return 0; return 0;
} }
if (a->flags & GPU_ALLOC_GVA_SPACE) if (a->flags & GPU_ALLOC_GVA_SPACE) {
pte_size = __get_pte_size(a->vm, 0, len); pte_size = __get_pte_size(a->vm, 0, len);
else } else {
pte_size = BALLOC_PTE_SIZE_ANY; pte_size = BALLOC_PTE_SIZE_ANY;
}
addr = __balloc_do_alloc(a, order, pte_size); addr = __balloc_do_alloc(a, order, pte_size);
@@ -845,25 +871,29 @@ static u64 __nvgpu_balloc_fixed_buddy(struct nvgpu_allocator *__a,
struct nvgpu_buddy_allocator *a = __a->priv; struct nvgpu_buddy_allocator *a = __a->priv;
/* If base isn't aligned to an order 0 block, fail. */ /* If base isn't aligned to an order 0 block, fail. */
if (base & (a->blk_size - 1)) if (base & (a->blk_size - 1)) {
goto fail;
if (len == 0)
goto fail;
/* Check that the page size is valid. */
if (a->flags & GPU_ALLOC_GVA_SPACE && a->vm->big_pages) {
if (page_size == a->vm->big_page_size)
pte_size = gmmu_page_size_big;
else if (page_size == SZ_4K)
pte_size = gmmu_page_size_small;
else
goto fail; goto fail;
} }
falloc = nvgpu_kmalloc(nvgpu_alloc_to_gpu(__a), sizeof(*falloc)); if (len == 0) {
if (!falloc)
goto fail; goto fail;
}
/* Check that the page size is valid. */
if (a->flags & GPU_ALLOC_GVA_SPACE && a->vm->big_pages) {
if (page_size == a->vm->big_page_size) {
pte_size = gmmu_page_size_big;
} else if (page_size == SZ_4K) {
pte_size = gmmu_page_size_small;
} else {
goto fail;
}
}
falloc = nvgpu_kmalloc(nvgpu_alloc_to_gpu(__a), sizeof(*falloc));
if (!falloc) {
goto fail;
}
nvgpu_init_list_node(&falloc->buddies); nvgpu_init_list_node(&falloc->buddies);
falloc->start = base; falloc->start = base;
@@ -936,8 +966,9 @@ static void nvgpu_buddy_bfree(struct nvgpu_allocator *__a, u64 addr)
struct nvgpu_fixed_alloc *falloc; struct nvgpu_fixed_alloc *falloc;
struct nvgpu_buddy_allocator *a = __a->priv; struct nvgpu_buddy_allocator *a = __a->priv;
if (!addr) if (!addr) {
return; return;
}
alloc_lock(__a); alloc_lock(__a);
@@ -952,8 +983,9 @@ static void nvgpu_buddy_bfree(struct nvgpu_allocator *__a, u64 addr)
} }
bud = balloc_free_buddy(a, addr); bud = balloc_free_buddy(a, addr);
if (!bud) if (!bud) {
goto done; goto done;
}
balloc_blist_add(a, bud); balloc_blist_add(a, bud);
a->bytes_freed += balloc_order_to_len(a, bud->order); a->bytes_freed += balloc_order_to_len(a, bud->order);
@@ -987,9 +1019,10 @@ static bool nvgpu_buddy_reserve_is_possible(struct nvgpu_buddy_allocator *a,
if ((co_base >= tmp->base && if ((co_base >= tmp->base &&
co_base < (tmp->base + tmp->length)) || co_base < (tmp->base + tmp->length)) ||
(co_end >= tmp->base && (co_end >= tmp->base &&
co_end < (tmp->base + tmp->length))) co_end < (tmp->base + tmp->length))) {
return false; return false;
} }
}
return true; return true;
} }
@@ -1006,8 +1039,9 @@ static int nvgpu_buddy_reserve_co(struct nvgpu_allocator *__a,
int err = 0; int err = 0;
if (co->base < a->start || (co->base + co->length) > a->end || if (co->base < a->start || (co->base + co->length) > a->end ||
a->alloc_made) a->alloc_made) {
return -EINVAL; return -EINVAL;
}
alloc_lock(__a); alloc_lock(__a);
@@ -1221,25 +1255,31 @@ int __nvgpu_buddy_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a,
struct nvgpu_buddy_allocator *a; struct nvgpu_buddy_allocator *a;
/* blk_size must be greater than 0 and a power of 2. */ /* blk_size must be greater than 0 and a power of 2. */
if (blk_size == 0) if (blk_size == 0) {
return -EINVAL; return -EINVAL;
if (blk_size & (blk_size - 1)) }
if (blk_size & (blk_size - 1)) {
return -EINVAL; return -EINVAL;
}
if (max_order > GPU_BALLOC_MAX_ORDER) if (max_order > GPU_BALLOC_MAX_ORDER) {
return -EINVAL; return -EINVAL;
}
/* If this is to manage a GVA space we need a VM. */ /* If this is to manage a GVA space we need a VM. */
if (flags & GPU_ALLOC_GVA_SPACE && !vm) if (flags & GPU_ALLOC_GVA_SPACE && !vm) {
return -EINVAL; return -EINVAL;
}
a = nvgpu_kzalloc(g, sizeof(struct nvgpu_buddy_allocator)); a = nvgpu_kzalloc(g, sizeof(struct nvgpu_buddy_allocator));
if (!a) if (!a) {
return -ENOMEM; return -ENOMEM;
}
err = __nvgpu_alloc_common_init(__a, g, name, a, false, &buddy_ops); err = __nvgpu_alloc_common_init(__a, g, name, a, false, &buddy_ops);
if (err) if (err) {
goto fail; goto fail;
}
a->base = base; a->base = base;
a->length = size; a->length = size;
@@ -1269,8 +1309,9 @@ int __nvgpu_buddy_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a,
*/ */
if (flags & GPU_ALLOC_GVA_SPACE && vm->big_pages && if (flags & GPU_ALLOC_GVA_SPACE && vm->big_pages &&
(base & ((vm->big_page_size << 10) - 1) || (base & ((vm->big_page_size << 10) - 1) ||
size & ((vm->big_page_size << 10) - 1))) size & ((vm->big_page_size << 10) - 1))) {
return -EINVAL; return -EINVAL;
}
a->flags = flags; a->flags = flags;
a->max_order = max_order; a->max_order = max_order;
@@ -1288,8 +1329,9 @@ int __nvgpu_buddy_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a,
a->fixed_allocs = NULL; a->fixed_allocs = NULL;
nvgpu_init_list_node(&a->co_list); nvgpu_init_list_node(&a->co_list);
err = balloc_init_lists(a); err = balloc_init_lists(a);
if (err) if (err) {
goto fail; goto fail;
}
nvgpu_smp_wmb(); nvgpu_smp_wmb();
a->initialized = 1; a->initialized = 1;
@@ -1301,18 +1343,20 @@ int __nvgpu_buddy_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a,
alloc_dbg(__a, " base 0x%llx", a->base); alloc_dbg(__a, " base 0x%llx", a->base);
alloc_dbg(__a, " size 0x%llx", a->length); alloc_dbg(__a, " size 0x%llx", a->length);
alloc_dbg(__a, " blk_size 0x%llx", a->blk_size); alloc_dbg(__a, " blk_size 0x%llx", a->blk_size);
if (flags & GPU_ALLOC_GVA_SPACE) if (flags & GPU_ALLOC_GVA_SPACE) {
alloc_dbg(balloc_owner(a), alloc_dbg(balloc_owner(a),
" pde_size 0x%llx", " pde_size 0x%llx",
balloc_order_to_len(a, a->pte_blk_order)); balloc_order_to_len(a, a->pte_blk_order));
}
alloc_dbg(__a, " max_order %llu", a->max_order); alloc_dbg(__a, " max_order %llu", a->max_order);
alloc_dbg(__a, " flags 0x%llx", a->flags); alloc_dbg(__a, " flags 0x%llx", a->flags);
return 0; return 0;
fail: fail:
if (a->buddy_cache) if (a->buddy_cache) {
nvgpu_kmem_cache_destroy(a->buddy_cache); nvgpu_kmem_cache_destroy(a->buddy_cache);
}
nvgpu_kfree(g, a); nvgpu_kfree(g, a);
return err; return err;
} }

98
drivers/gpu/nvgpu/common/mm/gmmu.c
View File

@@ -38,18 +38,20 @@
#define __gmmu_dbg(g, attrs, fmt, args...) \ #define __gmmu_dbg(g, attrs, fmt, args...) \
do { \ do { \
if (attrs->debug) \ if (attrs->debug) { \
nvgpu_info(g, fmt, ##args); \ nvgpu_info(g, fmt, ##args); \
else \ } else { \
nvgpu_log(g, gpu_dbg_map, fmt, ##args); \ nvgpu_log(g, gpu_dbg_map, fmt, ##args); \
} \
} while (0) } while (0)
#define __gmmu_dbg_v(g, attrs, fmt, args...) \ #define __gmmu_dbg_v(g, attrs, fmt, args...) \
do { \ do { \
if (attrs->debug) \ if (attrs->debug) { \
nvgpu_info(g, fmt, ##args); \ nvgpu_info(g, fmt, ##args); \
else \ } else { \
nvgpu_log(g, gpu_dbg_map_v, fmt, ##args); \ nvgpu_log(g, gpu_dbg_map_v, fmt, ##args); \
} \
} while (0) } while (0)
static int pd_allocate(struct vm_gk20a *vm, static int pd_allocate(struct vm_gk20a *vm,
@@ -77,15 +79,17 @@ static u64 __nvgpu_gmmu_map(struct vm_gk20a *vm,
struct nvgpu_sgt *sgt = nvgpu_sgt_create_from_mem(g, mem); struct nvgpu_sgt *sgt = nvgpu_sgt_create_from_mem(g, mem);
if (!sgt) if (!sgt) {
return -ENOMEM; return -ENOMEM;
}
/* /*
* If the GPU is IO coherent and the DMA API is giving us IO coherent * If the GPU is IO coherent and the DMA API is giving us IO coherent
* CPU mappings then we gotta make sure we use the IO coherent aperture. * CPU mappings then we gotta make sure we use the IO coherent aperture.
*/ */
if (nvgpu_is_enabled(g, NVGPU_USE_COHERENT_SYSMEM)) if (nvgpu_is_enabled(g, NVGPU_USE_COHERENT_SYSMEM)) {
flags |= NVGPU_VM_MAP_IO_COHERENT; flags |= NVGPU_VM_MAP_IO_COHERENT;
}
/* /*
* Later on, when we free this nvgpu_mem's GPU mapping, we are going to * Later on, when we free this nvgpu_mem's GPU mapping, we are going to
@@ -94,10 +98,11 @@ static u64 __nvgpu_gmmu_map(struct vm_gk20a *vm,
* therefor we should not try and free it. But otherwise, if we do * therefor we should not try and free it. But otherwise, if we do
* manage the VA alloc, we obviously must free it. * manage the VA alloc, we obviously must free it.
*/ */
if (addr != 0) if (addr != 0) {
mem->free_gpu_va = false; mem->free_gpu_va = false;
else } else {
mem->free_gpu_va = true; mem->free_gpu_va = true;
}
nvgpu_mutex_acquire(&vm->update_gmmu_lock); nvgpu_mutex_acquire(&vm->update_gmmu_lock);
vaddr = g->ops.mm.gmmu_map(vm, addr, vaddr = g->ops.mm.gmmu_map(vm, addr,
@@ -196,8 +201,9 @@ int nvgpu_gmmu_init_page_table(struct vm_gk20a *vm)
pdb_size = ALIGN(pd_size(&vm->mmu_levels[0], &attrs), PAGE_SIZE); pdb_size = ALIGN(pd_size(&vm->mmu_levels[0], &attrs), PAGE_SIZE);
err = __nvgpu_pd_cache_alloc_direct(vm->mm->g, &vm->pdb, pdb_size); err = __nvgpu_pd_cache_alloc_direct(vm->mm->g, &vm->pdb, pdb_size);
if (WARN_ON(err)) if (WARN_ON(err)) {
return err; return err;
}
/* /*
* One nvgpu_mb() is done after all mapping operations. Don't need * One nvgpu_mb() is done after all mapping operations. Don't need
@@ -267,8 +273,9 @@ static int pd_allocate(struct vm_gk20a *vm,
{ {
int err; int err;
if (pd->mem) if (pd->mem) {
return 0; return 0;
}
err = __nvgpu_pd_alloc(vm, pd, pd_size(l, attrs)); err = __nvgpu_pd_alloc(vm, pd, pd_size(l, attrs));
if (err) { if (err) {
@@ -310,14 +317,16 @@ static int pd_allocate_children(struct vm_gk20a *vm,
{ {
struct gk20a *g = gk20a_from_vm(vm); struct gk20a *g = gk20a_from_vm(vm);
if (pd->entries) if (pd->entries) {
return 0; return 0;
}
pd->num_entries = pd_entries(l, attrs); pd->num_entries = pd_entries(l, attrs);
pd->entries = nvgpu_vzalloc(g, sizeof(struct nvgpu_gmmu_pd) * pd->entries = nvgpu_vzalloc(g, sizeof(struct nvgpu_gmmu_pd) *
pd->num_entries); pd->num_entries);
if (!pd->entries) if (!pd->entries) {
return -ENOMEM; return -ENOMEM;
}
return 0; return 0;
} }
@@ -398,8 +407,9 @@ static int __set_pd_level(struct vm_gk20a *vm,
* have a bunch of children PDs. * have a bunch of children PDs.
*/ */
if (next_l->update_entry) { if (next_l->update_entry) {
if (pd_allocate_children(vm, l, pd, attrs)) if (pd_allocate_children(vm, l, pd, attrs)) {
return -ENOMEM; return -ENOMEM;
}
/* /*
* Get the next PD so that we know what to put in this * Get the next PD so that we know what to put in this
@@ -412,9 +422,10 @@ static int __set_pd_level(struct vm_gk20a *vm,
/* /*
* Allocate the backing memory for next_pd. * Allocate the backing memory for next_pd.
*/ */
if (pd_allocate(vm, next_pd, next_l, attrs)) if (pd_allocate(vm, next_pd, next_l, attrs)) {
return -ENOMEM; return -ENOMEM;
} }
}
/* /*
* This is the address we want to program into the actual PDE/ * This is the address we want to program into the actual PDE/
@@ -440,9 +451,10 @@ static int __set_pd_level(struct vm_gk20a *vm,
chunk_size, chunk_size,
attrs); attrs);
if (err) if (err) {
return err; return err;
} }
}
virt_addr += chunk_size; virt_addr += chunk_size;
@@ -452,8 +464,9 @@ static int __set_pd_level(struct vm_gk20a *vm,
* non-zero phys addresses in the PTEs. A non-zero phys-addr * non-zero phys addresses in the PTEs. A non-zero phys-addr
* would also confuse the lower level PTE programming code. * would also confuse the lower level PTE programming code.
*/ */
if (phys_addr) if (phys_addr) {
phys_addr += chunk_size; phys_addr += chunk_size;
}
length -= chunk_size; length -= chunk_size;
} }
@@ -547,8 +560,9 @@ static int __nvgpu_gmmu_do_update_page_table(struct vm_gk20a *vm,
virt_addr, virt_addr,
chunk_length, chunk_length,
attrs); attrs);
if (err) if (err) {
break; break;
}
/* Space has been skipped so zero this for future chunks. */ /* Space has been skipped so zero this for future chunks. */
space_to_skip = 0; space_to_skip = 0;
@@ -559,9 +573,10 @@ static int __nvgpu_gmmu_do_update_page_table(struct vm_gk20a *vm,
virt_addr += chunk_length; virt_addr += chunk_length;
length -= chunk_length; length -= chunk_length;
if (length == 0) if (length == 0) {
break; break;
} }
}
return err; return err;
} }
@@ -594,13 +609,15 @@ static int __nvgpu_gmmu_update_page_table(struct vm_gk20a *vm,
/* note: here we need to map kernel to small, since the /* note: here we need to map kernel to small, since the
* low-level mmu code assumes 0 is small and 1 is big pages */ * low-level mmu code assumes 0 is small and 1 is big pages */
if (attrs->pgsz == gmmu_page_size_kernel) if (attrs->pgsz == gmmu_page_size_kernel) {
attrs->pgsz = gmmu_page_size_small; attrs->pgsz = gmmu_page_size_small;
}
page_size = vm->gmmu_page_sizes[attrs->pgsz]; page_size = vm->gmmu_page_sizes[attrs->pgsz];
if (space_to_skip & (page_size - 1)) if (space_to_skip & (page_size - 1)) {
return -EINVAL; return -EINVAL;
}
/* /*
* Update length to be aligned to the passed page size. * Update length to be aligned to the passed page size.
@@ -692,8 +709,9 @@ u64 gk20a_locked_gmmu_map(struct vm_gk20a *vm,
* the programmed ctagline gets increased at compression_page_size * the programmed ctagline gets increased at compression_page_size
* boundaries. * boundaries.
*/ */
if (attrs.ctag) if (attrs.ctag) {
attrs.ctag += buffer_offset & (ctag_granularity - 1U); attrs.ctag += buffer_offset & (ctag_granularity - 1U);
}
attrs.l3_alloc = (bool)(flags & NVGPU_VM_MAP_L3_ALLOC); attrs.l3_alloc = (bool)(flags & NVGPU_VM_MAP_L3_ALLOC);
@@ -701,8 +719,9 @@ u64 gk20a_locked_gmmu_map(struct vm_gk20a *vm,
* Handle the IO coherency aperture: make sure the .aperture field is * Handle the IO coherency aperture: make sure the .aperture field is
* correct based on the IO coherency flag. * correct based on the IO coherency flag.
*/ */
if (attrs.coherent && attrs.aperture == APERTURE_SYSMEM) if (attrs.coherent && attrs.aperture == APERTURE_SYSMEM) {
attrs.aperture = __APERTURE_SYSMEM_COH; attrs.aperture = __APERTURE_SYSMEM_COH;
}
/* /*
* Only allocate a new GPU VA range if we haven't already been passed a * Only allocate a new GPU VA range if we haven't already been passed a
@@ -725,16 +744,18 @@ u64 gk20a_locked_gmmu_map(struct vm_gk20a *vm,
goto fail_validate; goto fail_validate;
} }
if (!batch) if (!batch) {
g->ops.fb.tlb_invalidate(g, vm->pdb.mem); g->ops.fb.tlb_invalidate(g, vm->pdb.mem);
else } else {
batch->need_tlb_invalidate = true; batch->need_tlb_invalidate = true;
}
return vaddr; return vaddr;
fail_validate: fail_validate:
if (allocated) if (allocated) {
__nvgpu_vm_free_va(vm, vaddr, pgsz_idx); __nvgpu_vm_free_va(vm, vaddr, pgsz_idx);
}
fail_alloc: fail_alloc:
nvgpu_err(g, "%s: failed with err=%d", __func__, err); nvgpu_err(g, "%s: failed with err=%d", __func__, err);
return 0; return 0;
@@ -775,8 +796,9 @@ void gk20a_locked_gmmu_unmap(struct vm_gk20a *vm,
/* unmap here needs to know the page size we assigned at mapping */ /* unmap here needs to know the page size we assigned at mapping */
err = __nvgpu_gmmu_update_page_table(vm, NULL, 0, err = __nvgpu_gmmu_update_page_table(vm, NULL, 0,
vaddr, size, &attrs); vaddr, size, &attrs);
if (err) if (err) {
nvgpu_err(g, "failed to update gmmu ptes on unmap"); nvgpu_err(g, "failed to update gmmu ptes on unmap");
}
if (!batch) { if (!batch) {
gk20a_mm_l2_flush(g, true); gk20a_mm_l2_flush(g, true);
@@ -801,8 +823,9 @@ u32 __nvgpu_pte_words(struct gk20a *g)
*/ */
do { do {
next_l = l + 1; next_l = l + 1;
if (!next_l->update_entry) if (!next_l->update_entry) {
break; break;
}
l++; l++;
} while (true); } while (true);
@@ -836,13 +859,15 @@ static int __nvgpu_locate_pte(struct gk20a *g, struct vm_gk20a *vm,
struct nvgpu_gmmu_pd *pd_next = pd->entries + pd_idx; struct nvgpu_gmmu_pd *pd_next = pd->entries + pd_idx;
/* Invalid entry! */ /* Invalid entry! */
if (!pd_next->mem) if (!pd_next->mem) {
return -EINVAL; return -EINVAL;
}
attrs->pgsz = l->get_pgsz(g, l, pd, pd_idx); attrs->pgsz = l->get_pgsz(g, l, pd, pd_idx);
if (attrs->pgsz >= gmmu_nr_page_sizes) if (attrs->pgsz >= gmmu_nr_page_sizes) {
return -EINVAL; return -EINVAL;
}
return __nvgpu_locate_pte(g, vm, pd_next, return __nvgpu_locate_pte(g, vm, pd_next,
vaddr, lvl + 1, attrs, vaddr, lvl + 1, attrs,
@@ -850,8 +875,9 @@ static int __nvgpu_locate_pte(struct gk20a *g, struct vm_gk20a *vm,
pd_offs_out); pd_offs_out);
} }
if (!pd->mem) if (!pd->mem) {
return -EINVAL; return -EINVAL;
}
/* /*
* Take into account the real offset into the nvgpu_mem since the PD * Take into account the real offset into the nvgpu_mem since the PD
@@ -867,14 +893,17 @@ static int __nvgpu_locate_pte(struct gk20a *g, struct vm_gk20a *vm,
} }
} }
if (pd_out) if (pd_out) {
*pd_out = pd; *pd_out = pd;
}
if (pd_idx_out) if (pd_idx_out) {
*pd_idx_out = pd_idx; *pd_idx_out = pd_idx;
}
if (pd_offs_out) if (pd_offs_out) {
*pd_offs_out = pd_offset_from_index(l, pd_idx); *pd_offs_out = pd_offset_from_index(l, pd_idx);
}
return 0; return 0;
} }
@@ -903,8 +932,9 @@ int __nvgpu_set_pte(struct gk20a *g, struct vm_gk20a *vm, u64 vaddr, u32 *pte)
err = __nvgpu_locate_pte(g, vm, &vm->pdb, err = __nvgpu_locate_pte(g, vm, &vm->pdb,
vaddr, 0, &attrs, vaddr, 0, &attrs,
NULL, &pd, &pd_idx, &pd_offs); NULL, &pd, &pd_idx, &pd_offs);
if (err) if (err) {
return err; return err;
}
pte_size = __nvgpu_pte_words(g); pte_size = __nvgpu_pte_words(g);

20
drivers/gpu/nvgpu/common/mm/lockless_allocator.c
View File

@@ -63,8 +63,9 @@ static u64 nvgpu_lockless_alloc(struct nvgpu_allocator *a, u64 len)
int head, new_head, ret; int head, new_head, ret;
u64 addr = 0; u64 addr = 0;
if (len != pa->blk_size) if (len != pa->blk_size) {
return 0; return 0;
}
head = NV_ACCESS_ONCE(pa->head); head = NV_ACCESS_ONCE(pa->head);
while (head >= 0) { while (head >= 0) {
@@ -80,10 +81,11 @@ static u64 nvgpu_lockless_alloc(struct nvgpu_allocator *a, u64 len)
head = NV_ACCESS_ONCE(pa->head); head = NV_ACCESS_ONCE(pa->head);
} }
if (addr) if (addr) {
alloc_dbg(a, "Alloc node # %d @ addr 0x%llx", head, addr); alloc_dbg(a, "Alloc node # %d @ addr 0x%llx", head, addr);
else } else {
alloc_dbg(a, "Alloc failed!"); alloc_dbg(a, "Alloc failed!");
}
return addr; return addr;
} }
@@ -167,24 +169,28 @@ int nvgpu_lockless_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a,
u64 count; u64 count;
struct nvgpu_lockless_allocator *a; struct nvgpu_lockless_allocator *a;
if (!blk_size) if (!blk_size) {
return -EINVAL; return -EINVAL;
}
/* /*
* Ensure we have space for at least one node & there's no overflow. * Ensure we have space for at least one node & there's no overflow.
* In order to control memory footprint, we require count < INT_MAX * In order to control memory footprint, we require count < INT_MAX
*/ */
count = length / blk_size; count = length / blk_size;
if (!base || !count || count > INT_MAX) if (!base || !count || count > INT_MAX) {
return -EINVAL; return -EINVAL;
}
a = nvgpu_kzalloc(g, sizeof(struct nvgpu_lockless_allocator)); a = nvgpu_kzalloc(g, sizeof(struct nvgpu_lockless_allocator));
if (!a) if (!a) {
return -ENOMEM; return -ENOMEM;
}
err = __nvgpu_alloc_common_init(__a, g, name, a, false, &pool_ops); err = __nvgpu_alloc_common_init(__a, g, name, a, false, &pool_ops);
if (err) if (err) {
goto fail; goto fail;
}
a->next = nvgpu_vzalloc(g, sizeof(*a->next) * count); a->next = nvgpu_vzalloc(g, sizeof(*a->next) * count);
if (!a->next) { if (!a->next) {

119
drivers/gpu/nvgpu/common/mm/mm.c
View File

@@ -40,8 +40,9 @@ enum gmmu_pgsz_gk20a __get_pte_size_fixed_map(struct vm_gk20a *vm,
struct nvgpu_vm_area *vm_area; struct nvgpu_vm_area *vm_area;
vm_area = nvgpu_vm_area_find(vm, base); vm_area = nvgpu_vm_area_find(vm, base);
if (!vm_area) if (!vm_area) {
return gmmu_page_size_small; return gmmu_page_size_small;
}
return vm_area->pgsz_idx; return vm_area->pgsz_idx;
} }
@@ -53,15 +54,17 @@ static enum gmmu_pgsz_gk20a __get_pte_size_split_addr(struct vm_gk20a *vm,
u64 base, u64 size) u64 base, u64 size)
{ {
if (!base) { if (!base) {
if (size >= vm->gmmu_page_sizes[gmmu_page_size_big]) if (size >= vm->gmmu_page_sizes[gmmu_page_size_big]) {
return gmmu_page_size_big; return gmmu_page_size_big;
}
return gmmu_page_size_small; return gmmu_page_size_small;
} else { } else {
if (base < __nv_gmmu_va_small_page_limit()) if (base < __nv_gmmu_va_small_page_limit()) {
return gmmu_page_size_small; return gmmu_page_size_small;
else } else {
return gmmu_page_size_big; return gmmu_page_size_big;
} }
}
} }
/* /*
@@ -89,18 +92,22 @@ enum gmmu_pgsz_gk20a __get_pte_size(struct vm_gk20a *vm, u64 base, u64 size)
{ {
struct gk20a *g = gk20a_from_vm(vm); struct gk20a *g = gk20a_from_vm(vm);
if (!vm->big_pages) if (!vm->big_pages) {
return gmmu_page_size_small; return gmmu_page_size_small;
}
if (!nvgpu_is_enabled(g, NVGPU_MM_UNIFY_ADDRESS_SPACES)) if (!nvgpu_is_enabled(g, NVGPU_MM_UNIFY_ADDRESS_SPACES)) {
return __get_pte_size_split_addr(vm, base, size); return __get_pte_size_split_addr(vm, base, size);
}
if (base) if (base) {
return __get_pte_size_fixed_map(vm, base, size); return __get_pte_size_fixed_map(vm, base, size);
}
if (size >= vm->gmmu_page_sizes[gmmu_page_size_big] && if (size >= vm->gmmu_page_sizes[gmmu_page_size_big] &&
nvgpu_iommuable(g)) nvgpu_iommuable(g)) {
return gmmu_page_size_big; return gmmu_page_size_big;
}
return gmmu_page_size_small; return gmmu_page_size_small;
} }
@@ -137,8 +144,9 @@ u64 nvgpu_inst_block_addr(struct gk20a *g, struct nvgpu_mem *inst_block)
void nvgpu_free_inst_block(struct gk20a *g, struct nvgpu_mem *inst_block) void nvgpu_free_inst_block(struct gk20a *g, struct nvgpu_mem *inst_block)
{ {
if (nvgpu_mem_is_valid(inst_block)) if (nvgpu_mem_is_valid(inst_block)) {
nvgpu_dma_free(g, inst_block); nvgpu_dma_free(g, inst_block);
}
} }
static int nvgpu_alloc_sysmem_flush(struct gk20a *g) static int nvgpu_alloc_sysmem_flush(struct gk20a *g)
@@ -150,8 +158,9 @@ static void nvgpu_remove_mm_ce_support(struct mm_gk20a *mm)
{ {
struct gk20a *g = gk20a_from_mm(mm); struct gk20a *g = gk20a_from_mm(mm);
if (mm->vidmem.ce_ctx_id != (u32)~0) if (mm->vidmem.ce_ctx_id != (u32)~0) {
gk20a_ce_delete_context_priv(g, mm->vidmem.ce_ctx_id); gk20a_ce_delete_context_priv(g, mm->vidmem.ce_ctx_id);
}
mm->vidmem.ce_ctx_id = (u32)~0; mm->vidmem.ce_ctx_id = (u32)~0;
@@ -162,11 +171,13 @@ static void nvgpu_remove_mm_support(struct mm_gk20a *mm)
{ {
struct gk20a *g = gk20a_from_mm(mm); struct gk20a *g = gk20a_from_mm(mm);
if (g->ops.mm.fault_info_mem_destroy) if (g->ops.mm.fault_info_mem_destroy) {
g->ops.mm.fault_info_mem_destroy(g); g->ops.mm.fault_info_mem_destroy(g);
}
if (g->ops.mm.remove_bar2_vm) if (g->ops.mm.remove_bar2_vm) {
g->ops.mm.remove_bar2_vm(g); g->ops.mm.remove_bar2_vm(g);
}
nvgpu_free_inst_block(g, &mm->bar1.inst_block); nvgpu_free_inst_block(g, &mm->bar1.inst_block);
nvgpu_vm_put(mm->bar1.vm); nvgpu_vm_put(mm->bar1.vm);
@@ -175,8 +186,9 @@ static void nvgpu_remove_mm_support(struct mm_gk20a *mm)
nvgpu_free_inst_block(g, &mm->hwpm.inst_block); nvgpu_free_inst_block(g, &mm->hwpm.inst_block);
nvgpu_vm_put(mm->pmu.vm); nvgpu_vm_put(mm->pmu.vm);
if (g->has_cde) if (g->has_cde) {
nvgpu_vm_put(mm->cde.vm); nvgpu_vm_put(mm->cde.vm);
}
nvgpu_semaphore_sea_destroy(g); nvgpu_semaphore_sea_destroy(g);
nvgpu_vidmem_destroy(g); nvgpu_vidmem_destroy(g);
@@ -208,12 +220,14 @@ static int nvgpu_init_system_vm(struct mm_gk20a *mm)
true, true,
false, false,
"system"); "system");
if (!mm->pmu.vm) if (!mm->pmu.vm) {
return -ENOMEM; return -ENOMEM;
}
err = g->ops.mm.alloc_inst_block(g, inst_block); err = g->ops.mm.alloc_inst_block(g, inst_block);
if (err) if (err) {
goto clean_up_vm; goto clean_up_vm;
}
g->ops.mm.init_inst_block(inst_block, mm->pmu.vm, big_page_size); g->ops.mm.init_inst_block(inst_block, mm->pmu.vm, big_page_size);
return 0; return 0;
@@ -230,8 +244,9 @@ static int nvgpu_init_hwpm(struct mm_gk20a *mm)
struct nvgpu_mem *inst_block = &mm->hwpm.inst_block; struct nvgpu_mem *inst_block = &mm->hwpm.inst_block;
err = g->ops.mm.alloc_inst_block(g, inst_block); err = g->ops.mm.alloc_inst_block(g, inst_block);
if (err) if (err) {
return err; return err;
}
g->ops.mm.init_inst_block(inst_block, mm->pmu.vm, 0); g->ops.mm.init_inst_block(inst_block, mm->pmu.vm, 0);
return 0; return 0;
@@ -247,8 +262,9 @@ static int nvgpu_init_cde_vm(struct mm_gk20a *mm)
NV_MM_DEFAULT_KERNEL_SIZE, NV_MM_DEFAULT_KERNEL_SIZE,
NV_MM_DEFAULT_KERNEL_SIZE + NV_MM_DEFAULT_USER_SIZE, NV_MM_DEFAULT_KERNEL_SIZE + NV_MM_DEFAULT_USER_SIZE,
false, false, "cde"); false, false, "cde");
if (!mm->cde.vm) if (!mm->cde.vm) {
return -ENOMEM; return -ENOMEM;
}
return 0; return 0;
} }
@@ -262,8 +278,9 @@ static int nvgpu_init_ce_vm(struct mm_gk20a *mm)
NV_MM_DEFAULT_KERNEL_SIZE, NV_MM_DEFAULT_KERNEL_SIZE,
NV_MM_DEFAULT_KERNEL_SIZE + NV_MM_DEFAULT_USER_SIZE, NV_MM_DEFAULT_KERNEL_SIZE + NV_MM_DEFAULT_USER_SIZE,
false, false, "ce"); false, false, "ce");
if (!mm->ce.vm) if (!mm->ce.vm) {
return -ENOMEM; return -ENOMEM;
}
return 0; return 0;
} }
@@ -286,24 +303,30 @@ void nvgpu_init_mm_ce_context(struct gk20a *g)
static int nvgpu_init_mm_reset_enable_hw(struct gk20a *g) static int nvgpu_init_mm_reset_enable_hw(struct gk20a *g)
{ {
if (g->ops.fb.reset) if (g->ops.fb.reset) {
g->ops.fb.reset(g); g->ops.fb.reset(g);
}
if (g->ops.clock_gating.slcg_fb_load_gating_prod) if (g->ops.clock_gating.slcg_fb_load_gating_prod) {
g->ops.clock_gating.slcg_fb_load_gating_prod(g, g->ops.clock_gating.slcg_fb_load_gating_prod(g,
g->slcg_enabled); g->slcg_enabled);
if (g->ops.clock_gating.slcg_ltc_load_gating_prod) }
if (g->ops.clock_gating.slcg_ltc_load_gating_prod) {
g->ops.clock_gating.slcg_ltc_load_gating_prod(g, g->ops.clock_gating.slcg_ltc_load_gating_prod(g,
g->slcg_enabled); g->slcg_enabled);
if (g->ops.clock_gating.blcg_fb_load_gating_prod) }
if (g->ops.clock_gating.blcg_fb_load_gating_prod) {
g->ops.clock_gating.blcg_fb_load_gating_prod(g, g->ops.clock_gating.blcg_fb_load_gating_prod(g,
g->blcg_enabled); g->blcg_enabled);
if (g->ops.clock_gating.blcg_ltc_load_gating_prod) }
if (g->ops.clock_gating.blcg_ltc_load_gating_prod) {
g->ops.clock_gating.blcg_ltc_load_gating_prod(g, g->ops.clock_gating.blcg_ltc_load_gating_prod(g,
g->blcg_enabled); g->blcg_enabled);
}
if (g->ops.fb.init_fs_state) if (g->ops.fb.init_fs_state) {
g->ops.fb.init_fs_state(g); g->ops.fb.init_fs_state(g);
}
return 0; return 0;
} }
@@ -324,12 +347,14 @@ static int nvgpu_init_bar1_vm(struct mm_gk20a *mm)
mm->bar1.aperture_size, mm->bar1.aperture_size,
true, false, true, false,
"bar1"); "bar1");
if (!mm->bar1.vm) if (!mm->bar1.vm) {
return -ENOMEM; return -ENOMEM;
}
err = g->ops.mm.alloc_inst_block(g, inst_block); err = g->ops.mm.alloc_inst_block(g, inst_block);
if (err) if (err) {
goto clean_up_vm; goto clean_up_vm;
}
g->ops.mm.init_inst_block(inst_block, mm->bar1.vm, big_page_size); g->ops.mm.init_inst_block(inst_block, mm->bar1.vm, big_page_size);
return 0; return 0;
@@ -366,8 +391,9 @@ static int nvgpu_init_mm_setup_sw(struct gk20a *g)
mm->vidmem.ce_ctx_id = (u32)~0; mm->vidmem.ce_ctx_id = (u32)~0;
err = nvgpu_vidmem_init(mm); err = nvgpu_vidmem_init(mm);
if (err) if (err) {
return err; return err;
}
/* /*
* this requires fixed allocations in vidmem which must be * this requires fixed allocations in vidmem which must be
@@ -376,40 +402,48 @@ static int nvgpu_init_mm_setup_sw(struct gk20a *g)
if (g->ops.pmu.alloc_blob_space if (g->ops.pmu.alloc_blob_space
&& !nvgpu_is_enabled(g, NVGPU_MM_UNIFIED_MEMORY)) { && !nvgpu_is_enabled(g, NVGPU_MM_UNIFIED_MEMORY)) {
err = g->ops.pmu.alloc_blob_space(g, 0, &g->acr.ucode_blob); err = g->ops.pmu.alloc_blob_space(g, 0, &g->acr.ucode_blob);
if (err) if (err) {
return err; return err;
} }
}
err = nvgpu_alloc_sysmem_flush(g); err = nvgpu_alloc_sysmem_flush(g);
if (err) if (err) {
return err; return err;
}
err = nvgpu_init_bar1_vm(mm); err = nvgpu_init_bar1_vm(mm);
if (err) if (err) {
return err; return err;
}
if (g->ops.mm.init_bar2_vm) { if (g->ops.mm.init_bar2_vm) {
err = g->ops.mm.init_bar2_vm(g); err = g->ops.mm.init_bar2_vm(g);
if (err) if (err) {
return err; return err;
} }
}
err = nvgpu_init_system_vm(mm); err = nvgpu_init_system_vm(mm);
if (err) if (err) {
return err; return err;
}
err = nvgpu_init_hwpm(mm); err = nvgpu_init_hwpm(mm);
if (err) if (err) {
return err; return err;
}
if (g->has_cde) { if (g->has_cde) {
err = nvgpu_init_cde_vm(mm); err = nvgpu_init_cde_vm(mm);
if (err) if (err) {
return err; return err;
} }
}
err = nvgpu_init_ce_vm(mm); err = nvgpu_init_ce_vm(mm);
if (err) if (err) {
return err; return err;
}
mm->remove_support = nvgpu_remove_mm_support; mm->remove_support = nvgpu_remove_mm_support;
mm->remove_ce_support = nvgpu_remove_mm_ce_support; mm->remove_ce_support = nvgpu_remove_mm_ce_support;
@@ -424,15 +458,18 @@ int nvgpu_init_mm_support(struct gk20a *g)
u32 err; u32 err;
err = nvgpu_init_mm_reset_enable_hw(g); err = nvgpu_init_mm_reset_enable_hw(g);
if (err) if (err) {
return err; return err;
}
err = nvgpu_init_mm_setup_sw(g); err = nvgpu_init_mm_setup_sw(g);
if (err) if (err) {
return err; return err;
}
if (g->ops.mm.init_mm_setup_hw) if (g->ops.mm.init_mm_setup_hw) {
err = g->ops.mm.init_mm_setup_hw(g); err = g->ops.mm.init_mm_setup_hw(g);
}
return err; return err;
} }
@@ -443,8 +480,9 @@ u32 nvgpu_mm_get_default_big_page_size(struct gk20a *g)
big_page_size = g->ops.mm.get_default_big_page_size(); big_page_size = g->ops.mm.get_default_big_page_size();
if (g->mm.disable_bigpage) if (g->mm.disable_bigpage) {
big_page_size = 0; big_page_size = 0;
}
return big_page_size; return big_page_size;
} }
@@ -456,9 +494,10 @@ u32 nvgpu_mm_get_available_big_page_sizes(struct gk20a *g)
if (!g->mm.disable_bigpage) { if (!g->mm.disable_bigpage) {
available_big_page_sizes = available_big_page_sizes =
g->ops.mm.get_default_big_page_size(); g->ops.mm.get_default_big_page_size();
if (g->ops.mm.get_big_page_sizes) if (g->ops.mm.get_big_page_sizes) {
available_big_page_sizes |= g->ops.mm.get_big_page_sizes(); available_big_page_sizes |= g->ops.mm.get_big_page_sizes();
} }
}
return available_big_page_sizes; return available_big_page_sizes;
} }

23
drivers/gpu/nvgpu/common/mm/nvgpu_mem.c
View File

@@ -40,8 +40,9 @@ u32 __nvgpu_aperture_mask(struct gk20a *g, enum nvgpu_aperture aperture,
* Some iGPUs treat sysmem (i.e SoC DRAM) as vidmem. In these cases the * Some iGPUs treat sysmem (i.e SoC DRAM) as vidmem. In these cases the
* "sysmem" aperture should really be translated to VIDMEM. * "sysmem" aperture should really be translated to VIDMEM.
*/ */
if (!nvgpu_is_enabled(g, NVGPU_MM_HONORS_APERTURE)) if (!nvgpu_is_enabled(g, NVGPU_MM_HONORS_APERTURE)) {
aperture = APERTURE_VIDMEM; aperture = APERTURE_VIDMEM;
}
switch (aperture) { switch (aperture) {
case __APERTURE_SYSMEM_COH: case __APERTURE_SYSMEM_COH:
@@ -67,8 +68,9 @@ u32 nvgpu_aperture_mask(struct gk20a *g, struct nvgpu_mem *mem,
* we add this translation step here. * we add this translation step here.
*/ */
if (nvgpu_is_enabled(g, NVGPU_USE_COHERENT_SYSMEM) && if (nvgpu_is_enabled(g, NVGPU_USE_COHERENT_SYSMEM) &&
ap == APERTURE_SYSMEM) ap == APERTURE_SYSMEM) {
ap = __APERTURE_SYSMEM_COH; ap = __APERTURE_SYSMEM_COH;
}
return __nvgpu_aperture_mask(g, ap, return __nvgpu_aperture_mask(g, ap,
sysmem_mask, sysmem_coh_mask, vidmem_mask); sysmem_mask, sysmem_coh_mask, vidmem_mask);
@@ -115,15 +117,17 @@ u64 nvgpu_sgt_get_gpu_addr(struct gk20a *g, struct nvgpu_sgt *sgt,
bool nvgpu_sgt_iommuable(struct gk20a *g, struct nvgpu_sgt *sgt) bool nvgpu_sgt_iommuable(struct gk20a *g, struct nvgpu_sgt *sgt)
{ {
if (sgt->ops->sgt_iommuable) if (sgt->ops->sgt_iommuable) {
return sgt->ops->sgt_iommuable(g, sgt); return sgt->ops->sgt_iommuable(g, sgt);
}
return false; return false;
} }
void nvgpu_sgt_free(struct gk20a *g, struct nvgpu_sgt *sgt) void nvgpu_sgt_free(struct gk20a *g, struct nvgpu_sgt *sgt)
{ {
if (sgt && sgt->ops->sgt_free) if (sgt && sgt->ops->sgt_free) {
sgt->ops->sgt_free(g, sgt); sgt->ops->sgt_free(g, sgt);
}
} }
u64 nvgpu_mem_iommu_translate(struct gk20a *g, u64 phys) u64 nvgpu_mem_iommu_translate(struct gk20a *g, u64 phys)
@@ -131,8 +135,9 @@ u64 nvgpu_mem_iommu_translate(struct gk20a *g, u64 phys)
/* ensure it is not vidmem allocation */ /* ensure it is not vidmem allocation */
WARN_ON(nvgpu_addr_is_vidmem_page_alloc(phys)); WARN_ON(nvgpu_addr_is_vidmem_page_alloc(phys));
if (nvgpu_iommuable(g) && g->ops.mm.get_iommu_bit) if (nvgpu_iommuable(g) && g->ops.mm.get_iommu_bit) {
return phys | 1ULL << g->ops.mm.get_iommu_bit(g); return phys | 1ULL << g->ops.mm.get_iommu_bit(g);
}
return phys; return phys;
} }
@@ -157,8 +162,9 @@ u64 nvgpu_sgt_alignment(struct gk20a *g, struct nvgpu_sgt *sgt)
*/ */
if (nvgpu_iommuable(g) && if (nvgpu_iommuable(g) &&
nvgpu_sgt_iommuable(g, sgt) && nvgpu_sgt_iommuable(g, sgt) &&
nvgpu_sgt_get_dma(sgt, sgt->sgl)) nvgpu_sgt_get_dma(sgt, sgt->sgl)) {
return 1ULL << __ffs(nvgpu_sgt_get_dma(sgt, sgt->sgl)); return 1ULL << __ffs(nvgpu_sgt_get_dma(sgt, sgt->sgl));
}
/* /*
* Otherwise the buffer is not iommuable (VIDMEM, for example) or we are * Otherwise the buffer is not iommuable (VIDMEM, for example) or we are
@@ -169,11 +175,12 @@ u64 nvgpu_sgt_alignment(struct gk20a *g, struct nvgpu_sgt *sgt)
chunk_align = 1ULL << __ffs(nvgpu_sgt_get_phys(g, sgt, sgl) | chunk_align = 1ULL << __ffs(nvgpu_sgt_get_phys(g, sgt, sgl) |
nvgpu_sgt_get_length(sgt, sgl)); nvgpu_sgt_get_length(sgt, sgl));
if (align) if (align) {
align = min(align, chunk_align); align = min(align, chunk_align);
else } else {
align = chunk_align; align = chunk_align;
} }
}
return align; return align;
} }

123
drivers/gpu/nvgpu/common/mm/vm.c
View File

@@ -111,8 +111,9 @@ static void nvgpu_vm_free_entries(struct vm_gk20a *vm,
__nvgpu_pd_cache_free_direct(g, pdb); __nvgpu_pd_cache_free_direct(g, pdb);
if (!pdb->entries) if (!pdb->entries) {
return; return;
}
for (i = 0; i < pdb->num_entries; i++) { for (i = 0; i < pdb->num_entries; i++) {
__nvgpu_vm_free_entries(vm, &pdb->entries[i], 1); __nvgpu_vm_free_entries(vm, &pdb->entries[i], 1);
@@ -204,8 +205,9 @@ int nvgpu_big_pages_possible(struct vm_gk20a *vm, u64 base, u64 size)
{ {
u64 mask = ((u64)vm->big_page_size << 10) - 1; u64 mask = ((u64)vm->big_page_size << 10) - 1;
if (base & mask || size & mask) if (base & mask || size & mask) {
return 0; return 0;
}
return 1; return 1;
} }
@@ -223,19 +225,23 @@ static int nvgpu_init_sema_pool(struct vm_gk20a *vm)
/* /*
* Don't waste the memory on semaphores if we don't need them. * Don't waste the memory on semaphores if we don't need them.
*/ */
if (nvgpu_is_enabled(g, NVGPU_HAS_SYNCPOINTS)) if (nvgpu_is_enabled(g, NVGPU_HAS_SYNCPOINTS)) {
return 0; return 0;
}
if (vm->sema_pool) if (vm->sema_pool) {
return 0; return 0;
}
sema_sea = nvgpu_semaphore_sea_create(g); sema_sea = nvgpu_semaphore_sea_create(g);
if (!sema_sea) if (!sema_sea) {
return -ENOMEM; return -ENOMEM;
}
err = nvgpu_semaphore_pool_alloc(sema_sea, &vm->sema_pool); err = nvgpu_semaphore_pool_alloc(sema_sea, &vm->sema_pool);
if (err) if (err) {
return err; return err;
}
/* /*
* Allocate a chunk of GPU VA space for mapping the semaphores. We will * Allocate a chunk of GPU VA space for mapping the semaphores. We will
@@ -287,11 +293,13 @@ int __nvgpu_vm_init(struct mm_gk20a *mm,
u64 kernel_vma_start, kernel_vma_limit; u64 kernel_vma_start, kernel_vma_limit;
struct gk20a *g = gk20a_from_mm(mm); struct gk20a *g = gk20a_from_mm(mm);
if (WARN_ON(kernel_reserved + low_hole > aperture_size)) if (WARN_ON(kernel_reserved + low_hole > aperture_size)) {
return -ENOMEM; return -ENOMEM;
}
if (WARN_ON(vm->guest_managed && kernel_reserved != 0)) if (WARN_ON(vm->guest_managed && kernel_reserved != 0)) {
return -EINVAL; return -EINVAL;
}
nvgpu_log_info(g, "Init space for %s: valimit=0x%llx, " nvgpu_log_info(g, "Init space for %s: valimit=0x%llx, "
"LP size=0x%x lowhole=0x%llx", "LP size=0x%x lowhole=0x%llx",
@@ -308,8 +316,9 @@ int __nvgpu_vm_init(struct mm_gk20a *mm,
vm->vma[gmmu_page_size_small] = &vm->user; vm->vma[gmmu_page_size_small] = &vm->user;
vm->vma[gmmu_page_size_big] = &vm->user; vm->vma[gmmu_page_size_big] = &vm->user;
vm->vma[gmmu_page_size_kernel] = &vm->kernel; vm->vma[gmmu_page_size_kernel] = &vm->kernel;
if (!nvgpu_is_enabled(g, NVGPU_MM_UNIFY_ADDRESS_SPACES)) if (!nvgpu_is_enabled(g, NVGPU_MM_UNIFY_ADDRESS_SPACES)) {
vm->vma[gmmu_page_size_big] = &vm->user_lp; vm->vma[gmmu_page_size_big] = &vm->user_lp;
}
vm->va_start = low_hole; vm->va_start = low_hole;
vm->va_limit = aperture_size; vm->va_limit = aperture_size;
@@ -332,8 +341,9 @@ int __nvgpu_vm_init(struct mm_gk20a *mm,
/* Initialize the page table data structures. */ /* Initialize the page table data structures. */
strncpy(vm->name, name, min(strlen(name), sizeof(vm->name))); strncpy(vm->name, name, min(strlen(name), sizeof(vm->name)));
err = nvgpu_gmmu_init_page_table(vm); err = nvgpu_gmmu_init_page_table(vm);
if (err) if (err) {
goto clean_up_vgpu_vm; goto clean_up_vgpu_vm;
}
/* Setup vma limits. */ /* Setup vma limits. */
if (kernel_reserved + low_hole < aperture_size) { if (kernel_reserved + low_hole < aperture_size) {
@@ -396,14 +406,15 @@ int __nvgpu_vm_init(struct mm_gk20a *mm,
* Determine if big pages are possible in this VM. If a split address * Determine if big pages are possible in this VM. If a split address
* space is used then check the user_lp vma instead of the user vma. * space is used then check the user_lp vma instead of the user vma.
*/ */
if (nvgpu_is_enabled(g, NVGPU_MM_UNIFY_ADDRESS_SPACES)) if (nvgpu_is_enabled(g, NVGPU_MM_UNIFY_ADDRESS_SPACES)) {
vm->big_pages = big_pages && vm->big_pages = big_pages &&
nvgpu_big_pages_possible(vm, user_vma_start, nvgpu_big_pages_possible(vm, user_vma_start,
user_vma_limit - user_vma_start); user_vma_limit - user_vma_start);
else } else {
vm->big_pages = big_pages && vm->big_pages = big_pages &&
nvgpu_big_pages_possible(vm, user_lp_vma_start, nvgpu_big_pages_possible(vm, user_lp_vma_start,
user_lp_vma_limit - user_lp_vma_start); user_lp_vma_limit - user_lp_vma_start);
}
/* /*
* User VMA. * User VMA.
@@ -418,8 +429,9 @@ int __nvgpu_vm_init(struct mm_gk20a *mm,
SZ_4K, SZ_4K,
GPU_BALLOC_MAX_ORDER, GPU_BALLOC_MAX_ORDER,
GPU_ALLOC_GVA_SPACE); GPU_ALLOC_GVA_SPACE);
if (err) if (err) {
goto clean_up_page_tables; goto clean_up_page_tables;
}
} else { } else {
/* /*
* Make these allocator pointers point to the kernel allocator * Make these allocator pointers point to the kernel allocator
@@ -443,9 +455,10 @@ int __nvgpu_vm_init(struct mm_gk20a *mm,
vm->big_page_size, vm->big_page_size,
GPU_BALLOC_MAX_ORDER, GPU_BALLOC_MAX_ORDER,
GPU_ALLOC_GVA_SPACE); GPU_ALLOC_GVA_SPACE);
if (err) if (err) {
goto clean_up_allocators; goto clean_up_allocators;
} }
}
/* /*
* Kernel VMA. Must always exist for an address space. * Kernel VMA. Must always exist for an address space.
@@ -458,8 +471,9 @@ int __nvgpu_vm_init(struct mm_gk20a *mm,
SZ_4K, SZ_4K,
GPU_BALLOC_MAX_ORDER, GPU_BALLOC_MAX_ORDER,
kernel_vma_flags); kernel_vma_flags);
if (err) if (err) {
goto clean_up_allocators; goto clean_up_allocators;
}
vm->mapped_buffers = NULL; vm->mapped_buffers = NULL;
@@ -475,19 +489,23 @@ int __nvgpu_vm_init(struct mm_gk20a *mm,
*/ */
if (vm->va_limit > 4ULL * SZ_1G) { if (vm->va_limit > 4ULL * SZ_1G) {
err = nvgpu_init_sema_pool(vm); err = nvgpu_init_sema_pool(vm);
if (err) if (err) {
goto clean_up_allocators; goto clean_up_allocators;
} }
}
return 0; return 0;
clean_up_allocators: clean_up_allocators:
if (nvgpu_alloc_initialized(&vm->kernel)) if (nvgpu_alloc_initialized(&vm->kernel)) {
nvgpu_alloc_destroy(&vm->kernel); nvgpu_alloc_destroy(&vm->kernel);
if (nvgpu_alloc_initialized(&vm->user)) }
if (nvgpu_alloc_initialized(&vm->user)) {
nvgpu_alloc_destroy(&vm->user); nvgpu_alloc_destroy(&vm->user);
if (nvgpu_alloc_initialized(&vm->user_lp)) }
if (nvgpu_alloc_initialized(&vm->user_lp)) {
nvgpu_alloc_destroy(&vm->user_lp); nvgpu_alloc_destroy(&vm->user_lp);
}
clean_up_page_tables: clean_up_page_tables:
/* Cleans up nvgpu_gmmu_init_page_table() */ /* Cleans up nvgpu_gmmu_init_page_table() */
__nvgpu_pd_cache_free_direct(g, &vm->pdb); __nvgpu_pd_cache_free_direct(g, &vm->pdb);
@@ -547,8 +565,9 @@ struct vm_gk20a *nvgpu_vm_init(struct gk20a *g,
{ {
struct vm_gk20a *vm = nvgpu_kzalloc(g, sizeof(*vm)); struct vm_gk20a *vm = nvgpu_kzalloc(g, sizeof(*vm));
if (!vm) if (!vm) {
return NULL; return NULL;
}
if (__nvgpu_vm_init(&g->mm, vm, big_page_size, low_hole, if (__nvgpu_vm_init(&g->mm, vm, big_page_size, low_hole,
kernel_reserved, aperture_size, big_pages, kernel_reserved, aperture_size, big_pages,
@@ -582,9 +601,10 @@ static void __nvgpu_vm_remove(struct vm_gk20a *vm)
} }
} }
if (nvgpu_mem_is_valid(&g->syncpt_mem) && vm->syncpt_ro_map_gpu_va) if (nvgpu_mem_is_valid(&g->syncpt_mem) && vm->syncpt_ro_map_gpu_va) {
nvgpu_gmmu_unmap(vm, &g->syncpt_mem, nvgpu_gmmu_unmap(vm, &g->syncpt_mem,
vm->syncpt_ro_map_gpu_va); vm->syncpt_ro_map_gpu_va);
}
nvgpu_mutex_acquire(&vm->update_gmmu_lock); nvgpu_mutex_acquire(&vm->update_gmmu_lock);
@@ -603,12 +623,15 @@ static void __nvgpu_vm_remove(struct vm_gk20a *vm)
nvgpu_kfree(vm->mm->g, vm_area); nvgpu_kfree(vm->mm->g, vm_area);
} }
if (nvgpu_alloc_initialized(&vm->kernel)) if (nvgpu_alloc_initialized(&vm->kernel)) {
nvgpu_alloc_destroy(&vm->kernel); nvgpu_alloc_destroy(&vm->kernel);
if (nvgpu_alloc_initialized(&vm->user)) }
if (nvgpu_alloc_initialized(&vm->user)) {
nvgpu_alloc_destroy(&vm->user); nvgpu_alloc_destroy(&vm->user);
if (nvgpu_alloc_initialized(&vm->user_lp)) }
if (nvgpu_alloc_initialized(&vm->user_lp)) {
nvgpu_alloc_destroy(&vm->user_lp); nvgpu_alloc_destroy(&vm->user_lp);
}
nvgpu_vm_free_entries(vm, &vm->pdb); nvgpu_vm_free_entries(vm, &vm->pdb);
@@ -664,8 +687,9 @@ struct nvgpu_mapped_buf *__nvgpu_vm_find_mapped_buf(
struct nvgpu_rbtree_node *root = vm->mapped_buffers; struct nvgpu_rbtree_node *root = vm->mapped_buffers;
nvgpu_rbtree_search(addr, &node, root); nvgpu_rbtree_search(addr, &node, root);
if (!node) if (!node) {
return NULL; return NULL;
}
return mapped_buffer_from_rbtree_node(node); return mapped_buffer_from_rbtree_node(node);
} }
@@ -677,8 +701,9 @@ struct nvgpu_mapped_buf *__nvgpu_vm_find_mapped_buf_range(
struct nvgpu_rbtree_node *root = vm->mapped_buffers; struct nvgpu_rbtree_node *root = vm->mapped_buffers;
nvgpu_rbtree_range_search(addr, &node, root); nvgpu_rbtree_range_search(addr, &node, root);
if (!node) if (!node) {
return NULL; return NULL;
}
return mapped_buffer_from_rbtree_node(node); return mapped_buffer_from_rbtree_node(node);
} }
@@ -690,8 +715,9 @@ struct nvgpu_mapped_buf *__nvgpu_vm_find_mapped_buf_less_than(
struct nvgpu_rbtree_node *root = vm->mapped_buffers; struct nvgpu_rbtree_node *root = vm->mapped_buffers;
nvgpu_rbtree_less_than_search(addr, &node, root); nvgpu_rbtree_less_than_search(addr, &node, root);
if (!node) if (!node) {
return NULL; return NULL;
}
return mapped_buffer_from_rbtree_node(node); return mapped_buffer_from_rbtree_node(node);
} }
@@ -746,8 +772,9 @@ void nvgpu_vm_put_buffers(struct vm_gk20a *vm,
int i; int i;
struct vm_gk20a_mapping_batch batch; struct vm_gk20a_mapping_batch batch;
if (num_buffers == 0) if (num_buffers == 0) {
return; return;
}
nvgpu_mutex_acquire(&vm->update_gmmu_lock); nvgpu_mutex_acquire(&vm->update_gmmu_lock);
nvgpu_vm_mapping_batch_start(&batch); nvgpu_vm_mapping_batch_start(&batch);
@@ -814,10 +841,11 @@ struct nvgpu_mapped_buf *nvgpu_vm_map(struct vm_gk20a *vm,
compr_kind : NVGPU_KIND_INVALID); compr_kind : NVGPU_KIND_INVALID);
binfo.incompr_kind = incompr_kind; binfo.incompr_kind = incompr_kind;
if (compr_kind != NVGPU_KIND_INVALID) if (compr_kind != NVGPU_KIND_INVALID) {
map_key_kind = compr_kind; map_key_kind = compr_kind;
else } else {
map_key_kind = incompr_kind; map_key_kind = incompr_kind;
}
/* /*
* Check if this buffer is already mapped. * Check if this buffer is already mapped.
@@ -847,11 +875,12 @@ struct nvgpu_mapped_buf *nvgpu_vm_map(struct vm_gk20a *vm,
} }
align = nvgpu_sgt_alignment(g, sgt); align = nvgpu_sgt_alignment(g, sgt);
if (g->mm.disable_bigpage) if (g->mm.disable_bigpage) {
binfo.pgsz_idx = gmmu_page_size_small; binfo.pgsz_idx = gmmu_page_size_small;
else } else {
binfo.pgsz_idx = __get_pte_size(vm, map_addr, binfo.pgsz_idx = __get_pte_size(vm, map_addr,
min_t(u64, binfo.size, align)); min_t(u64, binfo.size, align));
}
map_size = map_size ? map_size : binfo.size; map_size = map_size ? map_size : binfo.size;
map_size = ALIGN(map_size, SZ_4K); map_size = ALIGN(map_size, SZ_4K);
@@ -872,8 +901,9 @@ struct nvgpu_mapped_buf *nvgpu_vm_map(struct vm_gk20a *vm,
map_size, map_size,
binfo.pgsz_idx, binfo.pgsz_idx,
&vm_area); &vm_area);
if (err) if (err) {
goto clean_up; goto clean_up;
}
va_allocated = false; va_allocated = false;
} }
@@ -941,9 +971,10 @@ struct nvgpu_mapped_buf *nvgpu_vm_map(struct vm_gk20a *vm,
comptags.lines - 1)); comptags.lines - 1));
gk20a_comptags_finish_clear( gk20a_comptags_finish_clear(
os_buf, err == 0); os_buf, err == 0);
if (err) if (err) {
goto clean_up; goto clean_up;
} }
}
} else { } else {
/* /*
* Cleared as part of gmmu map * Cleared as part of gmmu map
@@ -955,9 +986,10 @@ struct nvgpu_mapped_buf *nvgpu_vm_map(struct vm_gk20a *vm,
/* /*
* Store the ctag offset for later use if we got the comptags * Store the ctag offset for later use if we got the comptags
*/ */
if (comptags.lines) if (comptags.lines) {
ctag_offset = comptags.offset; ctag_offset = comptags.offset;
} }
}
/* /*
* Figure out the kind and ctag offset for the GMMU page tables * Figure out the kind and ctag offset for the GMMU page tables
@@ -984,8 +1016,9 @@ struct nvgpu_mapped_buf *nvgpu_vm_map(struct vm_gk20a *vm,
goto clean_up; goto clean_up;
} }
if (clear_ctags) if (clear_ctags) {
clear_ctags = gk20a_comptags_start_clear(os_buf); clear_ctags = gk20a_comptags_start_clear(os_buf);
}
map_addr = g->ops.mm.gmmu_map(vm, map_addr = g->ops.mm.gmmu_map(vm,
map_addr, map_addr,
@@ -1003,8 +1036,9 @@ struct nvgpu_mapped_buf *nvgpu_vm_map(struct vm_gk20a *vm,
batch, batch,
aperture); aperture);
if (clear_ctags) if (clear_ctags) {
gk20a_comptags_finish_clear(os_buf, map_addr != 0); gk20a_comptags_finish_clear(os_buf, map_addr != 0);
}
if (!map_addr) { if (!map_addr) {
err = -ENOMEM; err = -ENOMEM;
@@ -1041,7 +1075,7 @@ struct nvgpu_mapped_buf *nvgpu_vm_map(struct vm_gk20a *vm,
return mapped_buffer; return mapped_buffer;
clean_up: clean_up:
if (mapped_buffer->addr) if (mapped_buffer->addr) {
g->ops.mm.gmmu_unmap(vm, g->ops.mm.gmmu_unmap(vm,
mapped_buffer->addr, mapped_buffer->addr,
mapped_buffer->size, mapped_buffer->size,
@@ -1051,6 +1085,7 @@ clean_up:
mapped_buffer->vm_area ? mapped_buffer->vm_area ?
mapped_buffer->vm_area->sparse : false, mapped_buffer->vm_area->sparse : false,
NULL); NULL);
}
nvgpu_mutex_release(&vm->update_gmmu_lock); nvgpu_mutex_release(&vm->update_gmmu_lock);
clean_up_nolock: clean_up_nolock:
nvgpu_kfree(g, mapped_buffer); nvgpu_kfree(g, mapped_buffer);
@@ -1132,14 +1167,16 @@ static int nvgpu_vm_unmap_sync_buffer(struct vm_gk20a *vm,
nvgpu_timeout_init(vm->mm->g, &timeout, 50, NVGPU_TIMER_CPU_TIMER); nvgpu_timeout_init(vm->mm->g, &timeout, 50, NVGPU_TIMER_CPU_TIMER);
do { do {
if (nvgpu_atomic_read(&mapped_buffer->ref.refcount) == 1) if (nvgpu_atomic_read(&mapped_buffer->ref.refcount) == 1) {
break; break;
}
nvgpu_msleep(10); nvgpu_msleep(10);
} while (!nvgpu_timeout_expired_msg(&timeout, } while (!nvgpu_timeout_expired_msg(&timeout,
"sync-unmap failed on 0x%llx")); "sync-unmap failed on 0x%llx"));
if (nvgpu_timeout_expired(&timeout)) if (nvgpu_timeout_expired(&timeout)) {
ret = -ETIMEDOUT; ret = -ETIMEDOUT;
}
nvgpu_mutex_acquire(&vm->update_gmmu_lock); nvgpu_mutex_acquire(&vm->update_gmmu_lock);
@@ -1154,17 +1191,19 @@ void nvgpu_vm_unmap(struct vm_gk20a *vm, u64 offset,
nvgpu_mutex_acquire(&vm->update_gmmu_lock); nvgpu_mutex_acquire(&vm->update_gmmu_lock);
mapped_buffer = __nvgpu_vm_find_mapped_buf(vm, offset); mapped_buffer = __nvgpu_vm_find_mapped_buf(vm, offset);
if (!mapped_buffer) if (!mapped_buffer) {
goto done; goto done;
}
if (mapped_buffer->flags & NVGPU_VM_MAP_FIXED_OFFSET) { if (mapped_buffer->flags & NVGPU_VM_MAP_FIXED_OFFSET) {
if (nvgpu_vm_unmap_sync_buffer(vm, mapped_buffer)) if (nvgpu_vm_unmap_sync_buffer(vm, mapped_buffer)) {
/* /*
* Looks like we have failed... Better not continue in * Looks like we have failed... Better not continue in
* case the buffer is in use. * case the buffer is in use.
*/ */
goto done; goto done;
} }
}
/* /*
* Make sure we have access to the batch if we end up calling through to * Make sure we have access to the batch if we end up calling through to