mirror of
git://nv-tegra.nvidia.com/linux-nvgpu.git
synced 2025-12-22 17:36:20 +03:00
gpu: nvgpu: common: mm: Fix MISRA 15.6 violations
MISRA Rule-15.6 requires that all if-else blocks be enclosed in braces, including single statement blocks. Fix errors due to single statement if blocks without braces, introducing the braces. JIRA NVGPU-671 Change-Id: Ieeecf719dca9acc1a116d2893637bf770caf4f5b Signed-off-by: Srirangan <smadhavan@nvidia.com> Reviewed-on: https://git-master.nvidia.com/r/1794241 GVS: Gerrit_Virtual_Submit Reviewed-by: Adeel Raza <araza@nvidia.com> Reviewed-by: Alex Waterman <alexw@nvidia.com> Reviewed-by: mobile promotions <svcmobile_promotions@nvidia.com> Tested-by: mobile promotions <svcmobile_promotions@nvidia.com>
This commit is contained in:
committed by
mobile promotions
parent
553fdf3534
commit
70c20bb75b
@@ -79,10 +79,12 @@ static void balloc_compute_max_order(struct nvgpu_buddy_allocator *a)
|
||||
return;
|
||||
}
|
||||
|
||||
if (a->max_order > true_max_order)
|
||||
if (a->max_order > true_max_order) {
|
||||
a->max_order = true_max_order;
|
||||
if (a->max_order > GPU_BALLOC_MAX_ORDER)
|
||||
}
|
||||
if (a->max_order > GPU_BALLOC_MAX_ORDER) {
|
||||
a->max_order = GPU_BALLOC_MAX_ORDER;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -108,8 +110,9 @@ static struct nvgpu_buddy *balloc_new_buddy(struct nvgpu_buddy_allocator *a,
|
||||
struct nvgpu_buddy *new_buddy;
|
||||
|
||||
new_buddy = nvgpu_kmem_cache_alloc(a->buddy_cache);
|
||||
if (!new_buddy)
|
||||
if (!new_buddy) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
memset(new_buddy, 0, sizeof(struct nvgpu_buddy));
|
||||
|
||||
@@ -139,10 +142,11 @@ static void __balloc_buddy_list_add(struct nvgpu_buddy_allocator *a,
|
||||
* without cycling through the entire list.
|
||||
*/
|
||||
if (a->flags & GPU_ALLOC_GVA_SPACE &&
|
||||
b->pte_size == gmmu_page_size_big)
|
||||
b->pte_size == gmmu_page_size_big) {
|
||||
nvgpu_list_add_tail(&b->buddy_entry, list);
|
||||
else
|
||||
} else {
|
||||
nvgpu_list_add(&b->buddy_entry, list);
|
||||
}
|
||||
|
||||
buddy_set_in_list(b);
|
||||
}
|
||||
@@ -181,8 +185,9 @@ static void balloc_blist_rem(struct nvgpu_buddy_allocator *a,
|
||||
|
||||
static u64 balloc_get_order(struct nvgpu_buddy_allocator *a, u64 len)
|
||||
{
|
||||
if (len == 0)
|
||||
if (len == 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
len--;
|
||||
len >>= a->blk_shift;
|
||||
@@ -195,10 +200,11 @@ static u64 __balloc_max_order_in(struct nvgpu_buddy_allocator *a,
|
||||
{
|
||||
u64 size = (end - start) >> a->blk_shift;
|
||||
|
||||
if (size > 0)
|
||||
if (size > 0) {
|
||||
return min_t(u64, ilog2(size), a->max_order);
|
||||
else
|
||||
} else {
|
||||
return GPU_BALLOC_MAX_ORDER;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -222,8 +228,9 @@ static int balloc_init_lists(struct nvgpu_buddy_allocator *a)
|
||||
order = __balloc_max_order_in(a, bstart, bend);
|
||||
|
||||
buddy = balloc_new_buddy(a, NULL, bstart, order);
|
||||
if (!buddy)
|
||||
if (!buddy) {
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
balloc_blist_add(a, buddy);
|
||||
bstart += balloc_order_to_len(a, order);
|
||||
@@ -340,17 +347,20 @@ static void balloc_coalesce(struct nvgpu_buddy_allocator *a,
|
||||
{
|
||||
struct nvgpu_buddy *parent;
|
||||
|
||||
if (buddy_is_alloced(b) || buddy_is_split(b))
|
||||
if (buddy_is_alloced(b) || buddy_is_split(b)) {
|
||||
return;
|
||||
}
|
||||
|
||||
/*
|
||||
* If both our buddy and I are both not allocated and not split then
|
||||
* we can coalesce ourselves.
|
||||
*/
|
||||
if (!b->buddy)
|
||||
if (!b->buddy) {
|
||||
return;
|
||||
if (buddy_is_alloced(b->buddy) || buddy_is_split(b->buddy))
|
||||
}
|
||||
if (buddy_is_alloced(b->buddy) || buddy_is_split(b->buddy)) {
|
||||
return;
|
||||
}
|
||||
|
||||
parent = b->parent;
|
||||
|
||||
@@ -383,8 +393,9 @@ static int balloc_split_buddy(struct nvgpu_buddy_allocator *a,
|
||||
u64 half;
|
||||
|
||||
left = balloc_new_buddy(a, b, b->start, b->order - 1);
|
||||
if (!left)
|
||||
if (!left) {
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
half = (b->end - b->start) / 2;
|
||||
|
||||
@@ -449,8 +460,9 @@ static struct nvgpu_buddy *balloc_free_buddy(struct nvgpu_buddy_allocator *a,
|
||||
struct nvgpu_buddy *bud;
|
||||
|
||||
nvgpu_rbtree_search(addr, &node, a->alloced_buddies);
|
||||
if (!node)
|
||||
if (!node) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
bud = nvgpu_buddy_from_rbtree_node(node);
|
||||
|
||||
@@ -470,21 +482,24 @@ static struct nvgpu_buddy *__balloc_find_buddy(struct nvgpu_buddy_allocator *a,
|
||||
struct nvgpu_buddy *bud;
|
||||
|
||||
if (order > a->max_order ||
|
||||
nvgpu_list_empty(balloc_get_order_list(a, order)))
|
||||
nvgpu_list_empty(balloc_get_order_list(a, order))) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (a->flags & GPU_ALLOC_GVA_SPACE &&
|
||||
pte_size == gmmu_page_size_big)
|
||||
pte_size == gmmu_page_size_big) {
|
||||
bud = nvgpu_list_last_entry(balloc_get_order_list(a, order),
|
||||
nvgpu_buddy, buddy_entry);
|
||||
else
|
||||
} else {
|
||||
bud = nvgpu_list_first_entry(balloc_get_order_list(a, order),
|
||||
nvgpu_buddy, buddy_entry);
|
||||
}
|
||||
|
||||
if (pte_size != BALLOC_PTE_SIZE_ANY &&
|
||||
pte_size != bud->pte_size &&
|
||||
bud->pte_size != BALLOC_PTE_SIZE_ANY)
|
||||
bud->pte_size != BALLOC_PTE_SIZE_ANY) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
return bud;
|
||||
}
|
||||
@@ -511,12 +526,14 @@ static u64 __balloc_do_alloc(struct nvgpu_buddy_allocator *a,
|
||||
}
|
||||
|
||||
/* Out of memory! */
|
||||
if (!bud)
|
||||
if (!bud) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
while (bud->order != order) {
|
||||
if (balloc_split_buddy(a, bud, pte_size))
|
||||
if (balloc_split_buddy(a, bud, pte_size)) {
|
||||
return 0; /* No mem... */
|
||||
}
|
||||
bud = bud->left;
|
||||
}
|
||||
|
||||
@@ -540,19 +557,22 @@ static int balloc_is_range_free(struct nvgpu_buddy_allocator *a,
|
||||
struct nvgpu_buddy *bud;
|
||||
|
||||
nvgpu_rbtree_enum_start(0, &node, a->alloced_buddies);
|
||||
if (!node)
|
||||
if (!node) {
|
||||
return 1; /* No allocs yet. */
|
||||
}
|
||||
|
||||
bud = nvgpu_buddy_from_rbtree_node(node);
|
||||
|
||||
while (bud->start < end) {
|
||||
if ((bud->start > base && bud->start < end) ||
|
||||
(bud->end > base && bud->end < end))
|
||||
(bud->end > base && bud->end < end)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
nvgpu_rbtree_enum_next(&node, node);
|
||||
if (!node)
|
||||
if (!node) {
|
||||
break;
|
||||
}
|
||||
bud = nvgpu_buddy_from_rbtree_node(node);
|
||||
}
|
||||
|
||||
@@ -581,8 +601,9 @@ static struct nvgpu_fixed_alloc *balloc_free_fixed(
|
||||
struct nvgpu_rbtree_node *node = NULL;
|
||||
|
||||
nvgpu_rbtree_search(addr, &node, a->fixed_allocs);
|
||||
if (!node)
|
||||
if (!node) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
falloc = nvgpu_fixed_alloc_from_rbtree_node(node);
|
||||
|
||||
@@ -657,8 +678,9 @@ static struct nvgpu_buddy *__balloc_make_fixed_buddy(
|
||||
}
|
||||
}
|
||||
|
||||
if (found)
|
||||
if (found) {
|
||||
break;
|
||||
}
|
||||
|
||||
__balloc_get_parent_range(a, cur_base, cur_order,
|
||||
&cur_base, &cur_order);
|
||||
@@ -679,10 +701,11 @@ static struct nvgpu_buddy *__balloc_make_fixed_buddy(
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (base < bud->right->start)
|
||||
if (base < bud->right->start) {
|
||||
bud = bud->left;
|
||||
else
|
||||
} else {
|
||||
bud = bud->right;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -697,12 +720,13 @@ static u64 __balloc_do_alloc_fixed(struct nvgpu_buddy_allocator *a,
|
||||
u64 align_order;
|
||||
|
||||
shifted_base = balloc_base_shift(a, base);
|
||||
if (shifted_base == 0)
|
||||
if (shifted_base == 0) {
|
||||
align_order = __fls(len >> a->blk_shift);
|
||||
else
|
||||
} else {
|
||||
align_order = min_t(u64,
|
||||
__ffs(shifted_base >> a->blk_shift),
|
||||
__fls(len >> a->blk_shift));
|
||||
}
|
||||
|
||||
if (align_order > a->max_order) {
|
||||
alloc_dbg(balloc_owner(a),
|
||||
@@ -741,9 +765,10 @@ static u64 __balloc_do_alloc_fixed(struct nvgpu_buddy_allocator *a,
|
||||
align_order = __ffs(inc_base >> a->blk_shift);
|
||||
|
||||
/* If we don't have much left - trim down align_order. */
|
||||
if (balloc_order_to_len(a, align_order) > remaining)
|
||||
if (balloc_order_to_len(a, align_order) > remaining) {
|
||||
align_order = __balloc_max_order_in(a, inc_base,
|
||||
inc_base + remaining);
|
||||
}
|
||||
}
|
||||
|
||||
return base;
|
||||
@@ -805,10 +830,11 @@ static u64 nvgpu_buddy_balloc(struct nvgpu_allocator *__a, u64 len)
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (a->flags & GPU_ALLOC_GVA_SPACE)
|
||||
if (a->flags & GPU_ALLOC_GVA_SPACE) {
|
||||
pte_size = __get_pte_size(a->vm, 0, len);
|
||||
else
|
||||
} else {
|
||||
pte_size = BALLOC_PTE_SIZE_ANY;
|
||||
}
|
||||
|
||||
addr = __balloc_do_alloc(a, order, pte_size);
|
||||
|
||||
@@ -845,25 +871,29 @@ static u64 __nvgpu_balloc_fixed_buddy(struct nvgpu_allocator *__a,
|
||||
struct nvgpu_buddy_allocator *a = __a->priv;
|
||||
|
||||
/* If base isn't aligned to an order 0 block, fail. */
|
||||
if (base & (a->blk_size - 1))
|
||||
if (base & (a->blk_size - 1)) {
|
||||
goto fail;
|
||||
}
|
||||
|
||||
if (len == 0)
|
||||
if (len == 0) {
|
||||
goto fail;
|
||||
}
|
||||
|
||||
/* Check that the page size is valid. */
|
||||
if (a->flags & GPU_ALLOC_GVA_SPACE && a->vm->big_pages) {
|
||||
if (page_size == a->vm->big_page_size)
|
||||
if (page_size == a->vm->big_page_size) {
|
||||
pte_size = gmmu_page_size_big;
|
||||
else if (page_size == SZ_4K)
|
||||
} else if (page_size == SZ_4K) {
|
||||
pte_size = gmmu_page_size_small;
|
||||
else
|
||||
} else {
|
||||
goto fail;
|
||||
}
|
||||
}
|
||||
|
||||
falloc = nvgpu_kmalloc(nvgpu_alloc_to_gpu(__a), sizeof(*falloc));
|
||||
if (!falloc)
|
||||
if (!falloc) {
|
||||
goto fail;
|
||||
}
|
||||
|
||||
nvgpu_init_list_node(&falloc->buddies);
|
||||
falloc->start = base;
|
||||
@@ -936,8 +966,9 @@ static void nvgpu_buddy_bfree(struct nvgpu_allocator *__a, u64 addr)
|
||||
struct nvgpu_fixed_alloc *falloc;
|
||||
struct nvgpu_buddy_allocator *a = __a->priv;
|
||||
|
||||
if (!addr)
|
||||
if (!addr) {
|
||||
return;
|
||||
}
|
||||
|
||||
alloc_lock(__a);
|
||||
|
||||
@@ -952,8 +983,9 @@ static void nvgpu_buddy_bfree(struct nvgpu_allocator *__a, u64 addr)
|
||||
}
|
||||
|
||||
bud = balloc_free_buddy(a, addr);
|
||||
if (!bud)
|
||||
if (!bud) {
|
||||
goto done;
|
||||
}
|
||||
|
||||
balloc_blist_add(a, bud);
|
||||
a->bytes_freed += balloc_order_to_len(a, bud->order);
|
||||
@@ -987,8 +1019,9 @@ static bool nvgpu_buddy_reserve_is_possible(struct nvgpu_buddy_allocator *a,
|
||||
if ((co_base >= tmp->base &&
|
||||
co_base < (tmp->base + tmp->length)) ||
|
||||
(co_end >= tmp->base &&
|
||||
co_end < (tmp->base + tmp->length)))
|
||||
co_end < (tmp->base + tmp->length))) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
@@ -1006,8 +1039,9 @@ static int nvgpu_buddy_reserve_co(struct nvgpu_allocator *__a,
|
||||
int err = 0;
|
||||
|
||||
if (co->base < a->start || (co->base + co->length) > a->end ||
|
||||
a->alloc_made)
|
||||
a->alloc_made) {
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
alloc_lock(__a);
|
||||
|
||||
@@ -1221,25 +1255,31 @@ int __nvgpu_buddy_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a,
|
||||
struct nvgpu_buddy_allocator *a;
|
||||
|
||||
/* blk_size must be greater than 0 and a power of 2. */
|
||||
if (blk_size == 0)
|
||||
if (blk_size == 0) {
|
||||
return -EINVAL;
|
||||
if (blk_size & (blk_size - 1))
|
||||
}
|
||||
if (blk_size & (blk_size - 1)) {
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
if (max_order > GPU_BALLOC_MAX_ORDER)
|
||||
if (max_order > GPU_BALLOC_MAX_ORDER) {
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
/* If this is to manage a GVA space we need a VM. */
|
||||
if (flags & GPU_ALLOC_GVA_SPACE && !vm)
|
||||
if (flags & GPU_ALLOC_GVA_SPACE && !vm) {
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
a = nvgpu_kzalloc(g, sizeof(struct nvgpu_buddy_allocator));
|
||||
if (!a)
|
||||
if (!a) {
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
err = __nvgpu_alloc_common_init(__a, g, name, a, false, &buddy_ops);
|
||||
if (err)
|
||||
if (err) {
|
||||
goto fail;
|
||||
}
|
||||
|
||||
a->base = base;
|
||||
a->length = size;
|
||||
@@ -1269,8 +1309,9 @@ int __nvgpu_buddy_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a,
|
||||
*/
|
||||
if (flags & GPU_ALLOC_GVA_SPACE && vm->big_pages &&
|
||||
(base & ((vm->big_page_size << 10) - 1) ||
|
||||
size & ((vm->big_page_size << 10) - 1)))
|
||||
size & ((vm->big_page_size << 10) - 1))) {
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
a->flags = flags;
|
||||
a->max_order = max_order;
|
||||
@@ -1288,8 +1329,9 @@ int __nvgpu_buddy_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a,
|
||||
a->fixed_allocs = NULL;
|
||||
nvgpu_init_list_node(&a->co_list);
|
||||
err = balloc_init_lists(a);
|
||||
if (err)
|
||||
if (err) {
|
||||
goto fail;
|
||||
}
|
||||
|
||||
nvgpu_smp_wmb();
|
||||
a->initialized = 1;
|
||||
@@ -1301,18 +1343,20 @@ int __nvgpu_buddy_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a,
|
||||
alloc_dbg(__a, " base 0x%llx", a->base);
|
||||
alloc_dbg(__a, " size 0x%llx", a->length);
|
||||
alloc_dbg(__a, " blk_size 0x%llx", a->blk_size);
|
||||
if (flags & GPU_ALLOC_GVA_SPACE)
|
||||
if (flags & GPU_ALLOC_GVA_SPACE) {
|
||||
alloc_dbg(balloc_owner(a),
|
||||
" pde_size 0x%llx",
|
||||
balloc_order_to_len(a, a->pte_blk_order));
|
||||
}
|
||||
alloc_dbg(__a, " max_order %llu", a->max_order);
|
||||
alloc_dbg(__a, " flags 0x%llx", a->flags);
|
||||
|
||||
return 0;
|
||||
|
||||
fail:
|
||||
if (a->buddy_cache)
|
||||
if (a->buddy_cache) {
|
||||
nvgpu_kmem_cache_destroy(a->buddy_cache);
|
||||
}
|
||||
nvgpu_kfree(g, a);
|
||||
return err;
|
||||
}
|
||||
|
||||
@@ -38,18 +38,20 @@
|
||||
|
||||
#define __gmmu_dbg(g, attrs, fmt, args...) \
|
||||
do { \
|
||||
if (attrs->debug) \
|
||||
if (attrs->debug) { \
|
||||
nvgpu_info(g, fmt, ##args); \
|
||||
else \
|
||||
} else { \
|
||||
nvgpu_log(g, gpu_dbg_map, fmt, ##args); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
#define __gmmu_dbg_v(g, attrs, fmt, args...) \
|
||||
do { \
|
||||
if (attrs->debug) \
|
||||
if (attrs->debug) { \
|
||||
nvgpu_info(g, fmt, ##args); \
|
||||
else \
|
||||
} else { \
|
||||
nvgpu_log(g, gpu_dbg_map_v, fmt, ##args); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
static int pd_allocate(struct vm_gk20a *vm,
|
||||
@@ -77,15 +79,17 @@ static u64 __nvgpu_gmmu_map(struct vm_gk20a *vm,
|
||||
|
||||
struct nvgpu_sgt *sgt = nvgpu_sgt_create_from_mem(g, mem);
|
||||
|
||||
if (!sgt)
|
||||
if (!sgt) {
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
/*
|
||||
* If the GPU is IO coherent and the DMA API is giving us IO coherent
|
||||
* CPU mappings then we gotta make sure we use the IO coherent aperture.
|
||||
*/
|
||||
if (nvgpu_is_enabled(g, NVGPU_USE_COHERENT_SYSMEM))
|
||||
if (nvgpu_is_enabled(g, NVGPU_USE_COHERENT_SYSMEM)) {
|
||||
flags |= NVGPU_VM_MAP_IO_COHERENT;
|
||||
}
|
||||
|
||||
/*
|
||||
* Later on, when we free this nvgpu_mem's GPU mapping, we are going to
|
||||
@@ -94,10 +98,11 @@ static u64 __nvgpu_gmmu_map(struct vm_gk20a *vm,
|
||||
* therefor we should not try and free it. But otherwise, if we do
|
||||
* manage the VA alloc, we obviously must free it.
|
||||
*/
|
||||
if (addr != 0)
|
||||
if (addr != 0) {
|
||||
mem->free_gpu_va = false;
|
||||
else
|
||||
} else {
|
||||
mem->free_gpu_va = true;
|
||||
}
|
||||
|
||||
nvgpu_mutex_acquire(&vm->update_gmmu_lock);
|
||||
vaddr = g->ops.mm.gmmu_map(vm, addr,
|
||||
@@ -196,8 +201,9 @@ int nvgpu_gmmu_init_page_table(struct vm_gk20a *vm)
|
||||
pdb_size = ALIGN(pd_size(&vm->mmu_levels[0], &attrs), PAGE_SIZE);
|
||||
|
||||
err = __nvgpu_pd_cache_alloc_direct(vm->mm->g, &vm->pdb, pdb_size);
|
||||
if (WARN_ON(err))
|
||||
if (WARN_ON(err)) {
|
||||
return err;
|
||||
}
|
||||
|
||||
/*
|
||||
* One nvgpu_mb() is done after all mapping operations. Don't need
|
||||
@@ -267,8 +273,9 @@ static int pd_allocate(struct vm_gk20a *vm,
|
||||
{
|
||||
int err;
|
||||
|
||||
if (pd->mem)
|
||||
if (pd->mem) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
err = __nvgpu_pd_alloc(vm, pd, pd_size(l, attrs));
|
||||
if (err) {
|
||||
@@ -310,14 +317,16 @@ static int pd_allocate_children(struct vm_gk20a *vm,
|
||||
{
|
||||
struct gk20a *g = gk20a_from_vm(vm);
|
||||
|
||||
if (pd->entries)
|
||||
if (pd->entries) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
pd->num_entries = pd_entries(l, attrs);
|
||||
pd->entries = nvgpu_vzalloc(g, sizeof(struct nvgpu_gmmu_pd) *
|
||||
pd->num_entries);
|
||||
if (!pd->entries)
|
||||
if (!pd->entries) {
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -398,8 +407,9 @@ static int __set_pd_level(struct vm_gk20a *vm,
|
||||
* have a bunch of children PDs.
|
||||
*/
|
||||
if (next_l->update_entry) {
|
||||
if (pd_allocate_children(vm, l, pd, attrs))
|
||||
if (pd_allocate_children(vm, l, pd, attrs)) {
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
/*
|
||||
* Get the next PD so that we know what to put in this
|
||||
@@ -412,8 +422,9 @@ static int __set_pd_level(struct vm_gk20a *vm,
|
||||
/*
|
||||
* Allocate the backing memory for next_pd.
|
||||
*/
|
||||
if (pd_allocate(vm, next_pd, next_l, attrs))
|
||||
if (pd_allocate(vm, next_pd, next_l, attrs)) {
|
||||
return -ENOMEM;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -440,8 +451,9 @@ static int __set_pd_level(struct vm_gk20a *vm,
|
||||
chunk_size,
|
||||
attrs);
|
||||
|
||||
if (err)
|
||||
if (err) {
|
||||
return err;
|
||||
}
|
||||
}
|
||||
|
||||
virt_addr += chunk_size;
|
||||
@@ -452,8 +464,9 @@ static int __set_pd_level(struct vm_gk20a *vm,
|
||||
* non-zero phys addresses in the PTEs. A non-zero phys-addr
|
||||
* would also confuse the lower level PTE programming code.
|
||||
*/
|
||||
if (phys_addr)
|
||||
if (phys_addr) {
|
||||
phys_addr += chunk_size;
|
||||
}
|
||||
length -= chunk_size;
|
||||
}
|
||||
|
||||
@@ -547,8 +560,9 @@ static int __nvgpu_gmmu_do_update_page_table(struct vm_gk20a *vm,
|
||||
virt_addr,
|
||||
chunk_length,
|
||||
attrs);
|
||||
if (err)
|
||||
if (err) {
|
||||
break;
|
||||
}
|
||||
|
||||
/* Space has been skipped so zero this for future chunks. */
|
||||
space_to_skip = 0;
|
||||
@@ -559,8 +573,9 @@ static int __nvgpu_gmmu_do_update_page_table(struct vm_gk20a *vm,
|
||||
virt_addr += chunk_length;
|
||||
length -= chunk_length;
|
||||
|
||||
if (length == 0)
|
||||
if (length == 0) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return err;
|
||||
@@ -594,13 +609,15 @@ static int __nvgpu_gmmu_update_page_table(struct vm_gk20a *vm,
|
||||
|
||||
/* note: here we need to map kernel to small, since the
|
||||
* low-level mmu code assumes 0 is small and 1 is big pages */
|
||||
if (attrs->pgsz == gmmu_page_size_kernel)
|
||||
if (attrs->pgsz == gmmu_page_size_kernel) {
|
||||
attrs->pgsz = gmmu_page_size_small;
|
||||
}
|
||||
|
||||
page_size = vm->gmmu_page_sizes[attrs->pgsz];
|
||||
|
||||
if (space_to_skip & (page_size - 1))
|
||||
if (space_to_skip & (page_size - 1)) {
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
/*
|
||||
* Update length to be aligned to the passed page size.
|
||||
@@ -692,8 +709,9 @@ u64 gk20a_locked_gmmu_map(struct vm_gk20a *vm,
|
||||
* the programmed ctagline gets increased at compression_page_size
|
||||
* boundaries.
|
||||
*/
|
||||
if (attrs.ctag)
|
||||
if (attrs.ctag) {
|
||||
attrs.ctag += buffer_offset & (ctag_granularity - 1U);
|
||||
}
|
||||
|
||||
attrs.l3_alloc = (bool)(flags & NVGPU_VM_MAP_L3_ALLOC);
|
||||
|
||||
@@ -701,8 +719,9 @@ u64 gk20a_locked_gmmu_map(struct vm_gk20a *vm,
|
||||
* Handle the IO coherency aperture: make sure the .aperture field is
|
||||
* correct based on the IO coherency flag.
|
||||
*/
|
||||
if (attrs.coherent && attrs.aperture == APERTURE_SYSMEM)
|
||||
if (attrs.coherent && attrs.aperture == APERTURE_SYSMEM) {
|
||||
attrs.aperture = __APERTURE_SYSMEM_COH;
|
||||
}
|
||||
|
||||
/*
|
||||
* Only allocate a new GPU VA range if we haven't already been passed a
|
||||
@@ -725,16 +744,18 @@ u64 gk20a_locked_gmmu_map(struct vm_gk20a *vm,
|
||||
goto fail_validate;
|
||||
}
|
||||
|
||||
if (!batch)
|
||||
if (!batch) {
|
||||
g->ops.fb.tlb_invalidate(g, vm->pdb.mem);
|
||||
else
|
||||
} else {
|
||||
batch->need_tlb_invalidate = true;
|
||||
}
|
||||
|
||||
return vaddr;
|
||||
|
||||
fail_validate:
|
||||
if (allocated)
|
||||
if (allocated) {
|
||||
__nvgpu_vm_free_va(vm, vaddr, pgsz_idx);
|
||||
}
|
||||
fail_alloc:
|
||||
nvgpu_err(g, "%s: failed with err=%d", __func__, err);
|
||||
return 0;
|
||||
@@ -775,8 +796,9 @@ void gk20a_locked_gmmu_unmap(struct vm_gk20a *vm,
|
||||
/* unmap here needs to know the page size we assigned at mapping */
|
||||
err = __nvgpu_gmmu_update_page_table(vm, NULL, 0,
|
||||
vaddr, size, &attrs);
|
||||
if (err)
|
||||
if (err) {
|
||||
nvgpu_err(g, "failed to update gmmu ptes on unmap");
|
||||
}
|
||||
|
||||
if (!batch) {
|
||||
gk20a_mm_l2_flush(g, true);
|
||||
@@ -801,8 +823,9 @@ u32 __nvgpu_pte_words(struct gk20a *g)
|
||||
*/
|
||||
do {
|
||||
next_l = l + 1;
|
||||
if (!next_l->update_entry)
|
||||
if (!next_l->update_entry) {
|
||||
break;
|
||||
}
|
||||
|
||||
l++;
|
||||
} while (true);
|
||||
@@ -836,13 +859,15 @@ static int __nvgpu_locate_pte(struct gk20a *g, struct vm_gk20a *vm,
|
||||
struct nvgpu_gmmu_pd *pd_next = pd->entries + pd_idx;
|
||||
|
||||
/* Invalid entry! */
|
||||
if (!pd_next->mem)
|
||||
if (!pd_next->mem) {
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
attrs->pgsz = l->get_pgsz(g, l, pd, pd_idx);
|
||||
|
||||
if (attrs->pgsz >= gmmu_nr_page_sizes)
|
||||
if (attrs->pgsz >= gmmu_nr_page_sizes) {
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
return __nvgpu_locate_pte(g, vm, pd_next,
|
||||
vaddr, lvl + 1, attrs,
|
||||
@@ -850,8 +875,9 @@ static int __nvgpu_locate_pte(struct gk20a *g, struct vm_gk20a *vm,
|
||||
pd_offs_out);
|
||||
}
|
||||
|
||||
if (!pd->mem)
|
||||
if (!pd->mem) {
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
/*
|
||||
* Take into account the real offset into the nvgpu_mem since the PD
|
||||
@@ -867,14 +893,17 @@ static int __nvgpu_locate_pte(struct gk20a *g, struct vm_gk20a *vm,
|
||||
}
|
||||
}
|
||||
|
||||
if (pd_out)
|
||||
if (pd_out) {
|
||||
*pd_out = pd;
|
||||
}
|
||||
|
||||
if (pd_idx_out)
|
||||
if (pd_idx_out) {
|
||||
*pd_idx_out = pd_idx;
|
||||
}
|
||||
|
||||
if (pd_offs_out)
|
||||
if (pd_offs_out) {
|
||||
*pd_offs_out = pd_offset_from_index(l, pd_idx);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -903,8 +932,9 @@ int __nvgpu_set_pte(struct gk20a *g, struct vm_gk20a *vm, u64 vaddr, u32 *pte)
|
||||
err = __nvgpu_locate_pte(g, vm, &vm->pdb,
|
||||
vaddr, 0, &attrs,
|
||||
NULL, &pd, &pd_idx, &pd_offs);
|
||||
if (err)
|
||||
if (err) {
|
||||
return err;
|
||||
}
|
||||
|
||||
pte_size = __nvgpu_pte_words(g);
|
||||
|
||||
|
||||
@@ -63,8 +63,9 @@ static u64 nvgpu_lockless_alloc(struct nvgpu_allocator *a, u64 len)
|
||||
int head, new_head, ret;
|
||||
u64 addr = 0;
|
||||
|
||||
if (len != pa->blk_size)
|
||||
if (len != pa->blk_size) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
head = NV_ACCESS_ONCE(pa->head);
|
||||
while (head >= 0) {
|
||||
@@ -80,10 +81,11 @@ static u64 nvgpu_lockless_alloc(struct nvgpu_allocator *a, u64 len)
|
||||
head = NV_ACCESS_ONCE(pa->head);
|
||||
}
|
||||
|
||||
if (addr)
|
||||
if (addr) {
|
||||
alloc_dbg(a, "Alloc node # %d @ addr 0x%llx", head, addr);
|
||||
else
|
||||
} else {
|
||||
alloc_dbg(a, "Alloc failed!");
|
||||
}
|
||||
|
||||
return addr;
|
||||
}
|
||||
@@ -167,24 +169,28 @@ int nvgpu_lockless_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a,
|
||||
u64 count;
|
||||
struct nvgpu_lockless_allocator *a;
|
||||
|
||||
if (!blk_size)
|
||||
if (!blk_size) {
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
/*
|
||||
* Ensure we have space for at least one node & there's no overflow.
|
||||
* In order to control memory footprint, we require count < INT_MAX
|
||||
*/
|
||||
count = length / blk_size;
|
||||
if (!base || !count || count > INT_MAX)
|
||||
if (!base || !count || count > INT_MAX) {
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
a = nvgpu_kzalloc(g, sizeof(struct nvgpu_lockless_allocator));
|
||||
if (!a)
|
||||
if (!a) {
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
err = __nvgpu_alloc_common_init(__a, g, name, a, false, &pool_ops);
|
||||
if (err)
|
||||
if (err) {
|
||||
goto fail;
|
||||
}
|
||||
|
||||
a->next = nvgpu_vzalloc(g, sizeof(*a->next) * count);
|
||||
if (!a->next) {
|
||||
|
||||
@@ -40,8 +40,9 @@ enum gmmu_pgsz_gk20a __get_pte_size_fixed_map(struct vm_gk20a *vm,
|
||||
struct nvgpu_vm_area *vm_area;
|
||||
|
||||
vm_area = nvgpu_vm_area_find(vm, base);
|
||||
if (!vm_area)
|
||||
if (!vm_area) {
|
||||
return gmmu_page_size_small;
|
||||
}
|
||||
|
||||
return vm_area->pgsz_idx;
|
||||
}
|
||||
@@ -53,14 +54,16 @@ static enum gmmu_pgsz_gk20a __get_pte_size_split_addr(struct vm_gk20a *vm,
|
||||
u64 base, u64 size)
|
||||
{
|
||||
if (!base) {
|
||||
if (size >= vm->gmmu_page_sizes[gmmu_page_size_big])
|
||||
if (size >= vm->gmmu_page_sizes[gmmu_page_size_big]) {
|
||||
return gmmu_page_size_big;
|
||||
}
|
||||
return gmmu_page_size_small;
|
||||
} else {
|
||||
if (base < __nv_gmmu_va_small_page_limit())
|
||||
if (base < __nv_gmmu_va_small_page_limit()) {
|
||||
return gmmu_page_size_small;
|
||||
else
|
||||
} else {
|
||||
return gmmu_page_size_big;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -89,18 +92,22 @@ enum gmmu_pgsz_gk20a __get_pte_size(struct vm_gk20a *vm, u64 base, u64 size)
|
||||
{
|
||||
struct gk20a *g = gk20a_from_vm(vm);
|
||||
|
||||
if (!vm->big_pages)
|
||||
if (!vm->big_pages) {
|
||||
return gmmu_page_size_small;
|
||||
}
|
||||
|
||||
if (!nvgpu_is_enabled(g, NVGPU_MM_UNIFY_ADDRESS_SPACES))
|
||||
if (!nvgpu_is_enabled(g, NVGPU_MM_UNIFY_ADDRESS_SPACES)) {
|
||||
return __get_pte_size_split_addr(vm, base, size);
|
||||
}
|
||||
|
||||
if (base)
|
||||
if (base) {
|
||||
return __get_pte_size_fixed_map(vm, base, size);
|
||||
}
|
||||
|
||||
if (size >= vm->gmmu_page_sizes[gmmu_page_size_big] &&
|
||||
nvgpu_iommuable(g))
|
||||
nvgpu_iommuable(g)) {
|
||||
return gmmu_page_size_big;
|
||||
}
|
||||
return gmmu_page_size_small;
|
||||
}
|
||||
|
||||
@@ -137,8 +144,9 @@ u64 nvgpu_inst_block_addr(struct gk20a *g, struct nvgpu_mem *inst_block)
|
||||
|
||||
void nvgpu_free_inst_block(struct gk20a *g, struct nvgpu_mem *inst_block)
|
||||
{
|
||||
if (nvgpu_mem_is_valid(inst_block))
|
||||
if (nvgpu_mem_is_valid(inst_block)) {
|
||||
nvgpu_dma_free(g, inst_block);
|
||||
}
|
||||
}
|
||||
|
||||
static int nvgpu_alloc_sysmem_flush(struct gk20a *g)
|
||||
@@ -150,8 +158,9 @@ static void nvgpu_remove_mm_ce_support(struct mm_gk20a *mm)
|
||||
{
|
||||
struct gk20a *g = gk20a_from_mm(mm);
|
||||
|
||||
if (mm->vidmem.ce_ctx_id != (u32)~0)
|
||||
if (mm->vidmem.ce_ctx_id != (u32)~0) {
|
||||
gk20a_ce_delete_context_priv(g, mm->vidmem.ce_ctx_id);
|
||||
}
|
||||
|
||||
mm->vidmem.ce_ctx_id = (u32)~0;
|
||||
|
||||
@@ -162,11 +171,13 @@ static void nvgpu_remove_mm_support(struct mm_gk20a *mm)
|
||||
{
|
||||
struct gk20a *g = gk20a_from_mm(mm);
|
||||
|
||||
if (g->ops.mm.fault_info_mem_destroy)
|
||||
if (g->ops.mm.fault_info_mem_destroy) {
|
||||
g->ops.mm.fault_info_mem_destroy(g);
|
||||
}
|
||||
|
||||
if (g->ops.mm.remove_bar2_vm)
|
||||
if (g->ops.mm.remove_bar2_vm) {
|
||||
g->ops.mm.remove_bar2_vm(g);
|
||||
}
|
||||
|
||||
nvgpu_free_inst_block(g, &mm->bar1.inst_block);
|
||||
nvgpu_vm_put(mm->bar1.vm);
|
||||
@@ -175,8 +186,9 @@ static void nvgpu_remove_mm_support(struct mm_gk20a *mm)
|
||||
nvgpu_free_inst_block(g, &mm->hwpm.inst_block);
|
||||
nvgpu_vm_put(mm->pmu.vm);
|
||||
|
||||
if (g->has_cde)
|
||||
if (g->has_cde) {
|
||||
nvgpu_vm_put(mm->cde.vm);
|
||||
}
|
||||
|
||||
nvgpu_semaphore_sea_destroy(g);
|
||||
nvgpu_vidmem_destroy(g);
|
||||
@@ -208,12 +220,14 @@ static int nvgpu_init_system_vm(struct mm_gk20a *mm)
|
||||
true,
|
||||
false,
|
||||
"system");
|
||||
if (!mm->pmu.vm)
|
||||
if (!mm->pmu.vm) {
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
err = g->ops.mm.alloc_inst_block(g, inst_block);
|
||||
if (err)
|
||||
if (err) {
|
||||
goto clean_up_vm;
|
||||
}
|
||||
g->ops.mm.init_inst_block(inst_block, mm->pmu.vm, big_page_size);
|
||||
|
||||
return 0;
|
||||
@@ -230,8 +244,9 @@ static int nvgpu_init_hwpm(struct mm_gk20a *mm)
|
||||
struct nvgpu_mem *inst_block = &mm->hwpm.inst_block;
|
||||
|
||||
err = g->ops.mm.alloc_inst_block(g, inst_block);
|
||||
if (err)
|
||||
if (err) {
|
||||
return err;
|
||||
}
|
||||
g->ops.mm.init_inst_block(inst_block, mm->pmu.vm, 0);
|
||||
|
||||
return 0;
|
||||
@@ -247,8 +262,9 @@ static int nvgpu_init_cde_vm(struct mm_gk20a *mm)
|
||||
NV_MM_DEFAULT_KERNEL_SIZE,
|
||||
NV_MM_DEFAULT_KERNEL_SIZE + NV_MM_DEFAULT_USER_SIZE,
|
||||
false, false, "cde");
|
||||
if (!mm->cde.vm)
|
||||
if (!mm->cde.vm) {
|
||||
return -ENOMEM;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -262,8 +278,9 @@ static int nvgpu_init_ce_vm(struct mm_gk20a *mm)
|
||||
NV_MM_DEFAULT_KERNEL_SIZE,
|
||||
NV_MM_DEFAULT_KERNEL_SIZE + NV_MM_DEFAULT_USER_SIZE,
|
||||
false, false, "ce");
|
||||
if (!mm->ce.vm)
|
||||
if (!mm->ce.vm) {
|
||||
return -ENOMEM;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -286,24 +303,30 @@ void nvgpu_init_mm_ce_context(struct gk20a *g)
|
||||
|
||||
static int nvgpu_init_mm_reset_enable_hw(struct gk20a *g)
|
||||
{
|
||||
if (g->ops.fb.reset)
|
||||
if (g->ops.fb.reset) {
|
||||
g->ops.fb.reset(g);
|
||||
}
|
||||
|
||||
if (g->ops.clock_gating.slcg_fb_load_gating_prod)
|
||||
if (g->ops.clock_gating.slcg_fb_load_gating_prod) {
|
||||
g->ops.clock_gating.slcg_fb_load_gating_prod(g,
|
||||
g->slcg_enabled);
|
||||
if (g->ops.clock_gating.slcg_ltc_load_gating_prod)
|
||||
}
|
||||
if (g->ops.clock_gating.slcg_ltc_load_gating_prod) {
|
||||
g->ops.clock_gating.slcg_ltc_load_gating_prod(g,
|
||||
g->slcg_enabled);
|
||||
if (g->ops.clock_gating.blcg_fb_load_gating_prod)
|
||||
}
|
||||
if (g->ops.clock_gating.blcg_fb_load_gating_prod) {
|
||||
g->ops.clock_gating.blcg_fb_load_gating_prod(g,
|
||||
g->blcg_enabled);
|
||||
if (g->ops.clock_gating.blcg_ltc_load_gating_prod)
|
||||
}
|
||||
if (g->ops.clock_gating.blcg_ltc_load_gating_prod) {
|
||||
g->ops.clock_gating.blcg_ltc_load_gating_prod(g,
|
||||
g->blcg_enabled);
|
||||
}
|
||||
|
||||
if (g->ops.fb.init_fs_state)
|
||||
if (g->ops.fb.init_fs_state) {
|
||||
g->ops.fb.init_fs_state(g);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -324,12 +347,14 @@ static int nvgpu_init_bar1_vm(struct mm_gk20a *mm)
|
||||
mm->bar1.aperture_size,
|
||||
true, false,
|
||||
"bar1");
|
||||
if (!mm->bar1.vm)
|
||||
if (!mm->bar1.vm) {
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
err = g->ops.mm.alloc_inst_block(g, inst_block);
|
||||
if (err)
|
||||
if (err) {
|
||||
goto clean_up_vm;
|
||||
}
|
||||
g->ops.mm.init_inst_block(inst_block, mm->bar1.vm, big_page_size);
|
||||
|
||||
return 0;
|
||||
@@ -366,8 +391,9 @@ static int nvgpu_init_mm_setup_sw(struct gk20a *g)
|
||||
mm->vidmem.ce_ctx_id = (u32)~0;
|
||||
|
||||
err = nvgpu_vidmem_init(mm);
|
||||
if (err)
|
||||
if (err) {
|
||||
return err;
|
||||
}
|
||||
|
||||
/*
|
||||
* this requires fixed allocations in vidmem which must be
|
||||
@@ -376,40 +402,48 @@ static int nvgpu_init_mm_setup_sw(struct gk20a *g)
|
||||
if (g->ops.pmu.alloc_blob_space
|
||||
&& !nvgpu_is_enabled(g, NVGPU_MM_UNIFIED_MEMORY)) {
|
||||
err = g->ops.pmu.alloc_blob_space(g, 0, &g->acr.ucode_blob);
|
||||
if (err)
|
||||
if (err) {
|
||||
return err;
|
||||
}
|
||||
}
|
||||
|
||||
err = nvgpu_alloc_sysmem_flush(g);
|
||||
if (err)
|
||||
if (err) {
|
||||
return err;
|
||||
}
|
||||
|
||||
err = nvgpu_init_bar1_vm(mm);
|
||||
if (err)
|
||||
if (err) {
|
||||
return err;
|
||||
}
|
||||
|
||||
if (g->ops.mm.init_bar2_vm) {
|
||||
err = g->ops.mm.init_bar2_vm(g);
|
||||
if (err)
|
||||
if (err) {
|
||||
return err;
|
||||
}
|
||||
}
|
||||
err = nvgpu_init_system_vm(mm);
|
||||
if (err)
|
||||
if (err) {
|
||||
return err;
|
||||
}
|
||||
|
||||
err = nvgpu_init_hwpm(mm);
|
||||
if (err)
|
||||
if (err) {
|
||||
return err;
|
||||
}
|
||||
|
||||
if (g->has_cde) {
|
||||
err = nvgpu_init_cde_vm(mm);
|
||||
if (err)
|
||||
if (err) {
|
||||
return err;
|
||||
}
|
||||
}
|
||||
|
||||
err = nvgpu_init_ce_vm(mm);
|
||||
if (err)
|
||||
if (err) {
|
||||
return err;
|
||||
}
|
||||
|
||||
mm->remove_support = nvgpu_remove_mm_support;
|
||||
mm->remove_ce_support = nvgpu_remove_mm_ce_support;
|
||||
@@ -424,15 +458,18 @@ int nvgpu_init_mm_support(struct gk20a *g)
|
||||
u32 err;
|
||||
|
||||
err = nvgpu_init_mm_reset_enable_hw(g);
|
||||
if (err)
|
||||
if (err) {
|
||||
return err;
|
||||
}
|
||||
|
||||
err = nvgpu_init_mm_setup_sw(g);
|
||||
if (err)
|
||||
if (err) {
|
||||
return err;
|
||||
}
|
||||
|
||||
if (g->ops.mm.init_mm_setup_hw)
|
||||
if (g->ops.mm.init_mm_setup_hw) {
|
||||
err = g->ops.mm.init_mm_setup_hw(g);
|
||||
}
|
||||
|
||||
return err;
|
||||
}
|
||||
@@ -443,8 +480,9 @@ u32 nvgpu_mm_get_default_big_page_size(struct gk20a *g)
|
||||
|
||||
big_page_size = g->ops.mm.get_default_big_page_size();
|
||||
|
||||
if (g->mm.disable_bigpage)
|
||||
if (g->mm.disable_bigpage) {
|
||||
big_page_size = 0;
|
||||
}
|
||||
|
||||
return big_page_size;
|
||||
}
|
||||
@@ -456,8 +494,9 @@ u32 nvgpu_mm_get_available_big_page_sizes(struct gk20a *g)
|
||||
if (!g->mm.disable_bigpage) {
|
||||
available_big_page_sizes =
|
||||
g->ops.mm.get_default_big_page_size();
|
||||
if (g->ops.mm.get_big_page_sizes)
|
||||
if (g->ops.mm.get_big_page_sizes) {
|
||||
available_big_page_sizes |= g->ops.mm.get_big_page_sizes();
|
||||
}
|
||||
}
|
||||
|
||||
return available_big_page_sizes;
|
||||
|
||||
@@ -40,8 +40,9 @@ u32 __nvgpu_aperture_mask(struct gk20a *g, enum nvgpu_aperture aperture,
|
||||
* Some iGPUs treat sysmem (i.e SoC DRAM) as vidmem. In these cases the
|
||||
* "sysmem" aperture should really be translated to VIDMEM.
|
||||
*/
|
||||
if (!nvgpu_is_enabled(g, NVGPU_MM_HONORS_APERTURE))
|
||||
if (!nvgpu_is_enabled(g, NVGPU_MM_HONORS_APERTURE)) {
|
||||
aperture = APERTURE_VIDMEM;
|
||||
}
|
||||
|
||||
switch (aperture) {
|
||||
case __APERTURE_SYSMEM_COH:
|
||||
@@ -67,8 +68,9 @@ u32 nvgpu_aperture_mask(struct gk20a *g, struct nvgpu_mem *mem,
|
||||
* we add this translation step here.
|
||||
*/
|
||||
if (nvgpu_is_enabled(g, NVGPU_USE_COHERENT_SYSMEM) &&
|
||||
ap == APERTURE_SYSMEM)
|
||||
ap == APERTURE_SYSMEM) {
|
||||
ap = __APERTURE_SYSMEM_COH;
|
||||
}
|
||||
|
||||
return __nvgpu_aperture_mask(g, ap,
|
||||
sysmem_mask, sysmem_coh_mask, vidmem_mask);
|
||||
@@ -115,15 +117,17 @@ u64 nvgpu_sgt_get_gpu_addr(struct gk20a *g, struct nvgpu_sgt *sgt,
|
||||
|
||||
bool nvgpu_sgt_iommuable(struct gk20a *g, struct nvgpu_sgt *sgt)
|
||||
{
|
||||
if (sgt->ops->sgt_iommuable)
|
||||
if (sgt->ops->sgt_iommuable) {
|
||||
return sgt->ops->sgt_iommuable(g, sgt);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
void nvgpu_sgt_free(struct gk20a *g, struct nvgpu_sgt *sgt)
|
||||
{
|
||||
if (sgt && sgt->ops->sgt_free)
|
||||
if (sgt && sgt->ops->sgt_free) {
|
||||
sgt->ops->sgt_free(g, sgt);
|
||||
}
|
||||
}
|
||||
|
||||
u64 nvgpu_mem_iommu_translate(struct gk20a *g, u64 phys)
|
||||
@@ -131,8 +135,9 @@ u64 nvgpu_mem_iommu_translate(struct gk20a *g, u64 phys)
|
||||
/* ensure it is not vidmem allocation */
|
||||
WARN_ON(nvgpu_addr_is_vidmem_page_alloc(phys));
|
||||
|
||||
if (nvgpu_iommuable(g) && g->ops.mm.get_iommu_bit)
|
||||
if (nvgpu_iommuable(g) && g->ops.mm.get_iommu_bit) {
|
||||
return phys | 1ULL << g->ops.mm.get_iommu_bit(g);
|
||||
}
|
||||
|
||||
return phys;
|
||||
}
|
||||
@@ -157,8 +162,9 @@ u64 nvgpu_sgt_alignment(struct gk20a *g, struct nvgpu_sgt *sgt)
|
||||
*/
|
||||
if (nvgpu_iommuable(g) &&
|
||||
nvgpu_sgt_iommuable(g, sgt) &&
|
||||
nvgpu_sgt_get_dma(sgt, sgt->sgl))
|
||||
nvgpu_sgt_get_dma(sgt, sgt->sgl)) {
|
||||
return 1ULL << __ffs(nvgpu_sgt_get_dma(sgt, sgt->sgl));
|
||||
}
|
||||
|
||||
/*
|
||||
* Otherwise the buffer is not iommuable (VIDMEM, for example) or we are
|
||||
@@ -169,10 +175,11 @@ u64 nvgpu_sgt_alignment(struct gk20a *g, struct nvgpu_sgt *sgt)
|
||||
chunk_align = 1ULL << __ffs(nvgpu_sgt_get_phys(g, sgt, sgl) |
|
||||
nvgpu_sgt_get_length(sgt, sgl));
|
||||
|
||||
if (align)
|
||||
if (align) {
|
||||
align = min(align, chunk_align);
|
||||
else
|
||||
} else {
|
||||
align = chunk_align;
|
||||
}
|
||||
}
|
||||
|
||||
return align;
|
||||
|
||||
@@ -111,8 +111,9 @@ static void nvgpu_vm_free_entries(struct vm_gk20a *vm,
|
||||
|
||||
__nvgpu_pd_cache_free_direct(g, pdb);
|
||||
|
||||
if (!pdb->entries)
|
||||
if (!pdb->entries) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (i = 0; i < pdb->num_entries; i++) {
|
||||
__nvgpu_vm_free_entries(vm, &pdb->entries[i], 1);
|
||||
@@ -204,8 +205,9 @@ int nvgpu_big_pages_possible(struct vm_gk20a *vm, u64 base, u64 size)
|
||||
{
|
||||
u64 mask = ((u64)vm->big_page_size << 10) - 1;
|
||||
|
||||
if (base & mask || size & mask)
|
||||
if (base & mask || size & mask) {
|
||||
return 0;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
@@ -223,19 +225,23 @@ static int nvgpu_init_sema_pool(struct vm_gk20a *vm)
|
||||
/*
|
||||
* Don't waste the memory on semaphores if we don't need them.
|
||||
*/
|
||||
if (nvgpu_is_enabled(g, NVGPU_HAS_SYNCPOINTS))
|
||||
if (nvgpu_is_enabled(g, NVGPU_HAS_SYNCPOINTS)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (vm->sema_pool)
|
||||
if (vm->sema_pool) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
sema_sea = nvgpu_semaphore_sea_create(g);
|
||||
if (!sema_sea)
|
||||
if (!sema_sea) {
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
err = nvgpu_semaphore_pool_alloc(sema_sea, &vm->sema_pool);
|
||||
if (err)
|
||||
if (err) {
|
||||
return err;
|
||||
}
|
||||
|
||||
/*
|
||||
* Allocate a chunk of GPU VA space for mapping the semaphores. We will
|
||||
@@ -287,11 +293,13 @@ int __nvgpu_vm_init(struct mm_gk20a *mm,
|
||||
u64 kernel_vma_start, kernel_vma_limit;
|
||||
struct gk20a *g = gk20a_from_mm(mm);
|
||||
|
||||
if (WARN_ON(kernel_reserved + low_hole > aperture_size))
|
||||
if (WARN_ON(kernel_reserved + low_hole > aperture_size)) {
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
if (WARN_ON(vm->guest_managed && kernel_reserved != 0))
|
||||
if (WARN_ON(vm->guest_managed && kernel_reserved != 0)) {
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
nvgpu_log_info(g, "Init space for %s: valimit=0x%llx, "
|
||||
"LP size=0x%x lowhole=0x%llx",
|
||||
@@ -308,8 +316,9 @@ int __nvgpu_vm_init(struct mm_gk20a *mm,
|
||||
vm->vma[gmmu_page_size_small] = &vm->user;
|
||||
vm->vma[gmmu_page_size_big] = &vm->user;
|
||||
vm->vma[gmmu_page_size_kernel] = &vm->kernel;
|
||||
if (!nvgpu_is_enabled(g, NVGPU_MM_UNIFY_ADDRESS_SPACES))
|
||||
if (!nvgpu_is_enabled(g, NVGPU_MM_UNIFY_ADDRESS_SPACES)) {
|
||||
vm->vma[gmmu_page_size_big] = &vm->user_lp;
|
||||
}
|
||||
|
||||
vm->va_start = low_hole;
|
||||
vm->va_limit = aperture_size;
|
||||
@@ -332,8 +341,9 @@ int __nvgpu_vm_init(struct mm_gk20a *mm,
|
||||
/* Initialize the page table data structures. */
|
||||
strncpy(vm->name, name, min(strlen(name), sizeof(vm->name)));
|
||||
err = nvgpu_gmmu_init_page_table(vm);
|
||||
if (err)
|
||||
if (err) {
|
||||
goto clean_up_vgpu_vm;
|
||||
}
|
||||
|
||||
/* Setup vma limits. */
|
||||
if (kernel_reserved + low_hole < aperture_size) {
|
||||
@@ -396,14 +406,15 @@ int __nvgpu_vm_init(struct mm_gk20a *mm,
|
||||
* Determine if big pages are possible in this VM. If a split address
|
||||
* space is used then check the user_lp vma instead of the user vma.
|
||||
*/
|
||||
if (nvgpu_is_enabled(g, NVGPU_MM_UNIFY_ADDRESS_SPACES))
|
||||
if (nvgpu_is_enabled(g, NVGPU_MM_UNIFY_ADDRESS_SPACES)) {
|
||||
vm->big_pages = big_pages &&
|
||||
nvgpu_big_pages_possible(vm, user_vma_start,
|
||||
user_vma_limit - user_vma_start);
|
||||
else
|
||||
} else {
|
||||
vm->big_pages = big_pages &&
|
||||
nvgpu_big_pages_possible(vm, user_lp_vma_start,
|
||||
user_lp_vma_limit - user_lp_vma_start);
|
||||
}
|
||||
|
||||
/*
|
||||
* User VMA.
|
||||
@@ -418,8 +429,9 @@ int __nvgpu_vm_init(struct mm_gk20a *mm,
|
||||
SZ_4K,
|
||||
GPU_BALLOC_MAX_ORDER,
|
||||
GPU_ALLOC_GVA_SPACE);
|
||||
if (err)
|
||||
if (err) {
|
||||
goto clean_up_page_tables;
|
||||
}
|
||||
} else {
|
||||
/*
|
||||
* Make these allocator pointers point to the kernel allocator
|
||||
@@ -443,8 +455,9 @@ int __nvgpu_vm_init(struct mm_gk20a *mm,
|
||||
vm->big_page_size,
|
||||
GPU_BALLOC_MAX_ORDER,
|
||||
GPU_ALLOC_GVA_SPACE);
|
||||
if (err)
|
||||
if (err) {
|
||||
goto clean_up_allocators;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -458,8 +471,9 @@ int __nvgpu_vm_init(struct mm_gk20a *mm,
|
||||
SZ_4K,
|
||||
GPU_BALLOC_MAX_ORDER,
|
||||
kernel_vma_flags);
|
||||
if (err)
|
||||
if (err) {
|
||||
goto clean_up_allocators;
|
||||
}
|
||||
|
||||
vm->mapped_buffers = NULL;
|
||||
|
||||
@@ -475,19 +489,23 @@ int __nvgpu_vm_init(struct mm_gk20a *mm,
|
||||
*/
|
||||
if (vm->va_limit > 4ULL * SZ_1G) {
|
||||
err = nvgpu_init_sema_pool(vm);
|
||||
if (err)
|
||||
if (err) {
|
||||
goto clean_up_allocators;
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
|
||||
clean_up_allocators:
|
||||
if (nvgpu_alloc_initialized(&vm->kernel))
|
||||
if (nvgpu_alloc_initialized(&vm->kernel)) {
|
||||
nvgpu_alloc_destroy(&vm->kernel);
|
||||
if (nvgpu_alloc_initialized(&vm->user))
|
||||
}
|
||||
if (nvgpu_alloc_initialized(&vm->user)) {
|
||||
nvgpu_alloc_destroy(&vm->user);
|
||||
if (nvgpu_alloc_initialized(&vm->user_lp))
|
||||
}
|
||||
if (nvgpu_alloc_initialized(&vm->user_lp)) {
|
||||
nvgpu_alloc_destroy(&vm->user_lp);
|
||||
}
|
||||
clean_up_page_tables:
|
||||
/* Cleans up nvgpu_gmmu_init_page_table() */
|
||||
__nvgpu_pd_cache_free_direct(g, &vm->pdb);
|
||||
@@ -547,8 +565,9 @@ struct vm_gk20a *nvgpu_vm_init(struct gk20a *g,
|
||||
{
|
||||
struct vm_gk20a *vm = nvgpu_kzalloc(g, sizeof(*vm));
|
||||
|
||||
if (!vm)
|
||||
if (!vm) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (__nvgpu_vm_init(&g->mm, vm, big_page_size, low_hole,
|
||||
kernel_reserved, aperture_size, big_pages,
|
||||
@@ -582,9 +601,10 @@ static void __nvgpu_vm_remove(struct vm_gk20a *vm)
|
||||
}
|
||||
}
|
||||
|
||||
if (nvgpu_mem_is_valid(&g->syncpt_mem) && vm->syncpt_ro_map_gpu_va)
|
||||
if (nvgpu_mem_is_valid(&g->syncpt_mem) && vm->syncpt_ro_map_gpu_va) {
|
||||
nvgpu_gmmu_unmap(vm, &g->syncpt_mem,
|
||||
vm->syncpt_ro_map_gpu_va);
|
||||
}
|
||||
|
||||
nvgpu_mutex_acquire(&vm->update_gmmu_lock);
|
||||
|
||||
@@ -603,12 +623,15 @@ static void __nvgpu_vm_remove(struct vm_gk20a *vm)
|
||||
nvgpu_kfree(vm->mm->g, vm_area);
|
||||
}
|
||||
|
||||
if (nvgpu_alloc_initialized(&vm->kernel))
|
||||
if (nvgpu_alloc_initialized(&vm->kernel)) {
|
||||
nvgpu_alloc_destroy(&vm->kernel);
|
||||
if (nvgpu_alloc_initialized(&vm->user))
|
||||
}
|
||||
if (nvgpu_alloc_initialized(&vm->user)) {
|
||||
nvgpu_alloc_destroy(&vm->user);
|
||||
if (nvgpu_alloc_initialized(&vm->user_lp))
|
||||
}
|
||||
if (nvgpu_alloc_initialized(&vm->user_lp)) {
|
||||
nvgpu_alloc_destroy(&vm->user_lp);
|
||||
}
|
||||
|
||||
nvgpu_vm_free_entries(vm, &vm->pdb);
|
||||
|
||||
@@ -664,8 +687,9 @@ struct nvgpu_mapped_buf *__nvgpu_vm_find_mapped_buf(
|
||||
struct nvgpu_rbtree_node *root = vm->mapped_buffers;
|
||||
|
||||
nvgpu_rbtree_search(addr, &node, root);
|
||||
if (!node)
|
||||
if (!node) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
return mapped_buffer_from_rbtree_node(node);
|
||||
}
|
||||
@@ -677,8 +701,9 @@ struct nvgpu_mapped_buf *__nvgpu_vm_find_mapped_buf_range(
|
||||
struct nvgpu_rbtree_node *root = vm->mapped_buffers;
|
||||
|
||||
nvgpu_rbtree_range_search(addr, &node, root);
|
||||
if (!node)
|
||||
if (!node) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
return mapped_buffer_from_rbtree_node(node);
|
||||
}
|
||||
@@ -690,8 +715,9 @@ struct nvgpu_mapped_buf *__nvgpu_vm_find_mapped_buf_less_than(
|
||||
struct nvgpu_rbtree_node *root = vm->mapped_buffers;
|
||||
|
||||
nvgpu_rbtree_less_than_search(addr, &node, root);
|
||||
if (!node)
|
||||
if (!node) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
return mapped_buffer_from_rbtree_node(node);
|
||||
}
|
||||
@@ -746,8 +772,9 @@ void nvgpu_vm_put_buffers(struct vm_gk20a *vm,
|
||||
int i;
|
||||
struct vm_gk20a_mapping_batch batch;
|
||||
|
||||
if (num_buffers == 0)
|
||||
if (num_buffers == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
nvgpu_mutex_acquire(&vm->update_gmmu_lock);
|
||||
nvgpu_vm_mapping_batch_start(&batch);
|
||||
@@ -814,10 +841,11 @@ struct nvgpu_mapped_buf *nvgpu_vm_map(struct vm_gk20a *vm,
|
||||
compr_kind : NVGPU_KIND_INVALID);
|
||||
binfo.incompr_kind = incompr_kind;
|
||||
|
||||
if (compr_kind != NVGPU_KIND_INVALID)
|
||||
if (compr_kind != NVGPU_KIND_INVALID) {
|
||||
map_key_kind = compr_kind;
|
||||
else
|
||||
} else {
|
||||
map_key_kind = incompr_kind;
|
||||
}
|
||||
|
||||
/*
|
||||
* Check if this buffer is already mapped.
|
||||
@@ -847,11 +875,12 @@ struct nvgpu_mapped_buf *nvgpu_vm_map(struct vm_gk20a *vm,
|
||||
}
|
||||
|
||||
align = nvgpu_sgt_alignment(g, sgt);
|
||||
if (g->mm.disable_bigpage)
|
||||
if (g->mm.disable_bigpage) {
|
||||
binfo.pgsz_idx = gmmu_page_size_small;
|
||||
else
|
||||
} else {
|
||||
binfo.pgsz_idx = __get_pte_size(vm, map_addr,
|
||||
min_t(u64, binfo.size, align));
|
||||
}
|
||||
map_size = map_size ? map_size : binfo.size;
|
||||
map_size = ALIGN(map_size, SZ_4K);
|
||||
|
||||
@@ -872,8 +901,9 @@ struct nvgpu_mapped_buf *nvgpu_vm_map(struct vm_gk20a *vm,
|
||||
map_size,
|
||||
binfo.pgsz_idx,
|
||||
&vm_area);
|
||||
if (err)
|
||||
if (err) {
|
||||
goto clean_up;
|
||||
}
|
||||
|
||||
va_allocated = false;
|
||||
}
|
||||
@@ -941,8 +971,9 @@ struct nvgpu_mapped_buf *nvgpu_vm_map(struct vm_gk20a *vm,
|
||||
comptags.lines - 1));
|
||||
gk20a_comptags_finish_clear(
|
||||
os_buf, err == 0);
|
||||
if (err)
|
||||
if (err) {
|
||||
goto clean_up;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
/*
|
||||
@@ -955,8 +986,9 @@ struct nvgpu_mapped_buf *nvgpu_vm_map(struct vm_gk20a *vm,
|
||||
/*
|
||||
* Store the ctag offset for later use if we got the comptags
|
||||
*/
|
||||
if (comptags.lines)
|
||||
if (comptags.lines) {
|
||||
ctag_offset = comptags.offset;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -984,8 +1016,9 @@ struct nvgpu_mapped_buf *nvgpu_vm_map(struct vm_gk20a *vm,
|
||||
goto clean_up;
|
||||
}
|
||||
|
||||
if (clear_ctags)
|
||||
if (clear_ctags) {
|
||||
clear_ctags = gk20a_comptags_start_clear(os_buf);
|
||||
}
|
||||
|
||||
map_addr = g->ops.mm.gmmu_map(vm,
|
||||
map_addr,
|
||||
@@ -1003,8 +1036,9 @@ struct nvgpu_mapped_buf *nvgpu_vm_map(struct vm_gk20a *vm,
|
||||
batch,
|
||||
aperture);
|
||||
|
||||
if (clear_ctags)
|
||||
if (clear_ctags) {
|
||||
gk20a_comptags_finish_clear(os_buf, map_addr != 0);
|
||||
}
|
||||
|
||||
if (!map_addr) {
|
||||
err = -ENOMEM;
|
||||
@@ -1041,7 +1075,7 @@ struct nvgpu_mapped_buf *nvgpu_vm_map(struct vm_gk20a *vm,
|
||||
return mapped_buffer;
|
||||
|
||||
clean_up:
|
||||
if (mapped_buffer->addr)
|
||||
if (mapped_buffer->addr) {
|
||||
g->ops.mm.gmmu_unmap(vm,
|
||||
mapped_buffer->addr,
|
||||
mapped_buffer->size,
|
||||
@@ -1051,6 +1085,7 @@ clean_up:
|
||||
mapped_buffer->vm_area ?
|
||||
mapped_buffer->vm_area->sparse : false,
|
||||
NULL);
|
||||
}
|
||||
nvgpu_mutex_release(&vm->update_gmmu_lock);
|
||||
clean_up_nolock:
|
||||
nvgpu_kfree(g, mapped_buffer);
|
||||
@@ -1132,14 +1167,16 @@ static int nvgpu_vm_unmap_sync_buffer(struct vm_gk20a *vm,
|
||||
nvgpu_timeout_init(vm->mm->g, &timeout, 50, NVGPU_TIMER_CPU_TIMER);
|
||||
|
||||
do {
|
||||
if (nvgpu_atomic_read(&mapped_buffer->ref.refcount) == 1)
|
||||
if (nvgpu_atomic_read(&mapped_buffer->ref.refcount) == 1) {
|
||||
break;
|
||||
}
|
||||
nvgpu_msleep(10);
|
||||
} while (!nvgpu_timeout_expired_msg(&timeout,
|
||||
"sync-unmap failed on 0x%llx"));
|
||||
|
||||
if (nvgpu_timeout_expired(&timeout))
|
||||
if (nvgpu_timeout_expired(&timeout)) {
|
||||
ret = -ETIMEDOUT;
|
||||
}
|
||||
|
||||
nvgpu_mutex_acquire(&vm->update_gmmu_lock);
|
||||
|
||||
@@ -1154,16 +1191,18 @@ void nvgpu_vm_unmap(struct vm_gk20a *vm, u64 offset,
|
||||
nvgpu_mutex_acquire(&vm->update_gmmu_lock);
|
||||
|
||||
mapped_buffer = __nvgpu_vm_find_mapped_buf(vm, offset);
|
||||
if (!mapped_buffer)
|
||||
if (!mapped_buffer) {
|
||||
goto done;
|
||||
}
|
||||
|
||||
if (mapped_buffer->flags & NVGPU_VM_MAP_FIXED_OFFSET) {
|
||||
if (nvgpu_vm_unmap_sync_buffer(vm, mapped_buffer))
|
||||
if (nvgpu_vm_unmap_sync_buffer(vm, mapped_buffer)) {
|
||||
/*
|
||||
* Looks like we have failed... Better not continue in
|
||||
* case the buffer is in use.
|
||||
*/
|
||||
goto done;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
Reference in New Issue
Block a user