diff --git a/drivers/gpu/nvgpu/Makefile.nvgpu b/drivers/gpu/nvgpu/Makefile.nvgpu index 93629eff7..afce062b9 100644 --- a/drivers/gpu/nvgpu/Makefile.nvgpu +++ b/drivers/gpu/nvgpu/Makefile.nvgpu @@ -23,6 +23,11 @@ obj-$(CONFIG_GK20A) := nvgpu.o nvgpu-y := \ common/linux/timers.o \ + common/mm/nvgpu_allocator.o \ + common/mm/bitmap_allocator.o \ + common/mm/buddy_allocator.o \ + common/mm/page_allocator.o \ + common/mm/lockless_allocator.o \ nvgpu_common.o \ gk20a/gk20a.o \ gk20a/sched_gk20a.o \ @@ -51,11 +56,6 @@ nvgpu-y := \ gk20a/fb_gk20a.o \ gk20a/hal.o \ gk20a/hal_gk20a.o \ - gk20a/gk20a_allocator.o \ - gk20a/gk20a_allocator_bitmap.o \ - gk20a/gk20a_allocator_buddy.o \ - gk20a/gk20a_allocator_page.o \ - gk20a/gk20a_allocator_lockless.o \ gk20a/cde_gk20a.o \ gk20a/platform_gk20a_generic.o \ gk20a/tsg_gk20a.o \ diff --git a/drivers/gpu/nvgpu/gk20a/gk20a_allocator_bitmap.c b/drivers/gpu/nvgpu/common/mm/bitmap_allocator.c similarity index 76% rename from drivers/gpu/nvgpu/gk20a/gk20a_allocator_bitmap.c rename to drivers/gpu/nvgpu/common/mm/bitmap_allocator.c index f98e07820..6f267c852 100644 --- a/drivers/gpu/nvgpu/gk20a/gk20a_allocator_bitmap.c +++ b/drivers/gpu/nvgpu/common/mm/bitmap_allocator.c @@ -18,46 +18,47 @@ #include #include -#include "gk20a_allocator.h" +#include + #include "bitmap_allocator_priv.h" static struct kmem_cache *meta_data_cache; /* slab cache for meta data. */ static DEFINE_MUTEX(meta_data_cache_lock); -static u64 gk20a_bitmap_alloc_length(struct gk20a_allocator *a) +static u64 nvgpu_bitmap_alloc_length(struct nvgpu_allocator *a) { - struct gk20a_bitmap_allocator *ba = a->priv; + struct nvgpu_bitmap_allocator *ba = a->priv; return ba->length; } -static u64 gk20a_bitmap_alloc_base(struct gk20a_allocator *a) +static u64 nvgpu_bitmap_alloc_base(struct nvgpu_allocator *a) { - struct gk20a_bitmap_allocator *ba = a->priv; + struct nvgpu_bitmap_allocator *ba = a->priv; return ba->base; } -static int gk20a_bitmap_alloc_inited(struct gk20a_allocator *a) +static int nvgpu_bitmap_alloc_inited(struct nvgpu_allocator *a) { - struct gk20a_bitmap_allocator *ba = a->priv; + struct nvgpu_bitmap_allocator *ba = a->priv; int inited = ba->inited; rmb(); return inited; } -static u64 gk20a_bitmap_alloc_end(struct gk20a_allocator *a) +static u64 nvgpu_bitmap_alloc_end(struct nvgpu_allocator *a) { - struct gk20a_bitmap_allocator *ba = a->priv; + struct nvgpu_bitmap_allocator *ba = a->priv; return ba->base + ba->length; } -static u64 gk20a_bitmap_alloc_fixed(struct gk20a_allocator *__a, +static u64 nvgpu_bitmap_alloc_fixed(struct nvgpu_allocator *__a, u64 base, u64 len) { - struct gk20a_bitmap_allocator *a = bitmap_allocator(__a); + struct nvgpu_bitmap_allocator *a = bitmap_allocator(__a); u64 blks, offs, ret; /* Compute the bit offset and make sure it's aligned to a block. */ @@ -101,10 +102,10 @@ fail: * Note: this function won't do much error checking. Thus you could really * confuse the allocator if you misuse this function. */ -static void gk20a_bitmap_free_fixed(struct gk20a_allocator *__a, +static void nvgpu_bitmap_free_fixed(struct nvgpu_allocator *__a, u64 base, u64 len) { - struct gk20a_bitmap_allocator *a = bitmap_allocator(__a); + struct nvgpu_bitmap_allocator *a = bitmap_allocator(__a); u64 blks, offs; offs = base >> a->blk_shift; @@ -129,15 +130,15 @@ static void gk20a_bitmap_free_fixed(struct gk20a_allocator *__a, /* * Add the passed alloc to the tree of stored allocations. */ -static void insert_alloc_metadata(struct gk20a_bitmap_allocator *a, - struct gk20a_bitmap_alloc *alloc) +static void insert_alloc_metadata(struct nvgpu_bitmap_allocator *a, + struct nvgpu_bitmap_alloc *alloc) { struct rb_node **new = &a->allocs.rb_node; struct rb_node *parent = NULL; - struct gk20a_bitmap_alloc *tmp; + struct nvgpu_bitmap_alloc *tmp; while (*new) { - tmp = container_of(*new, struct gk20a_bitmap_alloc, + tmp = container_of(*new, struct nvgpu_bitmap_alloc, alloc_entry); parent = *new; @@ -158,14 +159,14 @@ static void insert_alloc_metadata(struct gk20a_bitmap_allocator *a, /* * Find and remove meta-data from the outstanding allocations. */ -static struct gk20a_bitmap_alloc *find_alloc_metadata( - struct gk20a_bitmap_allocator *a, u64 addr) +static struct nvgpu_bitmap_alloc *find_alloc_metadata( + struct nvgpu_bitmap_allocator *a, u64 addr) { struct rb_node *node = a->allocs.rb_node; - struct gk20a_bitmap_alloc *alloc; + struct nvgpu_bitmap_alloc *alloc; while (node) { - alloc = container_of(node, struct gk20a_bitmap_alloc, + alloc = container_of(node, struct nvgpu_bitmap_alloc, alloc_entry); if (addr < alloc->base) @@ -187,10 +188,10 @@ static struct gk20a_bitmap_alloc *find_alloc_metadata( /* * Tree of alloc meta data stores the address of the alloc not the bit offset. */ -static int __gk20a_bitmap_store_alloc(struct gk20a_bitmap_allocator *a, +static int __nvgpu_bitmap_store_alloc(struct nvgpu_bitmap_allocator *a, u64 addr, u64 len) { - struct gk20a_bitmap_alloc *alloc = + struct nvgpu_bitmap_alloc *alloc = kmem_cache_alloc(meta_data_cache, GFP_KERNEL); if (!alloc) @@ -208,11 +209,11 @@ static int __gk20a_bitmap_store_alloc(struct gk20a_bitmap_allocator *a, * @len is in bytes. This routine will figure out the right number of bits to * actually allocate. The return is the address in bytes as well. */ -static u64 gk20a_bitmap_alloc(struct gk20a_allocator *__a, u64 len) +static u64 nvgpu_bitmap_alloc(struct nvgpu_allocator *__a, u64 len) { u64 blks, addr; unsigned long offs, adjusted_offs, limit; - struct gk20a_bitmap_allocator *a = bitmap_allocator(__a); + struct nvgpu_bitmap_allocator *a = bitmap_allocator(__a); blks = len >> a->blk_shift; @@ -255,7 +256,7 @@ static u64 gk20a_bitmap_alloc(struct gk20a_allocator *__a, u64 len) * allocation. */ if (!(a->flags & GPU_ALLOC_NO_ALLOC_PAGE) && - __gk20a_bitmap_store_alloc(a, addr, blks * a->blk_size)) + __nvgpu_bitmap_store_alloc(a, addr, blks * a->blk_size)) goto fail_reset_bitmap; alloc_dbg(__a, "Alloc 0x%-10llx 0x%-5llx [bits=0x%llx (%llu)]\n", @@ -276,10 +277,10 @@ fail: return 0; } -static void gk20a_bitmap_free(struct gk20a_allocator *__a, u64 addr) +static void nvgpu_bitmap_free(struct nvgpu_allocator *__a, u64 addr) { - struct gk20a_bitmap_allocator *a = bitmap_allocator(__a); - struct gk20a_bitmap_alloc *alloc = NULL; + struct nvgpu_bitmap_allocator *a = bitmap_allocator(__a); + struct nvgpu_bitmap_alloc *alloc = NULL; u64 offs, adjusted_offs, blks; alloc_lock(__a); @@ -312,17 +313,17 @@ done: alloc_unlock(__a); } -static void gk20a_bitmap_alloc_destroy(struct gk20a_allocator *__a) +static void nvgpu_bitmap_alloc_destroy(struct nvgpu_allocator *__a) { - struct gk20a_bitmap_allocator *a = bitmap_allocator(__a); - struct gk20a_bitmap_alloc *alloc; + struct nvgpu_bitmap_allocator *a = bitmap_allocator(__a); + struct nvgpu_bitmap_alloc *alloc; struct rb_node *node; /* * Kill any outstanding allocations. */ while ((node = rb_first(&a->allocs)) != NULL) { - alloc = container_of(node, struct gk20a_bitmap_alloc, + alloc = container_of(node, struct nvgpu_bitmap_alloc, alloc_entry); rb_erase(node, &a->allocs); @@ -333,10 +334,10 @@ static void gk20a_bitmap_alloc_destroy(struct gk20a_allocator *__a) kfree(a); } -static void gk20a_bitmap_print_stats(struct gk20a_allocator *__a, +static void nvgpu_bitmap_print_stats(struct nvgpu_allocator *__a, struct seq_file *s, int lock) { - struct gk20a_bitmap_allocator *a = bitmap_allocator(__a); + struct nvgpu_bitmap_allocator *a = bitmap_allocator(__a); __alloc_pstat(s, __a, "Bitmap allocator params:\n"); __alloc_pstat(s, __a, " start = 0x%llx\n", a->base); @@ -353,34 +354,34 @@ static void gk20a_bitmap_print_stats(struct gk20a_allocator *__a, a->bytes_alloced - a->bytes_freed); } -static const struct gk20a_allocator_ops bitmap_ops = { - .alloc = gk20a_bitmap_alloc, - .free = gk20a_bitmap_free, +static const struct nvgpu_allocator_ops bitmap_ops = { + .alloc = nvgpu_bitmap_alloc, + .free = nvgpu_bitmap_free, - .alloc_fixed = gk20a_bitmap_alloc_fixed, - .free_fixed = gk20a_bitmap_free_fixed, + .alloc_fixed = nvgpu_bitmap_alloc_fixed, + .free_fixed = nvgpu_bitmap_free_fixed, - .base = gk20a_bitmap_alloc_base, - .length = gk20a_bitmap_alloc_length, - .end = gk20a_bitmap_alloc_end, - .inited = gk20a_bitmap_alloc_inited, + .base = nvgpu_bitmap_alloc_base, + .length = nvgpu_bitmap_alloc_length, + .end = nvgpu_bitmap_alloc_end, + .inited = nvgpu_bitmap_alloc_inited, - .fini = gk20a_bitmap_alloc_destroy, + .fini = nvgpu_bitmap_alloc_destroy, - .print_stats = gk20a_bitmap_print_stats, + .print_stats = nvgpu_bitmap_print_stats, }; -int gk20a_bitmap_allocator_init(struct gk20a *g, struct gk20a_allocator *__a, +int nvgpu_bitmap_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a, const char *name, u64 base, u64 length, u64 blk_size, u64 flags) { int err; - struct gk20a_bitmap_allocator *a; + struct nvgpu_bitmap_allocator *a; mutex_lock(&meta_data_cache_lock); if (!meta_data_cache) - meta_data_cache = KMEM_CACHE(gk20a_bitmap_alloc, 0); + meta_data_cache = KMEM_CACHE(nvgpu_bitmap_alloc, 0); mutex_unlock(&meta_data_cache_lock); if (!meta_data_cache) @@ -402,11 +403,11 @@ int gk20a_bitmap_allocator_init(struct gk20a *g, struct gk20a_allocator *__a, length -= blk_size; } - a = kzalloc(sizeof(struct gk20a_bitmap_allocator), GFP_KERNEL); + a = kzalloc(sizeof(struct nvgpu_bitmap_allocator), GFP_KERNEL); if (!a) return -ENOMEM; - err = __gk20a_alloc_common_init(__a, name, a, false, &bitmap_ops); + err = __nvgpu_alloc_common_init(__a, name, a, false, &bitmap_ops); if (err) goto fail; @@ -426,7 +427,7 @@ int gk20a_bitmap_allocator_init(struct gk20a *g, struct gk20a_allocator *__a, wmb(); a->inited = true; - gk20a_init_alloc_debug(g, __a); + nvgpu_init_alloc_debug(g, __a); alloc_dbg(__a, "New allocator: type bitmap\n"); alloc_dbg(__a, " base 0x%llx\n", a->base); alloc_dbg(__a, " bit_offs 0x%llx\n", a->bit_offs); diff --git a/drivers/gpu/nvgpu/gk20a/bitmap_allocator_priv.h b/drivers/gpu/nvgpu/common/mm/bitmap_allocator_priv.h similarity index 87% rename from drivers/gpu/nvgpu/gk20a/bitmap_allocator_priv.h rename to drivers/gpu/nvgpu/common/mm/bitmap_allocator_priv.h index a686b7043..9802b9db6 100644 --- a/drivers/gpu/nvgpu/gk20a/bitmap_allocator_priv.h +++ b/drivers/gpu/nvgpu/common/mm/bitmap_allocator_priv.h @@ -19,10 +19,10 @@ #include -struct gk20a_allocator; +struct nvgpu_allocator; -struct gk20a_bitmap_allocator { - struct gk20a_allocator *owner; +struct nvgpu_bitmap_allocator { + struct nvgpu_allocator *owner; u64 base; /* Base address of the space. */ u64 length; /* Length of the space. */ @@ -54,16 +54,16 @@ struct gk20a_bitmap_allocator { u64 bytes_freed; }; -struct gk20a_bitmap_alloc { +struct nvgpu_bitmap_alloc { u64 base; u64 length; struct rb_node alloc_entry; /* RB tree of allocations. */ }; -static inline struct gk20a_bitmap_allocator *bitmap_allocator( - struct gk20a_allocator *a) +static inline struct nvgpu_bitmap_allocator *bitmap_allocator( + struct nvgpu_allocator *a) { - return (struct gk20a_bitmap_allocator *)(a)->priv; + return (struct nvgpu_bitmap_allocator *)(a)->priv; } diff --git a/drivers/gpu/nvgpu/gk20a/gk20a_allocator_buddy.c b/drivers/gpu/nvgpu/common/mm/buddy_allocator.c similarity index 79% rename from drivers/gpu/nvgpu/gk20a/gk20a_allocator_buddy.c rename to drivers/gpu/nvgpu/common/mm/buddy_allocator.c index 3715e9f80..39a53801f 100644 --- a/drivers/gpu/nvgpu/gk20a/gk20a_allocator_buddy.c +++ b/drivers/gpu/nvgpu/common/mm/buddy_allocator.c @@ -17,20 +17,22 @@ #include #include -#include "mm_gk20a.h" -#include "platform_gk20a.h" -#include "gk20a_allocator.h" +#include + +#include "gk20a/mm_gk20a.h" +#include "gk20a/platform_gk20a.h" + #include "buddy_allocator_priv.h" static struct kmem_cache *buddy_cache; /* slab cache for meta data. */ /* Some other buddy allocator functions. */ -static struct gk20a_buddy *balloc_free_buddy(struct gk20a_buddy_allocator *a, +static struct nvgpu_buddy *balloc_free_buddy(struct nvgpu_buddy_allocator *a, u64 addr); -static void balloc_coalesce(struct gk20a_buddy_allocator *a, - struct gk20a_buddy *b); -static void __balloc_do_free_fixed(struct gk20a_buddy_allocator *a, - struct gk20a_fixed_alloc *falloc); +static void balloc_coalesce(struct nvgpu_buddy_allocator *a, + struct nvgpu_buddy *b); +static void __balloc_do_free_fixed(struct nvgpu_buddy_allocator *a, + struct nvgpu_fixed_alloc *falloc); /* * This function is not present in older kernel's list.h code. @@ -64,7 +66,7 @@ static void __balloc_do_free_fixed(struct gk20a_buddy_allocator *a, * Hueristic: Just guessing that the best max order is the largest single * block that will fit in the address space. */ -static void balloc_compute_max_order(struct gk20a_buddy_allocator *a) +static void balloc_compute_max_order(struct nvgpu_buddy_allocator *a) { u64 true_max_order = ilog2(a->blks); @@ -83,7 +85,7 @@ static void balloc_compute_max_order(struct gk20a_buddy_allocator *a) * Since we can only allocate in chucks of a->blk_size we need to trim off * any excess data that is not aligned to a->blk_size. */ -static void balloc_allocator_align(struct gk20a_buddy_allocator *a) +static void balloc_allocator_align(struct nvgpu_buddy_allocator *a) { a->start = ALIGN(a->base, a->blk_size); WARN_ON(a->start != a->base); @@ -95,17 +97,17 @@ static void balloc_allocator_align(struct gk20a_buddy_allocator *a) /* * Pass NULL for parent if you want a top level buddy. */ -static struct gk20a_buddy *balloc_new_buddy(struct gk20a_buddy_allocator *a, - struct gk20a_buddy *parent, +static struct nvgpu_buddy *balloc_new_buddy(struct nvgpu_buddy_allocator *a, + struct nvgpu_buddy *parent, u64 start, u64 order) { - struct gk20a_buddy *new_buddy; + struct nvgpu_buddy *new_buddy; new_buddy = kmem_cache_alloc(buddy_cache, GFP_KERNEL); if (!new_buddy) return NULL; - memset(new_buddy, 0, sizeof(struct gk20a_buddy)); + memset(new_buddy, 0, sizeof(struct nvgpu_buddy)); new_buddy->parent = parent; new_buddy->start = start; @@ -116,8 +118,8 @@ static struct gk20a_buddy *balloc_new_buddy(struct gk20a_buddy_allocator *a, return new_buddy; } -static void __balloc_buddy_list_add(struct gk20a_buddy_allocator *a, - struct gk20a_buddy *b, +static void __balloc_buddy_list_add(struct nvgpu_buddy_allocator *a, + struct nvgpu_buddy *b, struct list_head *list) { if (buddy_is_in_list(b)) { @@ -141,8 +143,8 @@ static void __balloc_buddy_list_add(struct gk20a_buddy_allocator *a, buddy_set_in_list(b); } -static void __balloc_buddy_list_rem(struct gk20a_buddy_allocator *a, - struct gk20a_buddy *b) +static void __balloc_buddy_list_rem(struct nvgpu_buddy_allocator *a, + struct nvgpu_buddy *b) { if (!buddy_is_in_list(b)) { alloc_dbg(balloc_owner(a), @@ -159,21 +161,21 @@ static void __balloc_buddy_list_rem(struct gk20a_buddy_allocator *a, * Add a buddy to one of the buddy lists and deal with the necessary * book keeping. Adds the buddy to the list specified by the buddy's order. */ -static void balloc_blist_add(struct gk20a_buddy_allocator *a, - struct gk20a_buddy *b) +static void balloc_blist_add(struct nvgpu_buddy_allocator *a, + struct nvgpu_buddy *b) { __balloc_buddy_list_add(a, b, balloc_get_order_list(a, b->order)); a->buddy_list_len[b->order]++; } -static void balloc_blist_rem(struct gk20a_buddy_allocator *a, - struct gk20a_buddy *b) +static void balloc_blist_rem(struct nvgpu_buddy_allocator *a, + struct nvgpu_buddy *b) { __balloc_buddy_list_rem(a, b); a->buddy_list_len[b->order]--; } -static u64 balloc_get_order(struct gk20a_buddy_allocator *a, u64 len) +static u64 balloc_get_order(struct nvgpu_buddy_allocator *a, u64 len) { if (len == 0) return 0; @@ -184,7 +186,7 @@ static u64 balloc_get_order(struct gk20a_buddy_allocator *a, u64 len) return fls(len); } -static u64 __balloc_max_order_in(struct gk20a_buddy_allocator *a, +static u64 __balloc_max_order_in(struct nvgpu_buddy_allocator *a, u64 start, u64 end) { u64 size = (end - start) >> a->blk_shift; @@ -198,11 +200,11 @@ static u64 __balloc_max_order_in(struct gk20a_buddy_allocator *a, /* * Initialize the buddy lists. */ -static int balloc_init_lists(struct gk20a_buddy_allocator *a) +static int balloc_init_lists(struct nvgpu_buddy_allocator *a) { int i; u64 bstart, bend, order; - struct gk20a_buddy *buddy; + struct nvgpu_buddy *buddy; bstart = a->start; bend = a->end; @@ -228,7 +230,7 @@ cleanup: for (i = 0; i < GPU_BALLOC_ORDER_LIST_LEN; i++) { if (!list_empty(balloc_get_order_list(a, i))) { buddy = list_first_entry(balloc_get_order_list(a, i), - struct gk20a_buddy, buddy_entry); + struct nvgpu_buddy, buddy_entry); balloc_blist_rem(a, buddy); kmem_cache_free(buddy_cache, buddy); } @@ -240,24 +242,24 @@ cleanup: /* * Clean up and destroy the passed allocator. */ -static void gk20a_buddy_allocator_destroy(struct gk20a_allocator *__a) +static void nvgpu_buddy_allocator_destroy(struct nvgpu_allocator *__a) { int i; struct rb_node *node; - struct gk20a_buddy *bud; - struct gk20a_fixed_alloc *falloc; - struct gk20a_buddy_allocator *a = __a->priv; + struct nvgpu_buddy *bud; + struct nvgpu_fixed_alloc *falloc; + struct nvgpu_buddy_allocator *a = __a->priv; alloc_lock(__a); - gk20a_fini_alloc_debug(__a); + nvgpu_fini_alloc_debug(__a); /* * Free the fixed allocs first. */ while ((node = rb_first(&a->fixed_allocs)) != NULL) { falloc = container_of(node, - struct gk20a_fixed_alloc, alloced_entry); + struct nvgpu_fixed_alloc, alloced_entry); rb_erase(node, &a->fixed_allocs); __balloc_do_free_fixed(a, falloc); @@ -267,7 +269,7 @@ static void gk20a_buddy_allocator_destroy(struct gk20a_allocator *__a) * And now free all outstanding allocations. */ while ((node = rb_first(&a->alloced_buddies)) != NULL) { - bud = container_of(node, struct gk20a_buddy, alloced_entry); + bud = container_of(node, struct nvgpu_buddy, alloced_entry); balloc_free_buddy(a, bud->start); balloc_blist_add(a, bud); balloc_coalesce(a, bud); @@ -281,7 +283,7 @@ static void gk20a_buddy_allocator_destroy(struct gk20a_allocator *__a) while (!list_empty(balloc_get_order_list(a, i))) { bud = list_first_entry(balloc_get_order_list(a, i), - struct gk20a_buddy, buddy_entry); + struct nvgpu_buddy, buddy_entry); balloc_blist_rem(a, bud); kmem_cache_free(buddy_cache, bud); } @@ -314,10 +316,10 @@ static void gk20a_buddy_allocator_destroy(struct gk20a_allocator *__a) * * @a must be locked. */ -static void balloc_coalesce(struct gk20a_buddy_allocator *a, - struct gk20a_buddy *b) +static void balloc_coalesce(struct nvgpu_buddy_allocator *a, + struct nvgpu_buddy *b) { - struct gk20a_buddy *parent; + struct nvgpu_buddy *parent; if (buddy_is_alloced(b) || buddy_is_split(b)) return; @@ -355,10 +357,10 @@ static void balloc_coalesce(struct gk20a_buddy_allocator *a, * * @a must be locked. */ -static int balloc_split_buddy(struct gk20a_buddy_allocator *a, - struct gk20a_buddy *b, int pte_size) +static int balloc_split_buddy(struct nvgpu_buddy_allocator *a, + struct nvgpu_buddy *b, int pte_size) { - struct gk20a_buddy *left, *right; + struct nvgpu_buddy *left, *right; u64 half; left = balloc_new_buddy(a, b, b->start, b->order - 1); @@ -403,14 +405,14 @@ static int balloc_split_buddy(struct gk20a_buddy_allocator *a, * * @a must be locked. */ -static void balloc_alloc_buddy(struct gk20a_buddy_allocator *a, - struct gk20a_buddy *b) +static void balloc_alloc_buddy(struct nvgpu_buddy_allocator *a, + struct nvgpu_buddy *b) { struct rb_node **new = &(a->alloced_buddies.rb_node); struct rb_node *parent = NULL; while (*new) { - struct gk20a_buddy *bud = container_of(*new, struct gk20a_buddy, + struct nvgpu_buddy *bud = container_of(*new, struct nvgpu_buddy, alloced_entry); parent = *new; @@ -435,14 +437,14 @@ static void balloc_alloc_buddy(struct gk20a_buddy_allocator *a, * * @a must be locked. */ -static struct gk20a_buddy *balloc_free_buddy(struct gk20a_buddy_allocator *a, +static struct nvgpu_buddy *balloc_free_buddy(struct nvgpu_buddy_allocator *a, u64 addr) { struct rb_node *node = a->alloced_buddies.rb_node; - struct gk20a_buddy *bud; + struct nvgpu_buddy *bud; while (node) { - bud = container_of(node, struct gk20a_buddy, alloced_entry); + bud = container_of(node, struct nvgpu_buddy, alloced_entry); if (addr < bud->start) node = node->rb_left; @@ -465,10 +467,10 @@ static struct gk20a_buddy *balloc_free_buddy(struct gk20a_buddy_allocator *a, /* * Find a suitable buddy for the given order and PTE type (big or little). */ -static struct gk20a_buddy *__balloc_find_buddy(struct gk20a_buddy_allocator *a, +static struct nvgpu_buddy *__balloc_find_buddy(struct nvgpu_buddy_allocator *a, u64 order, int pte_size) { - struct gk20a_buddy *bud; + struct nvgpu_buddy *bud; if (order > a->max_order || list_empty(balloc_get_order_list(a, order))) @@ -477,10 +479,10 @@ static struct gk20a_buddy *__balloc_find_buddy(struct gk20a_buddy_allocator *a, if (a->flags & GPU_ALLOC_GVA_SPACE && pte_size == gmmu_page_size_big) bud = list_last_entry(balloc_get_order_list(a, order), - struct gk20a_buddy, buddy_entry); + struct nvgpu_buddy, buddy_entry); else bud = list_first_entry(balloc_get_order_list(a, order), - struct gk20a_buddy, buddy_entry); + struct nvgpu_buddy, buddy_entry); if (bud->pte_size != BALLOC_PTE_SIZE_ANY && bud->pte_size != pte_size) @@ -498,11 +500,11 @@ static struct gk20a_buddy *__balloc_find_buddy(struct gk20a_buddy_allocator *a, * * @a must be locked. */ -static u64 __balloc_do_alloc(struct gk20a_buddy_allocator *a, +static u64 __balloc_do_alloc(struct nvgpu_buddy_allocator *a, u64 order, int pte_size) { u64 split_order; - struct gk20a_buddy *bud = NULL; + struct nvgpu_buddy *bud = NULL; split_order = order; while (split_order <= a->max_order && @@ -532,17 +534,17 @@ static u64 __balloc_do_alloc(struct gk20a_buddy_allocator *a, * TODO: Right now this uses the unoptimal approach of going through all * outstanding allocations and checking their base/ends. This could be better. */ -static int balloc_is_range_free(struct gk20a_buddy_allocator *a, +static int balloc_is_range_free(struct nvgpu_buddy_allocator *a, u64 base, u64 end) { struct rb_node *node; - struct gk20a_buddy *bud; + struct nvgpu_buddy *bud; node = rb_first(&a->alloced_buddies); if (!node) return 1; /* No allocs yet. */ - bud = container_of(node, struct gk20a_buddy, alloced_entry); + bud = container_of(node, struct nvgpu_buddy, alloced_entry); while (bud->start < end) { if ((bud->start > base && bud->start < end) || @@ -552,21 +554,21 @@ static int balloc_is_range_free(struct gk20a_buddy_allocator *a, node = rb_next(node); if (!node) break; - bud = container_of(node, struct gk20a_buddy, alloced_entry); + bud = container_of(node, struct nvgpu_buddy, alloced_entry); } return 1; } -static void balloc_alloc_fixed(struct gk20a_buddy_allocator *a, - struct gk20a_fixed_alloc *f) +static void balloc_alloc_fixed(struct nvgpu_buddy_allocator *a, + struct nvgpu_fixed_alloc *f) { struct rb_node **new = &(a->fixed_allocs.rb_node); struct rb_node *parent = NULL; while (*new) { - struct gk20a_fixed_alloc *falloc = - container_of(*new, struct gk20a_fixed_alloc, + struct nvgpu_fixed_alloc *falloc = + container_of(*new, struct nvgpu_fixed_alloc, alloced_entry); BUG_ON(!virt_addr_valid(falloc)); @@ -590,15 +592,15 @@ static void balloc_alloc_fixed(struct gk20a_buddy_allocator *a, * * @a must be locked. */ -static struct gk20a_fixed_alloc *balloc_free_fixed( - struct gk20a_buddy_allocator *a, u64 addr) +static struct nvgpu_fixed_alloc *balloc_free_fixed( + struct nvgpu_buddy_allocator *a, u64 addr) { struct rb_node *node = a->fixed_allocs.rb_node; - struct gk20a_fixed_alloc *falloc; + struct nvgpu_fixed_alloc *falloc; while (node) { falloc = container_of(node, - struct gk20a_fixed_alloc, alloced_entry); + struct nvgpu_fixed_alloc, alloced_entry); if (addr < falloc->start) node = node->rb_left; @@ -620,7 +622,7 @@ static struct gk20a_fixed_alloc *balloc_free_fixed( * Find the parent range - doesn't necessarily need the parent to actually exist * as a buddy. Finding an existing parent comes later... */ -static void __balloc_get_parent_range(struct gk20a_buddy_allocator *a, +static void __balloc_get_parent_range(struct nvgpu_buddy_allocator *a, u64 base, u64 order, u64 *pbase, u64 *porder) { @@ -640,10 +642,10 @@ static void __balloc_get_parent_range(struct gk20a_buddy_allocator *a, * Makes a buddy at the passed address. This will make all parent buddies * necessary for this buddy to exist as well. */ -static struct gk20a_buddy *__balloc_make_fixed_buddy( - struct gk20a_buddy_allocator *a, u64 base, u64 order) +static struct nvgpu_buddy *__balloc_make_fixed_buddy( + struct nvgpu_buddy_allocator *a, u64 base, u64 order) { - struct gk20a_buddy *bud = NULL; + struct nvgpu_buddy *bud = NULL; struct list_head *order_list; u64 cur_order = order, cur_base = base; @@ -696,8 +698,8 @@ static struct gk20a_buddy *__balloc_make_fixed_buddy( return bud; } -static u64 __balloc_do_alloc_fixed(struct gk20a_buddy_allocator *a, - struct gk20a_fixed_alloc *falloc, +static u64 __balloc_do_alloc_fixed(struct nvgpu_buddy_allocator *a, + struct nvgpu_fixed_alloc *falloc, u64 base, u64 len) { u64 shifted_base, inc_base; @@ -725,7 +727,7 @@ static u64 __balloc_do_alloc_fixed(struct gk20a_buddy_allocator *a, while (inc_base < (shifted_base + len)) { u64 order_len = balloc_order_to_len(a, align_order); u64 remaining; - struct gk20a_buddy *bud; + struct nvgpu_buddy *bud; bud = __balloc_make_fixed_buddy(a, balloc_base_unshift(a, inc_base), @@ -757,8 +759,8 @@ static u64 __balloc_do_alloc_fixed(struct gk20a_buddy_allocator *a, err_and_cleanup: while (!list_empty(&falloc->buddies)) { - struct gk20a_buddy *bud = list_first_entry(&falloc->buddies, - struct gk20a_buddy, + struct nvgpu_buddy *bud = list_first_entry(&falloc->buddies, + struct nvgpu_buddy, buddy_entry); __balloc_buddy_list_rem(a, bud); @@ -769,14 +771,14 @@ err_and_cleanup: return 0; } -static void __balloc_do_free_fixed(struct gk20a_buddy_allocator *a, - struct gk20a_fixed_alloc *falloc) +static void __balloc_do_free_fixed(struct nvgpu_buddy_allocator *a, + struct nvgpu_fixed_alloc *falloc) { - struct gk20a_buddy *bud; + struct nvgpu_buddy *bud; while (!list_empty(&falloc->buddies)) { bud = list_first_entry(&falloc->buddies, - struct gk20a_buddy, + struct nvgpu_buddy, buddy_entry); __balloc_buddy_list_rem(a, bud); @@ -796,13 +798,13 @@ static void __balloc_do_free_fixed(struct gk20a_buddy_allocator *a, /* * Allocate memory from the passed allocator. */ -static u64 gk20a_buddy_balloc(struct gk20a_allocator *__a, u64 len) +static u64 nvgpu_buddy_balloc(struct nvgpu_allocator *__a, u64 len) { u64 order, addr; int pte_size; - struct gk20a_buddy_allocator *a = __a->priv; + struct nvgpu_buddy_allocator *a = __a->priv; - gk20a_alloc_trace_func(); + nvgpu_alloc_trace_func(); alloc_lock(__a); @@ -811,7 +813,7 @@ static u64 gk20a_buddy_balloc(struct gk20a_allocator *__a, u64 len) if (order > a->max_order) { alloc_unlock(__a); alloc_dbg(balloc_owner(a), "Alloc fail\n"); - gk20a_alloc_trace_func_done(); + nvgpu_alloc_trace_func_done(); return 0; } @@ -848,22 +850,22 @@ static u64 gk20a_buddy_balloc(struct gk20a_allocator *__a, u64 len) alloc_unlock(__a); - gk20a_alloc_trace_func_done(); + nvgpu_alloc_trace_func_done(); return addr; } /* * Requires @__a to be locked. */ -static u64 __gk20a_balloc_fixed_buddy(struct gk20a_allocator *__a, +static u64 __nvgpu_balloc_fixed_buddy(struct nvgpu_allocator *__a, u64 base, u64 len) { u64 ret, real_bytes = 0; - struct gk20a_buddy *bud; - struct gk20a_fixed_alloc *falloc = NULL; - struct gk20a_buddy_allocator *a = __a->priv; + struct nvgpu_buddy *bud; + struct nvgpu_fixed_alloc *falloc = NULL; + struct nvgpu_buddy_allocator *a = __a->priv; - gk20a_alloc_trace_func(); + nvgpu_alloc_trace_func(); /* If base isn't aligned to an order 0 block, fail. */ if (base & (a->blk_size - 1)) @@ -905,14 +907,14 @@ static u64 __gk20a_balloc_fixed_buddy(struct gk20a_allocator *__a, alloc_dbg(balloc_owner(a), "Alloc (fixed) 0x%llx\n", base); - gk20a_alloc_trace_func_done(); + nvgpu_alloc_trace_func_done(); return base; fail_unlock: alloc_unlock(__a); fail: kfree(falloc); - gk20a_alloc_trace_func_done(); + nvgpu_alloc_trace_func_done(); return 0; } @@ -924,14 +926,14 @@ fail: * * Please do not use this function unless _absolutely_ necessary. */ -static u64 gk20a_balloc_fixed_buddy(struct gk20a_allocator *__a, +static u64 nvgpu_balloc_fixed_buddy(struct nvgpu_allocator *__a, u64 base, u64 len) { u64 alloc; - struct gk20a_buddy_allocator *a = __a->priv; + struct nvgpu_buddy_allocator *a = __a->priv; alloc_lock(__a); - alloc = __gk20a_balloc_fixed_buddy(__a, base, len); + alloc = __nvgpu_balloc_fixed_buddy(__a, base, len); a->alloc_made = 1; alloc_unlock(__a); @@ -941,16 +943,16 @@ static u64 gk20a_balloc_fixed_buddy(struct gk20a_allocator *__a, /* * Free the passed allocation. */ -static void gk20a_buddy_bfree(struct gk20a_allocator *__a, u64 addr) +static void nvgpu_buddy_bfree(struct nvgpu_allocator *__a, u64 addr) { - struct gk20a_buddy *bud; - struct gk20a_fixed_alloc *falloc; - struct gk20a_buddy_allocator *a = __a->priv; + struct nvgpu_buddy *bud; + struct nvgpu_fixed_alloc *falloc; + struct nvgpu_buddy_allocator *a = __a->priv; - gk20a_alloc_trace_func(); + nvgpu_alloc_trace_func(); if (!addr) { - gk20a_alloc_trace_func_done(); + nvgpu_alloc_trace_func_done(); return; } @@ -981,14 +983,14 @@ static void gk20a_buddy_bfree(struct gk20a_allocator *__a, u64 addr) done: alloc_unlock(__a); alloc_dbg(balloc_owner(a), "Free 0x%llx\n", addr); - gk20a_alloc_trace_func_done(); + nvgpu_alloc_trace_func_done(); return; } -static bool gk20a_buddy_reserve_is_possible(struct gk20a_buddy_allocator *a, - struct gk20a_alloc_carveout *co) +static bool nvgpu_buddy_reserve_is_possible(struct nvgpu_buddy_allocator *a, + struct nvgpu_alloc_carveout *co) { - struct gk20a_alloc_carveout *tmp; + struct nvgpu_alloc_carveout *tmp; u64 co_base, co_end; co_base = co->base; @@ -1013,10 +1015,10 @@ static bool gk20a_buddy_reserve_is_possible(struct gk20a_buddy_allocator *a, * Carveouts can only be reserved before any regular allocations have been * made. */ -static int gk20a_buddy_reserve_co(struct gk20a_allocator *__a, - struct gk20a_alloc_carveout *co) +static int nvgpu_buddy_reserve_co(struct nvgpu_allocator *__a, + struct nvgpu_alloc_carveout *co) { - struct gk20a_buddy_allocator *a = __a->priv; + struct nvgpu_buddy_allocator *a = __a->priv; u64 addr; int err = 0; @@ -1026,13 +1028,13 @@ static int gk20a_buddy_reserve_co(struct gk20a_allocator *__a, alloc_lock(__a); - if (!gk20a_buddy_reserve_is_possible(a, co)) { + if (!nvgpu_buddy_reserve_is_possible(a, co)) { err = -EBUSY; goto done; } /* Should not be possible to fail... */ - addr = __gk20a_balloc_fixed_buddy(__a, co->base, co->length); + addr = __nvgpu_balloc_fixed_buddy(__a, co->base, co->length); if (!addr) { err = -ENOMEM; pr_warn("%s: Failed to reserve a valid carveout!\n", __func__); @@ -1049,50 +1051,50 @@ done: /* * Carveouts can be release at any time. */ -static void gk20a_buddy_release_co(struct gk20a_allocator *__a, - struct gk20a_alloc_carveout *co) +static void nvgpu_buddy_release_co(struct nvgpu_allocator *__a, + struct nvgpu_alloc_carveout *co) { alloc_lock(__a); list_del_init(&co->co_entry); - gk20a_free(__a, co->base); + nvgpu_free(__a, co->base); alloc_unlock(__a); } -static u64 gk20a_buddy_alloc_length(struct gk20a_allocator *a) +static u64 nvgpu_buddy_alloc_length(struct nvgpu_allocator *a) { - struct gk20a_buddy_allocator *ba = a->priv; + struct nvgpu_buddy_allocator *ba = a->priv; return ba->length; } -static u64 gk20a_buddy_alloc_base(struct gk20a_allocator *a) +static u64 nvgpu_buddy_alloc_base(struct nvgpu_allocator *a) { - struct gk20a_buddy_allocator *ba = a->priv; + struct nvgpu_buddy_allocator *ba = a->priv; return ba->start; } -static int gk20a_buddy_alloc_inited(struct gk20a_allocator *a) +static int nvgpu_buddy_alloc_inited(struct nvgpu_allocator *a) { - struct gk20a_buddy_allocator *ba = a->priv; + struct nvgpu_buddy_allocator *ba = a->priv; int inited = ba->initialized; rmb(); return inited; } -static u64 gk20a_buddy_alloc_end(struct gk20a_allocator *a) +static u64 nvgpu_buddy_alloc_end(struct nvgpu_allocator *a) { - struct gk20a_buddy_allocator *ba = a->priv; + struct nvgpu_buddy_allocator *ba = a->priv; return ba->end; } -static u64 gk20a_buddy_alloc_space(struct gk20a_allocator *a) +static u64 nvgpu_buddy_alloc_space(struct nvgpu_allocator *a) { - struct gk20a_buddy_allocator *ba = a->priv; + struct nvgpu_buddy_allocator *ba = a->priv; u64 space; alloc_lock(a); @@ -1108,14 +1110,14 @@ static u64 gk20a_buddy_alloc_space(struct gk20a_allocator *a) * stats are printed to the kernel log. This lets this code be used for * debugging purposes internal to the allocator. */ -static void gk20a_buddy_print_stats(struct gk20a_allocator *__a, +static void nvgpu_buddy_print_stats(struct nvgpu_allocator *__a, struct seq_file *s, int lock) { int i = 0; struct rb_node *node; - struct gk20a_fixed_alloc *falloc; - struct gk20a_alloc_carveout *tmp; - struct gk20a_buddy_allocator *a = __a->priv; + struct nvgpu_fixed_alloc *falloc; + struct nvgpu_alloc_carveout *tmp; + struct nvgpu_buddy_allocator *a = __a->priv; __alloc_pstat(s, __a, "base = %llu, limit = %llu, blk_size = %llu\n", a->base, a->length, a->blk_size); @@ -1161,7 +1163,7 @@ static void gk20a_buddy_print_stats(struct gk20a_allocator *__a, node != NULL; node = rb_next(node)) { falloc = container_of(node, - struct gk20a_fixed_alloc, alloced_entry); + struct nvgpu_fixed_alloc, alloced_entry); __alloc_pstat(s, __a, "Fixed alloc (%d): [0x%llx -> 0x%llx]\n", i, falloc->start, falloc->end); @@ -1179,25 +1181,25 @@ static void gk20a_buddy_print_stats(struct gk20a_allocator *__a, alloc_unlock(__a); } -static const struct gk20a_allocator_ops buddy_ops = { - .alloc = gk20a_buddy_balloc, - .free = gk20a_buddy_bfree, +static const struct nvgpu_allocator_ops buddy_ops = { + .alloc = nvgpu_buddy_balloc, + .free = nvgpu_buddy_bfree, - .alloc_fixed = gk20a_balloc_fixed_buddy, + .alloc_fixed = nvgpu_balloc_fixed_buddy, /* .free_fixed not needed. */ - .reserve_carveout = gk20a_buddy_reserve_co, - .release_carveout = gk20a_buddy_release_co, + .reserve_carveout = nvgpu_buddy_reserve_co, + .release_carveout = nvgpu_buddy_release_co, - .base = gk20a_buddy_alloc_base, - .length = gk20a_buddy_alloc_length, - .end = gk20a_buddy_alloc_end, - .inited = gk20a_buddy_alloc_inited, - .space = gk20a_buddy_alloc_space, + .base = nvgpu_buddy_alloc_base, + .length = nvgpu_buddy_alloc_length, + .end = nvgpu_buddy_alloc_end, + .inited = nvgpu_buddy_alloc_inited, + .space = nvgpu_buddy_alloc_space, - .fini = gk20a_buddy_allocator_destroy, + .fini = nvgpu_buddy_allocator_destroy, - .print_stats = gk20a_buddy_print_stats, + .print_stats = nvgpu_buddy_print_stats, }; /* @@ -1218,14 +1220,14 @@ static const struct gk20a_allocator_ops buddy_ops = { * will try and pick a reasonable max order. * @flags: Extra flags necessary. See GPU_BALLOC_*. */ -int __gk20a_buddy_allocator_init(struct gk20a *g, struct gk20a_allocator *__a, +int __nvgpu_buddy_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a, struct vm_gk20a *vm, const char *name, u64 base, u64 size, u64 blk_size, u64 max_order, u64 flags) { int err; u64 pde_size; - struct gk20a_buddy_allocator *a; + struct nvgpu_buddy_allocator *a; /* blk_size must be greater than 0 and a power of 2. */ if (blk_size == 0) @@ -1240,11 +1242,11 @@ int __gk20a_buddy_allocator_init(struct gk20a *g, struct gk20a_allocator *__a, if (flags & GPU_ALLOC_GVA_SPACE && !vm) return -EINVAL; - a = kzalloc(sizeof(struct gk20a_buddy_allocator), GFP_KERNEL); + a = kzalloc(sizeof(struct nvgpu_buddy_allocator), GFP_KERNEL); if (!a) return -ENOMEM; - err = __gk20a_alloc_common_init(__a, name, a, false, &buddy_ops); + err = __nvgpu_alloc_common_init(__a, name, a, false, &buddy_ops); if (err) goto fail; @@ -1287,7 +1289,7 @@ int __gk20a_buddy_allocator_init(struct gk20a *g, struct gk20a_allocator *__a, /* Shared buddy kmem_cache for all allocators. */ if (!buddy_cache) - buddy_cache = KMEM_CACHE(gk20a_buddy, 0); + buddy_cache = KMEM_CACHE(nvgpu_buddy, 0); if (!buddy_cache) { err = -ENOMEM; goto fail; @@ -1303,7 +1305,7 @@ int __gk20a_buddy_allocator_init(struct gk20a *g, struct gk20a_allocator *__a, wmb(); a->initialized = 1; - gk20a_init_alloc_debug(g, __a); + nvgpu_init_alloc_debug(g, __a); alloc_dbg(__a, "New allocator: type buddy\n"); alloc_dbg(__a, " base 0x%llx\n", a->base); alloc_dbg(__a, " size 0x%llx\n", a->length); @@ -1318,10 +1320,10 @@ fail: return err; } -int gk20a_buddy_allocator_init(struct gk20a *g, struct gk20a_allocator *a, +int nvgpu_buddy_allocator_init(struct gk20a *g, struct nvgpu_allocator *a, const char *name, u64 base, u64 size, u64 blk_size, u64 flags) { - return __gk20a_buddy_allocator_init(g, a, NULL, name, + return __nvgpu_buddy_allocator_init(g, a, NULL, name, base, size, blk_size, 0, 0); } diff --git a/drivers/gpu/nvgpu/gk20a/buddy_allocator_priv.h b/drivers/gpu/nvgpu/common/mm/buddy_allocator_priv.h similarity index 73% rename from drivers/gpu/nvgpu/gk20a/buddy_allocator_priv.h rename to drivers/gpu/nvgpu/common/mm/buddy_allocator_priv.h index bb8b307b0..50a11f14e 100644 --- a/drivers/gpu/nvgpu/gk20a/buddy_allocator_priv.h +++ b/drivers/gpu/nvgpu/common/mm/buddy_allocator_priv.h @@ -20,17 +20,17 @@ #include #include -struct gk20a_allocator; +struct nvgpu_allocator; struct vm_gk20a; /* * Each buddy is an element in a binary tree. */ -struct gk20a_buddy { - struct gk20a_buddy *parent; /* Parent node. */ - struct gk20a_buddy *buddy; /* This node's buddy. */ - struct gk20a_buddy *left; /* Lower address sub-node. */ - struct gk20a_buddy *right; /* Higher address sub-node. */ +struct nvgpu_buddy { + struct nvgpu_buddy *parent; /* Parent node. */ + struct nvgpu_buddy *buddy; /* This node's buddy. */ + struct nvgpu_buddy *left; /* Lower address sub-node. */ + struct nvgpu_buddy *right; /* Higher address sub-node. */ struct list_head buddy_entry; /* List entry for various lists. */ struct rb_node alloced_entry; /* RB tree of allocations. */ @@ -54,31 +54,31 @@ struct gk20a_buddy { }; #define __buddy_flag_ops(flag, flag_up) \ - static inline int buddy_is_ ## flag(struct gk20a_buddy *b) \ + static inline int buddy_is_ ## flag(struct nvgpu_buddy *b) \ { \ return b->flags & BALLOC_BUDDY_ ## flag_up; \ } \ - static inline void buddy_set_ ## flag(struct gk20a_buddy *b) \ + static inline void buddy_set_ ## flag(struct nvgpu_buddy *b) \ { \ b->flags |= BALLOC_BUDDY_ ## flag_up; \ } \ - static inline void buddy_clr_ ## flag(struct gk20a_buddy *b) \ + static inline void buddy_clr_ ## flag(struct nvgpu_buddy *b) \ { \ b->flags &= ~BALLOC_BUDDY_ ## flag_up; \ } /* - * int buddy_is_alloced(struct gk20a_buddy *b); - * void buddy_set_alloced(struct gk20a_buddy *b); - * void buddy_clr_alloced(struct gk20a_buddy *b); + * int buddy_is_alloced(struct nvgpu_buddy *b); + * void buddy_set_alloced(struct nvgpu_buddy *b); + * void buddy_clr_alloced(struct nvgpu_buddy *b); * - * int buddy_is_split(struct gk20a_buddy *b); - * void buddy_set_split(struct gk20a_buddy *b); - * void buddy_clr_split(struct gk20a_buddy *b); + * int buddy_is_split(struct nvgpu_buddy *b); + * void buddy_set_split(struct nvgpu_buddy *b); + * void buddy_clr_split(struct nvgpu_buddy *b); * - * int buddy_is_in_list(struct gk20a_buddy *b); - * void buddy_set_in_list(struct gk20a_buddy *b); - * void buddy_clr_in_list(struct gk20a_buddy *b); + * int buddy_is_in_list(struct nvgpu_buddy *b); + * void buddy_set_in_list(struct nvgpu_buddy *b); + * void buddy_clr_in_list(struct nvgpu_buddy *b); */ __buddy_flag_ops(alloced, ALLOCED); __buddy_flag_ops(split, SPLIT); @@ -87,7 +87,7 @@ __buddy_flag_ops(in_list, IN_LIST); /* * Keeps info for a fixed allocation. */ -struct gk20a_fixed_alloc { +struct nvgpu_fixed_alloc { struct list_head buddies; /* List of buddies. */ struct rb_node alloced_entry; /* RB tree of fixed allocations. */ @@ -105,8 +105,8 @@ struct gk20a_fixed_alloc { * * order_size is the size of an order 0 buddy. */ -struct gk20a_buddy_allocator { - struct gk20a_allocator *owner; /* Owner of this buddy allocator. */ +struct nvgpu_buddy_allocator { + struct nvgpu_allocator *owner; /* Owner of this buddy allocator. */ struct vm_gk20a *vm; /* Parent VM - can be NULL. */ u64 base; /* Base address of the space. */ @@ -153,38 +153,38 @@ struct gk20a_buddy_allocator { u64 bytes_freed; }; -static inline struct gk20a_buddy_allocator *buddy_allocator( - struct gk20a_allocator *a) +static inline struct nvgpu_buddy_allocator *buddy_allocator( + struct nvgpu_allocator *a) { - return (struct gk20a_buddy_allocator *)(a)->priv; + return (struct nvgpu_buddy_allocator *)(a)->priv; } static inline struct list_head *balloc_get_order_list( - struct gk20a_buddy_allocator *a, int order) + struct nvgpu_buddy_allocator *a, int order) { return &a->buddy_list[order]; } -static inline u64 balloc_order_to_len(struct gk20a_buddy_allocator *a, +static inline u64 balloc_order_to_len(struct nvgpu_buddy_allocator *a, int order) { return (1 << order) * a->blk_size; } -static inline u64 balloc_base_shift(struct gk20a_buddy_allocator *a, +static inline u64 balloc_base_shift(struct nvgpu_buddy_allocator *a, u64 base) { return base - a->start; } -static inline u64 balloc_base_unshift(struct gk20a_buddy_allocator *a, +static inline u64 balloc_base_unshift(struct nvgpu_buddy_allocator *a, u64 base) { return base + a->start; } -static inline struct gk20a_allocator *balloc_owner( - struct gk20a_buddy_allocator *a) +static inline struct nvgpu_allocator *balloc_owner( + struct nvgpu_buddy_allocator *a) { return a->owner; } diff --git a/drivers/gpu/nvgpu/gk20a/gk20a_allocator_lockless.c b/drivers/gpu/nvgpu/common/mm/lockless_allocator.c similarity index 69% rename from drivers/gpu/nvgpu/gk20a/gk20a_allocator_lockless.c rename to drivers/gpu/nvgpu/common/mm/lockless_allocator.c index 5b011d8c3..e3063a428 100644 --- a/drivers/gpu/nvgpu/gk20a/gk20a_allocator_lockless.c +++ b/drivers/gpu/nvgpu/common/mm/lockless_allocator.c @@ -19,42 +19,43 @@ #include #include -#include "gk20a_allocator.h" +#include + #include "lockless_allocator_priv.h" -static u64 gk20a_lockless_alloc_length(struct gk20a_allocator *a) +static u64 nvgpu_lockless_alloc_length(struct nvgpu_allocator *a) { - struct gk20a_lockless_allocator *pa = a->priv; + struct nvgpu_lockless_allocator *pa = a->priv; return pa->length; } -static u64 gk20a_lockless_alloc_base(struct gk20a_allocator *a) +static u64 nvgpu_lockless_alloc_base(struct nvgpu_allocator *a) { - struct gk20a_lockless_allocator *pa = a->priv; + struct nvgpu_lockless_allocator *pa = a->priv; return pa->base; } -static int gk20a_lockless_alloc_inited(struct gk20a_allocator *a) +static int nvgpu_lockless_alloc_inited(struct nvgpu_allocator *a) { - struct gk20a_lockless_allocator *pa = a->priv; + struct nvgpu_lockless_allocator *pa = a->priv; int inited = pa->inited; rmb(); return inited; } -static u64 gk20a_lockless_alloc_end(struct gk20a_allocator *a) +static u64 nvgpu_lockless_alloc_end(struct nvgpu_allocator *a) { - struct gk20a_lockless_allocator *pa = a->priv; + struct nvgpu_lockless_allocator *pa = a->priv; return pa->base + pa->length; } -static u64 gk20a_lockless_alloc(struct gk20a_allocator *a, u64 len) +static u64 nvgpu_lockless_alloc(struct nvgpu_allocator *a, u64 len) { - struct gk20a_lockless_allocator *pa = a->priv; + struct nvgpu_lockless_allocator *pa = a->priv; int head, new_head, ret; u64 addr = 0; @@ -77,9 +78,9 @@ static u64 gk20a_lockless_alloc(struct gk20a_allocator *a, u64 len) return addr; } -static void gk20a_lockless_free(struct gk20a_allocator *a, u64 addr) +static void nvgpu_lockless_free(struct nvgpu_allocator *a, u64 addr) { - struct gk20a_lockless_allocator *pa = a->priv; + struct nvgpu_lockless_allocator *pa = a->priv; int head, ret; u64 cur_idx, rem; @@ -98,20 +99,20 @@ static void gk20a_lockless_free(struct gk20a_allocator *a, u64 addr) } } -static void gk20a_lockless_alloc_destroy(struct gk20a_allocator *a) +static void nvgpu_lockless_alloc_destroy(struct nvgpu_allocator *a) { - struct gk20a_lockless_allocator *pa = a->priv; + struct nvgpu_lockless_allocator *pa = a->priv; - gk20a_fini_alloc_debug(a); + nvgpu_fini_alloc_debug(a); vfree(pa->next); kfree(pa); } -static void gk20a_lockless_print_stats(struct gk20a_allocator *a, +static void nvgpu_lockless_print_stats(struct nvgpu_allocator *a, struct seq_file *s, int lock) { - struct gk20a_lockless_allocator *pa = a->priv; + struct nvgpu_lockless_allocator *pa = a->priv; __alloc_pstat(s, a, "Lockless allocator params:\n"); __alloc_pstat(s, a, " start = 0x%llx\n", pa->base); @@ -125,21 +126,21 @@ static void gk20a_lockless_print_stats(struct gk20a_allocator *a, pa->nr_nodes - atomic_read(&pa->nr_allocs)); } -static const struct gk20a_allocator_ops pool_ops = { - .alloc = gk20a_lockless_alloc, - .free = gk20a_lockless_free, +static const struct nvgpu_allocator_ops pool_ops = { + .alloc = nvgpu_lockless_alloc, + .free = nvgpu_lockless_free, - .base = gk20a_lockless_alloc_base, - .length = gk20a_lockless_alloc_length, - .end = gk20a_lockless_alloc_end, - .inited = gk20a_lockless_alloc_inited, + .base = nvgpu_lockless_alloc_base, + .length = nvgpu_lockless_alloc_length, + .end = nvgpu_lockless_alloc_end, + .inited = nvgpu_lockless_alloc_inited, - .fini = gk20a_lockless_alloc_destroy, + .fini = nvgpu_lockless_alloc_destroy, - .print_stats = gk20a_lockless_print_stats, + .print_stats = nvgpu_lockless_print_stats, }; -int gk20a_lockless_allocator_init(struct gk20a *g, struct gk20a_allocator *__a, +int nvgpu_lockless_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a, const char *name, u64 base, u64 length, u64 blk_size, u64 flags) { @@ -147,7 +148,7 @@ int gk20a_lockless_allocator_init(struct gk20a *g, struct gk20a_allocator *__a, int err; int nr_nodes; u64 count, rem; - struct gk20a_lockless_allocator *a; + struct nvgpu_lockless_allocator *a; if (!blk_size) return -EINVAL; @@ -161,11 +162,11 @@ int gk20a_lockless_allocator_init(struct gk20a *g, struct gk20a_allocator *__a, if (!base || !count || count > INT_MAX) return -EINVAL; - a = kzalloc(sizeof(struct gk20a_lockless_allocator), GFP_KERNEL); + a = kzalloc(sizeof(struct nvgpu_lockless_allocator), GFP_KERNEL); if (!a) return -ENOMEM; - err = __gk20a_alloc_common_init(__a, name, a, false, &pool_ops); + err = __nvgpu_alloc_common_init(__a, name, a, false, &pool_ops); if (err) goto fail; @@ -191,7 +192,7 @@ int gk20a_lockless_allocator_init(struct gk20a *g, struct gk20a_allocator *__a, wmb(); a->inited = true; - gk20a_init_alloc_debug(g, __a); + nvgpu_init_alloc_debug(g, __a); alloc_dbg(__a, "New allocator: type lockless\n"); alloc_dbg(__a, " base 0x%llx\n", a->base); alloc_dbg(__a, " nodes %d\n", a->nr_nodes); diff --git a/drivers/gpu/nvgpu/gk20a/lockless_allocator_priv.h b/drivers/gpu/nvgpu/common/mm/lockless_allocator_priv.h similarity index 94% rename from drivers/gpu/nvgpu/gk20a/lockless_allocator_priv.h rename to drivers/gpu/nvgpu/common/mm/lockless_allocator_priv.h index f9b03e0e8..32421ac1c 100644 --- a/drivers/gpu/nvgpu/gk20a/lockless_allocator_priv.h +++ b/drivers/gpu/nvgpu/common/mm/lockless_allocator_priv.h @@ -91,10 +91,10 @@ #ifndef LOCKLESS_ALLOCATOR_PRIV_H #define LOCKLESS_ALLOCATOR_PRIV_H -struct gk20a_allocator; +struct nvgpu_allocator; -struct gk20a_lockless_allocator { - struct gk20a_allocator *owner; +struct nvgpu_lockless_allocator { + struct nvgpu_allocator *owner; u64 base; /* Base address of the space. */ u64 length; /* Length of the space. */ @@ -112,10 +112,10 @@ struct gk20a_lockless_allocator { atomic_t nr_allocs; }; -static inline struct gk20a_lockless_allocator *lockless_allocator( - struct gk20a_allocator *a) +static inline struct nvgpu_lockless_allocator *lockless_allocator( + struct nvgpu_allocator *a) { - return (struct gk20a_lockless_allocator *)(a)->priv; + return (struct nvgpu_lockless_allocator *)(a)->priv; } #endif diff --git a/drivers/gpu/nvgpu/gk20a/gk20a_allocator.c b/drivers/gpu/nvgpu/common/mm/nvgpu_allocator.c similarity index 70% rename from drivers/gpu/nvgpu/gk20a/gk20a_allocator.c rename to drivers/gpu/nvgpu/common/mm/nvgpu_allocator.c index 3129b07c4..ebd779c06 100644 --- a/drivers/gpu/nvgpu/gk20a/gk20a_allocator.c +++ b/drivers/gpu/nvgpu/common/mm/nvgpu_allocator.c @@ -19,14 +19,15 @@ #include #include -#include "gk20a.h" -#include "mm_gk20a.h" -#include "platform_gk20a.h" -#include "gk20a_allocator.h" +#include -u32 gk20a_alloc_tracing_on; +#include "gk20a/gk20a.h" +#include "gk20a/mm_gk20a.h" +#include "gk20a/platform_gk20a.h" -u64 gk20a_alloc_length(struct gk20a_allocator *a) +u32 nvgpu_alloc_tracing_on; + +u64 nvgpu_alloc_length(struct nvgpu_allocator *a) { if (a->ops->length) return a->ops->length(a); @@ -34,7 +35,7 @@ u64 gk20a_alloc_length(struct gk20a_allocator *a) return 0; } -u64 gk20a_alloc_base(struct gk20a_allocator *a) +u64 nvgpu_alloc_base(struct nvgpu_allocator *a) { if (a->ops->base) return a->ops->base(a); @@ -42,7 +43,7 @@ u64 gk20a_alloc_base(struct gk20a_allocator *a) return 0; } -u64 gk20a_alloc_initialized(struct gk20a_allocator *a) +u64 nvgpu_alloc_initialized(struct nvgpu_allocator *a) { if (!a->ops || !a->ops->inited) return 0; @@ -50,7 +51,7 @@ u64 gk20a_alloc_initialized(struct gk20a_allocator *a) return a->ops->inited(a); } -u64 gk20a_alloc_end(struct gk20a_allocator *a) +u64 nvgpu_alloc_end(struct nvgpu_allocator *a) { if (a->ops->end) return a->ops->end(a); @@ -58,7 +59,7 @@ u64 gk20a_alloc_end(struct gk20a_allocator *a) return 0; } -u64 gk20a_alloc_space(struct gk20a_allocator *a) +u64 nvgpu_alloc_space(struct nvgpu_allocator *a) { if (a->ops->space) return a->ops->space(a); @@ -66,17 +67,17 @@ u64 gk20a_alloc_space(struct gk20a_allocator *a) return 0; } -u64 gk20a_alloc(struct gk20a_allocator *a, u64 len) +u64 nvgpu_alloc(struct nvgpu_allocator *a, u64 len) { return a->ops->alloc(a, len); } -void gk20a_free(struct gk20a_allocator *a, u64 addr) +void nvgpu_free(struct nvgpu_allocator *a, u64 addr) { a->ops->free(a, addr); } -u64 gk20a_alloc_fixed(struct gk20a_allocator *a, u64 base, u64 len) +u64 nvgpu_alloc_fixed(struct nvgpu_allocator *a, u64 base, u64 len) { if (a->ops->alloc_fixed) return a->ops->alloc_fixed(a, base, len); @@ -84,7 +85,7 @@ u64 gk20a_alloc_fixed(struct gk20a_allocator *a, u64 base, u64 len) return 0; } -void gk20a_free_fixed(struct gk20a_allocator *a, u64 base, u64 len) +void nvgpu_free_fixed(struct nvgpu_allocator *a, u64 base, u64 len) { /* * If this operation is not defined for the allocator then just do @@ -95,8 +96,8 @@ void gk20a_free_fixed(struct gk20a_allocator *a, u64 base, u64 len) a->ops->free_fixed(a, base, len); } -int gk20a_alloc_reserve_carveout(struct gk20a_allocator *a, - struct gk20a_alloc_carveout *co) +int nvgpu_alloc_reserve_carveout(struct nvgpu_allocator *a, + struct nvgpu_alloc_carveout *co) { if (a->ops->reserve_carveout) return a->ops->reserve_carveout(a, co); @@ -104,25 +105,25 @@ int gk20a_alloc_reserve_carveout(struct gk20a_allocator *a, return -ENODEV; } -void gk20a_alloc_release_carveout(struct gk20a_allocator *a, - struct gk20a_alloc_carveout *co) +void nvgpu_alloc_release_carveout(struct nvgpu_allocator *a, + struct nvgpu_alloc_carveout *co) { if (a->ops->release_carveout) a->ops->release_carveout(a, co); } -void gk20a_alloc_destroy(struct gk20a_allocator *a) +void nvgpu_alloc_destroy(struct nvgpu_allocator *a) { a->ops->fini(a); memset(a, 0, sizeof(*a)); } /* - * Handle the common init stuff for a gk20a_allocator. + * Handle the common init stuff for a nvgpu_allocator. */ -int __gk20a_alloc_common_init(struct gk20a_allocator *a, +int __nvgpu_alloc_common_init(struct nvgpu_allocator *a, const char *name, void *priv, bool dbg, - const struct gk20a_allocator_ops *ops) + const struct nvgpu_allocator_ops *ops) { if (!ops) return -EINVAL; @@ -145,7 +146,7 @@ int __gk20a_alloc_common_init(struct gk20a_allocator *a, return 0; } -void gk20a_alloc_print_stats(struct gk20a_allocator *__a, +void nvgpu_alloc_print_stats(struct nvgpu_allocator *__a, struct seq_file *s, int lock) { __a->ops->print_stats(__a, s, lock); @@ -154,9 +155,9 @@ void gk20a_alloc_print_stats(struct gk20a_allocator *__a, #ifdef CONFIG_DEBUG_FS static int __alloc_show(struct seq_file *s, void *unused) { - struct gk20a_allocator *a = s->private; + struct nvgpu_allocator *a = s->private; - gk20a_alloc_print_stats(a, s, 1); + nvgpu_alloc_print_stats(a, s, 1); return 0; } @@ -174,7 +175,7 @@ static const struct file_operations __alloc_fops = { }; #endif -void gk20a_init_alloc_debug(struct gk20a *g, struct gk20a_allocator *a) +void nvgpu_init_alloc_debug(struct gk20a *g, struct nvgpu_allocator *a) { #ifdef CONFIG_DEBUG_FS if (!g->debugfs_allocators) @@ -186,7 +187,7 @@ void gk20a_init_alloc_debug(struct gk20a *g, struct gk20a_allocator *a) #endif } -void gk20a_fini_alloc_debug(struct gk20a_allocator *a) +void nvgpu_fini_alloc_debug(struct nvgpu_allocator *a) { #ifdef CONFIG_DEBUG_FS if (!IS_ERR_OR_NULL(a->debugfs_entry)) @@ -194,7 +195,7 @@ void gk20a_fini_alloc_debug(struct gk20a_allocator *a) #endif } -void gk20a_alloc_debugfs_init(struct device *dev) +void nvgpu_alloc_debugfs_init(struct device *dev) { #ifdef CONFIG_DEBUG_FS struct gk20a_platform *platform = dev_get_drvdata(dev); @@ -206,6 +207,6 @@ void gk20a_alloc_debugfs_init(struct device *dev) return; debugfs_create_u32("tracing", 0664, g->debugfs_allocators, - &gk20a_alloc_tracing_on); + &nvgpu_alloc_tracing_on); #endif } diff --git a/drivers/gpu/nvgpu/gk20a/gk20a_allocator_page.c b/drivers/gpu/nvgpu/common/mm/page_allocator.c similarity index 76% rename from drivers/gpu/nvgpu/gk20a/gk20a_allocator_page.c rename to drivers/gpu/nvgpu/common/mm/page_allocator.c index 9717a7266..c61b22380 100644 --- a/drivers/gpu/nvgpu/gk20a/gk20a_allocator_page.c +++ b/drivers/gpu/nvgpu/common/mm/page_allocator.c @@ -19,9 +19,10 @@ #include #include -#include "gk20a_allocator.h" +#include +#include + #include "buddy_allocator_priv.h" -#include "page_allocator_priv.h" #define palloc_dbg(a, fmt, arg...) \ alloc_dbg(palloc_owner(a), fmt, ##arg) @@ -81,59 +82,59 @@ static inline void del_slab_page_from_full(struct page_alloc_slab *slab, page->state = SP_NONE; } -static u64 gk20a_page_alloc_length(struct gk20a_allocator *a) +static u64 nvgpu_page_alloc_length(struct nvgpu_allocator *a) { - struct gk20a_page_allocator *va = a->priv; + struct nvgpu_page_allocator *va = a->priv; - return gk20a_alloc_length(&va->source_allocator); + return nvgpu_alloc_length(&va->source_allocator); } -static u64 gk20a_page_alloc_base(struct gk20a_allocator *a) +static u64 nvgpu_page_alloc_base(struct nvgpu_allocator *a) { - struct gk20a_page_allocator *va = a->priv; + struct nvgpu_page_allocator *va = a->priv; - return gk20a_alloc_base(&va->source_allocator); + return nvgpu_alloc_base(&va->source_allocator); } -static int gk20a_page_alloc_inited(struct gk20a_allocator *a) +static int nvgpu_page_alloc_inited(struct nvgpu_allocator *a) { - struct gk20a_page_allocator *va = a->priv; + struct nvgpu_page_allocator *va = a->priv; - return gk20a_alloc_initialized(&va->source_allocator); + return nvgpu_alloc_initialized(&va->source_allocator); } -static u64 gk20a_page_alloc_end(struct gk20a_allocator *a) +static u64 nvgpu_page_alloc_end(struct nvgpu_allocator *a) { - struct gk20a_page_allocator *va = a->priv; + struct nvgpu_page_allocator *va = a->priv; - return gk20a_alloc_end(&va->source_allocator); + return nvgpu_alloc_end(&va->source_allocator); } -static u64 gk20a_page_alloc_space(struct gk20a_allocator *a) +static u64 nvgpu_page_alloc_space(struct nvgpu_allocator *a) { - struct gk20a_page_allocator *va = a->priv; + struct nvgpu_page_allocator *va = a->priv; - return gk20a_alloc_space(&va->source_allocator); + return nvgpu_alloc_space(&va->source_allocator); } -static int gk20a_page_reserve_co(struct gk20a_allocator *a, - struct gk20a_alloc_carveout *co) +static int nvgpu_page_reserve_co(struct nvgpu_allocator *a, + struct nvgpu_alloc_carveout *co) { - struct gk20a_page_allocator *va = a->priv; + struct nvgpu_page_allocator *va = a->priv; - return gk20a_alloc_reserve_carveout(&va->source_allocator, co); + return nvgpu_alloc_reserve_carveout(&va->source_allocator, co); } -static void gk20a_page_release_co(struct gk20a_allocator *a, - struct gk20a_alloc_carveout *co) +static void nvgpu_page_release_co(struct nvgpu_allocator *a, + struct nvgpu_alloc_carveout *co) { - struct gk20a_page_allocator *va = a->priv; + struct nvgpu_page_allocator *va = a->priv; - gk20a_alloc_release_carveout(&va->source_allocator, co); + nvgpu_alloc_release_carveout(&va->source_allocator, co); } -static void __gk20a_free_pages(struct gk20a_page_allocator *a, - struct gk20a_page_alloc *alloc, +static void __nvgpu_free_pages(struct nvgpu_page_allocator *a, + struct nvgpu_page_alloc *alloc, bool free_buddy_alloc) { struct page_alloc_chunk *chunk; @@ -145,22 +146,22 @@ static void __gk20a_free_pages(struct gk20a_page_allocator *a, list_del(&chunk->list_entry); if (free_buddy_alloc) - gk20a_free(&a->source_allocator, chunk->base); + nvgpu_free(&a->source_allocator, chunk->base); kfree(chunk); } kfree(alloc); } -static int __insert_page_alloc(struct gk20a_page_allocator *a, - struct gk20a_page_alloc *alloc) +static int __insert_page_alloc(struct nvgpu_page_allocator *a, + struct nvgpu_page_alloc *alloc) { struct rb_node **new = &a->allocs.rb_node; struct rb_node *parent = NULL; while (*new) { - struct gk20a_page_alloc *tmp = - container_of(*new, struct gk20a_page_alloc, + struct nvgpu_page_alloc *tmp = + container_of(*new, struct nvgpu_page_alloc, tree_entry); parent = *new; @@ -180,15 +181,15 @@ static int __insert_page_alloc(struct gk20a_page_allocator *a, return 0; } -static struct gk20a_page_alloc *__find_page_alloc( - struct gk20a_page_allocator *a, +static struct nvgpu_page_alloc *__find_page_alloc( + struct nvgpu_page_allocator *a, u64 addr) { struct rb_node *node = a->allocs.rb_node; - struct gk20a_page_alloc *alloc; + struct nvgpu_page_alloc *alloc; while (node) { - alloc = container_of(node, struct gk20a_page_alloc, tree_entry); + alloc = container_of(node, struct nvgpu_page_alloc, tree_entry); if (addr < alloc->base) node = node->rb_left; @@ -207,7 +208,7 @@ static struct gk20a_page_alloc *__find_page_alloc( } static struct page_alloc_slab_page *alloc_slab_page( - struct gk20a_page_allocator *a, + struct nvgpu_page_allocator *a, struct page_alloc_slab *slab) { struct page_alloc_slab_page *slab_page; @@ -220,7 +221,7 @@ static struct page_alloc_slab_page *alloc_slab_page( memset(slab_page, 0, sizeof(*slab_page)); - slab_page->page_addr = gk20a_alloc(&a->source_allocator, a->page_size); + slab_page->page_addr = nvgpu_alloc(&a->source_allocator, a->page_size); if (!slab_page->page_addr) { kfree(slab_page); palloc_dbg(a, "OOM: vidmem is full!\n"); @@ -242,7 +243,7 @@ static struct page_alloc_slab_page *alloc_slab_page( return slab_page; } -static void free_slab_page(struct gk20a_page_allocator *a, +static void free_slab_page(struct nvgpu_page_allocator *a, struct page_alloc_slab_page *slab_page) { palloc_dbg(a, "Freeing slab page @ 0x%012llx\n", slab_page->page_addr); @@ -251,7 +252,7 @@ static void free_slab_page(struct gk20a_page_allocator *a, slab_page->nr_objects_alloced != 0 || slab_page->bitmap != 0); - gk20a_free(&a->source_allocator, slab_page->page_addr); + nvgpu_free(&a->source_allocator, slab_page->page_addr); a->pages_freed++; kmem_cache_free(page_alloc_slab_page_cache, slab_page); @@ -261,9 +262,9 @@ static void free_slab_page(struct gk20a_page_allocator *a, * This expects @alloc to have 1 empty page_alloc_chunk already added to the * alloc_chunks list. */ -static int __do_slab_alloc(struct gk20a_page_allocator *a, +static int __do_slab_alloc(struct nvgpu_page_allocator *a, struct page_alloc_slab *slab, - struct gk20a_page_alloc *alloc) + struct nvgpu_page_alloc *alloc) { struct page_alloc_slab_page *slab_page = NULL; struct page_alloc_chunk *chunk; @@ -317,7 +318,7 @@ static int __do_slab_alloc(struct gk20a_page_allocator *a, BUG(); /* Should be impossible to hit this. */ /* - * Handle building the gk20a_page_alloc struct. We expect one + * Handle building the nvgpu_page_alloc struct. We expect one * page_alloc_chunk to be present. */ alloc->slab_page = slab_page; @@ -336,12 +337,12 @@ static int __do_slab_alloc(struct gk20a_page_allocator *a, /* * Allocate from a slab instead of directly from the page allocator. */ -static struct gk20a_page_alloc *__gk20a_alloc_slab( - struct gk20a_page_allocator *a, u64 len) +static struct nvgpu_page_alloc *__nvgpu_alloc_slab( + struct nvgpu_page_allocator *a, u64 len) { int err, slab_nr; struct page_alloc_slab *slab; - struct gk20a_page_alloc *alloc = NULL; + struct nvgpu_page_alloc *alloc = NULL; struct page_alloc_chunk *chunk = NULL; /* @@ -381,8 +382,8 @@ fail: return NULL; } -static void __gk20a_free_slab(struct gk20a_page_allocator *a, - struct gk20a_page_alloc *alloc) +static void __nvgpu_free_slab(struct nvgpu_page_allocator *a, + struct nvgpu_page_alloc *alloc) { struct page_alloc_slab_page *slab_page = alloc->slab_page; struct page_alloc_slab *slab = slab_page->owner; @@ -423,7 +424,7 @@ static void __gk20a_free_slab(struct gk20a_page_allocator *a, /* * Now handle the page_alloc. */ - __gk20a_free_pages(a, alloc, false); + __nvgpu_free_pages(a, alloc, false); a->nr_slab_frees++; return; @@ -435,10 +436,10 @@ static void __gk20a_free_slab(struct gk20a_page_allocator *a, * fragmentation in the space this allocator will collate smaller non-contiguous * allocations together if necessary. */ -static struct gk20a_page_alloc *__do_gk20a_alloc_pages( - struct gk20a_page_allocator *a, u64 pages) +static struct nvgpu_page_alloc *__do_nvgpu_alloc_pages( + struct nvgpu_page_allocator *a, u64 pages) { - struct gk20a_page_alloc *alloc; + struct nvgpu_page_alloc *alloc; struct page_alloc_chunk *c; u64 max_chunk_len = pages << a->page_shift; int i = 0; @@ -476,7 +477,7 @@ static struct gk20a_page_alloc *__do_gk20a_alloc_pages( * allocator (i.e the allocator is OOM). */ do { - chunk_addr = gk20a_alloc(&a->source_allocator, + chunk_addr = nvgpu_alloc(&a->source_allocator, chunk_len); /* Divide by 2 and try again */ @@ -497,7 +498,7 @@ static struct gk20a_page_alloc *__do_gk20a_alloc_pages( c = kmem_cache_alloc(page_alloc_chunk_cache, GFP_KERNEL); if (!c) { - gk20a_free(&a->source_allocator, chunk_addr); + nvgpu_free(&a->source_allocator, chunk_addr); goto fail_cleanup; } @@ -522,7 +523,7 @@ fail_cleanup: c = list_first_entry(&alloc->alloc_chunks, struct page_alloc_chunk, list_entry); list_del(&c->list_entry); - gk20a_free(&a->source_allocator, c->base); + nvgpu_free(&a->source_allocator, c->base); kfree(c); } kfree(alloc); @@ -530,17 +531,17 @@ fail: return ERR_PTR(-ENOMEM); } -static struct gk20a_page_alloc *__gk20a_alloc_pages( - struct gk20a_page_allocator *a, u64 len) +static struct nvgpu_page_alloc *__nvgpu_alloc_pages( + struct nvgpu_page_allocator *a, u64 len) { - struct gk20a_page_alloc *alloc = NULL; + struct nvgpu_page_alloc *alloc = NULL; struct page_alloc_chunk *c; u64 pages; int i = 0; pages = ALIGN(len, a->page_size) >> a->page_shift; - alloc = __do_gk20a_alloc_pages(a, pages); + alloc = __do_nvgpu_alloc_pages(a, pages); if (IS_ERR(alloc)) { palloc_dbg(a, "Alloc 0x%llx (%llu) (failed)\n", pages << a->page_shift, pages); @@ -561,16 +562,16 @@ static struct gk20a_page_alloc *__gk20a_alloc_pages( * Allocate enough pages to satisfy @len. Page size is determined at * initialization of the allocator. * - * The return is actually a pointer to a struct gk20a_page_alloc pointer. This + * The return is actually a pointer to a struct nvgpu_page_alloc pointer. This * is because it doesn't make a lot of sense to return the address of the first * page in the list of pages (since they could be discontiguous). This has * precedent in the dma_alloc APIs, though, it's really just an annoying - * artifact of the fact that the gk20a_alloc() API requires a u64 return type. + * artifact of the fact that the nvgpu_alloc() API requires a u64 return type. */ -static u64 gk20a_page_alloc(struct gk20a_allocator *__a, u64 len) +static u64 nvgpu_page_alloc(struct nvgpu_allocator *__a, u64 len) { - struct gk20a_page_allocator *a = page_allocator(__a); - struct gk20a_page_alloc *alloc = NULL; + struct nvgpu_page_allocator *a = page_allocator(__a); + struct nvgpu_page_alloc *alloc = NULL; u64 real_len; /* @@ -583,9 +584,9 @@ static u64 gk20a_page_alloc(struct gk20a_allocator *__a, u64 len) alloc_lock(__a); if (a->flags & GPU_ALLOC_4K_VIDMEM_PAGES && real_len <= (a->page_size / 2)) - alloc = __gk20a_alloc_slab(a, real_len); + alloc = __nvgpu_alloc_slab(a, real_len); else - alloc = __gk20a_alloc_pages(a, real_len); + alloc = __nvgpu_alloc_pages(a, real_len); if (!alloc) { alloc_unlock(__a); @@ -606,13 +607,13 @@ static u64 gk20a_page_alloc(struct gk20a_allocator *__a, u64 len) } /* - * Note: this will remove the gk20a_page_alloc struct from the RB tree + * Note: this will remove the nvgpu_page_alloc struct from the RB tree * if it's found. */ -static void gk20a_page_free(struct gk20a_allocator *__a, u64 base) +static void nvgpu_page_free(struct nvgpu_allocator *__a, u64 base) { - struct gk20a_page_allocator *a = page_allocator(__a); - struct gk20a_page_alloc *alloc; + struct nvgpu_page_allocator *a = page_allocator(__a); + struct nvgpu_page_alloc *alloc; alloc_lock(__a); @@ -620,7 +621,7 @@ static void gk20a_page_free(struct gk20a_allocator *__a, u64 base) alloc = __find_page_alloc(a, base); else alloc = __find_page_alloc(a, - ((struct gk20a_page_alloc *)(uintptr_t)base)->base); + ((struct nvgpu_page_alloc *)(uintptr_t)base)->base); if (!alloc) { palloc_dbg(a, "Hrm, found no alloc?\n"); @@ -636,20 +637,20 @@ static void gk20a_page_free(struct gk20a_allocator *__a, u64 base) * Frees *alloc. */ if (alloc->slab_page) { - __gk20a_free_slab(a, alloc); + __nvgpu_free_slab(a, alloc); } else { a->pages_freed += (alloc->length >> a->page_shift); - __gk20a_free_pages(a, alloc, true); + __nvgpu_free_pages(a, alloc, true); } done: alloc_unlock(__a); } -static struct gk20a_page_alloc *__gk20a_alloc_pages_fixed( - struct gk20a_page_allocator *a, u64 base, u64 length) +static struct nvgpu_page_alloc *__nvgpu_alloc_pages_fixed( + struct nvgpu_page_allocator *a, u64 base, u64 length) { - struct gk20a_page_alloc *alloc; + struct nvgpu_page_alloc *alloc; struct page_alloc_chunk *c; alloc = kmem_cache_alloc(page_alloc_cache, GFP_KERNEL); @@ -657,9 +658,9 @@ static struct gk20a_page_alloc *__gk20a_alloc_pages_fixed( if (!alloc || !c) goto fail; - alloc->base = gk20a_alloc_fixed(&a->source_allocator, base, length); + alloc->base = nvgpu_alloc_fixed(&a->source_allocator, base, length); if (!alloc->base) { - WARN(1, "gk20a: failed to fixed alloc pages @ 0x%010llx", base); + WARN(1, "nvgpu: failed to fixed alloc pages @ 0x%010llx", base); goto fail; } @@ -679,11 +680,11 @@ fail: return ERR_PTR(-ENOMEM); } -static u64 gk20a_page_alloc_fixed(struct gk20a_allocator *__a, +static u64 nvgpu_page_alloc_fixed(struct nvgpu_allocator *__a, u64 base, u64 len) { - struct gk20a_page_allocator *a = page_allocator(__a); - struct gk20a_page_alloc *alloc = NULL; + struct nvgpu_page_allocator *a = page_allocator(__a); + struct nvgpu_page_alloc *alloc = NULL; struct page_alloc_chunk *c; u64 aligned_len, pages; int i = 0; @@ -693,7 +694,7 @@ static u64 gk20a_page_alloc_fixed(struct gk20a_allocator *__a, alloc_lock(__a); - alloc = __gk20a_alloc_pages_fixed(a, base, aligned_len); + alloc = __nvgpu_alloc_pages_fixed(a, base, aligned_len); if (IS_ERR(alloc)) { alloc_unlock(__a); return 0; @@ -718,11 +719,11 @@ static u64 gk20a_page_alloc_fixed(struct gk20a_allocator *__a, return (u64) (uintptr_t) alloc; } -static void gk20a_page_free_fixed(struct gk20a_allocator *__a, +static void nvgpu_page_free_fixed(struct nvgpu_allocator *__a, u64 base, u64 len) { - struct gk20a_page_allocator *a = page_allocator(__a); - struct gk20a_page_alloc *alloc; + struct nvgpu_page_allocator *a = page_allocator(__a); + struct nvgpu_page_alloc *alloc; alloc_lock(__a); @@ -731,7 +732,7 @@ static void gk20a_page_free_fixed(struct gk20a_allocator *__a, if (!alloc) goto done; } else { - alloc = (struct gk20a_page_alloc *) (uintptr_t) base; + alloc = (struct nvgpu_page_alloc *) (uintptr_t) base; } palloc_dbg(a, "Free [fixed] 0x%010llx + 0x%llx\n", @@ -746,15 +747,15 @@ static void gk20a_page_free_fixed(struct gk20a_allocator *__a, * allocs. This would have to be updated if the underlying * allocator were to change. */ - __gk20a_free_pages(a, alloc, true); + __nvgpu_free_pages(a, alloc, true); done: alloc_unlock(__a); } -static void gk20a_page_allocator_destroy(struct gk20a_allocator *__a) +static void nvgpu_page_allocator_destroy(struct nvgpu_allocator *__a) { - struct gk20a_page_allocator *a = page_allocator(__a); + struct nvgpu_page_allocator *a = page_allocator(__a); alloc_lock(__a); kfree(a); @@ -762,10 +763,10 @@ static void gk20a_page_allocator_destroy(struct gk20a_allocator *__a) alloc_unlock(__a); } -static void gk20a_page_print_stats(struct gk20a_allocator *__a, +static void nvgpu_page_print_stats(struct nvgpu_allocator *__a, struct seq_file *s, int lock) { - struct gk20a_page_allocator *a = page_allocator(__a); + struct nvgpu_page_allocator *a = page_allocator(__a); int i; if (lock) @@ -803,31 +804,31 @@ static void gk20a_page_print_stats(struct gk20a_allocator *__a, __alloc_pstat(s, __a, "Source alloc: %s\n", a->source_allocator.name); - gk20a_alloc_print_stats(&a->source_allocator, s, lock); + nvgpu_alloc_print_stats(&a->source_allocator, s, lock); if (lock) alloc_unlock(__a); } -static const struct gk20a_allocator_ops page_ops = { - .alloc = gk20a_page_alloc, - .free = gk20a_page_free, +static const struct nvgpu_allocator_ops page_ops = { + .alloc = nvgpu_page_alloc, + .free = nvgpu_page_free, - .alloc_fixed = gk20a_page_alloc_fixed, - .free_fixed = gk20a_page_free_fixed, + .alloc_fixed = nvgpu_page_alloc_fixed, + .free_fixed = nvgpu_page_free_fixed, - .reserve_carveout = gk20a_page_reserve_co, - .release_carveout = gk20a_page_release_co, + .reserve_carveout = nvgpu_page_reserve_co, + .release_carveout = nvgpu_page_release_co, - .base = gk20a_page_alloc_base, - .length = gk20a_page_alloc_length, - .end = gk20a_page_alloc_end, - .inited = gk20a_page_alloc_inited, - .space = gk20a_page_alloc_space, + .base = nvgpu_page_alloc_base, + .length = nvgpu_page_alloc_length, + .end = nvgpu_page_alloc_end, + .inited = nvgpu_page_alloc_inited, + .space = nvgpu_page_alloc_space, - .fini = gk20a_page_allocator_destroy, + .fini = nvgpu_page_allocator_destroy, - .print_stats = gk20a_page_print_stats, + .print_stats = nvgpu_page_print_stats, }; /* @@ -840,7 +841,7 @@ static const struct gk20a_allocator_ops page_ops = { * * That gives buckets of 1, 2, 4, and 8 pages (i.e 4k, 8k, 16k, 32k). */ -static int gk20a_page_alloc_init_slabs(struct gk20a_page_allocator *a) +static int nvgpu_page_alloc_init_slabs(struct nvgpu_page_allocator *a) { size_t nr_slabs = ilog2(a->page_size >> 12); unsigned int i; @@ -867,17 +868,17 @@ static int gk20a_page_alloc_init_slabs(struct gk20a_page_allocator *a) return 0; } -int gk20a_page_allocator_init(struct gk20a *g, struct gk20a_allocator *__a, +int nvgpu_page_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a, const char *name, u64 base, u64 length, u64 blk_size, u64 flags) { - struct gk20a_page_allocator *a; + struct nvgpu_page_allocator *a; char buddy_name[sizeof(__a->name)]; int err; mutex_lock(&meta_data_cache_lock); if (!page_alloc_cache) - page_alloc_cache = KMEM_CACHE(gk20a_page_alloc, 0); + page_alloc_cache = KMEM_CACHE(nvgpu_page_alloc, 0); if (!page_alloc_chunk_cache) page_alloc_chunk_cache = KMEM_CACHE(page_alloc_chunk, 0); if (!page_alloc_slab_page_cache) @@ -891,11 +892,11 @@ int gk20a_page_allocator_init(struct gk20a *g, struct gk20a_allocator *__a, if (blk_size < SZ_4K) return -EINVAL; - a = kzalloc(sizeof(struct gk20a_page_allocator), GFP_KERNEL); + a = kzalloc(sizeof(struct nvgpu_page_allocator), GFP_KERNEL); if (!a) return -ENOMEM; - err = __gk20a_alloc_common_init(__a, name, a, false, &page_ops); + err = __nvgpu_alloc_common_init(__a, name, a, false, &page_ops); if (err) goto fail; @@ -908,19 +909,19 @@ int gk20a_page_allocator_init(struct gk20a *g, struct gk20a_allocator *__a, a->flags = flags; if (flags & GPU_ALLOC_4K_VIDMEM_PAGES && blk_size > SZ_4K) { - err = gk20a_page_alloc_init_slabs(a); + err = nvgpu_page_alloc_init_slabs(a); if (err) goto fail; } snprintf(buddy_name, sizeof(buddy_name), "%s-src", name); - err = gk20a_buddy_allocator_init(g, &a->source_allocator, buddy_name, + err = nvgpu_buddy_allocator_init(g, &a->source_allocator, buddy_name, base, length, blk_size, 0); if (err) goto fail; - gk20a_init_alloc_debug(g, __a); + nvgpu_init_alloc_debug(g, __a); palloc_dbg(a, "New allocator: type page\n"); palloc_dbg(a, " base 0x%llx\n", a->base); palloc_dbg(a, " size 0x%llx\n", a->length); diff --git a/drivers/gpu/nvgpu/gk20a/as_gk20a.c b/drivers/gpu/nvgpu/gk20a/as_gk20a.c index 0b90090aa..07601d424 100644 --- a/drivers/gpu/nvgpu/gk20a/as_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/as_gk20a.c @@ -279,17 +279,17 @@ static int gk20a_as_ioctl_get_va_regions( for (i = 0; i < write_entries; ++i) { struct nvgpu_as_va_region region; - struct gk20a_allocator *vma = - gk20a_alloc_initialized(&vm->fixed) ? + struct nvgpu_allocator *vma = + nvgpu_alloc_initialized(&vm->fixed) ? &vm->fixed : &vm->vma[i]; memset(®ion, 0, sizeof(struct nvgpu_as_va_region)); region.page_size = vm->gmmu_page_sizes[i]; - region.offset = gk20a_alloc_base(vma); + region.offset = nvgpu_alloc_base(vma); /* No __aeabi_uldivmod() on some platforms... */ - region.pages = (gk20a_alloc_end(vma) - - gk20a_alloc_base(vma)) >> ilog2(region.page_size); + region.pages = (nvgpu_alloc_end(vma) - + nvgpu_alloc_base(vma)) >> ilog2(region.page_size); if (copy_to_user(user_region_ptr + i, ®ion, sizeof(region))) return -EFAULT; diff --git a/drivers/gpu/nvgpu/gk20a/channel_gk20a.c b/drivers/gpu/nvgpu/gk20a/channel_gk20a.c index 136c28d00..be01e0e98 100644 --- a/drivers/gpu/nvgpu/gk20a/channel_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/channel_gk20a.c @@ -976,7 +976,7 @@ static void gk20a_free_channel(struct channel_gk20a *ch, bool force) memset(&ch->ramfc, 0, sizeof(struct mem_desc_sub)); gk20a_gmmu_unmap_free(ch_vm, &ch->gpfifo.mem); - nvgpu_free(ch->gpfifo.pipe); + nvgpu_kfree(ch->gpfifo.pipe); memset(&ch->gpfifo, 0, sizeof(struct gpfifo_desc)); #if defined(CONFIG_GK20A_CYCLE_STATS) @@ -1778,7 +1778,7 @@ int gk20a_alloc_channel_gpfifo(struct channel_gk20a *c, } if (c->gpfifo.mem.aperture == APERTURE_VIDMEM || g->mm.force_pramin) { - c->gpfifo.pipe = nvgpu_alloc( + c->gpfifo.pipe = nvgpu_kalloc( gpfifo_size * sizeof(struct nvgpu_gpfifo), false); if (!c->gpfifo.pipe) { @@ -1850,7 +1850,7 @@ clean_up_sync: c->sync = NULL; } clean_up_unmap: - nvgpu_free(c->gpfifo.pipe); + nvgpu_kfree(c->gpfifo.pipe); gk20a_gmmu_unmap_free(ch_vm, &c->gpfifo.mem); clean_up: memset(&c->gpfifo, 0, sizeof(struct gpfifo_desc)); @@ -1980,12 +1980,12 @@ static void trace_write_pushbuffer_range(struct channel_gk20a *c, if (!g) { size = count * sizeof(struct nvgpu_gpfifo); if (size) { - g = nvgpu_alloc(size, false); + g = nvgpu_kalloc(size, false); if (!g) return; if (copy_from_user(g, user_gpfifo, size)) { - nvgpu_free(g); + nvgpu_kfree(g); return; } } @@ -1997,7 +1997,7 @@ static void trace_write_pushbuffer_range(struct channel_gk20a *c, trace_write_pushbuffer(c, gp); if (gpfifo_allocated) - nvgpu_free(g); + nvgpu_kfree(g); } static void gk20a_channel_timeout_start(struct channel_gk20a *ch, diff --git a/drivers/gpu/nvgpu/gk20a/channel_gk20a.h b/drivers/gpu/nvgpu/gk20a/channel_gk20a.h index 0a0d94b7b..697d1603b 100644 --- a/drivers/gpu/nvgpu/gk20a/channel_gk20a.h +++ b/drivers/gpu/nvgpu/gk20a/channel_gk20a.h @@ -143,7 +143,7 @@ struct channel_gk20a { struct list_head ch_entry; /* channel's entry in TSG */ struct channel_gk20a_joblist joblist; - struct gk20a_allocator fence_allocator; + struct nvgpu_allocator fence_allocator; struct vm_gk20a *vm; diff --git a/drivers/gpu/nvgpu/gk20a/dbg_gpu_gk20a.c b/drivers/gpu/nvgpu/gk20a/dbg_gpu_gk20a.c index e55292957..ac96036f9 100644 --- a/drivers/gpu/nvgpu/gk20a/dbg_gpu_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/dbg_gpu_gk20a.c @@ -815,7 +815,7 @@ static int nvgpu_dbg_gpu_ioctl_access_fb_memory(struct dbg_session_gk20a *dbg_s, goto fail_dmabuf_put; } - buffer = nvgpu_alloc(access_limit_size, true); + buffer = nvgpu_kalloc(access_limit_size, true); if (!buffer) { err = -ENOMEM; goto fail_dmabuf_put; @@ -861,7 +861,7 @@ static int nvgpu_dbg_gpu_ioctl_access_fb_memory(struct dbg_session_gk20a *dbg_s, fail_idle: gk20a_idle(g->dev); fail_free_buffer: - nvgpu_free(buffer); + nvgpu_kfree(buffer); fail_dmabuf_put: dma_buf_put(dmabuf); diff --git a/drivers/gpu/nvgpu/gk20a/debug_gk20a.c b/drivers/gpu/nvgpu/gk20a/debug_gk20a.c index b84db9337..8fa108c2a 100644 --- a/drivers/gpu/nvgpu/gk20a/debug_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/debug_gk20a.c @@ -477,7 +477,7 @@ void gk20a_debug_init(struct device *dev, const char *debugfs_symlink) gk20a_railgating_debugfs_init(g->dev); gk20a_cde_debugfs_init(g->dev); gk20a_ce_debugfs_init(g->dev); - gk20a_alloc_debugfs_init(g->dev); + nvgpu_alloc_debugfs_init(g->dev); gk20a_mm_debugfs_init(g->dev); gk20a_fifo_debugfs_init(g->dev); gk20a_sched_debugfs_init(g->dev); diff --git a/drivers/gpu/nvgpu/gk20a/fence_gk20a.c b/drivers/gpu/nvgpu/gk20a/fence_gk20a.c index 323caa8fb..b8a1dcbc0 100644 --- a/drivers/gpu/nvgpu/gk20a/fence_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/fence_gk20a.c @@ -49,8 +49,8 @@ static void gk20a_fence_free(struct kref *ref) gk20a_semaphore_put(f->semaphore); if (f->allocator) { - if (gk20a_alloc_initialized(f->allocator)) - gk20a_free(f->allocator, (size_t)f); + if (nvgpu_alloc_initialized(f->allocator)) + nvgpu_free(f->allocator, (size_t)f); } else kfree(f); } @@ -129,7 +129,7 @@ int gk20a_alloc_fence_pool(struct channel_gk20a *c, unsigned int count) if (!fence_pool) return -ENOMEM; - err = gk20a_lockless_allocator_init(c->g, &c->fence_allocator, + err = nvgpu_lockless_allocator_init(c->g, &c->fence_allocator, "fence_pool", (size_t)fence_pool, size, sizeof(struct gk20a_fence), 0); if (err) @@ -144,11 +144,11 @@ fail: void gk20a_free_fence_pool(struct channel_gk20a *c) { - if (gk20a_alloc_initialized(&c->fence_allocator)) { + if (nvgpu_alloc_initialized(&c->fence_allocator)) { void *base = (void *)(uintptr_t) - gk20a_alloc_base(&c->fence_allocator); + nvgpu_alloc_base(&c->fence_allocator); - gk20a_alloc_destroy(&c->fence_allocator); + nvgpu_alloc_destroy(&c->fence_allocator); vfree(base); } } @@ -158,9 +158,9 @@ struct gk20a_fence *gk20a_alloc_fence(struct channel_gk20a *c) struct gk20a_fence *fence = NULL; if (channel_gk20a_is_prealloc_enabled(c)) { - if (gk20a_alloc_initialized(&c->fence_allocator)) { + if (nvgpu_alloc_initialized(&c->fence_allocator)) { fence = (struct gk20a_fence *)(uintptr_t) - gk20a_alloc(&c->fence_allocator, + nvgpu_alloc(&c->fence_allocator, sizeof(struct gk20a_fence)); /* clear the node and reset the allocator pointer */ diff --git a/drivers/gpu/nvgpu/gk20a/fence_gk20a.h b/drivers/gpu/nvgpu/gk20a/fence_gk20a.h index beba761ab..f38fcbe7f 100644 --- a/drivers/gpu/nvgpu/gk20a/fence_gk20a.h +++ b/drivers/gpu/nvgpu/gk20a/fence_gk20a.h @@ -47,7 +47,7 @@ struct gk20a_fence { u32 syncpt_value; /* Valid for fences part of a pre-allocated fence pool */ - struct gk20a_allocator *allocator; + struct nvgpu_allocator *allocator; }; /* Fences can be created from semaphores or syncpoint (id, value) pairs */ diff --git a/drivers/gpu/nvgpu/gk20a/gk20a.c b/drivers/gpu/nvgpu/gk20a/gk20a.c index b1e90bd8b..753f031a4 100644 --- a/drivers/gpu/nvgpu/gk20a/gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/gk20a.c @@ -43,6 +43,8 @@ #include #include +#include + #include "gk20a.h" #include "nvgpu_common.h" #include "debug_gk20a.h" @@ -60,7 +62,6 @@ #include "gk20a_scale.h" #include "ctxsw_trace_gk20a.h" #include "dbg_gpu_gk20a.h" -#include "gk20a_allocator.h" #include "hal.h" #include "vgpu/vgpu.h" #include "pci.h" diff --git a/drivers/gpu/nvgpu/gk20a/gr_gk20a.c b/drivers/gpu/nvgpu/gk20a/gr_gk20a.c index 39562ec13..2ee2dd434 100644 --- a/drivers/gpu/nvgpu/gk20a/gr_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/gr_gk20a.c @@ -3400,7 +3400,7 @@ static void gk20a_remove_gr_support(struct gr_gk20a *gr) gr->ctx_vars.local_golden_image = NULL; if (gr->ctx_vars.hwpm_ctxsw_buffer_offset_map) - nvgpu_free(gr->ctx_vars.hwpm_ctxsw_buffer_offset_map); + nvgpu_kfree(gr->ctx_vars.hwpm_ctxsw_buffer_offset_map); gr->ctx_vars.hwpm_ctxsw_buffer_offset_map = NULL; gk20a_comptag_allocator_destroy(&gr->comp_tags); @@ -7998,7 +7998,7 @@ static int gr_gk20a_create_hwpm_ctxsw_buffer_offset_map(struct gk20a *g) hwpm_ctxsw_reg_count_max = hwpm_ctxsw_buffer_size >> 2; map_size = hwpm_ctxsw_reg_count_max * sizeof(*map); - map = nvgpu_alloc(map_size, true); + map = nvgpu_kalloc(map_size, true); if (!map) return -ENOMEM; @@ -8088,7 +8088,7 @@ static int gr_gk20a_create_hwpm_ctxsw_buffer_offset_map(struct gk20a *g) return 0; cleanup: gk20a_err(dev_from_gk20a(g), "Failed to create HWPM buffer offset map"); - nvgpu_free(map); + nvgpu_kfree(map); return -EINVAL; } diff --git a/drivers/gpu/nvgpu/gk20a/mm_gk20a.c b/drivers/gpu/nvgpu/gk20a/mm_gk20a.c index 2e338fefb..d594a5a44 100644 --- a/drivers/gpu/nvgpu/gk20a/mm_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/mm_gk20a.c @@ -31,9 +31,9 @@ #include #include -#include - #include +#include +#include #include "gk20a.h" #include "mm_gk20a.h" @@ -74,7 +74,7 @@ is_vidmem_page_alloc(u64 addr) return !!(addr & 1ULL); } -static inline struct gk20a_page_alloc * +static inline struct nvgpu_page_alloc * get_vidmem_page_alloc(struct scatterlist *sgl) { u64 addr; @@ -86,7 +86,7 @@ get_vidmem_page_alloc(struct scatterlist *sgl) else WARN_ON(1); - return (struct gk20a_page_alloc *)(uintptr_t)addr; + return (struct nvgpu_page_alloc *)(uintptr_t)addr; } int gk20a_mem_begin(struct gk20a *g, struct mem_desc *mem) @@ -176,7 +176,7 @@ typedef void (*pramin_access_batch_fn)(struct gk20a *g, u32 start, u32 words, static inline void pramin_access_batched(struct gk20a *g, struct mem_desc *mem, u32 offset, u32 size, pramin_access_batch_fn loop, u32 **arg) { - struct gk20a_page_alloc *alloc = NULL; + struct nvgpu_page_alloc *alloc = NULL; struct page_alloc_chunk *chunk = NULL; u32 byteoff, start_reg, until_end, n; @@ -797,8 +797,8 @@ void gk20a_remove_vm(struct vm_gk20a *vm, struct mem_desc *inst_block) static void gk20a_vidmem_destroy(struct gk20a *g) { #if defined(CONFIG_GK20A_VIDMEM) - if (gk20a_alloc_initialized(&g->mm.vidmem.allocator)) - gk20a_alloc_destroy(&g->mm.vidmem.allocator); + if (nvgpu_alloc_initialized(&g->mm.vidmem.allocator)) + nvgpu_alloc_destroy(&g->mm.vidmem.allocator); #endif } @@ -928,8 +928,8 @@ static int gk20a_init_vidmem(struct mm_gk20a *mm) u64 default_page_size = SZ_64K; int err; - static struct gk20a_alloc_carveout wpr_co = - GK20A_CARVEOUT("wpr-region", 0, SZ_16M); + static struct nvgpu_alloc_carveout wpr_co = + NVGPU_CARVEOUT("wpr-region", 0, SZ_16M); if (!size) return 0; @@ -944,12 +944,12 @@ static int gk20a_init_vidmem(struct mm_gk20a *mm) * initialization requires vidmem but we want to use the CE to zero * out vidmem before allocating it... */ - err = gk20a_page_allocator_init(g, &g->mm.vidmem.bootstrap_allocator, + err = nvgpu_page_allocator_init(g, &g->mm.vidmem.bootstrap_allocator, "vidmem-bootstrap", bootstrap_base, bootstrap_size, SZ_4K, 0); - err = gk20a_page_allocator_init(g, &g->mm.vidmem.allocator, + err = nvgpu_page_allocator_init(g, &g->mm.vidmem.allocator, "vidmem", base, size - base, default_page_size, @@ -961,7 +961,7 @@ static int gk20a_init_vidmem(struct mm_gk20a *mm) } /* Reserve bootstrap region in vidmem allocator */ - gk20a_alloc_reserve_carveout(&g->mm.vidmem.allocator, &wpr_co); + nvgpu_alloc_reserve_carveout(&g->mm.vidmem.allocator, &wpr_co); mm->vidmem.base = base; mm->vidmem.size = size - base; @@ -1482,7 +1482,7 @@ int gk20a_vm_get_buffers(struct vm_gk20a *vm, mutex_lock(&vm->update_gmmu_lock); - buffer_list = nvgpu_alloc(sizeof(*buffer_list) * + buffer_list = nvgpu_kalloc(sizeof(*buffer_list) * vm->num_user_mapped_buffers, true); if (!buffer_list) { mutex_unlock(&vm->update_gmmu_lock); @@ -1567,7 +1567,7 @@ void gk20a_vm_put_buffers(struct vm_gk20a *vm, gk20a_vm_mapping_batch_finish_locked(vm, &batch); mutex_unlock(&vm->update_gmmu_lock); - nvgpu_free(mapped_buffers); + nvgpu_kfree(mapped_buffers); } static void gk20a_vm_unmap_user(struct vm_gk20a *vm, u64 offset, @@ -1623,7 +1623,7 @@ u64 gk20a_vm_alloc_va(struct vm_gk20a *vm, enum gmmu_pgsz_gk20a gmmu_pgsz_idx) { - struct gk20a_allocator *vma = &vm->vma[gmmu_pgsz_idx]; + struct nvgpu_allocator *vma = &vm->vma[gmmu_pgsz_idx]; u64 offset; u64 gmmu_page_size = vm->gmmu_page_sizes[gmmu_pgsz_idx]; @@ -1645,7 +1645,7 @@ u64 gk20a_vm_alloc_va(struct vm_gk20a *vm, gk20a_dbg_info("size=0x%llx @ pgsz=%dKB", size, vm->gmmu_page_sizes[gmmu_pgsz_idx]>>10); - offset = gk20a_alloc(vma, size); + offset = nvgpu_alloc(vma, size); if (!offset) { gk20a_err(dev_from_vm(vm), "%s oom: sz=0x%llx", vma->name, size); @@ -1660,11 +1660,11 @@ int gk20a_vm_free_va(struct vm_gk20a *vm, u64 offset, u64 size, enum gmmu_pgsz_gk20a pgsz_idx) { - struct gk20a_allocator *vma = &vm->vma[pgsz_idx]; + struct nvgpu_allocator *vma = &vm->vma[pgsz_idx]; gk20a_dbg_info("%s free addr=0x%llx, size=0x%llx", vma->name, offset, size); - gk20a_free(vma, offset); + nvgpu_free(vma, offset); return 0; } @@ -2302,15 +2302,15 @@ err_kfree: int gk20a_vidmem_get_space(struct gk20a *g, u64 *space) { #if defined(CONFIG_GK20A_VIDMEM) - struct gk20a_allocator *allocator = &g->mm.vidmem.allocator; + struct nvgpu_allocator *allocator = &g->mm.vidmem.allocator; gk20a_dbg_fn(""); - if (!gk20a_alloc_initialized(allocator)) + if (!nvgpu_alloc_initialized(allocator)) return -ENOSYS; mutex_lock(&g->mm.vidmem.clear_list_mutex); - *space = gk20a_alloc_space(allocator) + + *space = nvgpu_alloc_space(allocator) + atomic64_read(&g->mm.vidmem.bytes_pending); mutex_unlock(&g->mm.vidmem.clear_list_mutex); return 0; @@ -2359,7 +2359,7 @@ static u64 gk20a_mm_get_align(struct gk20a *g, struct scatterlist *sgl, u64 buf_addr; if (aperture == APERTURE_VIDMEM) { - struct gk20a_page_alloc *alloc = get_vidmem_page_alloc(sgl); + struct nvgpu_page_alloc *alloc = get_vidmem_page_alloc(sgl); struct page_alloc_chunk *chunk = NULL; list_for_each_entry(chunk, &alloc->alloc_chunks, list_entry) { @@ -3068,7 +3068,7 @@ static int gk20a_gmmu_clear_vidmem_mem(struct gk20a *g, struct mem_desc *mem) { struct gk20a_fence *gk20a_fence_out = NULL; struct gk20a_fence *gk20a_last_fence = NULL; - struct gk20a_page_alloc *alloc = NULL; + struct nvgpu_page_alloc *alloc = NULL; struct page_alloc_chunk *chunk = NULL; int err = 0; @@ -3134,15 +3134,15 @@ int gk20a_gmmu_alloc_attr_vid(struct gk20a *g, enum dma_attr attr, } #if defined(CONFIG_GK20A_VIDMEM) -static u64 __gk20a_gmmu_alloc(struct gk20a_allocator *allocator, dma_addr_t at, +static u64 __gk20a_gmmu_alloc(struct nvgpu_allocator *allocator, dma_addr_t at, size_t size) { u64 addr = 0; if (at) - addr = gk20a_alloc_fixed(allocator, at, size); + addr = nvgpu_alloc_fixed(allocator, at, size); else - addr = gk20a_alloc(allocator, size); + addr = nvgpu_alloc(allocator, size); return addr; } @@ -3154,14 +3154,14 @@ int gk20a_gmmu_alloc_attr_vid_at(struct gk20a *g, enum dma_attr attr, #if defined(CONFIG_GK20A_VIDMEM) u64 addr; int err; - struct gk20a_allocator *vidmem_alloc = g->mm.vidmem.cleared ? + struct nvgpu_allocator *vidmem_alloc = g->mm.vidmem.cleared ? &g->mm.vidmem.allocator : &g->mm.vidmem.bootstrap_allocator; int before_pending; gk20a_dbg_fn(""); - if (!gk20a_alloc_initialized(&g->mm.vidmem.allocator)) + if (!nvgpu_alloc_initialized(&g->mm.vidmem.allocator)) return -ENOSYS; /* we don't support dma attributes here, except that kernel mappings @@ -3214,7 +3214,7 @@ int gk20a_gmmu_alloc_attr_vid_at(struct gk20a *g, enum dma_attr attr, fail_kfree: kfree(mem->sgt); fail_physfree: - gk20a_free(&g->mm.vidmem.allocator, addr); + nvgpu_free(&g->mm.vidmem.allocator, addr); return err; #else return -ENOSYS; @@ -3241,7 +3241,7 @@ static void gk20a_gmmu_free_attr_vid(struct gk20a *g, enum dma_attr attr, } } else { gk20a_memset(g, mem, 0, 0, mem->size); - gk20a_free(mem->allocator, + nvgpu_free(mem->allocator, (u64)get_vidmem_page_alloc(mem->sgt->sgl)); gk20a_free_sgtable(&mem->sgt); @@ -3276,7 +3276,7 @@ void gk20a_gmmu_free(struct gk20a *g, struct mem_desc *mem) u64 gk20a_mem_get_base_addr(struct gk20a *g, struct mem_desc *mem, u32 flags) { - struct gk20a_page_alloc *alloc; + struct nvgpu_page_alloc *alloc; u64 addr; if (mem->aperture == APERTURE_VIDMEM) { @@ -3317,7 +3317,7 @@ static void gk20a_vidmem_clear_mem_worker(struct work_struct *work) while ((mem = get_pending_mem_desc(mm)) != NULL) { gk20a_gmmu_clear_vidmem_mem(g, mem); - gk20a_free(mem->allocator, + nvgpu_free(mem->allocator, (u64)get_vidmem_page_alloc(mem->sgt->sgl)); gk20a_free_sgtable(&mem->sgt); @@ -3905,7 +3905,7 @@ static int update_gmmu_ptes_locked(struct vm_gk20a *vm, u32 page_size = vm->gmmu_page_sizes[pgsz_idx]; int err; struct scatterlist *sgl = NULL; - struct gk20a_page_alloc *alloc = NULL; + struct nvgpu_page_alloc *alloc = NULL; struct page_alloc_chunk *chunk = NULL; u64 length; @@ -4251,12 +4251,12 @@ static int gk20a_init_sema_pool(struct vm_gk20a *vm) * * !!! TODO: cleanup. */ - sema_sea->gpu_va = gk20a_alloc_fixed(&vm->vma[gmmu_page_size_kernel], + sema_sea->gpu_va = nvgpu_alloc_fixed(&vm->vma[gmmu_page_size_kernel], vm->va_limit - mm->channel.kernel_size, 512 * PAGE_SIZE); if (!sema_sea->gpu_va) { - gk20a_free(&vm->vma[gmmu_page_size_small], sema_sea->gpu_va); + nvgpu_free(&vm->vma[gmmu_page_size_small], sema_sea->gpu_va); gk20a_vm_put(vm); return -ENOMEM; } @@ -4264,7 +4264,7 @@ static int gk20a_init_sema_pool(struct vm_gk20a *vm) err = gk20a_semaphore_pool_map(vm->sema_pool, vm); if (err) { gk20a_semaphore_pool_unmap(vm->sema_pool, vm); - gk20a_free(&vm->vma[gmmu_page_size_small], + nvgpu_free(&vm->vma[gmmu_page_size_small], vm->sema_pool->gpu_va); gk20a_vm_put(vm); } @@ -4387,7 +4387,7 @@ int gk20a_init_vm(struct mm_gk20a *mm, snprintf(alloc_name, sizeof(alloc_name), "gk20a_%s-fixed", name); - err = __gk20a_buddy_allocator_init(g, &vm->fixed, + err = __nvgpu_buddy_allocator_init(g, &vm->fixed, vm, alloc_name, small_vma_start, g->separate_fixed_allocs, @@ -4404,7 +4404,7 @@ int gk20a_init_vm(struct mm_gk20a *mm, if (small_vma_start < small_vma_limit) { snprintf(alloc_name, sizeof(alloc_name), "gk20a_%s-%dKB", name, vm->gmmu_page_sizes[gmmu_page_size_small] >> 10); - err = __gk20a_buddy_allocator_init( + err = __nvgpu_buddy_allocator_init( g, &vm->vma[gmmu_page_size_small], vm, alloc_name, @@ -4420,7 +4420,7 @@ int gk20a_init_vm(struct mm_gk20a *mm, if (large_vma_start < large_vma_limit) { snprintf(alloc_name, sizeof(alloc_name), "gk20a_%s-%dKB", name, vm->gmmu_page_sizes[gmmu_page_size_big] >> 10); - err = __gk20a_buddy_allocator_init( + err = __nvgpu_buddy_allocator_init( g, &vm->vma[gmmu_page_size_big], vm, alloc_name, @@ -4438,7 +4438,7 @@ int gk20a_init_vm(struct mm_gk20a *mm, /* * kernel reserved VMA is at the end of the aperture */ - err = __gk20a_buddy_allocator_init(g, &vm->vma[gmmu_page_size_kernel], + err = __nvgpu_buddy_allocator_init(g, &vm->vma[gmmu_page_size_kernel], vm, alloc_name, kernel_vma_start, kernel_vma_limit - kernel_vma_start, @@ -4469,10 +4469,10 @@ int gk20a_init_vm(struct mm_gk20a *mm, clean_up_big_allocator: if (large_vma_start < large_vma_limit) - gk20a_alloc_destroy(&vm->vma[gmmu_page_size_big]); + nvgpu_alloc_destroy(&vm->vma[gmmu_page_size_big]); clean_up_small_allocator: if (small_vma_start < small_vma_limit) - gk20a_alloc_destroy(&vm->vma[gmmu_page_size_small]); + nvgpu_alloc_destroy(&vm->vma[gmmu_page_size_small]); clean_up_ptes: free_gmmu_pages(vm, &vm->pdb); clean_up_pdes: @@ -4547,7 +4547,7 @@ int gk20a_vm_alloc_space(struct gk20a_as_share *as_share, { int err = -ENOMEM; int pgsz_idx = gmmu_page_size_small; - struct gk20a_allocator *vma; + struct nvgpu_allocator *vma; struct vm_gk20a *vm = as_share->vm; struct gk20a *g = vm->mm->g; struct vm_reserved_va_node *va_node; @@ -4579,13 +4579,13 @@ int gk20a_vm_alloc_space(struct gk20a_as_share *as_share, vma = &vm->vma[pgsz_idx]; if (args->flags & NVGPU_AS_ALLOC_SPACE_FLAGS_FIXED_OFFSET) { - if (gk20a_alloc_initialized(&vm->fixed)) + if (nvgpu_alloc_initialized(&vm->fixed)) vma = &vm->fixed; - vaddr_start = gk20a_alloc_fixed(vma, args->o_a.offset, + vaddr_start = nvgpu_alloc_fixed(vma, args->o_a.offset, (u64)args->pages * (u64)args->page_size); } else { - vaddr_start = gk20a_alloc(vma, + vaddr_start = nvgpu_alloc(vma, (u64)args->pages * (u64)args->page_size); } @@ -4621,7 +4621,7 @@ int gk20a_vm_alloc_space(struct gk20a_as_share *as_share, APERTURE_INVALID); if (!map_offset) { mutex_unlock(&vm->update_gmmu_lock); - gk20a_free(vma, vaddr_start); + nvgpu_free(vma, vaddr_start); kfree(va_node); goto clean_up; } @@ -4644,7 +4644,7 @@ int gk20a_vm_free_space(struct gk20a_as_share *as_share, { int err = -ENOMEM; int pgsz_idx; - struct gk20a_allocator *vma; + struct nvgpu_allocator *vma; struct vm_gk20a *vm = as_share->vm; struct vm_reserved_va_node *va_node; struct gk20a *g = gk20a_from_vm(vm); @@ -4656,11 +4656,11 @@ int gk20a_vm_free_space(struct gk20a_as_share *as_share, pgsz_idx = __nv_gmmu_va_is_big_page_region(vm, args->offset) ? gmmu_page_size_big : gmmu_page_size_small; - if (gk20a_alloc_initialized(&vm->fixed)) + if (nvgpu_alloc_initialized(&vm->fixed)) vma = &vm->fixed; else vma = &vm->vma[pgsz_idx]; - gk20a_free(vma, args->offset); + nvgpu_free(vma, args->offset); mutex_lock(&vm->update_gmmu_lock); va_node = addr_to_reservation(vm, args->offset); @@ -4844,13 +4844,13 @@ int gk20a_vm_unmap_buffer(struct vm_gk20a *vm, u64 offset, void gk20a_deinit_vm(struct vm_gk20a *vm) { - gk20a_alloc_destroy(&vm->vma[gmmu_page_size_kernel]); - if (gk20a_alloc_initialized(&vm->vma[gmmu_page_size_big])) - gk20a_alloc_destroy(&vm->vma[gmmu_page_size_big]); - if (gk20a_alloc_initialized(&vm->vma[gmmu_page_size_small])) - gk20a_alloc_destroy(&vm->vma[gmmu_page_size_small]); - if (gk20a_alloc_initialized(&vm->fixed)) - gk20a_alloc_destroy(&vm->fixed); + nvgpu_alloc_destroy(&vm->vma[gmmu_page_size_kernel]); + if (nvgpu_alloc_initialized(&vm->vma[gmmu_page_size_big])) + nvgpu_alloc_destroy(&vm->vma[gmmu_page_size_big]); + if (nvgpu_alloc_initialized(&vm->vma[gmmu_page_size_small])) + nvgpu_alloc_destroy(&vm->vma[gmmu_page_size_small]); + if (nvgpu_alloc_initialized(&vm->fixed)) + nvgpu_alloc_destroy(&vm->fixed); gk20a_vm_free_entries(vm, &vm->pdb, 0); } diff --git a/drivers/gpu/nvgpu/gk20a/mm_gk20a.h b/drivers/gpu/nvgpu/gk20a/mm_gk20a.h index d32e121a0..f58b5df5b 100644 --- a/drivers/gpu/nvgpu/gk20a/mm_gk20a.h +++ b/drivers/gpu/nvgpu/gk20a/mm_gk20a.h @@ -27,7 +27,8 @@ #include #include #include -#include "gk20a_allocator.h" + +#include #ifdef CONFIG_ARM64 #define outer_flush_range(a, b) @@ -70,7 +71,7 @@ struct mem_desc { u64 gpu_va; bool fixed; /* vidmem only */ bool user_mem; /* vidmem only */ - struct gk20a_allocator *allocator; /* vidmem only */ + struct nvgpu_allocator *allocator; /* vidmem only */ struct list_head clear_list_entry; /* vidmem only */ bool skip_wmb; }; @@ -295,10 +296,10 @@ struct vm_gk20a { struct gk20a_mm_entry pdb; - struct gk20a_allocator vma[gmmu_nr_page_sizes]; + struct nvgpu_allocator vma[gmmu_nr_page_sizes]; /* If necessary, split fixed from non-fixed. */ - struct gk20a_allocator fixed; + struct nvgpu_allocator fixed; struct rb_root mapped_buffers; @@ -421,8 +422,8 @@ struct mm_gk20a { size_t bootstrap_size; u64 bootstrap_base; - struct gk20a_allocator allocator; - struct gk20a_allocator bootstrap_allocator; + struct nvgpu_allocator allocator; + struct nvgpu_allocator bootstrap_allocator; u32 ce_ctx_id; volatile bool cleared; @@ -470,13 +471,13 @@ static inline u64 __nv_gmmu_va_small_page_limit(void) static inline int __nv_gmmu_va_is_big_page_region(struct vm_gk20a *vm, u64 addr) { - struct gk20a_allocator *a = &vm->vma[gmmu_page_size_big]; + struct nvgpu_allocator *a = &vm->vma[gmmu_page_size_big]; if (!vm->big_pages) return 0; - return addr >= gk20a_alloc_base(a) && - addr < gk20a_alloc_base(a) + gk20a_alloc_length(a); + return addr >= nvgpu_alloc_base(a) && + addr < nvgpu_alloc_base(a) + nvgpu_alloc_length(a); } /* @@ -825,7 +826,7 @@ void gk20a_remove_vm(struct vm_gk20a *vm, struct mem_desc *inst_block); extern const struct gk20a_mmu_level gk20a_mm_levels_64k[]; extern const struct gk20a_mmu_level gk20a_mm_levels_128k[]; -static inline void *nvgpu_alloc(size_t size, bool clear) +static inline void *nvgpu_kalloc(size_t size, bool clear) { void *p; @@ -844,7 +845,7 @@ static inline void *nvgpu_alloc(size_t size, bool clear) return p; } -static inline void nvgpu_free(void *p) +static inline void nvgpu_kfree(void *p) { if (virt_addr_valid(p)) kfree(p); diff --git a/drivers/gpu/nvgpu/gk20a/pmu_gk20a.c b/drivers/gpu/nvgpu/gk20a/pmu_gk20a.c index e221be11c..56ebda1a6 100644 --- a/drivers/gpu/nvgpu/gk20a/pmu_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/pmu_gk20a.c @@ -2896,8 +2896,8 @@ void gk20a_remove_pmu_support(struct pmu_gk20a *pmu) { gk20a_dbg_fn(""); - if (gk20a_alloc_initialized(&pmu->dmem)) - gk20a_alloc_destroy(&pmu->dmem); + if (nvgpu_alloc_initialized(&pmu->dmem)) + nvgpu_alloc_destroy(&pmu->dmem); release_firmware(pmu->fw); } @@ -3607,7 +3607,7 @@ static int pmu_init_perfmon(struct pmu_gk20a *pmu) gk20a_writel(g, pwr_pmu_idle_ctrl_r(2), data); if (!pmu->sample_buffer) - pmu->sample_buffer = gk20a_alloc(&pmu->dmem, + pmu->sample_buffer = nvgpu_alloc(&pmu->dmem, 2 * sizeof(u16)); if (!pmu->sample_buffer) { gk20a_err(dev_from_gk20a(g), @@ -3708,7 +3708,7 @@ static int pmu_process_init_msg(struct pmu_gk20a *pmu, for (i = 0; i < PMU_QUEUE_COUNT; i++) pmu_queue_init(pmu, i, init); - if (!gk20a_alloc_initialized(&pmu->dmem)) { + if (!nvgpu_alloc_initialized(&pmu->dmem)) { /* Align start and end addresses */ u32 start = ALIGN(pv->get_pmu_init_msg_pmu_sw_mg_off(init), PMU_DMEM_ALLOC_ALIGNMENT); @@ -3716,9 +3716,9 @@ static int pmu_process_init_msg(struct pmu_gk20a *pmu, pv->get_pmu_init_msg_pmu_sw_mg_size(init)) & ~(PMU_DMEM_ALLOC_ALIGNMENT - 1); u32 size = end - start; - gk20a_bitmap_allocator_init(g, &pmu->dmem, "gk20a_pmu_dmem", - start, size, - PMU_DMEM_ALLOC_ALIGNMENT, 0); + nvgpu_bitmap_allocator_init(g, &pmu->dmem, "gk20a_pmu_dmem", + start, size, + PMU_DMEM_ALLOC_ALIGNMENT, 0); } pmu->pmu_ready = true; @@ -3855,12 +3855,12 @@ static int pmu_response_handle(struct pmu_gk20a *pmu, seq->callback = NULL; if (pv->pmu_allocation_get_dmem_size(pmu, pv->get_pmu_seq_in_a_ptr(seq)) != 0) - gk20a_free(&pmu->dmem, + nvgpu_free(&pmu->dmem, pv->pmu_allocation_get_dmem_offset(pmu, pv->get_pmu_seq_in_a_ptr(seq))); if (pv->pmu_allocation_get_dmem_size(pmu, pv->get_pmu_seq_out_a_ptr(seq)) != 0) - gk20a_free(&pmu->dmem, + nvgpu_free(&pmu->dmem, pv->pmu_allocation_get_dmem_offset(pmu, pv->get_pmu_seq_out_a_ptr(seq))); @@ -4601,7 +4601,7 @@ int gk20a_pmu_cmd_post(struct gk20a *g, struct pmu_cmd *cmd, (u16)max(payload->in.size, payload->out.size)); *(pv->pmu_allocation_get_dmem_offset_addr(pmu, in)) = - gk20a_alloc(&pmu->dmem, + nvgpu_alloc(&pmu->dmem, pv->pmu_allocation_get_dmem_size(pmu, in)); if (!*(pv->pmu_allocation_get_dmem_offset_addr(pmu, in))) goto clean_up; @@ -4644,7 +4644,7 @@ int gk20a_pmu_cmd_post(struct gk20a *g, struct pmu_cmd *cmd, if (payload->in.buf != payload->out.buf) { *(pv->pmu_allocation_get_dmem_offset_addr(pmu, out)) = - gk20a_alloc(&pmu->dmem, + nvgpu_alloc(&pmu->dmem, pv->pmu_allocation_get_dmem_size(pmu, out)); if (!*(pv->pmu_allocation_get_dmem_offset_addr(pmu, out))) @@ -4694,10 +4694,10 @@ int gk20a_pmu_cmd_post(struct gk20a *g, struct pmu_cmd *cmd, clean_up: gk20a_dbg_fn("fail"); if (in) - gk20a_free(&pmu->dmem, + nvgpu_free(&pmu->dmem, pv->pmu_allocation_get_dmem_offset(pmu, in)); if (out) - gk20a_free(&pmu->dmem, + nvgpu_free(&pmu->dmem, pv->pmu_allocation_get_dmem_offset(pmu, out)); pmu_seq_release(pmu, seq); diff --git a/drivers/gpu/nvgpu/gk20a/pmu_gk20a.h b/drivers/gpu/nvgpu/gk20a/pmu_gk20a.h index cf4f3b52e..32e2ef542 100644 --- a/drivers/gpu/nvgpu/gk20a/pmu_gk20a.h +++ b/drivers/gpu/nvgpu/gk20a/pmu_gk20a.h @@ -709,7 +709,7 @@ struct pmu_gk20a { struct mutex pmu_copy_lock; struct mutex pmu_seq_lock; - struct gk20a_allocator dmem; + struct nvgpu_allocator dmem; u32 *ucode_image; bool pmu_ready; diff --git a/drivers/gpu/nvgpu/gk20a/semaphore_gk20a.h b/drivers/gpu/nvgpu/gk20a/semaphore_gk20a.h index cf724fdb4..8e09fcfca 100644 --- a/drivers/gpu/nvgpu/gk20a/semaphore_gk20a.h +++ b/drivers/gpu/nvgpu/gk20a/semaphore_gk20a.h @@ -18,10 +18,11 @@ #include #include +#include + #include "gk20a.h" #include "mm_gk20a.h" #include "channel_gk20a.h" -#include "gk20a_allocator.h" #define gpu_sema_dbg(fmt, args...) \ gk20a_dbg(gpu_dbg_sema, fmt, ##args) diff --git a/drivers/gpu/nvgpu/gk20a/gk20a_allocator.h b/drivers/gpu/nvgpu/include/nvgpu/allocator.h similarity index 67% rename from drivers/gpu/nvgpu/gk20a/gk20a_allocator.h rename to drivers/gpu/nvgpu/include/nvgpu/allocator.h index b12926b3b..dee9b5625 100644 --- a/drivers/gpu/nvgpu/gk20a/gk20a_allocator.h +++ b/drivers/gpu/nvgpu/include/nvgpu/allocator.h @@ -14,8 +14,8 @@ * along with this program. If not, see . */ -#ifndef GK20A_ALLOCATOR_H -#define GK20A_ALLOCATOR_H +#ifndef NVGPU_ALLOCATOR_H +#define NVGPU_ALLOCATOR_H #include #include @@ -23,17 +23,17 @@ /* #define ALLOCATOR_DEBUG */ -struct gk20a_allocator; -struct gk20a_alloc_carveout; +struct nvgpu_allocator; +struct nvgpu_alloc_carveout; struct vm_gk20a; struct gk20a; /* * Operations for an allocator to implement. */ -struct gk20a_allocator_ops { - u64 (*alloc)(struct gk20a_allocator *allocator, u64 len); - void (*free)(struct gk20a_allocator *allocator, u64 addr); +struct nvgpu_allocator_ops { + u64 (*alloc)(struct nvgpu_allocator *allocator, u64 len); + void (*free)(struct nvgpu_allocator *allocator, u64 addr); /* * Special interface to allocate a memory region with a specific @@ -42,53 +42,53 @@ struct gk20a_allocator_ops { * be implemented. This behavior exists for legacy reasons and should * not be propagated to new allocators. */ - u64 (*alloc_fixed)(struct gk20a_allocator *allocator, + u64 (*alloc_fixed)(struct nvgpu_allocator *allocator, u64 base, u64 len); - void (*free_fixed)(struct gk20a_allocator *allocator, + void (*free_fixed)(struct nvgpu_allocator *allocator, u64 base, u64 len); /* * Allow allocators to reserve space for carveouts. */ - int (*reserve_carveout)(struct gk20a_allocator *allocator, - struct gk20a_alloc_carveout *co); - void (*release_carveout)(struct gk20a_allocator *allocator, - struct gk20a_alloc_carveout *co); + int (*reserve_carveout)(struct nvgpu_allocator *allocator, + struct nvgpu_alloc_carveout *co); + void (*release_carveout)(struct nvgpu_allocator *allocator, + struct nvgpu_alloc_carveout *co); /* * Returns info about the allocator. */ - u64 (*base)(struct gk20a_allocator *allocator); - u64 (*length)(struct gk20a_allocator *allocator); - u64 (*end)(struct gk20a_allocator *allocator); - int (*inited)(struct gk20a_allocator *allocator); - u64 (*space)(struct gk20a_allocator *allocator); + u64 (*base)(struct nvgpu_allocator *allocator); + u64 (*length)(struct nvgpu_allocator *allocator); + u64 (*end)(struct nvgpu_allocator *allocator); + int (*inited)(struct nvgpu_allocator *allocator); + u64 (*space)(struct nvgpu_allocator *allocator); /* Destructor. */ - void (*fini)(struct gk20a_allocator *allocator); + void (*fini)(struct nvgpu_allocator *allocator); /* Debugging. */ - void (*print_stats)(struct gk20a_allocator *allocator, + void (*print_stats)(struct nvgpu_allocator *allocator, struct seq_file *s, int lock); }; -struct gk20a_allocator { +struct nvgpu_allocator { char name[32]; struct mutex lock; void *priv; - const struct gk20a_allocator_ops *ops; + const struct nvgpu_allocator_ops *ops; struct dentry *debugfs_entry; bool debug; /* Control for debug msgs. */ }; -struct gk20a_alloc_carveout { +struct nvgpu_alloc_carveout { const char *name; u64 base; u64 length; - struct gk20a_allocator *allocator; + struct nvgpu_allocator *allocator; /* * For usage by the allocator implementation. @@ -96,7 +96,7 @@ struct gk20a_alloc_carveout { struct list_head co_entry; }; -#define GK20A_CARVEOUT(__name, __base, __length) \ +#define NVGPU_CARVEOUT(__name, __base, __length) \ { \ .name = (__name), \ .base = (__base), \ @@ -161,12 +161,12 @@ struct gk20a_alloc_carveout { #define GPU_ALLOC_FORCE_CONTIG 0x8 #define GPU_ALLOC_NO_SCATTER_GATHER 0x10 -static inline void alloc_lock(struct gk20a_allocator *a) +static inline void alloc_lock(struct nvgpu_allocator *a) { mutex_lock(&a->lock); } -static inline void alloc_unlock(struct gk20a_allocator *a) +static inline void alloc_unlock(struct nvgpu_allocator *a) { mutex_unlock(&a->lock); } @@ -174,25 +174,25 @@ static inline void alloc_unlock(struct gk20a_allocator *a) /* * Buddy allocator specific initializers. */ -int __gk20a_buddy_allocator_init(struct gk20a *g, struct gk20a_allocator *a, +int __nvgpu_buddy_allocator_init(struct gk20a *g, struct nvgpu_allocator *a, struct vm_gk20a *vm, const char *name, u64 base, u64 size, u64 blk_size, u64 max_order, u64 flags); -int gk20a_buddy_allocator_init(struct gk20a *g, struct gk20a_allocator *a, +int nvgpu_buddy_allocator_init(struct gk20a *g, struct nvgpu_allocator *a, const char *name, u64 base, u64 size, u64 blk_size, u64 flags); /* * Bitmap initializers. */ -int gk20a_bitmap_allocator_init(struct gk20a *g, struct gk20a_allocator *a, +int nvgpu_bitmap_allocator_init(struct gk20a *g, struct nvgpu_allocator *a, const char *name, u64 base, u64 length, u64 blk_size, u64 flags); /* * Page allocator initializers. */ -int gk20a_page_allocator_init(struct gk20a *g, struct gk20a_allocator *a, +int nvgpu_page_allocator_init(struct gk20a *g, struct nvgpu_allocator *a, const char *name, u64 base, u64 length, u64 blk_size, u64 flags); @@ -201,7 +201,7 @@ int gk20a_page_allocator_init(struct gk20a *g, struct gk20a_allocator *a, * Note: This allocator can only allocate fixed-size structures of a * pre-defined size. */ -int gk20a_lockless_allocator_init(struct gk20a *g, struct gk20a_allocator *a, +int nvgpu_lockless_allocator_init(struct gk20a *g, struct nvgpu_allocator *a, const char *name, u64 base, u64 length, u64 struct_size, u64 flags); @@ -210,44 +210,44 @@ int gk20a_lockless_allocator_init(struct gk20a *g, struct gk20a_allocator *a, /* * Allocator APIs. */ -u64 gk20a_alloc(struct gk20a_allocator *allocator, u64 len); -void gk20a_free(struct gk20a_allocator *allocator, u64 addr); +u64 nvgpu_alloc(struct nvgpu_allocator *allocator, u64 len); +void nvgpu_free(struct nvgpu_allocator *allocator, u64 addr); -u64 gk20a_alloc_fixed(struct gk20a_allocator *allocator, u64 base, u64 len); -void gk20a_free_fixed(struct gk20a_allocator *allocator, u64 base, u64 len); +u64 nvgpu_alloc_fixed(struct nvgpu_allocator *allocator, u64 base, u64 len); +void nvgpu_free_fixed(struct nvgpu_allocator *allocator, u64 base, u64 len); -int gk20a_alloc_reserve_carveout(struct gk20a_allocator *a, - struct gk20a_alloc_carveout *co); -void gk20a_alloc_release_carveout(struct gk20a_allocator *a, - struct gk20a_alloc_carveout *co); +int nvgpu_alloc_reserve_carveout(struct nvgpu_allocator *a, + struct nvgpu_alloc_carveout *co); +void nvgpu_alloc_release_carveout(struct nvgpu_allocator *a, + struct nvgpu_alloc_carveout *co); -u64 gk20a_alloc_base(struct gk20a_allocator *a); -u64 gk20a_alloc_length(struct gk20a_allocator *a); -u64 gk20a_alloc_end(struct gk20a_allocator *a); -u64 gk20a_alloc_initialized(struct gk20a_allocator *a); -u64 gk20a_alloc_space(struct gk20a_allocator *a); +u64 nvgpu_alloc_base(struct nvgpu_allocator *a); +u64 nvgpu_alloc_length(struct nvgpu_allocator *a); +u64 nvgpu_alloc_end(struct nvgpu_allocator *a); +u64 nvgpu_alloc_initialized(struct nvgpu_allocator *a); +u64 nvgpu_alloc_space(struct nvgpu_allocator *a); -void gk20a_alloc_destroy(struct gk20a_allocator *allocator); +void nvgpu_alloc_destroy(struct nvgpu_allocator *allocator); -void gk20a_alloc_print_stats(struct gk20a_allocator *a, +void nvgpu_alloc_print_stats(struct nvgpu_allocator *a, struct seq_file *s, int lock); /* * Common functionality for the internals of the allocators. */ -void gk20a_init_alloc_debug(struct gk20a *g, struct gk20a_allocator *a); -void gk20a_fini_alloc_debug(struct gk20a_allocator *a); +void nvgpu_init_alloc_debug(struct gk20a *g, struct nvgpu_allocator *a); +void nvgpu_fini_alloc_debug(struct nvgpu_allocator *a); -int __gk20a_alloc_common_init(struct gk20a_allocator *a, +int __nvgpu_alloc_common_init(struct nvgpu_allocator *a, const char *name, void *priv, bool dbg, - const struct gk20a_allocator_ops *ops); + const struct nvgpu_allocator_ops *ops); -static inline void gk20a_alloc_enable_dbg(struct gk20a_allocator *a) +static inline void nvgpu_alloc_enable_dbg(struct nvgpu_allocator *a) { a->debug = true; } -static inline void gk20a_alloc_disable_dbg(struct gk20a_allocator *a) +static inline void nvgpu_alloc_disable_dbg(struct nvgpu_allocator *a) { a->debug = false; } @@ -255,19 +255,19 @@ static inline void gk20a_alloc_disable_dbg(struct gk20a_allocator *a) /* * Debug stuff. */ -extern u32 gk20a_alloc_tracing_on; +extern u32 nvgpu_alloc_tracing_on; -void gk20a_alloc_debugfs_init(struct device *dev); +void nvgpu_alloc_debugfs_init(struct device *dev); -#define gk20a_alloc_trace_func() \ +#define nvgpu_alloc_trace_func() \ do { \ - if (gk20a_alloc_tracing_on) \ + if (nvgpu_alloc_tracing_on) \ trace_printk("%s\n", __func__); \ } while (0) -#define gk20a_alloc_trace_func_done() \ +#define nvgpu_alloc_trace_func_done() \ do { \ - if (gk20a_alloc_tracing_on) \ + if (nvgpu_alloc_tracing_on) \ trace_printk("%s_done\n", __func__); \ } while (0) @@ -299,4 +299,4 @@ void gk20a_alloc_debugfs_init(struct device *dev); #endif -#endif /* GK20A_ALLOCATOR_H */ +#endif /* NVGPU_ALLOCATOR_H */ diff --git a/drivers/gpu/nvgpu/gk20a/page_allocator_priv.h b/drivers/gpu/nvgpu/include/nvgpu/page_allocator.h similarity index 88% rename from drivers/gpu/nvgpu/gk20a/page_allocator_priv.h rename to drivers/gpu/nvgpu/include/nvgpu/page_allocator.h index 7d7f43c20..7c21c1170 100644 --- a/drivers/gpu/nvgpu/gk20a/page_allocator_priv.h +++ b/drivers/gpu/nvgpu/include/nvgpu/page_allocator.h @@ -20,9 +20,9 @@ #include #include -#include "gk20a_allocator.h" +#include -struct gk20a_allocator; +struct nvgpu_allocator; /* * This allocator implements the ability to do SLAB style allocation since the @@ -88,7 +88,7 @@ struct page_alloc_chunk { * of the chunks of pages that make up the overall allocation - much like a * scatter gather table. */ -struct gk20a_page_alloc { +struct nvgpu_page_alloc { struct list_head alloc_chunks; int nr_chunks; @@ -111,15 +111,15 @@ struct gk20a_page_alloc { struct page_alloc_slab_page *slab_page; }; -struct gk20a_page_allocator { - struct gk20a_allocator *owner; /* Owner of this allocator. */ +struct nvgpu_page_allocator { + struct nvgpu_allocator *owner; /* Owner of this allocator. */ /* * Use a buddy allocator to manage the allocation of the underlying * pages. This lets us abstract the discontiguous allocation handling * out of the annoyingly complicated buddy allocator. */ - struct gk20a_allocator source_allocator; + struct nvgpu_allocator source_allocator; /* * Page params. @@ -149,14 +149,14 @@ struct gk20a_page_allocator { u64 pages_freed; }; -static inline struct gk20a_page_allocator *page_allocator( - struct gk20a_allocator *a) +static inline struct nvgpu_page_allocator *page_allocator( + struct nvgpu_allocator *a) { - return (struct gk20a_page_allocator *)(a)->priv; + return (struct nvgpu_page_allocator *)(a)->priv; } -static inline struct gk20a_allocator *palloc_owner( - struct gk20a_page_allocator *a) +static inline struct nvgpu_allocator *palloc_owner( + struct nvgpu_page_allocator *a) { return a->owner; } diff --git a/drivers/gpu/nvgpu/vgpu/mm_vgpu.c b/drivers/gpu/nvgpu/vgpu/mm_vgpu.c index 69f6fcaf9..66c9344b6 100644 --- a/drivers/gpu/nvgpu/vgpu/mm_vgpu.c +++ b/drivers/gpu/nvgpu/vgpu/mm_vgpu.c @@ -227,11 +227,11 @@ static void vgpu_vm_remove_support(struct vm_gk20a *vm) err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg)); WARN_ON(err || msg.ret); - gk20a_alloc_destroy(&vm->vma[gmmu_page_size_kernel]); - if (gk20a_alloc_initialized(&vm->vma[gmmu_page_size_small])) - gk20a_alloc_destroy(&vm->vma[gmmu_page_size_small]); - if (gk20a_alloc_initialized(&vm->vma[gmmu_page_size_big])) - gk20a_alloc_destroy(&vm->vma[gmmu_page_size_big]); + nvgpu_alloc_destroy(&vm->vma[gmmu_page_size_kernel]); + if (nvgpu_alloc_initialized(&vm->vma[gmmu_page_size_small])) + nvgpu_alloc_destroy(&vm->vma[gmmu_page_size_small]); + if (nvgpu_alloc_initialized(&vm->vma[gmmu_page_size_big])) + nvgpu_alloc_destroy(&vm->vma[gmmu_page_size_big]); mutex_unlock(&vm->update_gmmu_lock); @@ -370,7 +370,7 @@ static int vgpu_vm_alloc_share(struct gk20a_as_share *as_share, snprintf(name, sizeof(name), "gk20a_as_%d-%dKB", as_share->id, gmmu_page_sizes[gmmu_page_size_small] >> 10); - err = __gk20a_buddy_allocator_init( + err = __nvgpu_buddy_allocator_init( g, &vm->vma[gmmu_page_size_small], vm, name, @@ -386,7 +386,7 @@ static int vgpu_vm_alloc_share(struct gk20a_as_share *as_share, if (large_vma_start < large_vma_limit) { snprintf(name, sizeof(name), "gk20a_as_%d-%dKB", as_share->id, gmmu_page_sizes[gmmu_page_size_big] >> 10); - err = __gk20a_buddy_allocator_init( + err = __nvgpu_buddy_allocator_init( g, &vm->vma[gmmu_page_size_big], vm, name, @@ -404,7 +404,7 @@ static int vgpu_vm_alloc_share(struct gk20a_as_share *as_share, /* * kernel reserved VMA is at the end of the aperture */ - err = __gk20a_buddy_allocator_init( + err = __nvgpu_buddy_allocator_init( g, &vm->vma[gmmu_page_size_kernel], vm, name, @@ -428,10 +428,10 @@ static int vgpu_vm_alloc_share(struct gk20a_as_share *as_share, clean_up_big_allocator: if (large_vma_start < large_vma_limit) - gk20a_alloc_destroy(&vm->vma[gmmu_page_size_big]); + nvgpu_alloc_destroy(&vm->vma[gmmu_page_size_big]); clean_up_small_allocator: if (small_vma_start < small_vma_limit) - gk20a_alloc_destroy(&vm->vma[gmmu_page_size_small]); + nvgpu_alloc_destroy(&vm->vma[gmmu_page_size_small]); clean_up_share: msg.cmd = TEGRA_VGPU_CMD_AS_FREE_SHARE; msg.handle = vgpu_get_handle(g);