gpu: nvgpu: Use new kmem API functions (common/*)

Use the new kmem API functions in common/* and common/mm/*.

Add a struct gk20a pointer to struct nvgpu_allocator in order
to store the gk20a pointer used for allocating memory.

Bug 1799159
Bug 1823380

Change-Id: I881ea9545e8a8f0b75d77a1e35dd1812e0bb654e
Signed-off-by: Alex Waterman <alexw@nvidia.com>
Reviewed-on: http://git-master/r/1318315
Reviewed-by: svccoveritychecker <svccoveritychecker@nvidia.com>
GVS: Gerrit_Virtual_Submit
Reviewed-by: Terje Bergstrom <tbergstrom@nvidia.com>

drivers/gpu/nvgpu/common/mm/bitmap_allocator.c

@@ -19,6 +19,7 @@
 #include <linux/bitops.h>
 
 #include <nvgpu/allocator.h>
+#include <nvgpu/kmem.h>
 
 #include "bitmap_allocator_priv.h"
 
@@ -248,12 +249,11 @@ static u64 nvgpu_bitmap_alloc(struct nvgpu_allocator *__a, u64 len)
 
 	/*
 	 * Only do meta-data storage if we are allowed to allocate storage for
-	 * that meta-data. The issue with using kmalloc() and friends is that
+	 * that meta-data. The issue with using malloc and friends is that
 	 * in latency and success critical paths an alloc_page() call can either
-	 * sleep for potentially a long time or, assuming GFP_ATOMIC, fail.
-	 * Since we might not want either of these possibilities assume that the
-	 * caller will keep what data it needs around to successfully free this
-	 * allocation.
+	 * sleep for potentially a long time or fail. Since we might not want
+	 * either of these possibilities assume that the caller will keep what
+	 * data it needs around to successfully free this allocation.
 	 */
 	if (!(a->flags & GPU_ALLOC_NO_ALLOC_PAGE) &&
 	    __nvgpu_bitmap_store_alloc(a, addr, blks * a->blk_size))
@@ -332,8 +332,8 @@ static void nvgpu_bitmap_alloc_destroy(struct nvgpu_allocator *__a)
 	}
 
 	nvgpu_kmem_cache_destroy(a->meta_data_cache);
-	kfree(a->bitmap);
-	kfree(a);
+	nvgpu_kfree(nvgpu_alloc_to_gpu(__a), a->bitmap);
+	nvgpu_kfree(nvgpu_alloc_to_gpu(__a), a);
 }
 
 static void nvgpu_bitmap_print_stats(struct nvgpu_allocator *__a,
@@ -397,11 +397,11 @@ int nvgpu_bitmap_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a,
 		length -= blk_size;
 	}
 
-	a = kzalloc(sizeof(struct nvgpu_bitmap_allocator), GFP_KERNEL);
+	a = nvgpu_kzalloc(g, sizeof(struct nvgpu_bitmap_allocator));
 	if (!a)
 		return -ENOMEM;
 
-	err = __nvgpu_alloc_common_init(__a, name, a, false, &bitmap_ops);
+	err = __nvgpu_alloc_common_init(__a, g, name, a, false, &bitmap_ops);
 	if (err)
 		goto fail;
 
@@ -422,8 +422,8 @@ int nvgpu_bitmap_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a,
 	a->bit_offs = a->base >> a->blk_shift;
 	a->flags = flags;
 
-	a->bitmap = kcalloc(BITS_TO_LONGS(a->num_bits), sizeof(*a->bitmap),
-			    GFP_KERNEL);
+	a->bitmap = nvgpu_kcalloc(g, BITS_TO_LONGS(a->num_bits),
+				  sizeof(*a->bitmap));
 	if (!a->bitmap) {
 		err = -ENOMEM;
 		goto fail;
@@ -445,6 +445,6 @@ int nvgpu_bitmap_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a,
 fail:
 	if (a->meta_data_cache)
 		nvgpu_kmem_cache_destroy(a->meta_data_cache);
-	kfree(a);
+	nvgpu_kfree(g, a);
 	return err;
 }

drivers/gpu/nvgpu/common/mm/buddy_allocator.c

@@ -18,6 +18,7 @@
 #include <linux/slab.h>
 
 #include <nvgpu/allocator.h>
+#include <nvgpu/kmem.h>
 
 #include "gk20a/mm_gk20a.h"
 #include "gk20a/platform_gk20a.h"
@@ -304,7 +305,7 @@ static void nvgpu_buddy_allocator_destroy(struct nvgpu_allocator *__a)
 	}
 
 	nvgpu_kmem_cache_destroy(a->buddy_cache);
-	kfree(a);
+	nvgpu_kfree(nvgpu_alloc_to_gpu(__a), a);
 
 	alloc_unlock(__a);
 }
@@ -809,7 +810,7 @@ static void __balloc_do_free_fixed(struct nvgpu_buddy_allocator *a,
 		balloc_coalesce(a, bud);
 	}
 
-	kfree(falloc);
+	nvgpu_kfree(nvgpu_alloc_to_gpu(a->owner), falloc);
 }
 
 /*
@@ -893,7 +894,7 @@ static u64 __nvgpu_balloc_fixed_buddy(struct nvgpu_allocator *__a,
 			goto fail;
 	}
 
-	falloc = kmalloc(sizeof(*falloc), GFP_KERNEL);
+	falloc = nvgpu_kmalloc(nvgpu_alloc_to_gpu(__a), sizeof(*falloc));
 	if (!falloc)
 		goto fail;
 
@@ -932,7 +933,7 @@ static u64 __nvgpu_balloc_fixed_buddy(struct nvgpu_allocator *__a,
 fail_unlock:
 	alloc_unlock(__a);
 fail:
-	kfree(falloc);
+	nvgpu_kfree(nvgpu_alloc_to_gpu(__a), falloc);
 	nvgpu_alloc_trace_func_done();
 	return 0;
 }
@@ -1261,11 +1262,11 @@ int __nvgpu_buddy_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a,
 	if (flags & GPU_ALLOC_GVA_SPACE && !vm)
 		return -EINVAL;
 
-	a = kzalloc(sizeof(struct nvgpu_buddy_allocator), GFP_KERNEL);
+	a = nvgpu_kzalloc(g, sizeof(struct nvgpu_buddy_allocator));
 	if (!a)
 		return -ENOMEM;
 
-	err = __nvgpu_alloc_common_init(__a, name, a, false, &buddy_ops);
+	err = __nvgpu_alloc_common_init(__a, g, name, a, false, &buddy_ops);
 	if (err)
 		goto fail;
 
@@ -1339,7 +1340,7 @@ int __nvgpu_buddy_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a,
 fail:
 	if (a->buddy_cache)
 		nvgpu_kmem_cache_destroy(a->buddy_cache);
-	kfree(a);
+	nvgpu_kfree(g, a);
 	return err;
 }
 

drivers/gpu/nvgpu/common/mm/buddy_allocator_priv.h

@@ -20,8 +20,7 @@
 #include <linux/list.h>
 #include <linux/rbtree.h>
 
-#include <nvgpu/kmem.h>
-
+struct nvgpu_kmem_cache;
 struct nvgpu_allocator;
 struct vm_gk20a;
 

drivers/gpu/nvgpu/common/mm/lockless_allocator.c

@@ -20,6 +20,7 @@
 #include <linux/atomic.h>
 
 #include <nvgpu/allocator.h>
+#include <nvgpu/kmem.h>
 
 #include "lockless_allocator_priv.h"
 
@@ -106,7 +107,7 @@ static void nvgpu_lockless_alloc_destroy(struct nvgpu_allocator *a)
 	nvgpu_fini_alloc_debug(a);
 
 	vfree(pa->next);
-	kfree(pa);
+	nvgpu_kfree(nvgpu_alloc_to_gpu(a), pa);
 }
 
 static void nvgpu_lockless_print_stats(struct nvgpu_allocator *a,
@@ -154,7 +155,7 @@ int nvgpu_lockless_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a,
 		return -EINVAL;
 
 	/*
-	 * Ensure we have space for atleast one node & there's no overflow.
+	 * Ensure we have space for at least one node & there's no overflow.
 	 * In order to control memory footprint, we require count < INT_MAX
 	 */
 	count = length;
@@ -162,11 +163,11 @@ int nvgpu_lockless_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a,
 	if (!base || !count || count > INT_MAX)
 		return -EINVAL;
 
-	a = kzalloc(sizeof(struct nvgpu_lockless_allocator), GFP_KERNEL);
+	a = nvgpu_kzalloc(g, sizeof(struct nvgpu_lockless_allocator));
 	if (!a)
 		return -ENOMEM;
 
-	err = __nvgpu_alloc_common_init(__a, name, a, false, &pool_ops);
+	err = __nvgpu_alloc_common_init(__a, g, name, a, false, &pool_ops);
 	if (err)
 		goto fail;
 
@@ -202,6 +203,6 @@ int nvgpu_lockless_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a,
 	return 0;
 
 fail:
-	kfree(a);
+	nvgpu_kfree(g, a);
 	return err;
 }

drivers/gpu/nvgpu/common/mm/nvgpu_allocator.c

@@ -123,7 +123,7 @@ void nvgpu_alloc_destroy(struct nvgpu_allocator *a)
 /*
  * Handle the common init stuff for a nvgpu_allocator.
  */
-int __nvgpu_alloc_common_init(struct nvgpu_allocator *a,
+int __nvgpu_alloc_common_init(struct nvgpu_allocator *a, struct gk20a *g,
 			      const char *name, void *priv, bool dbg,
 			      const struct nvgpu_allocator_ops *ops)
 {
@@ -143,6 +143,7 @@ int __nvgpu_alloc_common_init(struct nvgpu_allocator *a,
 	if (err)
 		return err;
 
+	a->g = g;
 	a->ops = ops;
 	a->priv = priv;
 	a->debug = dbg;

drivers/gpu/nvgpu/common/mm/page_allocator.c

@@ -21,6 +21,7 @@
 
 #include <nvgpu/allocator.h>
 #include <nvgpu/page_allocator.h>
+#include <nvgpu/kmem.h>
 
 #include "buddy_allocator_priv.h"
 
@@ -760,7 +761,7 @@ static void nvgpu_page_allocator_destroy(struct nvgpu_allocator *__a)
 	struct nvgpu_page_allocator *a = page_allocator(__a);
 
 	alloc_lock(__a);
-	kfree(a);
+	nvgpu_kfree(nvgpu_alloc_to_gpu(__a), a);
 	__a->priv = NULL;
 	alloc_unlock(__a);
 }
@@ -848,9 +849,9 @@ static int nvgpu_page_alloc_init_slabs(struct nvgpu_page_allocator *a)
 	size_t nr_slabs = ilog2(a->page_size >> 12);
 	unsigned int i;
 
-	a->slabs = kcalloc(nr_slabs,
-			   sizeof(struct page_alloc_slab),
-			   GFP_KERNEL);
+	a->slabs = nvgpu_kcalloc(nvgpu_alloc_to_gpu(a->owner),
+				 nr_slabs,
+				 sizeof(struct page_alloc_slab));
 	if (!a->slabs)
 		return -ENOMEM;
 	a->nr_slabs = nr_slabs;
@@ -881,11 +882,11 @@ int nvgpu_page_allocator_init(struct gk20a *g, struct nvgpu_allocator *__a,
 	if (blk_size < SZ_4K)
 		return -EINVAL;
 
-	a = kzalloc(sizeof(struct nvgpu_page_allocator), GFP_KERNEL);
+	a = nvgpu_kzalloc(g, sizeof(struct nvgpu_page_allocator));
 	if (!a)
 		return -ENOMEM;
 
-	err = __nvgpu_alloc_common_init(__a, name, a, false, &page_ops);
+	err = __nvgpu_alloc_common_init(__a, g, name, a, false, &page_ops);
 	if (err)
 		goto fail;
 
@@ -938,6 +939,6 @@ fail:
 		nvgpu_kmem_cache_destroy(a->chunk_cache);
 	if (a->slab_page_cache)
 		nvgpu_kmem_cache_destroy(a->slab_page_cache);
-	kfree(a);
+	nvgpu_kfree(g, a);
 	return err;
 }

drivers/gpu/nvgpu/common/nvgpu_common.c

@@ -17,11 +17,12 @@
 #include <linux/dma-mapping.h>
 #include <linux/firmware.h>
 
+#include <nvgpu/kmem.h>
+#include <nvgpu/nvgpu_common.h>
+
 #include "gk20a/gk20a_scale.h"
 #include "gk20a/gk20a.h"
 
-#include <nvgpu/nvgpu_common.h>
-
 #define EMC3D_DEFAULT_RATIO 750
 
 static void nvgpu_init_vars(struct gk20a *g)
@@ -164,7 +165,7 @@ int nvgpu_probe(struct gk20a *g,
 	gk20a_create_sysfs(g->dev);
 	gk20a_debug_init(g->dev, debugfs_symlink);
 
-	g->dbg_regops_tmp_buf = kzalloc(SZ_4K, GFP_KERNEL);
+	g->dbg_regops_tmp_buf = nvgpu_kzalloc(g, SZ_4K);
 	if (!g->dbg_regops_tmp_buf) {
 		dev_err(g->dev, "couldn't allocate regops tmp buf");
 		return -ENOMEM;
@@ -190,7 +191,8 @@ static const struct firmware *do_request_firmware(struct device *dev,
 		path_len = strlen(prefix) + strlen(fw_name);
 		path_len += 2; /* for the path separator and zero terminator*/
 
-		fw_path = kzalloc(sizeof(*fw_path) * path_len, GFP_KERNEL);
+		fw_path = nvgpu_kzalloc(get_gk20a(dev),
+					sizeof(*fw_path) * path_len);
 		if (!fw_path)
 			return NULL;
 
@@ -207,7 +209,7 @@ static const struct firmware *do_request_firmware(struct device *dev,
 		err = request_firmware(&fw, fw_name, dev);
 #endif
 
-	kfree(fw_path);
+	nvgpu_kfree(get_gk20a(dev), fw_path);
 	if (err)
 		return NULL;
 	return fw;

drivers/gpu/nvgpu/common/semaphore.c

@@ -20,6 +20,10 @@
 #include <linux/slab.h>
 
 #include <nvgpu/semaphore.h>
+#include <nvgpu/kmem.h>
+
+#include "gk20a/gk20a.h"
+#include "gk20a/mm_gk20a.h"
 
 #define __lock_sema_sea(s)						\
 	do {								\
@@ -83,7 +87,7 @@ struct nvgpu_semaphore_sea *nvgpu_semaphore_sea_create(struct gk20a *g)
 	if (g->sema_sea)
 		return g->sema_sea;
 
-	g->sema_sea = kzalloc(sizeof(*g->sema_sea), GFP_KERNEL);
+	g->sema_sea = nvgpu_kzalloc(g, sizeof(*g->sema_sea));
 	if (!g->sema_sea)
 		return NULL;
 
@@ -103,7 +107,7 @@ struct nvgpu_semaphore_sea *nvgpu_semaphore_sea_create(struct gk20a *g)
 cleanup_destroy:
 	nvgpu_mutex_destroy(&g->sema_sea->sea_lock);
 cleanup_free:
-	kfree(g->sema_sea);
+	nvgpu_kfree(g, g->sema_sea);
 	g->sema_sea = NULL;
 	gpu_sema_dbg("Failed to creat semaphore sea!");
 	return NULL;
@@ -131,7 +135,7 @@ struct nvgpu_semaphore_pool *nvgpu_semaphore_pool_alloc(
 	unsigned long page_idx;
 	int ret, err = 0;
 
-	p = kzalloc(sizeof(*p), GFP_KERNEL);
+	p = nvgpu_kzalloc(sea->gk20a, sizeof(*p));
 	if (!p)
 		return ERR_PTR(-ENOMEM);
 
@@ -168,7 +172,7 @@ fail_alloc:
 	nvgpu_mutex_destroy(&p->pool_lock);
 fail:
 	__unlock_sema_sea(sea);
-	kfree(p);
+	nvgpu_kfree(sea->gk20a, p);
 	gpu_sema_dbg("Failed to allocate semaphore pool!");
 	return ERR_PTR(err);
 }
@@ -191,7 +195,8 @@ int nvgpu_semaphore_pool_map(struct nvgpu_semaphore_pool *p,
 	gpu_sema_dbg("  %d: CPU VA = 0x%p!", p->page_idx, p->cpu_va);
 
 	/* First do the RW mapping. */
-	p->rw_sg_table = kzalloc(sizeof(*p->rw_sg_table), GFP_KERNEL);
+	p->rw_sg_table = nvgpu_kzalloc(p->sema_sea->gk20a,
+				       sizeof(*p->rw_sg_table));
 	if (!p->rw_sg_table)
 		return -ENOMEM;
 
@@ -261,7 +266,7 @@ fail_unmap_sgt:
 fail_free_sgt:
 	sg_free_table(p->rw_sg_table);
 fail:
-	kfree(p->rw_sg_table);
+	nvgpu_kfree(p->sema_sea->gk20a, p->rw_sg_table);
 	p->rw_sg_table = NULL;
 	gpu_sema_dbg("  %d: Failed to map semaphore pool!", p->page_idx);
 	return err;
@@ -292,7 +297,7 @@ void nvgpu_semaphore_pool_unmap(struct nvgpu_semaphore_pool *p,
 		     DMA_BIDIRECTIONAL);
 
 	sg_free_table(p->rw_sg_table);
-	kfree(p->rw_sg_table);
+	nvgpu_kfree(p->sema_sea->gk20a, p->rw_sg_table);
 	p->rw_sg_table = NULL;
 
 	list_for_each_entry(hw_sema, &p->hw_semas, hw_sema_list)
@@ -325,12 +330,12 @@ static void nvgpu_semaphore_pool_free(struct kref *ref)
 	__unlock_sema_sea(s);
 
 	list_for_each_entry_safe(hw_sema, tmp, &p->hw_semas, hw_sema_list)
-		kfree(hw_sema);
+		nvgpu_kfree(p->sema_sea->gk20a, hw_sema);
 
 	nvgpu_mutex_destroy(&p->pool_lock);
 
 	gpu_sema_dbg("Freed semaphore pool! (idx=%d)", p->page_idx);
-	kfree(p);
+	nvgpu_kfree(p->sema_sea->gk20a, p);
 }
 
 void nvgpu_semaphore_pool_get(struct nvgpu_semaphore_pool *p)
@@ -374,7 +379,7 @@ static int __nvgpu_init_hw_sema(struct channel_gk20a *ch)
 		goto fail;
 	}
 
-	hw_sema = kzalloc(sizeof(struct nvgpu_semaphore_int), GFP_KERNEL);
+	hw_sema = nvgpu_kzalloc(ch->g, sizeof(struct nvgpu_semaphore_int));
 	if (!hw_sema) {
 		ret = -ENOMEM;
 		goto fail_free_idx;
@@ -417,7 +422,7 @@ void nvgpu_semaphore_free_hw_sema(struct channel_gk20a *ch)
 
 	/* Make sure that when the ch is re-opened it will get a new HW sema. */
 	list_del(&ch->hw_sema->hw_sema_list);
-	kfree(ch->hw_sema);
+	nvgpu_kfree(ch->g, ch->hw_sema);
 	ch->hw_sema = NULL;
 
 	nvgpu_mutex_release(&p->pool_lock);
@@ -440,7 +445,7 @@ struct nvgpu_semaphore *nvgpu_semaphore_alloc(struct channel_gk20a *ch)
 			return NULL;
 	}
 
-	s = kzalloc(sizeof(*s), GFP_KERNEL);
+	s = nvgpu_kzalloc(ch->g, sizeof(*s));
 	if (!s)
 		return NULL;
 
@@ -466,7 +471,7 @@ static void nvgpu_semaphore_free(struct kref *ref)
 
 	nvgpu_semaphore_pool_put(s->hw_sema->p);
 
-	kfree(s);
+	nvgpu_kfree(s->hw_sema->ch->g, s);
 }
 
 void nvgpu_semaphore_put(struct nvgpu_semaphore *s)

drivers/gpu/nvgpu/include/nvgpu/allocator.h

@@ -20,6 +20,7 @@
 #include <linux/debugfs.h>
 #include <linux/seq_file.h>
 #include <linux/platform_device.h>
+
 #include <nvgpu/lock.h>
 
 /* #define ALLOCATOR_DEBUG */
@@ -78,6 +79,8 @@ struct nvgpu_allocator_ops {
 };
 
 struct nvgpu_allocator {
+	struct gk20a *g;
+
 	char name[32];
 	struct nvgpu_mutex lock;
 
@@ -238,13 +241,18 @@ void nvgpu_alloc_destroy(struct nvgpu_allocator *allocator);
 void nvgpu_alloc_print_stats(struct nvgpu_allocator *a,
 			     struct seq_file *s, int lock);
 
+static inline struct gk20a *nvgpu_alloc_to_gpu(struct nvgpu_allocator *a)
+{
+	return a->g;
+}
+
 /*
  * Common functionality for the internals of the allocators.
  */
 void nvgpu_init_alloc_debug(struct gk20a *g, struct nvgpu_allocator *a);
 void nvgpu_fini_alloc_debug(struct nvgpu_allocator *a);
 
-int  __nvgpu_alloc_common_init(struct nvgpu_allocator *a,
+int  __nvgpu_alloc_common_init(struct nvgpu_allocator *a, struct gk20a *g,
 			       const char *name, void *priv, bool dbg,
 			       const struct nvgpu_allocator_ops *ops);