Open source GPL/LGPL release

drivers/gpu/nvgpu/common/mm/allocators/bitmap_allocator.c

@@ -0,0 +1,514 @@
+/*
+ * Copyright (c) 2016-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/bitops.h>
+#include <nvgpu/allocator.h>
+#include <nvgpu/kmem.h>
+#include <nvgpu/bug.h>
+#include <nvgpu/barrier.h>
+
+#include "bitmap_allocator_priv.h"
+
+static u64 nvgpu_bitmap_alloc_length(struct nvgpu_allocator *a)
+{
+	struct nvgpu_bitmap_allocator *ba = a->priv;
+
+	return ba->length;
+}
+
+static u64 nvgpu_bitmap_alloc_base(struct nvgpu_allocator *a)
+{
+	struct nvgpu_bitmap_allocator *ba = a->priv;
+
+	return ba->base;
+}
+
+static bool nvgpu_bitmap_alloc_inited(struct nvgpu_allocator *a)
+{
+	struct nvgpu_bitmap_allocator *ba = a->priv;
+	bool inited = ba->inited;
+
+	nvgpu_smp_rmb();
+	return inited;
+}
+
+static u64 nvgpu_bitmap_alloc_end(struct nvgpu_allocator *a)
+{
+	struct nvgpu_bitmap_allocator *ba = a->priv;
+
+	return nvgpu_safe_add_u64(ba->base, ba->length);
+}
+
+/*
+ * @page_size is ignored.
+ */
+static u64 nvgpu_bitmap_balloc_fixed(struct nvgpu_allocator *na,
+				    u64 base, u64 len, u32 page_size)
+{
+	struct nvgpu_bitmap_allocator *a = bitmap_allocator(na);
+	u64 blks, offs, ret;
+
+	/* Compute the bit offset and make sure it's aligned to a block.  */
+	offs = base >> a->blk_shift;
+	if (nvgpu_safe_mult_u64(offs, a->blk_size) != base) {
+		return 0;
+	}
+
+	offs = nvgpu_safe_sub_u64(offs, a->bit_offs);
+
+	blks = len >> a->blk_shift;
+	if (nvgpu_safe_mult_u64(blks, a->blk_size) != len) {
+		blks++;
+	}
+	nvgpu_assert(blks <= U32_MAX);
+
+	alloc_lock(na);
+
+	/* Check if the space requested is already occupied. */
+	ret = bitmap_find_next_zero_area(a->bitmap, a->num_bits, offs,
+					(u32)blks, 0UL);
+	if (ret != offs) {
+		goto fail;
+	}
+
+	nvgpu_assert(blks <= U32_MAX);
+	nvgpu_bitmap_set(a->bitmap, (u32)offs, U32(blks));
+
+	a->bytes_alloced = nvgpu_safe_add_u64(a->bytes_alloced,
+				nvgpu_safe_mult_u64(blks, a->blk_size));
+NVGPU_COV_WHITELIST(false_positive, NVGPU_MISRA(Rule, 14_3), "Bug 2615925")
+	nvgpu_assert(a->nr_fixed_allocs < U64_MAX);
+	a->nr_fixed_allocs++;
+	alloc_unlock(na);
+
+	alloc_dbg(na, "Alloc-fixed 0x%-10llx 0x%-5llx [bits=0x%llx (%llu)]",
+		  base, len, blks, blks);
+	return base;
+
+fail:
+	alloc_unlock(na);
+	alloc_dbg(na, "Alloc-fixed failed! (0x%llx)", base);
+	return 0;
+}
+
+/*
+ * Two possibilities for this function: either we are freeing a fixed allocation
+ * or we are freeing a regular alloc but with GPU_ALLOC_NO_ALLOC_PAGE defined.
+ *
+ * Note: this function won't do much error checking. Thus you could really
+ * confuse the allocator if you misuse this function.
+ */
+static void nvgpu_bitmap_free_fixed(struct nvgpu_allocator *na,
+				    u64 base, u64 len)
+{
+	struct nvgpu_bitmap_allocator *a = bitmap_allocator(na);
+	u64 blks, offs;
+
+	offs = base >> a->blk_shift;
+	if (nvgpu_safe_mult_u64(offs, a->blk_size) != base) {
+		nvgpu_do_assert();
+		return;
+	}
+
+	offs = nvgpu_safe_sub_u64(offs, a->bit_offs);
+
+	blks = len >> a->blk_shift;
+	if (nvgpu_safe_mult_u64(blks, a->blk_size) != len) {
+		blks++;
+	}
+
+	alloc_lock(na);
+	nvgpu_assert(offs <= U32_MAX);
+	nvgpu_assert(blks <= (u32)INT_MAX);
+	nvgpu_bitmap_clear(a->bitmap, (u32)offs, (u32)blks);
+	a->bytes_freed = nvgpu_safe_add_u64(a->bytes_freed,
+				nvgpu_safe_mult_u64(blks, a->blk_size));
+	alloc_unlock(na);
+
+	alloc_dbg(na, "Free-fixed 0x%-10llx 0x%-5llx [bits=0x%llx (%llu)]",
+		  base, len, blks, blks);
+}
+
+/*
+ * Add the passed alloc to the tree of stored allocations.
+ */
+static void insert_alloc_metadata(struct nvgpu_bitmap_allocator *a,
+				  struct nvgpu_bitmap_alloc *alloc)
+{
+	alloc->alloc_entry.key_start = alloc->base;
+	alloc->alloc_entry.key_end = nvgpu_safe_add_u64(alloc->base,
+							alloc->length);
+
+	nvgpu_rbtree_insert(&alloc->alloc_entry, &a->allocs);
+}
+
+/*
+ * Find and remove meta-data from the outstanding allocations.
+ */
+static struct nvgpu_bitmap_alloc *find_alloc_metadata(
+	struct nvgpu_bitmap_allocator *a, u64 addr)
+{
+	struct nvgpu_bitmap_alloc *alloc;
+	struct nvgpu_rbtree_node *node = NULL;
+
+	nvgpu_rbtree_search(addr, &node, a->allocs);
+	if (node == NULL) {
+		return NULL;
+	}
+
+	alloc = nvgpu_bitmap_alloc_from_rbtree_node(node);
+
+	nvgpu_rbtree_unlink(node, &a->allocs);
+
+	return alloc;
+}
+
+/*
+ * Tree of alloc meta data stores the address of the alloc not the bit offset.
+ */
+static int nvgpu_bitmap_store_alloc(struct nvgpu_bitmap_allocator *a,
+				      u64 addr, u64 len)
+{
+	struct nvgpu_bitmap_alloc *alloc =
+		nvgpu_kmem_cache_alloc(a->meta_data_cache);
+
+	if (alloc == NULL) {
+		return -ENOMEM;
+	}
+
+	alloc->base = addr;
+	alloc->length = len;
+
+	insert_alloc_metadata(a, alloc);
+
+	return 0;
+}
+
+/*
+ * @len is in bytes. This routine will figure out the right number of bits to
+ * actually allocate. The return is the address in bytes as well.
+ *
+ * This is a find-first-fit allocator.
+ * Check the input parameter validity.
+ * Acquire the alloc_lock.
+ * Searche a bitmap for the first space that is large enough to satisfy the
+ *  requested size of bits by walking the next available free blocks by
+ *  bitmap_find_next_zero_area().
+ * Release the alloc_lock.
+ */
+static u64 nvgpu_bitmap_balloc(struct nvgpu_allocator *na, u64 len)
+{
+	u64 tmp_u64, addr;
+	u32 blks;
+	unsigned long offs, adjusted_offs, limit;
+	struct nvgpu_bitmap_allocator *a = bitmap_allocator(na);
+
+	if (len == 0ULL) {
+		alloc_dbg(na, "len = 0, Alloc failed!");
+		return 0;
+	}
+
+	tmp_u64 = len >> a->blk_shift;
+	nvgpu_assert(tmp_u64 <= U32_MAX);
+	blks = (u32)tmp_u64;
+
+	if (nvgpu_safe_mult_u64(blks, a->blk_size) != len) {
+		blks++;
+	}
+
+	alloc_lock(na);
+
+	/*
+	 * First look from next_blk and onwards...
+	 */
+	offs = bitmap_find_next_zero_area(a->bitmap, a->num_bits,
+					  a->next_blk, blks, 0);
+	if (offs >= a->num_bits) {
+		/*
+		 * If that didn't work try the remaining area. Since there can
+		 * be available space that spans across a->next_blk we need to
+		 * search up to the first set bit after that.
+		 */
+		limit = find_next_bit(a->bitmap, a->num_bits, a->next_blk);
+		offs = bitmap_find_next_zero_area(a->bitmap, limit,
+						  0, blks, 0);
+		if (offs >= a->next_blk) {
+			goto fail;
+		}
+	}
+
+	nvgpu_assert(offs <= U32_MAX);
+	nvgpu_bitmap_set(a->bitmap, (u32)offs, blks);
+	a->next_blk = offs + blks;
+
+	adjusted_offs = nvgpu_safe_add_u64(offs, a->bit_offs);
+	addr = nvgpu_safe_mult_u64(((u64)adjusted_offs), a->blk_size);
+
+	/*
+	 * Only do meta-data storage if we are allowed to allocate storage for
+	 * that meta-data. The issue with using malloc and friends is that
+	 * in latency and success critical paths an alloc_page() call can either
+	 * sleep for potentially a long time or fail. Since we might not want
+	 * either of these possibilities assume that the caller will keep what
+	 * data it needs around to successfully free this allocation.
+	 */
+	if ((a->flags & GPU_ALLOC_NO_ALLOC_PAGE) == 0ULL) {
+		if (nvgpu_bitmap_store_alloc(a, addr,
+						blks * a->blk_size) != 0) {
+			goto fail_reset_bitmap;
+		}
+	}
+
+	alloc_dbg(na, "Alloc 0x%-10llx 0x%-5llx [bits=0x%x (%u)]",
+		  addr, len, blks, blks);
+
+NVGPU_COV_WHITELIST(false_positive, NVGPU_MISRA(Rule, 14_3), "Bug 2615925")
+	nvgpu_assert(a->nr_allocs < U64_MAX);
+	a->nr_allocs++;
+	a->bytes_alloced = nvgpu_safe_add_u64(a->bytes_alloced,
+				nvgpu_safe_mult_u64(blks, a->blk_size));
+	alloc_unlock(na);
+
+	return addr;
+
+fail_reset_bitmap:
+	nvgpu_assert(blks <= (u32)INT_MAX);
+	nvgpu_assert(offs <= U32_MAX);
+	nvgpu_bitmap_clear(a->bitmap, (u32)offs, blks);
+fail:
+	a->next_blk = 0;
+	alloc_unlock(na);
+	alloc_dbg(na, "Alloc failed!");
+	return 0;
+}
+
+static void nvgpu_bitmap_free(struct nvgpu_allocator *na, u64 addr)
+{
+	struct nvgpu_bitmap_allocator *a = bitmap_allocator(na);
+	struct nvgpu_bitmap_alloc *alloc = NULL;
+	u64 offs, adjusted_offs, blks;
+
+	alloc_lock(na);
+
+	if ((a->flags & GPU_ALLOC_NO_ALLOC_PAGE) != 0ULL) {
+		(void) WARN(true,
+			"Using wrong free for NO_ALLOC_PAGE bitmap allocator");
+		goto done;
+	}
+
+	alloc = find_alloc_metadata(a, addr);
+	if (alloc == NULL) {
+		goto done;
+	}
+
+	/*
+	 * Address comes from adjusted offset (i.e the bit offset with
+	 * a->bit_offs added. So start with that and then work out the real
+	 * offs into the bitmap.
+	 */
+	adjusted_offs = addr >> a->blk_shift;
+	offs = adjusted_offs - a->bit_offs;
+	blks = alloc->length >> a->blk_shift;
+
+	nvgpu_assert(blks <= (u32)INT_MAX);
+	nvgpu_assert(offs <= U32_MAX);
+	nvgpu_bitmap_clear(a->bitmap, (u32)offs, (u32)blks);
+	alloc_dbg(na, "Free  0x%-10llx", addr);
+
+	a->bytes_freed = nvgpu_safe_add_u64(a->bytes_freed, alloc->length);
+
+done:
+	if ((a->meta_data_cache != NULL) && (alloc != NULL)) {
+		nvgpu_kmem_cache_free(a->meta_data_cache, alloc);
+	}
+	alloc_unlock(na);
+}
+
+static void nvgpu_bitmap_alloc_destroy(struct nvgpu_allocator *na)
+{
+	struct nvgpu_bitmap_allocator *a = bitmap_allocator(na);
+	struct nvgpu_bitmap_alloc *alloc;
+	struct nvgpu_rbtree_node *node = NULL;
+
+	/*
+	 * Kill any outstanding allocations.
+	 */
+	nvgpu_rbtree_enum_start(0, &node, a->allocs);
+	while (node != NULL) {
+		alloc = nvgpu_bitmap_alloc_from_rbtree_node(node);
+
+		nvgpu_rbtree_unlink(node, &a->allocs);
+		nvgpu_kmem_cache_free(a->meta_data_cache, alloc);
+
+		nvgpu_rbtree_enum_start(0, &node, a->allocs);
+	}
+
+	nvgpu_kmem_cache_destroy(a->meta_data_cache);
+	nvgpu_kfree(nvgpu_alloc_to_gpu(na), a->bitmap);
+	nvgpu_kfree(nvgpu_alloc_to_gpu(na), a);
+}
+
+#ifdef __KERNEL__
+static void nvgpu_bitmap_print_stats(struct nvgpu_allocator *na,
+				     struct seq_file *s, int lock)
+{
+	struct nvgpu_bitmap_allocator *a = bitmap_allocator(na);
+
+	alloc_pstat(s, na, "Bitmap allocator params:");
+	alloc_pstat(s, na, "  start = 0x%llx", a->base);
+	alloc_pstat(s, na, "  end   = 0x%llx", a->base + a->length);
+	alloc_pstat(s, na, "  blks  = 0x%llx", a->num_bits);
+
+	/* Actual stats. */
+	alloc_pstat(s, na, "Stats:");
+	alloc_pstat(s, na, "  Number allocs = 0x%llx", a->nr_allocs);
+	alloc_pstat(s, na, "  Number fixed  = 0x%llx", a->nr_fixed_allocs);
+	alloc_pstat(s, na, "  Bytes alloced = 0x%llx", a->bytes_alloced);
+	alloc_pstat(s, na, "  Bytes freed   = 0x%llx", a->bytes_freed);
+	alloc_pstat(s, na, "  Outstanding   = 0x%llx",
+		      a->bytes_alloced - a->bytes_freed);
+}
+#endif
+
+NVGPU_COV_WHITELIST_BLOCK_BEGIN(false_positive, 1, NVGPU_MISRA(Rule, 8_7), "Bug 2823817")
+static const struct nvgpu_allocator_ops bitmap_ops = {
+NVGPU_COV_WHITELIST_BLOCK_END(NVGPU_MISRA(Rule, 8_7))
+	.alloc		= nvgpu_bitmap_balloc,
+	.free_alloc	= nvgpu_bitmap_free,
+
+	.alloc_fixed	= nvgpu_bitmap_balloc_fixed,
+	.free_fixed	= nvgpu_bitmap_free_fixed,
+
+	.base		= nvgpu_bitmap_alloc_base,
+	.length		= nvgpu_bitmap_alloc_length,
+	.end		= nvgpu_bitmap_alloc_end,
+	.inited		= nvgpu_bitmap_alloc_inited,
+
+	.fini		= nvgpu_bitmap_alloc_destroy,
+
+#ifdef __KERNEL__
+	.print_stats	= nvgpu_bitmap_print_stats,
+#endif
+};
+
+static int nvgpu_bitmap_check_argument_limits(u64 base, u64 length,
+								u64 blk_size)
+{
+	bool is_blk_size_pwr_2;
+	bool is_base_aligned;
+	bool is_length_aligned;
+
+	nvgpu_assert(blk_size > 0ULL);
+	is_blk_size_pwr_2 = (blk_size & (blk_size - 1ULL)) == 0ULL;
+	is_base_aligned =  (base & (blk_size - 1ULL)) == 0ULL;
+	is_length_aligned = (length & (blk_size - 1ULL)) == 0ULL;
+
+	if (!is_blk_size_pwr_2) {
+		nvgpu_do_assert();
+		return -EINVAL;
+	}
+
+	if (!is_base_aligned || !is_length_aligned) {
+		return -EINVAL;
+	}
+
+	if (length == 0ULL) {
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+int nvgpu_bitmap_allocator_init(struct gk20a *g, struct nvgpu_allocator *na,
+				const char *name, u64 base, u64 length,
+				u64 blk_size, u64 flags)
+{
+	int err;
+	struct nvgpu_bitmap_allocator *a;
+
+	err = nvgpu_bitmap_check_argument_limits(base, length, blk_size);
+	if (err != 0) {
+		return err;
+	}
+
+	if (base == 0ULL) {
+		base = blk_size;
+		length = nvgpu_safe_sub_u64(length, blk_size);
+	}
+
+	a = nvgpu_kzalloc(g, sizeof(struct nvgpu_bitmap_allocator));
+	if (a == NULL) {
+		return -ENOMEM;
+	}
+
+	err = nvgpu_alloc_common_init(na, g, name, a, false, &bitmap_ops);
+	if (err != 0) {
+		goto fail;
+	}
+
+	if ((flags & GPU_ALLOC_NO_ALLOC_PAGE) == 0ULL) {
+		a->meta_data_cache = nvgpu_kmem_cache_create(g,
+					sizeof(struct nvgpu_bitmap_alloc));
+		if (a->meta_data_cache == NULL) {
+			err = -ENOMEM;
+			goto fail;
+		}
+	}
+
+	a->base = base;
+	a->length = length;
+	a->blk_size = blk_size;
+	a->blk_shift = nvgpu_safe_sub_u64(nvgpu_ffs(a->blk_size), 1UL);
+	a->num_bits = length >> a->blk_shift;
+	a->bit_offs = a->base >> a->blk_shift;
+	a->flags = flags;
+	a->allocs = NULL;
+
+	a->bitmap = nvgpu_kcalloc(g, BITS_TO_LONGS(a->num_bits),
+				  sizeof(*a->bitmap));
+	if (a->bitmap == NULL) {
+		err = -ENOMEM;
+		goto fail;
+	}
+
+	nvgpu_smp_wmb();
+	a->inited = true;
+
+#ifdef CONFIG_DEBUG_FS
+	nvgpu_init_alloc_debug(g, na);
+#endif
+	alloc_dbg(na, "New allocator: type      bitmap");
+	alloc_dbg(na, "               base      0x%llx", a->base);
+	alloc_dbg(na, "               bit_offs  0x%llx", a->bit_offs);
+	alloc_dbg(na, "               size      0x%llx", a->length);
+	alloc_dbg(na, "               blk_size  0x%llx", a->blk_size);
+	alloc_dbg(na, "               flags     0x%llx", a->flags);
+
+	return 0;
+
+fail:
+	if (a->meta_data_cache != NULL) {
+		nvgpu_kmem_cache_destroy(a->meta_data_cache);
+	}
+	nvgpu_kfree(g, a);
+	return err;
+}

drivers/gpu/nvgpu/common/mm/allocators/bitmap_allocator_priv.h

@@ -0,0 +1,183 @@
+/*
+ * Copyright (c) 2016-2017, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef BITMAP_ALLOCATOR_PRIV_H
+#define BITMAP_ALLOCATOR_PRIV_H
+
+/**
+ * @file
+ *
+ * Implementation of a bitmap allocator.
+ */
+
+#include <nvgpu/rbtree.h>
+#include <nvgpu/kmem.h>
+
+struct nvgpu_allocator;
+
+/**
+ * Structure to hold the implementation details of the bitmap allocator.
+ */
+struct nvgpu_bitmap_allocator {
+	/**
+	 * Pointer to the common allocator structure.
+	 */
+	struct nvgpu_allocator *owner;
+
+	/**
+	 * Base address of the space.
+	 */
+	u64 base;
+
+	/**
+	 * Length of the space.
+	 */
+	u64 length;
+
+	/**
+	 * Size that corresponds to 1 bit.
+	 */
+	u64 blk_size;
+
+	/**
+	 * Bit shift to divide by blk_size.
+	 */
+	u64 blk_shift;
+
+	/**
+	 * Number of allocatable bits.
+	 */
+	u64 num_bits;
+
+	/**
+	 * Offset of bitmap.
+	 */
+	u64 bit_offs;
+
+	/**
+	 * Optimization for making repeated allocations faster. Keep track of
+	 * the next bit after the most recent allocation. This is where the next
+	 * search will start from. This should make allocation faster in cases
+	 * where lots of allocations get made one after another. It shouldn't
+	 * have a negative impact on the case where the allocator is fragmented.
+	 */
+	u64 next_blk;
+
+	/**
+	 * The actual bitmap used for allocations.
+	 */
+	unsigned long *bitmap;
+
+	/**
+	 * Tree of outstanding allocations.
+	 */
+	struct nvgpu_rbtree_node *allocs;
+
+	/**
+	 * Metadata cache of allocations (contains address and size of
+	 * allocations).
+	 */
+	struct nvgpu_kmem_cache *meta_data_cache;
+
+	/**
+	 * Configuration flags of the allocator. See \a GPU_ALLOC_* flags.
+	 */
+	u64 flags;
+
+	/**
+	 * Boolean to indicate if the allocator has been fully initialized.
+	 */
+	bool inited;
+
+	/**
+	 * Statistics: track the number of non-fixed allocations.
+	 */
+	u64 nr_allocs;
+
+	/**
+	 * Statistics: track the number of fixed allocations.
+	 */
+	u64 nr_fixed_allocs;
+
+	/**
+	 * Statistics: total number of bytes allocated for both fixed and non-
+	 * fixed allocations.
+	 */
+	u64 bytes_alloced;
+
+	/**
+	 * Statistics: total number of bytes freed for both fixed and non-fixed
+	 * allocations.
+	 */
+	u64 bytes_freed;
+};
+
+/**
+ * Structure to hold the allocation metadata.
+ */
+struct nvgpu_bitmap_alloc {
+	/**
+	 * Base address of the allocation.
+	 */
+	u64 base;
+
+	/**
+	 * Size of the allocation.
+	 */
+	u64 length;
+
+	/**
+	 * RB tree of allocations.
+	 */
+	struct nvgpu_rbtree_node alloc_entry;
+};
+
+/**
+ * @brief Given a tree node, retrieve the metdata of the allocation.
+ *
+ * @param[in] node	Pointer to the tree node.
+ *
+ * @return pointer to the struct nvgpu_bitmap_alloc of the node.
+ */
+static inline struct nvgpu_bitmap_alloc *
+nvgpu_bitmap_alloc_from_rbtree_node(struct nvgpu_rbtree_node *node)
+{
+	return (struct nvgpu_bitmap_alloc *)
+	((uintptr_t)node - offsetof(struct nvgpu_bitmap_alloc, alloc_entry));
+};
+
+/**
+ * @brief Given a generic allocator context, retrieve a pointer to the bitmap
+ * allocator context structure.
+ *
+ * @param[in] a		Pointer to nvgpu allocator.
+ *
+ * @return pointer to the struct nvgpu_bitmap_allocator.
+ */
+static inline struct nvgpu_bitmap_allocator *bitmap_allocator(
+	struct nvgpu_allocator *a)
+{
+	return (struct nvgpu_bitmap_allocator *)(a)->priv;
+}
+
+
+#endif

drivers/gpu/nvgpu/common/mm/allocators/buddy_allocator_priv.h

@@ -0,0 +1,451 @@
+/*
+ * Copyright (c) 2016-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef NVGPU_MM_BUDDY_ALLOCATOR_PRIV_H
+#define NVGPU_MM_BUDDY_ALLOCATOR_PRIV_H
+
+/**
+ * @file
+ *
+ * Implementation of the buddy allocator.
+ */
+
+#include <nvgpu/rbtree.h>
+#include <nvgpu/list.h>
+#include <nvgpu/static_analysis.h>
+
+struct nvgpu_kmem_cache;
+struct nvgpu_allocator;
+struct vm_gk20a;
+
+/**
+ * Structure that defines each buddy as an element in a binary tree.
+ */
+struct nvgpu_buddy {
+	/**
+	 * Parent node.
+	 */
+	struct nvgpu_buddy *parent;
+
+	/**
+	 * This node's buddy.
+	 */
+	struct nvgpu_buddy *buddy;
+
+	/**
+	 * Lower address sub-node.
+	 */
+	struct nvgpu_buddy *left;
+
+	/**
+	 * Higher address sub-node.
+	 */
+	struct nvgpu_buddy *right;
+
+	/**
+	 * List entry for various lists.
+	 */
+	struct nvgpu_list_node buddy_entry;
+
+	/**
+	 * RB tree of allocations.
+	 */
+	struct nvgpu_rbtree_node alloced_entry;
+
+	/**
+	 * Start address of this buddy.
+	 */
+	u64 start;
+
+	/**
+	 * End address of this buddy.
+	 */
+	u64 end;
+
+	/**
+	 * Buddy order.
+	 */
+	u64 order;
+
+	/**
+	 * Possible flags to use in the buddy allocator. Set in the #flags
+	 * member.
+	 * @addtogroup BALLOC_BUDDY_FLAGS
+	 * @{
+	 */
+#define BALLOC_BUDDY_ALLOCED	0x1U
+#define BALLOC_BUDDY_SPLIT	0x2U
+#define BALLOC_BUDDY_IN_LIST	0x4U
+	/**@}*/
+
+	/**
+	 * Buddy flags among the @ref BALLOC_BUDDY_FLAGS
+	 */
+	u32 flags;
+
+
+	/**
+	 * Possible PDE sizes. This allows for grouping like sized allocations
+	 * into the same PDE. Set in the #pte_size member.
+	 * @addtogroup BALLOC_PTE_SIZE
+	 * @{
+	 */
+#define BALLOC_PTE_SIZE_ANY	(~0U)
+#define BALLOC_PTE_SIZE_INVALID	0U
+#define BALLOC_PTE_SIZE_SMALL	1U
+#define BALLOC_PTE_SIZE_BIG	2U
+	/**@}*/
+
+	/**
+	 * Size of the PDE this buddy is using. Possible values in
+	 * @ref BALLOC_PTE_SIZE
+	 */
+	u32 pte_size;
+};
+
+/**
+ * @brief Given a list node, retrieve the buddy.
+ *
+ * @param[in] node	Pointer to the list node.
+ *
+ * @return pointer to the struct nvgpu_buddy of the node.
+ */
+static inline struct nvgpu_buddy *
+nvgpu_buddy_from_buddy_entry(struct nvgpu_list_node *node)
+{
+	return (struct nvgpu_buddy *)
+		((uintptr_t)node - offsetof(struct nvgpu_buddy, buddy_entry));
+};
+
+/**
+ * @brief Given a tree node, retrieve the buddy.
+ *
+ * @param[in] node	Pointer to the tree node.
+ *
+ * @return pointer to the struct nvgpu_buddy of the node.
+ */
+static inline struct nvgpu_buddy *
+nvgpu_buddy_from_rbtree_node(struct nvgpu_rbtree_node *node)
+{
+	return (struct nvgpu_buddy *)
+		((uintptr_t)node - offsetof(struct nvgpu_buddy, alloced_entry));
+};
+
+/**
+ * @brief Macro generator to create is/set/clr operations for each of the
+ * flags in @ref BALLOC_BUDDY_FLAGS.
+ *
+ * The created functions are:
+ *
+ *     bool buddy_is_alloced(struct nvgpu_buddy *b);
+ *     void buddy_set_alloced(struct nvgpu_buddy *b);
+ *     void buddy_clr_alloced(struct nvgpu_buddy *b);
+ *
+ *     bool buddy_is_split(struct nvgpu_buddy *b);
+ *     void buddy_set_split(struct nvgpu_buddy *b);
+ *     void buddy_clr_split(struct nvgpu_buddy *b);
+ *
+ *     bool buddy_is_in_list(struct nvgpu_buddy *b);
+ *     void buddy_set_in_list(struct nvgpu_buddy *b);
+ *     void buddy_clr_in_list(struct nvgpu_buddy *b);
+ *
+ * @param[in] flag	One of is, set or clr
+ * @param[in] flag_up	One of the @ref BALLOC_BUDDY_FLAGS
+ *
+ * @{
+ */
+#define nvgpu_buddy_allocator_flag_ops(flag, flag_up)			\
+	static inline bool buddy_is_ ## flag(struct nvgpu_buddy *b)	\
+	{								\
+		return (b->flags & BALLOC_BUDDY_ ## flag_up) != 0U;	\
+	}								\
+	static inline void buddy_set_ ## flag(struct nvgpu_buddy *b)	\
+	{								\
+		b->flags |= BALLOC_BUDDY_ ## flag_up;			\
+	}								\
+	static inline void buddy_clr_ ## flag(struct nvgpu_buddy *b)	\
+	{								\
+		b->flags &= ~BALLOC_BUDDY_ ## flag_up;			\
+	}
+
+nvgpu_buddy_allocator_flag_ops(alloced, ALLOCED);
+nvgpu_buddy_allocator_flag_ops(split,   SPLIT);
+nvgpu_buddy_allocator_flag_ops(in_list, IN_LIST);
+/**@} */
+
+/**
+ * Structure to keep information for a fixed allocation.
+ */
+struct nvgpu_fixed_alloc {
+	/**
+	 * List of buddies.
+	 */
+	struct nvgpu_list_node buddies;
+	/**
+	 * RB tree of fixed allocations.
+	 */
+	struct nvgpu_rbtree_node alloced_entry;
+	/**
+	 * Start of fixed block.
+	 */
+	u64 start;
+	/**
+	 * End address.
+	 */
+	u64 end;
+};
+
+/**
+ * @brief Given a tree node, retrieve the fixed allocation.
+ *
+ * @param[in] node	Pointer to the tree node.
+ *
+ * @return pointer to the struct nvgpu_fixed_alloc of the node.
+ */
+static inline struct nvgpu_fixed_alloc *
+nvgpu_fixed_alloc_from_rbtree_node(struct nvgpu_rbtree_node *node)
+{
+	return (struct nvgpu_fixed_alloc *)
+	((uintptr_t)node - offsetof(struct nvgpu_fixed_alloc, alloced_entry));
+};
+
+/**
+ * GPU buddy allocator for the various GPU address spaces. Each addressable unit
+ * doesn't have to correspond to a byte. In some cases each unit is a more
+ * complex object such as a comp_tag line or the like.
+ *
+ * The max order is computed based on the size of the minimum order and the size
+ * of the address space.
+ *
+ * #blk_size is the size of an order 0 buddy.
+ */
+struct nvgpu_buddy_allocator {
+	/**
+	 * Pointer to the common allocator structure.
+	 */
+	struct nvgpu_allocator *owner;
+	/**
+	 * Parent VM - can be NULL.
+	 */
+	struct vm_gk20a *vm;
+
+	/**
+	 * Base address of the space.
+	 */
+	u64 base;
+	/**
+	 * Length of the space.
+	 */
+	u64 length;
+	/**
+	 * Size of order 0 allocation.
+	 */
+	u64 blk_size;
+	/**
+	 * Shift to divide by blk_size.
+	 */
+	u64 blk_shift;
+
+	/**
+	 * Internal: real start (aligned to #blk_size).
+	 */
+	u64 start;
+	/**
+	 * Internal: real end, trimmed if needed.
+	 */
+	u64 end;
+	/**
+	 * Internal: count of objects in space.
+	 */
+	u64 count;
+	/**
+	 * Internal: count of blks in the space.
+	 */
+	u64 blks;
+	/**
+	 * Internal: specific maximum order.
+	 */
+	u64 max_order;
+
+	/**
+	 * Outstanding allocations.
+	 */
+	struct nvgpu_rbtree_node *alloced_buddies;
+	/**
+	 * Outstanding fixed allocations.
+	 */
+	struct nvgpu_rbtree_node *fixed_allocs;
+
+	/**
+	 * List of carveouts.
+	 */
+	struct nvgpu_list_node co_list;
+
+	/**
+	 * Cache of allocations (contains address and size of allocations).
+	 */
+	struct nvgpu_kmem_cache *buddy_cache;
+
+	/**
+	 * Impose an upper bound on the maximum order.
+	 */
+#define GPU_BALLOC_ORDER_LIST_LEN	(GPU_BALLOC_MAX_ORDER + 1U)
+
+	/**
+	 * List of buddies.
+	 */
+	struct nvgpu_list_node buddy_list[GPU_BALLOC_ORDER_LIST_LEN];
+	/**
+	 * Length of the buddy list.
+	 */
+	u64 buddy_list_len[GPU_BALLOC_ORDER_LIST_LEN];
+	/**
+	 * Number of split nodes.
+	 */
+	u64 buddy_list_split[GPU_BALLOC_ORDER_LIST_LEN];
+	/**
+	 * Number of allocated nodes.
+	 */
+	u64 buddy_list_alloced[GPU_BALLOC_ORDER_LIST_LEN];
+
+	/**
+	 * This is for when the allocator is managing a GVA space (the
+	 * #GPU_ALLOC_GVA_SPACE bit is set in #flags). This requires
+	 * that we group like sized allocations into PDE blocks.
+	 */
+	u64 pte_blk_order;
+
+	/**
+	 * Boolean to indicate if the allocator has been fully initialized.
+	 */
+	bool initialized;
+	/**
+	 * Boolean set to true after the first allocation is made.
+	 */
+	bool alloc_made;
+
+	/**
+	 * Flags in used by the allocator as defined by @ref GPU_ALLOC_FLAGS
+	 */
+	u64 flags;
+
+	/**
+	 * Statistics: total number of bytes allocated.
+	 */
+	u64 bytes_alloced;
+	/**
+	 * Statistics: total number of bytes allocated taking into account the
+	 * buddy order.
+	 */
+	u64 bytes_alloced_real;
+	/**
+	 * Statistics: total number of bytes freed.
+	 */
+	u64 bytes_freed;
+};
+
+/**
+ * @brief Given a generic allocator context, retrieve a pointer to the buddy
+ * allocator context structure.
+ *
+ * @param[in] a		Pointer to nvgpu allocator.
+ *
+ * @return pointer to the struct nvgpu_bitmap_allocator.
+ */
+static inline struct nvgpu_buddy_allocator *buddy_allocator(
+	struct nvgpu_allocator *a)
+{
+	return (struct nvgpu_buddy_allocator *)(a)->priv;
+}
+
+/**
+ * @brief Given a buddy allocator, retrieve the list of buddies of the chosen
+ * order.
+ *
+ * @param[in] a		Pointer to the buddy allocator.
+ * @param[in] order	Buddy order.
+ *
+ * @return list of buddies whose order is \a order.
+ */
+static inline struct nvgpu_list_node *balloc_get_order_list(
+	struct nvgpu_buddy_allocator *a, u64 order)
+{
+	return &a->buddy_list[order];
+}
+
+/**
+ * @brief Convert a buddy order to a length in bytes, based on the block size.
+ *
+ * @param[in] a		Pointer to the buddy allocator.
+ * @param[in] order	Buddy order.
+ *
+ * @return length in bytes.
+ */
+static inline u64 balloc_order_to_len(struct nvgpu_buddy_allocator *a,
+				      u64 order)
+{
+	return nvgpu_safe_mult_u64(BIT64(order), a->blk_size);
+}
+
+/**
+ * @brief Given a base address, shift it by the base address of the buddy.
+ *
+ * @param[in] a		Pointer to the buddy allocator.
+ * @param[in] order	Base address.
+ *
+ * @return shifted address.
+ */
+static inline u64 balloc_base_shift(struct nvgpu_buddy_allocator *a,
+				    u64 base)
+{
+	return nvgpu_safe_sub_u64(base, a->start);
+}
+
+/**
+ * @brief Given a shifted address, unshift it by the base address of the buddy.
+ *
+ * @param[in] a		Pointer to the buddy allocator.
+ * @param[in] order	Shifted address.
+ *
+ * @return unshifted address.
+ */
+static inline u64 balloc_base_unshift(struct nvgpu_buddy_allocator *a,
+				      u64 base)
+{
+	return nvgpu_safe_add_u64(base, a->start);
+}
+
+/**
+ * @brief Given a buddy allocator context, retrieve a pointer to the generic
+ * allocator context structure.
+ *
+ * @param[in] a		Pointer to nvgpu buddy allocator.
+ *
+ * @return pointer to the struct nvgpu_allocator.
+ */
+static inline struct nvgpu_allocator *balloc_owner(
+	struct nvgpu_buddy_allocator *a)
+{
+	return a->owner;
+}
+
+#endif /* NVGPU_MM_BUDDY_ALLOCATOR_PRIV_H */

drivers/gpu/nvgpu/common/mm/allocators/nvgpu_allocator.c

@@ -0,0 +1,216 @@
+/*
+ * gk20a allocator
+ *
+ * Copyright (c) 2011-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/allocator.h>
+#include <nvgpu/gk20a.h>
+
+
+u64 nvgpu_alloc_length(struct nvgpu_allocator *a)
+{
+	if (a->ops->length != NULL) {
+		return a->ops->length(a);
+	}
+
+	return 0;
+}
+
+u64 nvgpu_alloc_base(struct nvgpu_allocator *a)
+{
+	if (a->ops->base != NULL) {
+		return a->ops->base(a);
+	}
+
+	return 0;
+}
+
+bool nvgpu_alloc_initialized(struct nvgpu_allocator *a)
+{
+	if ((a->ops == NULL) || (a->ops->inited == NULL)) {
+		return false;
+	}
+
+	return a->ops->inited(a);
+}
+
+u64 nvgpu_alloc_end(struct nvgpu_allocator *a)
+{
+	if (a->ops->end != NULL) {
+		return a->ops->end(a);
+	}
+
+	return 0;
+}
+
+u64 nvgpu_alloc_space(struct nvgpu_allocator *a)
+{
+	if (a->ops->space != NULL) {
+		return a->ops->space(a);
+	}
+
+	return 0;
+}
+
+u64 nvgpu_alloc(struct nvgpu_allocator *a, u64 len)
+{
+	return a->ops->alloc(a, len);
+}
+
+u64 nvgpu_alloc_pte(struct nvgpu_allocator *a, u64 len, u32 page_size)
+{
+	return a->ops->alloc_pte(a, len, page_size);
+}
+
+void nvgpu_free(struct nvgpu_allocator *a, u64 addr)
+{
+	a->ops->free_alloc(a, addr);
+}
+
+u64 nvgpu_alloc_fixed(struct nvgpu_allocator *a, u64 base, u64 len,
+		      u32 page_size)
+{
+	if ((U64_MAX - base) < len) {
+		return 0ULL;
+	}
+
+	if (a->ops->alloc_fixed != NULL) {
+		return a->ops->alloc_fixed(a, base, len, page_size);
+	}
+
+	return 0;
+}
+
+void nvgpu_free_fixed(struct nvgpu_allocator *a, u64 base, u64 len)
+{
+	/*
+	 * If this operation is not defined for the allocator then just do
+	 * nothing. The alternative would be to fall back on the regular
+	 * free but that may be harmful in unexpected ways.
+	 */
+	if (a->ops->free_fixed != NULL) {
+		a->ops->free_fixed(a, base, len);
+	}
+}
+
+int nvgpu_alloc_reserve_carveout(struct nvgpu_allocator *a,
+				 struct nvgpu_alloc_carveout *co)
+{
+	if (a->ops->reserve_carveout != NULL) {
+		return a->ops->reserve_carveout(a, co);
+	}
+
+	return -ENODEV;
+}
+
+void nvgpu_alloc_release_carveout(struct nvgpu_allocator *a,
+				  struct nvgpu_alloc_carveout *co)
+{
+	if (a->ops->release_carveout != NULL) {
+		a->ops->release_carveout(a, co);
+	}
+}
+
+void nvgpu_alloc_destroy(struct nvgpu_allocator *a)
+{
+	a->ops->fini(a);
+	nvgpu_mutex_destroy(&a->lock);
+	(void) memset(a, 0, sizeof(*a));
+}
+
+#ifdef __KERNEL__
+void nvgpu_alloc_print_stats(struct nvgpu_allocator *na,
+			     struct seq_file *s, int lock)
+{
+	na->ops->print_stats(na, s, lock);
+}
+#endif
+
+/*
+ * Handle the common init stuff for a nvgpu_allocator.
+ */
+int nvgpu_alloc_common_init(struct nvgpu_allocator *a, struct gk20a *g,
+			    const char *name, void *priv, bool dbg,
+			    const struct nvgpu_allocator_ops *ops)
+{
+	if (ops == NULL) {
+		return -EINVAL;
+	}
+
+	/*
+	 * This is the bare minimum operations required for a sensible
+	 * allocator.
+	 */
+	if ((ops->alloc == NULL) || (ops->free_alloc == NULL) ||
+		(ops->fini == NULL)) {
+		return -EINVAL;
+	}
+
+	nvgpu_mutex_init(&a->lock);
+
+	a->g = g;
+	a->ops = ops;
+	a->priv = priv;
+	a->debug = dbg;
+
+	(void) strncpy(a->name, name, sizeof(a->name));
+	a->name[sizeof(a->name) - 1U] = '\0';
+
+	return 0;
+}
+
+/*
+ * Initialize requested type of allocator
+ */
+
+int nvgpu_allocator_init(struct gk20a *g, struct nvgpu_allocator *na,
+			      struct vm_gk20a *vm, const char *name,
+			      u64 base, u64 length, u64 blk_size, u64 max_order,
+			      u64 flags, enum nvgpu_allocator_type alloc_type)
+{
+	int err = -EINVAL;
+
+	switch (alloc_type) {
+	case BUDDY_ALLOCATOR:
+		err = nvgpu_buddy_allocator_init(g, na, vm, name, base, length,
+						blk_size, max_order, flags);
+		break;
+#ifdef CONFIG_NVGPU_DGPU
+	case PAGE_ALLOCATOR:
+		err = nvgpu_page_allocator_init(g, na, name, base, length,
+							blk_size, flags);
+		break;
+#endif
+	case BITMAP_ALLOCATOR:
+		err = nvgpu_bitmap_allocator_init(g, na, name, base, length,
+							blk_size, flags);
+		break;
+	default:
+		nvgpu_err(g, "Incorrect allocator type, couldn't initialize");
+		break;
+	}
+
+	if (err < 0) {
+		nvgpu_err(g, "Failed!");
+	}
+	return err;
+}