diff --git a/arch/nvgpu-common.yaml b/arch/nvgpu-common.yaml index efdfc50f1..2050c882b 100644 --- a/arch/nvgpu-common.yaml +++ b/arch/nvgpu-common.yaml @@ -580,11 +580,6 @@ mm: sources: [ common/mm/allocators/page_allocator.c, include/nvgpu/page_allocator.h ] deps: [ ] - lockless: - safe: no - sources: [ common/mm/allocators/lockless_allocator.c, - common/mm/allocators/lockless_allocator_priv.h ] - tags: unit-testable dma: safe: yes sources: [ common/mm/dma.c ] diff --git a/drivers/gpu/nvgpu/Makefile b/drivers/gpu/nvgpu/Makefile index 086b79557..616197107 100644 --- a/drivers/gpu/nvgpu/Makefile +++ b/drivers/gpu/nvgpu/Makefile @@ -508,7 +508,6 @@ nvgpu-y += \ common/mm/allocators/nvgpu_allocator.o \ common/mm/allocators/bitmap_allocator.o \ common/mm/allocators/buddy_allocator.o \ - common/mm/allocators/lockless_allocator.o \ common/mm/gmmu/page_table.o \ common/mm/gmmu/pd_cache.o \ common/mm/gmmu/pte.o \ diff --git a/drivers/gpu/nvgpu/Makefile.sources b/drivers/gpu/nvgpu/Makefile.sources index 0b6014858..ba8221635 100644 --- a/drivers/gpu/nvgpu/Makefile.sources +++ b/drivers/gpu/nvgpu/Makefile.sources @@ -427,8 +427,7 @@ endif endif ifeq ($(CONFIG_NVGPU_FENCE),1) -srcs += common/fence/fence.c \ - common/mm/allocators/lockless_allocator.c +srcs += common/fence/fence.c endif ifeq ($(CONFIG_NVGPU_FECS_TRACE),1) diff --git a/drivers/gpu/nvgpu/common/mm/allocators/lockless_allocator.c b/drivers/gpu/nvgpu/common/mm/allocators/lockless_allocator.c deleted file mode 100644 index 395c61a87..000000000 --- a/drivers/gpu/nvgpu/common/mm/allocators/lockless_allocator.c +++ /dev/null @@ -1,234 +0,0 @@ -/* - * Copyright (c) 2016-2020, NVIDIA CORPORATION. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - */ - -#include -#include -#include -#include -#include - -#include "lockless_allocator_priv.h" - -static u64 nvgpu_lockless_alloc_length(struct nvgpu_allocator *a) -{ - struct nvgpu_lockless_allocator *pa = a->priv; - - return pa->length; -} - -static u64 nvgpu_lockless_alloc_base(struct nvgpu_allocator *a) -{ - struct nvgpu_lockless_allocator *pa = a->priv; - - return pa->base; -} - -static bool nvgpu_lockless_alloc_inited(struct nvgpu_allocator *a) -{ - struct nvgpu_lockless_allocator *pa = a->priv; - bool inited = pa->inited; - - nvgpu_smp_rmb(); - return inited; -} - -static u64 nvgpu_lockless_alloc_end(struct nvgpu_allocator *a) -{ - struct nvgpu_lockless_allocator *pa = a->priv; - - return nvgpu_safe_add_u64(pa->base, pa->length); -} - -static u64 nvgpu_lockless_alloc(struct nvgpu_allocator *a, u64 len) -{ - struct nvgpu_lockless_allocator *pa = a->priv; - int head, new_head, ret; - u64 addr = 0; - - if (len != pa->blk_size) { - return 0; - } - - head = NV_READ_ONCE(pa->head); - while (head >= 0) { - new_head = NV_READ_ONCE(pa->next[head]); - ret = cmpxchg(&pa->head, head, new_head); - if (ret == head) { - addr = pa->base + U64(head) * pa->blk_size; - nvgpu_atomic_inc(&pa->nr_allocs); - alloc_dbg(a, "Alloc node # %d @ addr 0x%llx", head, - addr); - break; - } - head = NV_READ_ONCE(pa->head); - } - - if (addr != 0ULL) { - alloc_dbg(a, "Alloc node # %d @ addr 0x%llx", head, addr); - } else { - alloc_dbg(a, "Alloc failed!"); - } - - return addr; -} - -static void nvgpu_lockless_free(struct nvgpu_allocator *a, u64 addr) -{ - struct nvgpu_lockless_allocator *pa = a->priv; - int head, ret; - u64 cur_idx; - - cur_idx = nvgpu_safe_sub_u64(addr, pa->base) / pa->blk_size; - - alloc_dbg(a, "Free node # %llu @ addr 0x%llx", cur_idx, addr); - - while (true) { - head = NV_READ_ONCE(pa->head); - NV_WRITE_ONCE(pa->next[cur_idx], head); - nvgpu_assert(cur_idx <= U64(INT_MAX)); - ret = cmpxchg(&pa->head, head, (int)cur_idx); - if (ret == head) { - nvgpu_atomic_dec(&pa->nr_allocs); - alloc_dbg(a, "Free node # %llu", cur_idx); - break; - } - } -} - -static void nvgpu_lockless_alloc_destroy(struct nvgpu_allocator *a) -{ - struct nvgpu_lockless_allocator *pa = a->priv; - -#ifdef CONFIG_DEBUG_FS - nvgpu_fini_alloc_debug(a); -#endif - - nvgpu_vfree(a->g, pa->next); - nvgpu_kfree(nvgpu_alloc_to_gpu(a), pa); -} - -#ifdef __KERNEL__ -static void nvgpu_lockless_print_stats(struct nvgpu_allocator *a, - struct seq_file *s, int lock) -{ - struct nvgpu_lockless_allocator *pa = a->priv; - - alloc_pstat(s, a, "Lockless allocator params:"); - alloc_pstat(s, a, " start = 0x%llx", pa->base); - alloc_pstat(s, a, " end = 0x%llx", pa->base + pa->length); - - /* Actual stats. */ - alloc_pstat(s, a, "Stats:"); - alloc_pstat(s, a, " Number allocs = %d", - nvgpu_atomic_read(&pa->nr_allocs)); - alloc_pstat(s, a, " Number free = %d", - pa->nr_nodes - nvgpu_atomic_read(&pa->nr_allocs)); -} -#endif - -static const struct nvgpu_allocator_ops pool_ops = { - .alloc = nvgpu_lockless_alloc, - .free_alloc = nvgpu_lockless_free, - - .base = nvgpu_lockless_alloc_base, - .length = nvgpu_lockless_alloc_length, - .end = nvgpu_lockless_alloc_end, - .inited = nvgpu_lockless_alloc_inited, - - .fini = nvgpu_lockless_alloc_destroy, - -#ifdef __KERNEL__ - .print_stats = nvgpu_lockless_print_stats, -#endif -}; - -int nvgpu_lockless_allocator_init(struct gk20a *g, struct nvgpu_allocator *na, - const char *name, u64 base, u64 length, - u64 blk_size, u64 flags) -{ - int i; - int err; - int nr_nodes; - u64 count; - struct nvgpu_lockless_allocator *a; - - if (blk_size == 0ULL) { - return -EINVAL; - } - - /* - * Ensure we have space for at least one node & there's no overflow. - * In order to control memory footprint, we require count < INT_MAX - */ - count = length / blk_size; - if (base == 0ULL || count == 0ULL || count > U64(INT_MAX)) { - return -EINVAL; - } - - a = nvgpu_kzalloc(g, sizeof(struct nvgpu_lockless_allocator)); - if (a == NULL) { - return -ENOMEM; - } - - err = nvgpu_alloc_common_init(na, g, name, a, false, &pool_ops); - if (err != 0) { - goto fail; - } - - a->next = nvgpu_vzalloc(g, sizeof(*a->next) * count); - if (a->next == NULL) { - err = -ENOMEM; - goto fail; - } - - /* chain the elements together to form the initial free list */ - nr_nodes = (int)count; - for (i = 0; i < nr_nodes; i++) { - a->next[i] = i + 1; - } - a->next[nr_nodes - 1] = -1; - - a->base = base; - a->length = length; - a->blk_size = blk_size; - a->nr_nodes = nr_nodes; - a->flags = flags; - nvgpu_atomic_set(&a->nr_allocs, 0); - - nvgpu_smp_wmb(); - a->inited = true; - -#ifdef CONFIG_DEBUG_FS - nvgpu_init_alloc_debug(g, na); -#endif - alloc_dbg(na, "New allocator: type lockless"); - alloc_dbg(na, " base 0x%llx", a->base); - alloc_dbg(na, " nodes %d", a->nr_nodes); - alloc_dbg(na, " blk_size 0x%llx", a->blk_size); - alloc_dbg(na, " flags 0x%llx", a->flags); - - return 0; - -fail: - nvgpu_kfree(g, a); - return err; -} diff --git a/drivers/gpu/nvgpu/common/mm/allocators/lockless_allocator_priv.h b/drivers/gpu/nvgpu/common/mm/allocators/lockless_allocator_priv.h deleted file mode 100644 index c2f6649a4..000000000 --- a/drivers/gpu/nvgpu/common/mm/allocators/lockless_allocator_priv.h +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright (c) 2016 - 2017, NVIDIA CORPORATION. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - */ - -/* - * Basics: - * - * - Lockless memory allocator for fixed-size structures, whose - * size is defined up front at init time. - * - Memory footprint scales linearly w/ the number of structures in - * the pool. It is ~= sizeof(int) * N. - * - Memory is pre-allocated by the client. The allocator itself - * only computes the addresses for allocations. - * - Limit of MAX_INT nodes that the allocator can be responsible for. - * - * Implementation details: - * - * The allocator maintains a single list of free nodes. We allocate & - * free nodes from the head of the list. We rely on the cmpxchg() operator - * to maintain atomicity on the head. - * - * So, both allocs & frees are O(1)!! - * - * -- Definitions -- - * Block Size - size of a single structure that this allocator will - * allocate. - * Node - one of the elements of size blk_size in the - * client-allocated buffer. - * Node Index - zero-based index of a node in the client-allocated - * contiguous buffer. - * - * -- Initial State -- - * We maintain the following to track the state of the free list: - * - * 1) A "head" index to track the index of the first free node in the list - * 2) A "next" array to track the index of the next free node in the list - * for every node. So next[head], will give the index to the 2nd free - * element in the list. - * - * So, to begin with, the free list consists of all node indices, and each - * position in the next array contains index N + 1: - * - * head = 0 - * next = [1, 2, 3, 4, -1] : Example for a user-allocated buffer of 5 nodes - * free_list = 0->1->2->3->4->-1 - * - * -- Allocations -- - * 1) Read the current head (aka acq_head) - * 2) Read next[acq_head], to get the 2nd free element (aka new_head) - * 3) cmp_xchg(&head, acq_head, new_head) - * 4) If it succeeds, compute the address of the node, based on - * base address, blk_size, & acq_head. - * - * head = 1; - * next = [1, 2, 3, 4, -1] : Example after allocating Node #0 - * free_list = 1->2->3->4->-1 - * - * head = 2; - * next = [1, 2, 3, 4, -1] : Example after allocating Node #1 - * free_list = 2->3->4->-1 - * - * -- Frees -- - * 1) Based on the address to be freed, calculate the index of the node - * being freed (cur_idx) - * 2) Read the current head (old_head) - * 3) So the freed node is going to go at the head of the list, and we - * want to put the old_head after it. So next[cur_idx] = old_head - * 4) cmpxchg(head, old_head, cur_idx) - * - * head = 0 - * next = [2, 2, 3, 4, -1] - * free_list = 0->2->3->4->-1 : Example after freeing Node #0 - * - * head = 1 - * next = [2, 0, 3, 4, -1] - * free_list = 1->0->2->3->4->-1 : Example after freeing Node #1 - */ - -#ifndef LOCKLESS_ALLOCATOR_PRIV_H -#define LOCKLESS_ALLOCATOR_PRIV_H - -struct nvgpu_allocator; - -struct nvgpu_lockless_allocator { - struct nvgpu_allocator *owner; - - u64 base; /* Base address of the space. */ - u64 length; /* Length of the space. */ - u64 blk_size; /* Size of the structure being allocated */ - int nr_nodes; /* Number of nodes available for allocation */ - - int *next; /* An array holding the next indices per node */ - int head; /* Current node at the top of the stack */ - - u64 flags; - - bool inited; - - /* Statistics */ - nvgpu_atomic_t nr_allocs; -}; - -static inline struct nvgpu_lockless_allocator *lockless_allocator( - struct nvgpu_allocator *a) -{ - return (struct nvgpu_lockless_allocator *)(a)->priv; -} - -#endif diff --git a/drivers/gpu/nvgpu/include/nvgpu/allocator.h b/drivers/gpu/nvgpu/include/nvgpu/allocator.h index 77629bbb4..26ed43103 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/allocator.h +++ b/drivers/gpu/nvgpu/include/nvgpu/allocator.h @@ -473,27 +473,6 @@ int nvgpu_allocator_init(struct gk20a *g, struct nvgpu_allocator *na, u64 base, u64 length, u64 blk_size, u64 max_order, u64 flags, enum nvgpu_allocator_type alloc_type); -#ifdef CONFIG_NVGPU_FENCE -/** - * @brief Initialize lockless allocator. - * - * @param[in] g Pointer to GPU structure. - * @param[in] na Pointer to allocator structure. - * @param[in] name Name of lockless allocator. - * @param[in] base Base address of lockless allocator. - * @param[in] length Size of lockless allocator. - * @param[in] blk_size Block size of lockless allocator. - * @param[in] flags Flags indicating lockless allocator conditions. - * - * @return 0 in case of success, < 0 otherwise. - * @retval -EINVAL in case of incorrect input value. - * @retval -ENOMEM in case there is not enough memory for allocation. - */ -int nvgpu_lockless_allocator_init(struct gk20a *g, struct nvgpu_allocator *na, - const char *name, u64 base, u64 length, - u64 blk_size, u64 flags); -#endif - /** * Largest block of resources that fits in address space. */ diff --git a/drivers/gpu/nvgpu/include/nvgpu/posix/atomic.h b/drivers/gpu/nvgpu/include/nvgpu/posix/atomic.h index 26e993a90..e19acdc2f 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/posix/atomic.h +++ b/drivers/gpu/nvgpu/include/nvgpu/posix/atomic.h @@ -701,21 +701,4 @@ static inline bool nvgpu_atomic64_sub_and_test_impl(long x, nvgpu_atomic64_t *v) return NVGPU_POSIX_ATOMIC_SUB_RETURN(v, x) == 0; } -/* - * The following is only used by the lockless allocator and makes direct use - * of the cmpxchg function. For POSIX, this is translated to a call to - * nvgpu_atomic_cmpxchg. - */ - -/** - * @brief Define for compare and exchange POSIX implementation. - * - * @param p Pointer to data. - * @param old Value to compare. - * @param new Value to exchange. - */ -#define cmpxchg(p, old, new) \ - nvgpu_atomic_cmpxchg((nvgpu_atomic_t *)(void *)(p), (old), (new)) - - #endif /* NVGPU_POSIX_ATOMIC_H */