diff --git a/drivers/video/tegra/nvmap/Makefile.memory.configs b/drivers/video/tegra/nvmap/Makefile.memory.configs index 16f7531d..3c092d37 100644 --- a/drivers/video/tegra/nvmap/Makefile.memory.configs +++ b/drivers/video/tegra/nvmap/Makefile.memory.configs @@ -1,5 +1,5 @@ # SPDX-License-Identifier: GPL-2.0-only -# Copyright (c) 2021-2023, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2021-2024, NVIDIA CORPORATION. All rights reserved. # This file consists of 4 sections # Section 1: This section is for doing prerequisite check. @@ -56,14 +56,6 @@ NVMAP_CONFIG_PAGE_POOL_DEBUG := n # Config for page pool size in pages NVMAP_CONFIG_PAGE_POOL_SIZE := 0x0 -# Config to enable page coloring -# Page coloring rearranges the pages allocated based on the color -# of the page. It can improve memory access performance. -# The coloring option enable can optionally overallocate a portion of -# reqeusted allcoation size to improve the probabilty of better -# page coloring. If unsure, say Y. -NVMAP_CONFIG_COLOR_PAGES := y - # Config for FD number to start allocation from # NvMap handles are represented with FD's in the user processes. # To avoid Linux FD usage limitations, NvMap allocates FD starting @@ -140,7 +132,6 @@ endif NVMAP_CONFIG_SCIIPC := y # For OOT build NVMAP_CONFIG_UPSTREAM_KERNEL := y -NVMAP_CONFIG_COLOR_PAGES := n endif ifeq ($(CONFIG_DMABUF_DEFERRED_UNMAPPING),y) NVMAP_CONFIG_DMABUF_DEFERRED_UNMAPPING := y @@ -170,13 +161,6 @@ ccflags-y += -DNVMAP_CONFIG_PAGE_POOL_SIZE=${NVMAP_CONFIG_PAGE_POOL_SIZE} endif #NVMAP_CONFIG_PAGE_POOL_SIZE endif #NVMAP_CONFIG_PAGE_POOLS -# NVMAP_CONFIG_COLOR_PAGES depends upon CONFIG_ARM64_4K_PAGES -ifeq ($(CONFIG_ARM64_4K_PAGES),y) -ifeq ($(NVMAP_CONFIG_COLOR_PAGES),y) -ccflags-y += -DNVMAP_CONFIG_COLOR_PAGES -endif #CONFIG_ARM64_4K_PAGES -endif #NVMAP_CONFIG_COLOR_PAGES - ifdef NVMAP_CONFIG_FD_START ccflags-y += -DNVMAP_CONFIG_FD_START=${NVMAP_CONFIG_FD_START} endif #NVMAP_CONFIG_FD_START diff --git a/drivers/video/tegra/nvmap/nvmap_alloc.c b/drivers/video/tegra/nvmap/nvmap_alloc.c index d674dee2..e6b19fa7 100644 --- a/drivers/video/tegra/nvmap/nvmap_alloc.c +++ b/drivers/video/tegra/nvmap/nvmap_alloc.c @@ -71,397 +71,6 @@ static struct page *nvmap_alloc_pages_exact(gfp_t gfp, size_t size, bool use_num return page; } -static uint s_nr_colors = 1; -module_param_named(nr_colors, s_nr_colors, uint, 0644); - -#define NVMAP_MAX_COLORS 16 - -struct color_list { - u32 *counts; - u32 *heads; - u32 *list; - struct page **pages; - u32 page_count; - u32 length; -}; - -static struct color_list *alloc_color_list(u32 nr_pages, u32 nr_colors) -{ - struct color_list *list; - u32 *temp = NULL; - u32 nr_u32; - - list = kzalloc(sizeof(struct color_list), GFP_KERNEL); - if (!list) - return NULL; - - list->pages = vmalloc(nr_pages * sizeof(struct page *)); - if (!list->pages) { - kfree(list); - return NULL; - } - - /* Allocate counts, heads, and list with a single allocation */ - nr_u32 = nr_pages + 2 * nr_colors; - temp = vmalloc(nr_u32 * sizeof(u32)); - if (!temp) - goto fail; - - memset(&temp[0], 0, 2 * nr_colors * sizeof(u32)); - list->counts = &temp[0]; - list->heads = &temp[nr_colors]; - list->list = &temp[2 * nr_colors]; - - list->page_count = nr_pages; - - return list; -fail: - if (list->pages) - vfree(list->pages); - kfree(list); - return NULL; -} - -static void free_color_list(struct color_list *list) -{ - vfree(list->pages); - vfree(list->counts); /* Frees counts, heads, and list */ - kfree(list); -} - -static struct page *list_pop_page(struct color_list *list, u32 color, char *who) -{ - u32 i; - - /* Debug check */ - if ((list->counts[color] == 0) || (list->counts[color] > 1 << 31)) { - pr_err("list_pop_page: OVER FREE!\n"); - pr_err(" called from: %s\n", who); - for (i = 0; i < s_nr_colors; i++) - pr_err(" color = %d: %d\n", i, list->counts[i]); - BUG(); - } - i = list->heads[color]; - list->heads[color] = list->list[i]; - list->counts[color]--; - return list->pages[i]; -} - -struct nvmap_alloc_state { - u32 nr_colors; - u32 (*addr_to_color)(uintptr_t phys); - u32 tile; - u32 output_count; - u32 nr_pages; - u32 max_color_per_tile; - struct color_list *list; -}; - -#define CHANNEL_MASK_0 0x27af5200 -#define CHANNEL_MASK_1 0x563ca400 -#define CHANNEL_MASK_2 0x3f264800 -#define CHANNEL_MASK_3 0xe2443000 -#define BANK_MASK_0 0x5ca78400 -#define BANK_MASK_1 0xe5724800 -#define BANK_MASK_2 0x973bb000 - -#define BIT_N(a, n) \ - (((a) >> (n)) & 1) - -#define BITS_XOR_9_TO_31(a) \ - (BIT_N((a), 9) ^ BIT_N((a), 10) ^ BIT_N((a), 11) ^ BIT_N((a), 12) ^ \ - BIT_N((a), 13) ^ BIT_N((a), 14) ^ BIT_N((a), 15) ^ BIT_N((a), 16) ^ \ - BIT_N((a), 17) ^ BIT_N((a), 18) ^ BIT_N((a), 19) ^ BIT_N((a), 20) ^ \ - BIT_N((a), 21) ^ BIT_N((a), 22) ^ BIT_N((a), 23) ^ BIT_N((a), 24) ^ \ - BIT_N((a), 25) ^ BIT_N((a), 26) ^ BIT_N((a), 27) ^ BIT_N((a), 28) ^ \ - BIT_N((a), 29) ^ BIT_N((a), 30) ^ BIT_N((a), 31)) - -#define BITS_XOR_10_TO_31(a) \ - (BIT_N((a), 10) ^ BIT_N((a), 11) ^ BIT_N((a), 12) ^ \ - BIT_N((a), 13) ^ BIT_N((a), 14) ^ BIT_N((a), 15) ^ BIT_N((a), 16) ^ \ - BIT_N((a), 17) ^ BIT_N((a), 18) ^ BIT_N((a), 19) ^ BIT_N((a), 20) ^ \ - BIT_N((a), 21) ^ BIT_N((a), 22) ^ BIT_N((a), 23) ^ BIT_N((a), 24) ^ \ - BIT_N((a), 25) ^ BIT_N((a), 26) ^ BIT_N((a), 27) ^ BIT_N((a), 28) ^ \ - BIT_N((a), 29) ^ BIT_N((a), 30) ^ BIT_N((a), 31)) - -static u32 addr_to_color_t19x(uintptr_t phys) -{ - int color, chan, bank; - u32 addr = (u32)phys; - u32 xaddr = (u32)(phys >> 4); - - - chan = (BITS_XOR_9_TO_31(addr & CHANNEL_MASK_0) << 0); - chan |= (BITS_XOR_9_TO_31(addr & CHANNEL_MASK_1) << 1); - chan |= (BITS_XOR_9_TO_31(addr & CHANNEL_MASK_2) << 2); - chan |= (BITS_XOR_9_TO_31(addr & CHANNEL_MASK_3) << 3); - - bank = (BITS_XOR_10_TO_31(xaddr & BANK_MASK_0) << 0); - bank |= (BITS_XOR_10_TO_31(xaddr & BANK_MASK_1) << 1); - bank |= (BITS_XOR_10_TO_31(xaddr & BANK_MASK_2) << 2); - - WARN_ON(chan > 15); - WARN_ON(bank > 7); - /* It is preferable to color pages based on even/odd banks - * as well. To limit the number of colors to 16, bank info - * is not used in page coloring. - */ - color = chan; - - return color; -} - -static struct color_list *init_color_list(struct nvmap_alloc_state *state, - u32 nr_pages) -{ - struct color_list *list; - u32 color, i, page_index = 0; - gfp_t gfp = GFP_NVMAP | __GFP_ZERO; - - list = alloc_color_list(nr_pages, state->nr_colors); - if (!list) - return NULL; - -#ifdef NVMAP_CONFIG_PAGE_POOLS - /* Allocated page from nvmap page pool if possible */ - page_index = nvmap_page_pool_alloc_lots(&nvmap_dev->pool, list->pages, nr_pages, - false, 0); -#endif - /* Fall back to general page allocator */ - for (i = page_index; i < nr_pages; i++) { - list->pages[i] = nvmap_alloc_pages_exact(gfp, PAGE_SIZE, false, 0); - if (!list->pages[i]) - goto fail; - } - /* Clean the cache for any page that didn't come from the page pool */ - if (page_index < nr_pages) - nvmap_clean_cache(&list->pages[page_index], - nr_pages - page_index); - - /* Create linked list of colors and compute the histogram */ - for (i = 0; i < nr_pages; i++) { - color = state->addr_to_color((uintptr_t) - page_to_phys(list->pages[i])); - list->list[i] = list->heads[color]; - list->heads[color] = i; - list->counts[color]++; - } - return list; -fail: - while (i--) - __free_page(list->pages[i]); - free_color_list(list); - return NULL; -} - -static void smooth_pages(struct color_list *list, u32 nr_extra, u32 nr_colors) -{ - u32 i, j, color, max; - u32 counts[NVMAP_MAX_COLORS] = {0}; - - if (nr_extra == 0) - return; - - /* Determine which colors need to be freed */ - for (i = 0; i < nr_extra; i++) { - /* Find the max */ - max = 0; - color = 0; - for (j = 0; j < nr_colors; j++) { - if (list->counts[j] - counts[j] > max) { - color = j; - max = list->counts[j] - counts[j]; - } - } - counts[color]++; - } - - /* Iterate through 0...nr_extra-1 in psuedorandom order */ - do { - /* Pop the max off and free it */ - for (color = 0; color < nr_colors; color++) { - while (counts[color]) { - __free_page(list_pop_page(list, - color, "smooth_pages")); - counts[color]--; - nr_extra--; - } - } - } while (nr_extra > 0); -} - -static void add_perfect(struct nvmap_alloc_state *state, u32 nr_pages, - struct page **out_pages) -{ - u32 i; - u32 color; - struct page *page; - uintptr_t virt_addr; - - - /* create a perfect tile */ - for (i = 0; - i < state->nr_colors && state->output_count < nr_pages; - i++) { - virt_addr = (i + (state->tile * state->nr_colors)) * PAGE_SIZE; - color = state->addr_to_color(virt_addr); - page = list_pop_page(state->list, color, "perfect"); - out_pages[state->output_count++] = page; - } -} - -static void add_imperfect(struct nvmap_alloc_state *state, u32 nr_pages, - struct page **out_pages) -{ - u32 i, j; - u32 max_count; - u32 color; - struct page *page; - uintptr_t virt_addr; - u32 counts[NVMAP_MAX_COLORS] = {0}; - - /* Determine which colors will go into the tile */ - for (i = 0; i < state->nr_colors; i++) { - max_count = 0; - color = 0; - for (j = 0; j < state->nr_colors; j++) { - u32 left = state->list->counts[j] - counts[j]; - - if (left > max_count && - counts[j] < state->max_color_per_tile) { - max_count = left; - color = j; - } - } - counts[color]++; - } - - /* Arrange the colors into the tile */ - for (i = 0; - i < state->nr_colors && state->output_count < nr_pages; - i++) { - virt_addr = (i + (i * state->nr_colors)) * PAGE_SIZE; - color = state->addr_to_color(virt_addr); - /* Find a substitute color */ - if (counts[color] == 0) { - /* Find the color used the most in the tile */ - max_count = 0; - for (j = 0; j < state->nr_colors; j++) { - if (counts[j] > max_count) { - max_count = counts[j]; - color = j; - } - } - } - page = list_pop_page(state->list, color, "imperfect"); - out_pages[state->output_count++] = page; - counts[color]--; - } -} - -static int alloc_colored(u32 nr_pages, - struct page **out_pages, u32 chipid) -{ - struct nvmap_alloc_state state; - u32 nr_alloc, max_count, min_count; - u32 nr_tiles, nr_perfect, nr_imperfect; - int dither_state; - u32 i; - - state.nr_colors = s_nr_colors; - state.addr_to_color = addr_to_color_t19x; - - /* Allocate pages for full 32-page tiles */ - nr_tiles = (nr_pages + state.nr_colors - 1) / state.nr_colors; - /* Overallocate pages by 1/16th */ - nr_alloc = state.nr_colors * nr_tiles; - nr_alloc += nr_alloc >> 4; - - /* Create lists of each page color */ - state.list = init_color_list(&state, nr_alloc); - if (!state.list) - return -ENOMEM; - - /* Smooth out the histogram by freeing over allocated pages */ - smooth_pages(state.list, nr_alloc - state.nr_colors * nr_tiles, - state.nr_colors); - - max_count = 0; - min_count = state.list->counts[0]; - for (i = 0; i < state.nr_colors; i++) { - if (state.list->counts[i] > max_count) - max_count = state.list->counts[i]; - if (state.list->counts[i] < min_count) - min_count = state.list->counts[i]; - } - - /* - * Compute the number of perfect / imperfect tiles and the maximum - * number of pages with the same color can be in a tile - * - * Perfect tile: A tile which consist of one page of each color i.e. 16 pages, - * each of different color - * Imperfect tile: A tile which is not perfect i.e. at least some color will repeat - * max_color_per_tile: How many max times any color can be present in a tile - */ - if (min_count == 0) { - /* - * If there is no page of at least one color, then not a sigle perefect tile can be - * created. The max color pages would need to be distributed equally among all - * tiles. - */ - nr_perfect = 0; - state.max_color_per_tile = max_count / nr_tiles; - if (max_count % nr_tiles != 0) - (state.max_color_per_tile)++; - } else if (min_count == nr_tiles) { - /* - * If pages with each color are at least the number of tiles, then all of the tiles - * can be perfect. - */ - nr_perfect = nr_tiles; - state.max_color_per_tile = 1; - } else { - /* - * Some of the tiles can be perfect and remaining will be imperfect. - * min_count number of perfect tiles can be created, hence the min_count number of - * pages - * having max color would be present in the perfect tiles, The remaining pages would - * be distributed equally among the imperfect tiles. - */ - nr_perfect = min_count; - nr_imperfect = nr_tiles - nr_perfect; - state.max_color_per_tile = (max_count - nr_perfect) / nr_imperfect; - if ((max_count - nr_perfect) % nr_imperfect != 0) - (state.max_color_per_tile)++; - } - nr_imperfect = nr_tiles - nr_perfect; - - /* Output tiles */ - dither_state = nr_perfect - nr_imperfect; - state.output_count = 0; - for (state.tile = 0; state.tile < nr_tiles; state.tile++) { - if (dither_state > 0) { - add_perfect(&state, nr_pages, out_pages); - dither_state -= nr_imperfect; - } else { - add_imperfect(&state, nr_pages, out_pages); - dither_state += nr_perfect; - } - } - - /* Free extra pages created when the buffer does not - * fill the last tile - */ - for (i = 0; i < state.nr_colors; i++) - while (state.list->counts[i] > 0) - __free_page(list_pop_page(state.list, i, "free")); - - free_color_list(state.list); - - return 0; -} - static int handle_page_alloc(struct nvmap_client *client, struct nvmap_handle *h, bool contiguous) { @@ -477,15 +86,6 @@ static int handle_page_alloc(struct nvmap_client *client, int pages_per_big_pg = 0; #endif #endif /* CONFIG_ARM64_4K_PAGES */ - static u8 chipid; - - if (!chipid) { -#ifdef NVMAP_CONFIG_COLOR_PAGES - chipid = tegra_get_chip_id(); - if (chipid == TEGRA194) - s_nr_colors = 16; -#endif - } pages = nvmap_altalloc(nr_page * sizeof(*pages)); if (!pages) @@ -531,31 +131,25 @@ static int handle_page_alloc(struct nvmap_client *client, } nvmap_big_page_allocs += page_index; #endif /* CONFIG_ARM64_4K_PAGES */ - if (s_nr_colors <= 1) { #ifdef NVMAP_CONFIG_PAGE_POOLS /* Get as many pages from the pool as possible. */ - page_index += nvmap_page_pool_alloc_lots( - &nvmap_dev->pool, &pages[page_index], - nr_page - page_index, true, h->numa_id); + page_index += nvmap_page_pool_alloc_lots( + &nvmap_dev->pool, &pages[page_index], + nr_page - page_index, true, h->numa_id); #endif - allocated = page_index; - if (page_index < nr_page) { - int nid = h->numa_id == NUMA_NO_NODE ? numa_mem_id() : h->numa_id; + allocated = page_index; + if (page_index < nr_page) { + int nid = h->numa_id == NUMA_NO_NODE ? numa_mem_id() : h->numa_id; - allocated = __alloc_pages_bulk(gfp, nid, NULL, - nr_page, NULL, pages); - } - for (i = allocated; i < nr_page; i++) { - pages[i] = nvmap_alloc_pages_exact(gfp, PAGE_SIZE, - true, h->numa_id); + allocated = __alloc_pages_bulk(gfp, nid, NULL, + nr_page, NULL, pages); + } + for (i = allocated; i < nr_page; i++) { + pages[i] = nvmap_alloc_pages_exact(gfp, PAGE_SIZE, + true, h->numa_id); - if (!pages[i]) - goto fail; - } - } else if (page_index < nr_page) { - if (alloc_colored(nr_page - page_index, &pages[page_index], chipid)) + if (!pages[i]) goto fail; - page_index = nr_page; } nvmap_total_page_allocs += nr_page; } diff --git a/drivers/video/tegra/nvmap/nvmap_ioctl.c b/drivers/video/tegra/nvmap/nvmap_ioctl.c index 53fb331e..262b2015 100644 --- a/drivers/video/tegra/nvmap/nvmap_ioctl.c +++ b/drivers/video/tegra/nvmap/nvmap_ioctl.c @@ -1360,10 +1360,9 @@ int nvmap_ioctl_handle_from_sci_ipc_id(struct file *filp, void __user *arg) /* * This function calculates allocatable free memory using following formula: - * free_mem = avail mem - cma free - (avail mem - cma free) / 16 + * free_mem = avail mem - cma free * The CMA memory is not allocatable by NvMap for regular allocations and it * is part of Available memory reported, so subtract it from available memory. - * NvMap allocates 1/16 extra memory in page coloring, so subtract it as well. */ int system_heap_free_mem(unsigned long *mem_val) { @@ -1383,9 +1382,6 @@ int system_heap_free_mem(unsigned long *mem_val) return 0; } free_mem = (available_mem << PAGE_SHIFT) - cma_free; -#ifdef NVMAP_CONFIG_COLOR_PAGES - free_mem = free_mem - (free_mem >> 4); -#endif /* NVMAP_CONFIG_COLOR_PAGES */ *mem_val = free_mem; return 0; }