From 594e39e4986658e8e49ad59ad666ec8e660535ad Mon Sep 17 00:00:00 2001 From: Ketan Patil Date: Thu, 23 Mar 2023 12:55:36 +0000 Subject: [PATCH] video: tegra: nvmap: Add NUMA support in nvmap In NUMA system, user can request nvmap to allocate from any particular node using NvRmMem APIs. Add code to request allocation from such requested node. While allocating pages from page pool, check if those pages belong to requested NUMA node. In some cases, we may not need to use NUMA support e.g. in color pages allocation code, we don't need to use NUMA, as page coloring is needed only for t19x chips. JIRA TMM-5287 Bug 4043165 Change-Id: If8043278122068773a802d83723d287d32914e14 Signed-off-by: Ketan Patil Reviewed-on: https://git-master.nvidia.com/r/c/linux-nv-oot/+/2889707 Reviewed-by: svc-mobile-coverity Reviewed-by: svc-mobile-cert Reviewed-by: Sachin Nikam GVS: Gerrit_Virtual_Submit --- drivers/video/tegra/nvmap/nvmap_alloc.c | 32 ++++++---- drivers/video/tegra/nvmap/nvmap_ioctl.c | 6 ++ drivers/video/tegra/nvmap/nvmap_pp.c | 82 +++++++++++++++++-------- drivers/video/tegra/nvmap/nvmap_priv.h | 5 +- include/uapi/linux/nvmap.h | 3 +- 5 files changed, 88 insertions(+), 40 deletions(-) diff --git a/drivers/video/tegra/nvmap/nvmap_alloc.c b/drivers/video/tegra/nvmap/nvmap_alloc.c index 93c47495..998f94cc 100644 --- a/drivers/video/tegra/nvmap/nvmap_alloc.c +++ b/drivers/video/tegra/nvmap/nvmap_alloc.c @@ -71,13 +71,16 @@ void nvmap_altfree(void *ptr, size_t len) kfree(ptr); } -static struct page *nvmap_alloc_pages_exact(gfp_t gfp, size_t size) +static struct page *nvmap_alloc_pages_exact(gfp_t gfp, size_t size, bool use_numa, int numa_id) { struct page *page, *p, *e; unsigned int order; order = get_order(size); - page = alloc_pages(gfp, order); + if (!use_numa) + page = alloc_pages(gfp, order); + else + page = alloc_pages_node(numa_id, gfp, order); if (!page) return NULL; @@ -243,11 +246,12 @@ static struct color_list *init_color_list(struct nvmap_alloc_state *state, #ifdef NVMAP_CONFIG_PAGE_POOLS /* Allocated page from nvmap page pool if possible */ - page_index = nvmap_page_pool_alloc_lots(&nvmap_dev->pool, list->pages, nr_pages); + page_index = nvmap_page_pool_alloc_lots(&nvmap_dev->pool, list->pages, nr_pages, + false, 0); #endif /* Fall back to general page allocator */ for (i = page_index; i < nr_pages; i++) { - list->pages[i] = nvmap_alloc_pages_exact(gfp, PAGE_SIZE); + list->pages[i] = nvmap_alloc_pages_exact(gfp, PAGE_SIZE, false, 0); if (!list->pages[i]) goto fail; } @@ -519,7 +523,7 @@ static int handle_page_alloc(struct nvmap_client *client, if (contiguous) { struct page *page; - page = nvmap_alloc_pages_exact(gfp, size); + page = nvmap_alloc_pages_exact(gfp, size, true, h->numa_id); if (!page) goto fail; @@ -531,7 +535,7 @@ static int handle_page_alloc(struct nvmap_client *client, #ifdef NVMAP_CONFIG_PAGE_POOLS /* Get as many big pages from the pool as possible. */ page_index = nvmap_page_pool_alloc_lots_bp(&nvmap_dev->pool, pages, - nr_page); + nr_page, true, h->numa_id); pages_per_big_pg = nvmap_dev->pool.pages_per_big_pg; #endif /* Try to allocate big pages from page allocator */ @@ -547,7 +551,7 @@ static int handle_page_alloc(struct nvmap_client *client, gfp_t gfp_no_reclaim = (gfp | __GFP_NOMEMALLOC) & ~__GFP_RECLAIM; page = nvmap_alloc_pages_exact(gfp_no_reclaim, - pages_per_big_pg << PAGE_SHIFT); + pages_per_big_pg << PAGE_SHIFT, true, h->numa_id); if (!page) break; @@ -562,17 +566,21 @@ static int handle_page_alloc(struct nvmap_client *client, /* Get as many pages from the pool as possible. */ page_index += nvmap_page_pool_alloc_lots( &nvmap_dev->pool, &pages[page_index], - nr_page - page_index); + nr_page - page_index, true, h->numa_id); #endif allocated = page_index; #if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 13, 0) - if (page_index < nr_page) - allocated = __alloc_pages_bulk(gfp, numa_mem_id(), NULL, + if (page_index < nr_page) { + int nid = h->numa_id == NUMA_NO_NODE ? numa_mem_id() : h->numa_id; + + allocated = __alloc_pages_bulk(gfp, nid, NULL, nr_page, NULL, pages); + } #endif for (i = allocated; i < nr_page; i++) { - pages[i] = nvmap_alloc_pages_exact(gfp, - PAGE_SIZE); + pages[i] = nvmap_alloc_pages_exact(gfp, PAGE_SIZE, + true, h->numa_id); + if (!pages[i]) goto fail; } diff --git a/drivers/video/tegra/nvmap/nvmap_ioctl.c b/drivers/video/tegra/nvmap/nvmap_ioctl.c index baea1d0d..3d69ed5b 100644 --- a/drivers/video/tegra/nvmap/nvmap_ioctl.c +++ b/drivers/video/tegra/nvmap/nvmap_ioctl.c @@ -203,6 +203,11 @@ int nvmap_ioctl_alloc(struct file *filp, void __user *arg) if (!op.handle) return -EINVAL; + if (op.numa_nid > MAX_NUMNODES || (op.numa_nid != NUMA_NO_NODE && op.numa_nid < 0)) { + pr_err("numa id:%d is invalid\n", op.numa_nid); + return -EINVAL; + } + handle = nvmap_handle_get_from_id(client, op.handle); if (IS_ERR_OR_NULL(handle)) return -EINVAL; @@ -220,6 +225,7 @@ int nvmap_ioctl_alloc(struct file *filp, void __user *arg) return -ENOMEM; } + handle->numa_id = op.numa_nid; /* user-space handles are aligned to page boundaries, to prevent * data leakage. */ op.align = max_t(size_t, op.align, page_sz); diff --git a/drivers/video/tegra/nvmap/nvmap_pp.c b/drivers/video/tegra/nvmap/nvmap_pp.c index 7a367572..36fe93fb 100644 --- a/drivers/video/tegra/nvmap/nvmap_pp.c +++ b/drivers/video/tegra/nvmap/nvmap_pp.c @@ -3,7 +3,7 @@ * * Manage page pools to speed up page allocation. * - * Copyright (c) 2009-2022, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2009-2023, NVIDIA CORPORATION. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, @@ -63,56 +63,86 @@ static inline void __pp_dbg_var_add(u64 *dbg_var, u32 nr) static int __nvmap_page_pool_fill_lots_locked(struct nvmap_page_pool *pool, struct page **pages, u32 nr); -static inline struct page *get_zero_list_page(struct nvmap_page_pool *pool) +static inline struct page *get_zero_list_page(struct nvmap_page_pool *pool, bool use_numa, + int numa_id) { - struct page *page; + struct page *page, *tmp; + int nid = numa_id == NUMA_NO_NODE ? numa_mem_id() : numa_id; trace_get_zero_list_page(pool->to_zero); if (list_empty(&pool->zero_list)) return NULL; - page = list_first_entry(&pool->zero_list, struct page, lru); + if (!use_numa) { + page = list_first_entry(&pool->zero_list, struct page, lru); + goto exit; + } else { + list_for_each_entry_safe(page, tmp, &pool->zero_list, lru) + if (page_to_nid(page) == nid) + goto exit; + } + return NULL; + +exit: list_del(&page->lru); - pool->to_zero--; - return page; } -static inline struct page *get_page_list_page(struct nvmap_page_pool *pool) +static inline struct page *get_page_list_page(struct nvmap_page_pool *pool, bool use_numa, + int numa_id) { - struct page *page; + struct page *page, *tmp; + int nid = numa_id == NUMA_NO_NODE ? numa_mem_id() : numa_id; trace_get_page_list_page(pool->count); if (list_empty(&pool->page_list)) return NULL; - page = list_first_entry(&pool->page_list, struct page, lru); + if (!use_numa) { + page = list_first_entry(&pool->page_list, struct page, lru); + goto exit; + } else { + list_for_each_entry_safe(page, tmp, &pool->page_list, lru) + if (page_to_nid(page) == nid) + goto exit; + } + return NULL; + +exit: list_del(&page->lru); - pool->count--; - return page; } #ifdef CONFIG_ARM64_4K_PAGES -static inline struct page *get_page_list_page_bp(struct nvmap_page_pool *pool) +static inline struct page *get_page_list_page_bp(struct nvmap_page_pool *pool, bool use_numa, + int numa_id) { - struct page *page; + struct page *page, *tmp; + int nid = numa_id == NUMA_NO_NODE ? numa_mem_id() : numa_id; trace_get_page_list_page_bp(pool->big_page_count); if (list_empty(&pool->page_list_bp)) return NULL; - page = list_first_entry(&pool->page_list_bp, struct page, lru); - list_del(&page->lru); + if (!use_numa) { + page = list_first_entry(&pool->page_list_bp, struct page, lru); + goto exit; + } else { + list_for_each_entry_safe(page, tmp, &pool->page_list_bp, lru) + if (page_to_nid(page) == nid) + goto exit; + } + return NULL; +exit: + list_del(&page->lru); pool->count -= pool->pages_per_big_pg; pool->big_page_count -= pool->pages_per_big_pg; - return page; } #endif /* CONFIG_ARM64_4K_PAGES */ @@ -147,7 +177,7 @@ static void nvmap_pp_do_background_zero_pages(struct nvmap_page_pool *pool) rt_mutex_lock(&pool->lock); for (i = 0; i < PENDING_PAGES_SIZE; i++) { - page = get_zero_list_page(pool); + page = get_zero_list_page(pool, false, 0); if (page == NULL) break; pending_zero_pages[i] = page; @@ -234,14 +264,14 @@ static ulong nvmap_page_pool_free_pages_locked(struct nvmap_page_pool *pool, #ifdef CONFIG_ARM64_4K_PAGES if (use_page_list_bp) - page = get_page_list_page_bp(pool); + page = get_page_list_page_bp(pool, false, 0); else if (use_page_list) #else if (use_page_list) #endif /* CONFIG_ARM64_4K_PAGES */ - page = get_page_list_page(pool); + page = get_page_list_page(pool, false, 0); else - page = get_zero_list_page(pool); + page = get_zero_list_page(pool, false, 0); if (!page) { if (!use_page_list) { @@ -286,7 +316,8 @@ static ulong nvmap_page_pool_free_pages_locked(struct nvmap_page_pool *pool, * array in a linear fashion starting from index 0. */ int nvmap_page_pool_alloc_lots(struct nvmap_page_pool *pool, - struct page **pages, u32 nr) + struct page **pages, u32 nr, + bool use_numa, int numa_id) { u32 ind = 0; u32 non_zero_idx; @@ -301,10 +332,10 @@ int nvmap_page_pool_alloc_lots(struct nvmap_page_pool *pool, struct page *page = NULL; if (!non_zero_cnt) - page = get_page_list_page(pool); + page = get_page_list_page(pool, use_numa, numa_id); if (!page) { - page = get_zero_list_page(pool); + page = get_zero_list_page(pool, use_numa, numa_id); if (!page) break; if (!non_zero_cnt) @@ -336,7 +367,8 @@ int nvmap_page_pool_alloc_lots(struct nvmap_page_pool *pool, #ifdef CONFIG_ARM64_4K_PAGES int nvmap_page_pool_alloc_lots_bp(struct nvmap_page_pool *pool, - struct page **pages, u32 nr) + struct page **pages, u32 nr, + bool use_numa, int numa_id) { u32 ind = 0, nr_pages = nr; struct page *page; @@ -350,7 +382,7 @@ int nvmap_page_pool_alloc_lots_bp(struct nvmap_page_pool *pool, while (nr_pages - ind >= pool->pages_per_big_pg) { int i; - page = get_page_list_page_bp(pool); + page = get_page_list_page_bp(pool, use_numa, numa_id); if (!page) break; diff --git a/drivers/video/tegra/nvmap/nvmap_priv.h b/drivers/video/tegra/nvmap/nvmap_priv.h index 230aa741..aca75a25 100644 --- a/drivers/video/tegra/nvmap/nvmap_priv.h +++ b/drivers/video/tegra/nvmap/nvmap_priv.h @@ -278,6 +278,7 @@ struct nvmap_handle { * waitq to wait on RO dmabuf release completion, if release is already in progress. */ wait_queue_head_t waitq; + int numa_id; }; struct nvmap_handle_info { @@ -345,10 +346,10 @@ int nvmap_page_pool_init(struct nvmap_device *dev); int nvmap_page_pool_fini(struct nvmap_device *dev); struct page *nvmap_page_pool_alloc(struct nvmap_page_pool *pool); int nvmap_page_pool_alloc_lots(struct nvmap_page_pool *pool, - struct page **pages, u32 nr); + struct page **pages, u32 nr, bool use_numa, int numa_id); #ifdef CONFIG_ARM64_4K_PAGES int nvmap_page_pool_alloc_lots_bp(struct nvmap_page_pool *pool, - struct page **pages, u32 nr); + struct page **pages, u32 nr, bool use_numa, int numa_id); #endif /* CONFIG_ARM64_4K_PAGES */ u32 nvmap_page_pool_fill_lots(struct nvmap_page_pool *pool, struct page **pages, u32 nr); diff --git a/include/uapi/linux/nvmap.h b/include/uapi/linux/nvmap.h index a8560e3d..de7d0f47 100644 --- a/include/uapi/linux/nvmap.h +++ b/include/uapi/linux/nvmap.h @@ -3,7 +3,7 @@ * * structure declarations for nvmem and nvmap user-space ioctls * - * Copyright (c) 2009-2022, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2009-2023, NVIDIA CORPORATION. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, @@ -103,6 +103,7 @@ struct nvmap_alloc_handle { __u32 heap_mask; /* heaps to allocate from */ __u32 flags; /* wb/wc/uc/iwb etc. */ __u32 align; /* min alignment necessary */ + __s32 numa_nid; /* NUMA node id */ }; struct nvmap_alloc_ivm_handle {