diff --git a/drivers/video/tegra/nvmap/nvmap_alloc.c b/drivers/video/tegra/nvmap/nvmap_alloc.c index 93c47495..998f94cc 100644 --- a/drivers/video/tegra/nvmap/nvmap_alloc.c +++ b/drivers/video/tegra/nvmap/nvmap_alloc.c @@ -71,13 +71,16 @@ void nvmap_altfree(void *ptr, size_t len) kfree(ptr); } -static struct page *nvmap_alloc_pages_exact(gfp_t gfp, size_t size) +static struct page *nvmap_alloc_pages_exact(gfp_t gfp, size_t size, bool use_numa, int numa_id) { struct page *page, *p, *e; unsigned int order; order = get_order(size); - page = alloc_pages(gfp, order); + if (!use_numa) + page = alloc_pages(gfp, order); + else + page = alloc_pages_node(numa_id, gfp, order); if (!page) return NULL; @@ -243,11 +246,12 @@ static struct color_list *init_color_list(struct nvmap_alloc_state *state, #ifdef NVMAP_CONFIG_PAGE_POOLS /* Allocated page from nvmap page pool if possible */ - page_index = nvmap_page_pool_alloc_lots(&nvmap_dev->pool, list->pages, nr_pages); + page_index = nvmap_page_pool_alloc_lots(&nvmap_dev->pool, list->pages, nr_pages, + false, 0); #endif /* Fall back to general page allocator */ for (i = page_index; i < nr_pages; i++) { - list->pages[i] = nvmap_alloc_pages_exact(gfp, PAGE_SIZE); + list->pages[i] = nvmap_alloc_pages_exact(gfp, PAGE_SIZE, false, 0); if (!list->pages[i]) goto fail; } @@ -519,7 +523,7 @@ static int handle_page_alloc(struct nvmap_client *client, if (contiguous) { struct page *page; - page = nvmap_alloc_pages_exact(gfp, size); + page = nvmap_alloc_pages_exact(gfp, size, true, h->numa_id); if (!page) goto fail; @@ -531,7 +535,7 @@ static int handle_page_alloc(struct nvmap_client *client, #ifdef NVMAP_CONFIG_PAGE_POOLS /* Get as many big pages from the pool as possible. */ page_index = nvmap_page_pool_alloc_lots_bp(&nvmap_dev->pool, pages, - nr_page); + nr_page, true, h->numa_id); pages_per_big_pg = nvmap_dev->pool.pages_per_big_pg; #endif /* Try to allocate big pages from page allocator */ @@ -547,7 +551,7 @@ static int handle_page_alloc(struct nvmap_client *client, gfp_t gfp_no_reclaim = (gfp | __GFP_NOMEMALLOC) & ~__GFP_RECLAIM; page = nvmap_alloc_pages_exact(gfp_no_reclaim, - pages_per_big_pg << PAGE_SHIFT); + pages_per_big_pg << PAGE_SHIFT, true, h->numa_id); if (!page) break; @@ -562,17 +566,21 @@ static int handle_page_alloc(struct nvmap_client *client, /* Get as many pages from the pool as possible. */ page_index += nvmap_page_pool_alloc_lots( &nvmap_dev->pool, &pages[page_index], - nr_page - page_index); + nr_page - page_index, true, h->numa_id); #endif allocated = page_index; #if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 13, 0) - if (page_index < nr_page) - allocated = __alloc_pages_bulk(gfp, numa_mem_id(), NULL, + if (page_index < nr_page) { + int nid = h->numa_id == NUMA_NO_NODE ? numa_mem_id() : h->numa_id; + + allocated = __alloc_pages_bulk(gfp, nid, NULL, nr_page, NULL, pages); + } #endif for (i = allocated; i < nr_page; i++) { - pages[i] = nvmap_alloc_pages_exact(gfp, - PAGE_SIZE); + pages[i] = nvmap_alloc_pages_exact(gfp, PAGE_SIZE, + true, h->numa_id); + if (!pages[i]) goto fail; } diff --git a/drivers/video/tegra/nvmap/nvmap_ioctl.c b/drivers/video/tegra/nvmap/nvmap_ioctl.c index baea1d0d..3d69ed5b 100644 --- a/drivers/video/tegra/nvmap/nvmap_ioctl.c +++ b/drivers/video/tegra/nvmap/nvmap_ioctl.c @@ -203,6 +203,11 @@ int nvmap_ioctl_alloc(struct file *filp, void __user *arg) if (!op.handle) return -EINVAL; + if (op.numa_nid > MAX_NUMNODES || (op.numa_nid != NUMA_NO_NODE && op.numa_nid < 0)) { + pr_err("numa id:%d is invalid\n", op.numa_nid); + return -EINVAL; + } + handle = nvmap_handle_get_from_id(client, op.handle); if (IS_ERR_OR_NULL(handle)) return -EINVAL; @@ -220,6 +225,7 @@ int nvmap_ioctl_alloc(struct file *filp, void __user *arg) return -ENOMEM; } + handle->numa_id = op.numa_nid; /* user-space handles are aligned to page boundaries, to prevent * data leakage. */ op.align = max_t(size_t, op.align, page_sz); diff --git a/drivers/video/tegra/nvmap/nvmap_pp.c b/drivers/video/tegra/nvmap/nvmap_pp.c index 7a367572..36fe93fb 100644 --- a/drivers/video/tegra/nvmap/nvmap_pp.c +++ b/drivers/video/tegra/nvmap/nvmap_pp.c @@ -3,7 +3,7 @@ * * Manage page pools to speed up page allocation. * - * Copyright (c) 2009-2022, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2009-2023, NVIDIA CORPORATION. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, @@ -63,56 +63,86 @@ static inline void __pp_dbg_var_add(u64 *dbg_var, u32 nr) static int __nvmap_page_pool_fill_lots_locked(struct nvmap_page_pool *pool, struct page **pages, u32 nr); -static inline struct page *get_zero_list_page(struct nvmap_page_pool *pool) +static inline struct page *get_zero_list_page(struct nvmap_page_pool *pool, bool use_numa, + int numa_id) { - struct page *page; + struct page *page, *tmp; + int nid = numa_id == NUMA_NO_NODE ? numa_mem_id() : numa_id; trace_get_zero_list_page(pool->to_zero); if (list_empty(&pool->zero_list)) return NULL; - page = list_first_entry(&pool->zero_list, struct page, lru); + if (!use_numa) { + page = list_first_entry(&pool->zero_list, struct page, lru); + goto exit; + } else { + list_for_each_entry_safe(page, tmp, &pool->zero_list, lru) + if (page_to_nid(page) == nid) + goto exit; + } + return NULL; + +exit: list_del(&page->lru); - pool->to_zero--; - return page; } -static inline struct page *get_page_list_page(struct nvmap_page_pool *pool) +static inline struct page *get_page_list_page(struct nvmap_page_pool *pool, bool use_numa, + int numa_id) { - struct page *page; + struct page *page, *tmp; + int nid = numa_id == NUMA_NO_NODE ? numa_mem_id() : numa_id; trace_get_page_list_page(pool->count); if (list_empty(&pool->page_list)) return NULL; - page = list_first_entry(&pool->page_list, struct page, lru); + if (!use_numa) { + page = list_first_entry(&pool->page_list, struct page, lru); + goto exit; + } else { + list_for_each_entry_safe(page, tmp, &pool->page_list, lru) + if (page_to_nid(page) == nid) + goto exit; + } + return NULL; + +exit: list_del(&page->lru); - pool->count--; - return page; } #ifdef CONFIG_ARM64_4K_PAGES -static inline struct page *get_page_list_page_bp(struct nvmap_page_pool *pool) +static inline struct page *get_page_list_page_bp(struct nvmap_page_pool *pool, bool use_numa, + int numa_id) { - struct page *page; + struct page *page, *tmp; + int nid = numa_id == NUMA_NO_NODE ? numa_mem_id() : numa_id; trace_get_page_list_page_bp(pool->big_page_count); if (list_empty(&pool->page_list_bp)) return NULL; - page = list_first_entry(&pool->page_list_bp, struct page, lru); - list_del(&page->lru); + if (!use_numa) { + page = list_first_entry(&pool->page_list_bp, struct page, lru); + goto exit; + } else { + list_for_each_entry_safe(page, tmp, &pool->page_list_bp, lru) + if (page_to_nid(page) == nid) + goto exit; + } + return NULL; +exit: + list_del(&page->lru); pool->count -= pool->pages_per_big_pg; pool->big_page_count -= pool->pages_per_big_pg; - return page; } #endif /* CONFIG_ARM64_4K_PAGES */ @@ -147,7 +177,7 @@ static void nvmap_pp_do_background_zero_pages(struct nvmap_page_pool *pool) rt_mutex_lock(&pool->lock); for (i = 0; i < PENDING_PAGES_SIZE; i++) { - page = get_zero_list_page(pool); + page = get_zero_list_page(pool, false, 0); if (page == NULL) break; pending_zero_pages[i] = page; @@ -234,14 +264,14 @@ static ulong nvmap_page_pool_free_pages_locked(struct nvmap_page_pool *pool, #ifdef CONFIG_ARM64_4K_PAGES if (use_page_list_bp) - page = get_page_list_page_bp(pool); + page = get_page_list_page_bp(pool, false, 0); else if (use_page_list) #else if (use_page_list) #endif /* CONFIG_ARM64_4K_PAGES */ - page = get_page_list_page(pool); + page = get_page_list_page(pool, false, 0); else - page = get_zero_list_page(pool); + page = get_zero_list_page(pool, false, 0); if (!page) { if (!use_page_list) { @@ -286,7 +316,8 @@ static ulong nvmap_page_pool_free_pages_locked(struct nvmap_page_pool *pool, * array in a linear fashion starting from index 0. */ int nvmap_page_pool_alloc_lots(struct nvmap_page_pool *pool, - struct page **pages, u32 nr) + struct page **pages, u32 nr, + bool use_numa, int numa_id) { u32 ind = 0; u32 non_zero_idx; @@ -301,10 +332,10 @@ int nvmap_page_pool_alloc_lots(struct nvmap_page_pool *pool, struct page *page = NULL; if (!non_zero_cnt) - page = get_page_list_page(pool); + page = get_page_list_page(pool, use_numa, numa_id); if (!page) { - page = get_zero_list_page(pool); + page = get_zero_list_page(pool, use_numa, numa_id); if (!page) break; if (!non_zero_cnt) @@ -336,7 +367,8 @@ int nvmap_page_pool_alloc_lots(struct nvmap_page_pool *pool, #ifdef CONFIG_ARM64_4K_PAGES int nvmap_page_pool_alloc_lots_bp(struct nvmap_page_pool *pool, - struct page **pages, u32 nr) + struct page **pages, u32 nr, + bool use_numa, int numa_id) { u32 ind = 0, nr_pages = nr; struct page *page; @@ -350,7 +382,7 @@ int nvmap_page_pool_alloc_lots_bp(struct nvmap_page_pool *pool, while (nr_pages - ind >= pool->pages_per_big_pg) { int i; - page = get_page_list_page_bp(pool); + page = get_page_list_page_bp(pool, use_numa, numa_id); if (!page) break; diff --git a/drivers/video/tegra/nvmap/nvmap_priv.h b/drivers/video/tegra/nvmap/nvmap_priv.h index 230aa741..aca75a25 100644 --- a/drivers/video/tegra/nvmap/nvmap_priv.h +++ b/drivers/video/tegra/nvmap/nvmap_priv.h @@ -278,6 +278,7 @@ struct nvmap_handle { * waitq to wait on RO dmabuf release completion, if release is already in progress. */ wait_queue_head_t waitq; + int numa_id; }; struct nvmap_handle_info { @@ -345,10 +346,10 @@ int nvmap_page_pool_init(struct nvmap_device *dev); int nvmap_page_pool_fini(struct nvmap_device *dev); struct page *nvmap_page_pool_alloc(struct nvmap_page_pool *pool); int nvmap_page_pool_alloc_lots(struct nvmap_page_pool *pool, - struct page **pages, u32 nr); + struct page **pages, u32 nr, bool use_numa, int numa_id); #ifdef CONFIG_ARM64_4K_PAGES int nvmap_page_pool_alloc_lots_bp(struct nvmap_page_pool *pool, - struct page **pages, u32 nr); + struct page **pages, u32 nr, bool use_numa, int numa_id); #endif /* CONFIG_ARM64_4K_PAGES */ u32 nvmap_page_pool_fill_lots(struct nvmap_page_pool *pool, struct page **pages, u32 nr); diff --git a/include/uapi/linux/nvmap.h b/include/uapi/linux/nvmap.h index a8560e3d..de7d0f47 100644 --- a/include/uapi/linux/nvmap.h +++ b/include/uapi/linux/nvmap.h @@ -3,7 +3,7 @@ * * structure declarations for nvmem and nvmap user-space ioctls * - * Copyright (c) 2009-2022, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2009-2023, NVIDIA CORPORATION. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, @@ -103,6 +103,7 @@ struct nvmap_alloc_handle { __u32 heap_mask; /* heaps to allocate from */ __u32 flags; /* wb/wc/uc/iwb etc. */ __u32 align; /* min alignment necessary */ + __s32 numa_nid; /* NUMA node id */ }; struct nvmap_alloc_ivm_handle {