video: tegra: nvmap: Add NUMA support in nvmap

In NUMA system, user can request nvmap to allocate from any particular
node using NvRmMem APIs. Add code to request allocation from such
requested node. While allocating pages from page pool, check if those
pages belong to requested NUMA node. In some cases, we may not need to
use NUMA support e.g. in color pages allocation code, we don't need to
use NUMA, as page coloring is needed only for t19x chips.

JIRA TMM-5287
Bug 4043165

Change-Id: If8043278122068773a802d83723d287d32914e14
Signed-off-by: Ketan Patil <ketanp@nvidia.com>
Reviewed-on: https://git-master.nvidia.com/r/c/linux-nv-oot/+/2889707
Reviewed-by: svc-mobile-coverity <svc-mobile-coverity@nvidia.com>
Reviewed-by: svc-mobile-cert <svc-mobile-cert@nvidia.com>
Reviewed-by: Sachin Nikam <snikam@nvidia.com>
GVS: Gerrit_Virtual_Submit <buildbot_gerritrpt@nvidia.com>

drivers/video/tegra/nvmap/nvmap_alloc.c

@@ -71,13 +71,16 @@ void nvmap_altfree(void *ptr, size_t len)
 		kfree(ptr);
 }
 
-static struct page *nvmap_alloc_pages_exact(gfp_t gfp, size_t size)
+static struct page *nvmap_alloc_pages_exact(gfp_t gfp, size_t size, bool use_numa, int numa_id)
 {
 	struct page *page, *p, *e;
 	unsigned int order;
 
 	order = get_order(size);
-	page = alloc_pages(gfp, order);
+	if (!use_numa)
+		page = alloc_pages(gfp, order);
+	else
+		page = alloc_pages_node(numa_id, gfp, order);
 
 	if (!page)
 		return NULL;
@@ -243,11 +246,12 @@ static struct color_list *init_color_list(struct nvmap_alloc_state *state,
 
 #ifdef NVMAP_CONFIG_PAGE_POOLS
 	/* Allocated page from nvmap page pool if possible */
-	page_index = nvmap_page_pool_alloc_lots(&nvmap_dev->pool, list->pages, nr_pages);
+	page_index = nvmap_page_pool_alloc_lots(&nvmap_dev->pool, list->pages, nr_pages,
+				false, 0);
 #endif
 	/* Fall back to general page allocator */
 	for (i = page_index; i < nr_pages; i++) {
-		list->pages[i] = nvmap_alloc_pages_exact(gfp, PAGE_SIZE);
+		list->pages[i] = nvmap_alloc_pages_exact(gfp, PAGE_SIZE, false, 0);
 		if (!list->pages[i])
 			goto fail;
 	}
@@ -519,7 +523,7 @@ static int handle_page_alloc(struct nvmap_client *client,
 
 	if (contiguous) {
 		struct page *page;
-		page = nvmap_alloc_pages_exact(gfp, size);
+		page = nvmap_alloc_pages_exact(gfp, size, true, h->numa_id);
 		if (!page)
 			goto fail;
 
@@ -531,7 +535,7 @@ static int handle_page_alloc(struct nvmap_client *client,
 #ifdef NVMAP_CONFIG_PAGE_POOLS
 		/* Get as many big pages from the pool as possible. */
 		page_index = nvmap_page_pool_alloc_lots_bp(&nvmap_dev->pool, pages,
-								 nr_page);
+							nr_page, true, h->numa_id);
 		pages_per_big_pg = nvmap_dev->pool.pages_per_big_pg;
 #endif
 		/* Try to allocate big pages from page allocator */
@@ -547,7 +551,7 @@ static int handle_page_alloc(struct nvmap_client *client,
 			gfp_t gfp_no_reclaim = (gfp | __GFP_NOMEMALLOC) & ~__GFP_RECLAIM;
 
 			page = nvmap_alloc_pages_exact(gfp_no_reclaim,
-					pages_per_big_pg << PAGE_SHIFT);
+					pages_per_big_pg << PAGE_SHIFT, true, h->numa_id);
 			if (!page)
 				break;
 
@@ -562,17 +566,21 @@ static int handle_page_alloc(struct nvmap_client *client,
 			/* Get as many pages from the pool as possible. */
 			page_index += nvmap_page_pool_alloc_lots(
 				      &nvmap_dev->pool, &pages[page_index],
-				      nr_page - page_index);
+				      nr_page - page_index, true, h->numa_id);
 #endif
 			allocated = page_index;
 #if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 13, 0)
-			if (page_index < nr_page)
+			if (page_index < nr_page) {
-				allocated = __alloc_pages_bulk(gfp, numa_mem_id(), NULL,
+				int nid = h->numa_id == NUMA_NO_NODE ? numa_mem_id() : h->numa_id;
+
+				allocated = __alloc_pages_bulk(gfp, nid, NULL,
 						nr_page, NULL, pages);
+			}
 #endif
 			for (i = allocated; i < nr_page; i++) {
-				pages[i] = nvmap_alloc_pages_exact(gfp,
+				pages[i] = nvmap_alloc_pages_exact(gfp, PAGE_SIZE,
-								   PAGE_SIZE);
+								   true, h->numa_id);
+
 				if (!pages[i])
 					goto fail;
 			}

drivers/video/tegra/nvmap/nvmap_ioctl.c

@@ -203,6 +203,11 @@ int nvmap_ioctl_alloc(struct file *filp, void __user *arg)
 	if (!op.handle)
 		return -EINVAL;
 
+	if (op.numa_nid > MAX_NUMNODES || (op.numa_nid != NUMA_NO_NODE && op.numa_nid < 0)) {
+		pr_err("numa id:%d is invalid\n", op.numa_nid);
+		return -EINVAL;
+	}
+
 	handle = nvmap_handle_get_from_id(client, op.handle);
 	if (IS_ERR_OR_NULL(handle))
 		return -EINVAL;
@@ -220,6 +225,7 @@ int nvmap_ioctl_alloc(struct file *filp, void __user *arg)
 		return -ENOMEM;
 	}
 
+	handle->numa_id = op.numa_nid;
 	/* user-space handles are aligned to page boundaries, to prevent
 	 * data leakage. */
 	op.align = max_t(size_t, op.align, page_sz);

drivers/video/tegra/nvmap/nvmap_pp.c

@@ -3,7 +3,7 @@
  *
  * Manage page pools to speed up page allocation.
  *
- * Copyright (c) 2009-2022, NVIDIA CORPORATION. All rights reserved.
+ * Copyright (c) 2009-2023, NVIDIA CORPORATION. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms and conditions of the GNU General Public License,
@@ -63,56 +63,86 @@ static inline void __pp_dbg_var_add(u64 *dbg_var, u32 nr)
 static int __nvmap_page_pool_fill_lots_locked(struct nvmap_page_pool *pool,
 				       struct page **pages, u32 nr);
 
-static inline struct page *get_zero_list_page(struct nvmap_page_pool *pool)
+static inline struct page *get_zero_list_page(struct nvmap_page_pool *pool, bool use_numa,
+					int numa_id)
 {
-	struct page *page;
+	struct page *page, *tmp;
+	int nid = numa_id == NUMA_NO_NODE ? numa_mem_id() : numa_id;
 
 	trace_get_zero_list_page(pool->to_zero);
 
 	if (list_empty(&pool->zero_list))
 		return NULL;
 
-	page = list_first_entry(&pool->zero_list, struct page, lru);
+	if (!use_numa) {
+		page = list_first_entry(&pool->zero_list, struct page, lru);
+		goto exit;
+	} else {
+		list_for_each_entry_safe(page, tmp, &pool->zero_list, lru)
+			if (page_to_nid(page) == nid)
+				goto exit;
+	}
+	return NULL;
+
+exit:
 	list_del(&page->lru);
-
 	pool->to_zero--;
-
 	return page;
 }
 
-static inline struct page *get_page_list_page(struct nvmap_page_pool *pool)
+static inline struct page *get_page_list_page(struct nvmap_page_pool *pool, bool use_numa,
+					int numa_id)
 {
-	struct page *page;
+	struct page *page, *tmp;
+	int nid = numa_id == NUMA_NO_NODE ? numa_mem_id() : numa_id;
 
 	trace_get_page_list_page(pool->count);
 
 	if (list_empty(&pool->page_list))
 		return NULL;
 
-	page = list_first_entry(&pool->page_list, struct page, lru);
+	if (!use_numa) {
+		page = list_first_entry(&pool->page_list, struct page, lru);
+		goto exit;
+	} else {
+		list_for_each_entry_safe(page, tmp, &pool->page_list, lru)
+			if (page_to_nid(page) == nid)
+				goto exit;
+	}
+	return NULL;
+
+exit:
 	list_del(&page->lru);
-
 	pool->count--;
-
 	return page;
 }
 
 #ifdef CONFIG_ARM64_4K_PAGES
-static inline struct page *get_page_list_page_bp(struct nvmap_page_pool *pool)
+static inline struct page *get_page_list_page_bp(struct nvmap_page_pool *pool, bool use_numa,
+					int numa_id)
 {
-	struct page *page;
+	struct page *page, *tmp;
+	int nid = numa_id == NUMA_NO_NODE ? numa_mem_id() : numa_id;
 
 	trace_get_page_list_page_bp(pool->big_page_count);
 
 	if (list_empty(&pool->page_list_bp))
 		return NULL;
 
-	page = list_first_entry(&pool->page_list_bp, struct page, lru);
+	if (!use_numa) {
-	list_del(&page->lru);
+		page = list_first_entry(&pool->page_list_bp, struct page, lru);
+		goto exit;
+	} else {
+		list_for_each_entry_safe(page, tmp, &pool->page_list_bp, lru)
+			if (page_to_nid(page) == nid)
+				goto exit;
+	}
+	return NULL;
 
+exit:
+	list_del(&page->lru);
 	pool->count -= pool->pages_per_big_pg;
 	pool->big_page_count -= pool->pages_per_big_pg;
-
 	return page;
 }
 #endif /* CONFIG_ARM64_4K_PAGES */
@@ -147,7 +177,7 @@ static void nvmap_pp_do_background_zero_pages(struct nvmap_page_pool *pool)
 
 	rt_mutex_lock(&pool->lock);
 	for (i = 0; i < PENDING_PAGES_SIZE; i++) {
-		page = get_zero_list_page(pool);
+		page = get_zero_list_page(pool, false, 0);
 		if (page == NULL)
 			break;
 		pending_zero_pages[i] = page;
@@ -234,14 +264,14 @@ static ulong nvmap_page_pool_free_pages_locked(struct nvmap_page_pool *pool,
 
 #ifdef CONFIG_ARM64_4K_PAGES
 		if (use_page_list_bp)
-			page = get_page_list_page_bp(pool);
+			page = get_page_list_page_bp(pool, false, 0);
 		else if (use_page_list)
 #else
 		if (use_page_list)
 #endif /* CONFIG_ARM64_4K_PAGES */
-			page = get_page_list_page(pool);
+			page = get_page_list_page(pool, false, 0);
 		else
-			page = get_zero_list_page(pool);
+			page = get_zero_list_page(pool, false, 0);
 
 		if (!page) {
 			if (!use_page_list) {
@@ -286,7 +316,8 @@ static ulong nvmap_page_pool_free_pages_locked(struct nvmap_page_pool *pool,
  * array in a linear fashion starting from index 0.
  */
 int nvmap_page_pool_alloc_lots(struct nvmap_page_pool *pool,
-				struct page **pages, u32 nr)
+				struct page **pages, u32 nr,
+				bool use_numa, int numa_id)
 {
 	u32 ind = 0;
 	u32 non_zero_idx;
@@ -301,10 +332,10 @@ int nvmap_page_pool_alloc_lots(struct nvmap_page_pool *pool,
 		struct page *page = NULL;
 
 		if (!non_zero_cnt)
-			page = get_page_list_page(pool);
+			page = get_page_list_page(pool, use_numa, numa_id);
 
 		if (!page) {
-			page = get_zero_list_page(pool);
+			page = get_zero_list_page(pool, use_numa, numa_id);
 			if (!page)
 				break;
 			if (!non_zero_cnt)
@@ -336,7 +367,8 @@ int nvmap_page_pool_alloc_lots(struct nvmap_page_pool *pool,
 
 #ifdef CONFIG_ARM64_4K_PAGES
 int nvmap_page_pool_alloc_lots_bp(struct nvmap_page_pool *pool,
-				struct page **pages, u32 nr)
+				struct page **pages, u32 nr,
+				bool use_numa, int numa_id)
 {
 	u32 ind = 0, nr_pages = nr;
 	struct page *page;
@@ -350,7 +382,7 @@ int nvmap_page_pool_alloc_lots_bp(struct nvmap_page_pool *pool,
 	while (nr_pages - ind >= pool->pages_per_big_pg) {
 		int i;
 
-		page = get_page_list_page_bp(pool);
+		page = get_page_list_page_bp(pool, use_numa, numa_id);
 		if (!page)
 			break;
 

drivers/video/tegra/nvmap/nvmap_priv.h

@@ -278,6 +278,7 @@ struct nvmap_handle {
 	 * waitq to wait on RO dmabuf release completion, if release is already in progress.
 	 */
 	wait_queue_head_t waitq;
+	int numa_id;
 };
 
 struct nvmap_handle_info {
@@ -345,10 +346,10 @@ int nvmap_page_pool_init(struct nvmap_device *dev);
 int nvmap_page_pool_fini(struct nvmap_device *dev);
 struct page *nvmap_page_pool_alloc(struct nvmap_page_pool *pool);
 int nvmap_page_pool_alloc_lots(struct nvmap_page_pool *pool,
-					struct page **pages, u32 nr);
+		struct page **pages, u32 nr, bool use_numa, int numa_id);
 #ifdef CONFIG_ARM64_4K_PAGES
 int nvmap_page_pool_alloc_lots_bp(struct nvmap_page_pool *pool,
-					struct page **pages, u32 nr);
+		struct page **pages, u32 nr, bool use_numa, int numa_id);
 #endif /* CONFIG_ARM64_4K_PAGES */
 u32 nvmap_page_pool_fill_lots(struct nvmap_page_pool *pool,
 				       struct page **pages, u32 nr);

include/uapi/linux/nvmap.h

@@ -3,7 +3,7 @@
  *
  * structure declarations for nvmem and nvmap user-space ioctls
  *
- * Copyright (c) 2009-2022, NVIDIA CORPORATION. All rights reserved.
+ * Copyright (c) 2009-2023, NVIDIA CORPORATION. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms and conditions of the GNU General Public License,
@@ -103,6 +103,7 @@ struct nvmap_alloc_handle {
 	__u32 heap_mask;	/* heaps to allocate from */
 	__u32 flags;		/* wb/wc/uc/iwb etc. */
 	__u32 align;		/* min alignment necessary */
+	__s32 numa_nid;		/* NUMA node id */
 };
 
 struct nvmap_alloc_ivm_handle {