gpu: nvgpu: Add compression support with added contig memory pool

This is adding compression support for Ampere gpus by
the given contig memory pool.

Bug 3426194

Change-Id: I1c2400094296eb5448fe18f76d021a10c33ef861
Signed-off-by: Dinesh T <dt@nvidia.com>
Reviewed-on: https://git-master.nvidia.com/r/c/linux-nvgpu/+/2673581
Tested-by: mobile promotions <svcmobile_promotions@nvidia.com>
Reviewed-by: mobile promotions <svcmobile_promotions@nvidia.com>

arch/nvgpu-common.yaml

@@ -127,7 +127,8 @@ cbc:
   sources: [ common/cbc/cbc.c,
              include/nvgpu/cbc.h,
              include/nvgpu/gops/cbc.h,
-             include/nvgpu/nvgpu_ivm.h ]
+             include/nvgpu/nvgpu_ivm.h,
+             common/cbc/contig_pool.c ]
 
 regops:
   safe: no

drivers/gpu/nvgpu/Makefile

@@ -57,6 +57,10 @@ ccflags-y += -DCONFIG_NVGPU_DEBUGGER
 ccflags-y += -DCONFIG_NVGPU_ENGINE_RESET
 endif
 
+ifeq ($(CONFIG_NVGPU_IVM_BUILD),y)
+ccflags-y += -DCONFIG_NVGPU_IVM_BUILD
+endif
+
 ccflags-y += -DCONFIG_NVGPU_DETERMINISTIC_CHANNELS
 ccflags-y += -DCONFIG_NVGPU_STATIC_POWERGATE
 ccflags-y += -DCONFIG_NVGPU_ACR_LEGACY
@@ -482,7 +486,9 @@ nvgpu-y += \
 	os/linux/dmabuf_priv.o \
 	os/linux/power_ops.o
 
-nvgpu-$(CONFIG_NVGPU_IVM_BUILD) += os/linux/nvgpu_ivm.o
+nvgpu-$(CONFIG_NVGPU_IVM_BUILD) += \
+	os/linux/nvgpu_ivm.o \
+	common/cbc/contig_pool.o
 
 nvgpu-$(CONFIG_NVGPU_VPR) += os/linux/vpr.o
 

drivers/gpu/nvgpu/Makefile.linux.configs

@@ -270,3 +270,6 @@ endif
 ifeq ($(CONFIG_NVS_PRESENT),y)
 ccflags-y += -DCONFIG_NVS_PRESENT
 endif
+ifeq ($(CONFIG_NVGPU_IVM_BUILD),y)
+ccflags-y += -DCONFIG_NVGPU_IVM_BUILD
+endif

drivers/gpu/nvgpu/Makefile.shared.configs

@@ -61,6 +61,9 @@ ifeq ($(CONFIG_NVGPU_DGPU),1)
 NVGPU_COMMON_CFLAGS		+= -DCONFIG_PCI_MSI
 endif
 
+CONFIG_NVGPU_IVM_BUILD               := 1
+NVGPU_COMMON_CFLAGS                  += -DCONFIG_NVGPU_IVM_BUILD
+
 CONFIG_NVGPU_LOGGING                 := 1
 NVGPU_COMMON_CFLAGS                  += -DCONFIG_NVGPU_LOGGING
 

drivers/gpu/nvgpu/Makefile.sources

@@ -639,6 +639,9 @@ srcs += common/mm/comptags.c \
 	hal/cbc/cbc_gm20b.c \
 	hal/cbc/cbc_gp10b.c \
 	hal/cbc/cbc_gv11b.c
+ifeq ($(CONFIG_NVGPU_IVM_BUILD),1)
+srcs += common/cbc/contig_pool.c
+endif
 endif
 
 ifeq ($(CONFIG_NVGPU_REMAP),1)

drivers/gpu/nvgpu/common/cbc/cbc.c

@@ -30,6 +30,7 @@
 #include <nvgpu/string.h>
 #include <nvgpu/nvgpu_mem.h>
 #include <nvgpu/comptags.h>
+#include <nvgpu/soc.h>
 
 void nvgpu_cbc_remove_support(struct gk20a *g)
 {
@@ -40,7 +41,9 @@ void nvgpu_cbc_remove_support(struct gk20a *g)
 	if (cbc == NULL) {
 		return;
 	}
-
+#ifdef CONFIG_NVGPU_IVM_BUILD
+	nvgpu_cbc_contig_deinit(g);
+#endif
 	if (nvgpu_mem_is_valid(&cbc->compbit_store.mem)) {
 		nvgpu_dma_free(g, &cbc->compbit_store.mem);
 		(void) memset(&cbc->compbit_store, 0,
@@ -66,6 +69,10 @@ int nvgpu_cbc_init_support(struct gk20a *g)
 
 	nvgpu_log_fn(g, " ");
 
+	if (!nvgpu_is_enabled(g, NVGPU_SUPPORT_COMPRESSION)) {
+		return 0;
+	}
+
 	/*
 	 * If cbc == NULL, the device is being powered-on for the first
 	 * time and hence nvgpu_cbc_init_support is not called as part of
@@ -97,11 +104,50 @@ int nvgpu_cbc_init_support(struct gk20a *g)
 	return err;
 }
 
+#ifdef CONFIG_NVGPU_IVM_BUILD
+static int nvgpu_init_cbc_mem(struct gk20a *g, u64 pa, u64 size)
+{
+	u64 nr_pages;
+	int err = 0;
+	struct nvgpu_cbc *cbc = g->cbc;
+
+	nr_pages = size / NVGPU_CPU_PAGE_SIZE;
+	err = nvgpu_mem_create_from_phys(g, &cbc->compbit_store.mem,
+		pa, nr_pages);
+	return err;
+}
+
+static int nvgpu_get_mem_from_contigpool(struct gk20a *g,
+			size_t size,
+			struct nvgpu_mem *mem)
+{
+	struct nvgpu_contig_cbcmempool *contig_pool;
+	u64 pa;
+	int err = 0;
+
+	contig_pool = g->cbc->cbc_contig_mempool;
+	if (contig_pool->size < size) {
+		return -ENOMEM;
+	}
+
+	pa = contig_pool->base_addr;
+	err = nvgpu_init_cbc_mem(g, pa, size);
+	if (err != 0) {
+		return err;
+	}
+
+	return 0;
+
+}
+#endif
+
 int nvgpu_cbc_alloc(struct gk20a *g, size_t compbit_backing_size,
 			bool vidmem_alloc)
 {
 	struct nvgpu_cbc *cbc = g->cbc;
-
+#ifdef CONFIG_NVGPU_IVM_BUILD
+	int err = 0;
+#endif
 	(void)vidmem_alloc;
 
 	if (nvgpu_mem_is_valid(&cbc->compbit_store.mem) != 0) {
@@ -121,12 +167,28 @@ int nvgpu_cbc_alloc(struct gk20a *g, size_t compbit_backing_size,
 		return nvgpu_dma_alloc_vid(g,
 					 compbit_backing_size,
 					 &cbc->compbit_store.mem);
+	}
+#endif
+#ifdef CONFIG_NVGPU_IVM_BUILD
+	if (nvgpu_is_hypervisor_mode(g) && !g->is_virtual &&
+			(g->ops.cbc.use_contig_pool != NULL)) {
+		if (cbc->cbc_contig_mempool == NULL) {
+			err = nvgpu_cbc_contig_init(g);
+			if (err != 0) {
+				nvgpu_err(g, "Contig pool initialization failed");
+				return -ENOMEM;
+			}
+		}
+
+		return nvgpu_get_mem_from_contigpool(g,
+				compbit_backing_size,
+				&cbc->compbit_store.mem);
 	} else
 #endif
 	{
 		return nvgpu_dma_alloc_flags_sys(g,
-					 NVGPU_DMA_PHYSICALLY_ADDRESSED,
-					 compbit_backing_size,
-					 &cbc->compbit_store.mem);
+				NVGPU_DMA_PHYSICALLY_ADDRESSED,
+				compbit_backing_size,
+				&cbc->compbit_store.mem);
 	}
 }

drivers/gpu/nvgpu/common/cbc/contig_pool.c

@@ -0,0 +1,112 @@
+/*
+ * Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <nvgpu/kmem.h>
+#include <nvgpu/enabled.h>
+#include <nvgpu/dt.h>
+#include <nvgpu/gk20a.h>
+#include <nvgpu/nvgpu_ivm.h>
+#include <nvgpu/nvgpu_mem.h>
+#include <nvgpu/soc.h>
+#ifdef __KERNEL__
+#include <linux/tegra-ivc.h>
+#else
+#include <tegra-ivc.h>
+#endif
+#include <nvgpu/nvgpu_sgt.h>
+
+static void nvgpu_init_cbc_contig_pa(struct nvgpu_contig_cbcmempool *contig_pool)
+{
+	contig_pool->base_addr = nvgpu_get_pa_from_ipa(contig_pool->g,
+					contig_pool->cookie->ipa);
+	contig_pool->size = contig_pool->cookie->size;
+}
+
+int nvgpu_cbc_contig_init(struct gk20a *g)
+{
+	struct nvgpu_contig_cbcmempool *contig_pool;
+	u32 mempool_id;
+	int err;
+
+	contig_pool = nvgpu_kzalloc(g, sizeof(*contig_pool));
+	if (!contig_pool) {
+		nvgpu_set_enabled(g, NVGPU_SUPPORT_COMPRESSION, false);
+		return -ENOMEM;
+	}
+
+	contig_pool->g = g;
+	nvgpu_mutex_init(&contig_pool->contigmem_mutex);
+	g->cbc->cbc_contig_mempool = contig_pool;
+	err = nvgpu_dt_read_u32_index(g, "phys_contiguous_mempool",
+			0, &mempool_id);
+	if (err) {
+		nvgpu_err(g, "Reading the contig_mempool from dt failed %d", err);
+		goto fail;
+	}
+
+	contig_pool->cookie = nvgpu_ivm_mempool_reserve(mempool_id);
+	if (contig_pool->cookie == NULL) {
+		nvgpu_err(g,
+			"mempool  %u reserve failed", mempool_id);
+		contig_pool->cookie = NULL;
+		goto fail;
+	}
+
+	contig_pool->cbc_cpuva = nvgpu_ivm_mempool_map(contig_pool->cookie);
+	if (contig_pool->cbc_cpuva == NULL) {
+		nvgpu_err(g, "nvgpu_ivm_mempool_map failed");
+		goto fail;
+	}
+
+	nvgpu_init_cbc_contig_pa(contig_pool);
+
+	return 0;
+fail:
+	nvgpu_cbc_contig_deinit(g);
+	err = -ENOMEM;
+	nvgpu_set_enabled(g, NVGPU_SUPPORT_COMPRESSION, false);
+	return err;
+}
+
+void nvgpu_cbc_contig_deinit(struct gk20a *g)
+{
+	struct nvgpu_contig_cbcmempool *contig_pool;
+	struct nvgpu_mem *mem;
+
+	if ((g->cbc == NULL) || (g->cbc->cbc_contig_mempool == NULL)) {
+		return;
+	}
+
+	contig_pool = g->cbc->cbc_contig_mempool;
+	if (contig_pool->cookie != NULL &&
+			contig_pool->cbc_cpuva != NULL) {
+		nvgpu_ivm_mempool_unmap(contig_pool->cookie,
+				contig_pool->cbc_cpuva);
+	}
+
+	if (contig_pool->cookie) {
+		nvgpu_ivm_mempool_unreserve(contig_pool->cookie);
+	}
+
+	nvgpu_kfree(g, contig_pool);
+	g->cbc->cbc_contig_mempool = NULL;
+	mem = &g->cbc->compbit_store.mem;
+	nvgpu_kfree(g, mem->phys_sgt->sgl);
+	nvgpu_kfree(g, mem->phys_sgt);
+	(void) memset(&g->cbc->compbit_store, 0,
+			sizeof(struct compbit_store_desc));
+}
+

drivers/gpu/nvgpu/common/init/nvgpu_init.c

@@ -938,7 +938,8 @@ int nvgpu_finalize_poweron(struct gk20a *g)
 		NVGPU_INIT_TABLE_ENTRY(&nvgpu_init_boot_clk_or_clk_arb, NO_FLAG),
 		NVGPU_INIT_TABLE_ENTRY(g->ops.therm.init_therm_support, NO_FLAG),
 #ifdef CONFIG_NVGPU_COMPRESSION
-		NVGPU_INIT_TABLE_ENTRY(g->ops.cbc.cbc_init_support, NO_FLAG),
+		NVGPU_INIT_TABLE_ENTRY(g->ops.cbc.cbc_init_support,
+				NVGPU_SUPPORT_COMPRESSION),
 #endif
 		NVGPU_INIT_TABLE_ENTRY(g->ops.chip_init_gpu_characteristics,
 				       NO_FLAG),

drivers/gpu/nvgpu/common/mm/nvgpu_mem.c

@@ -420,3 +420,12 @@ int nvgpu_mem_create_from_phys(struct gk20a *g, struct nvgpu_mem *dest,
 
 	return ret;
 }
+
+u64 nvgpu_mem_phys_get_addr(struct gk20a *g, struct nvgpu_mem *mem)
+{
+	struct nvgpu_mem_sgl *sgl_impl;
+
+	(void)g;
+	sgl_impl = (struct nvgpu_mem_sgl *)(mem->phys_sgt->sgl);
+	return sgl_impl->phys;
+}

drivers/gpu/nvgpu/hal/cbc/cbc_ga10b.c

@@ -168,3 +168,8 @@ int ga10b_cbc_alloc_comptags(struct gk20a *g, struct nvgpu_cbc *cbc)
 
 	return 0;
 }
+
+bool ga10b_cbc_use_contig_pool(struct gk20a *g)
+{
+	return true;
+}

drivers/gpu/nvgpu/hal/cbc/cbc_ga10b.h

@@ -31,6 +31,6 @@ struct gk20a;
 struct nvgpu_cbc;
 
 int ga10b_cbc_alloc_comptags(struct gk20a *g, struct nvgpu_cbc *cbc);
-
+bool ga10b_cbc_use_contig_pool(struct gk20a *g);
 #endif
 #endif /* CBC_GA10B_H */

drivers/gpu/nvgpu/hal/fb/fb_ga10b.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020-2021, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2020-2022, NVIDIA CORPORATION.  All rights reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -65,7 +65,16 @@ void ga10b_fb_cbc_configure(struct gk20a *g, struct nvgpu_cbc *cbc)
 		  fb_mmu_cbc_max_comptagline_f(cbc->max_comptag_lines));
 	nvgpu_writel(g, fb_mmu_cbc_max_r(), cbc_max_rval);
 
-	compbit_store_pa = nvgpu_mem_get_addr(g, &cbc->compbit_store.mem);
+	if (nvgpu_is_hypervisor_mode(g) &&
+			(g->ops.cbc.use_contig_pool != NULL)) {
+		/*
+		 * As the nvgpu_mem in ga10b holds the physical sgt, call
+		 * nvgpu_mem_phys_get_addr to get the physical address.
+		 */
+		compbit_store_pa = nvgpu_mem_phys_get_addr(g, &cbc->compbit_store.mem);
+	} else {
+		compbit_store_pa = nvgpu_mem_get_addr(g, &cbc->compbit_store.mem);
+	}
 	/* must be a multiple of 64KB within allocated memory */
 	compbit_store_base = round_up(compbit_store_pa, SZ_64K);
 	/* Calculate post-divide cbc address */

drivers/gpu/nvgpu/hal/init/hal_ga10b.c

@@ -382,6 +382,7 @@ static const struct gops_cbc ga10b_ops_cbc = {
 	.init = gv11b_cbc_init,
 	.alloc_comptags = ga10b_cbc_alloc_comptags,
 	.ctrl = tu104_cbc_ctrl,
+	.use_contig_pool = ga10b_cbc_use_contig_pool,
 };
 #endif
 
@@ -1944,10 +1945,10 @@ int ga10b_init_hal(struct gk20a *g)
 #endif
 
 #ifdef CONFIG_NVGPU_COMPRESSION
-	if (nvgpu_is_hypervisor_mode(g)) {
-		nvgpu_set_enabled(g, NVGPU_SUPPORT_COMPRESSION, false);
-	} else {
+	if (nvgpu_platform_is_silicon(g)) {
 		nvgpu_set_enabled(g, NVGPU_SUPPORT_COMPRESSION, true);
+	} else {
+		nvgpu_set_enabled(g, NVGPU_SUPPORT_COMPRESSION, false);
 	}
 
 	if (nvgpu_is_enabled(g, NVGPU_SUPPORT_COMPRESSION)) {

drivers/gpu/nvgpu/include/nvgpu/cbc.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2019-2022, NVIDIA CORPORATION.  All rights reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -47,6 +47,24 @@ struct compbit_store_desc {
 	u64 base_hw;
 };
 
+struct nvgpu_contig_cbcmempool {
+	struct gk20a *g;
+	/*
+	 * cookie to hold the information about the IVM.
+	 */
+	struct tegra_hv_ivm_cookie *cookie;
+	/*
+	 * base physical address of the contig pool.
+	 */
+	u64 base_addr;
+	/* size of the contig_pool */
+	u64 size;
+	/* Cpu mapped address for the given pool. */
+	void *cbc_cpuva;
+	/* Mutex to protect the allocation requests. */
+	struct nvgpu_mutex contigmem_mutex;
+};
+
 struct nvgpu_cbc {
 	u32 compbit_backing_size;
 	u32 comptags_per_cacheline;
@@ -54,11 +72,14 @@ struct nvgpu_cbc {
 	u32 max_comptag_lines;
 	struct gk20a_comptag_allocator comp_tags;
 	struct compbit_store_desc compbit_store;
+	struct nvgpu_contig_cbcmempool *cbc_contig_mempool;
 };
 
 int nvgpu_cbc_init_support(struct gk20a *g);
 void nvgpu_cbc_remove_support(struct gk20a *g);
 int nvgpu_cbc_alloc(struct gk20a *g, size_t compbit_backing_size,
 			bool vidmem_alloc);
+int  nvgpu_cbc_contig_init(struct gk20a *g);
+void nvgpu_cbc_contig_deinit(struct gk20a *g);
 #endif
 #endif /* NVGPU_CBC_H */

drivers/gpu/nvgpu/include/nvgpu/gops/cbc.h

@@ -32,6 +32,7 @@ struct gops_cbc {
 	int (*ctrl)(struct gk20a *g, enum nvgpu_cbc_op op,
 			u32 min, u32 max);
 	u32 (*fix_config)(struct gk20a *g, int base);
+	bool (*use_contig_pool)(struct gk20a *g);
 };
 #endif
 

drivers/gpu/nvgpu/include/nvgpu/nvgpu_mem.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-2021, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2022, NVIDIA CORPORATION.  All rights reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -488,4 +488,18 @@ u32 nvgpu_aperture_mask(struct gk20a *g, struct nvgpu_mem *mem,
  */
 u64 nvgpu_mem_iommu_translate(struct gk20a *g, u64 phys);
 
+/**
+ * @brief Get the physical address associated with the physical nvgpu_mem.
+ *
+ * @param[in]  g	Pointer to GPU structure.
+ * @param[in]  mem	Pointer to nvgpu_mem structure holds the physical
+ * 			scatter gather table.
+ *
+ * This fuction should not be used for normal nvgpumem that holds
+ * the sgt of intermediate or iova addresses.
+ *
+ * @return translated physical address.
+ */
+u64 nvgpu_mem_phys_get_addr(struct gk20a *g, struct nvgpu_mem *mem);
+
 #endif /* NVGPU_MEM_H */

drivers/gpu/nvgpu/include/nvgpu/soc.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-2021, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2022, NVIDIA CORPORATION.  All rights reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -142,4 +142,14 @@ static inline int nvgpu_init_soc_vars(struct gk20a *g)
 }
 #endif /* CONFIG_NVGPU_TEGRA_FUSE */
 
+/**
+ * @brief Get the physical address from the given intermediate physical address.
+ *
+ * @param[in]  g	Pointer to GPU structure.
+ * @param[in]  ipa	Intermediate physical address.
+ *
+ * @return translated physical address.
+ */
+u64 nvgpu_get_pa_from_ipa(struct gk20a *g, u64 ipa);
+
 #endif /* NVGPU_SOC_H */

drivers/gpu/nvgpu/os/linux/ioctl_ctrl.c

@@ -397,11 +397,13 @@ static long gk20a_ctrl_ioctl_gpu_characteristics(
 	gpu.bus_type = NVGPU_GPU_BUS_TYPE_AXI; /* always AXI for now */
 
 #ifdef CONFIG_NVGPU_COMPRESSION
-	gpu.compression_page_size = g->ops.fb.compression_page_size(g);
-	gpu.gr_compbit_store_base_hw = g->cbc->compbit_store.base_hw;
-	gpu.gr_gobs_per_comptagline_per_slice =
-		g->cbc->gobs_per_comptagline_per_slice;
-	gpu.cbc_comptags_per_line = g->cbc->comptags_per_cacheline;
+	if (nvgpu_is_enabled(g, NVGPU_SUPPORT_COMPRESSION)) {
+		gpu.compression_page_size = g->ops.fb.compression_page_size(g);
+		gpu.gr_compbit_store_base_hw = g->cbc->compbit_store.base_hw;
+		gpu.gr_gobs_per_comptagline_per_slice =
+			g->cbc->gobs_per_comptagline_per_slice;
+		gpu.cbc_comptags_per_line = g->cbc->comptags_per_cacheline;
+	}
 #endif
 
 	if (!nvgpu_is_enabled(g, NVGPU_SUPPORT_MIG) ||

drivers/gpu/nvgpu/os/linux/soc.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-2021, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2022, NVIDIA CORPORATION.  All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms and conditions of the GNU General Public License,
@@ -130,3 +130,15 @@ int nvgpu_init_soc_vars(struct gk20a *g)
 #endif
 	return 0;
 }
+
+u64 nvgpu_get_pa_from_ipa(struct gk20a *g, u64 ipa)
+{
+	struct device *dev = dev_from_gk20a(g);
+	struct gk20a_platform *platform = gk20a_get_platform(dev);
+	u64 pa_len = 0U;
+
+	if (platform->phys_addr) {
+		return platform->phys_addr(g, ipa, &pa_len);
+	}
+	return ipa;
+}

drivers/gpu/nvgpu/os/posix/soc.c

@@ -63,3 +63,9 @@ int nvgpu_init_soc_vars(struct gk20a *g)
 	(void)g;
 	return 0;
 }
+
+u64 nvgpu_get_pa_from_ipa(struct gk20a *g, u64 ipa)
+{
+	(void)g;
+	return ipa;
+}