gpu: nvgpu: Enable raw mode for compression

In raw addressing mode of CBC backing storage, comptaglines are not
required to be allocated or need to programmed in the ptes. Introduce a
flag to detect if the hardware supports raw mode and use that to skip
all the comptagline allocations and respective page table programming.

JIRA NVGPU-9717

Change-Id: I0a16881fc3e897c3c408b30d1835f30564649dad
Signed-off-by: Prathap Kumar Valsan <prathapk@nvidia.com>
Reviewed-on: https://git-master.nvidia.com/r/c/linux-nvgpu/+/2908278
Tested-by: mobile promotions <svcmobile_promotions@nvidia.com>
Reviewed-by: mobile promotions <svcmobile_promotions@nvidia.com>

drivers/gpu/nvgpu/common/mm/gmmu/page_table.c

@@ -1186,6 +1186,7 @@ u64 nvgpu_gmmu_map_locked(struct vm_gk20a *vm,
 		.aperture  = aperture,
 	};
 #ifdef CONFIG_NVGPU_COMPRESSION
+	if (!g->cbc_use_raw_mode) {
 		u64 ctag_granularity = g->ops.fb.compression_page_size(g);
 
 		attrs.ctag      = (u64)ctag_offset * ctag_granularity;
@@ -1203,6 +1204,7 @@ u64 nvgpu_gmmu_map_locked(struct vm_gk20a *vm,
 		attrs.cbc_comptagline_mode =
 			g->ops.fb.is_comptagline_mode_enabled != NULL ?
 			g->ops.fb.is_comptagline_mode_enabled(g) : true;
+	}
 #endif
 
 	attrs.l3_alloc  = ((flags & NVGPU_VM_MAP_L3_ALLOC)  != 0U);

drivers/gpu/nvgpu/common/mm/gmmu/pte.c

@@ -72,6 +72,7 @@ void nvgpu_pte_dbg_print(struct gk20a *g,
 	const char *aperture_str = nvgpu_aperture_str(attrs->aperture);
 	const char *perm_str = nvgpu_gmmu_perm_str(attrs->rw_flag);
 #ifdef CONFIG_NVGPU_COMPRESSION
+	if (!g->cbc_use_raw_mode) {
 		u64 ctag_tmp = attrs->ctag;
 		u32 str_len = 0U;
 		u32 ctag_num = 0U;
@@ -90,6 +91,7 @@ void nvgpu_pte_dbg_print(struct gk20a *g,
 		str_len = (u32)strlen(ctag_str);
 		(void)nvgpu_strnadd_u32(ctag_str + str_len, ctag_num,
 					nvgpu_safe_sub_u32(31U, str_len), 16U);
+	}
 #endif
 	(void)map_attrs_to_str(attrs_str, attrs);
 	pte_dbg(g, attrs,

drivers/gpu/nvgpu/common/mm/vm.c

@@ -1187,6 +1187,17 @@ static int nvgpu_vm_do_map(struct vm_gk20a *vm,
 	(void)os_buf;
 	(void)flags;
 #ifdef CONFIG_NVGPU_COMPRESSION
+	if (g->cbc_use_raw_mode) {
+		if (binfo_ptr->compr_kind != NVGPU_KIND_INVALID) {
+			pte_kind = (u8)binfo_ptr->compr_kind;
+		} else if ((binfo_ptr->incompr_kind >= 0)) {
+			pte_kind = (u8)binfo_ptr->incompr_kind;
+		} else {
+			err = -EINVAL;
+			goto ret_err;
+		}
+		goto kind_done;
+	}
 	err = nvgpu_vm_compute_compression(vm, binfo_ptr);
 	if (err != 0) {
 		nvgpu_err(g, "failure setting up compression");
@@ -1305,6 +1316,7 @@ static int nvgpu_vm_do_map(struct vm_gk20a *vm,
 		goto ret_err;
 	}
 
+kind_done:
 #ifdef CONFIG_NVGPU_COMPRESSION
 	if (clear_ctags) {
 		clear_ctags = gk20a_comptags_start_clear(os_buf);

drivers/gpu/nvgpu/common/mm/vm_remap.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-2022, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2023, NVIDIA CORPORATION.  All rights reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -202,6 +202,7 @@ static int nvgpu_vm_remap_validate_map(struct vm_gk20a *vm,
 			struct nvgpu_vm_remap_os_buffer *remap_os_buf)
 {
 	u64 page_size = nvgpu_vm_remap_page_size(op);
+	struct gk20a *g = gk20a_from_vm(vm);
 	u64 map_offset;
 	u64 map_size;
 	u64 os_buf_size;
@@ -216,9 +217,9 @@ static int nvgpu_vm_remap_validate_map(struct vm_gk20a *vm,
 	}
 
 #ifdef CONFIG_NVGPU_COMPRESSION
-	if (op->compr_kind != NVGPU_KIND_INVALID) {
+	if (op->compr_kind != NVGPU_KIND_INVALID &&
+	    !g->cbc_use_raw_mode) {
 
-		struct gk20a *g = gk20a_from_vm(vm);
 		struct gk20a_comptags comptags = { 0 };
 
 		/*
@@ -360,6 +361,10 @@ static u64 nvgpu_vm_remap_get_ctag_offset(struct vm_gk20a *vm,
 					page_size);
 	u64 compression_page_size;
 
+	if (g->cbc_use_raw_mode) {
+		return 0;
+	}
+
 	gk20a_get_comptags(os_buf, &comptags);
 
 	if (comptags.lines != 0) {

drivers/gpu/nvgpu/include/nvgpu/gk20a.h

@@ -752,6 +752,13 @@ struct gk20a {
 
 	u32 max_comptag_mem; /* max memory size (MB) for comptag */
 	struct nvgpu_cbc *cbc;
+	/*
+	 * In raw mode, L2 calculates the CBC backing storage address from
+	 * physical address of the compressible surface. The comptag lines are
+	 * not used in the calculation, so nvgpu doesn't need to allocate comptag
+	 * lines in the pagetable.
+	 */
+	bool cbc_use_raw_mode;
 #endif
 
 #ifdef CONFIG_NVGPU_NON_FUSA

drivers/gpu/nvgpu/os/linux/comptags.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-2022, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2023, NVIDIA CORPORATION.  All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms and conditions of the GNU General Public License,
@@ -51,6 +51,16 @@ int gk20a_alloc_comptags(struct gk20a *g, struct nvgpu_os_buffer *buf,
 	u32 lines = 0;
 	int err;
 
+	/*
+	 * In raw mode, raw address is used by the hardware to map the
+	 * compressible memory address to CBC address, comptaglines are never
+	 * used.
+	 */
+	if (g->cbc_use_raw_mode) {
+		nvgpu_err(g, "comptags should not be allocated in raw mode\n");
+		return -EINVAL;
+	}
+
 	ctag_granularity = g->ops.fb.compression_page_size(g);
 	lines = DIV_ROUND_UP_ULL(buf->dmabuf->size, ctag_granularity);
 

drivers/gpu/nvgpu/os/linux/ioctl_ctrl.c

@@ -428,11 +428,14 @@ static long gk20a_ctrl_ioctl_gpu_characteristics(
 #ifdef CONFIG_NVGPU_COMPRESSION
 	if (nvgpu_is_enabled(g, NVGPU_SUPPORT_COMPRESSION)) {
 		gpu.compression_page_size = g->ops.fb.compression_page_size(g);
+
+		if (!g->cbc_use_raw_mode) {
 			gpu.gr_compbit_store_base_hw = g->cbc->compbit_store.base_hw;
 			gpu.gr_gobs_per_comptagline_per_slice =
 				g->cbc->gobs_per_comptagline_per_slice;
 			gpu.cbc_comptags_per_line = g->cbc->comptags_per_cacheline;
 		}
+	}
 #endif
 
 	if (!nvgpu_is_enabled(g, NVGPU_SUPPORT_MIG) ||
@@ -2256,6 +2259,17 @@ static int nvgpu_handle_comptags_control(struct gk20a *g,
 	os_buf.dmabuf = dmabuf;
 	os_buf.dev = dev_from_gk20a(g);
 
+	/*
+	 * In raw mode, comptaglines are not used to map the compressible memory
+	 * address to the CBC address, instead raw address is used by the
+	 * hardrdware to offset to the CBC address. Mark comptags as enabled and
+	 * return. In the comptagline mode, comptags will be marked as enabled
+	 * once the comptags are successfuly allocated in the
+	 * gk20a_alloc_comptags().
+	 */
+	if (g->cbc_use_raw_mode) {
+		priv->comptags.enabled = true;
+	} else {
 		err = gk20a_alloc_comptags(g, &os_buf, &g->cbc->comp_tags);
 		if (err != 0) {
 			if (comptags_alloc_control ==
@@ -2268,6 +2282,7 @@ static int nvgpu_handle_comptags_control(struct gk20a *g,
 				err = 0;
 			}
 		}
+	}
 
 	return err;
 }

drivers/gpu/nvgpu/os/linux/vm.c

@@ -524,7 +524,8 @@ int nvgpu_vm_mapping_modify(struct vm_gk20a *vm,
 	 * If we support compression and there's a compressible kind, use it.
 	 */
 #ifdef CONFIG_NVGPU_COMPRESSION
-	if (mapped_buffer->ctag_offset != 0) {
+	if (mapped_buffer->ctag_offset != 0 ||
+	    g->cbc_use_raw_mode) {
 		if (compr_kind == NV_KIND_INVALID) {
 			kind = incompr_kind;
 		} else {
@@ -549,6 +550,7 @@ int nvgpu_vm_mapping_modify(struct vm_gk20a *vm,
 	}
 
 #ifdef CONFIG_NVGPU_COMPRESSION
+	if (!g->cbc_use_raw_mode) {
 		ctag_offset = mapped_buffer->ctag_offset;
 
 		compression_page_size = g->ops.fb.compression_page_size(g);
@@ -556,6 +558,7 @@ int nvgpu_vm_mapping_modify(struct vm_gk20a *vm,
 
 		ctag_offset += (u32)(buffer_offset >>
 				     nvgpu_ilog2(compression_page_size));
+	}
 #endif
 
 	if (g->ops.mm.gmmu.map(vm,

drivers/gpu/nvgpu/os/linux/vm_remap.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-2022, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2023, NVIDIA CORPORATION.  All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms and conditions of the GNU General Public License,
@@ -127,6 +127,7 @@ void nvgpu_vm_remap_os_buf_put(struct vm_gk20a *vm,
 		remap_os_buf->os_priv.attachment, remap_os_buf->os_priv.sgt);
 
 #ifdef CONFIG_NVGPU_COMPRESSION
+	if (!g->cbc_use_raw_mode) {
 		gk20a_get_comptags(&remap_os_buf->os_buf, &comptags);
 
 		/*
@@ -142,6 +143,7 @@ void nvgpu_vm_remap_os_buf_put(struct vm_gk20a *vm,
 				return;
 			}
 		}
+	}
 #endif
 
 	nvgpu_sgt_free(g, remap_os_buf->nv_sgt);