gpu: nvgpu: add changes for nvgpu-next

- Add new UAPI IOCTLs.
- Add nvgpu-next gops in fb and gr.
- Initialize and teardown vab during mm_support

Bug 2999621

Change-Id: Icc241f1a234bfee3fd20dc69b42c92e0af6d445c
Signed-off-by: Vedashree Vidwans <vvidwans@nvidia.com>
Signed-off-by: Sami Kiminki <skiminki@nvidia.com>
Reviewed-on: https://git-master.nvidia.com/r/c/linux-nvgpu/+/2447064
Tested-by: mobile promotions <svcmobile_promotions@nvidia.com>
Reviewed-by: mobile promotions <svcmobile_promotions@nvidia.com>

drivers/gpu/nvgpu/common/mm/mm.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-2020, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2017-2021, NVIDIA CORPORATION.  All rights reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -135,6 +135,13 @@ static void nvgpu_remove_mm_support(struct mm_gk20a *mm)
 	nvgpu_dma_free(g, &mm->mmu_wr_mem);
 	nvgpu_dma_free(g, &mm->mmu_rd_mem);
 
+#if defined(CONFIG_NVGPU_HAL_NON_FUSA) && defined(CONFIG_NVGPU_NEXT)
+	if (nvgpu_fb_vab_teardown_hal(g) != 0) {
+		nvgpu_err(g, "failed to teardown VAB");
+	}
+
+#endif
+
 	if (g->ops.mm.mmu_fault.info_mem_destroy != NULL) {
 		g->ops.mm.mmu_fault.info_mem_destroy(g);
 	}
@@ -559,6 +566,12 @@ static int nvgpu_init_mm_setup_sw(struct gk20a *g)
 		}
 	}
 
+#if defined(CONFIG_NVGPU_HAL_NON_FUSA) && defined(CONFIG_NVGPU_NEXT)
+	if (nvgpu_fb_vab_init_hal(g) != 0) {
+		nvgpu_err(g, "failed to init VAB");
+	}
+#endif
+
 	mm->remove_support = nvgpu_remove_mm_support;
 #ifdef CONFIG_NVGPU_DGPU
 	mm->remove_ce_support = nvgpu_remove_mm_ce_support;

drivers/gpu/nvgpu/include/nvgpu/enabled.h

@@ -210,6 +210,7 @@ struct gk20a;
 	DEFINE_FLAG(NVGPU_L2_MAX_WAYS_EVICT_LAST_ENABLED,               \
 			"Set L2 Max Ways Evict Last support"),		\
 	DEFINE_FLAG(NVGPU_CLK_ARB_ENABLED, "CLK_ARB support"),          \
+	DEFINE_FLAG(NVGPU_SUPPORT_VAB_ENABLED, "VAB feature supported"), \
 	DEFINE_FLAG(NVGPU_MAX_ENABLED_BITS, "Marks max number of flags"),
 
 /**

drivers/gpu/nvgpu/include/nvgpu/fb.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2020-2021, NVIDIA CORPORATION.  All rights reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -23,6 +23,10 @@
 #ifndef NVGPU_FB_H
 #define NVGPU_FB_H
 
+#if defined(CONFIG_NVGPU_NON_FUSA) && defined(CONFIG_NVGPU_NEXT)
+#include "include/nvgpu/nvgpu_next_fb.h"
+#endif
+
 /**
  * @brief Initializes the FB unit.
  *

drivers/gpu/nvgpu/include/nvgpu/gops/fb.h

@@ -161,6 +161,10 @@ struct gops_fb_ecc {
 			u32 *uncorrected_error_mask);
 };
 
+#if defined(CONFIG_NVGPU_HAL_NON_FUSA) && defined(CONFIG_NVGPU_NEXT)
+#include "include/nvgpu/nvgpu_next_gops_fb_vab.h"
+#endif
+
 /**
  * common.fb unit hal operations.
  *

drivers/gpu/nvgpu/include/nvgpu/gops/gr.h

@@ -454,7 +454,7 @@ struct gops_gr_intr {
 
 	/** @cond DOXYGEN_SHOULD_SKIP_THIS */
 #if defined(CONFIG_NVGPU_HAL_NON_FUSA) && defined(CONFIG_NVGPU_NEXT)
-#include "include/nvgpu/nvgpu_next_gops_gr.h"
+#include "include/nvgpu/nvgpu_next_gops_gr_intr.h"
 #endif
 	int (*handle_fecs_error)(struct gk20a *g,
 				 struct nvgpu_channel *ch,
@@ -1277,6 +1277,9 @@ struct gops_gr {
 	struct gops_gr_zbc		zbc;
 	struct gops_gr_zcull		zcull;
 #endif /* CONFIG_NVGPU_GRAPHICS */
+#if defined(CONFIG_NVGPU_HAL_NON_FUSA) && defined(CONFIG_NVGPU_NEXT)
+#include "include/nvgpu/nvgpu_next_gops_gr.h"
+#endif
 	/** @endcond */
 };
 

drivers/gpu/nvgpu/include/nvgpu/log_common.h

@@ -74,6 +74,7 @@ enum nvgpu_log_type {
 #define gpu_dbg_mig		BIT(33) /* MIG info */
 #define gpu_dbg_rec		BIT(34) /* Recovery sequence debugging. */
 #define gpu_dbg_zbc		BIT(35) /* Gr ZBC */
+#define gpu_dbg_vab		BIT(36) /* VAB */
 #define gpu_dbg_runlists	BIT(38) /* Runlist related debugging. */
 
 #endif

drivers/gpu/nvgpu/include/nvgpu/mm.h

@@ -166,6 +166,7 @@
 #include <nvgpu/list.h>
 #include <nvgpu/sizes.h>
 #include <nvgpu/mmu_fault.h>
+#include <nvgpu/fb.h>
 
 struct gk20a;
 struct vm_gk20a;
@@ -454,6 +455,10 @@ struct mm_gk20a {
 	struct nvgpu_mem mmu_wr_mem;
 	/** GMMU debug read buffer. */
 	struct nvgpu_mem mmu_rd_mem;
+
+#if defined(CONFIG_NVGPU_NON_FUSA) && defined(CONFIG_NVGPU_NEXT)
+#include "include/nvgpu/nvgpu_next_mm.h"
+#endif
 };
 
 /**

drivers/gpu/nvgpu/os/linux/ioctl_ctrl.c

@@ -295,6 +295,8 @@ static struct nvgpu_flags_mapping flags_mapping[] = {
 		NVGPU_SUPPORT_GET_GR_CONTEXT},
 	{NVGPU_GPU_FLAGS_L2_MAX_WAYS_EVICT_LAST_ENABLED,
 		NVGPU_L2_MAX_WAYS_EVICT_LAST_ENABLED},
+	{NVGPU_GPU_FLAGS_SUPPORT_VAB,
+		NVGPU_SUPPORT_VAB_ENABLED},
 };
 
 static u64 nvgpu_ctrl_ioctl_gpu_characteristics_flags(struct gk20a *g)

drivers/gpu/nvgpu/os/linux/ioctl_prof.c

@@ -30,6 +30,7 @@
 #include <nvgpu/perfbuf.h>
 #include <nvgpu/pm_reservation.h>
 #include <nvgpu/tsg.h>
+#include <nvgpu/fb.h>
 
 #include "platform_gk20a.h"
 #include "os_linux.h"
@@ -38,6 +39,12 @@
 #include "ioctl_tsg.h"
 #include "ioctl.h"
 
+/** @cond DOXYGEN_SHOULD_SKIP_THIS */
+#if defined(CONFIG_NVGPU_NON_FUSA) && defined(CONFIG_NVGPU_NEXT)
+#include "os/linux/nvgpu_next_ioctl_prof.h"
+#endif
+/** @endcond DOXYGEN_SHOULD_SKIP_THIS */
+
 #define NVGPU_PROF_UMD_COPY_WINDOW_SIZE		SZ_4K
 
 struct nvgpu_profiler_object_priv {
@@ -144,6 +151,7 @@ int nvgpu_prof_dev_fops_open(struct inode *inode, struct file *filp)
 	}
 
 	if (!nvgpu_is_enabled(g, NVGPU_SUPPORT_PROFILER_V2_DEVICE)) {
+		nvgpu_err(g, "Profiler V2 not supported");
 		nvgpu_put(g);
 		return -EINVAL;
 	}
@@ -793,8 +801,12 @@ long nvgpu_prof_fops_ioctl(struct file *filp, unsigned int cmd,
 		break;
 
 	default:
+#if defined(CONFIG_NVGPU_NON_FUSA) && defined(CONFIG_NVGPU_NEXT)
+		err = nvgpu_next_prof_fops_ioctl(prof, cmd, (void *)buf);
+#else
 		nvgpu_err(g, "unrecognized profiler ioctl cmd: 0x%x", cmd);
 		err = -ENOTTY;
+#endif
 		break;
 	}
 

include/uapi/linux/nvgpu-ctrl.h

@@ -188,6 +188,8 @@ struct nvgpu_gpu_zbc_query_table_args {
 #define NVGPU_GPU_FLAGS_SUPPORT_BUFFER_METADATA		(1ULL << 50)
 /* Flag to indicate whether configuring L2_MAXEVICTLAST_WAYS is supported */
 #define NVGPU_GPU_FLAGS_L2_MAX_WAYS_EVICT_LAST_ENABLED	(1ULL << 51)
+/* Vidmem access bits feature is supported */
+#define NVGPU_GPU_FLAGS_SUPPORT_VAB		(1ULL << 52)
 /* SM LRF ECC is enabled */
 #define NVGPU_GPU_FLAGS_ECC_ENABLED_SM_LRF	(1ULL << 60)
 /* SM SHM ECC is enabled */

include/uapi/linux/nvgpu.h

@@ -708,6 +708,50 @@ struct nvgpu_profiler_exec_reg_ops_args {
 	__u32 reserved[3];
 };
 
+struct nvgpu_profiler_vab_range_checker {
+
+	/*
+	 * in: starting physical address. Must be aligned by
+	 *     1 << (granularity_shift + bitmask_size_shift) where
+	 *     bitmask_size_shift is a HW specific constant.
+	 */
+	__u64 start_phys_addr;
+
+	/* in: log2 of coverage granularity per bit */
+	__u8  granularity_shift;
+
+	__u8  reserved[7];
+};
+
+/* Range checkers track all accesses (read and write) */
+#define NVGPU_PROFILER_VAB_RANGE_CHECKER_MODE_ACCESS  1U
+/* Range checkers track writes (writes and read-modify-writes) */
+#define NVGPU_PROFILER_VAB_RANGE_CHECKER_MODE_DIRTY   2U
+
+struct nvgpu_profiler_vab_reserve_args {
+
+	/* in: range checker mode */
+	__u8 vab_mode;
+
+	__u8 reserved[3];
+
+	/* in: number of range checkers, must match with the HW */
+	__u32 num_range_checkers;
+
+	/*
+	 * in: range checker parameters. Pointer to array of
+	 *     nvgpu_profiler_vab_range_checker elements
+	 */
+	__u64 range_checkers_ptr;
+};
+
+struct nvgpu_profiler_vab_flush_state_args {
+	__u64 buffer_ptr;	/* in: usermode pointer to receive the
+				 * VAB state buffer */
+	__u64 buffer_size;	/* in: VAB buffer size. Must match
+				 * with the hardware VAB state size */
+};
+
 #define NVGPU_PROFILER_IOCTL_BIND_CONTEXT \
 	_IOW(NVGPU_PROFILER_IOCTL_MAGIC, 1, struct nvgpu_profiler_bind_context_args)
 #define NVGPU_PROFILER_IOCTL_RESERVE_PM_RESOURCE \
@@ -728,10 +772,16 @@ struct nvgpu_profiler_exec_reg_ops_args {
 	_IOWR(NVGPU_PROFILER_IOCTL_MAGIC, 9, struct nvgpu_profiler_exec_reg_ops_args)
 #define NVGPU_PROFILER_IOCTL_UNBIND_CONTEXT \
 	_IO(NVGPU_PROFILER_IOCTL_MAGIC, 10)
+#define NVGPU_PROFILER_IOCTL_VAB_RESERVE \
+	_IOW(NVGPU_PROFILER_IOCTL_MAGIC, 11, struct nvgpu_profiler_vab_reserve_args)
+#define NVGPU_PROFILER_IOCTL_VAB_RELEASE \
+	_IO(NVGPU_PROFILER_IOCTL_MAGIC, 12)
+#define NVGPU_PROFILER_IOCTL_VAB_FLUSH_STATE \
+	_IOW(NVGPU_PROFILER_IOCTL_MAGIC, 13, struct nvgpu_profiler_vab_flush_state_args)
 #define NVGPU_PROFILER_IOCTL_MAX_ARG_SIZE	\
 		sizeof(struct nvgpu_profiler_alloc_pma_stream_args)
 #define NVGPU_PROFILER_IOCTL_LAST		\
-	_IOC_NR(NVGPU_PROFILER_IOCTL_UNBIND_CONTEXT)
+	_IOC_NR(NVGPU_PROFILER_IOCTL_VAB_FLUSH_STATE)
 
 
 /*