gpu: nvgpu: add support for gsp stress test

Add debugfs entries to support GSP stress test and other
functionalities to enable the test.

JIRA CORERM-3382

Change-Id: Iab20fcfe78807e76e91c64716502a2f036ed4d18
Signed-off-by: ajeshkv <akv@nvidia.com>
Reviewed-on: https://git-master.nvidia.com/r/c/linux-nvgpu/+/2589390
Reviewed-by: Amit Pabalkar <apabalkar@nvidia.com>
Reviewed-by: mobile promotions <svcmobile_promotions@nvidia.com>
GVS: Gerrit_Virtual_Submit
Tested-by: mobile promotions <svcmobile_promotions@nvidia.com>

arch/nvgpu-common.yaml

@@ -309,7 +309,9 @@ gsp:
              common/gsp/gsp_priv.h,
              common/gsp/gsp_bootstrap.c,
              common/gsp/gsp_bootstrap.h,
-             include/nvgpu/gsp.h ]
+             common/gsp/gsp_test.c,
+             include/nvgpu/gsp.h,
+             include/nvgpu/gsp/gsp_test.h ]
 
 engine_queues:
   owner: Sagar K

arch/nvgpu-linux.yaml

@@ -86,6 +86,8 @@ debug:
              os/linux/debug_s_param.h,
              os/linux/debug_volt.c,
              os/linux/debug_volt.h,
+             os/linux/debug_gsp.c,
+             os/linux/debug_gsp.h,
              os/linux/swprofile_debugfs.c,
              os/linux/swprofile_debugfs.h,
              os/linux/fecs_trace_linux.c,

drivers/gpu/nvgpu/Makefile

@@ -95,6 +95,10 @@ nvgpu-y += \
 	hal/rc/rc_gv11b.o
 endif
 
+ifeq ($(CONFIG_NVGPU_GSP_STRESS_TEST),y)
+ccflags-y += -DCONFIG_NVGPU_GSP_STRESS_TEST
+endif
+
 obj-$(CONFIG_GK20A) := nvgpu.o
 
 # OS independent parts of nvgpu. The work to collect files here
@@ -409,6 +413,11 @@ nvgpu-$(CONFIG_NVGPU_GSP_SCHEDULER) += \
 	common/gsp/gsp_bootstrap.o
 endif
 
+ifeq ($(CONFIG_NVGPU_GSP_STRESS_TEST),y)
+nvgpu-$(CONFIG_NVGPU_GSP_STRESS_TEST) += \
+	common/gsp/gsp_test.o
+endif
+
 # Linux specific parts of nvgpu.
 nvgpu-y += \
 	os/linux/os_ops.o \
@@ -465,7 +474,8 @@ nvgpu-$(CONFIG_DEBUG_FS) += \
 	os/linux/debug_ltc.o \
 	os/linux/debug_volt.o \
 	os/linux/debug_s_param.o \
-	os/linux/swprofile_debugfs.o
+	os/linux/swprofile_debugfs.o \
+	os/linux/debug_gsp.o
 
 nvgpu-$(CONFIG_NVGPU_LOGGING) += os/linux/log.o
 

drivers/gpu/nvgpu/Makefile.linux.configs

@@ -49,6 +49,11 @@ CONFIG_NVGPU_REMAP := y
 # Enable gsp scheduler support
 CONFIG_NVGPU_GSP_SCHEDULER := y
 
+#Enable stress test for GSP
+ifeq ($(CONFIG_NVGPU_GSP_SCHEDULER),y)
+CONFIG_NVGPU_GSP_STRESS_TEST := y
+endif
+
 ifeq ($(CONFIG_COMMON_CLK),y)
 ifeq ($(CONFIG_PM_DEVFREQ),y)
 # Select this entry to enable gk20a scaling
@@ -244,3 +249,6 @@ endif
 ifeq ($(CONFIG_NVGPU_GSP_SCHEDULER),y)
 ccflags-y += -DCONFIG_NVGPU_GSP_SCHEDULER
 endif
+ifeq ($(CONFIG_NVGPU_GSP_STRESS_TEST),y)
+ccflags-y += -DCONFIG_NVGPU_GSP_STRESS_TEST
+endif

drivers/gpu/nvgpu/Makefile.shared.configs

@@ -304,6 +304,12 @@ NVGPU_COMMON_CFLAGS                  += -DCONFIG_NVGPU_MIG
 # Enable gsp scheduler for normal build
 CONFIG_NVGPU_GSP_SCHEDULER......:= 1
 NVGPU_COMMON_CFLAGS.............+= -DCONFIG_NVGPU_GSP_SCHEDULER
+
+# Enable GSP stress test
+ifeq ($(CONFIG_NVGPU_GSP_SCHEDULER),1)
+CONFIG_NVGPU_GSP_STRESS_TEST    := 1
+NVGPU_COMMON_CFLAGS             += -DCONFIG_NVGPU_GSP_STRESS_TEST
+endif
 endif
 endif
 

drivers/gpu/nvgpu/common/gsp/gsp_bootstrap.c

@@ -32,11 +32,23 @@
 #include "gsp_priv.h"
 #include "gsp_bootstrap.h"
 
-#define GSP_SIM_WAIT_TIME_MS 10000U
+#define GSP_WAIT_TIME_MS 10000U
 #define GSP_DBG_RISCV_FW_MANIFEST  "sample-gsp.manifest.encrypt.bin.out.bin"
 #define GSP_DBG_RISCV_FW_CODE      "sample-gsp.text.encrypt.bin"
 #define GSP_DBG_RISCV_FW_DATA      "sample-gsp.data.encrypt.bin"
 
+#ifdef CONFIG_NVGPU_GSP_STRESS_TEST
+#define GSPDBG_RISCV_STRESS_TEST_FW_MANIFEST  "gsp-stress.manifest.encrypt.bin.out.bin"
+#define GSPDBG_RISCV_STRESS_TEST_FW_CODE      "gsp-stress.text.encrypt.bin"
+#define GSPDBG_RISCV_STRESS_TEST_FW_DATA      "gsp-stress.data.encrypt.bin"
+
+#define GSPPROD_RISCV_STRESS_TEST_FW_MANIFEST  "gsp-stress.manifest.encrypt.bin.out.bin.prod"
+#define GSPPROD_RISCV_STRESS_TEST_FW_CODE      "gsp-stress.text.encrypt.bin.prod"
+#define GSPPROD_RISCV_STRESS_TEST_FW_DATA      "gsp-stress.data.encrypt.bin.prod"
+
+#define GSP_STRESS_TEST_MAILBOX_PASS 0xAAAAAAAA
+#endif
+
 static void gsp_release_firmware(struct gk20a *g, struct nvgpu_gsp *gsp)
 {
 	if (gsp->gsp_ucode.manifest != NULL) {
@@ -54,26 +66,52 @@ static void gsp_release_firmware(struct gk20a *g, struct nvgpu_gsp *gsp)
 
 static int gsp_read_firmware(struct gk20a *g, struct gsp_fw *gsp_ucode)
 {
+	const char *gsp_code_name;
+	const char *gsp_data_name;
+	const char *gsp_manifest_name;
+
 	nvgpu_log_fn(g, " ");
 
+#ifdef CONFIG_NVGPU_GSP_STRESS_TEST
+	if (g->gsp->gsp_test.load_stress_test) {
+		/*
+		 * TODO Switch to GSP specific register
+		 */
+		if (g->ops.pmu.is_debug_mode_enabled(g)) {
+			gsp_code_name = GSPDBG_RISCV_STRESS_TEST_FW_CODE;
+			gsp_data_name = GSPDBG_RISCV_STRESS_TEST_FW_DATA;
+			gsp_manifest_name = GSPDBG_RISCV_STRESS_TEST_FW_MANIFEST;
+		} else {
+			gsp_code_name = GSPPROD_RISCV_STRESS_TEST_FW_CODE;
+			gsp_data_name = GSPPROD_RISCV_STRESS_TEST_FW_DATA;
+			gsp_manifest_name = GSPPROD_RISCV_STRESS_TEST_FW_MANIFEST;
+		}
+	} else
+#endif
+	{
+		gsp_code_name = GSP_DBG_RISCV_FW_CODE;
+		gsp_data_name = GSP_DBG_RISCV_FW_DATA;
+		gsp_manifest_name = GSP_DBG_RISCV_FW_MANIFEST;
+	}
+
 	gsp_ucode->manifest = nvgpu_request_firmware(g,
-		GSP_DBG_RISCV_FW_MANIFEST, NVGPU_REQUEST_FIRMWARE_NO_WARN);
+		gsp_manifest_name, NVGPU_REQUEST_FIRMWARE_NO_WARN);
 	if (gsp_ucode->manifest == NULL) {
-		nvgpu_err(g, "GSP_DBG_RISCV_FW_MANIFEST ucode get failed");
+		nvgpu_err(g, "%s ucode get failed", gsp_manifest_name);
 		goto fw_release;
 	}
 
 	gsp_ucode->code = nvgpu_request_firmware(g,
-		GSP_DBG_RISCV_FW_CODE, NVGPU_REQUEST_FIRMWARE_NO_WARN);
+		gsp_code_name, NVGPU_REQUEST_FIRMWARE_NO_WARN);
 	if (gsp_ucode->code == NULL) {
-		nvgpu_err(g, "GSP_DBG_RISCV_FW_CODE ucode get failed");
+		nvgpu_err(g, "%s ucode get failed", gsp_code_name);
 		goto fw_release;
 	}
 
 	gsp_ucode->data = nvgpu_request_firmware(g,
-		GSP_DBG_RISCV_FW_DATA, NVGPU_REQUEST_FIRMWARE_NO_WARN);
+		gsp_data_name, NVGPU_REQUEST_FIRMWARE_NO_WARN);
 	if (gsp_ucode->data == NULL) {
-		nvgpu_err(g, "GSP_DBG_RISCV_FW_DATA ucode get failed");
+		nvgpu_err(g, "%s ucode get failed", gsp_data_name);
 		goto fw_release;
 	}
 
@@ -84,6 +122,25 @@ fw_release:
 	return -ENOENT;
 }
 
+#ifdef CONFIG_NVGPU_GSP_STRESS_TEST
+static void gsp_write_test_sysmem_addr(struct nvgpu_gsp *gsp)
+{
+	struct gk20a *g;
+	struct nvgpu_falcon *flcn;
+	u64 sysmem_addr;
+
+	g = gsp->g;
+	flcn = gsp->gsp_flcn;
+
+	sysmem_addr = nvgpu_mem_get_addr(g, &gsp->gsp_test.gsp_test_sysmem_block);
+
+	nvgpu_falcon_mailbox_write(flcn, FALCON_MAILBOX_0, u64_lo32(sysmem_addr));
+
+	nvgpu_falcon_mailbox_write(flcn, FALCON_MAILBOX_1, u64_hi32(sysmem_addr));
+
+}
+#endif
+
 static int gsp_ucode_load_and_bootstrap(struct gk20a *g,
 		struct nvgpu_falcon *flcn,
 		struct gsp_fw *gsp_ucode)
@@ -135,6 +192,14 @@ static int gsp_ucode_load_and_bootstrap(struct gk20a *g,
 	 * gsp ucode to denote its return status.
 	 */
 	nvgpu_falcon_mailbox_write(flcn, FALCON_MAILBOX_0, 0x0U);
+#ifdef CONFIG_NVGPU_GSP_STRESS_TEST
+	/*
+	 * Update the address of the allocated sysmem block in the
+	 * mailbox register for stress test.
+	 */
+	if (g->gsp->gsp_test.load_stress_test)
+		gsp_write_test_sysmem_addr(g->gsp);
+#endif
 
 	g->ops.falcon.bootstrap(flcn, 0x0);
 exit:
@@ -175,22 +240,32 @@ exit:
 	return -1;
 }
 
-static int gsp_wait_for_mailbox_update(struct nvgpu_falcon *flcn,
+static int gsp_wait_for_mailbox_update(struct nvgpu_gsp *gsp,
 		u32 mailbox_index, signed int timeoutms)
 {
 	u32 mail_box_data = 0;
+	u32 pass_val = 0;
+	struct nvgpu_falcon *flcn = gsp->gsp_flcn;
 
 	nvgpu_log_fn(flcn->g, " ");
 
+#ifdef CONFIG_NVGPU_GSP_STRESS_TEST
+	if (gsp->gsp_test.load_stress_test) {
+		pass_val = GSP_STRESS_TEST_MAILBOX_PASS;
+	}
+#endif
+
 	do {
 		mail_box_data = flcn->g->ops.falcon.mailbox_read(
 				flcn, mailbox_index);
 		if (mail_box_data != 0U) {
+			if ((pass_val == 0U) || (mail_box_data == pass_val)) {
 				nvgpu_info(flcn->g,
 					"gsp mailbox-0 updated successful with 0x%x",
 					mail_box_data);
 				break;
 			}
+		}
 
 		if (timeoutms <= 0) {
 			nvgpu_err(flcn->g, "gsp mailbox check timedout");
@@ -234,16 +309,14 @@ int gsp_bootstrap_ns(struct gk20a *g, struct nvgpu_gsp *gsp)
 		goto exit;
 	}
 
-	err = gsp_check_for_brom_completion(gsp->gsp_flcn,
+	err = gsp_check_for_brom_completion(gsp->gsp_flcn, GSP_WAIT_TIME_MS);
-			GSP_SIM_WAIT_TIME_MS);
 	if (err != 0) {
 		nvgpu_err(g, "gsp BROM failed");
 		goto exit;
 	}
 
 	/* wait for mailbox-0 update with non-zero value */
-	err = gsp_wait_for_mailbox_update(gsp->gsp_flcn, 0x0,
+	err = gsp_wait_for_mailbox_update(gsp, 0x0, GSP_WAIT_TIME_MS);
-			GSP_SIM_WAIT_TIME_MS);
 	if (err != 0) {
 		nvgpu_err(g, "gsp ucode failed to update mailbox-0");
 	}

drivers/gpu/nvgpu/common/gsp/gsp_init.c

@@ -25,6 +25,9 @@
 #include <nvgpu/gk20a.h>
 #include <nvgpu/log.h>
 #include <nvgpu/gsp.h>
+#ifdef CONFIG_NVGPU_GSP_STRESS_TEST
+#include <nvgpu/dma.h>
+#endif
 
 #include "gsp_priv.h"
 #include "gsp_bootstrap.h"
@@ -49,6 +52,9 @@ void nvgpu_gsp_sw_deinit(struct gk20a *g)
 		nvgpu_mutex_destroy(&g->gsp->isr_mutex);
 #ifdef CONFIG_NVGPU_FALCON_DEBUG
 		nvgpu_falcon_dbg_buf_destroy(g->gsp->gsp_flcn);
+#endif
+#ifdef CONFIG_NVGPU_GSP_STRESS_TEST
+		nvgpu_dma_free(g, &g->gsp->gsp_test.gsp_test_sysmem_block);
 #endif
 		nvgpu_kfree(g, g->gsp);
 		g->gsp = NULL;
@@ -68,6 +74,9 @@ int nvgpu_gsp_sw_init(struct gk20a *g)
 		 * gsp is set during cold boot & doesn't execute gsp clean up as
 		 * part of power off sequence, so reuse to perform faster boot.
 		 */
+#ifdef CONFIG_NVGPU_GSP_STRESS_TEST
+		nvgpu_gsp_stress_test_bootstrap(g, false);
+#endif
 		return err;
 	}
 
@@ -85,20 +94,6 @@ int nvgpu_gsp_sw_init(struct gk20a *g)
 	/* gsp falcon software state */
 	gsp->gsp_flcn = &g->gsp_flcn;
 
-	/* enable debug buffer support */
-#ifdef CONFIG_NVGPU_FALCON_DEBUG
-	if ((g->ops.gsp.gsp_get_queue_head != NULL) &&
-			(g->ops.gsp.gsp_get_queue_tail != NULL)) {
-		err = nvgpu_falcon_dbg_buf_init(
-			gsp->gsp_flcn, GSP_DMESG_BUFFER_SIZE,
-			g->ops.gsp.gsp_get_queue_head(GSP_DEBUG_BUFFER_QUEUE),
-			g->ops.gsp.gsp_get_queue_tail(GSP_DEBUG_BUFFER_QUEUE));
-		if (err != 0) {
-			nvgpu_err(g, "GSP debug init failed");
-			goto exit;
-		}
-	}
-#endif
 	/* Init isr mutex */
 	nvgpu_mutex_init(&gsp->isr_mutex);
 
@@ -112,6 +107,21 @@ int nvgpu_gsp_bootstrap(struct gk20a *g)
 
 	nvgpu_log_fn(g, " ");
 
+	/* enable debug buffer support */
+#ifdef CONFIG_NVGPU_FALCON_DEBUG
+	if ((g->ops.gsp.gsp_get_queue_head != NULL) &&
+			(g->ops.gsp.gsp_get_queue_tail != NULL)) {
+		err = nvgpu_falcon_dbg_buf_init(
+					g->gsp->gsp_flcn, GSP_DMESG_BUFFER_SIZE,
+					g->ops.gsp.gsp_get_queue_head(GSP_DEBUG_BUFFER_QUEUE),
+					g->ops.gsp.gsp_get_queue_tail(GSP_DEBUG_BUFFER_QUEUE));
+		if (err != 0) {
+			nvgpu_err(g, "GSP debug init failed");
+			goto de_init;
+		}
+	}
+#endif
+
 	err = gsp_bootstrap_ns(g, g->gsp);
 	if (err != 0) {
 		nvgpu_err(g, "GSP bootstrap failed");
@@ -151,3 +161,97 @@ struct nvgpu_falcon *nvgpu_gsp_falcon_instance(struct gk20a *g)
 
 	return gsp->gsp_flcn;
 }
+
+#ifdef CONFIG_NVGPU_GSP_STRESS_TEST
+int nvgpu_gsp_stress_test_bootstrap(struct gk20a *g, bool start)
+{
+	int err = 0;
+	struct nvgpu_gsp *gsp;
+
+	nvgpu_log_fn(g, " ");
+
+	gsp = g->gsp;
+
+	if (gsp == NULL) {
+		nvgpu_err(g, "GSP not initialized");
+		err = -EFAULT;
+		goto exit;
+	}
+
+	if (!start && !(gsp->gsp_test.load_stress_test))
+		return err;
+
+	if (start) {
+		err = nvgpu_dma_alloc_flags_sys(g,
+						NVGPU_DMA_PHYSICALLY_ADDRESSED,
+						SZ_64K,
+						&g->gsp->gsp_test.gsp_test_sysmem_block);
+		if (err != 0) {
+			nvgpu_err(g, "GSP test memory alloc failed");
+			goto exit;
+		}
+	}
+
+	gsp->gsp_test.load_stress_test = true;
+
+	err = nvgpu_gsp_bootstrap(g);
+	if (err != 0) {
+		nvgpu_err(g, "GSP bootstrap failed for stress test");
+		goto exit;
+	}
+
+	if (gsp->gsp_test.enable_stress_test) {
+		nvgpu_info(g, "Restarting GSP stress test");
+		nvgpu_falcon_mailbox_write(gsp->gsp_flcn, FALCON_MAILBOX_1, 0xFFFFFFFF);
+	}
+
+	return err;
+
+exit:
+	gsp->gsp_test.load_stress_test = false;
+
+	return err;
+}
+
+int nvgpu_gsp_stress_test_halt(struct gk20a *g, bool restart)
+{
+	int err = 0;
+	struct nvgpu_gsp *gsp;
+
+	nvgpu_log_fn(g, " ");
+
+	gsp = g->gsp;
+
+	if ((gsp == NULL)) {
+		nvgpu_info(g, "GSP not initialized");
+		goto exit;
+	}
+
+	if (restart && (gsp->gsp_test.load_stress_test == false)) {
+		nvgpu_info(g, "GSP stress test not loaded ");
+		goto exit;
+	}
+
+	err = nvgpu_falcon_reset(gsp->gsp_flcn);
+	if (err != 0) {
+		nvgpu_err(g, "gsp reset failed err=%d", err);
+		goto exit;
+	}
+
+	if (!restart) {
+		gsp->gsp_test.load_stress_test = false;
+		nvgpu_dma_free(g, &g->gsp->gsp_test.gsp_test_sysmem_block);
+	}
+
+exit:
+	return err;
+}
+
+bool nvgpu_gsp_is_stress_test(struct gk20a *g)
+{
+	if (g->gsp->gsp_test.load_stress_test)
+		return true;
+	else
+		return false;
+}
+#endif

drivers/gpu/nvgpu/common/gsp/gsp_priv.h

@@ -26,7 +26,7 @@
 #include <nvgpu/lock.h>
 
 #define GSP_DEBUG_BUFFER_QUEUE	3U
-#define GSP_DMESG_BUFFER_SIZE	0x1000U
+#define GSP_DMESG_BUFFER_SIZE	0xC00U
 
 struct gsp_fw {
 	/* gsp ucode */
@@ -35,6 +35,16 @@ struct gsp_fw {
 	struct nvgpu_firmware *manifest;
 };
 
+#ifdef CONFIG_NVGPU_GSP_STRESS_TEST
+struct gsp_stress_test {
+	bool load_stress_test;
+	bool enable_stress_test;
+	bool stress_test_fail_status;
+	u32 test_iterations;
+	u32 test_name;
+	struct nvgpu_mem gsp_test_sysmem_block;
+};
+#endif
 /* GSP descriptor's */
 struct nvgpu_gsp {
 	struct gk20a *g;
@@ -44,5 +54,8 @@ struct nvgpu_gsp {
 
 	bool isr_enabled;
 	struct nvgpu_mutex isr_mutex;
+#ifdef CONFIG_NVGPU_GSP_STRESS_TEST
+	struct gsp_stress_test gsp_test;
+#endif
 };
 #endif /* NVGPU_GSP_PRIV */

drivers/gpu/nvgpu/common/gsp/gsp_test.c

@@ -0,0 +1,114 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * GSP Test Functions
+ *
+ * Copyright (c) 2021, NVIDIA Corporation.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#include <nvgpu/firmware.h>
+#include <nvgpu/falcon.h>
+#include <nvgpu/gk20a.h>
+#include <nvgpu/log.h>
+#include <nvgpu/gsp.h>
+#include <nvgpu/gsp/gsp_test.h>
+
+#include "gsp_priv.h"
+#include "gsp_bootstrap.h"
+
+u32 nvgpu_gsp_get_current_iteration(struct gk20a *g)
+{
+	u32 data = 0;
+	struct nvgpu_gsp *gsp = g->gsp;
+
+	nvgpu_log_fn(g, " ");
+
+	data = nvgpu_falcon_mailbox_read(gsp->gsp_flcn, FALCON_MAILBOX_1);
+
+	return data;
+}
+
+u32 nvgpu_gsp_get_current_test(struct gk20a *g)
+{
+	u32 data = 0;
+	struct nvgpu_gsp *gsp = g->gsp;
+
+	nvgpu_log_fn(g, " ");
+
+	data = nvgpu_falcon_mailbox_read(gsp->gsp_flcn, FALCON_MAILBOX_0);
+
+	return data;
+}
+
+bool nvgpu_gsp_get_test_fail_status(struct gk20a *g)
+{
+	struct nvgpu_gsp *gsp = g->gsp;
+
+	return gsp->gsp_test.stress_test_fail_status;
+}
+
+bool nvgpu_gsp_get_stress_test_start(struct gk20a *g)
+{
+	struct nvgpu_gsp *gsp = g->gsp;
+
+	return gsp->gsp_test.enable_stress_test;
+}
+
+bool nvgpu_gsp_get_stress_test_load(struct gk20a *g)
+{
+	struct nvgpu_gsp *gsp = g->gsp;
+
+	if (gsp == NULL)
+		return false;
+
+	return gsp->gsp_test.load_stress_test;
+}
+
+void nvgpu_gsp_set_test_fail_status(struct gk20a *g, bool val)
+{
+	struct nvgpu_gsp *gsp = g->gsp;
+
+	gsp->gsp_test.stress_test_fail_status = val;
+}
+
+int nvgpu_gsp_set_stress_test_start(struct gk20a *g, bool flag)
+{
+	int err = 0;
+	struct nvgpu_gsp *gsp = g->gsp;
+
+	nvgpu_log_fn(g, " ");
+
+	if (flag) {
+		nvgpu_info(g, "Enabling GSP test");
+		nvgpu_falcon_mailbox_write(gsp->gsp_flcn, FALCON_MAILBOX_1, 0xFFFFFFFF);
+	} else {
+		nvgpu_info(g, "Halting GSP test");
+		nvgpu_gsp_stress_test_halt(g, false);
+	}
+
+	gsp->gsp_test.enable_stress_test = flag;
+
+	return err;
+}
+
+int nvgpu_gsp_set_stress_test_load(struct gk20a *g, bool flag)
+{
+	int err = 0;
+
+	nvgpu_log_fn(g, " ");
+
+	if (flag)
+		err = nvgpu_gsp_stress_test_bootstrap(g, flag);
+
+	return err;
+}

drivers/gpu/nvgpu/common/init/nvgpu_init.c

@@ -338,6 +338,12 @@ int nvgpu_prepare_poweroff(struct gk20a *g)
 		ret = tmp_ret;
 	}
 
+#ifdef CONFIG_NVGPU_GSP_STRESS_TEST
+	ret = nvgpu_gsp_stress_test_halt(g, true);
+	if (ret != 0)
+		nvgpu_err(g, "Failed to halt GSP stress test");
+#endif
+
 	nvgpu_falcons_sw_free(g);
 
 #ifdef CONFIG_NVGPU_DGPU
@@ -360,6 +366,7 @@ int nvgpu_prepare_poweroff(struct gk20a *g)
 		g->ops.clk_arb.stop_clk_arb_threads(g);
 	}
 #endif
+
 	gk20a_mask_interrupts(g);
 
 	/* Disable CIC after the interrupts are masked;
@@ -818,10 +825,6 @@ int nvgpu_finalize_poweron(struct gk20a *g)
 #ifdef CONFIG_NVGPU_DGPU
 		NVGPU_INIT_TABLE_ENTRY(g->ops.sec2.init_sec2_setup_sw,
 				       NVGPU_SUPPORT_SEC2_RTOS),
-#endif
-#ifdef CONFIG_NVGPU_GSP_SCHEDULER
-		/* Init gsp ops */
-		NVGPU_INIT_TABLE_ENTRY(&nvgpu_gsp_sw_init, NO_FLAG),
 #endif
 		NVGPU_INIT_TABLE_ENTRY(g->ops.acr.acr_init,
 				       NVGPU_SEC_PRIVSECURITY),
@@ -902,6 +905,10 @@ int nvgpu_finalize_poweron(struct gk20a *g)
 #endif
 		NVGPU_INIT_TABLE_ENTRY(g->ops.channel.resume_all_serviceable_ch,
 				       NO_FLAG),
+#if defined(CONFIG_NVGPU_GSP_SCHEDULER) || defined(CONFIG_NVGPU_GSP_STRESS_TEST)
+		/* Init gsp ops */
+		NVGPU_INIT_TABLE_ENTRY(&nvgpu_gsp_sw_init, NO_FLAG),
+#endif
 	};
 	size_t i;
 

drivers/gpu/nvgpu/hal/gsp/gsp_ga10b.c

@@ -29,6 +29,9 @@
 #ifdef CONFIG_NVGPU_GSP_SCHEDULER
 #include <nvgpu/gsp.h>
 #endif
+#ifdef CONFIG_NVGPU_GSP_STRESS_TEST
+#include <nvgpu/gsp/gsp_test.h>
+#endif
 
 #include "gsp_ga10b.h"
 
@@ -120,9 +123,18 @@ static void ga10b_gsp_handle_swgen1_irq(struct gk20a *g)
 #endif
 }
 
+static void ga10b_gsp_handle_halt_irq(struct gk20a *g)
+{
+	nvgpu_err(g, "GSP Halt Interrupt Fired");
+
+#ifdef CONFIG_NVGPU_GSP_STRESS_TEST
+	nvgpu_gsp_set_test_fail_status(g, true);
+#endif
+}
+
 static void ga10b_gsp_clr_intr(struct gk20a *g, u32 intr)
 {
-	gk20a_writel(g, pgsp_riscv_irqmclr_r(), intr);
+	gk20a_writel(g, pgsp_falcon_irqsclr_r(), intr);
 }
 
 void ga10b_gsp_handle_interrupts(struct gk20a *g, u32 intr)
@@ -136,7 +148,7 @@ void ga10b_gsp_handle_interrupts(struct gk20a *g, u32 intr)
 
 	/* halt interrupt handle */
 	if ((intr & pgsp_falcon_irqstat_halt_true_f()) != 0U) {
-		nvgpu_err(g, "gsp halt intr not implemented");
+		ga10b_gsp_handle_halt_irq(g);
 	}
 }
 
@@ -193,16 +205,24 @@ void ga10b_gsp_enable_irq(struct gk20a *g, bool enable)
 {
 	u32 intr_mask;
 	u32 intr_dest;
+	bool skip_priv = false;
 
 	nvgpu_log_fn(g, " ");
 
+#ifdef CONFIG_NVGPU_GSP_STRESS_TEST
+	if (nvgpu_gsp_is_stress_test(g))
+		skip_priv = true;
+#endif
+
 	/* clear before setting required irq */
+	if ((!skip_priv) || (!enable))
 		ga10b_riscv_set_irq(g, false, 0x0, 0x0);
 
 	nvgpu_cic_mon_intr_stall_unit_config(g,
 			NVGPU_CIC_INTR_UNIT_GSP, NVGPU_CIC_INTR_DISABLE);
 
 	if (enable) {
+		if (!skip_priv) {
 			/* dest 0=falcon, 1=host; level 0=irq0, 1=irq1 */
 			intr_dest = pgsp_riscv_irqdest_gptmr_f(0)    |
 				pgsp_riscv_irqdest_wdtmr_f(1)    |
@@ -226,6 +246,7 @@ void ga10b_gsp_enable_irq(struct gk20a *g, bool enable)
 
 			/* set required irq */
 			ga10b_riscv_set_irq(g, true, intr_mask, intr_dest);
+		}
 
 		nvgpu_cic_mon_intr_stall_unit_config(g,
 				NVGPU_CIC_INTR_UNIT_GSP, NVGPU_CIC_INTR_ENABLE);

drivers/gpu/nvgpu/include/nvgpu/gops/gsp.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2020-2021, NVIDIA CORPORATION.  All rights reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),

drivers/gpu/nvgpu/include/nvgpu/gsp.h

@@ -33,4 +33,9 @@ void nvgpu_gsp_isr_mutex_aquire(struct gk20a *g);
 void nvgpu_gsp_isr_mutex_release(struct gk20a *g);
 bool nvgpu_gsp_is_isr_enable(struct gk20a *g);
 struct nvgpu_falcon *nvgpu_gsp_falcon_instance(struct gk20a *g);
+#ifdef CONFIG_NVGPU_GSP_STRESS_TEST
+int nvgpu_gsp_stress_test_bootstrap(struct gk20a *g, bool start);
+int nvgpu_gsp_stress_test_halt(struct gk20a *g, bool restart);
+bool nvgpu_gsp_is_stress_test(struct gk20a *g);
+#endif
 #endif /* NVGPU_GSP */

drivers/gpu/nvgpu/include/nvgpu/gsp/gsp_test.h

@@ -0,0 +1,33 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * GSP Tests
+ *
+ * Copyright (c) 2021, NVIDIA Corporation.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef NVGPU_GSP_TEST
+#define NVGPU_GSP_TEST
+
+#include <nvgpu/types.h>
+struct gk20a;
+
+u32 nvgpu_gsp_get_current_iteration(struct gk20a *g);
+u32 nvgpu_gsp_get_current_test(struct gk20a *g);
+bool nvgpu_gsp_get_test_fail_status(struct gk20a *g);
+void nvgpu_gsp_set_test_fail_status(struct gk20a *g, bool val);
+bool nvgpu_gsp_get_stress_test_start(struct gk20a *g);
+int nvgpu_gsp_set_stress_test_start(struct gk20a *g, bool flag);
+bool nvgpu_gsp_get_stress_test_load(struct gk20a *g);
+int nvgpu_gsp_set_stress_test_load(struct gk20a *g, bool flag);
+#endif /* NVGPU_GSP_TEST */

drivers/gpu/nvgpu/include/nvgpu/hw/ga10b/hw_pgsp_ga10b.h

@@ -61,6 +61,7 @@
 
 #define pgsp_falcon2_gsp_base_r()                                  (0x00111000U)
 #define pgsp_falcon_irqsset_r()                                    (0x00110000U)
+#define pgsp_falcon_irqsclr_r()                                    (0x00110004U)
 #define pgsp_falcon_engine_r()                                     (0x001103c0U)
 #define pgsp_falcon_engine_reset_true_f()                                 (0x1U)
 #define pgsp_falcon_engine_reset_false_f()                                (0x0U)

drivers/gpu/nvgpu/os/linux/debug.c

@@ -25,6 +25,9 @@
 #include "debug_xve.h"
 #include "debug_ltc.h"
 #include "debug_bios.h"
+#ifdef CONFIG_NVGPU_GSP_STRESS_TEST
+#include "debug_gsp.h"
+#endif
 #include "os_linux.h"
 #include "platform_gk20a.h"
 
@@ -524,6 +527,9 @@ void gk20a_debug_init(struct gk20a *g, const char *debugfs_symlink)
 		nvgpu_bios_debugfs_init(g);
 	}
 #endif
+#ifdef CONFIG_NVGPU_GSP_STRESS_TEST
+	nvgpu_gsp_debugfs_init(g);
+#endif
 }
 
 void gk20a_debug_deinit(struct gk20a *g)

drivers/gpu/nvgpu/os/linux/debug_gsp.c

@@ -0,0 +1,357 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * GSP Debug Nodes
+ *
+ * Copyright (c) 2021, NVIDIA Corporation.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <nvgpu/nvgpu_init.h>
+#include <nvgpu/gsp/gsp_test.h>
+
+#include "debug_gsp.h"
+#include "os_linux.h"
+
+#include <linux/debugfs.h>
+#include <linux/seq_file.h>
+#include <linux/uaccess.h>
+
+#ifdef CONFIG_NVGPU_GSP_STRESS_TEST
+
+static int gsp_test_iterations_show(struct seq_file *s, void *data)
+{
+	struct gk20a *g = s->private;
+	u32 current_iteration = 0;
+	int err = 0;
+
+	if (nvgpu_is_powered_on(g)) {
+		err = gk20a_busy(g);
+		if (err)
+			return err;
+		current_iteration = nvgpu_gsp_get_current_iteration(g);
+		gk20a_idle(g);
+	}
+
+	seq_printf(s, "%d\n", current_iteration);
+
+	return err;
+}
+
+static int gsp_test_iterations_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, gsp_test_iterations_show, inode->i_private);
+}
+
+static const struct file_operations nvgpu_gsp_test_iterations_debugfs_fops = {
+	.open = gsp_test_iterations_open,
+	.read = seq_read,
+	.llseek = seq_lseek,
+	.release = single_release
+};
+
+static int gsp_current_test_show(struct seq_file *s, void *data)
+{
+	struct gk20a *g = s->private;
+	u32 current_test = 0;
+	int err = 0;
+
+	if (nvgpu_is_powered_on(g)) {
+		err = gk20a_busy(g);
+		if (err)
+			return err;
+		current_test = nvgpu_gsp_get_current_test(g);
+		gk20a_idle(g);
+	}
+
+	seq_printf(s, "%d\n", current_test);
+
+	return err;
+}
+
+static int gsp_current_test_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, gsp_current_test_show, inode->i_private);
+}
+
+static const struct file_operations nvgpu_gsp_current_test_debugfs_fops = {
+	.open = gsp_current_test_open,
+	.read = seq_read,
+	.llseek = seq_lseek,
+	.release = single_release
+};
+
+static int gsp_test_status_show(struct seq_file *s, void *data)
+{
+	struct gk20a *g = s->private;
+	u32 test_status = 0;
+	int err = 0;
+
+	if (nvgpu_is_powered_on(g)) {
+		err = gk20a_busy(g);
+		if (err)
+			return err;
+		test_status = (u32)nvgpu_gsp_get_test_fail_status(g);
+		gk20a_idle(g);
+	}
+
+	seq_printf(s, "%d\n", test_status);
+
+	return err;
+}
+
+static int gsp_test_status_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, gsp_test_status_show, inode->i_private);
+}
+
+static const struct file_operations nvgpu_gsp_test_status_debugfs_fops = {
+	.open = gsp_test_status_open,
+	.read = seq_read,
+	.llseek = seq_lseek,
+	.release = single_release
+};
+
+static int gsp_test_summary_show(struct seq_file *s, void *data)
+{
+	struct gk20a *g = s->private;
+	u32 test_state = 0;
+	u32 test_iterations = 0;
+	u32 current_test = 0;
+	u32 test_fail_status = 0;
+	int err = 0;
+
+	if (nvgpu_is_powered_on(g)) {
+		err = gk20a_busy(g);
+		if (err)
+			return err;
+		test_state = nvgpu_gsp_get_stress_test_start(g);
+		test_iterations = nvgpu_gsp_get_current_iteration(g);
+		current_test = nvgpu_gsp_get_current_test(g);
+		test_fail_status = (u32)nvgpu_gsp_get_test_fail_status(g);
+		gk20a_idle(g);
+	}
+
+	seq_printf(s,
+		"Test Started: %d\n"
+		"Passed Test: %d\n"
+		"Test Iterations: %d\n"
+		"Test State: %d\n"
+		, test_state, current_test, test_iterations, test_fail_status);
+
+	return err;
+}
+
+static int gsp_test_summary_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, gsp_test_summary_show, inode->i_private);
+}
+
+static const struct file_operations nvgpu_gsp_test_summary_debugfs_fops = {
+	.open = gsp_test_summary_open,
+	.read = seq_read,
+	.llseek = seq_lseek,
+	.release = single_release
+};
+
+static ssize_t gsp_start_test_read(struct file *file, char __user *user_buf,
+								   size_t count, loff_t *ppos)
+{
+	char buf[3];
+	struct gk20a *g = file->private_data;
+
+
+	buf[0] = 'N';
+	if (nvgpu_is_powered_on(g)) {
+		if (nvgpu_gsp_get_stress_test_start(g))
+			buf[0] = 'Y';
+	}
+
+	buf[1] = '\n';
+	buf[2] = 0x00;
+
+	return simple_read_from_buffer(user_buf, count, ppos, buf, 2);
+}
+
+static ssize_t gsp_start_test_write(struct file *file,
+									const char __user *user_buf,
+									size_t count, loff_t *ppos)
+{
+	char buf[32];
+	int buf_size;
+	bool bv;
+	struct gk20a *g = file->private_data;
+	int err;
+
+	buf_size = min(count, (sizeof(buf)-1));
+	if (copy_from_user(buf, user_buf, buf_size))
+		return -EFAULT;
+
+	if (strtobool(buf, &bv) == 0) {
+		if (nvgpu_is_powered_on(g) && nvgpu_gsp_get_stress_test_load(g)) {
+			err = nvgpu_gsp_set_stress_test_start(g, bv);
+			if (err != 0) {
+				nvgpu_err(g, "failed to start GSP stress test");
+				return -EFAULT;
+			}
+		} else {
+			nvgpu_err(g,
+					  "Unable to start GSP stress test, check GPU state");
+			return -EFAULT;
+		}
+	}
+
+	return count;
+}
+
+static const struct file_operations nvgpu_gsp_start_test_debugfs_fops = {
+	.open = simple_open,
+	.read = gsp_start_test_read,
+	.write = gsp_start_test_write,
+};
+
+static ssize_t gsp_load_test_read(struct file *file, char __user *user_buf,
+								   size_t count, loff_t *ppos)
+{
+	char buf[3];
+	struct gk20a *g = file->private_data;
+
+
+	buf[0] = 'N';
+	if (nvgpu_is_powered_on(g)) {
+		if (nvgpu_gsp_get_stress_test_load(g))
+			buf[0] = 'Y';
+	}
+
+	buf[1] = '\n';
+	buf[2] = 0x00;
+
+	return simple_read_from_buffer(user_buf, count, ppos, buf, 2);
+}
+
+static ssize_t gsp_load_test_write(struct file *file,
+								   const char __user *user_buf,
+								   size_t count, loff_t *ppos)
+{
+	char buf[32];
+	int buf_size;
+	bool bv;
+	struct gk20a *g = file->private_data;
+	int err;
+
+	buf_size = min(count, (sizeof(buf)-1));
+	if (copy_from_user(buf, user_buf, buf_size))
+		return -EFAULT;
+
+	if (strtobool(buf, &bv) == 0) {
+		if (nvgpu_is_powered_on(g)) {
+			err = nvgpu_gsp_set_stress_test_load(g, bv);
+			if (err != 0) {
+				nvgpu_err(g, "failed to load GSP stress test");
+				return -EFAULT;
+			}
+		} else {
+			nvgpu_err(g,
+					  "Unable to load GSP stress test, check GPU state");
+			return -EFAULT;
+		}
+	}
+
+	return count;
+}
+
+static const struct file_operations nvgpu_gsp_load_test_debugfs_fops = {
+	.open = simple_open,
+	.read = gsp_load_test_read,
+	.write = gsp_load_test_write,
+};
+#endif
+
+void nvgpu_gsp_debugfs_fini(struct gk20a *g)
+{
+	struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
+
+	if (!(l->debugfs_gsp == NULL))
+		debugfs_remove_recursive(l->debugfs_gsp);
+}
+
+int nvgpu_gsp_debugfs_init(struct gk20a *g)
+{
+	struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
+	struct dentry *gpu_root = l->debugfs;
+	struct dentry *d;
+	int err = 0;
+
+	if (!gpu_root) {
+		err = -ENODEV;
+		goto exit;
+	}
+
+	d = debugfs_create_dir("gsp", gpu_root);
+	if (IS_ERR(l->debugfs_gsp)) {
+		err = PTR_ERR(d);
+		goto exit;
+	}
+
+	l->debugfs_gsp = d;
+
+	nvgpu_log(g, gpu_dbg_info, "g=%p", g);
+
+#ifdef CONFIG_NVGPU_GSP_STRESS_TEST
+	d = debugfs_create_file("load_test", 0644, l->debugfs_gsp, g,
+							&nvgpu_gsp_load_test_debugfs_fops);
+	if (IS_ERR(d)) {
+		err = PTR_ERR(d);
+		goto exit;
+	}
+
+	d = debugfs_create_file("start_test", 0644, l->debugfs_gsp, g,
+							&nvgpu_gsp_start_test_debugfs_fops);
+	if (IS_ERR(d)) {
+		err = PTR_ERR(d);
+		goto exit;
+	}
+
+	d = debugfs_create_file("test_iterations", 0444, l->debugfs_gsp, g,
+							&nvgpu_gsp_test_iterations_debugfs_fops);
+	if (IS_ERR(d)) {
+		err = PTR_ERR(d);
+		goto exit;
+	}
+
+	d = debugfs_create_file("current_test", 0444, l->debugfs_gsp, g,
+							&nvgpu_gsp_current_test_debugfs_fops);
+	if (IS_ERR(d)) {
+		err = PTR_ERR(d);
+		goto exit;
+	}
+
+	d = debugfs_create_file("test_status", 0444, l->debugfs_gsp, g,
+							&nvgpu_gsp_test_status_debugfs_fops);
+	if (IS_ERR(d)) {
+		err = PTR_ERR(d);
+		goto exit;
+	}
+
+	d = debugfs_create_file("test_summary", 0444, l->debugfs_gsp, g,
+							&nvgpu_gsp_test_summary_debugfs_fops);
+	if (IS_ERR(d)) {
+		err = PTR_ERR(d);
+		goto exit;
+	}
+#endif
+	return err;
+exit:
+	nvgpu_gsp_debugfs_fini(g);
+	return err;
+}

drivers/gpu/nvgpu/os/linux/debug_gsp.h

@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * GSP Debug
+ *
+ * Copyright (c) 2021, NVIDIA Corporation.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef __NVGPU_DEBUG_GSP_H__
+#define __NVGPU_DEBUG_GSP_H__
+
+struct gk20a;
+
+void nvgpu_gsp_debugfs_fini(struct gk20a *g);
+int nvgpu_gsp_debugfs_init(struct gk20a *g);
+
+#endif /* __NVGPU_DEBUG_GSP_H__ */

drivers/gpu/nvgpu/os/linux/os_linux.h

@@ -122,6 +122,7 @@ struct nvgpu_os_linux {
 	struct dentry *debugfs_ltc;
 
 	struct dentry *debugfs_dump_ctxsw_stats;
+	struct dentry *debugfs_gsp;
 #endif
 	struct dev_ext_attribute *ecc_attrs;