gpu: nvgpu: add NVGPU_SUBMIT_GPFIFO_FLAGS_RESCHEDULE_RUNLIST

NVGPU_SUBMIT_GPFIFO_FLAGS_RESCHEDULE_RUNLIST causes host to expire
current timeslice and reschedule from front of runlist.
This can be used with NVGPU_RUNLIST_INTERLEAVE_LEVEL_HIGH to make a
channel start sooner after submit rather than waiting for natural
timeslice expiration or block/finish of currently running channel.

Bug 1968813

Change-Id: I632e87c5f583a09ec8bf521dc73f595150abebb0
Signed-off-by: David Li <davli@nvidia.com>
Reviewed-on: https://git-master.nvidia.com/r/1537218
GVS: Gerrit_Virtual_Submit
Reviewed-by: Aingara Paramakuru <aparamakuru@nvidia.com>
Reviewed-by: Terje Bergstrom <tbergstrom@nvidia.com>

drivers/gpu/nvgpu/gk20a/channel_gk20a.c

@@ -3225,6 +3225,10 @@ int gk20a_submit_channel_gpfifo(struct channel_gk20a *c,
 
 	g->ops.fifo.userd_gp_put(g, c);
 
+	if ((NVGPU_SUBMIT_GPFIFO_FLAGS_RESCHEDULE_RUNLIST & flags) &&
+		g->ops.fifo.reschedule_runlist)
+		g->ops.fifo.reschedule_runlist(g, c->runlist_id);
+
 	trace_gk20a_channel_submitted_gpfifo(dev_name(c->g->dev),
 				c->hw_chid,
 				num_entries,

drivers/gpu/nvgpu/gk20a/fifo_gk20a.c

@@ -3076,6 +3076,34 @@ end:
 	return ret;
 }
 
+/* trigger host to expire current timeslice and reschedule runlist from front */
+int gk20a_fifo_reschedule_runlist(struct gk20a *g, u32 runlist_id)
+{
+	struct fifo_runlist_info_gk20a *runlist;
+	u32 token = PMU_INVALID_MUTEX_OWNER_ID;
+	u32 mutex_ret;
+	int ret = 0;
+
+	runlist = &g->fifo.runlist_info[runlist_id];
+	if (nvgpu_mutex_tryacquire(&runlist->mutex)) {
+		mutex_ret = pmu_mutex_acquire(
+			&g->pmu, PMU_MUTEX_ID_FIFO, &token);
+
+		gk20a_writel(g, fifo_runlist_r(),
+			gk20a_readl(g, fifo_runlist_r()));
+		gk20a_fifo_runlist_wait_pending(g, runlist_id);
+
+		if (!mutex_ret)
+			pmu_mutex_release(
+				&g->pmu, PMU_MUTEX_ID_FIFO, &token);
+		nvgpu_mutex_release(&runlist->mutex);
+	} else {
+		/* someone else is writing fifo_runlist_r so not needed here */
+		ret = -EBUSY;
+	}
+	return ret;
+}
+
 /* add/remove a channel from runlist
    special cases below: runlist->active_channels will NOT be changed.
    (hw_chid == ~0 && !add) means remove all active channels from runlist.

drivers/gpu/nvgpu/gk20a/fifo_gk20a.h

@@ -230,6 +230,8 @@ int gk20a_fifo_disable_all_engine_activity(struct gk20a *g,
 				bool wait_for_idle);
 u32 gk20a_fifo_engines_on_ch(struct gk20a *g, u32 hw_chid);
 
+int gk20a_fifo_reschedule_runlist(struct gk20a *g, u32 runlist_id);
+
 int gk20a_fifo_update_runlist(struct gk20a *g, u32 engine_id, u32 hw_chid,
 			      bool add, bool wait_for_finish);
 

drivers/gpu/nvgpu/gk20a/gk20a.h

@@ -403,6 +403,7 @@ struct gpu_ops {
 		int (*resetup_ramfc)(struct channel_gk20a *c);
 		int (*preempt_channel)(struct gk20a *g, u32 hw_chid);
 		int (*preempt_tsg)(struct gk20a *g, u32 tsgid);
+		int (*reschedule_runlist)(struct gk20a *g, u32 runlist_id);
 		int (*update_runlist)(struct gk20a *g, u32 runlist_id,
 				u32 hw_chid, bool add,
 				bool wait_for_finish);

drivers/gpu/nvgpu/gp10b/fifo_gp10b.c

@@ -240,4 +240,5 @@ void gp10b_init_fifo(struct gpu_ops *gops)
 	gops->fifo.device_info_data_parse = gp10b_device_info_data_parse;
 	gops->fifo.eng_runlist_base_size = fifo_eng_runlist_base__size_1_v;
 	gops->fifo.device_info_fault_id = top_device_info_data_fault_id_enum_v;
+	gops->fifo.reschedule_runlist = gk20a_fifo_reschedule_runlist;
 }

drivers/gpu/nvgpu/gp10b/gp10b.c

@@ -1,7 +1,7 @@
 /*
  * GP10B Graphics
  *
- * Copyright (c) 2016, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2016-2017, NVIDIA CORPORATION.  All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms and conditions of the GNU General Public License,
@@ -106,6 +106,7 @@ int gp10b_init_gpu_characteristics(struct gk20a *g)
 {
 	gk20a_init_gpu_characteristics(g);
 	g->gpu_characteristics.flags |= gp10b_detect_ecc_enabled_units(g);
-
+	g->gpu_characteristics.flags |=
+		NVGPU_GPU_FLAGS_SUPPORT_RESCHEDULE_RUNLIST;
 	return 0;
 }

drivers/gpu/nvgpu/vgpu/gp10b/vgpu_fifo_gp10b.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015-2016, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2015-2017, NVIDIA CORPORATION.  All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms and conditions of the GNU General Public License,
@@ -45,4 +45,5 @@ void vgpu_gp10b_init_fifo_ops(struct gpu_ops *gops)
 	/* syncpoint protection not supported yet */
 	gops->fifo.init_engine_info = vgpu_gp10b_fifo_init_engine_info;
 	gops->fifo.resetup_ramfc = NULL;
+	gops->fifo.reschedule_runlist = NULL;
 }

drivers/gpu/nvgpu/vgpu/vgpu.c

@@ -330,6 +330,8 @@ static int vgpu_init_gpu_characteristics(struct gk20a *g)
 	/* features vgpu does not support */
 	g->gpu_characteristics.flags &= ~NVGPU_GPU_FLAGS_SUPPORT_CYCLE_STATS;
 	g->gpu_characteristics.flags &= ~NVGPU_GPU_FLAGS_SUPPORT_MAP_COMPBITS;
+	g->gpu_characteristics.flags &=
+		~NVGPU_GPU_FLAGS_SUPPORT_RESCHEDULE_RUNLIST;
 
 	return 0;
 }

include/uapi/linux/nvgpu.h

@@ -137,6 +137,8 @@ struct nvgpu_gpu_zbc_query_table_args {
 #define NVGPU_GPU_FLAGS_SUPPORT_FECS_CTXSW_TRACE	(1ULL << 16)
 /* NVGPU_AS_IOCTL_MAP_BUFFER_COMPBITS is available */
 #define NVGPU_GPU_FLAGS_SUPPORT_MAP_COMPBITS		(1ULL << 17)
+/* NVGPU_SUBMIT_GPFIFO_FLAGS_RESCHEDULE_RUNLIST is available */
+#define NVGPU_GPU_FLAGS_SUPPORT_RESCHEDULE_RUNLIST	(1ULL << 21)
 
 struct nvgpu_gpu_characteristics {
 	__u32 arch;
@@ -1376,6 +1378,8 @@ struct nvgpu_fence {
 #define NVGPU_SUBMIT_GPFIFO_FLAGS_SUPPRESS_WFI	(1 << 4)
 /* skip buffer refcounting during submit */
 #define NVGPU_SUBMIT_GPFIFO_FLAGS_SKIP_BUFFER_REFCOUNTING	(1 << 5)
+/* expire current timeslice and reschedule runlist from front */
+#define NVGPU_SUBMIT_GPFIFO_FLAGS_RESCHEDULE_RUNLIST	(1 << 6)
 
 struct nvgpu_submit_gpfifo_args {
 	__u64 gpfifo;