gpu: nvgpu: clean runlist dependencies

Split existing runlist HALs into:
- runlist HALs depending on ram hw headers
- runlist HALs depending on fifo hw headers

hal/fifo/runlist_<chip>.c implement
- runlist.entry_size
- runlist.get_tsg_entry
- runlist.get_ch_entry

hal/fifo/runlist_fifo_<chip>.c implement
- runlist.reschedule
- runlist.count_max
- runlist.entry_size
- runlist.hw_submit

Renamed
- nvgpu_fifo_reschedule_runlist -> nvgpu_runlist_reschedule

Jira NVGPU-3198

Change-Id: Icf835b0a4a45e5987e3db9d0931a9f111f418137
Signed-off-by: Thomas Fleury <tfleury@nvidia.com>
Reviewed-on: https://git-master.nvidia.com/r/2107603
Reviewed-by: mobile promotions <svcmobile_promotions@nvidia.com>
Tested-by: mobile promotions <svcmobile_promotions@nvidia.com>

drivers/gpu/nvgpu/Makefile

@@ -285,8 +285,11 @@ nvgpu-y += \
 	hal/fifo/ramin_tu104.o \
 	hal/fifo/runlist_gk20a.o \
 	hal/fifo/runlist_gv11b.o \
-	hal/fifo/runlist_gv100.o \
 	hal/fifo/runlist_tu104.o \
+	hal/fifo/runlist_fifo_gk20a.o \
+	hal/fifo/runlist_fifo_gv11b.o \
+	hal/fifo/runlist_fifo_gv100.o \
+	hal/fifo/runlist_fifo_tu104.o \
 	hal/fifo/tsg_gk20a.o \
 	hal/fifo/tsg_gv11b.o \
 	hal/fifo/userd_gk20a.o \

drivers/gpu/nvgpu/Makefile.sources

@@ -387,8 +387,11 @@ srcs += common/sim/sim.c \
 	hal/fifo/ramin_tu104.c \
 	hal/fifo/runlist_gk20a.c \
 	hal/fifo/runlist_gv11b.c \
-	hal/fifo/runlist_gv100.c \
 	hal/fifo/runlist_tu104.c \
+	hal/fifo/runlist_fifo_gk20a.c \
+	hal/fifo/runlist_fifo_gv11b.c \
+	hal/fifo/runlist_fifo_gv100.c \
+	hal/fifo/runlist_fifo_tu104.c \
 	hal/fifo/tsg_gk20a.c \
 	hal/fifo/tsg_gv11b.c \
 	hal/fifo/userd_gk20a.c \

drivers/gpu/nvgpu/common/fifo/runlist.c

@@ -442,7 +442,7 @@ int nvgpu_runlist_update_locked(struct gk20a *g, u32 runlist_id,
 }
 
 /* trigger host to expire current timeslice and reschedule runlist from front */
-int nvgpu_fifo_reschedule_runlist(struct channel_gk20a *ch, bool preempt_next,
+int nvgpu_runlist_reschedule(struct channel_gk20a *ch, bool preempt_next,
 		bool wait_preempt)
 {
 	struct gk20a *g = ch->g;

drivers/gpu/nvgpu/common/vgpu/gp10b/vgpu_hal_gp10b.c

@@ -44,6 +44,7 @@
 #include "hal/fifo/ramin_gm20b.h"
 #include "hal/fifo/ramin_gp10b.h"
 #include "hal/fifo/runlist_gk20a.h"
+#include "hal/fifo/runlist_fifo_gk20a.h"
 #include "hal/fifo/userd_gk20a.h"
 #include "hal/fifo/mmu_fault_gm20b.h"
 #include "hal/fifo/mmu_fault_gp10b.h"

drivers/gpu/nvgpu/common/vgpu/gv11b/vgpu_hal_gv11b.c

@@ -37,6 +37,7 @@
 #include "hal/fifo/ramin_gp10b.h"
 #include "hal/fifo/ramin_gv11b.h"
 #include "hal/fifo/runlist_gv11b.h"
+#include "hal/fifo/runlist_fifo_gv11b.h"
 #include "hal/fifo/tsg_gv11b.h"
 #include "hal/fifo/userd_gk20a.h"
 #include "hal/fifo/userd_gv11b.h"

drivers/gpu/nvgpu/hal/fifo/runlist_fifo_gk20a.c

@@ -0,0 +1,209 @@
+/*
+ * Copyright (c) 2011-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/io.h>
+#include <nvgpu/channel.h>
+#include <nvgpu/runlist.h>
+#include <nvgpu/gk20a.h>
+#include <nvgpu/engine_status.h>
+#include <nvgpu/engines.h>
+#include <nvgpu/gr/gr_falcon.h>
+
+#include "runlist_fifo_gk20a.h"
+
+#include <nvgpu/hw/gk20a/hw_fifo_gk20a.h>
+
+#define FECS_MAILBOX_0_ACK_RESTORE 0x4U
+
+int gk20a_runlist_reschedule(struct channel_gk20a *ch, bool preempt_next)
+{
+	return nvgpu_runlist_reschedule(ch, preempt_next, true);
+}
+
+u32 gk20a_runlist_count_max(void)
+{
+	return fifo_eng_runlist_base__size_1_v();
+}
+
+u32 gk20a_runlist_length_max(struct gk20a *g)
+{
+	return fifo_eng_runlist_length_max_v();
+}
+
+void gk20a_runlist_hw_submit(struct gk20a *g, u32 runlist_id,
+       u32 count, u32 buffer_index)
+{
+       struct fifo_runlist_info_gk20a *runlist = NULL;
+       u64 runlist_iova;
+
+       runlist = g->fifo.runlist_info[runlist_id];
+       runlist_iova = nvgpu_mem_get_addr(g, &runlist->mem[buffer_index]);
+
+       nvgpu_spinlock_acquire(&g->fifo.runlist_submit_lock);
+
+       if (count != 0U) {
+               nvgpu_writel(g, fifo_runlist_base_r(),
+                       fifo_runlist_base_ptr_f(u64_lo32(runlist_iova >> 12)) |
+                       nvgpu_aperture_mask(g, &runlist->mem[buffer_index],
+                               fifo_runlist_base_target_sys_mem_ncoh_f(),
+                               fifo_runlist_base_target_sys_mem_coh_f(),
+                               fifo_runlist_base_target_vid_mem_f()));
+       }
+
+       nvgpu_writel(g, fifo_runlist_r(),
+               fifo_runlist_engine_f(runlist_id) |
+               fifo_eng_runlist_length_f(count));
+
+       nvgpu_spinlock_release(&g->fifo.runlist_submit_lock);
+}
+
+int gk20a_runlist_wait_pending(struct gk20a *g, u32 runlist_id)
+{
+	struct nvgpu_timeout timeout;
+	u32 delay = POLL_DELAY_MIN_US;
+	int ret = 0;
+
+	ret = nvgpu_timeout_init(g, &timeout, nvgpu_get_poll_timeout(g),
+			   NVGPU_TIMER_CPU_TIMER);
+	if (ret != 0) {
+		nvgpu_err(g, "nvgpu_timeout_init failed err=%d", ret);
+		return ret;
+	}
+
+	ret = -ETIMEDOUT;
+	do {
+		if ((nvgpu_readl(g, fifo_eng_runlist_r(runlist_id)) &
+				fifo_eng_runlist_pending_true_f()) == 0U) {
+			ret = 0;
+			break;
+		}
+
+		nvgpu_usleep_range(delay, delay * 2U);
+		delay = min_t(u32, delay << 1, POLL_DELAY_MAX_US);
+	} while (nvgpu_timeout_expired(&timeout) == 0);
+
+	if (ret != 0) {
+		nvgpu_err(g, "runlist wait timeout: runlist id: %u",
+			runlist_id);
+	}
+
+	return ret;
+}
+
+void gk20a_runlist_write_state(struct gk20a *g, u32 runlists_mask,
+					 u32 runlist_state)
+{
+	u32 reg_val;
+	u32 reg_mask = 0U;
+	u32 i = 0U;
+
+	while (runlists_mask != 0U) {
+		if ((runlists_mask & BIT32(i)) != 0U) {
+			reg_mask |= fifo_sched_disable_runlist_m(i);
+		}
+		runlists_mask &= ~BIT32(i);
+		i++;
+	}
+
+	reg_val = nvgpu_readl(g, fifo_sched_disable_r());
+
+	if (runlist_state == RUNLIST_DISABLED) {
+		reg_val |= reg_mask;
+	} else {
+		reg_val &= ~reg_mask;
+	}
+
+	nvgpu_writel(g, fifo_sched_disable_r(), reg_val);
+}
+
+/* trigger host preempt of GR pending load ctx if that ctx is not for ch */
+int gk20a_fifo_reschedule_preempt_next(struct channel_gk20a *ch,
+		bool wait_preempt)
+{
+	struct gk20a *g = ch->g;
+	struct fifo_runlist_info_gk20a *runlist =
+		g->fifo.runlist_info[ch->runlist_id];
+	int ret = 0;
+	u32 gr_eng_id = 0;
+	u32 fecsstat0 = 0, fecsstat1 = 0;
+	u32 preempt_id;
+	u32 preempt_type = 0;
+	struct nvgpu_engine_status_info engine_status;
+
+	if (1U != nvgpu_engine_get_ids(
+		g, &gr_eng_id, 1, NVGPU_ENGINE_GR_GK20A)) {
+		return ret;
+	}
+	if ((runlist->eng_bitmask & BIT32(gr_eng_id)) == 0U) {
+		return ret;
+	}
+
+	if (wait_preempt && ((nvgpu_readl(g, fifo_preempt_r()) &
+				fifo_preempt_pending_true_f()) != 0U)) {
+		return ret;
+	}
+
+	fecsstat0 = g->ops.gr.falcon.read_fecs_ctxsw_mailbox(g,
+			NVGPU_GR_FALCON_FECS_CTXSW_MAILBOX0);
+	g->ops.engine_status.read_engine_status_info(g, gr_eng_id, &engine_status);
+	if (nvgpu_engine_status_is_ctxsw_switch(&engine_status)) {
+		nvgpu_engine_status_get_next_ctx_id_type(&engine_status,
+			&preempt_id, &preempt_type);
+	} else {
+		return ret;
+	}
+	if ((preempt_id == ch->tsgid) && (preempt_type != 0U)) {
+		return ret;
+	}
+	fecsstat1 = g->ops.gr.falcon.read_fecs_ctxsw_mailbox(g,
+			NVGPU_GR_FALCON_FECS_CTXSW_MAILBOX0);
+	if (fecsstat0 != FECS_MAILBOX_0_ACK_RESTORE ||
+		fecsstat1 != FECS_MAILBOX_0_ACK_RESTORE) {
+		/* preempt useless if FECS acked save and started restore */
+		return ret;
+	}
+
+	g->ops.fifo.preempt_trigger(g, preempt_id, preempt_type != 0U);
+#ifdef TRACEPOINTS_ENABLED
+	trace_gk20a_reschedule_preempt_next(ch->chid, fecsstat0,
+		engine_status.reg_data, fecsstat1,
+		g->ops.gr.falcon.read_fecs_ctxsw_mailbox(g,
+			NVGPU_GR_FALCON_FECS_CTXSW_MAILBOX0),
+		nvgpu_readl(g, fifo_preempt_r()));
+#endif
+	if (wait_preempt) {
+		if (g->ops.fifo.is_preempt_pending(g, preempt_id,
+			preempt_type) != 0) {
+				nvgpu_err(g, "fifo preempt timed out");
+				/*
+				 * This function does not care if preempt
+				 * times out since it is here only to improve
+				 * latency. If a timeout happens, it will be
+				 * handled by other fifo handling code.
+				 */
+		}
+	}
+#ifdef TRACEPOINTS_ENABLED
+	trace_gk20a_reschedule_preempted_next(ch->chid);
+#endif
+	return ret;
+}

drivers/gpu/nvgpu/hal/fifo/runlist_fifo_gk20a.h

@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2011-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef NVGPU_RUNLIST_FIFO_GK20A_H
+#define NVGPU_RUNLIST_FIFO_GK20A_H
+
+#include <nvgpu/types.h>
+
+struct channel_gk20a;
+struct tsg_gk20a;
+struct gk20a;
+
+int gk20a_runlist_reschedule(struct channel_gk20a *ch, bool preempt_next);
+int gk20a_fifo_reschedule_preempt_next(struct channel_gk20a *ch,
+		bool wait_preempt);
+u32 gk20a_runlist_count_max(void);
+u32 gk20a_runlist_length_max(struct gk20a *g);
+void gk20a_runlist_hw_submit(struct gk20a *g, u32 runlist_id,
+		u32 count, u32 buffer_index);
+int gk20a_runlist_wait_pending(struct gk20a *g, u32 runlist_id);
+void gk20a_runlist_write_state(struct gk20a *g, u32 runlists_mask,
+		u32 runlist_state);
+
+#endif /* NVGPU_RUNLIST_FIFO_GK20A_H */

drivers/gpu/nvgpu/hal/fifo/runlist_fifo_gv100.c

@@ -20,7 +20,7 @@
  * DEALINGS IN THE SOFTWARE.
  */
 
-#include "runlist_gv100.h"
+#include "runlist_fifo_gv100.h"
 
 #include <nvgpu/hw/gv100/hw_fifo_gv100.h>
 

drivers/gpu/nvgpu/hal/fifo/runlist_fifo_gv100.h

@@ -20,10 +20,11 @@
  * DEALINGS IN THE SOFTWARE.
  */
 
-#ifndef NVGPU_RUNLIST_GV100_H
+#ifndef NVGPU_RUNLIST_FIFO_GV100_H
+#define NVGPU_RUNLIST_FIFO_GV100_H
 
 #include <nvgpu/types.h>
 
 u32 gv100_runlist_count_max(void);
 
-#endif /* NVGPU_RUNLIST_GV100_H */
+#endif /* NVGPU_RUNLIST_FIFO_GV100_H */

drivers/gpu/nvgpu/hal/fifo/runlist_fifo_gv11b.c

@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2016-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/runlist.h>
+
+#include "runlist_fifo_gv11b.h"
+
+#include <nvgpu/hw/gv11b/hw_fifo_gv11b.h>
+
+int gv11b_runlist_reschedule(struct channel_gk20a *ch, bool preempt_next)
+{
+	/*
+	 * gv11b allows multiple outstanding preempts,
+	 * so always preempt next for best reschedule effect
+	 */
+	return nvgpu_runlist_reschedule(ch, true, false);
+}
+
+u32 gv11b_runlist_count_max(void)
+{
+	return fifo_eng_runlist_base__size_1_v();
+}

drivers/gpu/nvgpu/hal/fifo/runlist_fifo_gv11b.h

@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2016-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef NVGPU_RUNLIST_FIFO_GV11B_H
+#define NVGPU_RUNLIST_FIFO_GV11B_H
+
+#include <nvgpu/types.h>
+
+struct channel_gk20a;
+
+int gv11b_runlist_reschedule(struct channel_gk20a *ch, bool preempt_next);
+u32 gv11b_runlist_count_max(void);
+
+#endif /* NVGPU_RUNLIST_FIFO_GV11B_H */

drivers/gpu/nvgpu/hal/fifo/runlist_fifo_tu104.c

@@ -0,0 +1,91 @@
+/*
+ * Copyright (c) 2018-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/io.h>
+#include <nvgpu/runlist.h>
+#include <nvgpu/gk20a.h>
+
+#include "runlist_fifo_tu104.h"
+
+#include <nvgpu/hw/tu104/hw_fifo_tu104.h>
+
+u32 tu104_runlist_count_max(void)
+{
+	return fifo_runlist_base_lo__size_1_v();
+}
+
+void tu104_runlist_hw_submit(struct gk20a *g, u32 runlist_id,
+	u32 count, u32 buffer_index)
+{
+	struct fifo_runlist_info_gk20a *runlist = NULL;
+	u64 runlist_iova;
+	u32 runlist_iova_lo, runlist_iova_hi;
+
+	runlist = g->fifo.runlist_info[runlist_id];
+	runlist_iova = nvgpu_mem_get_addr(g, &runlist->mem[buffer_index]);
+
+	runlist_iova_lo = u64_lo32(runlist_iova) >>
+				fifo_runlist_base_lo_ptr_align_shift_v();
+	runlist_iova_hi = u64_hi32(runlist_iova);
+
+	if (count != 0U) {
+		nvgpu_writel(g, fifo_runlist_base_lo_r(runlist_id),
+			fifo_runlist_base_lo_ptr_lo_f(runlist_iova_lo) |
+			nvgpu_aperture_mask(g, &runlist->mem[buffer_index],
+				fifo_runlist_base_lo_target_sys_mem_ncoh_f(),
+				fifo_runlist_base_lo_target_sys_mem_coh_f(),
+				fifo_runlist_base_lo_target_vid_mem_f()));
+
+		nvgpu_writel(g, fifo_runlist_base_hi_r(runlist_id),
+			fifo_runlist_base_hi_ptr_hi_f(runlist_iova_hi));
+	}
+
+	nvgpu_writel(g, fifo_runlist_submit_r(runlist_id),
+		fifo_runlist_submit_length_f(count));
+}
+
+int tu104_runlist_wait_pending(struct gk20a *g, u32 runlist_id)
+{
+	struct nvgpu_timeout timeout;
+	u32 delay = POLL_DELAY_MIN_US;
+	int ret = -ETIMEDOUT;
+
+	ret = nvgpu_timeout_init(g, &timeout, nvgpu_get_poll_timeout(g),
+			   NVGPU_TIMER_CPU_TIMER);
+	if (ret != 0) {
+		return ret;
+	}
+
+	ret = -ETIMEDOUT;
+	do {
+		if ((nvgpu_readl(g, fifo_runlist_submit_info_r(runlist_id)) &
+			fifo_runlist_submit_info_pending_true_f()) == 0U) {
+			ret = 0;
+			break;
+		}
+
+		nvgpu_usleep_range(delay, delay * 2U);
+		delay = min_t(u32, delay << 1, POLL_DELAY_MAX_US);
+	} while (nvgpu_timeout_expired(&timeout) == 0);
+
+	return ret;
+}

drivers/gpu/nvgpu/hal/fifo/runlist_fifo_tu104.h

@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2018-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef NVGPU_RUNLIST_FIFO_TU104_H
+#define NVGPU_RUNLIST_FIFO_TU104_H
+
+#include <nvgpu/types.h>
+
+struct gk20a;
+
+u32 tu104_runlist_count_max(void);
+void tu104_runlist_hw_submit(struct gk20a *g, u32 runlist_id,
+		u32 count, u32 buffer_index);
+int tu104_runlist_wait_pending(struct gk20a *g, u32 runlist_id);
+
+#endif /* NVGPU_RUNLIST_FIFO_TU104_H */

drivers/gpu/nvgpu/hal/fifo/runlist_gk20a.c

@@ -28,11 +28,8 @@
 #include <nvgpu/engines.h>
 #include <nvgpu/gr/gr_falcon.h>
 
-#include <gk20a/fifo_gk20a.h>
-
 #include "runlist_gk20a.h"
 
-#include <nvgpu/hw/gk20a/hw_fifo_gk20a.h>
 #include <nvgpu/hw/gk20a/hw_ram_gk20a.h>
 
 #define FECS_MAILBOX_0_ACK_RESTORE 0x4U
@@ -40,99 +37,11 @@
 #define RL_MAX_TIMESLICE_TIMEOUT ram_rl_entry_timeslice_timeout_v(U32_MAX)
 #define RL_MAX_TIMESLICE_SCALE ram_rl_entry_timeslice_scale_v(U32_MAX)
 
-int gk20a_runlist_reschedule(struct channel_gk20a *ch, bool preempt_next)
-{
-	return nvgpu_fifo_reschedule_runlist(ch, preempt_next, true);
-}
-
-/* trigger host preempt of GR pending load ctx if that ctx is not for ch */
-int gk20a_fifo_reschedule_preempt_next(struct channel_gk20a *ch,
-		bool wait_preempt)
-{
-	struct gk20a *g = ch->g;
-	struct fifo_runlist_info_gk20a *runlist =
-		g->fifo.runlist_info[ch->runlist_id];
-	int ret = 0;
-	u32 gr_eng_id = 0;
-	u32 fecsstat0 = 0, fecsstat1 = 0;
-	u32 preempt_id;
-	u32 preempt_type = 0;
-	struct nvgpu_engine_status_info engine_status;
-
-	if (1U != nvgpu_engine_get_ids(
-		g, &gr_eng_id, 1, NVGPU_ENGINE_GR_GK20A)) {
-		return ret;
-	}
-	if ((runlist->eng_bitmask & BIT32(gr_eng_id)) == 0U) {
-		return ret;
-	}
-
-	if (wait_preempt && ((nvgpu_readl(g, fifo_preempt_r()) &
-				fifo_preempt_pending_true_f()) != 0U)) {
-		return ret;
-	}
-
-	fecsstat0 = g->ops.gr.falcon.read_fecs_ctxsw_mailbox(g,
-			NVGPU_GR_FALCON_FECS_CTXSW_MAILBOX0);
-	g->ops.engine_status.read_engine_status_info(g, gr_eng_id, &engine_status);
-	if (nvgpu_engine_status_is_ctxsw_switch(&engine_status)) {
-		nvgpu_engine_status_get_next_ctx_id_type(&engine_status,
-			&preempt_id, &preempt_type);
-	} else {
-		return ret;
-	}
-	if ((preempt_id == ch->tsgid) && (preempt_type != 0U)) {
-		return ret;
-	}
-	fecsstat1 = g->ops.gr.falcon.read_fecs_ctxsw_mailbox(g,
-			NVGPU_GR_FALCON_FECS_CTXSW_MAILBOX0);
-	if (fecsstat0 != FECS_MAILBOX_0_ACK_RESTORE ||
-		fecsstat1 != FECS_MAILBOX_0_ACK_RESTORE) {
-		/* preempt useless if FECS acked save and started restore */
-		return ret;
-	}
-
-	g->ops.fifo.preempt_trigger(g, preempt_id, preempt_type != 0U);
-#ifdef TRACEPOINTS_ENABLED
-	trace_gk20a_reschedule_preempt_next(ch->chid, fecsstat0,
-		engine_status.reg_data, fecsstat1,
-		g->ops.gr.falcon.read_fecs_ctxsw_mailbox(g,
-			NVGPU_GR_FALCON_FECS_CTXSW_MAILBOX0),
-		nvgpu_readl(g, fifo_preempt_r()));
-#endif
-	if (wait_preempt) {
-		if (g->ops.fifo.is_preempt_pending(g, preempt_id,
-			preempt_type) != 0) {
-				nvgpu_err(g, "fifo preempt timed out");
-				/*
-				 * This function does not care if preempt
-				 * times out since it is here only to improve
-				 * latency. If a timeout happens, it will be
-				 * handled by other fifo handling code.
-				 */
-		}
-	}
-#ifdef TRACEPOINTS_ENABLED
-	trace_gk20a_reschedule_preempted_next(ch->chid);
-#endif
-	return ret;
-}
-
-u32 gk20a_runlist_count_max(void)
-{
-	return fifo_eng_runlist_base__size_1_v();
-}
-
 u32 gk20a_runlist_entry_size(struct gk20a *g)
 {
 	return ram_rl_entry_size_v();
 }
 
-u32 gk20a_runlist_length_max(struct gk20a *g)
-{
-	return fifo_eng_runlist_length_max_v();
-}
-
 void gk20a_runlist_get_tsg_entry(struct tsg_gk20a *tsg,
 		u32 *runlist, u32 timeslice)
 {
@@ -166,90 +75,3 @@ void gk20a_runlist_get_ch_entry(struct channel_gk20a *ch, u32 *runlist)
 	runlist[0] = ram_rl_entry_chid_f(ch->chid);
 	runlist[1] = 0;
 }
-
-void gk20a_runlist_hw_submit(struct gk20a *g, u32 runlist_id,
-	u32 count, u32 buffer_index)
-{
-	struct fifo_runlist_info_gk20a *runlist = NULL;
-	u64 runlist_iova;
-
-	runlist = g->fifo.runlist_info[runlist_id];
-	runlist_iova = nvgpu_mem_get_addr(g, &runlist->mem[buffer_index]);
-
-	nvgpu_spinlock_acquire(&g->fifo.runlist_submit_lock);
-
-	if (count != 0U) {
-		nvgpu_writel(g, fifo_runlist_base_r(),
-			fifo_runlist_base_ptr_f(u64_lo32(runlist_iova >> 12)) |
-			nvgpu_aperture_mask(g, &runlist->mem[buffer_index],
-				fifo_runlist_base_target_sys_mem_ncoh_f(),
-				fifo_runlist_base_target_sys_mem_coh_f(),
-				fifo_runlist_base_target_vid_mem_f()));
-	}
-
-	nvgpu_writel(g, fifo_runlist_r(),
-		fifo_runlist_engine_f(runlist_id) |
-		fifo_eng_runlist_length_f(count));
-
-	nvgpu_spinlock_release(&g->fifo.runlist_submit_lock);
-}
-
-int gk20a_runlist_wait_pending(struct gk20a *g, u32 runlist_id)
-{
-	struct nvgpu_timeout timeout;
-	u32 delay = POLL_DELAY_MIN_US;
-	int ret = 0;
-
-	ret = nvgpu_timeout_init(g, &timeout, nvgpu_get_poll_timeout(g),
-			   NVGPU_TIMER_CPU_TIMER);
-	if (ret != 0) {
-		nvgpu_err(g, "nvgpu_timeout_init failed err=%d", ret);
-		return ret;
-	}
-
-	ret = -ETIMEDOUT;
-	do {
-		if ((nvgpu_readl(g, fifo_eng_runlist_r(runlist_id)) &
-				fifo_eng_runlist_pending_true_f()) == 0U) {
-			ret = 0;
-			break;
-		}
-
-		nvgpu_usleep_range(delay, delay * 2U);
-		delay = min_t(u32, delay << 1, POLL_DELAY_MAX_US);
-	} while (nvgpu_timeout_expired(&timeout) == 0);
-
-	if (ret != 0) {
-		nvgpu_err(g, "runlist wait timeout: runlist id: %u",
-			runlist_id);
-	}
-
-	return ret;
-}
-
-void gk20a_runlist_write_state(struct gk20a *g, u32 runlists_mask,
-					 u32 runlist_state)
-{
-	u32 reg_val;
-	u32 reg_mask = 0U;
-	u32 i = 0U;
-
-	while (runlists_mask != 0U) {
-		if ((runlists_mask & BIT32(i)) != 0U) {
-			reg_mask |= fifo_sched_disable_runlist_m(i);
-		}
-		runlists_mask &= ~BIT32(i);
-		i++;
-	}
-
-	reg_val = nvgpu_readl(g, fifo_sched_disable_r());
-
-	if (runlist_state == RUNLIST_DISABLED) {
-		reg_val |= reg_mask;
-	} else {
-		reg_val &= ~reg_mask;
-	}
-
-	nvgpu_writel(g, fifo_sched_disable_r(), reg_val);
-
-}

drivers/gpu/nvgpu/hal/fifo/runlist_gk20a.h

@@ -21,6 +21,7 @@
  */
 
 #ifndef NVGPU_RUNLIST_GK20A_H
+#define NVGPU_RUNLIST_GK20A_H
 
 #include <nvgpu/types.h>
 
@@ -28,19 +29,9 @@ struct channel_gk20a;
 struct tsg_gk20a;
 struct gk20a;
 
-int gk20a_runlist_reschedule(struct channel_gk20a *ch, bool preempt_next);
-int gk20a_fifo_reschedule_preempt_next(struct channel_gk20a *ch,
-		bool wait_preempt);
-u32 gk20a_runlist_count_max(void);
 u32 gk20a_runlist_entry_size(struct gk20a *g);
-u32 gk20a_runlist_length_max(struct gk20a *g);
 void gk20a_runlist_get_tsg_entry(struct tsg_gk20a *tsg,
 		u32 *runlist, u32 timeslice);
 void gk20a_runlist_get_ch_entry(struct channel_gk20a *ch, u32 *runlist);
-void gk20a_runlist_hw_submit(struct gk20a *g, u32 runlist_id,
-		u32 count, u32 buffer_index);
-int gk20a_runlist_wait_pending(struct gk20a *g, u32 runlist_id);
-void gk20a_runlist_write_state(struct gk20a *g, u32 runlists_mask,
-		u32 runlist_state);
 
 #endif /* NVGPU_RUNLIST_GK20A_H */

drivers/gpu/nvgpu/hal/fifo/runlist_gv11b.c

@@ -26,24 +26,11 @@
 
 #include "runlist_gv11b.h"
 
-#include <nvgpu/hw/gv11b/hw_fifo_gv11b.h>
 #include <nvgpu/hw/gv11b/hw_ram_gv11b.h>
 
 #define RL_MAX_TIMESLICE_TIMEOUT ram_rl_entry_tsg_timeslice_timeout_v(U32_MAX)
 #define RL_MAX_TIMESLICE_SCALE ram_rl_entry_tsg_timeslice_scale_v(U32_MAX)
 
-int gv11b_runlist_reschedule(struct channel_gk20a *ch, bool preempt_next)
-{
-	/* gv11b allows multiple outstanding preempts,
-	   so always preempt next for best reschedule effect */
-	return nvgpu_fifo_reschedule_runlist(ch, true, false);
-}
-
-u32 gv11b_runlist_count_max(void)
-{
-	return fifo_eng_runlist_base__size_1_v();
-}
-
 u32 gv11b_runlist_entry_size(struct gk20a *g)
 {
 	return ram_rl_entry_size_v();

drivers/gpu/nvgpu/hal/fifo/runlist_gv11b.h

@@ -21,14 +21,13 @@
  */
 
 #ifndef NVGPU_RUNLIST_GV11B_H
+#define NVGPU_RUNLIST_GV11B_H
 
 #include <nvgpu/types.h>
 
 struct channel_gk20a;
 struct tsg_gk20a;
 
-int gv11b_runlist_reschedule(struct channel_gk20a *ch, bool preempt_next);
-u32 gv11b_runlist_count_max(void);
 u32 gv11b_runlist_entry_size(struct gk20a *g);
 void gv11b_runlist_get_tsg_entry(struct tsg_gk20a *tsg,
 		u32 *runlist, u32 timeslice);

drivers/gpu/nvgpu/hal/fifo/runlist_tu104.c

@@ -20,81 +20,13 @@
  * DEALINGS IN THE SOFTWARE.
  */
 
-#include <nvgpu/io.h>
-#include <nvgpu/channel.h>
-#include <nvgpu/runlist.h>
-#include <nvgpu/gk20a.h>
-
-#include <tu104/fifo_tu104.h>
-
 #include "runlist_tu104.h"
 
-#include <nvgpu/hw/tu104/hw_fifo_tu104.h>
 #include <nvgpu/hw/tu104/hw_ram_tu104.h>
 
-u32 tu104_runlist_count_max(void)
-{
-	return fifo_runlist_base_lo__size_1_v();
-}
-
 u32 tu104_runlist_entry_size(struct gk20a *g)
 {
 	return ram_rl_entry_size_v();
 }
 
-void tu104_runlist_hw_submit(struct gk20a *g, u32 runlist_id,
-	u32 count, u32 buffer_index)
-{
-	struct fifo_runlist_info_gk20a *runlist = NULL;
-	u64 runlist_iova;
-	u32 runlist_iova_lo, runlist_iova_hi;
 
-	runlist = g->fifo.runlist_info[runlist_id];
-	runlist_iova = nvgpu_mem_get_addr(g, &runlist->mem[buffer_index]);
-
-	runlist_iova_lo = u64_lo32(runlist_iova) >>
-				fifo_runlist_base_lo_ptr_align_shift_v();
-	runlist_iova_hi = u64_hi32(runlist_iova);
-
-	if (count != 0U) {
-		nvgpu_writel(g, fifo_runlist_base_lo_r(runlist_id),
-			fifo_runlist_base_lo_ptr_lo_f(runlist_iova_lo) |
-			nvgpu_aperture_mask(g, &runlist->mem[buffer_index],
-				fifo_runlist_base_lo_target_sys_mem_ncoh_f(),
-				fifo_runlist_base_lo_target_sys_mem_coh_f(),
-				fifo_runlist_base_lo_target_vid_mem_f()));
-
-		nvgpu_writel(g, fifo_runlist_base_hi_r(runlist_id),
-			fifo_runlist_base_hi_ptr_hi_f(runlist_iova_hi));
-	}
-
-	nvgpu_writel(g, fifo_runlist_submit_r(runlist_id),
-		fifo_runlist_submit_length_f(count));
-}
-
-int tu104_runlist_wait_pending(struct gk20a *g, u32 runlist_id)
-{
-	struct nvgpu_timeout timeout;
-	u32 delay = POLL_DELAY_MIN_US;
-	int ret = -ETIMEDOUT;
-
-	ret = nvgpu_timeout_init(g, &timeout, nvgpu_get_poll_timeout(g),
-			   NVGPU_TIMER_CPU_TIMER);
-	if (ret != 0) {
-		return ret;
-	}
-
-	ret = -ETIMEDOUT;
-	do {
-		if ((nvgpu_readl(g, fifo_runlist_submit_info_r(runlist_id)) &
-			fifo_runlist_submit_info_pending_true_f()) == 0U) {
-			ret = 0;
-			break;
-		}
-
-		nvgpu_usleep_range(delay, delay * 2U);
-		delay = min_t(u32, delay << 1, POLL_DELAY_MAX_US);
-	} while (nvgpu_timeout_expired(&timeout) == 0);
-
-	return ret;
-}

drivers/gpu/nvgpu/hal/fifo/runlist_tu104.h

@@ -21,15 +21,12 @@
  */
 
 #ifndef NVGPU_RUNLIST_TU104_H
+#define NVGPU_RUNLIST_TU104_H
 
 #include <nvgpu/types.h>
 
 struct gk20a;
 
-u32 tu104_runlist_count_max(void);
 u32 tu104_runlist_entry_size(struct gk20a *g);
-void tu104_runlist_hw_submit(struct gk20a *g, u32 runlist_id,
-		u32 count, u32 buffer_index);
-int tu104_runlist_wait_pending(struct gk20a *g, u32 runlist_id);
 
 #endif /* NVGPU_RUNLIST_TU104_H */

drivers/gpu/nvgpu/hal/init/hal_gm20b.c

@@ -68,6 +68,7 @@
 #include "hal/fifo/ramin_gk20a.h"
 #include "hal/fifo/ramin_gm20b.h"
 #include "hal/fifo/runlist_gk20a.h"
+#include "hal/fifo/runlist_fifo_gk20a.h"
 #include "hal/fifo/tsg_gk20a.h"
 #include "hal/fifo/userd_gk20a.h"
 #include "hal/fifo/fifo_intr_gk20a.h"

drivers/gpu/nvgpu/hal/init/hal_gp10b.c

@@ -80,6 +80,7 @@
 #include "hal/fifo/ramin_gm20b.h"
 #include "hal/fifo/ramin_gp10b.h"
 #include "hal/fifo/runlist_gk20a.h"
+#include "hal/fifo/runlist_fifo_gk20a.h"
 #include "hal/fifo/tsg_gk20a.h"
 #include "hal/fifo/userd_gk20a.h"
 #include "hal/fifo/fifo_intr_gk20a.h"

drivers/gpu/nvgpu/hal/init/hal_gv100.c

@@ -79,7 +79,9 @@
 #include "hal/fifo/ramin_gv11b.h"
 #include "hal/fifo/runlist_gk20a.h"
 #include "hal/fifo/runlist_gv11b.h"
-#include "hal/fifo/runlist_gv100.h"
+#include "hal/fifo/runlist_fifo_gk20a.h"
+#include "hal/fifo/runlist_fifo_gv11b.h"
+#include "hal/fifo/runlist_fifo_gv100.h"
 #include "hal/fifo/tsg_gv11b.h"
 #include "hal/fifo/userd_gk20a.h"
 #include "hal/fifo/userd_gv11b.h"

drivers/gpu/nvgpu/hal/init/hal_gv11b.c

@@ -82,6 +82,8 @@
 #include "hal/fifo/ramin_gv11b.h"
 #include "hal/fifo/runlist_gk20a.h"
 #include "hal/fifo/runlist_gv11b.h"
+#include "hal/fifo/runlist_fifo_gk20a.h"
+#include "hal/fifo/runlist_fifo_gv11b.h"
 #include "hal/fifo/tsg_gv11b.h"
 #include "hal/fifo/userd_gk20a.h"
 #include "hal/fifo/userd_gv11b.h"

drivers/gpu/nvgpu/hal/init/hal_tu104.c

@@ -85,6 +85,9 @@
 #include "hal/fifo/runlist_gk20a.h"
 #include "hal/fifo/runlist_gv11b.h"
 #include "hal/fifo/runlist_tu104.h"
+#include "hal/fifo/runlist_fifo_gk20a.h"
+#include "hal/fifo/runlist_fifo_gv11b.h"
+#include "hal/fifo/runlist_fifo_tu104.h"
 #include "hal/fifo/tsg_gv11b.h"
 #include "hal/fifo/userd_gk20a.h"
 #include "hal/fifo/userd_gv11b.h"

drivers/gpu/nvgpu/include/nvgpu/runlist.h

@@ -47,7 +47,7 @@ int nvgpu_runlist_update_locked(struct gk20a *g, u32 runlist_id,
 					    struct channel_gk20a *ch, bool add,
 					    bool wait_for_finish);
 
-int nvgpu_fifo_reschedule_runlist(struct channel_gk20a *ch, bool preempt_next,
+int nvgpu_runlist_reschedule(struct channel_gk20a *ch, bool preempt_next,
 		bool wait_preempt);
 
 int gk20a_runlist_update_for_channel(struct gk20a *g, u32 runlist_id,