gpu: nvgpu: add runlist unit to common

Extract non-chip-specific code that manages the runlists (init, update,
reschedule etc.) to a new file in the common directory. Move the
declarations to a new matching runlist.h header.

Jira NVGPU-1309

Change-Id: I3c7e0032899516487037f47ddc9a7e7aa4b0b33a
Signed-off-by: Konsta Holtta <kholtta@nvidia.com>
Reviewed-on: https://git-master.nvidia.com/r/1978058
Reviewed-by: mobile promotions <svcmobile_promotions@nvidia.com>
Tested-by: mobile promotions <svcmobile_promotions@nvidia.com>

drivers/gpu/nvgpu/Makefile

@@ -278,6 +278,7 @@ nvgpu-y += \
 	common/fifo/channel.o \
 	common/fifo/submit.o \
 	common/fifo/tsg.o \
+	common/fifo/runlist.o \
 	common/ecc.o \
 	common/ce2.o \
 	common/debugger.o \

drivers/gpu/nvgpu/Makefile.sources

@@ -1,6 +1,6 @@
 # -*- mode: makefile -*-
 #
-# Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+# Copyright (c) 2018-2019, NVIDIA CORPORATION.  All rights reserved.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
 # copy of this software and associated documentation files (the "Software"),
@@ -150,6 +150,7 @@ srcs :=	os/posix/nvgpu.c \
 	common/fifo/channel.c \
 	common/fifo/submit.c \
 	common/fifo/tsg.c \
+	common/fifo/runlist.c \
 	common/mc/mc.c \
 	common/mc/mc_gm20b.c \
 	common/mc/mc_gp10b.c  \

drivers/gpu/nvgpu/common/fifo/channel.c

@@ -1,7 +1,7 @@
 /*
  * GK20A Graphics channel
  *
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2011-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -47,6 +47,7 @@
 #include <nvgpu/gk20a.h>
 #include <nvgpu/channel.h>
 #include <nvgpu/channel_sync.h>
+#include <nvgpu/runlist.h>
 
 #include "gk20a/fence_gk20a.h"
 #include "gk20a/gr_gk20a.h"

drivers/gpu/nvgpu/common/fifo/runlist.c

@@ -0,0 +1,642 @@
+/*
+ * Copyright (c) 2011-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/gk20a.h>
+#include <nvgpu/channel.h>
+#include <nvgpu/fifo.h>
+#include <nvgpu/runlist.h>
+#include <nvgpu/bug.h>
+
+static u32 nvgpu_runlist_append_tsg(struct gk20a *g,
+		struct fifo_runlist_info_gk20a *runlist,
+		u32 **runlist_entry,
+		u32 *entries_left,
+		struct tsg_gk20a *tsg)
+{
+	struct fifo_gk20a *f = &g->fifo;
+	u32 runlist_entry_words = f->runlist_entry_size / (u32)sizeof(u32);
+	struct channel_gk20a *ch;
+	u32 count = 0;
+
+	nvgpu_log_fn(f->g, " ");
+
+	if (*entries_left == 0U) {
+		return RUNLIST_APPEND_FAILURE;
+	}
+
+	/* add TSG entry */
+	nvgpu_log_info(g, "add TSG %d to runlist", tsg->tsgid);
+	g->ops.fifo.get_tsg_runlist_entry(tsg, *runlist_entry);
+	nvgpu_log_info(g, "tsg rl entries left %d runlist [0] %x [1] %x",
+			*entries_left,
+			(*runlist_entry)[0], (*runlist_entry)[1]);
+	*runlist_entry += runlist_entry_words;
+	count++;
+	(*entries_left)--;
+
+	nvgpu_rwsem_down_read(&tsg->ch_list_lock);
+	/* add runnable channels bound to this TSG */
+	nvgpu_list_for_each_entry(ch, &tsg->ch_list,
+			channel_gk20a, ch_entry) {
+		if (!test_bit((int)ch->chid,
+			      runlist->active_channels)) {
+			continue;
+		}
+
+		if (*entries_left == 0U) {
+			nvgpu_rwsem_up_read(&tsg->ch_list_lock);
+			return RUNLIST_APPEND_FAILURE;
+		}
+
+		nvgpu_log_info(g, "add channel %d to runlist",
+			ch->chid);
+		g->ops.fifo.get_ch_runlist_entry(ch, *runlist_entry);
+		nvgpu_log_info(g, "rl entries left %d runlist [0] %x [1] %x",
+			*entries_left,
+			(*runlist_entry)[0], (*runlist_entry)[1]);
+		count++;
+		*runlist_entry += runlist_entry_words;
+		(*entries_left)--;
+	}
+	nvgpu_rwsem_up_read(&tsg->ch_list_lock);
+
+	return count;
+}
+
+
+static u32 nvgpu_runlist_append_prio(struct fifo_gk20a *f,
+				struct fifo_runlist_info_gk20a *runlist,
+				u32 **runlist_entry,
+				u32 *entries_left,
+				u32 interleave_level)
+{
+	u32 count = 0;
+	unsigned long tsgid;
+
+	nvgpu_log_fn(f->g, " ");
+
+	for_each_set_bit(tsgid, runlist->active_tsgs, f->num_channels) {
+		struct tsg_gk20a *tsg = &f->tsg[tsgid];
+		u32 entries;
+
+		if (tsg->interleave_level == interleave_level) {
+			entries = nvgpu_runlist_append_tsg(f->g, runlist,
+					runlist_entry, entries_left, tsg);
+			if (entries == RUNLIST_APPEND_FAILURE) {
+				return RUNLIST_APPEND_FAILURE;
+			}
+			count += entries;
+		}
+	}
+
+	return count;
+}
+
+static u32 nvgpu_runlist_append_hi(struct fifo_gk20a *f,
+				struct fifo_runlist_info_gk20a *runlist,
+				u32 **runlist_entry,
+				u32 *entries_left)
+{
+	nvgpu_log_fn(f->g, " ");
+
+	/*
+	 * No higher levels - this is where the "recursion" ends; just add all
+	 * active TSGs at this level.
+	 */
+	return nvgpu_runlist_append_prio(f, runlist, runlist_entry,
+			entries_left,
+			NVGPU_FIFO_RUNLIST_INTERLEAVE_LEVEL_HIGH);
+}
+
+static u32 nvgpu_runlist_append_med(struct fifo_gk20a *f,
+				struct fifo_runlist_info_gk20a *runlist,
+				u32 **runlist_entry,
+				u32 *entries_left)
+{
+	u32 count = 0;
+	unsigned long tsgid;
+
+	nvgpu_log_fn(f->g, " ");
+
+	for_each_set_bit(tsgid, runlist->active_tsgs, f->num_channels) {
+		struct tsg_gk20a *tsg = &f->tsg[tsgid];
+		u32 entries;
+
+		if (tsg->interleave_level !=
+				NVGPU_FIFO_RUNLIST_INTERLEAVE_LEVEL_MEDIUM) {
+			continue;
+		}
+
+		/* LEVEL_MEDIUM list starts with a LEVEL_HIGH, if any */
+
+		entries = nvgpu_runlist_append_hi(f, runlist,
+				runlist_entry, entries_left);
+		if (entries == RUNLIST_APPEND_FAILURE) {
+			return RUNLIST_APPEND_FAILURE;
+		}
+		count += entries;
+
+		entries = nvgpu_runlist_append_tsg(f->g, runlist,
+				runlist_entry, entries_left, tsg);
+		if (entries == RUNLIST_APPEND_FAILURE) {
+			return RUNLIST_APPEND_FAILURE;
+		}
+		count += entries;
+	}
+
+	return count;
+}
+
+static u32 nvgpu_runlist_append_low(struct fifo_gk20a *f,
+				struct fifo_runlist_info_gk20a *runlist,
+				u32 **runlist_entry,
+				u32 *entries_left)
+{
+	u32 count = 0;
+	unsigned long tsgid;
+
+	nvgpu_log_fn(f->g, " ");
+
+	for_each_set_bit(tsgid, runlist->active_tsgs, f->num_channels) {
+		struct tsg_gk20a *tsg = &f->tsg[tsgid];
+		u32 entries;
+
+		if (tsg->interleave_level !=
+				NVGPU_FIFO_RUNLIST_INTERLEAVE_LEVEL_LOW) {
+			continue;
+		}
+
+		/* The medium level starts with the highs, if any. */
+
+		entries = nvgpu_runlist_append_med(f, runlist,
+				runlist_entry, entries_left);
+		if (entries == RUNLIST_APPEND_FAILURE) {
+			return RUNLIST_APPEND_FAILURE;
+		}
+		count += entries;
+
+		entries = nvgpu_runlist_append_hi(f, runlist,
+				runlist_entry, entries_left);
+		if (entries == RUNLIST_APPEND_FAILURE) {
+			return RUNLIST_APPEND_FAILURE;
+		}
+		count += entries;
+
+		entries = nvgpu_runlist_append_tsg(f->g, runlist,
+				runlist_entry, entries_left, tsg);
+		if (entries == RUNLIST_APPEND_FAILURE) {
+			return RUNLIST_APPEND_FAILURE;
+		}
+		count += entries;
+	}
+
+	if (count == 0U) {
+		/*
+		 * No transitions to fill with higher levels, so add
+		 * the next level once. If that's empty too, we have only
+		 * LEVEL_HIGH jobs.
+		 */
+		count = nvgpu_runlist_append_med(f, runlist,
+				runlist_entry, entries_left);
+		if (count == 0U) {
+			count = nvgpu_runlist_append_hi(f, runlist,
+					runlist_entry, entries_left);
+		}
+	}
+
+	return count;
+}
+
+static u32 nvgpu_runlist_append_flat(struct fifo_gk20a *f,
+				struct fifo_runlist_info_gk20a *runlist,
+				u32 **runlist_entry,
+				u32 *entries_left)
+{
+	u32 count = 0, entries, i;
+
+	nvgpu_log_fn(f->g, " ");
+
+	/* Group by priority but don't interleave. High comes first. */
+
+	for (i = 0; i < NVGPU_FIFO_RUNLIST_INTERLEAVE_NUM_LEVELS; i++) {
+		u32 level = NVGPU_FIFO_RUNLIST_INTERLEAVE_LEVEL_HIGH - i;
+
+		entries = nvgpu_runlist_append_prio(f, runlist, runlist_entry,
+				entries_left, level);
+		if (entries == RUNLIST_APPEND_FAILURE) {
+			return RUNLIST_APPEND_FAILURE;
+		}
+		count += entries;
+	}
+
+	return count;
+}
+
+u32 nvgpu_runlist_construct_locked(struct fifo_gk20a *f,
+				struct fifo_runlist_info_gk20a *runlist,
+				u32 buf_id,
+				u32 max_entries)
+{
+	u32 *runlist_entry_base = runlist->mem[buf_id].cpu_va;
+
+	nvgpu_log_fn(f->g, " ");
+
+	/*
+	 * The entry pointer and capacity counter that live on the stack here
+	 * keep track of the current position and the remaining space when tsg
+	 * and channel entries are ultimately appended.
+	 */
+	if (f->g->runlist_interleave) {
+		return nvgpu_runlist_append_low(f, runlist,
+				&runlist_entry_base, &max_entries);
+	} else {
+		return nvgpu_runlist_append_flat(f, runlist,
+				&runlist_entry_base, &max_entries);
+	}
+}
+
+int gk20a_fifo_update_runlist_locked(struct gk20a *g, u32 runlist_id,
+					    u32 chid, bool add,
+					    bool wait_for_finish)
+{
+	int ret = 0;
+	struct fifo_gk20a *f = &g->fifo;
+	struct fifo_runlist_info_gk20a *runlist = NULL;
+	u64 runlist_iova;
+	u32 new_buf;
+	struct channel_gk20a *ch = NULL;
+	struct tsg_gk20a *tsg = NULL;
+
+	runlist = &f->runlist_info[runlist_id];
+
+	/* valid channel, add/remove it from active list.
+	   Otherwise, keep active list untouched for suspend/resume. */
+	if (chid != FIFO_INVAL_CHANNEL_ID) {
+		ch = &f->channel[chid];
+		if (gk20a_is_channel_marked_as_tsg(ch)) {
+			tsg = &f->tsg[ch->tsgid];
+		}
+
+		if (add) {
+			if (test_and_set_bit(chid,
+				runlist->active_channels)) {
+				return 0;
+			}
+			if ((tsg != NULL) && (++tsg->num_active_channels != 0U)) {
+				set_bit((int)f->channel[chid].tsgid,
+					runlist->active_tsgs);
+			}
+		} else {
+			if (!test_and_clear_bit(chid,
+				runlist->active_channels)) {
+				return 0;
+			}
+			if ((tsg != NULL) &&
+			    (--tsg->num_active_channels == 0U)) {
+				clear_bit((int)f->channel[chid].tsgid,
+					runlist->active_tsgs);
+			}
+		}
+	}
+
+	/* There just 2 buffers */
+	new_buf = runlist->cur_buffer == 0U ? 1U : 0U;
+
+	runlist_iova = nvgpu_mem_get_addr(g, &runlist->mem[new_buf]);
+
+	nvgpu_log_info(g, "runlist_id : %d, switch to new buffer 0x%16llx",
+		runlist_id, (u64)runlist_iova);
+
+	if (runlist_iova == 0ULL) {
+		ret = -EINVAL;
+		goto clean_up;
+	}
+
+	if (chid != FIFO_INVAL_CHANNEL_ID || /* add/remove a valid channel */
+	    add /* resume to add all channels back */) {
+		u32 num_entries;
+
+		num_entries = nvgpu_runlist_construct_locked(f,
+						runlist,
+						new_buf,
+						f->num_runlist_entries);
+		if (num_entries == RUNLIST_APPEND_FAILURE) {
+			ret = -E2BIG;
+			goto clean_up;
+		}
+		runlist->count = num_entries;
+		WARN_ON(runlist->count > f->num_runlist_entries);
+	} else {
+		/* suspend to remove all channels */
+		runlist->count = 0;
+	}
+
+	g->ops.fifo.runlist_hw_submit(g, runlist_id, runlist->count, new_buf);
+
+	if (wait_for_finish) {
+		ret = g->ops.fifo.runlist_wait_pending(g, runlist_id);
+
+		if (ret == -ETIMEDOUT) {
+			nvgpu_err(g, "runlist %d update timeout", runlist_id);
+			/* trigger runlist update timeout recovery */
+			return ret;
+
+		} else if (ret == -EINTR) {
+			nvgpu_err(g, "runlist update interrupted");
+		}
+	}
+
+	runlist->cur_buffer = new_buf;
+
+clean_up:
+	return ret;
+}
+
+/* trigger host to expire current timeslice and reschedule runlist from front */
+int nvgpu_fifo_reschedule_runlist(struct channel_gk20a *ch, bool preempt_next,
+		bool wait_preempt)
+{
+	struct gk20a *g = ch->g;
+	struct fifo_runlist_info_gk20a *runlist;
+	u32 token = PMU_INVALID_MUTEX_OWNER_ID;
+	int mutex_ret;
+	int ret = 0;
+
+	runlist = &g->fifo.runlist_info[ch->runlist_id];
+	if (nvgpu_mutex_tryacquire(&runlist->runlist_lock) == 0) {
+		return -EBUSY;
+	}
+
+	mutex_ret = nvgpu_pmu_mutex_acquire(
+		&g->pmu, PMU_MUTEX_ID_FIFO, &token);
+
+	g->ops.fifo.runlist_hw_submit(
+		g, ch->runlist_id, runlist->count, runlist->cur_buffer);
+
+	if (preempt_next) {
+		g->ops.fifo.reschedule_preempt_next_locked(ch, wait_preempt);
+	}
+
+	g->ops.fifo.runlist_wait_pending(g, ch->runlist_id);
+
+	if (mutex_ret == 0) {
+		nvgpu_pmu_mutex_release(
+			&g->pmu, PMU_MUTEX_ID_FIFO, &token);
+	}
+	nvgpu_mutex_release(&runlist->runlist_lock);
+
+	return ret;
+}
+
+static void gk20a_fifo_runlist_reset_engines(struct gk20a *g, u32 runlist_id)
+{
+	u32 engines = g->ops.fifo.runlist_busy_engines(g, runlist_id);
+
+	if (engines != 0U) {
+		gk20a_fifo_recover(g, engines, ~(u32)0, false, false, true,
+				RC_TYPE_RUNLIST_UPDATE_TIMEOUT);
+	}
+}
+
+/* add/remove a channel from runlist
+   special cases below: runlist->active_channels will NOT be changed.
+   (chid == ~0 && !add) means remove all active channels from runlist.
+   (chid == ~0 &&  add) means restore all active channels on runlist. */
+int gk20a_fifo_update_runlist(struct gk20a *g, u32 runlist_id, u32 chid,
+			      bool add, bool wait_for_finish)
+{
+	struct fifo_runlist_info_gk20a *runlist = NULL;
+	struct fifo_gk20a *f = &g->fifo;
+	u32 token = PMU_INVALID_MUTEX_OWNER_ID;
+	int mutex_ret;
+	int ret = 0;
+
+	nvgpu_log_fn(g, " ");
+
+	runlist = &f->runlist_info[runlist_id];
+
+	nvgpu_mutex_acquire(&runlist->runlist_lock);
+
+	mutex_ret = nvgpu_pmu_mutex_acquire(&g->pmu, PMU_MUTEX_ID_FIFO, &token);
+
+	ret = gk20a_fifo_update_runlist_locked(g, runlist_id, chid, add,
+					       wait_for_finish);
+
+	if (mutex_ret == 0) {
+		nvgpu_pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token);
+	}
+
+	nvgpu_mutex_release(&runlist->runlist_lock);
+
+	if (ret == -ETIMEDOUT) {
+		gk20a_fifo_runlist_reset_engines(g, runlist_id);
+	}
+
+	return ret;
+}
+
+int gk20a_fifo_update_runlist_ids(struct gk20a *g, u32 runlist_ids, u32 chid,
+				bool add, bool wait_for_finish)
+{
+	int ret = -EINVAL;
+	unsigned long runlist_id = 0;
+	int errcode;
+	unsigned long ulong_runlist_ids = (unsigned long)runlist_ids;
+
+	if (g == NULL) {
+		goto end;
+	}
+
+	ret = 0;
+	for_each_set_bit(runlist_id, &ulong_runlist_ids, 32U) {
+		/* Capture the last failure error code */
+		errcode = g->ops.fifo.update_runlist(g, (u32)runlist_id, chid,
+						add, wait_for_finish);
+		if (errcode != 0) {
+			nvgpu_err(g,
+				"failed to update_runlist %lu %d",
+				runlist_id, errcode);
+			ret = errcode;
+		}
+	}
+end:
+	return ret;
+}
+
+const char *gk20a_fifo_interleave_level_name(u32 interleave_level)
+{
+	const char *ret_string = NULL;
+
+	switch (interleave_level) {
+	case NVGPU_FIFO_RUNLIST_INTERLEAVE_LEVEL_LOW:
+		ret_string = "LOW";
+		break;
+
+	case NVGPU_FIFO_RUNLIST_INTERLEAVE_LEVEL_MEDIUM:
+		ret_string = "MEDIUM";
+		break;
+
+	case NVGPU_FIFO_RUNLIST_INTERLEAVE_LEVEL_HIGH:
+		ret_string = "HIGH";
+		break;
+
+	default:
+		ret_string = "?";
+		break;
+	}
+
+	return ret_string;
+}
+
+void gk20a_fifo_delete_runlist(struct fifo_gk20a *f)
+{
+	u32 i;
+	u32 runlist_id;
+	struct fifo_runlist_info_gk20a *runlist;
+	struct gk20a *g = NULL;
+
+	if ((f == NULL) || (f->runlist_info == NULL)) {
+		return;
+	}
+
+	g = f->g;
+
+	for (runlist_id = 0; runlist_id < f->max_runlists; runlist_id++) {
+		runlist = &f->runlist_info[runlist_id];
+		for (i = 0; i < MAX_RUNLIST_BUFFERS; i++) {
+			nvgpu_dma_free(g, &runlist->mem[i]);
+		}
+
+		nvgpu_kfree(g, runlist->active_channels);
+		runlist->active_channels = NULL;
+
+		nvgpu_kfree(g, runlist->active_tsgs);
+		runlist->active_tsgs = NULL;
+
+		nvgpu_mutex_destroy(&runlist->runlist_lock);
+
+	}
+	(void) memset(f->runlist_info, 0,
+		(sizeof(struct fifo_runlist_info_gk20a) * f->max_runlists));
+
+	nvgpu_kfree(g, f->runlist_info);
+	f->runlist_info = NULL;
+	f->max_runlists = 0;
+}
+
+int nvgpu_init_runlist(struct gk20a *g, struct fifo_gk20a *f)
+{
+	struct fifo_runlist_info_gk20a *runlist;
+	struct fifo_engine_info_gk20a *engine_info;
+	unsigned int runlist_id;
+	u32 i;
+	size_t runlist_size;
+	u32 active_engine_id, pbdma_id, engine_id;
+	int err = 0;
+
+	nvgpu_log_fn(g, " ");
+
+	f->max_runlists = g->ops.fifo.eng_runlist_base_size();
+	f->runlist_info = nvgpu_kzalloc(g,
+					sizeof(struct fifo_runlist_info_gk20a) *
+					f->max_runlists);
+	if (f->runlist_info == NULL) {
+		goto clean_up_runlist;
+	}
+
+	(void) memset(f->runlist_info, 0,
+		(sizeof(struct fifo_runlist_info_gk20a) * f->max_runlists));
+
+	for (runlist_id = 0; runlist_id < f->max_runlists; runlist_id++) {
+		runlist = &f->runlist_info[runlist_id];
+
+		runlist->active_channels =
+			nvgpu_kzalloc(g, DIV_ROUND_UP(f->num_channels,
+						      BITS_PER_BYTE));
+		if (runlist->active_channels == NULL) {
+			goto clean_up_runlist;
+		}
+
+		runlist->active_tsgs =
+			nvgpu_kzalloc(g, DIV_ROUND_UP(f->num_channels,
+						      BITS_PER_BYTE));
+		if (runlist->active_tsgs == NULL) {
+			goto clean_up_runlist;
+		}
+
+		runlist_size  = (size_t)f->runlist_entry_size *
+				(size_t)f->num_runlist_entries;
+		nvgpu_log(g, gpu_dbg_info,
+				"runlist_entries %d runlist size %zu",
+				f->num_runlist_entries, runlist_size);
+
+		for (i = 0; i < MAX_RUNLIST_BUFFERS; i++) {
+			err = nvgpu_dma_alloc_flags_sys(g,
+					NVGPU_DMA_PHYSICALLY_ADDRESSED,
+					runlist_size,
+					&runlist->mem[i]);
+			if (err != 0) {
+				nvgpu_err(g, "memory allocation failed");
+				goto clean_up_runlist;
+			}
+		}
+
+		err = nvgpu_mutex_init(&runlist->runlist_lock);
+		if (err != 0) {
+			nvgpu_err(g,
+				"Error in runlist_lock mutex initialization");
+			goto clean_up_runlist;
+		}
+
+		/* None of buffers is pinned if this value doesn't change.
+		    Otherwise, one of them (cur_buffer) must have been pinned. */
+		runlist->cur_buffer = MAX_RUNLIST_BUFFERS;
+
+		for (pbdma_id = 0; pbdma_id < f->num_pbdma; pbdma_id++) {
+			if ((f->pbdma_map[pbdma_id] & BIT32(runlist_id)) != 0U) {
+				runlist->pbdma_bitmask |= BIT32(pbdma_id);
+			}
+		}
+		nvgpu_log(g, gpu_dbg_info, "runlist %d : pbdma bitmask 0x%x",
+				 runlist_id, runlist->pbdma_bitmask);
+
+		for (engine_id = 0; engine_id < f->num_engines; ++engine_id) {
+			active_engine_id = f->active_engines_list[engine_id];
+			engine_info = &f->engine_info[active_engine_id];
+
+			if ((engine_info != NULL) &&
+			    (engine_info->runlist_id == runlist_id)) {
+				runlist->eng_bitmask |= BIT(active_engine_id);
+			}
+		}
+		nvgpu_log(g, gpu_dbg_info, "runlist %d : act eng bitmask 0x%x",
+				 runlist_id, runlist->eng_bitmask);
+	}
+
+	nvgpu_log_fn(g, "done");
+	return 0;
+
+clean_up_runlist:
+	gk20a_fifo_delete_runlist(f);
+	nvgpu_log_fn(g, "fail");
+	return err;
+}

drivers/gpu/nvgpu/gk20a/fifo_gk20a.c

@@ -44,6 +44,7 @@
 #include <nvgpu/io.h>
 #include <nvgpu/utils.h>
 #include <nvgpu/fifo.h>
+#include <nvgpu/runlist.h>
 #include <nvgpu/gk20a.h>
 #include <nvgpu/channel.h>
 #include <nvgpu/unit.h>
@@ -393,42 +394,6 @@ u32 gk20a_fifo_engine_interrupt_mask(struct gk20a *g)
 	return eng_intr_mask;
 }
 
-void gk20a_fifo_delete_runlist(struct fifo_gk20a *f)
-{
-	u32 i;
-	u32 runlist_id;
-	struct fifo_runlist_info_gk20a *runlist;
-	struct gk20a *g = NULL;
-
-	if ((f == NULL) || (f->runlist_info == NULL)) {
-		return;
-	}
-
-	g = f->g;
-
-	for (runlist_id = 0; runlist_id < f->max_runlists; runlist_id++) {
-		runlist = &f->runlist_info[runlist_id];
-		for (i = 0; i < MAX_RUNLIST_BUFFERS; i++) {
-			nvgpu_dma_free(g, &runlist->mem[i]);
-		}
-
-		nvgpu_kfree(g, runlist->active_channels);
-		runlist->active_channels = NULL;
-
-		nvgpu_kfree(g, runlist->active_tsgs);
-		runlist->active_tsgs = NULL;
-
-		nvgpu_mutex_destroy(&runlist->runlist_lock);
-
-	}
-	(void) memset(f->runlist_info, 0,
-		(sizeof(struct fifo_runlist_info_gk20a) * f->max_runlists));
-
-	nvgpu_kfree(g, f->runlist_info);
-	f->runlist_info = NULL;
-	f->max_runlists = 0;
-}
-
 static void gk20a_remove_fifo_support(struct fifo_gk20a *f)
 {
 	struct gk20a *g = f->g;
@@ -482,104 +447,6 @@ static void gk20a_remove_fifo_support(struct fifo_gk20a *f)
 	f->active_engines_list = NULL;
 }
 
-static int init_runlist(struct gk20a *g, struct fifo_gk20a *f)
-{
-	struct fifo_runlist_info_gk20a *runlist;
-	struct fifo_engine_info_gk20a *engine_info;
-	unsigned int runlist_id;
-	u32 i;
-	size_t runlist_size;
-	u32 active_engine_id, pbdma_id, engine_id;
-	int err = 0;
-
-	nvgpu_log_fn(g, " ");
-
-	f->max_runlists = g->ops.fifo.eng_runlist_base_size();
-	f->runlist_info = nvgpu_kzalloc(g,
-					sizeof(struct fifo_runlist_info_gk20a) *
-					f->max_runlists);
-	if (f->runlist_info == NULL) {
-		goto clean_up_runlist;
-	}
-
-	(void) memset(f->runlist_info, 0,
-		(sizeof(struct fifo_runlist_info_gk20a) * f->max_runlists));
-
-	for (runlist_id = 0; runlist_id < f->max_runlists; runlist_id++) {
-		runlist = &f->runlist_info[runlist_id];
-
-		runlist->active_channels =
-			nvgpu_kzalloc(g, DIV_ROUND_UP(f->num_channels,
-						      BITS_PER_BYTE));
-		if (runlist->active_channels == NULL) {
-			goto clean_up_runlist;
-		}
-
-		runlist->active_tsgs =
-			nvgpu_kzalloc(g, DIV_ROUND_UP(f->num_channels,
-						      BITS_PER_BYTE));
-		if (runlist->active_tsgs == NULL) {
-			goto clean_up_runlist;
-		}
-
-		runlist_size  = (size_t)f->runlist_entry_size *
-				(size_t)f->num_runlist_entries;
-		nvgpu_log(g, gpu_dbg_info,
-				"runlist_entries %d runlist size %zu",
-				f->num_runlist_entries, runlist_size);
-
-		for (i = 0; i < MAX_RUNLIST_BUFFERS; i++) {
-			err = nvgpu_dma_alloc_flags_sys(g,
-					NVGPU_DMA_PHYSICALLY_ADDRESSED,
-					runlist_size,
-					&runlist->mem[i]);
-			if (err != 0) {
-				nvgpu_err(g, "memory allocation failed");
-				goto clean_up_runlist;
-			}
-		}
-
-		err = nvgpu_mutex_init(&runlist->runlist_lock);
-		if (err != 0) {
-			nvgpu_err(g,
-				"Error in runlist_lock mutex initialization");
-			goto clean_up_runlist;
-		}
-
-		/* None of buffers is pinned if this value doesn't change.
-		    Otherwise, one of them (cur_buffer) must have been pinned. */
-		runlist->cur_buffer = MAX_RUNLIST_BUFFERS;
-
-		for (pbdma_id = 0; pbdma_id < f->num_pbdma; pbdma_id++) {
-			if ((f->pbdma_map[pbdma_id] & BIT32(runlist_id)) != 0U) {
-				runlist->pbdma_bitmask |= BIT32(pbdma_id);
-			}
-		}
-		nvgpu_log(g, gpu_dbg_info, "runlist %d : pbdma bitmask 0x%x",
-				 runlist_id, runlist->pbdma_bitmask);
-
-		for (engine_id = 0; engine_id < f->num_engines; ++engine_id) {
-			active_engine_id = f->active_engines_list[engine_id];
-			engine_info = &f->engine_info[active_engine_id];
-
-			if ((engine_info != NULL) &&
-			    (engine_info->runlist_id == runlist_id)) {
-				runlist->eng_bitmask |= BIT(active_engine_id);
-			}
-		}
-		nvgpu_log(g, gpu_dbg_info, "runlist %d : act eng bitmask 0x%x",
-				 runlist_id, runlist->eng_bitmask);
-	}
-
-	nvgpu_log_fn(g, "done");
-	return 0;
-
-clean_up_runlist:
-	gk20a_fifo_delete_runlist(f);
-	nvgpu_log_fn(g, "fail");
-	return err;
-}
-
 u32 gk20a_fifo_intr_0_error_mask(struct gk20a *g)
 {
 	u32 intr_0_error_mask =
@@ -752,7 +619,7 @@ int gk20a_init_fifo_setup_sw_common(struct gk20a *g)
 
 	g->ops.fifo.init_engine_info(f);
 
-	err = init_runlist(g, f);
+	err = nvgpu_init_runlist(g, f);
 	if (err != 0) {
 		nvgpu_err(g, "failed to init runlist");
 		goto clean_up;
@@ -2851,16 +2718,6 @@ u32 gk20a_fifo_runlist_busy_engines(struct gk20a *g, u32 runlist_id)
 	return engines;
 }
 
-static void gk20a_fifo_runlist_reset_engines(struct gk20a *g, u32 runlist_id)
-{
-	u32 engines = g->ops.fifo.runlist_busy_engines(g, runlist_id);
-
-	if (engines != 0U) {
-		gk20a_fifo_recover(g, engines, ~(u32)0, false, false, true,
-				RC_TYPE_RUNLIST_UPDATE_TIMEOUT);
-	}
-}
-
 int gk20a_fifo_runlist_wait_pending(struct gk20a *g, u32 runlist_id)
 {
 	struct nvgpu_timeout timeout;
@@ -2938,254 +2795,6 @@ void gk20a_get_ch_runlist_entry(struct channel_gk20a *ch, u32 *runlist)
 	runlist[1] = 0;
 }
 
-static u32 nvgpu_runlist_append_tsg(struct gk20a *g,
-		struct fifo_runlist_info_gk20a *runlist,
-		u32 **runlist_entry,
-		u32 *entries_left,
-		struct tsg_gk20a *tsg)
-{
-	struct fifo_gk20a *f = &g->fifo;
-	u32 runlist_entry_words = f->runlist_entry_size / (u32)sizeof(u32);
-	struct channel_gk20a *ch;
-	u32 count = 0;
-
-	nvgpu_log_fn(f->g, " ");
-
-	if (*entries_left == 0U) {
-		return RUNLIST_APPEND_FAILURE;
-	}
-
-	/* add TSG entry */
-	nvgpu_log_info(g, "add TSG %d to runlist", tsg->tsgid);
-	g->ops.fifo.get_tsg_runlist_entry(tsg, *runlist_entry);
-	nvgpu_log_info(g, "tsg rl entries left %d runlist [0] %x [1] %x",
-			*entries_left,
-			(*runlist_entry)[0], (*runlist_entry)[1]);
-	*runlist_entry += runlist_entry_words;
-	count++;
-	(*entries_left)--;
-
-	nvgpu_rwsem_down_read(&tsg->ch_list_lock);
-	/* add runnable channels bound to this TSG */
-	nvgpu_list_for_each_entry(ch, &tsg->ch_list,
-			channel_gk20a, ch_entry) {
-		if (!test_bit((int)ch->chid,
-			      runlist->active_channels)) {
-			continue;
-		}
-
-		if (*entries_left == 0U) {
-			nvgpu_rwsem_up_read(&tsg->ch_list_lock);
-			return RUNLIST_APPEND_FAILURE;
-		}
-
-		nvgpu_log_info(g, "add channel %d to runlist",
-			ch->chid);
-		g->ops.fifo.get_ch_runlist_entry(ch, *runlist_entry);
-		nvgpu_log_info(g, "rl entries left %d runlist [0] %x [1] %x",
-			*entries_left,
-			(*runlist_entry)[0], (*runlist_entry)[1]);
-		count++;
-		*runlist_entry += runlist_entry_words;
-		(*entries_left)--;
-	}
-	nvgpu_rwsem_up_read(&tsg->ch_list_lock);
-
-	return count;
-}
-
-
-static u32 nvgpu_runlist_append_prio(struct fifo_gk20a *f,
-				struct fifo_runlist_info_gk20a *runlist,
-				u32 **runlist_entry,
-				u32 *entries_left,
-				u32 interleave_level)
-{
-	u32 count = 0;
-	unsigned long tsgid;
-
-	nvgpu_log_fn(f->g, " ");
-
-	for_each_set_bit(tsgid, runlist->active_tsgs, f->num_channels) {
-		struct tsg_gk20a *tsg = &f->tsg[tsgid];
-		u32 entries;
-
-		if (tsg->interleave_level == interleave_level) {
-			entries = nvgpu_runlist_append_tsg(f->g, runlist,
-					runlist_entry, entries_left, tsg);
-			if (entries == RUNLIST_APPEND_FAILURE) {
-				return RUNLIST_APPEND_FAILURE;
-			}
-			count += entries;
-		}
-	}
-
-	return count;
-}
-
-static u32 nvgpu_runlist_append_hi(struct fifo_gk20a *f,
-				struct fifo_runlist_info_gk20a *runlist,
-				u32 **runlist_entry,
-				u32 *entries_left)
-{
-	nvgpu_log_fn(f->g, " ");
-
-	/*
-	 * No higher levels - this is where the "recursion" ends; just add all
-	 * active TSGs at this level.
-	 */
-	return nvgpu_runlist_append_prio(f, runlist, runlist_entry,
-			entries_left,
-			NVGPU_FIFO_RUNLIST_INTERLEAVE_LEVEL_HIGH);
-}
-
-static u32 nvgpu_runlist_append_med(struct fifo_gk20a *f,
-				struct fifo_runlist_info_gk20a *runlist,
-				u32 **runlist_entry,
-				u32 *entries_left)
-{
-	u32 count = 0;
-	unsigned long tsgid;
-
-	nvgpu_log_fn(f->g, " ");
-
-	for_each_set_bit(tsgid, runlist->active_tsgs, f->num_channels) {
-		struct tsg_gk20a *tsg = &f->tsg[tsgid];
-		u32 entries;
-
-		if (tsg->interleave_level !=
-				NVGPU_FIFO_RUNLIST_INTERLEAVE_LEVEL_MEDIUM) {
-			continue;
-		}
-
-		/* LEVEL_MEDIUM list starts with a LEVEL_HIGH, if any */
-
-		entries = nvgpu_runlist_append_hi(f, runlist,
-				runlist_entry, entries_left);
-		if (entries == RUNLIST_APPEND_FAILURE) {
-			return RUNLIST_APPEND_FAILURE;
-		}
-		count += entries;
-
-		entries = nvgpu_runlist_append_tsg(f->g, runlist,
-				runlist_entry, entries_left, tsg);
-		if (entries == RUNLIST_APPEND_FAILURE) {
-			return RUNLIST_APPEND_FAILURE;
-		}
-		count += entries;
-	}
-
-	return count;
-}
-
-static u32 nvgpu_runlist_append_low(struct fifo_gk20a *f,
-				struct fifo_runlist_info_gk20a *runlist,
-				u32 **runlist_entry,
-				u32 *entries_left)
-{
-	u32 count = 0;
-	unsigned long tsgid;
-
-	nvgpu_log_fn(f->g, " ");
-
-	for_each_set_bit(tsgid, runlist->active_tsgs, f->num_channels) {
-		struct tsg_gk20a *tsg = &f->tsg[tsgid];
-		u32 entries;
-
-		if (tsg->interleave_level !=
-				NVGPU_FIFO_RUNLIST_INTERLEAVE_LEVEL_LOW) {
-			continue;
-		}
-
-		/* The medium level starts with the highs, if any. */
-
-		entries = nvgpu_runlist_append_med(f, runlist,
-				runlist_entry, entries_left);
-		if (entries == RUNLIST_APPEND_FAILURE) {
-			return RUNLIST_APPEND_FAILURE;
-		}
-		count += entries;
-
-		entries = nvgpu_runlist_append_hi(f, runlist,
-				runlist_entry, entries_left);
-		if (entries == RUNLIST_APPEND_FAILURE) {
-			return RUNLIST_APPEND_FAILURE;
-		}
-		count += entries;
-
-		entries = nvgpu_runlist_append_tsg(f->g, runlist,
-				runlist_entry, entries_left, tsg);
-		if (entries == RUNLIST_APPEND_FAILURE) {
-			return RUNLIST_APPEND_FAILURE;
-		}
-		count += entries;
-	}
-
-	if (count == 0U) {
-		/*
-		 * No transitions to fill with higher levels, so add
-		 * the next level once. If that's empty too, we have only
-		 * LEVEL_HIGH jobs.
-		 */
-		count = nvgpu_runlist_append_med(f, runlist,
-				runlist_entry, entries_left);
-		if (count == 0U) {
-			count = nvgpu_runlist_append_hi(f, runlist,
-					runlist_entry, entries_left);
-		}
-	}
-
-	return count;
-}
-
-static u32 nvgpu_runlist_append_flat(struct fifo_gk20a *f,
-				struct fifo_runlist_info_gk20a *runlist,
-				u32 **runlist_entry,
-				u32 *entries_left)
-{
-	u32 count = 0, entries, i;
-
-	nvgpu_log_fn(f->g, " ");
-
-	/* Group by priority but don't interleave. High comes first. */
-
-	for (i = 0; i < NVGPU_FIFO_RUNLIST_INTERLEAVE_NUM_LEVELS; i++) {
-		u32 level = NVGPU_FIFO_RUNLIST_INTERLEAVE_LEVEL_HIGH - i;
-
-		entries = nvgpu_runlist_append_prio(f, runlist, runlist_entry,
-				entries_left, level);
-		if (entries == RUNLIST_APPEND_FAILURE) {
-			return RUNLIST_APPEND_FAILURE;
-		}
-		count += entries;
-	}
-
-	return count;
-}
-
-u32 nvgpu_runlist_construct_locked(struct fifo_gk20a *f,
-				struct fifo_runlist_info_gk20a *runlist,
-				u32 buf_id,
-				u32 max_entries)
-{
-	u32 *runlist_entry_base = runlist->mem[buf_id].cpu_va;
-
-	nvgpu_log_fn(f->g, " ");
-
-	/*
-	 * The entry pointer and capacity counter that live on the stack here
-	 * keep track of the current position and the remaining space when tsg
-	 * and channel entries are ultimately appended.
-	 */
-	if (f->g->runlist_interleave) {
-		return nvgpu_runlist_append_low(f, runlist,
-				&runlist_entry_base, &max_entries);
-	} else {
-		return nvgpu_runlist_append_flat(f, runlist,
-				&runlist_entry_base, &max_entries);
-	}
-}
-
 int gk20a_fifo_set_runlist_interleave(struct gk20a *g,
 				u32 id,
 				u32 runlist_id,
@@ -3242,131 +2851,6 @@ void gk20a_fifo_runlist_hw_submit(struct gk20a *g, u32 runlist_id,
 	nvgpu_spinlock_release(&g->fifo.runlist_submit_lock);
 }
 
-int gk20a_fifo_update_runlist_locked(struct gk20a *g, u32 runlist_id,
-					    u32 chid, bool add,
-					    bool wait_for_finish)
-{
-	int ret = 0;
-	struct fifo_gk20a *f = &g->fifo;
-	struct fifo_runlist_info_gk20a *runlist = NULL;
-	u64 runlist_iova;
-	u32 new_buf;
-	struct channel_gk20a *ch = NULL;
-	struct tsg_gk20a *tsg = NULL;
-
-	runlist = &f->runlist_info[runlist_id];
-
-	/* valid channel, add/remove it from active list.
-	   Otherwise, keep active list untouched for suspend/resume. */
-	if (chid != FIFO_INVAL_CHANNEL_ID) {
-		ch = &f->channel[chid];
-		if (gk20a_is_channel_marked_as_tsg(ch)) {
-			tsg = &f->tsg[ch->tsgid];
-		}
-
-		if (add) {
-			if (test_and_set_bit(chid,
-				runlist->active_channels)) {
-				return 0;
-			}
-			if ((tsg != NULL) && (++tsg->num_active_channels != 0U)) {
-				set_bit((int)f->channel[chid].tsgid,
-					runlist->active_tsgs);
-			}
-		} else {
-			if (!test_and_clear_bit(chid,
-				runlist->active_channels)) {
-				return 0;
-			}
-			if ((tsg != NULL) &&
-			    (--tsg->num_active_channels == 0U)) {
-				clear_bit((int)f->channel[chid].tsgid,
-					runlist->active_tsgs);
-			}
-		}
-	}
-
-	/* There just 2 buffers */
-	new_buf = runlist->cur_buffer == 0U ? 1U : 0U;
-
-	runlist_iova = nvgpu_mem_get_addr(g, &runlist->mem[new_buf]);
-
-	nvgpu_log_info(g, "runlist_id : %d, switch to new buffer 0x%16llx",
-		runlist_id, (u64)runlist_iova);
-
-	if (runlist_iova == 0ULL) {
-		ret = -EINVAL;
-		goto clean_up;
-	}
-
-	if (chid != FIFO_INVAL_CHANNEL_ID || /* add/remove a valid channel */
-	    add /* resume to add all channels back */) {
-		u32 num_entries;
-
-		num_entries = nvgpu_runlist_construct_locked(f,
-						runlist,
-						new_buf,
-						f->num_runlist_entries);
-		if (num_entries == RUNLIST_APPEND_FAILURE) {
-			ret = -E2BIG;
-			goto clean_up;
-		}
-		runlist->count = num_entries;
-		WARN_ON(runlist->count > f->num_runlist_entries);
-	} else {
-		/* suspend to remove all channels */
-		runlist->count = 0;
-	}
-
-	g->ops.fifo.runlist_hw_submit(g, runlist_id, runlist->count, new_buf);
-
-	if (wait_for_finish) {
-		ret = g->ops.fifo.runlist_wait_pending(g, runlist_id);
-
-		if (ret == -ETIMEDOUT) {
-			nvgpu_err(g, "runlist %d update timeout", runlist_id);
-			/* trigger runlist update timeout recovery */
-			return ret;
-
-		} else if (ret == -EINTR) {
-			nvgpu_err(g, "runlist update interrupted");
-		}
-	}
-
-	runlist->cur_buffer = new_buf;
-
-clean_up:
-	return ret;
-}
-
-int gk20a_fifo_update_runlist_ids(struct gk20a *g, u32 runlist_ids, u32 chid,
-				bool add, bool wait_for_finish)
-{
-	int ret = -EINVAL;
-	unsigned long runlist_id = 0;
-	int errcode;
-	unsigned long ulong_runlist_ids = (unsigned long)runlist_ids;
-
-	if (g == NULL) {
-		goto end;
-	}
-
-	ret = 0;
-	for_each_set_bit(runlist_id, &ulong_runlist_ids, 32U) {
-		/* Capture the last failure error code */
-		errcode = g->ops.fifo.update_runlist(g, (u32)runlist_id, chid,
-						add, wait_for_finish);
-		if (errcode != 0) {
-			nvgpu_err(g,
-				"failed to update_runlist %lu %d",
-				runlist_id, errcode);
-			ret = errcode;
-		}
-	}
-end:
-	return ret;
-}
-
 /* trigger host preempt of GR pending load ctx if that ctx is not for ch */
 int gk20a_fifo_reschedule_preempt_next(struct channel_gk20a *ch,
 		bool wait_preempt)
@@ -3433,79 +2917,6 @@ int gk20a_fifo_reschedule_runlist(struct channel_gk20a *ch, bool preempt_next)
 	return nvgpu_fifo_reschedule_runlist(ch, preempt_next, true);
 }
 
-/* trigger host to expire current timeslice and reschedule runlist from front */
-int nvgpu_fifo_reschedule_runlist(struct channel_gk20a *ch, bool preempt_next,
-		bool wait_preempt)
-{
-	struct gk20a *g = ch->g;
-	struct fifo_runlist_info_gk20a *runlist;
-	u32 token = PMU_INVALID_MUTEX_OWNER_ID;
-	int mutex_ret;
-	int ret = 0;
-
-	runlist = &g->fifo.runlist_info[ch->runlist_id];
-	if (nvgpu_mutex_tryacquire(&runlist->runlist_lock) == 0) {
-		return -EBUSY;
-	}
-
-	mutex_ret = nvgpu_pmu_mutex_acquire(
-		&g->pmu, PMU_MUTEX_ID_FIFO, &token);
-
-	g->ops.fifo.runlist_hw_submit(
-		g, ch->runlist_id, runlist->count, runlist->cur_buffer);
-
-	if (preempt_next) {
-		g->ops.fifo.reschedule_preempt_next_locked(ch, wait_preempt);
-	}
-
-	gk20a_fifo_runlist_wait_pending(g, ch->runlist_id);
-
-	if (mutex_ret == 0) {
-		nvgpu_pmu_mutex_release(
-			&g->pmu, PMU_MUTEX_ID_FIFO, &token);
-	}
-	nvgpu_mutex_release(&runlist->runlist_lock);
-
-	return ret;
-}
-
-/* add/remove a channel from runlist
-   special cases below: runlist->active_channels will NOT be changed.
-   (chid == ~0 && !add) means remove all active channels from runlist.
-   (chid == ~0 &&  add) means restore all active channels on runlist. */
-int gk20a_fifo_update_runlist(struct gk20a *g, u32 runlist_id, u32 chid,
-			      bool add, bool wait_for_finish)
-{
-	struct fifo_runlist_info_gk20a *runlist = NULL;
-	struct fifo_gk20a *f = &g->fifo;
-	u32 token = PMU_INVALID_MUTEX_OWNER_ID;
-	int mutex_ret;
-	int ret = 0;
-
-	nvgpu_log_fn(g, " ");
-
-	runlist = &f->runlist_info[runlist_id];
-
-	nvgpu_mutex_acquire(&runlist->runlist_lock);
-
-	mutex_ret = nvgpu_pmu_mutex_acquire(&g->pmu, PMU_MUTEX_ID_FIFO, &token);
-
-	ret = gk20a_fifo_update_runlist_locked(g, runlist_id, chid, add,
-					       wait_for_finish);
-
-	if (mutex_ret == 0) {
-		nvgpu_pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token);
-	}
-
-	nvgpu_mutex_release(&runlist->runlist_lock);
-
-	if (ret == -ETIMEDOUT) {
-		gk20a_fifo_runlist_reset_engines(g, runlist_id);
-	}
-
-	return ret;
-}
-
 int gk20a_fifo_suspend(struct gk20a *g)
 {
 	nvgpu_log_fn(g, " ");
@@ -4201,31 +3612,6 @@ u32 gk20a_fifo_pbdma_acquire_val(u64 timeout)
 	return val;
 }
 
-const char *gk20a_fifo_interleave_level_name(u32 interleave_level)
-{
-	const char *ret_string = NULL;
-
-	switch (interleave_level) {
-	case NVGPU_FIFO_RUNLIST_INTERLEAVE_LEVEL_LOW:
-		ret_string = "LOW";
-		break;
-
-	case NVGPU_FIFO_RUNLIST_INTERLEAVE_LEVEL_MEDIUM:
-		ret_string = "MEDIUM";
-		break;
-
-	case NVGPU_FIFO_RUNLIST_INTERLEAVE_LEVEL_HIGH:
-		ret_string = "HIGH";
-		break;
-
-	default:
-		ret_string = "?";
-		break;
-	}
-
-	return ret_string;
-}
-
 u32 gk20a_fifo_get_sema_wait_cmd_size(void)
 {
 	return 8;

drivers/gpu/nvgpu/gk20a/fifo_gk20a.h

@@ -266,18 +266,9 @@ void gk20a_fifo_disable_tsg_sched(struct gk20a *g, struct tsg_gk20a *tsg);
 
 u32 gk20a_fifo_engines_on_ch(struct gk20a *g, u32 chid);
 
-int gk20a_fifo_reschedule_runlist(struct channel_gk20a *ch, bool preempt_next);
-int nvgpu_fifo_reschedule_runlist(struct channel_gk20a *ch, bool preempt_next,
-		bool wait_preempt);
 int gk20a_fifo_reschedule_preempt_next(struct channel_gk20a *ch,
 		bool wait_preempt);
 
-int gk20a_fifo_update_runlist(struct gk20a *g, u32 runlist_id, u32 chid,
-			      bool add, bool wait_for_finish);
-
-int gk20a_fifo_update_runlist_locked(struct gk20a *g, u32 runlist_id,
-					    u32 chid, bool add,
-					    bool wait_for_finish);
 int gk20a_fifo_suspend(struct gk20a *g);
 
 bool gk20a_fifo_mmu_fault_pending(struct gk20a *g);
@@ -312,16 +303,12 @@ int gk20a_fifo_set_runlist_interleave(struct gk20a *g,
 				u32 new_level);
 int gk20a_fifo_tsg_set_timeslice(struct tsg_gk20a *tsg, u32 timeslice);
 
-const char *gk20a_fifo_interleave_level_name(u32 interleave_level);
-
 enum fifo_engine gk20a_fifo_engine_enum_from_type(struct gk20a *g,
 				u32 engine_type);
 
 u32 gk20a_fifo_get_engine_ids(struct gk20a *g, u32 engine_id[],
 			u32 engine_id_sz, enum fifo_engine engine_enum);
 
-void gk20a_fifo_delete_runlist(struct fifo_gk20a *f);
-
 struct fifo_engine_info_gk20a *gk20a_fifo_get_engine_info(struct gk20a *g,
 							 u32 engine_id);
 
@@ -339,9 +326,6 @@ u32 gk20a_fifo_get_gr_runlist_id(struct gk20a *g);
 
 bool gk20a_fifo_is_valid_runlist_id(struct gk20a *g, u32 runlist_id);
 
-int gk20a_fifo_update_runlist_ids(struct gk20a *g, u32 runlist_ids, u32 chid,
-		bool add, bool wait_for_finish);
-
 void gk20a_get_tsg_runlist_entry(struct tsg_gk20a *tsg, u32 *runlist);
 void gk20a_get_ch_runlist_entry(struct channel_gk20a *ch, u32 *runlist);
 void gk20a_fifo_set_runlist_state(struct gk20a *g, u32 runlists_mask,
@@ -414,11 +398,6 @@ int gk20a_fifo_setup_userd(struct channel_gk20a *c);
 u32 gk20a_fifo_pbdma_acquire_val(u64 timeout);
 
 
-#define RUNLIST_APPEND_FAILURE 0xffffffffU
-u32 nvgpu_runlist_construct_locked(struct fifo_gk20a *f,
-				struct fifo_runlist_info_gk20a *runlist,
-				u32 buf_id,
-				u32 max_entries);
 void gk20a_fifo_runlist_hw_submit(struct gk20a *g, u32 runlist_id,
 	u32 count, u32 buffer_index);
 u32 gk20a_fifo_runlist_busy_engines(struct gk20a *g, u32 runlist_id);

drivers/gpu/nvgpu/gm20b/hal_gm20b.c

@@ -29,6 +29,7 @@
 #include <nvgpu/gk20a.h>
 #include <nvgpu/debugger.h>
 #include <nvgpu/channel.h>
+#include <nvgpu/runlist.h>
 #include <nvgpu/tsg.h>
 #include <nvgpu/perfbuf.h>
 #include <nvgpu/cyclestats_snapshot.h>

drivers/gpu/nvgpu/gp10b/hal_gp10b.c

@@ -30,6 +30,7 @@
 #include <nvgpu/gk20a.h>
 #include <nvgpu/debugger.h>
 #include <nvgpu/channel.h>
+#include <nvgpu/runlist.h>
 #include <nvgpu/tsg.h>
 #include <nvgpu/perfbuf.h>
 #include <nvgpu/cyclestats_snapshot.h>

drivers/gpu/nvgpu/gv100/hal_gv100.c

@@ -120,6 +120,7 @@
 #include <nvgpu/gk20a.h>
 #include <nvgpu/debugger.h>
 #include <nvgpu/channel.h>
+#include <nvgpu/runlist.h>
 #include <nvgpu/perfbuf.h>
 #include <nvgpu/cyclestats_snapshot.h>
 

drivers/gpu/nvgpu/gv11b/fifo_gv11b.c

@@ -1,7 +1,7 @@
 /*
  * GV11B fifo
  *
- * Copyright (c) 2015-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2015-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -41,6 +41,7 @@
 #include <nvgpu/io.h>
 #include <nvgpu/utils.h>
 #include <nvgpu/fifo.h>
+#include <nvgpu/runlist.h>
 #include <nvgpu/gk20a.h>
 #include <nvgpu/channel.h>
 #include <nvgpu/unit.h>

drivers/gpu/nvgpu/gv11b/hal_gv11b.c

@@ -99,6 +99,7 @@
 #include <nvgpu/bug.h>
 #include <nvgpu/debugger.h>
 #include <nvgpu/channel.h>
+#include <nvgpu/runlist.h>
 #include <nvgpu/perfbuf.h>
 #include <nvgpu/cyclestats_snapshot.h>
 

drivers/gpu/nvgpu/include/nvgpu/runlist.h

@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef NVGPU_RUNLIST_H
+#define NVGPU_RUNLIST_H
+
+#include <nvgpu/types.h>
+
+struct gk20a;
+struct fifo_runlist_info_gk20a;
+struct tsg_gk20a;
+struct fifo_gk20a;
+struct channel_gk20a;
+
+#define RUNLIST_APPEND_FAILURE 0xffffffffU
+u32 nvgpu_runlist_construct_locked(struct fifo_gk20a *f,
+				struct fifo_runlist_info_gk20a *runlist,
+				u32 buf_id,
+				u32 max_entries);
+int gk20a_fifo_update_runlist_locked(struct gk20a *g, u32 runlist_id,
+					    u32 chid, bool add,
+					    bool wait_for_finish);
+
+int gk20a_fifo_reschedule_runlist(struct channel_gk20a *ch, bool preempt_next);
+int nvgpu_fifo_reschedule_runlist(struct channel_gk20a *ch, bool preempt_next,
+		bool wait_preempt);
+
+int gk20a_fifo_update_runlist(struct gk20a *g, u32 runlist_id, u32 chid,
+			      bool add, bool wait_for_finish);
+int gk20a_fifo_update_runlist_ids(struct gk20a *g, u32 runlist_ids, u32 chid,
+				bool add, bool wait_for_finish);
+
+const char *gk20a_fifo_interleave_level_name(u32 interleave_level);
+
+void gk20a_fifo_delete_runlist(struct fifo_gk20a *f);
+int nvgpu_init_runlist(struct gk20a *g, struct fifo_gk20a *f);
+
+#endif /* NVGPU_RUNLIST_H */

drivers/gpu/nvgpu/os/linux/ioctl_channel.c

@@ -1,7 +1,7 @@
 /*
  * GK20A Graphics channel
  *
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2011-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms and conditions of the GNU General Public License,
@@ -38,6 +38,7 @@
 #include <nvgpu/channel.h>
 #include <nvgpu/channel_sync.h>
 #include <nvgpu/channel_sync_syncpt.h>
+#include <nvgpu/runlist.h>
 
 #include "gk20a/fence_gk20a.h"
 

drivers/gpu/nvgpu/tu104/hal_tu104.c

@@ -135,6 +135,7 @@
 #include <nvgpu/clk_arb.h>
 #include <nvgpu/debugger.h>
 #include <nvgpu/channel.h>
+#include <nvgpu/runlist.h>
 #include <nvgpu/perfbuf.h>
 #include <nvgpu/cyclestats_snapshot.h>
 

drivers/gpu/nvgpu/vgpu/fifo_vgpu.c

@@ -1,7 +1,7 @@
 /*
  * Virtualized GPU Fifo
  *
- * Copyright (c) 2014-2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2014-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -35,6 +35,7 @@
 #include <nvgpu/vgpu/vgpu.h>
 #include <nvgpu/gk20a.h>
 #include <nvgpu/channel.h>
+#include <nvgpu/runlist.h>
 #include <nvgpu/string.h>
 #include <nvgpu/vm_area.h>
 

userspace/units/fifo/runlist/nvgpu-runlist.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2018-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -26,6 +26,7 @@
 #include <unit/unit.h>
 
 #include <nvgpu/channel.h>
+#include <nvgpu/runlist.h>
 #include <nvgpu/gk20a.h>
 
 static void setup_fifo(struct gk20a *g, unsigned long *tsg_map,