Open source GPL/LGPL release

drivers/gpu/nvgpu/common/vgpu/fifo/channel_vgpu.c

@@ -0,0 +1,199 @@
+/*
+ * Copyright (c) 2019-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/gk20a.h>
+#include <nvgpu/channel.h>
+#include <nvgpu/runlist.h>
+#include <nvgpu/error_notifier.h>
+#include <nvgpu/vgpu/vgpu_ivc.h>
+#include <nvgpu/vgpu/vgpu.h>
+
+#include "common/vgpu/ivc/comm_vgpu.h"
+#include "channel_vgpu.h"
+
+void vgpu_channel_bind(struct nvgpu_channel *ch)
+{
+	struct tegra_vgpu_cmd_msg msg;
+	struct tegra_vgpu_channel_config_params *p =
+			&msg.params.channel_config;
+	int err;
+	struct gk20a *g = ch->g;
+
+	nvgpu_log_info(g, "bind channel %d", ch->chid);
+
+	msg.cmd = TEGRA_VGPU_CMD_CHANNEL_BIND;
+	msg.handle = vgpu_get_handle(ch->g);
+	p->handle = ch->virt_ctx;
+	err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
+	WARN_ON(err || msg.ret);
+
+	nvgpu_smp_wmb();
+	nvgpu_atomic_set(&ch->bound, true);
+}
+
+void vgpu_channel_unbind(struct nvgpu_channel *ch)
+{
+	struct gk20a *g = ch->g;
+
+	nvgpu_log_fn(g, " ");
+
+	if (nvgpu_atomic_cmpxchg(&ch->bound, true, false)) {
+		struct tegra_vgpu_cmd_msg msg;
+		struct tegra_vgpu_channel_config_params *p =
+				&msg.params.channel_config;
+		int err;
+
+		msg.cmd = TEGRA_VGPU_CMD_CHANNEL_UNBIND;
+		msg.handle = vgpu_get_handle(ch->g);
+		p->handle = ch->virt_ctx;
+		err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
+		WARN_ON(err || msg.ret);
+	}
+
+}
+
+int vgpu_channel_alloc_inst(struct gk20a *g, struct nvgpu_channel *ch)
+{
+	struct tegra_vgpu_cmd_msg msg;
+	struct tegra_vgpu_channel_hwctx_params *p = &msg.params.channel_hwctx;
+	int err;
+
+	nvgpu_log_fn(g, " ");
+
+	msg.cmd = TEGRA_VGPU_CMD_CHANNEL_ALLOC_HWCTX;
+	msg.handle = vgpu_get_handle(g);
+	p->id = ch->chid;
+	p->runlist_id = ch->runlist->id;
+	p->pid = (u64)ch->pid;
+	err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
+	if (err || msg.ret) {
+		nvgpu_err(g, "fail");
+		return -ENOMEM;
+	}
+
+	ch->virt_ctx = p->handle;
+	nvgpu_log_fn(g, "done");
+	return 0;
+}
+
+void vgpu_channel_free_inst(struct gk20a *g, struct nvgpu_channel *ch)
+{
+	struct tegra_vgpu_cmd_msg msg;
+	struct tegra_vgpu_channel_hwctx_params *p = &msg.params.channel_hwctx;
+	int err;
+
+	nvgpu_log_fn(g, " ");
+
+	msg.cmd = TEGRA_VGPU_CMD_CHANNEL_FREE_HWCTX;
+	msg.handle = vgpu_get_handle(g);
+	p->handle = ch->virt_ctx;
+	err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
+	WARN_ON(err || msg.ret);
+}
+
+void vgpu_channel_enable(struct nvgpu_channel *ch)
+{
+	struct tegra_vgpu_cmd_msg msg;
+	struct tegra_vgpu_channel_config_params *p =
+			&msg.params.channel_config;
+	int err;
+	struct gk20a *g = ch->g;
+
+	nvgpu_log_fn(g, " ");
+
+	msg.cmd = TEGRA_VGPU_CMD_CHANNEL_ENABLE;
+	msg.handle = vgpu_get_handle(ch->g);
+	p->handle = ch->virt_ctx;
+	err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
+	WARN_ON(err || msg.ret);
+}
+
+void vgpu_channel_disable(struct nvgpu_channel *ch)
+{
+	struct tegra_vgpu_cmd_msg msg;
+	struct tegra_vgpu_channel_config_params *p =
+			&msg.params.channel_config;
+	int err;
+	struct gk20a *g = ch->g;
+
+	nvgpu_log_fn(g, " ");
+
+	msg.cmd = TEGRA_VGPU_CMD_CHANNEL_DISABLE;
+	msg.handle = vgpu_get_handle(ch->g);
+	p->handle = ch->virt_ctx;
+	err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
+	WARN_ON(err || msg.ret);
+}
+
+u32 vgpu_channel_count(struct gk20a *g)
+{
+	struct vgpu_priv_data *priv = vgpu_get_priv_data(g);
+
+	return priv->constants.num_channels;
+}
+
+void vgpu_channel_set_ctx_mmu_error(struct gk20a *g, struct nvgpu_channel *ch)
+{
+	/*
+	 * If error code is already set, this mmu fault
+	 * was triggered as part of recovery from other
+	 * error condition.
+	 * Don't overwrite error flag.
+	 */
+	g->ops.channel.set_error_notifier(ch,
+		NVGPU_ERR_NOTIFIER_FIFO_ERROR_MMU_ERR_FLT);
+
+	/* mark channel as faulted */
+	nvgpu_channel_set_unserviceable(ch);
+
+	/* unblock pending waits */
+	nvgpu_cond_broadcast_interruptible(&ch->semaphore_wq);
+	nvgpu_cond_broadcast_interruptible(&ch->notifier_wq);
+}
+
+void vgpu_channel_set_error_notifier(struct gk20a *g,
+				struct tegra_vgpu_channel_set_error_notifier *p)
+{
+	struct nvgpu_channel *ch;
+
+	if (p->chid >= g->fifo.num_channels) {
+		nvgpu_err(g, "invalid chid %d", p->chid);
+		return;
+	}
+
+	ch = &g->fifo.channel[p->chid];
+	g->ops.channel.set_error_notifier(ch, p->error);
+}
+
+void vgpu_channel_abort_cleanup(struct gk20a *g, u32 chid)
+{
+	struct nvgpu_channel *ch = nvgpu_channel_from_id(g, chid);
+
+	if (ch == NULL) {
+		nvgpu_err(g, "invalid channel id %d", chid);
+		return;
+	}
+
+	nvgpu_channel_set_unserviceable(ch);
+	g->ops.channel.abort_clean_up(ch);
+	nvgpu_channel_put(ch);
+}

drivers/gpu/nvgpu/common/vgpu/fifo/channel_vgpu.h

@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef NVGPU_CHANNEL_VGPU_H
+#define NVGPU_CHANNEL_VGPU_H
+
+struct gk20a;
+struct nvgpu_channel;
+
+void vgpu_channel_bind(struct nvgpu_channel *ch);
+void vgpu_channel_unbind(struct nvgpu_channel *ch);
+int vgpu_channel_alloc_inst(struct gk20a *g, struct nvgpu_channel *ch);
+void vgpu_channel_free_inst(struct gk20a *g, struct nvgpu_channel *ch);
+void vgpu_channel_enable(struct nvgpu_channel *ch);
+void vgpu_channel_disable(struct nvgpu_channel *ch);
+u32 vgpu_channel_count(struct gk20a *g);
+void vgpu_channel_set_ctx_mmu_error(struct gk20a *g, struct nvgpu_channel *ch);
+void vgpu_channel_set_error_notifier(struct gk20a *g,
+			struct tegra_vgpu_channel_set_error_notifier *p);
+void vgpu_channel_abort_cleanup(struct gk20a *g, u32 chid);
+
+#endif

drivers/gpu/nvgpu/common/vgpu/fifo/fifo_vgpu.c

@@ -0,0 +1,136 @@
+/*
+ * Virtualized GPU Fifo
+ *
+ * Copyright (c) 2014-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/trace.h>
+#include <nvgpu/kmem.h>
+#include <nvgpu/dma.h>
+#include <nvgpu/atomic.h>
+#include <nvgpu/bug.h>
+#include <nvgpu/barrier.h>
+#include <nvgpu/io.h>
+#include <nvgpu/error_notifier.h>
+#include <nvgpu/gk20a.h>
+#include <nvgpu/channel.h>
+#include <nvgpu/fifo.h>
+#include <nvgpu/runlist.h>
+#include <nvgpu/string.h>
+#include <nvgpu/vm_area.h>
+#include <nvgpu/vgpu/vgpu_ivc.h>
+#include <nvgpu/vgpu/vgpu.h>
+
+#include <hal/fifo/tsg_gk20a.h>
+
+#include "fifo_vgpu.h"
+#include "channel_vgpu.h"
+#include "tsg_vgpu.h"
+
+void vgpu_fifo_cleanup_sw(struct gk20a *g)
+{
+	u32 i;
+	struct nvgpu_fifo *f = &g->fifo;
+
+	for (i = 0U; i < f->max_engines; i++) {
+		if (f->host_engines[i] == NULL) {
+			continue;
+		}
+
+		/*
+		 * Cast to (void *) to get rid of the constness.
+		 */
+		nvgpu_kfree(g, (void *)f->host_engines[i]);
+	}
+	nvgpu_fifo_cleanup_sw_common(g);
+}
+
+int vgpu_fifo_setup_sw(struct gk20a *g)
+{
+	struct nvgpu_fifo *f = &g->fifo;
+	struct vgpu_priv_data *priv = vgpu_get_priv_data(g);
+	int err = 0;
+
+	nvgpu_log_fn(g, " ");
+
+	if (f->sw_ready) {
+		nvgpu_log_fn(g, "skip init");
+		return 0;
+	}
+
+	err = nvgpu_fifo_setup_sw_common(g);
+	if (err != 0) {
+		nvgpu_err(g, "fifo sw setup failed, err=%d", err);
+		return err;
+	}
+
+#ifdef CONFIG_NVGPU_KERNEL_MODE_SUBMIT
+	err = nvgpu_channel_worker_init(g);
+	if (err) {
+		goto clean_up;
+	}
+#endif
+
+	f->channel_base = priv->constants.channel_base;
+
+	f->sw_ready = true;
+
+	nvgpu_log_fn(g, "done");
+	return 0;
+
+#ifdef CONFIG_NVGPU_KERNEL_MODE_SUBMIT
+clean_up:
+	nvgpu_fifo_cleanup_sw_common(g);
+#endif
+
+	return err;
+}
+
+int vgpu_fifo_isr(struct gk20a *g, struct tegra_vgpu_fifo_intr_info *info)
+{
+	struct nvgpu_channel *ch = nvgpu_channel_from_id(g, info->chid);
+
+	nvgpu_log_fn(g, " ");
+
+	nvgpu_err(g, "fifo intr (%d) on ch %u",
+		info->type, info->chid);
+
+	switch (info->type) {
+	case TEGRA_VGPU_FIFO_INTR_PBDMA:
+		g->ops.channel.set_error_notifier(ch,
+			NVGPU_ERR_NOTIFIER_PBDMA_ERROR);
+		break;
+	case TEGRA_VGPU_FIFO_INTR_CTXSW_TIMEOUT:
+		g->ops.channel.set_error_notifier(ch,
+			NVGPU_ERR_NOTIFIER_FIFO_ERROR_IDLE_TIMEOUT);
+		break;
+	case TEGRA_VGPU_FIFO_INTR_MMU_FAULT:
+		vgpu_tsg_set_ctx_mmu_error(g, info->chid);
+		nvgpu_channel_abort(ch, false);
+		break;
+	default:
+		WARN_ON(1);
+		break;
+	}
+
+	nvgpu_channel_put(ch);
+	return 0;
+}

drivers/gpu/nvgpu/common/vgpu/fifo/fifo_vgpu.h

@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2017-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef NVGPU_FIFO_VGPU_H
+#define NVGPU_FIFO_VGPU_H
+
+#include <nvgpu/types.h>
+
+struct gk20a;
+struct tegra_vgpu_fifo_intr_info;
+
+int vgpu_fifo_setup_sw(struct gk20a *g);
+void vgpu_fifo_cleanup_sw(struct gk20a *g);
+int vgpu_fifo_isr(struct gk20a *g, struct tegra_vgpu_fifo_intr_info *info);
+
+#endif /* NVGPU_FIFO_VGPU_H */

drivers/gpu/nvgpu/common/vgpu/fifo/preempt_vgpu.c

@@ -0,0 +1,84 @@
+/*
+ * Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/atomic.h>
+#include <nvgpu/gk20a.h>
+#include <nvgpu/channel.h>
+#include <nvgpu/fifo.h>
+#include <nvgpu/tsg.h>
+#include <nvgpu/preempt.h>
+
+#include <nvgpu/vgpu/vgpu_ivc.h>
+#include <nvgpu/vgpu/vgpu.h>
+
+#include "preempt_vgpu.h"
+#include "common/vgpu/ivc/comm_vgpu.h"
+
+int vgpu_fifo_preempt_channel(struct gk20a *g, struct nvgpu_channel *ch)
+{
+	struct tegra_vgpu_cmd_msg msg;
+	struct tegra_vgpu_channel_config_params *p =
+			&msg.params.channel_config;
+	int err;
+
+	nvgpu_log_fn(g, " ");
+
+	if (!nvgpu_atomic_read(&ch->bound)) {
+		return 0;
+	}
+
+	msg.cmd = TEGRA_VGPU_CMD_CHANNEL_PREEMPT;
+	msg.handle = vgpu_get_handle(g);
+	p->handle = ch->virt_ctx;
+	err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
+
+	if (err || msg.ret) {
+		nvgpu_err(g,
+			"preempt channel %d failed", ch->chid);
+		err = -ENOMEM;
+	}
+
+	return err;
+}
+
+int vgpu_fifo_preempt_tsg(struct gk20a *g, struct nvgpu_tsg *tsg)
+{
+	struct tegra_vgpu_cmd_msg msg;
+	struct tegra_vgpu_tsg_preempt_params *p =
+			&msg.params.tsg_preempt;
+	int err;
+
+	nvgpu_log_fn(g, " ");
+
+	msg.cmd = TEGRA_VGPU_CMD_TSG_PREEMPT;
+	msg.handle = vgpu_get_handle(g);
+	p->tsg_id = tsg->tsgid;
+	err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
+	err = err ? err : msg.ret;
+
+	if (err) {
+		nvgpu_err(g,
+			"preempt tsg %u failed", tsg->tsgid);
+	}
+
+	return err;
+}

drivers/gpu/nvgpu/common/vgpu/fifo/preempt_vgpu.h

@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef NVGPU_PREEMPT_VGPU_H
+#define NVGPU_PREEMPT_VGPU_H
+
+struct gk20a;
+struct nvgpu_channel;
+struct nvgpu_tsg;
+
+int vgpu_fifo_preempt_channel(struct gk20a *g, struct nvgpu_channel *ch);
+int vgpu_fifo_preempt_tsg(struct gk20a *g, struct nvgpu_tsg *tsg);
+
+#endif

drivers/gpu/nvgpu/common/vgpu/fifo/ramfc_vgpu.c

@@ -0,0 +1,52 @@
+/*
+ * Virtualized GPU Channel RAMFC
+ *
+ * Copyright (c) 2014-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/vgpu/vgpu.h>
+#include <nvgpu/gk20a.h>
+#include <nvgpu/channel.h>
+
+#include "ramfc_vgpu.h"
+#include "common/vgpu/ivc/comm_vgpu.h"
+
+int vgpu_ramfc_setup(struct nvgpu_channel *ch, u64 gpfifo_base,
+		u32 gpfifo_entries, u64 pbdma_acquire_timeout, u32 flags)
+{
+	struct tegra_vgpu_cmd_msg msg;
+	struct tegra_vgpu_ramfc_params *p = &msg.params.ramfc;
+	int err;
+	struct gk20a *g = ch->g;
+
+	nvgpu_log_fn(g, " ");
+
+	msg.cmd = TEGRA_VGPU_CMD_CHANNEL_SETUP_RAMFC;
+	msg.handle = vgpu_get_handle(ch->g);
+	p->handle = ch->virt_ctx;
+	p->gpfifo_va = gpfifo_base;
+	p->num_entries = gpfifo_entries;
+	p->userd_addr = ch->userd_iova;
+	p->iova = 0;
+	err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
+
+	return (err || msg.ret) ? -ENOMEM : 0;
+}

drivers/gpu/nvgpu/common/vgpu/fifo/ramfc_vgpu.h

@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef NVGPU_RAMFC_VGPU_H
+#define NVGPU_RAMFC_VGPU_H
+
+#include <nvgpu/types.h>
+
+struct nvgpu_channel;
+
+int vgpu_ramfc_setup(struct nvgpu_channel *ch, u64 gpfifo_base,
+		u32 gpfifo_entries, u64 pbdma_acquire_timeout, u32 flags);
+
+#endif /* NVGPU_RAMFC_VGPU_H */

drivers/gpu/nvgpu/common/vgpu/fifo/runlist_vgpu.c

@@ -0,0 +1,212 @@
+/*
+ * Virtualized GPU Runlist
+ *
+ * Copyright (c) 2019-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/kmem.h>
+#include <nvgpu/string.h>
+#include <nvgpu/bug.h>
+#include <nvgpu/vgpu/vgpu_ivc.h>
+#include <nvgpu/vgpu/vgpu.h>
+#include <nvgpu/gk20a.h>
+#include <nvgpu/channel.h>
+#include <nvgpu/runlist.h>
+
+#include "runlist_vgpu.h"
+#include "common/vgpu/ivc/comm_vgpu.h"
+
+static int vgpu_submit_runlist(struct gk20a *g, u64 handle, u8 runlist_id,
+			       u16 *runlist, u32 num_entries)
+{
+	struct tegra_vgpu_cmd_msg msg;
+	struct tegra_vgpu_runlist_params *p;
+	int err;
+	void *oob_handle;
+	void *oob;
+	size_t size, oob_size;
+
+	oob_handle = vgpu_ivc_oob_get_ptr(vgpu_ivc_get_server_vmid(),
+			TEGRA_VGPU_QUEUE_CMD,
+			&oob, &oob_size);
+	if (!oob_handle) {
+		return -EINVAL;
+	}
+
+	size = sizeof(*runlist) * num_entries;
+	if (oob_size < size) {
+		err = -ENOMEM;
+		goto done;
+	}
+
+	msg.cmd = TEGRA_VGPU_CMD_SUBMIT_RUNLIST;
+	msg.handle = handle;
+	p = &msg.params.runlist;
+	p->runlist_id = runlist_id;
+	p->num_entries = num_entries;
+
+	nvgpu_memcpy((u8 *)oob, (u8 *)runlist, size);
+	err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
+
+	err = (err || msg.ret) ? -1 : 0;
+
+done:
+	vgpu_ivc_oob_put_ptr(oob_handle);
+	return err;
+}
+
+static bool vgpu_runlist_modify_active_locked(struct gk20a *g, u32 runlist_id,
+					    struct nvgpu_channel *ch, bool add)
+{
+	struct nvgpu_fifo *f = &g->fifo;
+	struct nvgpu_runlist *runlist;
+
+	runlist = f->runlists[runlist_id];
+
+	if (add) {
+		if (nvgpu_test_and_set_bit(ch->chid,
+				runlist->active_channels)) {
+			return false;
+			/* was already there */
+		}
+	} else {
+		if (!nvgpu_test_and_clear_bit(ch->chid,
+				runlist->active_channels)) {
+			/* wasn't there */
+			return false;
+		}
+	}
+
+	return true;
+}
+
+static void vgpu_runlist_reconstruct_locked(struct gk20a *g, u32 runlist_id,
+				     bool add_entries)
+{
+	struct nvgpu_fifo *f = &g->fifo;
+	struct nvgpu_runlist *runlist;
+
+	runlist = f->runlists[runlist_id];
+
+	if (add_entries) {
+		u16 *runlist_entry;
+		u32 count = 0;
+		unsigned long chid;
+
+		runlist_entry = runlist->mem[0].cpu_va;
+
+		nvgpu_assert(f->num_channels <= (unsigned int)U16_MAX);
+		for_each_set_bit(chid,
+				runlist->active_channels, f->num_channels) {
+			nvgpu_log_info(g, "add channel %lu to runlist", chid);
+			*runlist_entry++ = (u16)chid;
+			count++;
+		}
+
+		runlist->count = count;
+	} else {
+		runlist->count = 0;
+	}
+}
+
+static int vgpu_runlist_update_locked(struct gk20a *g, u32 runlist_id,
+					struct nvgpu_channel *ch, bool add,
+					bool wait_for_finish)
+{
+	struct nvgpu_fifo *f = &g->fifo;
+	struct nvgpu_runlist *runlist;
+	bool add_entries;
+
+	nvgpu_log_fn(g, " ");
+
+	if (ch != NULL) {
+		bool update = vgpu_runlist_modify_active_locked(g, runlist_id,
+				ch, add);
+		if (!update) {
+			/* no change in runlist contents */
+			return 0;
+		}
+		/* had a channel to update, so reconstruct */
+		add_entries = true;
+	} else {
+		/* no channel; add means update all, !add means clear all */
+		add_entries = add;
+	}
+
+	runlist = f->runlists[runlist_id];
+
+	vgpu_runlist_reconstruct_locked(g, runlist_id, add_entries);
+
+	return vgpu_submit_runlist(g, vgpu_get_handle(g), runlist_id,
+				runlist->mem[0].cpu_va, runlist->count);
+}
+
+/* add/remove a channel from runlist
+   special cases below: runlist->active_channels will NOT be changed.
+   (ch == NULL && !add) means remove all active channels from runlist.
+   (ch == NULL &&  add) means restore all active channels on runlist. */
+static int vgpu_runlist_do_update(struct gk20a *g, struct nvgpu_runlist *rl,
+				struct nvgpu_channel *ch,
+				bool add, bool wait_for_finish)
+{
+	u32 ret = 0;
+
+	nvgpu_log_fn(g, " ");
+
+	nvgpu_mutex_acquire(&rl->runlist_lock);
+
+	ret = vgpu_runlist_update_locked(g, rl->id, ch, add,
+					wait_for_finish);
+
+	nvgpu_mutex_release(&rl->runlist_lock);
+	return ret;
+}
+
+int vgpu_runlist_update(struct gk20a *g, struct nvgpu_runlist *rl,
+			struct nvgpu_channel *ch,
+			bool add, bool wait_for_finish)
+{
+	nvgpu_assert(ch != NULL);
+
+	return vgpu_runlist_do_update(g, rl, ch, add, wait_for_finish);
+}
+
+int vgpu_runlist_reload(struct gk20a *g, struct nvgpu_runlist *rl,
+			bool add, bool wait_for_finish)
+{
+	return vgpu_runlist_do_update(g, rl, NULL, add, wait_for_finish);
+}
+
+u32 vgpu_runlist_length_max(struct gk20a *g)
+{
+	struct vgpu_priv_data *priv = vgpu_get_priv_data(g);
+
+	return priv->constants.num_channels;
+}
+
+u32 vgpu_runlist_entry_size(struct gk20a *g)
+{
+	/*
+	 * This is not the HW format you're looking for (see
+	 * vgpu_fifo_update_runlist_locked(), vgpu_submit_runlist())
+	 */
+	return (u32)sizeof(u16);
+}

drivers/gpu/nvgpu/common/vgpu/fifo/runlist_vgpu.h

@@ -0,0 +1,35 @@
+/*
+ * Virtualized GPU Runlist
+ *
+ * Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+struct gk20a;
+struct nvgpu_channel;
+struct nvgpu_runlist;
+
+int vgpu_runlist_update(struct gk20a *g, struct nvgpu_runlist *rl,
+			struct nvgpu_channel *ch,
+			bool add, bool wait_for_finish);
+int vgpu_runlist_reload(struct gk20a *g, struct nvgpu_runlist *rl,
+				bool add, bool wait_for_finish);
+u32 vgpu_runlist_length_max(struct gk20a *g);
+u32 vgpu_runlist_entry_size(struct gk20a *g);

drivers/gpu/nvgpu/common/vgpu/fifo/tsg_vgpu.c

@@ -0,0 +1,316 @@
+/*
+ * Copyright (c) 2016-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/gk20a.h>
+#include <nvgpu/channel.h>
+#include <nvgpu/runlist.h>
+#include <nvgpu/tsg.h>
+#include <nvgpu/bug.h>
+
+#include <nvgpu/vgpu/tegra_vgpu.h>
+#include <nvgpu/vgpu/vgpu_ivc.h>
+#include <nvgpu/vgpu/vgpu.h>
+
+#include "tsg_vgpu.h"
+#include "channel_vgpu.h"
+#include "common/vgpu/ivc/comm_vgpu.h"
+
+int vgpu_tsg_open(struct nvgpu_tsg *tsg)
+{
+	struct tegra_vgpu_cmd_msg msg = {};
+	struct tegra_vgpu_tsg_open_rel_params *p =
+				&msg.params.tsg_open;
+	int err;
+	struct gk20a *g = tsg->g;
+
+	nvgpu_log_fn(g, " ");
+
+	msg.cmd = TEGRA_VGPU_CMD_TSG_OPEN;
+	msg.handle = vgpu_get_handle(tsg->g);
+	p->tsg_id = tsg->tsgid;
+	p->pid = tsg->tgid;
+	err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
+	err = err ? err : msg.ret;
+	if (err) {
+		nvgpu_err(tsg->g,
+			"vgpu_tsg_open failed, tsgid %d", tsg->tsgid);
+	}
+
+	return err;
+}
+
+void vgpu_tsg_release(struct nvgpu_tsg *tsg)
+{
+	struct tegra_vgpu_cmd_msg msg = {};
+	struct tegra_vgpu_tsg_open_rel_params *p =
+				&msg.params.tsg_release;
+	int err;
+	struct gk20a *g = tsg->g;
+
+	nvgpu_log_fn(g, " ");
+
+	msg.cmd = TEGRA_VGPU_CMD_TSG_RELEASE;
+	msg.handle = vgpu_get_handle(tsg->g);
+	p->tsg_id = tsg->tsgid;
+	err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
+	err = err ? err : msg.ret;
+	if (err) {
+		nvgpu_err(tsg->g,
+			"vgpu_tsg_release failed, tsgid %d", tsg->tsgid);
+	}
+}
+
+void vgpu_tsg_enable(struct nvgpu_tsg *tsg)
+{
+	struct gk20a *g = tsg->g;
+	struct nvgpu_channel *ch;
+
+	nvgpu_rwsem_down_read(&tsg->ch_list_lock);
+	nvgpu_list_for_each_entry(ch, &tsg->ch_list, nvgpu_channel, ch_entry) {
+		g->ops.channel.enable(ch);
+	}
+	nvgpu_rwsem_up_read(&tsg->ch_list_lock);
+}
+
+int vgpu_tsg_bind_channel(struct nvgpu_tsg *tsg, struct nvgpu_channel *ch)
+{
+	struct tegra_vgpu_cmd_msg msg = {};
+	int err;
+	struct gk20a *g = ch->g;
+
+	nvgpu_log_fn(g, " ");
+
+	if (!nvgpu_is_enabled(g, NVGPU_SUPPORT_TSG_SUBCONTEXTS)) {
+		struct tegra_vgpu_tsg_bind_unbind_channel_params *p =
+				&msg.params.tsg_bind_unbind_channel;
+
+		msg.cmd = TEGRA_VGPU_CMD_TSG_BIND_CHANNEL;
+		p->tsg_id = tsg->tsgid;
+		p->ch_handle = ch->virt_ctx;
+		p->runlist_id = ch->runlist->id;
+	} else {
+		struct tegra_vgpu_tsg_bind_channel_ex_params *p =
+				&msg.params.tsg_bind_channel_ex;
+
+		msg.cmd = TEGRA_VGPU_CMD_TSG_BIND_CHANNEL_EX;
+		p->tsg_id = tsg->tsgid;
+		p->ch_handle = ch->virt_ctx;
+		p->runlist_id = ch->runlist->id;
+		p->subctx_id = ch->subctx_id;
+		p->runqueue_sel = ch->runqueue_sel;
+	}
+
+	msg.handle = vgpu_get_handle(g);
+
+	err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
+	err = err ? err : msg.ret;
+	if (err) {
+		nvgpu_err(g, "vgpu_tsg_bind_channel failed, ch %d tsgid %d",
+			ch->chid, tsg->tsgid);
+	}
+
+	return err;
+}
+
+int vgpu_tsg_unbind_channel(struct nvgpu_tsg *tsg, struct nvgpu_channel *ch)
+{
+	struct tegra_vgpu_cmd_msg msg = {};
+	struct tegra_vgpu_tsg_bind_unbind_channel_params *p =
+				&msg.params.tsg_bind_unbind_channel;
+	int err;
+	struct gk20a *g = ch->g;
+
+	nvgpu_log_fn(g, " ");
+
+	msg.cmd = TEGRA_VGPU_CMD_TSG_UNBIND_CHANNEL;
+	msg.handle = vgpu_get_handle(g);
+	p->ch_handle = ch->virt_ctx;
+	err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
+	err = err ? err : msg.ret;
+	WARN_ON(err);
+
+	return err;
+}
+
+int vgpu_tsg_set_timeslice(struct nvgpu_tsg *tsg, u32 timeslice)
+{
+	struct tegra_vgpu_cmd_msg msg = {0};
+	struct tegra_vgpu_tsg_timeslice_params *p =
+				&msg.params.tsg_timeslice;
+	int err;
+	struct gk20a *g = tsg->g;
+
+	nvgpu_log_fn(g, " ");
+
+	msg.cmd = TEGRA_VGPU_CMD_TSG_SET_TIMESLICE;
+	msg.handle = vgpu_get_handle(g);
+	p->tsg_id = tsg->tsgid;
+	p->timeslice_us = timeslice;
+	err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
+	err = err ? err : msg.ret;
+	WARN_ON(err);
+	if (!err) {
+		tsg->timeslice_us = timeslice;
+	}
+
+	return err;
+}
+
+int vgpu_set_sm_exception_type_mask(struct nvgpu_channel *ch,
+		u32 exception_mask)
+{
+	struct tegra_vgpu_cmd_msg msg;
+	struct tegra_vgpu_set_sm_exception_type_mask_params *p =
+		&msg.params.set_sm_exception_mask;
+	int err = 0;
+	struct gk20a *g = ch->g;
+
+	nvgpu_log_fn(g, " ");
+
+	msg.cmd = TEGRA_VGPU_CMD_SET_SM_EXCEPTION_TYPE_MASK;
+	msg.handle = vgpu_get_handle(g);
+	p->handle = ch->virt_ctx;
+	p->mask = exception_mask;
+	err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
+	err = err ? err : msg.ret;
+	WARN_ON(err);
+
+	return err;
+}
+
+int vgpu_tsg_set_interleave(struct nvgpu_tsg *tsg, u32 new_level)
+{
+	struct tegra_vgpu_cmd_msg msg = {0};
+	struct tegra_vgpu_tsg_runlist_interleave_params *p =
+			&msg.params.tsg_interleave;
+	int err;
+	struct gk20a *g = tsg->g;
+
+	nvgpu_log_fn(g, " ");
+
+	msg.cmd = TEGRA_VGPU_CMD_TSG_SET_RUNLIST_INTERLEAVE;
+	msg.handle = vgpu_get_handle(g);
+	p->tsg_id = tsg->tsgid;
+	p->level = new_level;
+	err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
+	WARN_ON(err || msg.ret);
+	return err ? err : msg.ret;
+}
+
+int vgpu_tsg_force_reset_ch(struct nvgpu_channel *ch,
+					u32 err_code, bool verbose)
+{
+	struct nvgpu_tsg *tsg = NULL;
+	struct nvgpu_channel *ch_tsg = NULL;
+	struct gk20a *g = ch->g;
+	struct tegra_vgpu_cmd_msg msg = {0};
+	struct tegra_vgpu_channel_config_params *p =
+			&msg.params.channel_config;
+	int err;
+
+	nvgpu_log_fn(g, " ");
+
+	tsg = nvgpu_tsg_from_ch(ch);
+	if (tsg != NULL) {
+		nvgpu_rwsem_down_read(&tsg->ch_list_lock);
+
+		nvgpu_list_for_each_entry(ch_tsg, &tsg->ch_list,
+				nvgpu_channel, ch_entry) {
+			if (nvgpu_channel_get(ch_tsg)) {
+				nvgpu_channel_set_error_notifier(g, ch_tsg,
+								err_code);
+				nvgpu_channel_set_unserviceable(ch_tsg);
+				nvgpu_channel_put(ch_tsg);
+			}
+		}
+
+		nvgpu_rwsem_up_read(&tsg->ch_list_lock);
+	} else {
+		nvgpu_err(g, "chid: %d is not bound to tsg", ch->chid);
+	}
+
+	msg.cmd = TEGRA_VGPU_CMD_CHANNEL_FORCE_RESET;
+	msg.handle = vgpu_get_handle(ch->g);
+	p->handle = ch->virt_ctx;
+	err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg));
+	WARN_ON(err || msg.ret);
+	if (!err) {
+		nvgpu_channel_abort(ch, false);
+	}
+	return err ? err : msg.ret;
+}
+
+u32 vgpu_tsg_default_timeslice_us(struct gk20a *g)
+{
+	struct vgpu_priv_data *priv = vgpu_get_priv_data(g);
+
+	return priv->constants.default_timeslice_us;
+}
+
+void vgpu_tsg_set_ctx_mmu_error(struct gk20a *g, u32 chid)
+{
+	struct nvgpu_channel *ch = nvgpu_channel_from_id(g, chid);
+	struct nvgpu_tsg *tsg = NULL;
+
+	if (ch == NULL) {
+		return;
+	}
+
+	tsg = nvgpu_tsg_from_ch(ch);
+	if (tsg != NULL) {
+		struct nvgpu_channel *ch_tsg = NULL;
+
+		nvgpu_rwsem_down_read(&tsg->ch_list_lock);
+
+		nvgpu_list_for_each_entry(ch_tsg, &tsg->ch_list,
+				nvgpu_channel, ch_entry) {
+			if (nvgpu_channel_get(ch_tsg)) {
+				vgpu_channel_set_ctx_mmu_error(g, ch_tsg);
+				nvgpu_channel_put(ch_tsg);
+			}
+		}
+
+		nvgpu_rwsem_up_read(&tsg->ch_list_lock);
+	} else {
+		nvgpu_err(g, "chid: %d is not bound to tsg", ch->chid);
+	}
+}
+
+void vgpu_tsg_handle_event(struct gk20a *g,
+			struct tegra_vgpu_channel_event_info *info)
+{
+	struct nvgpu_tsg *tsg;
+
+	if (!info->is_tsg) {
+		nvgpu_err(g, "channel event posted");
+		return;
+	}
+
+	if (info->id >= g->fifo.num_channels ||
+		info->event_id >= TEGRA_VGPU_CHANNEL_EVENT_ID_MAX) {
+		nvgpu_err(g, "invalid channel event");
+		return;
+	}
+
+	tsg = &g->fifo.tsg[info->id];
+	g->ops.tsg.post_event_id(tsg, info->event_id);
+}

drivers/gpu/nvgpu/common/vgpu/fifo/tsg_vgpu.h

@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef NVGPU_TSG_VGPU_H
+#define NVGPU_TSG_VGPU_H
+
+#include <nvgpu/types.h>
+
+struct gk20a;
+struct nvgpu_channel;
+struct nvgpu_tsg;
+
+int vgpu_tsg_open(struct nvgpu_tsg *tsg);
+void vgpu_tsg_release(struct nvgpu_tsg *tsg);
+void vgpu_tsg_enable(struct nvgpu_tsg *tsg);
+int vgpu_tsg_bind_channel(struct nvgpu_tsg *tsg, struct nvgpu_channel *ch);
+int vgpu_tsg_unbind_channel(struct nvgpu_tsg *tsg, struct nvgpu_channel *ch);
+int vgpu_tsg_set_timeslice(struct nvgpu_tsg *tsg, u32 timeslice);
+int vgpu_set_sm_exception_type_mask(struct nvgpu_channel *ch,
+		u32 exception_mask);
+int vgpu_tsg_set_interleave(struct nvgpu_tsg *tsg, u32 new_level);
+int vgpu_tsg_force_reset_ch(struct nvgpu_channel *ch,
+					u32 err_code, bool verbose);
+u32 vgpu_tsg_default_timeslice_us(struct gk20a *g);
+void vgpu_tsg_set_ctx_mmu_error(struct gk20a *g, u32 chid);
+void vgpu_tsg_handle_event(struct gk20a *g,
+			struct tegra_vgpu_channel_event_info *info);
+
+#endif

drivers/gpu/nvgpu/common/vgpu/fifo/userd_vgpu.c

@@ -0,0 +1,39 @@
+/*
+ * Virtualized GPU USERD
+ *
+ * Copyright (c) 2014-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/trace.h>
+#include <nvgpu/gk20a.h>
+#include <nvgpu/fifo/userd.h>
+
+#include "userd_vgpu.h"
+
+int vgpu_userd_setup_sw(struct gk20a *g)
+{
+	return nvgpu_userd_init_slabs(g);
+}
+
+void vgpu_userd_cleanup_sw(struct gk20a *g)
+{
+	nvgpu_userd_free_slabs(g);
+}

drivers/gpu/nvgpu/common/vgpu/fifo/userd_vgpu.h

@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2011-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef USERD_VGPU_H
+#define USERD_VGPU_H
+
+struct gk20a;
+
+int vgpu_userd_setup_sw(struct gk20a *g);
+void vgpu_userd_cleanup_sw(struct gk20a *g);
+
+#endif /* USERD_VGPU_H */