gpu: nvgpu: add control-fifo queues

Added implementation for following IOCTLs
NVGPU_NVS_CTRL_FIFO_CREATE_QUEUE
NVGPU_NVS_CTRL_FIFO_RELEASE_QUEUE

The above ioctls are supported only for users with
R/W permissions.

1) NVGPU_NVS_CTRL_FIFO_CREATE_QUEUE constructs a memory region
via the nvgpu_dma_alloc_sys() API and creates the corresponding
GPU and kernel mappings. Upon successful creation, KMD exports
this buffer to the userspace via a dmabuf fd that the UMD
can use to mmap it into its process address space.

2) Added plumbing to store VMA's corresponding to different users
for event queue in future.

3) Added necessary validation checks for the IOCTLs

4) NVGPU_NVS_CTRL_FIFO_RELEASE_QUEUE is used to clear the queues.

5) Using a global queue lock to protect access to the queues. This
could be modified to be more fine-grained in future when there
is more clarity on GSP's implementation and access of queues.

6) Added plumbing to enable user subscription to queues.
NVGPU_NVS_CTRL_FIFO_RELEASE_QUEUE is used to unsubscribe
the user from the queue. Once, the last user is deleted,
all the queues will be cleared. User must ensure that
any mappings are removed before calling release queue.

7) Set the default queue_size for event queues to
PAGE_SIZE. This can be modified later. For event
queues, UMD shall fetch the queue_size.

Jira NVGPU-8129

Signed-off-by: Debarshi Dutta <ddutta@nvidia.com>
Change-Id: I31633174e960ec6feb77caede9d143b3b3c145d7
Reviewed-on: https://git-master.nvidia.com/r/c/linux-nvgpu/+/2723198
Reviewed-by: svc-mobile-cert <svc-mobile-cert@nvidia.com>
Reviewed-by: Vijayakumar Subbu <vsubbu@nvidia.com>
GVS: Gerrit_Virtual_Submit

arch/nvgpu-linux.yaml

@@ -39,7 +39,8 @@ dma:
 
 dmabuf:
   sources: [ os/linux/dmabuf_vidmem.c, os/linux/dmabuf_vidmem.h,
-             os/linux/dmabuf_priv.c, os/linux/dmabuf_priv.h ]
+             os/linux/dmabuf_priv.c, os/linux/dmabuf_priv.h,
+             os/linux/dmabuf_nvs.c, os/linux/dmabuf_nvs.h ]
 
 driver_common:
   sources: [ os/linux/driver_common.c, os/linux/driver_common.h ]

drivers/gpu/nvgpu/Makefile

@@ -127,6 +127,7 @@ nvgpu-$(CONFIG_NVGPU_DGPU) += \
 	os/linux/pci.o \
 	os/linux/pci_power.o \
 	os/linux/dmabuf_vidmem.o \
+	os/linux/dmabuf_nvs.o \
 	os/linux/os_ops_gv100.o \
 	os/linux/os_ops_tu104.o \
 	common/sec2/sec2.o \

drivers/gpu/nvgpu/common/nvs/nvs_sched_ctrl.c

@@ -26,6 +26,7 @@
 #include <nvgpu/kmem.h>
 #include <nvgpu/gk20a.h>
 #include <nvgpu/list.h>
+#include <nvgpu/dma.h>
 
 struct nvgpu_nvs_domain_ctrl_fifo_users {
 	/* Flag to reserve exclusive user */
@@ -40,6 +41,34 @@ struct nvgpu_nvs_domain_ctrl_fifo_users {
 	struct nvgpu_spinlock user_lock;
 };
 
+struct nvgpu_nvs_domain_ctrl_fifo_queues {
+	/*
+	 * send indicates a buffer having data(PUT) written by a userspace client
+	 * and queried by the scheduler(GET).
+	 */
+	struct nvgpu_nvs_ctrl_queue send;
+	/*
+	 * receive indicates a buffer having data(PUT) written by scheduler
+	 * and queried by the userspace client(GET).
+	 */
+	struct nvgpu_nvs_ctrl_queue receive;
+
+	/*
+	 * event indicates a buffer that is subscribed to by userspace clients to
+	 * receive events. This buffer is Read-Only for the users and only scheduler can
+	 * write to it.
+	 */
+	struct nvgpu_nvs_ctrl_queue event;
+
+	/*
+	 * Global mutex for coarse grained access control
+	 * of all Queues for all UMD interfaces. e.g. IOCTL/devctls
+	 * and mmap calls. Keeping this as coarse-grained for now till
+	 * GSP's implementation is complete.
+	 */
+	struct nvgpu_mutex queue_lock;
+};
+
 struct nvgpu_nvs_domain_ctrl_fifo {
 	/*
 	 * Instance of global struct gk20a;
@@ -47,6 +76,8 @@ struct nvgpu_nvs_domain_ctrl_fifo {
 	struct gk20a *g;
 
 	struct nvgpu_nvs_domain_ctrl_fifo_users users;
+
+	struct nvgpu_nvs_domain_ctrl_fifo_queues queues;
 };
 
 void nvgpu_nvs_ctrl_fifo_reset_exclusive_user(
@@ -173,6 +204,7 @@ struct nvgpu_nvs_domain_ctrl_fifo *nvgpu_nvs_ctrl_fifo_create(struct gk20a *g)
 	}
 
 	nvgpu_spinlock_init(&sched->users.user_lock);
+	nvgpu_mutex_init(&sched->queues.queue_lock);
 	nvgpu_init_list_node(&sched->users.exclusive_user);
 	nvgpu_init_list_node(&sched->users.list_non_exclusive_user);
 
@@ -200,6 +232,144 @@ void nvgpu_nvs_ctrl_fifo_destroy(struct gk20a *g)
 
 	nvgpu_assert(!nvgpu_nvs_ctrl_fifo_is_busy(sched_ctrl));
 
+	nvgpu_nvs_ctrl_fifo_erase_all_queues(g);
+
 	nvgpu_kfree(g, sched_ctrl);
 	g->sched_ctrl_fifo = NULL;
 }
+
+struct nvgpu_nvs_ctrl_queue *nvgpu_nvs_ctrl_fifo_get_queue(
+		struct nvgpu_nvs_domain_ctrl_fifo *sched_ctrl,
+		enum nvgpu_nvs_ctrl_queue_num queue_num,
+		enum nvgpu_nvs_ctrl_queue_direction queue_direction,
+		u8 *mask)
+{
+	struct nvgpu_nvs_ctrl_queue *queue = NULL;
+
+	if (sched_ctrl == NULL) {
+		return NULL;
+	}
+
+	if (mask == NULL) {
+		return NULL;
+	}
+
+	if (queue_num == NVGPU_NVS_NUM_CONTROL) {
+		if (queue_direction == NVGPU_NVS_DIR_CLIENT_TO_SCHEDULER) {
+			queue = &sched_ctrl->queues.send;
+			*mask = NVGPU_NVS_CTRL_FIFO_QUEUE_EXCLUSIVE_CLIENT_WRITE;
+		} else if (queue_direction == NVGPU_NVS_DIR_SCHEDULER_TO_CLIENT) {
+			queue = &sched_ctrl->queues.receive;
+			*mask = NVGPU_NVS_CTRL_FIFO_QUEUE_EXCLUSIVE_CLIENT_READ;
+		}
+	} else if (queue_num == NVGPU_NVS_NUM_EVENT) {
+		if (queue_direction == NVGPU_NVS_DIR_SCHEDULER_TO_CLIENT) {
+			queue = &sched_ctrl->queues.event;
+			*mask = NVGPU_NVS_CTRL_FIFO_QUEUE_CLIENT_EVENTS_READ;
+		}
+	}
+
+	return queue;
+}
+
+bool nvgpu_nvs_buffer_is_valid(struct gk20a *g, struct nvgpu_nvs_ctrl_queue *buf)
+{
+	return buf->valid;
+}
+
+int nvgpu_nvs_buffer_alloc(struct nvgpu_nvs_domain_ctrl_fifo *sched_ctrl,
+		size_t bytes, u8 mask, struct nvgpu_nvs_ctrl_queue *buf)
+{
+	int err;
+	struct gk20a *g = sched_ctrl->g;
+	struct vm_gk20a *system_vm = g->mm.pmu.vm;
+
+	(void)memset(buf, 0, sizeof(*buf));
+	buf->g = g;
+
+	err = nvgpu_dma_alloc_map_sys(system_vm, bytes, &buf->mem);
+	if (err != 0) {
+		nvgpu_err(g, "failed to allocate memory for dma");
+		goto fail;
+	}
+
+	buf->valid = true;
+	buf->mask = mask;
+
+	return 0;
+
+fail:
+	(void)memset(buf, 0, sizeof(*buf));
+
+	return err;
+}
+
+void nvgpu_nvs_buffer_free(struct nvgpu_nvs_domain_ctrl_fifo *sched_ctrl,
+		struct nvgpu_nvs_ctrl_queue *buf)
+{
+	struct gk20a *g = sched_ctrl->g;
+	struct vm_gk20a *system_vm = g->mm.pmu.vm;
+
+	if (nvgpu_mem_is_valid(&buf->mem)) {
+		nvgpu_dma_unmap_free(system_vm, &buf->mem);
+	}
+
+	/* Sets buf->valid as false */
+	(void)memset(buf, 0, sizeof(*buf));
+}
+
+void nvgpu_nvs_ctrl_fifo_lock_queues(struct gk20a *g)
+{
+	struct nvgpu_nvs_domain_ctrl_fifo *sched_ctrl = g->sched_ctrl_fifo;
+	nvgpu_mutex_acquire(&sched_ctrl->queues.queue_lock);
+}
+
+void nvgpu_nvs_ctrl_fifo_unlock_queues(struct gk20a *g)
+{
+	struct nvgpu_nvs_domain_ctrl_fifo *sched_ctrl = g->sched_ctrl_fifo;
+	nvgpu_mutex_release(&sched_ctrl->queues.queue_lock);
+}
+
+void nvgpu_nvs_ctrl_fifo_user_subscribe_queue(struct nvs_domain_ctrl_fifo_user *user,
+		struct nvgpu_nvs_ctrl_queue *queue)
+{
+	user->active_used_queues |= queue->mask;
+}
+void nvgpu_nvs_ctrl_fifo_user_unsubscribe_queue(struct nvs_domain_ctrl_fifo_user *user,
+		struct nvgpu_nvs_ctrl_queue *queue)
+{
+	user->active_used_queues &= ~queue->mask;
+}
+bool nvgpu_nvs_ctrl_fifo_user_is_subscribed_to_queue(struct nvs_domain_ctrl_fifo_user *user,
+		struct nvgpu_nvs_ctrl_queue *queue)
+{
+	return (user->active_used_queues & queue->mask);
+}
+
+void nvgpu_nvs_ctrl_fifo_erase_all_queues(struct gk20a *g)
+{
+	struct nvgpu_nvs_domain_ctrl_fifo *sched_ctrl = g->sched_ctrl_fifo;
+
+	nvgpu_nvs_ctrl_fifo_lock_queues(g);
+
+	if (nvgpu_nvs_buffer_is_valid(g, &sched_ctrl->queues.send)) {
+		nvgpu_nvs_ctrl_fifo_erase_queue(g, &sched_ctrl->queues.send);
+	}
+
+	if (nvgpu_nvs_buffer_is_valid(g, &sched_ctrl->queues.receive)) {
+		nvgpu_nvs_ctrl_fifo_erase_queue(g, &sched_ctrl->queues.receive);
+	}
+
+	if (nvgpu_nvs_buffer_is_valid(g, &sched_ctrl->queues.event)) {
+		nvgpu_nvs_ctrl_fifo_erase_queue(g, &sched_ctrl->queues.event);
+	}
+
+	nvgpu_nvs_ctrl_fifo_unlock_queues(g);
+}
+
+void nvgpu_nvs_ctrl_fifo_erase_queue(struct gk20a *g, struct nvgpu_nvs_ctrl_queue *queue)
+{
+	if (queue->free != NULL) {
+		queue->free(g, queue);
+	}
+}

drivers/gpu/nvgpu/include/nvgpu/nvs.h

@@ -31,11 +31,16 @@
 #include <nvgpu/lock.h>
 #include <nvgpu/worker.h>
 #include <nvgpu/timers.h>
+#include <nvgpu/nvgpu_mem.h>
 
 /*
  * Max size we'll parse from an NVS log entry.
  */
 #define NVS_LOG_BUF_SIZE	128
+/*
+ * Keep it to page size for now. Can be updated later.
+ */
+#define NVS_QUEUE_DEFAULT_SIZE (64 * 1024)
 
 struct gk20a;
 struct nvgpu_nvs_domain_ioctl;
@@ -126,6 +131,85 @@ struct nvgpu_nvs_scheduler {
 	struct nvgpu_nvs_domain *shadow_domain;
 };
 
+enum nvgpu_nvs_ctrl_queue_num {
+	NVGPU_NVS_NUM_CONTROL = 0,
+	NVGPU_NVS_NUM_EVENT,
+	NVGPU_NVS_INVALID,
+};
+
+enum nvgpu_nvs_ctrl_queue_direction {
+	NVGPU_NVS_DIR_CLIENT_TO_SCHEDULER = 0,
+	NVGPU_NVS_DIR_SCHEDULER_TO_CLIENT,
+	NVGPU_NVS_DIR_INVALID,
+};
+
+/*
+ * The below definitions mirror the nvgpu-nvs(UAPI)
+ * headers.
+ */
+
+/*
+ * Invalid domain scheduler.
+ * The value of 'domain_scheduler_implementation'
+ * when 'has_domain_scheduler_control_fifo' is 0.
+ */
+#define NVGPU_NVS_DOMAIN_SCHED_INVALID 0U
+/*
+ * CPU based scheduler implementation. Intended use is mainly
+ * for debug and testing purposes. Doesn't meet latency requirements.
+ * Implementation will be supported in the initial versions and eventually
+ * discarded.
+ */
+#define NVGPU_NVS_DOMAIN_SCHED_KMD 1U
+/*
+ * GSP based scheduler implementation that meets latency requirements.
+ * This implementation will eventually replace NVGPU_NVS_DOMAIN_SCHED_KMD.
+ */
+#define NVGPU_NVS_DOMAIN_SCHED_GSP 2U
+
+/* Queue meant for exclusive client write access. This shared queue will be
+ * used for communicating the scheduling metadata between the client(producer)
+ * and scheduler(consumer).
+ */
+#define NVGPU_NVS_CTRL_FIFO_QUEUE_EXCLUSIVE_CLIENT_WRITE 1U
+
+/* Queue meant for exclusive client read access. This shared queue will be
+ * used for communicating the scheduling metadata between the scheduler(producer)
+ * and client(consumer).
+ */
+#define NVGPU_NVS_CTRL_FIFO_QUEUE_EXCLUSIVE_CLIENT_READ 2U
+
+/* Queue meant for generic read access. Clients can subscribe to this read-only
+ * queue for processing events such as recovery, preemption etc.
+ */
+#define NVGPU_NVS_CTRL_FIFO_QUEUE_CLIENT_EVENTS_READ 4U
+
+/*
+ * Direction of the requested queue is from CLIENT(producer)
+ * to SCHEDULER(consumer).
+ */
+#define NVGPU_NVS_CTRL_FIFO_QUEUE_DIRECTION_CLIENT_TO_SCHEDULER 0
+
+/*
+ * Direction of the requested queue is from SCHEDULER(producer)
+ * to CLIENT(consumer).
+ */
+#define NVGPU_NVS_CTRL_FIFO_QUEUE_DIRECTION_SCHEDULER_TO_CLIENT 1
+
+/* Structure to hold control_queues. This can be then passed to GSP or Rm based subscheduler. */
+struct nvgpu_nvs_ctrl_queue {
+	struct nvgpu_mem	mem;
+	struct gk20a		*g;
+	/*
+	 * Filled in by each OS - this holds the necessary data to export this
+	 * buffer to userspace.
+	 */
+	void			*priv;
+	bool			valid;
+	u8				mask;
+	void (*free)(struct gk20a *g, struct nvgpu_nvs_ctrl_queue *queue);
+};
+
 #ifdef CONFIG_NVS_PRESENT
 int nvgpu_nvs_init(struct gk20a *g);
 int nvgpu_nvs_open(struct gk20a *g);
@@ -150,9 +234,14 @@ const char *nvgpu_nvs_domain_get_name(struct nvgpu_nvs_domain *dom);
 #define nvs_dbg(g, fmt, arg...)			\
 	nvgpu_log(g, gpu_dbg_nvs, fmt, ##arg)
 
+void nvgpu_nvs_ctrl_fifo_lock_queues(struct gk20a *g);
+void nvgpu_nvs_ctrl_fifo_unlock_queues(struct gk20a *g);
+
 struct nvgpu_nvs_domain_ctrl_fifo *nvgpu_nvs_ctrl_fifo_create(struct gk20a *g);
 bool nvgpu_nvs_ctrl_fifo_user_exists(struct nvgpu_nvs_domain_ctrl_fifo *sched_ctrl,
     int pid, bool rw);
+bool nvgpu_nvs_ctrl_fifo_is_busy(struct nvgpu_nvs_domain_ctrl_fifo *sched_ctrl);
+void nvgpu_nvs_ctrl_fifo_destroy(struct gk20a *g);
 bool nvgpu_nvs_ctrl_fifo_user_is_active(struct nvs_domain_ctrl_fifo_user *user);
 void nvgpu_nvs_ctrl_fifo_add_user(struct nvgpu_nvs_domain_ctrl_fifo *sched_ctrl,
     struct nvs_domain_ctrl_fifo_user *user);
@@ -163,9 +252,26 @@ void nvgpu_nvs_ctrl_fifo_reset_exclusive_user(
 int nvgpu_nvs_ctrl_fifo_reserve_exclusive_user(
 		struct nvgpu_nvs_domain_ctrl_fifo *sched_ctrl, struct nvs_domain_ctrl_fifo_user *user);
 void nvgpu_nvs_ctrl_fifo_remove_user(struct nvgpu_nvs_domain_ctrl_fifo *sched_ctrl,
-    struct nvs_domain_ctrl_fifo_user *user);
-bool nvgpu_nvs_ctrl_fifo_is_busy(struct nvgpu_nvs_domain_ctrl_fifo *sched_ctrl);
-void nvgpu_nvs_ctrl_fifo_destroy(struct gk20a *g);
+		struct nvs_domain_ctrl_fifo_user *user);
+struct nvgpu_nvs_ctrl_queue *nvgpu_nvs_ctrl_fifo_get_queue(
+		struct nvgpu_nvs_domain_ctrl_fifo *sched_ctrl,
+		enum nvgpu_nvs_ctrl_queue_num queue_num,
+		enum nvgpu_nvs_ctrl_queue_direction queue_direction,
+		u8 *mask);
+/* Below methods require nvgpu_nvs_ctrl_fifo_lock_queues() to be held. */
+bool nvgpu_nvs_buffer_is_valid(struct gk20a *g, struct nvgpu_nvs_ctrl_queue *buf);
+int nvgpu_nvs_buffer_alloc(struct nvgpu_nvs_domain_ctrl_fifo *sched_ctrl,
+		size_t bytes, u8 mask, struct nvgpu_nvs_ctrl_queue *buf);
+void nvgpu_nvs_buffer_free(struct nvgpu_nvs_domain_ctrl_fifo *sched_ctrl,
+		struct nvgpu_nvs_ctrl_queue *buf);
+void nvgpu_nvs_ctrl_fifo_user_subscribe_queue(struct nvs_domain_ctrl_fifo_user *user,
+		struct nvgpu_nvs_ctrl_queue *queue);
+void nvgpu_nvs_ctrl_fifo_user_unsubscribe_queue(struct nvs_domain_ctrl_fifo_user *user,
+		struct nvgpu_nvs_ctrl_queue *queue);
+bool nvgpu_nvs_ctrl_fifo_user_is_subscribed_to_queue(struct nvs_domain_ctrl_fifo_user *user,
+		struct nvgpu_nvs_ctrl_queue *queue);
+void nvgpu_nvs_ctrl_fifo_erase_queue(struct gk20a *g, struct nvgpu_nvs_ctrl_queue *queue);
+void nvgpu_nvs_ctrl_fifo_erase_all_queues(struct gk20a *g);
 
 #else
 static inline int nvgpu_nvs_init(struct gk20a *g)

drivers/gpu/nvgpu/os/linux/dmabuf_nvs.c

@@ -0,0 +1,290 @@
+/*
+ * Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/version.h>
+#include <linux/dma-buf.h>
+#include <uapi/linux/nvgpu.h>
+
+#ifdef CONFIG_NVGPU_USE_TEGRA_ALLOC_FD
+#include <linux/platform/tegra/tegra_fd.h>
+#endif
+
+#include <nvgpu/dma.h>
+#include <nvgpu/enabled.h>
+#include <nvgpu/nvgpu_mem.h>
+#include <nvgpu/kmem.h>
+#include <nvgpu/gk20a.h>
+#include <nvgpu/nvgpu_init.h>
+
+#include <nvgpu/linux/vm.h>
+#include <nvgpu/linux/dma.h>
+#include <nvgpu/nvs.h>
+
+#include "dmabuf_nvs.h"
+
+/* This constant string is used to determine if the dmabuf belongs
+ * to nvgpu.
+ */
+static const char exporter_name[] = "nvgpu_ctrl_fifo";
+
+static struct nvgpu_nvs_domain_ctrl_fifo_user_vma *
+		nvgpu_nvs_domain_ctrl_fifo_user_vma_from_node(struct nvgpu_list_node *node)
+{
+	nvgpu_assert(node != NULL);
+	return container_of(node, struct nvgpu_nvs_domain_ctrl_fifo_user_vma, node);
+}
+
+static int zap_vma_entries(struct gk20a *g, struct vm_area_struct *vma)
+{
+	int err = 0;
+
+	#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 8, 0)
+		mmap_write_lock(vma->vm_mm);
+	#else
+		down_write(&vma->vm_mm->mmap_sem);
+	#endif
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 18, 0)
+		zap_vma_ptes(vma, vma->vm_start, vma->vm_end - vma->vm_start);
+#else
+		err = zap_vma_ptes(vma, vma->vm_start,
+				   vma->vm_end - vma->vm_start);
+#endif
+
+	#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 8, 0)
+		mmap_write_unlock(vma->vm_mm);
+	#else
+		up_write(&vma->vm_mm->mmap_sem);
+	#endif
+
+	return err;
+}
+
+static int nvs_release_user_mappings_locked(struct gk20a *g, struct nvgpu_nvs_linux_buf_priv *linux_buf)
+{
+	struct nvgpu_nvs_domain_ctrl_fifo_user_vma *current_entry, *next_entry;
+	int err = 0;
+
+	nvgpu_list_for_each_entry_safe(current_entry, next_entry,
+			&linux_buf->list_mapped_user_vmas,
+			nvgpu_nvs_domain_ctrl_fifo_user_vma, node) {
+		struct vm_area_struct *vma = current_entry->vma;
+
+		zap_vma_entries(g, vma);
+		linux_buf->ref--;
+	}
+
+	return err;
+}
+
+static void nvs_vma_close(struct vm_area_struct *vma)
+{
+	struct nvgpu_nvs_domain_ctrl_fifo_user_vma *vma_metadata = vma->vm_private_data;
+	struct nvgpu_nvs_ctrl_queue *buf = vma_metadata->buf;
+	struct nvgpu_nvs_linux_buf_priv *linux_buf = (struct nvgpu_nvs_linux_buf_priv *)buf->priv;
+	struct gk20a *g = buf->g;
+
+	nvgpu_nvs_ctrl_fifo_lock_queues(g);
+
+	linux_buf->ref--;
+	nvgpu_list_del(&vma_metadata->node);
+
+	/* This VMA is freed now and points to invalid ptes */
+	vma_metadata->vma = NULL;
+	nvgpu_kfree(g, vma_metadata);
+	vma->vm_private_data = NULL;
+
+	nvgpu_nvs_ctrl_fifo_unlock_queues(g);
+
+	nvgpu_put(g);
+}
+
+const struct vm_operations_struct nvs_vma_ops = {
+	/* no .open - we use VM_DONTCOPY and don't support fork */
+	.close = nvs_vma_close,
+};
+
+static int nvgpu_nvs_buf_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma)
+{
+	struct nvgpu_nvs_ctrl_queue *buf = dmabuf->priv;
+	struct nvgpu_nvs_linux_buf_priv *linux_buf = (struct nvgpu_nvs_linux_buf_priv *)buf->priv;
+	struct gk20a *g = buf->g;
+	struct nvgpu_mem *mem = &buf->mem;
+	struct nvgpu_nvs_domain_ctrl_fifo_user_vma *vma_metadata = NULL;
+	int err;
+	const struct vm_operations_struct *vm_ops_old = NULL;
+
+	if (vma->vm_end - vma->vm_start > buf->mem.size) {
+		return -EINVAL;
+	}
+
+	if (vma->vm_pgoff != 0UL) {
+		return -EINVAL;
+	}
+
+	/*
+	 * This ref is released when the mapping is removed.
+	 */
+	if (!nvgpu_get(g))
+		return -ENODEV;
+
+	nvgpu_nvs_ctrl_fifo_lock_queues(g);
+
+	vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_NORESERVE |
+		VM_DONTDUMP;
+
+	if (linux_buf->read_only) {
+		vma->vm_flags |= VM_SHARED;
+	}
+
+	vm_ops_old = vma->vm_ops;
+
+	vma->vm_ops = &nvs_vma_ops;
+
+	err = nvgpu_dma_mmap_sys(g, vma, mem);
+	if (err != 0) {
+		goto fail;
+	}
+
+	vma_metadata = nvgpu_kzalloc(g, sizeof(*vma_metadata));
+	if (vma_metadata == NULL) {
+		err = -ENOMEM;
+		zap_vma_entries(g, vma);
+		goto fail;
+	}
+
+	vma_metadata->vma = vma;
+	vma_metadata->buf = buf;
+	nvgpu_init_list_node(&vma_metadata->node);
+
+	linux_buf->ref++;
+	nvgpu_list_add_tail(&vma_metadata->node, &linux_buf->list_mapped_user_vmas);
+
+	vma->vm_private_data = vma_metadata;
+
+	nvgpu_nvs_ctrl_fifo_unlock_queues(g);
+
+	return 0;
+fail:
+	vma->vm_ops = vm_ops_old;
+
+	if (vma_metadata != NULL) {
+		nvgpu_kfree(g, vma_metadata);
+	}
+
+	nvgpu_nvs_ctrl_fifo_unlock_queues(g);
+
+	return err;
+}
+
+static struct dma_buf_ops gk20a_nvs_ops = {
+	.mmap             = nvgpu_nvs_buf_mmap,
+};
+
+static struct dma_buf *nvgpu_nvs_buf_export_dmabuf(struct nvgpu_nvs_ctrl_queue *buf,
+		bool read_only)
+{
+	DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
+	struct nvgpu_mem *mem = &buf->mem;
+
+	int flags = (read_only ? O_RDONLY : O_RDWR) | O_CLOEXEC;
+
+	exp_info.priv = buf;
+	exp_info.ops = &gk20a_nvs_ops;
+	exp_info.size = mem->size;
+	exp_info.flags = flags;
+	exp_info.exp_name = exporter_name;
+
+	return dma_buf_export(&exp_info);
+}
+
+static void nvgpu_nvs_destroy_buf_linux_locked(struct gk20a *g, struct nvgpu_nvs_ctrl_queue *buf)
+{
+	struct nvgpu_nvs_domain_ctrl_fifo *sched_ctrl = g->sched_ctrl_fifo;
+	struct nvgpu_nvs_linux_buf_priv *priv;
+
+	priv = (struct nvgpu_nvs_linux_buf_priv *)buf->priv;
+
+	nvs_release_user_mappings_locked(g, priv);
+
+	dma_buf_put(priv->dmabuf);
+
+	nvgpu_nvs_buffer_free(sched_ctrl, buf);
+	nvgpu_kfree(g, priv);
+
+	nvgpu_put(g);
+}
+
+bool nvgpu_nvs_buf_linux_is_mapped(struct gk20a *g, struct nvgpu_nvs_ctrl_queue *buf)
+{
+	bool is_mapped;
+	struct nvgpu_nvs_linux_buf_priv *priv = NULL;
+
+	priv = (struct nvgpu_nvs_linux_buf_priv *)buf->priv;
+	is_mapped = (priv->ref != 0U);
+
+	return is_mapped;
+}
+
+int nvgpu_nvs_get_buf_linux(struct gk20a *g, struct nvgpu_nvs_ctrl_queue *buf,
+		size_t bytes, u8 mask, bool read_only)
+{
+	struct nvgpu_nvs_linux_buf_priv *priv;
+	int err;
+	struct nvgpu_nvs_domain_ctrl_fifo *sched_ctrl = g->sched_ctrl_fifo;
+
+	/*
+	 * This ref is released when the dma_buf is closed.
+	 */
+	if (!nvgpu_get(g))
+		return -ENODEV;
+
+	nvs_dbg(g, "Allocating buf: %zu bytes", bytes);
+
+	priv = nvgpu_kzalloc(g, sizeof(*priv));
+	if (!priv) {
+		err = -ENOMEM;
+		goto fail;
+	}
+
+	nvgpu_init_list_node(&priv->list_mapped_user_vmas);
+	priv->read_only = read_only;
+
+	err = nvgpu_nvs_buffer_alloc(sched_ctrl, bytes, mask, buf);
+	if (err != 0) {
+		nvgpu_err(g, "Unable to alloc memory");
+		goto fail;
+	}
+
+	priv->dmabuf = nvgpu_nvs_buf_export_dmabuf(buf, read_only);
+	if (IS_ERR(priv->dmabuf)) {
+		nvgpu_err(g, "Unable to export dma buf");
+		err = PTR_ERR(priv->dmabuf);
+		goto fail;
+	}
+
+	buf->priv = priv;
+	buf->free = nvgpu_nvs_destroy_buf_linux_locked;
+
+	return 0;
+fail:
+	nvgpu_kfree(g, priv);
+	nvgpu_nvs_buffer_free(sched_ctrl, buf);
+
+	nvgpu_put(g);
+
+	return err;
+}

drivers/gpu/nvgpu/os/linux/dmabuf_nvs.h

@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef NVGPU_DMABUF_NVS_H
+#define NVGPU_DMABUF_NVS_H
+
+#include <nvgpu/types.h>
+#include <nvgpu/list.h>
+
+struct dma_buf;
+struct gk20a;
+
+struct nvgpu_nvs_linux_buf_priv {
+	struct dma_buf	*dmabuf;
+	bool read_only;
+	u32 ref;
+	struct nvgpu_list_node list_mapped_user_vmas;
+};
+
+struct nvgpu_nvs_domain_ctrl_fifo_user_vma {
+	struct nvgpu_nvs_ctrl_queue *buf;
+	struct vm_area_struct *vma;
+	struct nvgpu_list_node node;
+};
+
+/**
+ * @brief Construct a buffer for use as a shared message passing
+ *        queue between user and backend scheduler. Function is
+ *        not safe from concurrent access by multiple external
+ *        users. Must be invoked between the calls of
+ *        nvgpu_nvs_ctrl_fifo_lock_queues() and
+ *        nvgpu_nvs_ctrl_fifo_unlock_queues().
+ *
+ * @param g instance of struct gk20a.
+ * @param buf instance of struct nvgpu_nvs_ctrl_queue to contain
+ *            the constructed buffer metadata.
+ * @param bytes size of buffer requested.
+ * @param mask Mask of queue requested.
+ * @param read_only Indicates whether a read-only buffer is requested.
+ * @return int 0 on success, else fail.
+ */
+int nvgpu_nvs_get_buf_linux(struct gk20a *g, struct nvgpu_nvs_ctrl_queue *buf,
+		size_t bytes, u8 mask, bool read_only);
+
+/**
+ * @brief Check whether user mappings exist for this buffer.
+ *
+ * @param g instance of struct gk20a.
+ * @param buf instance of struct nvgpu_nvs_ctrl_queue.
+ * @return true User mappings exist.
+ * @return false User mappings doesn't exist.
+ */
+bool nvgpu_nvs_buf_linux_is_mapped(struct gk20a *g, struct nvgpu_nvs_ctrl_queue *buf);
+
+#endif /* NVGPU_DMABUF_NVS_H */

drivers/gpu/nvgpu/os/linux/ioctl_nvs.c

@@ -15,6 +15,7 @@
 #include <linux/fs.h>
 #include <linux/file.h>
 #include <linux/uaccess.h>
+#include <linux/dma-buf.h>
 
 #include <uapi/linux/nvgpu-nvs.h>
 
@@ -28,6 +29,7 @@
 #include <nvs/domain.h>
 
 #include "ioctl.h"
+#include "dmabuf_nvs.h"
 
 /*
  * OS-specific layer to hold device node mapping for a domain.
@@ -665,6 +667,9 @@ int nvgpu_nvs_ctrl_fifo_ops_release(struct inode *inode, struct file *filp)
 	}
 
 	nvgpu_nvs_ctrl_fifo_remove_user(g->sched_ctrl_fifo, &linux_user->user);
+	if (!nvgpu_nvs_ctrl_fifo_is_busy(g->sched_ctrl_fifo)) {
+		nvgpu_nvs_ctrl_fifo_erase_all_queues(g);
+	}
 
 	filp->private_data = NULL;
 
@@ -675,3 +680,352 @@ int nvgpu_nvs_ctrl_fifo_ops_release(struct inode *inode, struct file *filp)
 }
 
 extern const struct file_operations nvgpu_nvs_ctrl_fifo_ops;
+
+static int nvgpu_nvs_ctrl_fifo_create_queue_verify_flags(struct gk20a *g,
+		struct nvs_domain_ctrl_fifo_user *user,
+		struct nvgpu_nvs_ctrl_fifo_ioctl_create_queue_args *args)
+{
+	int err = 0;
+
+	if (args->reserve0 != 0) {
+		return -EINVAL;
+	}
+
+	if (args->dmabuf_fd != 0) {
+		return -EINVAL;
+	}
+
+	if (args->queue_size != 0) {
+		return -EINVAL;
+	}
+
+	if (args->access_type == NVS_CTRL_FIFO_QUEUE_ACCESS_TYPE_EXCLUSIVE) {
+		if (args->queue_num == 0)
+			return -EINVAL;
+		if (args->direction == 0)
+			return -EINVAL;
+		if (!nvgpu_nvs_ctrl_fifo_is_exclusive_user(g->sched_ctrl_fifo, user)) {
+			err = nvgpu_nvs_ctrl_fifo_reserve_exclusive_user(g->sched_ctrl_fifo, user);
+			if (err != 0) {
+				return err;
+			}
+		}
+	} else {
+		if (args->queue_num != NVS_CTRL_FIFO_QUEUE_NUM_EVENT)
+			return -EINVAL;
+		if (args->direction != NVS_CTRL_FIFO_QUEUE_DIRECTION_SCHEDULER_TO_CLIENT)
+			return -EINVAL;
+	}
+
+	return 0;
+}
+
+static enum nvgpu_nvs_ctrl_queue_num nvgpu_nvs_translate_queue_num(u32 queue_num_arg)
+{
+	enum nvgpu_nvs_ctrl_queue_num num_queue = NVGPU_NVS_INVALID;
+	if (queue_num_arg == NVGPU_NVS_CTRL_FIFO_QUEUE_NUM_CONTROL)
+		num_queue = NVGPU_NVS_NUM_CONTROL;
+	else if (queue_num_arg == NVS_CTRL_FIFO_QUEUE_NUM_EVENT)
+		num_queue = NVGPU_NVS_NUM_EVENT;
+
+	return num_queue;
+}
+
+static enum nvgpu_nvs_ctrl_queue_direction
+	nvgpu_nvs_translate_queue_direction(u32 queue_direction)
+{
+	enum nvgpu_nvs_ctrl_queue_direction direction = NVGPU_NVS_DIR_INVALID;
+	if (queue_direction == NVS_CTRL_FIFO_QUEUE_DIRECTION_CLIENT_TO_SCHEDULER)
+		direction = NVGPU_NVS_DIR_CLIENT_TO_SCHEDULER;
+	else if (queue_direction == NVS_CTRL_FIFO_QUEUE_DIRECTION_SCHEDULER_TO_CLIENT)
+		direction = NVGPU_NVS_DIR_SCHEDULER_TO_CLIENT;
+
+	return direction;
+}
+
+static int nvgpu_nvs_ctrl_fifo_create_queue(struct gk20a *g,
+		struct nvs_domain_ctrl_fifo_user *user,
+		struct nvgpu_nvs_ctrl_fifo_ioctl_create_queue_args *args)
+{
+	enum nvgpu_nvs_ctrl_queue_num num_queue;
+	enum nvgpu_nvs_ctrl_queue_direction queue_direction;
+	struct nvgpu_nvs_ctrl_queue *queue;
+	struct nvgpu_nvs_linux_buf_priv *priv = NULL;
+	int err = 0;
+	int fd;
+	int flag = O_CLOEXEC;
+	bool read_only;
+	size_t queue_size;
+	u8 mask = 0;
+
+	err = nvgpu_nvs_ctrl_fifo_create_queue_verify_flags(g, user, args);
+	if (err != 0) {
+		args->dmabuf_fd = -1;
+		return err;
+	}
+
+	nvgpu_nvs_ctrl_fifo_lock_queues(g);
+
+	num_queue = nvgpu_nvs_translate_queue_num(args->queue_num);
+	queue_direction = nvgpu_nvs_translate_queue_direction(args->direction);
+
+	queue = nvgpu_nvs_ctrl_fifo_get_queue(g->sched_ctrl_fifo, num_queue, queue_direction, &mask);
+	if (queue == NULL) {
+		err = -EOPNOTSUPP;
+		goto fail;
+	}
+
+	read_only = (args->access_type == NVS_CTRL_FIFO_QUEUE_ACCESS_TYPE_EXCLUSIVE) ? false : true;
+	if (read_only) {
+		flag |= O_RDONLY;
+	} else {
+		flag |= O_RDWR;
+	}
+
+	/* Support for Read-Only Observers will be added later */
+	if (read_only) {
+		err = -EOPNOTSUPP;
+		goto fail;
+	}
+
+	if (args->access_type == NVS_CTRL_FIFO_QUEUE_ACCESS_TYPE_EXCLUSIVE) {
+		if (nvgpu_nvs_buffer_is_valid(g, queue)) {
+			err = -EBUSY;
+			goto fail;
+		}
+	}
+
+	/* For event queue, prevent multiple subscription by the same user */
+	if (nvgpu_nvs_ctrl_fifo_user_is_subscribed_to_queue(user, queue)) {
+		err = -EEXIST;
+		goto fail;
+	}
+
+	queue_size = NVS_QUEUE_DEFAULT_SIZE;
+
+	/* Ensure, event queue is constructed only once across all users. */
+	if (!nvgpu_nvs_buffer_is_valid(g, queue)) {
+		err = nvgpu_nvs_get_buf_linux(g, queue, queue_size, mask, read_only);
+		if (err != 0) {
+			goto fail;
+		}
+	}
+
+	priv = queue->priv;
+
+	fd = dma_buf_fd(priv->dmabuf, flag);
+	if (fd < 0) {
+		/* Might have valid user vmas for previous event queue users */
+		if (!nvgpu_nvs_buf_linux_is_mapped(g, queue)) {
+			nvgpu_nvs_ctrl_fifo_erase_queue(g, queue);
+		}
+		err = fd;
+		goto fail;
+	}
+
+	nvgpu_nvs_ctrl_fifo_user_subscribe_queue(user, queue);
+
+	args->dmabuf_fd = fd;
+	/* update the queue size correctly */
+	args->queue_size = queue_size;
+
+	nvgpu_nvs_ctrl_fifo_unlock_queues(g);
+	return 0;
+
+fail:
+	nvgpu_nvs_ctrl_fifo_unlock_queues(g);
+	nvgpu_err(g, "failed");
+
+	/* set dmabuf_fd to -1 for failure exclusively */
+	args->dmabuf_fd = -1;
+
+	return err;
+}
+
+static void nvgpu_nvs_ctrl_fifo_undo_create_queue(struct gk20a *g,
+		struct nvs_domain_ctrl_fifo_user *user,
+		struct nvgpu_nvs_ctrl_fifo_ioctl_create_queue_args *args)
+{
+	enum nvgpu_nvs_ctrl_queue_num num_queue;
+	enum nvgpu_nvs_ctrl_queue_direction queue_direction;
+	struct nvgpu_nvs_ctrl_queue *queue;
+	u8 mask = 0;
+
+	nvgpu_nvs_ctrl_fifo_lock_queues(g);
+
+	num_queue = nvgpu_nvs_translate_queue_num(args->queue_num);
+	queue_direction = nvgpu_nvs_translate_queue_direction(args->direction);
+
+	queue = nvgpu_nvs_ctrl_fifo_get_queue(g->sched_ctrl_fifo, num_queue, queue_direction, &mask);
+	if (queue == NULL) {
+		nvgpu_nvs_ctrl_fifo_unlock_queues(g);
+		return;
+	}
+
+	nvgpu_nvs_ctrl_fifo_user_unsubscribe_queue(user, queue);
+
+	/* For Control Queues, no mappings exist, For Event Queues, mappings might exist */
+	if (nvgpu_nvs_buffer_is_valid(g, queue) && !nvgpu_nvs_buf_linux_is_mapped(g, queue)) {
+		nvgpu_nvs_ctrl_fifo_erase_queue(g, queue);
+	}
+
+	if (args->dmabuf_fd != 0) {
+		put_unused_fd(args->dmabuf_fd);
+		args->dmabuf_fd = 0;
+	}
+
+	nvgpu_nvs_ctrl_fifo_unlock_queues(g);
+}
+
+static int nvgpu_nvs_ctrl_fifo_destroy_queue(struct gk20a *g,
+		struct nvs_domain_ctrl_fifo_user *user,
+		struct nvgpu_nvs_ctrl_fifo_ioctl_release_queue_args *args)
+{
+	enum nvgpu_nvs_ctrl_queue_num num_queue;
+	enum nvgpu_nvs_ctrl_queue_direction queue_direction;
+	struct nvgpu_nvs_ctrl_queue *queue;
+	bool is_exclusive_user;
+	int err = 0;
+	u8 mask = 0;
+
+	if (args->reserve0 != 0) {
+		return -EINVAL;
+	}
+
+	if (args->reserve1 != 0) {
+		return -EINVAL;
+	}
+
+	if (args->reserve2 != 0) {
+		return -EINVAL;
+	}
+
+	num_queue = nvgpu_nvs_translate_queue_num(args->queue_num);
+	queue_direction = nvgpu_nvs_translate_queue_direction(args->direction);
+
+	is_exclusive_user = nvgpu_nvs_ctrl_fifo_is_exclusive_user(g->sched_ctrl_fifo, user);
+	if (!is_exclusive_user) {
+		if ((num_queue == NVGPU_NVS_NUM_CONTROL) ||
+			(queue_direction == NVGPU_NVS_DIR_CLIENT_TO_SCHEDULER)) {
+			return -EPERM;
+		}
+	}
+
+	queue = nvgpu_nvs_ctrl_fifo_get_queue(g->sched_ctrl_fifo, num_queue, queue_direction, &mask);
+	if (queue == NULL) {
+		err = -EOPNOTSUPP;
+		goto fail;
+	}
+
+	nvgpu_nvs_ctrl_fifo_lock_queues(g);
+
+	if (!nvgpu_nvs_ctrl_fifo_user_is_subscribed_to_queue(user, queue)) {
+		err = -EPERM;
+		goto fail;
+	}
+
+	/* For Control Queues, no mappings should exist, For Event Queues, mappings might exist */
+	if (nvgpu_nvs_buffer_is_valid(g, queue)) {
+		if (!nvgpu_nvs_buf_linux_is_mapped(g, queue)) {
+			nvgpu_nvs_ctrl_fifo_erase_queue(g, queue);
+		} else if (is_exclusive_user) {
+			err = -EBUSY;
+			goto fail;
+		}
+	}
+
+	nvgpu_nvs_ctrl_fifo_user_unsubscribe_queue(user, queue);
+
+	nvgpu_nvs_ctrl_fifo_unlock_queues(g);
+
+	return 0;
+
+fail:
+	nvgpu_nvs_ctrl_fifo_unlock_queues(g);
+	nvgpu_err(g, "failed to destroy queue");
+
+	return err;
+}
+
+long nvgpu_nvs_ctrl_fifo_ops_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
+{
+	u8 buf[NVGPU_NVS_CTRL_FIFO_IOCTL_MAX_ARG_SIZE] = { 0 };
+	int err = 0;
+	struct nvgpu_nvs_domain_ctrl_fifo_user_linux *linux_user;
+	struct nvs_domain_ctrl_fifo_user *user;
+	struct gk20a *g;
+
+	linux_user = (struct nvgpu_nvs_domain_ctrl_fifo_user_linux *)filp->private_data;
+	if (linux_user == NULL) {
+		return -ENODEV;
+	}
+
+	user = &linux_user->user;
+	g = nvgpu_get_gk20a_from_cdev(linux_user->cdev);
+
+	nvs_dbg(g, "IOC_TYPE: %c", _IOC_TYPE(cmd));
+	nvs_dbg(g, "IOC_NR:   %u", _IOC_NR(cmd));
+	nvs_dbg(g, "IOC_SIZE: %u", _IOC_SIZE(cmd));
+
+	if ((_IOC_TYPE(cmd) != NVGPU_NVS_CTRL_FIFO_IOCTL_MAGIC) ||
+		(_IOC_NR(cmd) == 0) ||
+		(_IOC_NR(cmd) > NVGPU_NVS_CTRL_FIFO_IOCTL_LAST) ||
+		(_IOC_SIZE(cmd) > NVGPU_NVS_CTRL_FIFO_IOCTL_MAX_ARG_SIZE)) {
+		nvs_dbg(g, "-> BAD!!");
+		return -EINVAL;
+	}
+
+	if (_IOC_DIR(cmd) & _IOC_WRITE) {
+		if (copy_from_user(buf, (void __user *)arg, _IOC_SIZE(cmd)))
+			return -EFAULT;
+	}
+
+	err = gk20a_busy(g);
+	if (err != 0) {
+		return err;
+	}
+
+	switch (cmd) {
+	case NVGPU_NVS_CTRL_FIFO_CREATE_QUEUE:
+	{
+		struct nvgpu_nvs_ctrl_fifo_ioctl_create_queue_args *args =
+			(struct nvgpu_nvs_ctrl_fifo_ioctl_create_queue_args *)buf;
+
+		err = nvgpu_nvs_ctrl_fifo_create_queue(g, user, args);
+		if (err)
+			goto done;
+
+		if (copy_to_user((void __user *)arg, buf, _IOC_SIZE(cmd))) {
+			nvgpu_nvs_ctrl_fifo_undo_create_queue(g, user, args);
+			err = -EFAULT;
+			args->dmabuf_fd = -1;
+			goto done;
+		}
+
+		break;
+	}
+	case NVGPU_NVS_CTRL_FIFO_RELEASE_QUEUE:
+	{
+		struct nvgpu_nvs_ctrl_fifo_ioctl_release_queue_args *args =
+			(struct nvgpu_nvs_ctrl_fifo_ioctl_release_queue_args *)buf;
+
+		err = nvgpu_nvs_ctrl_fifo_destroy_queue(g, user, args);
+		if (err)
+			goto done;
+
+		break;
+	}
+	case NVGPU_NVS_CTRL_FIFO_ENABLE_EVENT:
+	{
+		err = -EOPNOTSUPP;
+		goto done;
+	}
+	default:
+		err = -ENOTTY;
+		goto done;
+	}
+
+done:
+	gk20a_idle(g);
+	return err;
+}

drivers/gpu/nvgpu/os/linux/ioctl_nvs.h

@@ -28,5 +28,6 @@ struct nvgpu_nvs_domain *nvgpu_nvs_domain_get_from_file(int fd);
 
 int     nvgpu_nvs_ctrl_fifo_ops_open(struct inode *inode, struct file *filp);
 int     nvgpu_nvs_ctrl_fifo_ops_release(struct inode *inode, struct file *filp);
+long    nvgpu_nvs_ctrl_fifo_ops_ioctl(struct file *filp, unsigned int cmd, unsigned long arg);
 
 #endif