diff --git a/arch/nvgpu-linux.yaml b/arch/nvgpu-linux.yaml index 2087ac487..c5114fc47 100644 --- a/arch/nvgpu-linux.yaml +++ b/arch/nvgpu-linux.yaml @@ -39,7 +39,8 @@ dma: dmabuf: sources: [ os/linux/dmabuf_vidmem.c, os/linux/dmabuf_vidmem.h, - os/linux/dmabuf_priv.c, os/linux/dmabuf_priv.h ] + os/linux/dmabuf_priv.c, os/linux/dmabuf_priv.h, + os/linux/dmabuf_nvs.c, os/linux/dmabuf_nvs.h ] driver_common: sources: [ os/linux/driver_common.c, os/linux/driver_common.h ] diff --git a/drivers/gpu/nvgpu/Makefile b/drivers/gpu/nvgpu/Makefile index a4e4d0cc0..a05c5076e 100644 --- a/drivers/gpu/nvgpu/Makefile +++ b/drivers/gpu/nvgpu/Makefile @@ -127,6 +127,7 @@ nvgpu-$(CONFIG_NVGPU_DGPU) += \ os/linux/pci.o \ os/linux/pci_power.o \ os/linux/dmabuf_vidmem.o \ + os/linux/dmabuf_nvs.o \ os/linux/os_ops_gv100.o \ os/linux/os_ops_tu104.o \ common/sec2/sec2.o \ diff --git a/drivers/gpu/nvgpu/common/nvs/nvs_sched_ctrl.c b/drivers/gpu/nvgpu/common/nvs/nvs_sched_ctrl.c index 742a50551..92a198111 100644 --- a/drivers/gpu/nvgpu/common/nvs/nvs_sched_ctrl.c +++ b/drivers/gpu/nvgpu/common/nvs/nvs_sched_ctrl.c @@ -26,6 +26,7 @@ #include #include #include +#include struct nvgpu_nvs_domain_ctrl_fifo_users { /* Flag to reserve exclusive user */ @@ -40,6 +41,34 @@ struct nvgpu_nvs_domain_ctrl_fifo_users { struct nvgpu_spinlock user_lock; }; +struct nvgpu_nvs_domain_ctrl_fifo_queues { + /* + * send indicates a buffer having data(PUT) written by a userspace client + * and queried by the scheduler(GET). + */ + struct nvgpu_nvs_ctrl_queue send; + /* + * receive indicates a buffer having data(PUT) written by scheduler + * and queried by the userspace client(GET). + */ + struct nvgpu_nvs_ctrl_queue receive; + + /* + * event indicates a buffer that is subscribed to by userspace clients to + * receive events. This buffer is Read-Only for the users and only scheduler can + * write to it. + */ + struct nvgpu_nvs_ctrl_queue event; + + /* + * Global mutex for coarse grained access control + * of all Queues for all UMD interfaces. e.g. IOCTL/devctls + * and mmap calls. Keeping this as coarse-grained for now till + * GSP's implementation is complete. + */ + struct nvgpu_mutex queue_lock; +}; + struct nvgpu_nvs_domain_ctrl_fifo { /* * Instance of global struct gk20a; @@ -47,6 +76,8 @@ struct nvgpu_nvs_domain_ctrl_fifo { struct gk20a *g; struct nvgpu_nvs_domain_ctrl_fifo_users users; + + struct nvgpu_nvs_domain_ctrl_fifo_queues queues; }; void nvgpu_nvs_ctrl_fifo_reset_exclusive_user( @@ -173,6 +204,7 @@ struct nvgpu_nvs_domain_ctrl_fifo *nvgpu_nvs_ctrl_fifo_create(struct gk20a *g) } nvgpu_spinlock_init(&sched->users.user_lock); + nvgpu_mutex_init(&sched->queues.queue_lock); nvgpu_init_list_node(&sched->users.exclusive_user); nvgpu_init_list_node(&sched->users.list_non_exclusive_user); @@ -200,6 +232,144 @@ void nvgpu_nvs_ctrl_fifo_destroy(struct gk20a *g) nvgpu_assert(!nvgpu_nvs_ctrl_fifo_is_busy(sched_ctrl)); + nvgpu_nvs_ctrl_fifo_erase_all_queues(g); + nvgpu_kfree(g, sched_ctrl); g->sched_ctrl_fifo = NULL; } + +struct nvgpu_nvs_ctrl_queue *nvgpu_nvs_ctrl_fifo_get_queue( + struct nvgpu_nvs_domain_ctrl_fifo *sched_ctrl, + enum nvgpu_nvs_ctrl_queue_num queue_num, + enum nvgpu_nvs_ctrl_queue_direction queue_direction, + u8 *mask) +{ + struct nvgpu_nvs_ctrl_queue *queue = NULL; + + if (sched_ctrl == NULL) { + return NULL; + } + + if (mask == NULL) { + return NULL; + } + + if (queue_num == NVGPU_NVS_NUM_CONTROL) { + if (queue_direction == NVGPU_NVS_DIR_CLIENT_TO_SCHEDULER) { + queue = &sched_ctrl->queues.send; + *mask = NVGPU_NVS_CTRL_FIFO_QUEUE_EXCLUSIVE_CLIENT_WRITE; + } else if (queue_direction == NVGPU_NVS_DIR_SCHEDULER_TO_CLIENT) { + queue = &sched_ctrl->queues.receive; + *mask = NVGPU_NVS_CTRL_FIFO_QUEUE_EXCLUSIVE_CLIENT_READ; + } + } else if (queue_num == NVGPU_NVS_NUM_EVENT) { + if (queue_direction == NVGPU_NVS_DIR_SCHEDULER_TO_CLIENT) { + queue = &sched_ctrl->queues.event; + *mask = NVGPU_NVS_CTRL_FIFO_QUEUE_CLIENT_EVENTS_READ; + } + } + + return queue; +} + +bool nvgpu_nvs_buffer_is_valid(struct gk20a *g, struct nvgpu_nvs_ctrl_queue *buf) +{ + return buf->valid; +} + +int nvgpu_nvs_buffer_alloc(struct nvgpu_nvs_domain_ctrl_fifo *sched_ctrl, + size_t bytes, u8 mask, struct nvgpu_nvs_ctrl_queue *buf) +{ + int err; + struct gk20a *g = sched_ctrl->g; + struct vm_gk20a *system_vm = g->mm.pmu.vm; + + (void)memset(buf, 0, sizeof(*buf)); + buf->g = g; + + err = nvgpu_dma_alloc_map_sys(system_vm, bytes, &buf->mem); + if (err != 0) { + nvgpu_err(g, "failed to allocate memory for dma"); + goto fail; + } + + buf->valid = true; + buf->mask = mask; + + return 0; + +fail: + (void)memset(buf, 0, sizeof(*buf)); + + return err; +} + +void nvgpu_nvs_buffer_free(struct nvgpu_nvs_domain_ctrl_fifo *sched_ctrl, + struct nvgpu_nvs_ctrl_queue *buf) +{ + struct gk20a *g = sched_ctrl->g; + struct vm_gk20a *system_vm = g->mm.pmu.vm; + + if (nvgpu_mem_is_valid(&buf->mem)) { + nvgpu_dma_unmap_free(system_vm, &buf->mem); + } + + /* Sets buf->valid as false */ + (void)memset(buf, 0, sizeof(*buf)); +} + +void nvgpu_nvs_ctrl_fifo_lock_queues(struct gk20a *g) +{ + struct nvgpu_nvs_domain_ctrl_fifo *sched_ctrl = g->sched_ctrl_fifo; + nvgpu_mutex_acquire(&sched_ctrl->queues.queue_lock); +} + +void nvgpu_nvs_ctrl_fifo_unlock_queues(struct gk20a *g) +{ + struct nvgpu_nvs_domain_ctrl_fifo *sched_ctrl = g->sched_ctrl_fifo; + nvgpu_mutex_release(&sched_ctrl->queues.queue_lock); +} + +void nvgpu_nvs_ctrl_fifo_user_subscribe_queue(struct nvs_domain_ctrl_fifo_user *user, + struct nvgpu_nvs_ctrl_queue *queue) +{ + user->active_used_queues |= queue->mask; +} +void nvgpu_nvs_ctrl_fifo_user_unsubscribe_queue(struct nvs_domain_ctrl_fifo_user *user, + struct nvgpu_nvs_ctrl_queue *queue) +{ + user->active_used_queues &= ~queue->mask; +} +bool nvgpu_nvs_ctrl_fifo_user_is_subscribed_to_queue(struct nvs_domain_ctrl_fifo_user *user, + struct nvgpu_nvs_ctrl_queue *queue) +{ + return (user->active_used_queues & queue->mask); +} + +void nvgpu_nvs_ctrl_fifo_erase_all_queues(struct gk20a *g) +{ + struct nvgpu_nvs_domain_ctrl_fifo *sched_ctrl = g->sched_ctrl_fifo; + + nvgpu_nvs_ctrl_fifo_lock_queues(g); + + if (nvgpu_nvs_buffer_is_valid(g, &sched_ctrl->queues.send)) { + nvgpu_nvs_ctrl_fifo_erase_queue(g, &sched_ctrl->queues.send); + } + + if (nvgpu_nvs_buffer_is_valid(g, &sched_ctrl->queues.receive)) { + nvgpu_nvs_ctrl_fifo_erase_queue(g, &sched_ctrl->queues.receive); + } + + if (nvgpu_nvs_buffer_is_valid(g, &sched_ctrl->queues.event)) { + nvgpu_nvs_ctrl_fifo_erase_queue(g, &sched_ctrl->queues.event); + } + + nvgpu_nvs_ctrl_fifo_unlock_queues(g); +} + +void nvgpu_nvs_ctrl_fifo_erase_queue(struct gk20a *g, struct nvgpu_nvs_ctrl_queue *queue) +{ + if (queue->free != NULL) { + queue->free(g, queue); + } +} diff --git a/drivers/gpu/nvgpu/include/nvgpu/nvs.h b/drivers/gpu/nvgpu/include/nvgpu/nvs.h index 879084bda..544759464 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/nvs.h +++ b/drivers/gpu/nvgpu/include/nvgpu/nvs.h @@ -31,11 +31,16 @@ #include #include #include +#include /* * Max size we'll parse from an NVS log entry. */ #define NVS_LOG_BUF_SIZE 128 +/* + * Keep it to page size for now. Can be updated later. + */ +#define NVS_QUEUE_DEFAULT_SIZE (64 * 1024) struct gk20a; struct nvgpu_nvs_domain_ioctl; @@ -126,6 +131,85 @@ struct nvgpu_nvs_scheduler { struct nvgpu_nvs_domain *shadow_domain; }; +enum nvgpu_nvs_ctrl_queue_num { + NVGPU_NVS_NUM_CONTROL = 0, + NVGPU_NVS_NUM_EVENT, + NVGPU_NVS_INVALID, +}; + +enum nvgpu_nvs_ctrl_queue_direction { + NVGPU_NVS_DIR_CLIENT_TO_SCHEDULER = 0, + NVGPU_NVS_DIR_SCHEDULER_TO_CLIENT, + NVGPU_NVS_DIR_INVALID, +}; + +/* + * The below definitions mirror the nvgpu-nvs(UAPI) + * headers. + */ + +/* + * Invalid domain scheduler. + * The value of 'domain_scheduler_implementation' + * when 'has_domain_scheduler_control_fifo' is 0. + */ +#define NVGPU_NVS_DOMAIN_SCHED_INVALID 0U +/* + * CPU based scheduler implementation. Intended use is mainly + * for debug and testing purposes. Doesn't meet latency requirements. + * Implementation will be supported in the initial versions and eventually + * discarded. + */ +#define NVGPU_NVS_DOMAIN_SCHED_KMD 1U +/* + * GSP based scheduler implementation that meets latency requirements. + * This implementation will eventually replace NVGPU_NVS_DOMAIN_SCHED_KMD. + */ +#define NVGPU_NVS_DOMAIN_SCHED_GSP 2U + +/* Queue meant for exclusive client write access. This shared queue will be + * used for communicating the scheduling metadata between the client(producer) + * and scheduler(consumer). + */ +#define NVGPU_NVS_CTRL_FIFO_QUEUE_EXCLUSIVE_CLIENT_WRITE 1U + +/* Queue meant for exclusive client read access. This shared queue will be + * used for communicating the scheduling metadata between the scheduler(producer) + * and client(consumer). + */ +#define NVGPU_NVS_CTRL_FIFO_QUEUE_EXCLUSIVE_CLIENT_READ 2U + +/* Queue meant for generic read access. Clients can subscribe to this read-only + * queue for processing events such as recovery, preemption etc. + */ +#define NVGPU_NVS_CTRL_FIFO_QUEUE_CLIENT_EVENTS_READ 4U + +/* + * Direction of the requested queue is from CLIENT(producer) + * to SCHEDULER(consumer). + */ +#define NVGPU_NVS_CTRL_FIFO_QUEUE_DIRECTION_CLIENT_TO_SCHEDULER 0 + +/* + * Direction of the requested queue is from SCHEDULER(producer) + * to CLIENT(consumer). + */ +#define NVGPU_NVS_CTRL_FIFO_QUEUE_DIRECTION_SCHEDULER_TO_CLIENT 1 + +/* Structure to hold control_queues. This can be then passed to GSP or Rm based subscheduler. */ +struct nvgpu_nvs_ctrl_queue { + struct nvgpu_mem mem; + struct gk20a *g; + /* + * Filled in by each OS - this holds the necessary data to export this + * buffer to userspace. + */ + void *priv; + bool valid; + u8 mask; + void (*free)(struct gk20a *g, struct nvgpu_nvs_ctrl_queue *queue); +}; + #ifdef CONFIG_NVS_PRESENT int nvgpu_nvs_init(struct gk20a *g); int nvgpu_nvs_open(struct gk20a *g); @@ -150,9 +234,14 @@ const char *nvgpu_nvs_domain_get_name(struct nvgpu_nvs_domain *dom); #define nvs_dbg(g, fmt, arg...) \ nvgpu_log(g, gpu_dbg_nvs, fmt, ##arg) +void nvgpu_nvs_ctrl_fifo_lock_queues(struct gk20a *g); +void nvgpu_nvs_ctrl_fifo_unlock_queues(struct gk20a *g); + struct nvgpu_nvs_domain_ctrl_fifo *nvgpu_nvs_ctrl_fifo_create(struct gk20a *g); bool nvgpu_nvs_ctrl_fifo_user_exists(struct nvgpu_nvs_domain_ctrl_fifo *sched_ctrl, int pid, bool rw); +bool nvgpu_nvs_ctrl_fifo_is_busy(struct nvgpu_nvs_domain_ctrl_fifo *sched_ctrl); +void nvgpu_nvs_ctrl_fifo_destroy(struct gk20a *g); bool nvgpu_nvs_ctrl_fifo_user_is_active(struct nvs_domain_ctrl_fifo_user *user); void nvgpu_nvs_ctrl_fifo_add_user(struct nvgpu_nvs_domain_ctrl_fifo *sched_ctrl, struct nvs_domain_ctrl_fifo_user *user); @@ -163,9 +252,26 @@ void nvgpu_nvs_ctrl_fifo_reset_exclusive_user( int nvgpu_nvs_ctrl_fifo_reserve_exclusive_user( struct nvgpu_nvs_domain_ctrl_fifo *sched_ctrl, struct nvs_domain_ctrl_fifo_user *user); void nvgpu_nvs_ctrl_fifo_remove_user(struct nvgpu_nvs_domain_ctrl_fifo *sched_ctrl, - struct nvs_domain_ctrl_fifo_user *user); -bool nvgpu_nvs_ctrl_fifo_is_busy(struct nvgpu_nvs_domain_ctrl_fifo *sched_ctrl); -void nvgpu_nvs_ctrl_fifo_destroy(struct gk20a *g); + struct nvs_domain_ctrl_fifo_user *user); +struct nvgpu_nvs_ctrl_queue *nvgpu_nvs_ctrl_fifo_get_queue( + struct nvgpu_nvs_domain_ctrl_fifo *sched_ctrl, + enum nvgpu_nvs_ctrl_queue_num queue_num, + enum nvgpu_nvs_ctrl_queue_direction queue_direction, + u8 *mask); +/* Below methods require nvgpu_nvs_ctrl_fifo_lock_queues() to be held. */ +bool nvgpu_nvs_buffer_is_valid(struct gk20a *g, struct nvgpu_nvs_ctrl_queue *buf); +int nvgpu_nvs_buffer_alloc(struct nvgpu_nvs_domain_ctrl_fifo *sched_ctrl, + size_t bytes, u8 mask, struct nvgpu_nvs_ctrl_queue *buf); +void nvgpu_nvs_buffer_free(struct nvgpu_nvs_domain_ctrl_fifo *sched_ctrl, + struct nvgpu_nvs_ctrl_queue *buf); +void nvgpu_nvs_ctrl_fifo_user_subscribe_queue(struct nvs_domain_ctrl_fifo_user *user, + struct nvgpu_nvs_ctrl_queue *queue); +void nvgpu_nvs_ctrl_fifo_user_unsubscribe_queue(struct nvs_domain_ctrl_fifo_user *user, + struct nvgpu_nvs_ctrl_queue *queue); +bool nvgpu_nvs_ctrl_fifo_user_is_subscribed_to_queue(struct nvs_domain_ctrl_fifo_user *user, + struct nvgpu_nvs_ctrl_queue *queue); +void nvgpu_nvs_ctrl_fifo_erase_queue(struct gk20a *g, struct nvgpu_nvs_ctrl_queue *queue); +void nvgpu_nvs_ctrl_fifo_erase_all_queues(struct gk20a *g); #else static inline int nvgpu_nvs_init(struct gk20a *g) diff --git a/drivers/gpu/nvgpu/os/linux/dmabuf_nvs.c b/drivers/gpu/nvgpu/os/linux/dmabuf_nvs.c new file mode 100644 index 000000000..420893872 --- /dev/null +++ b/drivers/gpu/nvgpu/os/linux/dmabuf_nvs.c @@ -0,0 +1,290 @@ +/* + * Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include +#include +#include + +#ifdef CONFIG_NVGPU_USE_TEGRA_ALLOC_FD +#include +#endif + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "dmabuf_nvs.h" + +/* This constant string is used to determine if the dmabuf belongs + * to nvgpu. + */ +static const char exporter_name[] = "nvgpu_ctrl_fifo"; + +static struct nvgpu_nvs_domain_ctrl_fifo_user_vma * + nvgpu_nvs_domain_ctrl_fifo_user_vma_from_node(struct nvgpu_list_node *node) +{ + nvgpu_assert(node != NULL); + return container_of(node, struct nvgpu_nvs_domain_ctrl_fifo_user_vma, node); +} + +static int zap_vma_entries(struct gk20a *g, struct vm_area_struct *vma) +{ + int err = 0; + + #if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 8, 0) + mmap_write_lock(vma->vm_mm); + #else + down_write(&vma->vm_mm->mmap_sem); + #endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 18, 0) + zap_vma_ptes(vma, vma->vm_start, vma->vm_end - vma->vm_start); +#else + err = zap_vma_ptes(vma, vma->vm_start, + vma->vm_end - vma->vm_start); +#endif + + #if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 8, 0) + mmap_write_unlock(vma->vm_mm); + #else + up_write(&vma->vm_mm->mmap_sem); + #endif + + return err; +} + +static int nvs_release_user_mappings_locked(struct gk20a *g, struct nvgpu_nvs_linux_buf_priv *linux_buf) +{ + struct nvgpu_nvs_domain_ctrl_fifo_user_vma *current_entry, *next_entry; + int err = 0; + + nvgpu_list_for_each_entry_safe(current_entry, next_entry, + &linux_buf->list_mapped_user_vmas, + nvgpu_nvs_domain_ctrl_fifo_user_vma, node) { + struct vm_area_struct *vma = current_entry->vma; + + zap_vma_entries(g, vma); + linux_buf->ref--; + } + + return err; +} + +static void nvs_vma_close(struct vm_area_struct *vma) +{ + struct nvgpu_nvs_domain_ctrl_fifo_user_vma *vma_metadata = vma->vm_private_data; + struct nvgpu_nvs_ctrl_queue *buf = vma_metadata->buf; + struct nvgpu_nvs_linux_buf_priv *linux_buf = (struct nvgpu_nvs_linux_buf_priv *)buf->priv; + struct gk20a *g = buf->g; + + nvgpu_nvs_ctrl_fifo_lock_queues(g); + + linux_buf->ref--; + nvgpu_list_del(&vma_metadata->node); + + /* This VMA is freed now and points to invalid ptes */ + vma_metadata->vma = NULL; + nvgpu_kfree(g, vma_metadata); + vma->vm_private_data = NULL; + + nvgpu_nvs_ctrl_fifo_unlock_queues(g); + + nvgpu_put(g); +} + +const struct vm_operations_struct nvs_vma_ops = { + /* no .open - we use VM_DONTCOPY and don't support fork */ + .close = nvs_vma_close, +}; + +static int nvgpu_nvs_buf_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma) +{ + struct nvgpu_nvs_ctrl_queue *buf = dmabuf->priv; + struct nvgpu_nvs_linux_buf_priv *linux_buf = (struct nvgpu_nvs_linux_buf_priv *)buf->priv; + struct gk20a *g = buf->g; + struct nvgpu_mem *mem = &buf->mem; + struct nvgpu_nvs_domain_ctrl_fifo_user_vma *vma_metadata = NULL; + int err; + const struct vm_operations_struct *vm_ops_old = NULL; + + if (vma->vm_end - vma->vm_start > buf->mem.size) { + return -EINVAL; + } + + if (vma->vm_pgoff != 0UL) { + return -EINVAL; + } + + /* + * This ref is released when the mapping is removed. + */ + if (!nvgpu_get(g)) + return -ENODEV; + + nvgpu_nvs_ctrl_fifo_lock_queues(g); + + vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_NORESERVE | + VM_DONTDUMP; + + if (linux_buf->read_only) { + vma->vm_flags |= VM_SHARED; + } + + vm_ops_old = vma->vm_ops; + + vma->vm_ops = &nvs_vma_ops; + + err = nvgpu_dma_mmap_sys(g, vma, mem); + if (err != 0) { + goto fail; + } + + vma_metadata = nvgpu_kzalloc(g, sizeof(*vma_metadata)); + if (vma_metadata == NULL) { + err = -ENOMEM; + zap_vma_entries(g, vma); + goto fail; + } + + vma_metadata->vma = vma; + vma_metadata->buf = buf; + nvgpu_init_list_node(&vma_metadata->node); + + linux_buf->ref++; + nvgpu_list_add_tail(&vma_metadata->node, &linux_buf->list_mapped_user_vmas); + + vma->vm_private_data = vma_metadata; + + nvgpu_nvs_ctrl_fifo_unlock_queues(g); + + return 0; +fail: + vma->vm_ops = vm_ops_old; + + if (vma_metadata != NULL) { + nvgpu_kfree(g, vma_metadata); + } + + nvgpu_nvs_ctrl_fifo_unlock_queues(g); + + return err; +} + +static struct dma_buf_ops gk20a_nvs_ops = { + .mmap = nvgpu_nvs_buf_mmap, +}; + +static struct dma_buf *nvgpu_nvs_buf_export_dmabuf(struct nvgpu_nvs_ctrl_queue *buf, + bool read_only) +{ + DEFINE_DMA_BUF_EXPORT_INFO(exp_info); + struct nvgpu_mem *mem = &buf->mem; + + int flags = (read_only ? O_RDONLY : O_RDWR) | O_CLOEXEC; + + exp_info.priv = buf; + exp_info.ops = &gk20a_nvs_ops; + exp_info.size = mem->size; + exp_info.flags = flags; + exp_info.exp_name = exporter_name; + + return dma_buf_export(&exp_info); +} + +static void nvgpu_nvs_destroy_buf_linux_locked(struct gk20a *g, struct nvgpu_nvs_ctrl_queue *buf) +{ + struct nvgpu_nvs_domain_ctrl_fifo *sched_ctrl = g->sched_ctrl_fifo; + struct nvgpu_nvs_linux_buf_priv *priv; + + priv = (struct nvgpu_nvs_linux_buf_priv *)buf->priv; + + nvs_release_user_mappings_locked(g, priv); + + dma_buf_put(priv->dmabuf); + + nvgpu_nvs_buffer_free(sched_ctrl, buf); + nvgpu_kfree(g, priv); + + nvgpu_put(g); +} + +bool nvgpu_nvs_buf_linux_is_mapped(struct gk20a *g, struct nvgpu_nvs_ctrl_queue *buf) +{ + bool is_mapped; + struct nvgpu_nvs_linux_buf_priv *priv = NULL; + + priv = (struct nvgpu_nvs_linux_buf_priv *)buf->priv; + is_mapped = (priv->ref != 0U); + + return is_mapped; +} + +int nvgpu_nvs_get_buf_linux(struct gk20a *g, struct nvgpu_nvs_ctrl_queue *buf, + size_t bytes, u8 mask, bool read_only) +{ + struct nvgpu_nvs_linux_buf_priv *priv; + int err; + struct nvgpu_nvs_domain_ctrl_fifo *sched_ctrl = g->sched_ctrl_fifo; + + /* + * This ref is released when the dma_buf is closed. + */ + if (!nvgpu_get(g)) + return -ENODEV; + + nvs_dbg(g, "Allocating buf: %zu bytes", bytes); + + priv = nvgpu_kzalloc(g, sizeof(*priv)); + if (!priv) { + err = -ENOMEM; + goto fail; + } + + nvgpu_init_list_node(&priv->list_mapped_user_vmas); + priv->read_only = read_only; + + err = nvgpu_nvs_buffer_alloc(sched_ctrl, bytes, mask, buf); + if (err != 0) { + nvgpu_err(g, "Unable to alloc memory"); + goto fail; + } + + priv->dmabuf = nvgpu_nvs_buf_export_dmabuf(buf, read_only); + if (IS_ERR(priv->dmabuf)) { + nvgpu_err(g, "Unable to export dma buf"); + err = PTR_ERR(priv->dmabuf); + goto fail; + } + + buf->priv = priv; + buf->free = nvgpu_nvs_destroy_buf_linux_locked; + + return 0; +fail: + nvgpu_kfree(g, priv); + nvgpu_nvs_buffer_free(sched_ctrl, buf); + + nvgpu_put(g); + + return err; +} diff --git a/drivers/gpu/nvgpu/os/linux/dmabuf_nvs.h b/drivers/gpu/nvgpu/os/linux/dmabuf_nvs.h new file mode 100644 index 000000000..8c76c9e01 --- /dev/null +++ b/drivers/gpu/nvgpu/os/linux/dmabuf_nvs.h @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef NVGPU_DMABUF_NVS_H +#define NVGPU_DMABUF_NVS_H + +#include +#include + +struct dma_buf; +struct gk20a; + +struct nvgpu_nvs_linux_buf_priv { + struct dma_buf *dmabuf; + bool read_only; + u32 ref; + struct nvgpu_list_node list_mapped_user_vmas; +}; + +struct nvgpu_nvs_domain_ctrl_fifo_user_vma { + struct nvgpu_nvs_ctrl_queue *buf; + struct vm_area_struct *vma; + struct nvgpu_list_node node; +}; + +/** + * @brief Construct a buffer for use as a shared message passing + * queue between user and backend scheduler. Function is + * not safe from concurrent access by multiple external + * users. Must be invoked between the calls of + * nvgpu_nvs_ctrl_fifo_lock_queues() and + * nvgpu_nvs_ctrl_fifo_unlock_queues(). + * + * @param g instance of struct gk20a. + * @param buf instance of struct nvgpu_nvs_ctrl_queue to contain + * the constructed buffer metadata. + * @param bytes size of buffer requested. + * @param mask Mask of queue requested. + * @param read_only Indicates whether a read-only buffer is requested. + * @return int 0 on success, else fail. + */ +int nvgpu_nvs_get_buf_linux(struct gk20a *g, struct nvgpu_nvs_ctrl_queue *buf, + size_t bytes, u8 mask, bool read_only); + +/** + * @brief Check whether user mappings exist for this buffer. + * + * @param g instance of struct gk20a. + * @param buf instance of struct nvgpu_nvs_ctrl_queue. + * @return true User mappings exist. + * @return false User mappings doesn't exist. + */ +bool nvgpu_nvs_buf_linux_is_mapped(struct gk20a *g, struct nvgpu_nvs_ctrl_queue *buf); + +#endif /* NVGPU_DMABUF_NVS_H */ diff --git a/drivers/gpu/nvgpu/os/linux/ioctl_nvs.c b/drivers/gpu/nvgpu/os/linux/ioctl_nvs.c index 6a9c863a9..ea53a178d 100644 --- a/drivers/gpu/nvgpu/os/linux/ioctl_nvs.c +++ b/drivers/gpu/nvgpu/os/linux/ioctl_nvs.c @@ -15,6 +15,7 @@ #include #include #include +#include #include @@ -28,6 +29,7 @@ #include #include "ioctl.h" +#include "dmabuf_nvs.h" /* * OS-specific layer to hold device node mapping for a domain. @@ -665,6 +667,9 @@ int nvgpu_nvs_ctrl_fifo_ops_release(struct inode *inode, struct file *filp) } nvgpu_nvs_ctrl_fifo_remove_user(g->sched_ctrl_fifo, &linux_user->user); + if (!nvgpu_nvs_ctrl_fifo_is_busy(g->sched_ctrl_fifo)) { + nvgpu_nvs_ctrl_fifo_erase_all_queues(g); + } filp->private_data = NULL; @@ -675,3 +680,352 @@ int nvgpu_nvs_ctrl_fifo_ops_release(struct inode *inode, struct file *filp) } extern const struct file_operations nvgpu_nvs_ctrl_fifo_ops; + +static int nvgpu_nvs_ctrl_fifo_create_queue_verify_flags(struct gk20a *g, + struct nvs_domain_ctrl_fifo_user *user, + struct nvgpu_nvs_ctrl_fifo_ioctl_create_queue_args *args) +{ + int err = 0; + + if (args->reserve0 != 0) { + return -EINVAL; + } + + if (args->dmabuf_fd != 0) { + return -EINVAL; + } + + if (args->queue_size != 0) { + return -EINVAL; + } + + if (args->access_type == NVS_CTRL_FIFO_QUEUE_ACCESS_TYPE_EXCLUSIVE) { + if (args->queue_num == 0) + return -EINVAL; + if (args->direction == 0) + return -EINVAL; + if (!nvgpu_nvs_ctrl_fifo_is_exclusive_user(g->sched_ctrl_fifo, user)) { + err = nvgpu_nvs_ctrl_fifo_reserve_exclusive_user(g->sched_ctrl_fifo, user); + if (err != 0) { + return err; + } + } + } else { + if (args->queue_num != NVS_CTRL_FIFO_QUEUE_NUM_EVENT) + return -EINVAL; + if (args->direction != NVS_CTRL_FIFO_QUEUE_DIRECTION_SCHEDULER_TO_CLIENT) + return -EINVAL; + } + + return 0; +} + +static enum nvgpu_nvs_ctrl_queue_num nvgpu_nvs_translate_queue_num(u32 queue_num_arg) +{ + enum nvgpu_nvs_ctrl_queue_num num_queue = NVGPU_NVS_INVALID; + if (queue_num_arg == NVGPU_NVS_CTRL_FIFO_QUEUE_NUM_CONTROL) + num_queue = NVGPU_NVS_NUM_CONTROL; + else if (queue_num_arg == NVS_CTRL_FIFO_QUEUE_NUM_EVENT) + num_queue = NVGPU_NVS_NUM_EVENT; + + return num_queue; +} + +static enum nvgpu_nvs_ctrl_queue_direction + nvgpu_nvs_translate_queue_direction(u32 queue_direction) +{ + enum nvgpu_nvs_ctrl_queue_direction direction = NVGPU_NVS_DIR_INVALID; + if (queue_direction == NVS_CTRL_FIFO_QUEUE_DIRECTION_CLIENT_TO_SCHEDULER) + direction = NVGPU_NVS_DIR_CLIENT_TO_SCHEDULER; + else if (queue_direction == NVS_CTRL_FIFO_QUEUE_DIRECTION_SCHEDULER_TO_CLIENT) + direction = NVGPU_NVS_DIR_SCHEDULER_TO_CLIENT; + + return direction; +} + +static int nvgpu_nvs_ctrl_fifo_create_queue(struct gk20a *g, + struct nvs_domain_ctrl_fifo_user *user, + struct nvgpu_nvs_ctrl_fifo_ioctl_create_queue_args *args) +{ + enum nvgpu_nvs_ctrl_queue_num num_queue; + enum nvgpu_nvs_ctrl_queue_direction queue_direction; + struct nvgpu_nvs_ctrl_queue *queue; + struct nvgpu_nvs_linux_buf_priv *priv = NULL; + int err = 0; + int fd; + int flag = O_CLOEXEC; + bool read_only; + size_t queue_size; + u8 mask = 0; + + err = nvgpu_nvs_ctrl_fifo_create_queue_verify_flags(g, user, args); + if (err != 0) { + args->dmabuf_fd = -1; + return err; + } + + nvgpu_nvs_ctrl_fifo_lock_queues(g); + + num_queue = nvgpu_nvs_translate_queue_num(args->queue_num); + queue_direction = nvgpu_nvs_translate_queue_direction(args->direction); + + queue = nvgpu_nvs_ctrl_fifo_get_queue(g->sched_ctrl_fifo, num_queue, queue_direction, &mask); + if (queue == NULL) { + err = -EOPNOTSUPP; + goto fail; + } + + read_only = (args->access_type == NVS_CTRL_FIFO_QUEUE_ACCESS_TYPE_EXCLUSIVE) ? false : true; + if (read_only) { + flag |= O_RDONLY; + } else { + flag |= O_RDWR; + } + + /* Support for Read-Only Observers will be added later */ + if (read_only) { + err = -EOPNOTSUPP; + goto fail; + } + + if (args->access_type == NVS_CTRL_FIFO_QUEUE_ACCESS_TYPE_EXCLUSIVE) { + if (nvgpu_nvs_buffer_is_valid(g, queue)) { + err = -EBUSY; + goto fail; + } + } + + /* For event queue, prevent multiple subscription by the same user */ + if (nvgpu_nvs_ctrl_fifo_user_is_subscribed_to_queue(user, queue)) { + err = -EEXIST; + goto fail; + } + + queue_size = NVS_QUEUE_DEFAULT_SIZE; + + /* Ensure, event queue is constructed only once across all users. */ + if (!nvgpu_nvs_buffer_is_valid(g, queue)) { + err = nvgpu_nvs_get_buf_linux(g, queue, queue_size, mask, read_only); + if (err != 0) { + goto fail; + } + } + + priv = queue->priv; + + fd = dma_buf_fd(priv->dmabuf, flag); + if (fd < 0) { + /* Might have valid user vmas for previous event queue users */ + if (!nvgpu_nvs_buf_linux_is_mapped(g, queue)) { + nvgpu_nvs_ctrl_fifo_erase_queue(g, queue); + } + err = fd; + goto fail; + } + + nvgpu_nvs_ctrl_fifo_user_subscribe_queue(user, queue); + + args->dmabuf_fd = fd; + /* update the queue size correctly */ + args->queue_size = queue_size; + + nvgpu_nvs_ctrl_fifo_unlock_queues(g); + return 0; + +fail: + nvgpu_nvs_ctrl_fifo_unlock_queues(g); + nvgpu_err(g, "failed"); + + /* set dmabuf_fd to -1 for failure exclusively */ + args->dmabuf_fd = -1; + + return err; +} + +static void nvgpu_nvs_ctrl_fifo_undo_create_queue(struct gk20a *g, + struct nvs_domain_ctrl_fifo_user *user, + struct nvgpu_nvs_ctrl_fifo_ioctl_create_queue_args *args) +{ + enum nvgpu_nvs_ctrl_queue_num num_queue; + enum nvgpu_nvs_ctrl_queue_direction queue_direction; + struct nvgpu_nvs_ctrl_queue *queue; + u8 mask = 0; + + nvgpu_nvs_ctrl_fifo_lock_queues(g); + + num_queue = nvgpu_nvs_translate_queue_num(args->queue_num); + queue_direction = nvgpu_nvs_translate_queue_direction(args->direction); + + queue = nvgpu_nvs_ctrl_fifo_get_queue(g->sched_ctrl_fifo, num_queue, queue_direction, &mask); + if (queue == NULL) { + nvgpu_nvs_ctrl_fifo_unlock_queues(g); + return; + } + + nvgpu_nvs_ctrl_fifo_user_unsubscribe_queue(user, queue); + + /* For Control Queues, no mappings exist, For Event Queues, mappings might exist */ + if (nvgpu_nvs_buffer_is_valid(g, queue) && !nvgpu_nvs_buf_linux_is_mapped(g, queue)) { + nvgpu_nvs_ctrl_fifo_erase_queue(g, queue); + } + + if (args->dmabuf_fd != 0) { + put_unused_fd(args->dmabuf_fd); + args->dmabuf_fd = 0; + } + + nvgpu_nvs_ctrl_fifo_unlock_queues(g); +} + +static int nvgpu_nvs_ctrl_fifo_destroy_queue(struct gk20a *g, + struct nvs_domain_ctrl_fifo_user *user, + struct nvgpu_nvs_ctrl_fifo_ioctl_release_queue_args *args) +{ + enum nvgpu_nvs_ctrl_queue_num num_queue; + enum nvgpu_nvs_ctrl_queue_direction queue_direction; + struct nvgpu_nvs_ctrl_queue *queue; + bool is_exclusive_user; + int err = 0; + u8 mask = 0; + + if (args->reserve0 != 0) { + return -EINVAL; + } + + if (args->reserve1 != 0) { + return -EINVAL; + } + + if (args->reserve2 != 0) { + return -EINVAL; + } + + num_queue = nvgpu_nvs_translate_queue_num(args->queue_num); + queue_direction = nvgpu_nvs_translate_queue_direction(args->direction); + + is_exclusive_user = nvgpu_nvs_ctrl_fifo_is_exclusive_user(g->sched_ctrl_fifo, user); + if (!is_exclusive_user) { + if ((num_queue == NVGPU_NVS_NUM_CONTROL) || + (queue_direction == NVGPU_NVS_DIR_CLIENT_TO_SCHEDULER)) { + return -EPERM; + } + } + + queue = nvgpu_nvs_ctrl_fifo_get_queue(g->sched_ctrl_fifo, num_queue, queue_direction, &mask); + if (queue == NULL) { + err = -EOPNOTSUPP; + goto fail; + } + + nvgpu_nvs_ctrl_fifo_lock_queues(g); + + if (!nvgpu_nvs_ctrl_fifo_user_is_subscribed_to_queue(user, queue)) { + err = -EPERM; + goto fail; + } + + /* For Control Queues, no mappings should exist, For Event Queues, mappings might exist */ + if (nvgpu_nvs_buffer_is_valid(g, queue)) { + if (!nvgpu_nvs_buf_linux_is_mapped(g, queue)) { + nvgpu_nvs_ctrl_fifo_erase_queue(g, queue); + } else if (is_exclusive_user) { + err = -EBUSY; + goto fail; + } + } + + nvgpu_nvs_ctrl_fifo_user_unsubscribe_queue(user, queue); + + nvgpu_nvs_ctrl_fifo_unlock_queues(g); + + return 0; + +fail: + nvgpu_nvs_ctrl_fifo_unlock_queues(g); + nvgpu_err(g, "failed to destroy queue"); + + return err; +} + +long nvgpu_nvs_ctrl_fifo_ops_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + u8 buf[NVGPU_NVS_CTRL_FIFO_IOCTL_MAX_ARG_SIZE] = { 0 }; + int err = 0; + struct nvgpu_nvs_domain_ctrl_fifo_user_linux *linux_user; + struct nvs_domain_ctrl_fifo_user *user; + struct gk20a *g; + + linux_user = (struct nvgpu_nvs_domain_ctrl_fifo_user_linux *)filp->private_data; + if (linux_user == NULL) { + return -ENODEV; + } + + user = &linux_user->user; + g = nvgpu_get_gk20a_from_cdev(linux_user->cdev); + + nvs_dbg(g, "IOC_TYPE: %c", _IOC_TYPE(cmd)); + nvs_dbg(g, "IOC_NR: %u", _IOC_NR(cmd)); + nvs_dbg(g, "IOC_SIZE: %u", _IOC_SIZE(cmd)); + + if ((_IOC_TYPE(cmd) != NVGPU_NVS_CTRL_FIFO_IOCTL_MAGIC) || + (_IOC_NR(cmd) == 0) || + (_IOC_NR(cmd) > NVGPU_NVS_CTRL_FIFO_IOCTL_LAST) || + (_IOC_SIZE(cmd) > NVGPU_NVS_CTRL_FIFO_IOCTL_MAX_ARG_SIZE)) { + nvs_dbg(g, "-> BAD!!"); + return -EINVAL; + } + + if (_IOC_DIR(cmd) & _IOC_WRITE) { + if (copy_from_user(buf, (void __user *)arg, _IOC_SIZE(cmd))) + return -EFAULT; + } + + err = gk20a_busy(g); + if (err != 0) { + return err; + } + + switch (cmd) { + case NVGPU_NVS_CTRL_FIFO_CREATE_QUEUE: + { + struct nvgpu_nvs_ctrl_fifo_ioctl_create_queue_args *args = + (struct nvgpu_nvs_ctrl_fifo_ioctl_create_queue_args *)buf; + + err = nvgpu_nvs_ctrl_fifo_create_queue(g, user, args); + if (err) + goto done; + + if (copy_to_user((void __user *)arg, buf, _IOC_SIZE(cmd))) { + nvgpu_nvs_ctrl_fifo_undo_create_queue(g, user, args); + err = -EFAULT; + args->dmabuf_fd = -1; + goto done; + } + + break; + } + case NVGPU_NVS_CTRL_FIFO_RELEASE_QUEUE: + { + struct nvgpu_nvs_ctrl_fifo_ioctl_release_queue_args *args = + (struct nvgpu_nvs_ctrl_fifo_ioctl_release_queue_args *)buf; + + err = nvgpu_nvs_ctrl_fifo_destroy_queue(g, user, args); + if (err) + goto done; + + break; + } + case NVGPU_NVS_CTRL_FIFO_ENABLE_EVENT: + { + err = -EOPNOTSUPP; + goto done; + } + default: + err = -ENOTTY; + goto done; + } + +done: + gk20a_idle(g); + return err; +} diff --git a/drivers/gpu/nvgpu/os/linux/ioctl_nvs.h b/drivers/gpu/nvgpu/os/linux/ioctl_nvs.h index a859c5161..50f3f0f9c 100644 --- a/drivers/gpu/nvgpu/os/linux/ioctl_nvs.h +++ b/drivers/gpu/nvgpu/os/linux/ioctl_nvs.h @@ -28,5 +28,6 @@ struct nvgpu_nvs_domain *nvgpu_nvs_domain_get_from_file(int fd); int nvgpu_nvs_ctrl_fifo_ops_open(struct inode *inode, struct file *filp); int nvgpu_nvs_ctrl_fifo_ops_release(struct inode *inode, struct file *filp); +long nvgpu_nvs_ctrl_fifo_ops_ioctl(struct file *filp, unsigned int cmd, unsigned long arg); #endif