diff --git a/arch/nvgpu-linux.yaml b/arch/nvgpu-linux.yaml index f8201585b..bea0afccf 100644 --- a/arch/nvgpu-linux.yaml +++ b/arch/nvgpu-linux.yaml @@ -37,8 +37,7 @@ dma: sources: [ os/linux/linux-dma.c ] dmabuf: - sources: [ os/linux/dmabuf.c, os/linux/dmabuf_vidmem.c, - os/linux/dmabuf.h, os/linux/dmabuf_vidmem.h, + sources: [ os/linux/dmabuf_vidmem.c, os/linux/dmabuf_vidmem.h, os/linux/dmabuf_priv.c, os/linux/dmabuf_priv.h ] driver_common: diff --git a/drivers/gpu/nvgpu/Kconfig b/drivers/gpu/nvgpu/Kconfig index 8c6d5d6e3..4d8eeabbf 100644 --- a/drivers/gpu/nvgpu/Kconfig +++ b/drivers/gpu/nvgpu/Kconfig @@ -189,17 +189,9 @@ config NVGPU_GR_VIRTUALIZATION help Support NVGPU Virtualization -config NVGPU_DMABUF_HAS_DRVDATA - bool "NVGPU DMABUF private driver data support" - depends on GK20A - default y - help - Support NVGPU DMABUF private driver data needed for compression - config NVGPU_COMPRESSION bool "Compression support" depends on GK20A - depends on NVGPU_DMABUF_HAS_DRVDATA default y help Support for compression diff --git a/drivers/gpu/nvgpu/Makefile b/drivers/gpu/nvgpu/Makefile index 26d9ba579..609401b3d 100644 --- a/drivers/gpu/nvgpu/Makefile +++ b/drivers/gpu/nvgpu/Makefile @@ -380,7 +380,6 @@ nvgpu-y += \ os/linux/linux-io.o \ os/linux/io_usermode.o \ os/linux/rwsem.o \ - os/linux/dmabuf.o \ os/linux/sched.o \ os/linux/linux-channel.o \ os/linux/sim.o \ @@ -389,12 +388,11 @@ nvgpu-y += \ os/linux/dt.o \ os/linux/ecc_sysfs.o \ os/linux/bsearch.o \ - os/linux/sdl/sdl_stub.o + os/linux/sdl/sdl_stub.o \ + os/linux/dmabuf_priv.o nvgpu-$(CONFIG_NVGPU_VPR) += os/linux/vpr.o -nvgpu-$(CONFIG_NVGPU_DMABUF_HAS_DRVDATA) += os/linux/dmabuf_priv.o - nvgpu-$(CONFIG_DEBUG_FS) += \ os/linux/debug.o \ os/linux/debug_gr.o \ diff --git a/drivers/gpu/nvgpu/os/linux/cde.c b/drivers/gpu/nvgpu/os/linux/cde.c index 320fe41f6..c757d83f1 100644 --- a/drivers/gpu/nvgpu/os/linux/cde.c +++ b/drivers/gpu/nvgpu/os/linux/cde.c @@ -45,7 +45,6 @@ #include "cde.h" #include "os_linux.h" -#include "dmabuf.h" #include "dmabuf_priv.h" #include "channel.h" #include "cde_gm20b.h" @@ -1141,7 +1140,7 @@ __releases(&l->cde_app->mutex) nvgpu_log(g, gpu_dbg_cde, "surface=0x%p scatterBuffer=0x%p", surface, scatter_buffer); - sgt = gk20a_mm_pin(dev_from_gk20a(g), compbits_scatter_buf, + sgt = nvgpu_mm_pin_privdata(dev_from_gk20a(g), compbits_scatter_buf, &attachment); if (IS_ERR(sgt)) { nvgpu_warn(g, @@ -1154,7 +1153,7 @@ __releases(&l->cde_app->mutex) scatterbuffer_size); WARN_ON(err); - gk20a_mm_unpin(dev_from_gk20a(g), compbits_scatter_buf, + nvgpu_mm_unpin_privdata(dev_from_gk20a(g), compbits_scatter_buf, attachment, sgt); if (err) goto exit_unmap_surface; diff --git a/drivers/gpu/nvgpu/os/linux/comptags.c b/drivers/gpu/nvgpu/os/linux/comptags.c index 22c6a45e9..b6c667dd9 100644 --- a/drivers/gpu/nvgpu/os/linux/comptags.c +++ b/drivers/gpu/nvgpu/os/linux/comptags.c @@ -21,14 +21,13 @@ #include -#include "dmabuf.h" #include "dmabuf_priv.h" void gk20a_get_comptags(struct nvgpu_os_buffer *buf, struct gk20a_comptags *comptags) { - struct gk20a_dmabuf_priv *priv = dma_buf_get_drvdata(buf->dmabuf, - buf->dev); + struct gk20a_dmabuf_priv *priv = gk20a_dma_buf_get_drvdata(buf->dmabuf, + buf->dev); if (!comptags) return; @@ -48,8 +47,8 @@ int gk20a_alloc_or_get_comptags(struct gk20a *g, struct gk20a_comptag_allocator *allocator, struct gk20a_comptags *comptags) { - struct gk20a_dmabuf_priv *priv = dma_buf_get_drvdata(buf->dmabuf, - buf->dev); + struct gk20a_dmabuf_priv *priv = gk20a_dma_buf_get_drvdata( + buf->dmabuf, buf->dev); u32 offset; int err; u64 ctag_granularity; @@ -111,8 +110,8 @@ exit_locked: bool gk20a_comptags_start_clear(struct nvgpu_os_buffer *buf) { - struct gk20a_dmabuf_priv *priv = dma_buf_get_drvdata(buf->dmabuf, - buf->dev); + struct gk20a_dmabuf_priv *priv = gk20a_dma_buf_get_drvdata(buf->dmabuf, + buf->dev); bool clear_started = false; if (priv) { @@ -130,8 +129,8 @@ bool gk20a_comptags_start_clear(struct nvgpu_os_buffer *buf) void gk20a_comptags_finish_clear(struct nvgpu_os_buffer *buf, bool clear_successful) { - struct gk20a_dmabuf_priv *priv = dma_buf_get_drvdata(buf->dmabuf, - buf->dev); + struct gk20a_dmabuf_priv *priv = gk20a_dma_buf_get_drvdata(buf->dmabuf, + buf->dev); if (priv) { if (clear_successful) priv->comptags.needs_clear = false; diff --git a/drivers/gpu/nvgpu/os/linux/dmabuf.c b/drivers/gpu/nvgpu/os/linux/dmabuf.c deleted file mode 100644 index c3d00b673..000000000 --- a/drivers/gpu/nvgpu/os/linux/dmabuf.c +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms and conditions of the GNU General Public License, - * version 2, as published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#include "platform_gk20a.h" -#include "dmabuf.h" -#include "dmabuf_priv.h" -#include "os_linux.h" -#include "dmabuf_vidmem.h" - -enum nvgpu_aperture gk20a_dmabuf_aperture(struct gk20a *g, - struct dma_buf *dmabuf) -{ -#ifdef CONFIG_NVGPU_DGPU - struct gk20a *buf_owner = nvgpu_vidmem_buf_owner(dmabuf); - bool unified_memory = nvgpu_is_enabled(g, NVGPU_MM_UNIFIED_MEMORY); - - if (buf_owner == NULL) { - /* Not nvgpu-allocated, assume system memory */ - return APERTURE_SYSMEM; - } else if ((buf_owner == g) && unified_memory) { - /* Looks like our video memory, but this gpu doesn't support - * it. Warn about a bug and bail out */ - nvgpu_do_assert_print(g, - "dmabuf is our vidmem but we don't have local vidmem"); - return APERTURE_INVALID; - } else if (buf_owner != g) { - /* Someone else's vidmem */ - return APERTURE_INVALID; - } else { - /* Yay, buf_owner == g */ - return APERTURE_VIDMEM; - } -#else - return APERTURE_SYSMEM; -#endif -} - -struct sg_table *gk20a_mm_pin(struct device *dev, struct dma_buf *dmabuf, - struct dma_buf_attachment **attachment) -{ -#ifdef CONFIG_NVGPU_DMABUF_HAS_DRVDATA - return gk20a_mm_pin_has_drvdata(dev, dmabuf, attachment); -#else - struct dma_buf_attachment *attach = NULL; - struct gk20a *g = get_gk20a(dev); - struct sg_table *sgt = NULL; - - attach = dma_buf_attach(dmabuf, dev); - if (IS_ERR(attach)) { - nvgpu_err(g, "Failed to attach dma_buf (err = %ld)!", - PTR_ERR(attach)); - return ERR_CAST(attach); - } - - sgt = dma_buf_map_attachment(attach, DMA_BIDIRECTIONAL); - if (IS_ERR(sgt)) { - dma_buf_detach(dmabuf, attach); - nvgpu_err(g, "Failed to map attachment (err = %ld)!", - PTR_ERR(sgt)); - return ERR_CAST(sgt); - } - - *attachment = attach; - return sgt; -#endif -} - -void gk20a_mm_unpin(struct device *dev, struct dma_buf *dmabuf, - struct dma_buf_attachment *attachment, - struct sg_table *sgt) -{ -#ifdef CONFIG_NVGPU_DMABUF_HAS_DRVDATA - gk20a_mm_unpin_has_drvdata(dev, dmabuf, attachment, sgt); -#else - dma_buf_unmap_attachment(attachment, sgt, DMA_BIDIRECTIONAL); - dma_buf_detach(dmabuf, attachment); -#endif -} diff --git a/drivers/gpu/nvgpu/os/linux/dmabuf.h b/drivers/gpu/nvgpu/os/linux/dmabuf.h deleted file mode 100644 index acd73124a..000000000 --- a/drivers/gpu/nvgpu/os/linux/dmabuf.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms and conditions of the GNU General Public License, - * version 2, as published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -#ifndef NVGPU_DMABUF_H -#define NVGPU_DMABUF_H - -struct sg_table; -struct dma_buf; -struct dma_buf_attachment; -struct device; - -struct sg_table *gk20a_mm_pin(struct device *dev, struct dma_buf *dmabuf, - struct dma_buf_attachment **attachment); -void gk20a_mm_unpin(struct device *dev, struct dma_buf *dmabuf, - struct dma_buf_attachment *attachment, - struct sg_table *sgt); - -#endif diff --git a/drivers/gpu/nvgpu/os/linux/dmabuf_priv.c b/drivers/gpu/nvgpu/os/linux/dmabuf_priv.c index d963e99f6..4e940408c 100644 --- a/drivers/gpu/nvgpu/os/linux/dmabuf_priv.c +++ b/drivers/gpu/nvgpu/os/linux/dmabuf_priv.c @@ -24,21 +24,123 @@ #include #include #include +#include #include "platform_gk20a.h" -#include "dmabuf.h" #include "dmabuf_priv.h" #include "os_linux.h" #include "dmabuf_vidmem.h" -struct sg_table *gk20a_mm_pin_has_drvdata(struct device *dev, - struct dma_buf *dmabuf, - struct dma_buf_attachment **attachment) +void gk20a_mm_delete_priv(struct gk20a_dmabuf_priv *priv); + +enum nvgpu_aperture gk20a_dmabuf_aperture(struct gk20a *g, + struct dma_buf *dmabuf) +{ +#ifdef CONFIG_NVGPU_DGPU + struct gk20a *buf_owner = nvgpu_vidmem_buf_owner(dmabuf); + bool unified_memory = nvgpu_is_enabled(g, NVGPU_MM_UNIFIED_MEMORY); + + if (buf_owner == NULL) { + /* Not nvgpu-allocated, assume system memory */ + return APERTURE_SYSMEM; + } else if ((buf_owner == g) && unified_memory) { + /* Looks like our video memory, but this gpu doesn't support + * it. Warn about a bug and bail out */ + nvgpu_do_assert_print(g, + "dmabuf is our vidmem but we don't have local vidmem"); + return APERTURE_INVALID; + } else if (buf_owner != g) { + /* Someone else's vidmem */ + return APERTURE_INVALID; + } else { + /* Yay, buf_owner == g */ + return APERTURE_VIDMEM; + } +#else + return APERTURE_SYSMEM; +#endif +} + +static struct gk20a_dmabuf_priv *dma_buf_ops_to_gk20a_priv( + struct dma_buf_ops *ops) +{ + struct gk20a_dmabuf_priv *priv = container_of(ops, + struct gk20a_dmabuf_priv, local_ops); + + return priv; +} + +static void nvgpu_dma_buf_release(struct dma_buf *dmabuf) +{ + struct gk20a_dmabuf_priv *priv = NULL; + struct nvgpu_os_linux *l = NULL; + + priv = dma_buf_ops_to_gk20a_priv((struct dma_buf_ops *)dmabuf->ops); + if (priv != NULL) { + l = nvgpu_os_linux_from_gk20a(priv->g); + } else { + BUG(); + return; + } + + /* remove this entry from the global tracking list */ + nvgpu_mutex_acquire(&l->dmabuf_priv_list_lock); + gk20a_mm_delete_priv(priv); + nvgpu_mutex_release(&l->dmabuf_priv_list_lock); + + dmabuf->ops->release(dmabuf); +} + +int gk20a_dma_buf_set_drvdata(struct dma_buf *dmabuf, struct device *device, + struct gk20a_dmabuf_priv *priv) +{ + nvgpu_mutex_acquire(&priv->lock); + + priv->dmabuf = dmabuf; + + mutex_lock(&dmabuf->lock); + priv->previous_ops = dmabuf->ops; + /* + * Make a copy of the original ops struct and then update the + * release pointer + */ + priv->local_ops = *(dmabuf->ops); + priv->local_ops.release = nvgpu_dma_buf_release; + dmabuf->ops = &priv->local_ops; + mutex_unlock(&dmabuf->lock); + + nvgpu_mutex_release(&priv->lock); + + return 0; +} + +static struct gk20a_dmabuf_priv *gk20a_dmabuf_priv_from_list( + struct nvgpu_list_node *node) +{ + return container_of(node, struct gk20a_dmabuf_priv, list); +} + +struct gk20a_dmabuf_priv *gk20a_dma_buf_get_drvdata( + struct dma_buf *dmabuf, struct device *device) +{ + struct gk20a_dmabuf_priv *priv = NULL; + + mutex_lock(&dmabuf->lock); + if (dmabuf->ops->release == nvgpu_dma_buf_release) { + priv = dma_buf_ops_to_gk20a_priv((struct dma_buf_ops *)dmabuf->ops); + } + mutex_unlock(&dmabuf->lock); + + return priv; +} + +struct sg_table *nvgpu_mm_pin_privdata(struct device *dev, + struct dma_buf *dmabuf, struct dma_buf_attachment **attachment) { struct gk20a *g = get_gk20a(dev); - struct gk20a_dmabuf_priv *priv; + struct gk20a_dmabuf_priv *priv = NULL; - priv = dma_buf_get_drvdata(dmabuf, dev); + priv = gk20a_dma_buf_get_drvdata(dmabuf, dev); if (!priv) { nvgpu_do_assert(); return ERR_PTR(-EINVAL); @@ -72,12 +174,12 @@ struct sg_table *gk20a_mm_pin_has_drvdata(struct device *dev, return priv->sgt; } -void gk20a_mm_unpin_has_drvdata(struct device *dev, +void nvgpu_mm_unpin_privdata(struct device *dev, struct dma_buf *dmabuf, struct dma_buf_attachment *attachment, struct sg_table *sgt) { - struct gk20a_dmabuf_priv *priv = dma_buf_get_drvdata(dmabuf, dev); + struct gk20a_dmabuf_priv *priv = gk20a_dma_buf_get_drvdata(dmabuf, dev); dma_addr_t dma_addr; if (IS_ERR(priv) || !priv) @@ -97,17 +199,22 @@ void gk20a_mm_unpin_has_drvdata(struct device *dev, nvgpu_mutex_release(&priv->lock); } -static void gk20a_mm_delete_priv(void *_priv) +/* This function must be called after acquiring the global level + * dmabuf_priv_list_lock. + */ +void gk20a_mm_delete_priv(struct gk20a_dmabuf_priv *priv) { struct gk20a_buffer_state *s, *s_tmp; - struct gk20a_dmabuf_priv *priv = _priv; struct gk20a *g; + struct dma_buf *dmabuf; if (!priv) return; g = priv->g; + dmabuf = priv->dmabuf; + if (priv->comptags.allocated && priv->comptags.lines) { WARN_ON(!priv->comptag_allocator); gk20a_comptaglines_free(priv->comptag_allocator, @@ -123,20 +230,42 @@ static void gk20a_mm_delete_priv(void *_priv) nvgpu_kfree(g, s); } + /* The original pointer to dma_buf_ops is always put back here*/ + mutex_lock(&dmabuf->lock); + dmabuf->ops = priv->previous_ops; + mutex_unlock(&dmabuf->lock); + + /* Remove this entry from the global tracking list */ + nvgpu_list_del(&priv->list); + nvgpu_kfree(g, priv); } +void gk20a_dma_buf_priv_list_clear(struct nvgpu_os_linux *l) +{ + struct gk20a_dmabuf_priv *priv, *priv_next; + + nvgpu_mutex_acquire(&l->dmabuf_priv_list_lock); + nvgpu_list_for_each_entry_safe(priv, priv_next, &l->dmabuf_priv_list, + gk20a_dmabuf_priv, list) { + gk20a_mm_delete_priv(priv); + } + nvgpu_mutex_release(&l->dmabuf_priv_list_lock); +} + int gk20a_dmabuf_alloc_drvdata(struct dma_buf *dmabuf, struct device *dev) { struct gk20a *g = gk20a_get_platform(dev)->g; struct gk20a_dmabuf_priv *priv; + struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); + + priv = gk20a_dma_buf_get_drvdata(dmabuf, dev); - priv = dma_buf_get_drvdata(dmabuf, dev); if (likely(priv)) return 0; nvgpu_mutex_acquire(&g->mm.priv_lock); - priv = dma_buf_get_drvdata(dmabuf, dev); + priv = gk20a_dma_buf_get_drvdata(dmabuf, dev); if (priv) goto priv_exist_or_err; @@ -149,7 +278,14 @@ int gk20a_dmabuf_alloc_drvdata(struct dma_buf *dmabuf, struct device *dev) nvgpu_mutex_init(&priv->lock); nvgpu_init_list_node(&priv->states); priv->g = g; - dma_buf_set_drvdata(dmabuf, dev, priv, gk20a_mm_delete_priv); + gk20a_dma_buf_set_drvdata(dmabuf, dev, priv); + + nvgpu_init_list_node(&priv->list); + + /* Append this priv to the global tracker */ + nvgpu_mutex_acquire(&l->dmabuf_priv_list_lock); + nvgpu_list_add_tail(&l->dmabuf_priv_list, &priv->list); + nvgpu_mutex_release(&l->dmabuf_priv_list_lock); priv_exist_or_err: nvgpu_mutex_release(&g->mm.priv_lock); @@ -176,7 +312,7 @@ int gk20a_dmabuf_get_state(struct dma_buf *dmabuf, struct gk20a *g, if (err) return err; - priv = dma_buf_get_drvdata(dmabuf, dev); + priv = gk20a_dma_buf_get_drvdata(dmabuf, dev); if (!priv) { nvgpu_do_assert(); return -ENOSYS; diff --git a/drivers/gpu/nvgpu/os/linux/dmabuf_priv.h b/drivers/gpu/nvgpu/os/linux/dmabuf_priv.h index 99c1d2224..0842bb01c 100644 --- a/drivers/gpu/nvgpu/os/linux/dmabuf_priv.h +++ b/drivers/gpu/nvgpu/os/linux/dmabuf_priv.h @@ -17,7 +17,7 @@ #ifndef NVGPU_DMABUF_PRIV_H #define NVGPU_DMABUF_PRIV_H -#ifdef CONFIG_NVGPU_DMABUF_HAS_DRVDATA +#include #include #include @@ -28,9 +28,9 @@ struct sg_table; struct dma_buf; struct dma_buf_attachment; struct device; +struct nvgpu_os_linux; struct gk20a; - struct gk20a_buffer_state { struct nvgpu_list_node list; @@ -80,20 +80,46 @@ struct gk20a_dmabuf_priv { struct nvgpu_list_node states; u64 buffer_id; + + /* Used for retrieving the associated dmabuf from the priv */ + struct dma_buf *dmabuf; + /* 'dma_buf->ops' which is a pointer to a constant struct is + * altered to point to the local copy for the entire lifetime + * of this existing dma-buf until the driver is shutdown or + * the last reference to this dma_buf instance is put. This local + * copy replaces the 'release' callback with nvgpu's custom + * release function handler. This custom function handler frees the + * priv structure and replaces back the original pointer associated + * with the 'producer' of the dma_buf. + */ + struct dma_buf_ops local_ops; + /* Store a copy of the original ops for later restoration */ + const struct dma_buf_ops *previous_ops; + + /* list node for tracking the dmabuf_priv instances per gpu */ + struct nvgpu_list_node list; }; -struct sg_table *gk20a_mm_pin_has_drvdata(struct device *dev, +struct sg_table *nvgpu_mm_pin_privdata(struct device *dev, struct dma_buf *dmabuf, struct dma_buf_attachment **attachment); -void gk20a_mm_unpin_has_drvdata(struct device *dev, +void nvgpu_mm_unpin_privdata(struct device *dev, struct dma_buf *dmabuf, struct dma_buf_attachment *attachment, struct sg_table *sgt); +void gk20a_mm_delete_priv(struct gk20a_dmabuf_priv *priv); + int gk20a_dmabuf_alloc_drvdata(struct dma_buf *dmabuf, struct device *dev); int gk20a_dmabuf_get_state(struct dma_buf *dmabuf, struct gk20a *g, u64 offset, struct gk20a_buffer_state **state); -#endif + +int gk20a_dma_buf_set_drvdata(struct dma_buf *dmabuf, struct device *device, + struct gk20a_dmabuf_priv *priv); +void gk20a_dma_buf_priv_list_clear(struct nvgpu_os_linux *l); +struct gk20a_dmabuf_priv *gk20a_dma_buf_get_drvdata( + struct dma_buf *dmabuf, struct device *device); + #endif diff --git a/drivers/gpu/nvgpu/os/linux/dmabuf_vidmem.c b/drivers/gpu/nvgpu/os/linux/dmabuf_vidmem.c index 3e8f4962e..f40bbd8a6 100644 --- a/drivers/gpu/nvgpu/os/linux/dmabuf_vidmem.c +++ b/drivers/gpu/nvgpu/os/linux/dmabuf_vidmem.c @@ -14,8 +14,10 @@ * along with this program. If not, see . */ -#include #include +#include +#include +#include #include #ifdef CONFIG_NVGPU_USE_TEGRA_ALLOC_FD @@ -40,6 +42,11 @@ bool nvgpu_addr_is_vidmem_page_alloc(u64 addr) return !!(addr & 1ULL); } +/* This constant string is used to determine if the dmabuf belongs + * to nvgpu. + */ +static const char exporter_name[] = "nvgpu"; + void nvgpu_vidmem_set_page_alloc(struct scatterlist *sgl, u64 addr) { /* set bit 0 to indicate vidmem allocation */ @@ -74,24 +81,6 @@ static void gk20a_vidbuf_unmap_dma_buf(struct dma_buf_attachment *attach, { } -static void gk20a_vidbuf_release(struct dma_buf *dmabuf) -{ - struct nvgpu_vidmem_buf *buf = dmabuf->priv; - struct nvgpu_vidmem_linux *linux_buf = buf->priv; - struct gk20a *g = buf->g; - - vidmem_dbg(g, "Releasing Linux VIDMEM buf: dmabuf=0x%p size=%zuKB", - dmabuf, buf->mem->size >> 10); - - if (linux_buf && linux_buf->dmabuf_priv_delete) - linux_buf->dmabuf_priv_delete(linux_buf->dmabuf_priv); - - nvgpu_kfree(g, linux_buf); - nvgpu_vidmem_buf_free(g, buf); - - nvgpu_put(g); -} - #if LINUX_VERSION_CODE <= KERNEL_VERSION(5, 5, 0) static void *gk20a_vidbuf_kmap(struct dma_buf *dmabuf, unsigned long page_num) { @@ -114,30 +103,22 @@ static int gk20a_vidbuf_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma) return -EINVAL; } -#ifdef CONFIG_NVGPU_DMABUF_HAS_DRVDATA -static int gk20a_vidbuf_set_private(struct dma_buf *dmabuf, - struct device *dev, void *priv, void (*delete)(void *priv)) +static void gk20a_vidbuf_release(struct dma_buf *dmabuf) { struct nvgpu_vidmem_buf *buf = dmabuf->priv; struct nvgpu_vidmem_linux *linux_buf = buf->priv; + struct gk20a *g = buf->g; - linux_buf->dmabuf_priv = priv; - linux_buf->dmabuf_priv_delete = delete; + vidmem_dbg(g, "Releasing Linux VIDMEM buf: dmabuf=0x%p size=%zuKB", + dmabuf, buf->mem->size >> 10); - return 0; + nvgpu_kfree(g, linux_buf); + nvgpu_vidmem_buf_free(g, buf); + + nvgpu_put(g); } -static void *gk20a_vidbuf_get_private(struct dma_buf *dmabuf, - struct device *dev) -{ - struct nvgpu_vidmem_buf *buf = dmabuf->priv; - struct nvgpu_vidmem_linux *linux_buf = buf->priv; - - return linux_buf->dmabuf_priv; -} -#endif - -static const struct dma_buf_ops gk20a_vidbuf_ops = { +static struct dma_buf_ops gk20a_vidbuf_ops = { .map_dma_buf = gk20a_vidbuf_map_dma_buf, .unmap_dma_buf = gk20a_vidbuf_unmap_dma_buf, .release = gk20a_vidbuf_release, @@ -153,10 +134,6 @@ static const struct dma_buf_ops gk20a_vidbuf_ops = { .kmap = gk20a_vidbuf_kmap, #endif .mmap = gk20a_vidbuf_mmap, -#ifdef CONFIG_NVGPU_DMABUF_HAS_DRVDATA - .set_drvdata = gk20a_vidbuf_set_private, - .get_drvdata = gk20a_vidbuf_get_private, -#endif }; static struct dma_buf *gk20a_vidbuf_export(struct nvgpu_vidmem_buf *buf) @@ -167,6 +144,7 @@ static struct dma_buf *gk20a_vidbuf_export(struct nvgpu_vidmem_buf *buf) exp_info.ops = &gk20a_vidbuf_ops; exp_info.size = buf->mem->size; exp_info.flags = O_RDWR; + exp_info.exp_name = exporter_name; return dma_buf_export(&exp_info); } @@ -175,8 +153,9 @@ struct gk20a *nvgpu_vidmem_buf_owner(struct dma_buf *dmabuf) { struct nvgpu_vidmem_buf *buf = dmabuf->priv; - if (dmabuf->ops != &gk20a_vidbuf_ops) + if (dmabuf->exp_name != exporter_name) { return NULL; + } return buf->g; } diff --git a/drivers/gpu/nvgpu/os/linux/ioctl_ctrl.c b/drivers/gpu/nvgpu/os/linux/ioctl_ctrl.c index 311d752d2..f1071a2b4 100644 --- a/drivers/gpu/nvgpu/os/linux/ioctl_ctrl.c +++ b/drivers/gpu/nvgpu/os/linux/ioctl_ctrl.c @@ -61,7 +61,6 @@ #include "platform_gk20a.h" #include "os_linux.h" -#include "dmabuf.h" #include "channel.h" #include "dmabuf_vidmem.h" #include "fecs_trace_linux.h" diff --git a/drivers/gpu/nvgpu/os/linux/linux-channel.c b/drivers/gpu/nvgpu/os/linux/linux-channel.c index 1d77681a1..8a9491d7a 100644 --- a/drivers/gpu/nvgpu/os/linux/linux-channel.c +++ b/drivers/gpu/nvgpu/os/linux/linux-channel.c @@ -33,7 +33,6 @@ #include "channel.h" #include "ioctl_channel.h" #include "os_linux.h" -#include "dmabuf.h" #include "dmabuf_priv.h" #include @@ -434,14 +433,12 @@ int nvgpu_usermode_buf_from_dmabuf(struct gk20a *g, int dmabuf_fd, goto put_dmabuf; } -#ifdef CONFIG_NVGPU_DMABUF_HAS_DRVDATA err = gk20a_dmabuf_alloc_drvdata(dmabuf, dev); if (err != 0) { goto put_dmabuf; } -#endif - sgt = gk20a_mm_pin(dev, dmabuf, &attachment); + sgt = nvgpu_mm_pin_privdata(dev, dmabuf, &attachment); if (IS_ERR(sgt)) { nvgpu_warn(g, "Failed to pin dma_buf!"); err = PTR_ERR(sgt); @@ -479,7 +476,7 @@ void nvgpu_os_channel_free_usermode_buffers(struct nvgpu_channel *c) struct device *dev = dev_from_gk20a(g); if (priv->usermode.gpfifo.dmabuf != NULL) { - gk20a_mm_unpin(dev, priv->usermode.gpfifo.dmabuf, + nvgpu_mm_unpin_privdata(dev, priv->usermode.gpfifo.dmabuf, priv->usermode.gpfifo.attachment, priv->usermode.gpfifo.sgt); dma_buf_put(priv->usermode.gpfifo.dmabuf); @@ -487,7 +484,7 @@ void nvgpu_os_channel_free_usermode_buffers(struct nvgpu_channel *c) } if (priv->usermode.userd.dmabuf != NULL) { - gk20a_mm_unpin(dev, priv->usermode.userd.dmabuf, + nvgpu_mm_unpin_privdata(dev, priv->usermode.userd.dmabuf, priv->usermode.userd.attachment, priv->usermode.userd.sgt); dma_buf_put(priv->usermode.userd.dmabuf); @@ -550,7 +547,7 @@ static int nvgpu_channel_alloc_usermode_buffers(struct nvgpu_channel *c, unmap_free_gpfifo: nvgpu_dma_unmap_free(c->vm, &c->usermode_gpfifo); free_gpfifo: - gk20a_mm_unpin(dev, priv->usermode.gpfifo.dmabuf, + nvgpu_mm_unpin_privdata(dev, priv->usermode.gpfifo.dmabuf, priv->usermode.gpfifo.attachment, priv->usermode.gpfifo.sgt); dma_buf_put(priv->usermode.gpfifo.dmabuf); diff --git a/drivers/gpu/nvgpu/os/linux/module.c b/drivers/gpu/nvgpu/os/linux/module.c index a10be0e4f..0b50102df 100644 --- a/drivers/gpu/nvgpu/os/linux/module.c +++ b/drivers/gpu/nvgpu/os/linux/module.c @@ -78,6 +78,7 @@ #include "driver_common.h" #include "channel.h" #include "debug_pmgr.h" +#include "dmabuf_priv.h" #ifdef CONFIG_NVGPU_SUPPORT_CDE #include "cde.h" @@ -1713,6 +1714,9 @@ static int gk20a_probe(struct platform_device *dev) if (err) goto return_err; + nvgpu_mutex_init(&l->dmabuf_priv_list_lock); + nvgpu_init_list_node(&l->dmabuf_priv_list); + return 0; return_err: @@ -1793,6 +1797,9 @@ static int __exit gk20a_remove(struct platform_device *pdev) err = nvgpu_remove(dev, &nvgpu_class); + gk20a_dma_buf_priv_list_clear(l); + nvgpu_mutex_destroy(&l->dmabuf_priv_list_lock); + unregister_reboot_notifier(&l->nvgpu_reboot_nb); set_gk20a(pdev, NULL); diff --git a/drivers/gpu/nvgpu/os/linux/os_linux.h b/drivers/gpu/nvgpu/os/linux/os_linux.h index dab169cca..8f1ff5dd2 100644 --- a/drivers/gpu/nvgpu/os/linux/os_linux.h +++ b/drivers/gpu/nvgpu/os/linux/os_linux.h @@ -170,6 +170,9 @@ struct nvgpu_os_linux { struct rw_semaphore busy_lock; + struct nvgpu_mutex dmabuf_priv_list_lock; + struct nvgpu_list_node dmabuf_priv_list; + bool init_done; /** Debugfs knob for forcing syncpt support off in runtime. */ diff --git a/drivers/gpu/nvgpu/os/linux/pci.c b/drivers/gpu/nvgpu/os/linux/pci.c index 1cc10cccf..ee47be07c 100644 --- a/drivers/gpu/nvgpu/os/linux/pci.c +++ b/drivers/gpu/nvgpu/os/linux/pci.c @@ -43,6 +43,7 @@ #include "pci_power.h" #include "driver_common.h" +#include "dmabuf_priv.h" #define BOOT_GPC2CLK_MHZ 2581U #define PCI_INTERFACE_NAME "card-%s%%s" @@ -684,6 +685,9 @@ static int nvgpu_pci_probe(struct pci_dev *pdev, goto err_free_irq; } + nvgpu_mutex_init(&l->dmabuf_priv_list_lock); + nvgpu_init_list_node(&l->dmabuf_priv_list); + return 0; err_free_irq: @@ -719,11 +723,15 @@ static void nvgpu_pci_remove(struct pci_dev *pdev) struct gk20a *g = get_gk20a(&pdev->dev); struct device *dev = dev_from_gk20a(g); int err; + struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); /* no support yet for unbind if DGPU is in VGPU mode */ if (gk20a_gpu_is_virtual(dev)) return; + gk20a_dma_buf_priv_list_clear(l); + nvgpu_mutex_destroy(&l->dmabuf_priv_list_lock); + err = nvgpu_pci_clear_pci_power(dev_name(dev)); WARN(err, "gpu failed to clear pci power"); diff --git a/drivers/gpu/nvgpu/os/linux/vgpu/vgpu_linux.c b/drivers/gpu/nvgpu/os/linux/vgpu/vgpu_linux.c index a82983c24..a578c427c 100644 --- a/drivers/gpu/nvgpu/os/linux/vgpu/vgpu_linux.c +++ b/drivers/gpu/nvgpu/os/linux/vgpu/vgpu_linux.c @@ -60,6 +60,7 @@ #include "os/linux/driver_common.h" #include "os/linux/platform_gk20a.h" #include "os/linux/vgpu/platform_vgpu_tegra.h" +#include "os/linux/dmabuf_priv.h" struct vgpu_priv_data *vgpu_get_priv_data(struct gk20a *g) { @@ -487,6 +488,9 @@ int vgpu_probe(struct platform_device *pdev) #endif gk20a->max_comptag_mem = totalram_size_in_mb; + nvgpu_mutex_init(&l->dmabuf_priv_list_lock); + nvgpu_init_list_node(&l->dmabuf_priv_list); + nvgpu_ref_init(&gk20a->refcount); return 0; @@ -496,9 +500,13 @@ int vgpu_remove(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct gk20a *g = get_gk20a(dev); + struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); nvgpu_log_fn(g, " "); + gk20a_dma_buf_priv_list_clear(l); + nvgpu_mutex_destroy(&l->dmabuf_priv_list_lock); + vgpu_pm_qos_remove(dev); if (g->remove_support) g->remove_support(g); diff --git a/drivers/gpu/nvgpu/os/linux/vm.c b/drivers/gpu/nvgpu/os/linux/vm.c index b843711ce..830b40f64 100644 --- a/drivers/gpu/nvgpu/os/linux/vm.c +++ b/drivers/gpu/nvgpu/os/linux/vm.c @@ -35,7 +35,6 @@ #include "platform_gk20a.h" #include "os_linux.h" -#include "dmabuf.h" #include "dmabuf_priv.h" #include "dmabuf_vidmem.h" @@ -174,7 +173,7 @@ struct nvgpu_mapped_buf *nvgpu_vm_find_mapping(struct vm_gk20a *vm, * the dmabuf doesn't support drvdata, prior SGT is unpinned as the * new SGT was pinned at the beginning of the current map call. */ - gk20a_mm_unpin(os_buf->dev, os_buf->dmabuf, + nvgpu_mm_unpin_privdata(os_buf->dev, os_buf->dmabuf, mapped_buffer->os_priv.attachment, mapped_buffer->os_priv.sgt); dma_buf_put(os_buf->dmabuf); @@ -204,7 +203,7 @@ int nvgpu_vm_map_linux(struct vm_gk20a *vm, struct dma_buf_attachment *attachment; int err = 0; - sgt = gk20a_mm_pin(dev, dmabuf, &attachment); + sgt = nvgpu_mm_pin_privdata(dev, dmabuf, &attachment); if (IS_ERR(sgt)) { nvgpu_warn(g, "Failed to pin dma_buf!"); return PTR_ERR(sgt); @@ -252,7 +251,7 @@ int nvgpu_vm_map_linux(struct vm_gk20a *vm, return 0; clean_up: - gk20a_mm_unpin(dev, dmabuf, attachment, sgt); + nvgpu_mm_unpin_privdata(dev, dmabuf, attachment, sgt); return err; } @@ -319,13 +318,11 @@ int nvgpu_vm_map_buffer(struct vm_gk20a *vm, return -EINVAL; } -#ifdef CONFIG_NVGPU_DMABUF_HAS_DRVDATA err = gk20a_dmabuf_alloc_drvdata(dmabuf, dev_from_vm(vm)); if (err) { dma_buf_put(dmabuf); return err; } -#endif err = nvgpu_vm_map_linux(vm, dmabuf, *map_addr, nvgpu_vm_translate_linux_flags(g, flags), @@ -356,7 +353,7 @@ void nvgpu_vm_unmap_system(struct nvgpu_mapped_buf *mapped_buffer) { struct vm_gk20a *vm = mapped_buffer->vm; - gk20a_mm_unpin(dev_from_vm(vm), mapped_buffer->os_priv.dmabuf, + nvgpu_mm_unpin_privdata(dev_from_vm(vm), mapped_buffer->os_priv.dmabuf, mapped_buffer->os_priv.attachment, mapped_buffer->os_priv.sgt);