gpu: nvgpu: alternative implementation of dma_buf_get/set_data

Historically, nvgpu has supported a struct gk20a_dmabuf_priv and
associated it with a dmabuf instance. This was aided by Nvmap's
dma_buf_set_drv_data() and dma_buf_get_drvdata() APIs. gk20a_dmabuf_priv
is used to store Comptag IDs i.e. (1 per 64 kb) as well as can store the
dmabuf attachments to avoid multiple attach/detach calls. dma_buf_set_drv_data()
allows Nvgpu to associate an instance of struct gk20a_dmabuf_priv with the instance
of the dmabuf and also provide a release callback to delete the
instance when the last reference to the dmabuf is put. Nvmap accomplishes
this by modifying the struct dma_buf_ops definition to include the
set_drv_data and get_drv_data callbacks in the kernel code.

The above approach won't work for upstream Kstable and Nvmap
plans to remove these APIs for upcoming newer downstream kernels as
well.

In order to implement the same functionality without depending on Nvmap,
Nvgpu will implement a release chaining mechanism. Dmabuf's 'ops' pointer
points to a constant struct and hence a whole copy of the ops is made
followed by altering the new copy's release pointer.

struct gk20a_dmabuf_priv stores the new copy and the dmabuf's 'ops' is
changed to point to this. This allows Nvgpu to retrieve
the corresponding gk20a_dmabuf_priv instance using container_of.

Nvgpu's custom release callback will invoke the original release
callback of the dmabuf's producer as a last step, thus completing the
full circle. In case, the driver is removed, Nvgpu restores the
dmabuf's 'ops' back to the original state. In order to accomplish this,
every instance of a struct nvgpu_os_linux maintains a linkedlist of the
gk20a_dma_buf instances. During the driver removal, this linkedlist is
traversed and the corresponding dmabuf's 'ops' pointer is put back to
its original state followed by freeing of this instance.

Nvgpu is a producer of dmabuf's for vidmem and needs
a way to check whether the given dmabuf belongs to itself.
Its no longer reliable to depend on a comparision of
the 'ops' pointer. Instead dmabuf_export_info() allows a name to be set by the
exporter and this can be used to compare with a memory location
that belongs to Nvgpu. Similarly for sysmem dmabufs, Nvmap makes a
similar change in the way it identifies whether a dmabuf belongs to
itself.

Removed NVGPU_DMABUF_HAS_DRVDATA and moved to a unified mechanism for
both downstream as well as upstream kernel.

Some of the other changes in this file include the following.
1) Deletion of dmabuf.c and moving its contents over to dmabuf_priv.c
2) Replacing gk20a_mm_pin_has_drvdata with nvgpu_mm_pin_privdata and
vice-versa for unpin.

Bug 2878569

Change-Id: Icf8e79b05a25ad5a85f478c3ee0fc1eb7747e22d
Signed-off-by: Debarshi Dutta <ddutta@nvidia.com>
Reviewed-on: https://git-master.nvidia.com/r/c/linux-nvgpu/+/2341001
Reviewed-by: automaticguardword <automaticguardword@nvidia.com>
Reviewed-by: Puneet Saxena <puneets@nvidia.com>
Reviewed-by: Konsta Holtta <kholtta@nvidia.com>
Reviewed-by: Vijayakumar Subbu <vsubbu@nvidia.com>
Reviewed-by: mobile promotions <svcmobile_promotions@nvidia.com>
GVS: Gerrit_Virtual_Submit
Tested-by: mobile promotions <svcmobile_promotions@nvidia.com>

arch/nvgpu-linux.yaml

@@ -37,8 +37,7 @@ dma:
   sources: [ os/linux/linux-dma.c ]
 
 dmabuf:
-  sources: [ os/linux/dmabuf.c, os/linux/dmabuf_vidmem.c,
+  sources: [ os/linux/dmabuf_vidmem.c, os/linux/dmabuf_vidmem.h,
-             os/linux/dmabuf.h, os/linux/dmabuf_vidmem.h,
              os/linux/dmabuf_priv.c, os/linux/dmabuf_priv.h ]
 
 driver_common:

drivers/gpu/nvgpu/Kconfig

@@ -189,17 +189,9 @@ config NVGPU_GR_VIRTUALIZATION
 	help
 	  Support NVGPU Virtualization
 
-config NVGPU_DMABUF_HAS_DRVDATA
-	bool "NVGPU DMABUF private driver data support"
-	depends on GK20A
-	default y
-	help
-	  Support NVGPU DMABUF private driver data needed for compression
-
 config NVGPU_COMPRESSION
 	bool "Compression support"
 	depends on GK20A
-	depends on NVGPU_DMABUF_HAS_DRVDATA
 	default y
 	help
 	  Support for compression

drivers/gpu/nvgpu/Makefile

@@ -380,7 +380,6 @@ nvgpu-y += \
 	os/linux/linux-io.o \
 	os/linux/io_usermode.o \
 	os/linux/rwsem.o \
-	os/linux/dmabuf.o \
 	os/linux/sched.o \
 	os/linux/linux-channel.o \
 	os/linux/sim.o \
@@ -389,12 +388,11 @@ nvgpu-y += \
 	os/linux/dt.o \
 	os/linux/ecc_sysfs.o \
 	os/linux/bsearch.o \
-	os/linux/sdl/sdl_stub.o
+	os/linux/sdl/sdl_stub.o \
+	os/linux/dmabuf_priv.o
 
 nvgpu-$(CONFIG_NVGPU_VPR) += os/linux/vpr.o
 
-nvgpu-$(CONFIG_NVGPU_DMABUF_HAS_DRVDATA) += os/linux/dmabuf_priv.o
-
 nvgpu-$(CONFIG_DEBUG_FS) += \
 	os/linux/debug.o \
 	os/linux/debug_gr.o \

drivers/gpu/nvgpu/os/linux/cde.c

@@ -45,7 +45,6 @@
 
 #include "cde.h"
 #include "os_linux.h"
-#include "dmabuf.h"
 #include "dmabuf_priv.h"
 #include "channel.h"
 #include "cde_gm20b.h"
@@ -1141,7 +1140,7 @@ __releases(&l->cde_app->mutex)
 
 		nvgpu_log(g, gpu_dbg_cde, "surface=0x%p scatterBuffer=0x%p",
 			  surface, scatter_buffer);
-		sgt = gk20a_mm_pin(dev_from_gk20a(g), compbits_scatter_buf,
+		sgt = nvgpu_mm_pin_privdata(dev_from_gk20a(g), compbits_scatter_buf,
 				   &attachment);
 		if (IS_ERR(sgt)) {
 			nvgpu_warn(g,
@@ -1154,7 +1153,7 @@ __releases(&l->cde_app->mutex)
 					scatterbuffer_size);
 			WARN_ON(err);
 
-			gk20a_mm_unpin(dev_from_gk20a(g), compbits_scatter_buf,
+			nvgpu_mm_unpin_privdata(dev_from_gk20a(g), compbits_scatter_buf,
 				       attachment, sgt);
 			if (err)
 				goto exit_unmap_surface;

drivers/gpu/nvgpu/os/linux/comptags.c

@@ -21,13 +21,12 @@
 
 #include <nvgpu/linux/vm.h>
 
-#include "dmabuf.h"
 #include "dmabuf_priv.h"
 
 void gk20a_get_comptags(struct nvgpu_os_buffer *buf,
 			struct gk20a_comptags *comptags)
 {
-	struct gk20a_dmabuf_priv *priv = dma_buf_get_drvdata(buf->dmabuf,
+	struct gk20a_dmabuf_priv *priv = gk20a_dma_buf_get_drvdata(buf->dmabuf,
 							buf->dev);
 
 	if (!comptags)
@@ -48,8 +47,8 @@ int gk20a_alloc_or_get_comptags(struct gk20a *g,
 				struct gk20a_comptag_allocator *allocator,
 				struct gk20a_comptags *comptags)
 {
-	struct gk20a_dmabuf_priv *priv = dma_buf_get_drvdata(buf->dmabuf,
+	struct gk20a_dmabuf_priv *priv = gk20a_dma_buf_get_drvdata(
-							     buf->dev);
+						buf->dmabuf, buf->dev);
 	u32 offset;
 	int err;
 	u64 ctag_granularity;
@@ -111,7 +110,7 @@ exit_locked:
 
 bool gk20a_comptags_start_clear(struct nvgpu_os_buffer *buf)
 {
-	struct gk20a_dmabuf_priv *priv = dma_buf_get_drvdata(buf->dmabuf,
+	struct gk20a_dmabuf_priv *priv = gk20a_dma_buf_get_drvdata(buf->dmabuf,
 						buf->dev);
 	bool clear_started = false;
 
@@ -130,7 +129,7 @@ bool gk20a_comptags_start_clear(struct nvgpu_os_buffer *buf)
 void gk20a_comptags_finish_clear(struct nvgpu_os_buffer *buf,
 				 bool clear_successful)
 {
-	struct gk20a_dmabuf_priv *priv = dma_buf_get_drvdata(buf->dmabuf,
+	struct gk20a_dmabuf_priv *priv = gk20a_dma_buf_get_drvdata(buf->dmabuf,
 						buf->dev);
 	if (priv) {
 		if (clear_successful)

drivers/gpu/nvgpu/os/linux/dmabuf.c

@@ -1,102 +0,0 @@
-/*
- * Copyright (c) 2017-2020, NVIDIA CORPORATION.  All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-#include <linux/device.h>
-#include <linux/dma-buf.h>
-#include <linux/scatterlist.h>
-
-#include <nvgpu/comptags.h>
-#include <nvgpu/enabled.h>
-#include <nvgpu/gk20a.h>
-#include <nvgpu/linux/vm.h>
-#include <nvgpu/bug.h>
-#include <nvgpu/fence.h>
-
-#include "platform_gk20a.h"
-#include "dmabuf.h"
-#include "dmabuf_priv.h"
-#include "os_linux.h"
-#include "dmabuf_vidmem.h"
-
-enum nvgpu_aperture gk20a_dmabuf_aperture(struct gk20a *g,
-					  struct dma_buf *dmabuf)
-{
-#ifdef CONFIG_NVGPU_DGPU
-	struct gk20a *buf_owner = nvgpu_vidmem_buf_owner(dmabuf);
-	bool unified_memory = nvgpu_is_enabled(g, NVGPU_MM_UNIFIED_MEMORY);
-
-	if (buf_owner == NULL) {
-		/* Not nvgpu-allocated, assume system memory */
-		return APERTURE_SYSMEM;
-	} else if ((buf_owner == g) && unified_memory) {
-		/* Looks like our video memory, but this gpu doesn't support
-		 * it. Warn about a bug and bail out */
-		nvgpu_do_assert_print(g,
-			"dmabuf is our vidmem but we don't have local vidmem");
-		return APERTURE_INVALID;
-	} else if (buf_owner != g) {
-		/* Someone else's vidmem */
-		return APERTURE_INVALID;
-	} else {
-		/* Yay, buf_owner == g */
-		return APERTURE_VIDMEM;
-	}
-#else
-	return APERTURE_SYSMEM;
-#endif
-}
-
-struct sg_table *gk20a_mm_pin(struct device *dev, struct dma_buf *dmabuf,
-			      struct dma_buf_attachment **attachment)
-{
-#ifdef CONFIG_NVGPU_DMABUF_HAS_DRVDATA
-	return gk20a_mm_pin_has_drvdata(dev, dmabuf, attachment);
-#else
-	struct dma_buf_attachment *attach = NULL;
-	struct gk20a *g = get_gk20a(dev);
-	struct sg_table *sgt = NULL;
-
-	attach = dma_buf_attach(dmabuf, dev);
-	if (IS_ERR(attach)) {
-		nvgpu_err(g, "Failed to attach dma_buf (err = %ld)!",
-			  PTR_ERR(attach));
-		return ERR_CAST(attach);
-	}
-
-	sgt = dma_buf_map_attachment(attach, DMA_BIDIRECTIONAL);
-	if (IS_ERR(sgt)) {
-		dma_buf_detach(dmabuf, attach);
-		nvgpu_err(g, "Failed to map attachment (err = %ld)!",
-			  PTR_ERR(sgt));
-		return ERR_CAST(sgt);
-	}
-
-	*attachment = attach;
-	return sgt;
-#endif
-}
-
-void gk20a_mm_unpin(struct device *dev, struct dma_buf *dmabuf,
-		    struct dma_buf_attachment *attachment,
-		    struct sg_table *sgt)
-{
-#ifdef CONFIG_NVGPU_DMABUF_HAS_DRVDATA
-	gk20a_mm_unpin_has_drvdata(dev, dmabuf, attachment, sgt);
-#else
-	dma_buf_unmap_attachment(attachment, sgt, DMA_BIDIRECTIONAL);
-	dma_buf_detach(dmabuf, attachment);
-#endif
-}

drivers/gpu/nvgpu/os/linux/dmabuf.h

@@ -1,31 +0,0 @@
-/*
- * Copyright (c) 2017-2020, NVIDIA CORPORATION.  All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef NVGPU_DMABUF_H
-#define NVGPU_DMABUF_H
-
-struct sg_table;
-struct dma_buf;
-struct dma_buf_attachment;
-struct device;
-
-struct sg_table *gk20a_mm_pin(struct device *dev, struct dma_buf *dmabuf,
-			      struct dma_buf_attachment **attachment);
-void gk20a_mm_unpin(struct device *dev, struct dma_buf *dmabuf,
-		    struct dma_buf_attachment *attachment,
-		    struct sg_table *sgt);
-
-#endif

drivers/gpu/nvgpu/os/linux/dmabuf_priv.c

@@ -24,21 +24,123 @@
 #include <nvgpu/linux/vm.h>
 #include <nvgpu/bug.h>
 #include <nvgpu/fence.h>
+#include <nvgpu/vm.h>
 
 #include "platform_gk20a.h"
-#include "dmabuf.h"
 #include "dmabuf_priv.h"
 #include "os_linux.h"
 #include "dmabuf_vidmem.h"
 
-struct sg_table *gk20a_mm_pin_has_drvdata(struct device *dev,
+void gk20a_mm_delete_priv(struct gk20a_dmabuf_priv *priv);
-				struct dma_buf *dmabuf,
+
-				struct dma_buf_attachment **attachment)
+enum nvgpu_aperture gk20a_dmabuf_aperture(struct gk20a *g,
+					  struct dma_buf *dmabuf)
+{
+#ifdef CONFIG_NVGPU_DGPU
+	struct gk20a *buf_owner = nvgpu_vidmem_buf_owner(dmabuf);
+	bool unified_memory = nvgpu_is_enabled(g, NVGPU_MM_UNIFIED_MEMORY);
+
+	if (buf_owner == NULL) {
+		/* Not nvgpu-allocated, assume system memory */
+		return APERTURE_SYSMEM;
+	} else if ((buf_owner == g) && unified_memory) {
+		/* Looks like our video memory, but this gpu doesn't support
+		 * it. Warn about a bug and bail out */
+		nvgpu_do_assert_print(g,
+			"dmabuf is our vidmem but we don't have local vidmem");
+		return APERTURE_INVALID;
+	} else if (buf_owner != g) {
+		/* Someone else's vidmem */
+		return APERTURE_INVALID;
+	} else {
+		/* Yay, buf_owner == g */
+		return APERTURE_VIDMEM;
+	}
+#else
+	return APERTURE_SYSMEM;
+#endif
+}
+
+static struct gk20a_dmabuf_priv *dma_buf_ops_to_gk20a_priv(
+		struct dma_buf_ops *ops)
+{
+	struct gk20a_dmabuf_priv *priv = container_of(ops,
+			struct gk20a_dmabuf_priv, local_ops);
+
+	return priv;
+}
+
+static void nvgpu_dma_buf_release(struct dma_buf *dmabuf)
+{
+	struct gk20a_dmabuf_priv *priv = NULL;
+	struct nvgpu_os_linux *l = NULL;
+
+	priv = dma_buf_ops_to_gk20a_priv((struct dma_buf_ops *)dmabuf->ops);
+	if (priv != NULL) {
+		l = nvgpu_os_linux_from_gk20a(priv->g);
+	} else {
+		BUG();
+		return;
+	}
+
+	/* remove this entry from the global tracking list */
+	nvgpu_mutex_acquire(&l->dmabuf_priv_list_lock);
+	gk20a_mm_delete_priv(priv);
+	nvgpu_mutex_release(&l->dmabuf_priv_list_lock);
+
+	dmabuf->ops->release(dmabuf);
+}
+
+int gk20a_dma_buf_set_drvdata(struct dma_buf *dmabuf, struct device *device,
+			struct gk20a_dmabuf_priv *priv)
+{
+	nvgpu_mutex_acquire(&priv->lock);
+
+	priv->dmabuf = dmabuf;
+
+	mutex_lock(&dmabuf->lock);
+	priv->previous_ops = dmabuf->ops;
+	/*
+	 * Make a copy of the original ops struct and then update the
+	 * release pointer
+	 */
+	priv->local_ops = *(dmabuf->ops);
+	priv->local_ops.release = nvgpu_dma_buf_release;
+	dmabuf->ops = &priv->local_ops;
+	mutex_unlock(&dmabuf->lock);
+
+	nvgpu_mutex_release(&priv->lock);
+
+	return 0;
+}
+
+static struct gk20a_dmabuf_priv *gk20a_dmabuf_priv_from_list(
+						struct nvgpu_list_node *node)
+{
+	return container_of(node, struct gk20a_dmabuf_priv, list);
+}
+
+struct gk20a_dmabuf_priv *gk20a_dma_buf_get_drvdata(
+		struct dma_buf *dmabuf, struct device *device)
+{
+	struct gk20a_dmabuf_priv *priv = NULL;
+
+	mutex_lock(&dmabuf->lock);
+	if (dmabuf->ops->release == nvgpu_dma_buf_release) {
+		priv = dma_buf_ops_to_gk20a_priv((struct dma_buf_ops *)dmabuf->ops);
+	}
+	mutex_unlock(&dmabuf->lock);
+
+	return priv;
+}
+
+struct sg_table *nvgpu_mm_pin_privdata(struct device *dev,
+			struct dma_buf *dmabuf, struct dma_buf_attachment **attachment)
 {
 	struct gk20a *g = get_gk20a(dev);
-	struct gk20a_dmabuf_priv *priv;
+	struct gk20a_dmabuf_priv *priv = NULL;
 
-	priv = dma_buf_get_drvdata(dmabuf, dev);
+	priv = gk20a_dma_buf_get_drvdata(dmabuf, dev);
 	if (!priv) {
 		nvgpu_do_assert();
 		return ERR_PTR(-EINVAL);
@@ -72,12 +174,12 @@ struct sg_table *gk20a_mm_pin_has_drvdata(struct device *dev,
 	return priv->sgt;
 }
 
-void gk20a_mm_unpin_has_drvdata(struct device *dev,
+void nvgpu_mm_unpin_privdata(struct device *dev,
 			struct dma_buf *dmabuf,
 			struct dma_buf_attachment *attachment,
 			struct sg_table *sgt)
 {
-	struct gk20a_dmabuf_priv *priv = dma_buf_get_drvdata(dmabuf, dev);
+	struct gk20a_dmabuf_priv *priv = gk20a_dma_buf_get_drvdata(dmabuf, dev);
 	dma_addr_t dma_addr;
 
 	if (IS_ERR(priv) || !priv)
@@ -97,17 +199,22 @@ void gk20a_mm_unpin_has_drvdata(struct device *dev,
 	nvgpu_mutex_release(&priv->lock);
 }
 
-static void gk20a_mm_delete_priv(void *_priv)
+/* This function must be called after acquiring the global level
+ * dmabuf_priv_list_lock.
+ */
+void gk20a_mm_delete_priv(struct gk20a_dmabuf_priv *priv)
 {
 	struct gk20a_buffer_state *s, *s_tmp;
-	struct gk20a_dmabuf_priv *priv = _priv;
 	struct gk20a *g;
+	struct dma_buf *dmabuf;
 
 	if (!priv)
 		return;
 
 	g = priv->g;
 
+	dmabuf = priv->dmabuf;
+
 	if (priv->comptags.allocated && priv->comptags.lines) {
 		WARN_ON(!priv->comptag_allocator);
 		gk20a_comptaglines_free(priv->comptag_allocator,
@@ -123,20 +230,42 @@ static void gk20a_mm_delete_priv(void *_priv)
 		nvgpu_kfree(g, s);
 	}
 
+	/* The original pointer to dma_buf_ops is always put back here*/
+	mutex_lock(&dmabuf->lock);
+	dmabuf->ops = priv->previous_ops;
+	mutex_unlock(&dmabuf->lock);
+
+	/* Remove this entry from the global tracking list */
+	nvgpu_list_del(&priv->list);
+
 	nvgpu_kfree(g, priv);
 }
 
+void gk20a_dma_buf_priv_list_clear(struct nvgpu_os_linux *l)
+{
+	struct gk20a_dmabuf_priv *priv, *priv_next;
+
+	nvgpu_mutex_acquire(&l->dmabuf_priv_list_lock);
+	nvgpu_list_for_each_entry_safe(priv, priv_next, &l->dmabuf_priv_list,
+				gk20a_dmabuf_priv, list) {
+		gk20a_mm_delete_priv(priv);
+	}
+	nvgpu_mutex_release(&l->dmabuf_priv_list_lock);
+}
+
 int gk20a_dmabuf_alloc_drvdata(struct dma_buf *dmabuf, struct device *dev)
 {
 	struct gk20a *g = gk20a_get_platform(dev)->g;
 	struct gk20a_dmabuf_priv *priv;
+	struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
+
+	priv = gk20a_dma_buf_get_drvdata(dmabuf, dev);
 
-	priv = dma_buf_get_drvdata(dmabuf, dev);
 	if (likely(priv))
 		return 0;
 
 	nvgpu_mutex_acquire(&g->mm.priv_lock);
-	priv = dma_buf_get_drvdata(dmabuf, dev);
+	priv = gk20a_dma_buf_get_drvdata(dmabuf, dev);
 	if (priv)
 		goto priv_exist_or_err;
 
@@ -149,7 +278,14 @@ int gk20a_dmabuf_alloc_drvdata(struct dma_buf *dmabuf, struct device *dev)
 	nvgpu_mutex_init(&priv->lock);
 	nvgpu_init_list_node(&priv->states);
 	priv->g = g;
-	dma_buf_set_drvdata(dmabuf, dev, priv, gk20a_mm_delete_priv);
+	gk20a_dma_buf_set_drvdata(dmabuf, dev, priv);
+
+	nvgpu_init_list_node(&priv->list);
+
+	/* Append this priv to the global tracker */
+	nvgpu_mutex_acquire(&l->dmabuf_priv_list_lock);
+	nvgpu_list_add_tail(&l->dmabuf_priv_list, &priv->list);
+	nvgpu_mutex_release(&l->dmabuf_priv_list_lock);
 
 priv_exist_or_err:
 	nvgpu_mutex_release(&g->mm.priv_lock);
@@ -176,7 +312,7 @@ int gk20a_dmabuf_get_state(struct dma_buf *dmabuf, struct gk20a *g,
 	if (err)
 		return err;
 
-	priv = dma_buf_get_drvdata(dmabuf, dev);
+	priv = gk20a_dma_buf_get_drvdata(dmabuf, dev);
 	if (!priv) {
 		nvgpu_do_assert();
 		return -ENOSYS;

drivers/gpu/nvgpu/os/linux/dmabuf_priv.h

@@ -17,7 +17,7 @@
 #ifndef NVGPU_DMABUF_PRIV_H
 #define NVGPU_DMABUF_PRIV_H
 
-#ifdef CONFIG_NVGPU_DMABUF_HAS_DRVDATA
+#include <linux/dma-buf.h>
 
 #include <nvgpu/comptags.h>
 #include <nvgpu/list.h>
@@ -28,9 +28,9 @@ struct sg_table;
 struct dma_buf;
 struct dma_buf_attachment;
 struct device;
+struct nvgpu_os_linux;
 
 struct gk20a;
-
 struct gk20a_buffer_state {
 	struct nvgpu_list_node list;
 
@@ -80,20 +80,46 @@ struct gk20a_dmabuf_priv {
 	struct nvgpu_list_node states;
 
 	u64 buffer_id;
+
+	/* Used for retrieving the associated dmabuf from the priv */
+	struct dma_buf *dmabuf;
+	/* 'dma_buf->ops' which is a pointer to a constant struct is
+	 * altered to point to the local copy for the entire lifetime
+	 * of this existing dma-buf until the driver is shutdown or
+	 * the last reference to this dma_buf instance is put. This local
+	 * copy replaces the 'release' callback with nvgpu's custom
+	 * release function handler. This custom function handler frees the
+	 * priv structure and replaces back the original pointer associated
+	 * with the 'producer' of the dma_buf.
+	 */
+	struct dma_buf_ops local_ops;
+	/* Store a copy of the original ops for later restoration */
+	const struct dma_buf_ops *previous_ops;
+
+	/* list node for tracking the dmabuf_priv instances per gpu */
+	struct nvgpu_list_node list;
 };
 
-struct sg_table *gk20a_mm_pin_has_drvdata(struct device *dev,
+struct sg_table *nvgpu_mm_pin_privdata(struct device *dev,
 			struct dma_buf *dmabuf,
 			struct dma_buf_attachment **attachment);
 
-void gk20a_mm_unpin_has_drvdata(struct device *dev,
+void nvgpu_mm_unpin_privdata(struct device *dev,
 		struct dma_buf *dmabuf,
 		struct dma_buf_attachment *attachment,
 		struct sg_table *sgt);
 
+void gk20a_mm_delete_priv(struct gk20a_dmabuf_priv *priv);
+
 int gk20a_dmabuf_alloc_drvdata(struct dma_buf *dmabuf, struct device *dev);
 
 int gk20a_dmabuf_get_state(struct dma_buf *dmabuf, struct gk20a *g,
 			   u64 offset, struct gk20a_buffer_state **state);
-#endif
+
+int gk20a_dma_buf_set_drvdata(struct dma_buf *dmabuf, struct device *device,
+			struct gk20a_dmabuf_priv *priv);
+void gk20a_dma_buf_priv_list_clear(struct nvgpu_os_linux *l);
+struct gk20a_dmabuf_priv *gk20a_dma_buf_get_drvdata(
+		struct dma_buf *dmabuf, struct device *device);
+
 #endif

drivers/gpu/nvgpu/os/linux/dmabuf_vidmem.c

@@ -14,8 +14,10 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <linux/dma-buf.h>
 #include <linux/version.h>
+#include <linux/scatterlist.h>
+#include <linux/dma-direction.h>
+#include <linux/dma-buf.h>
 #include <uapi/linux/nvgpu.h>
 
 #ifdef CONFIG_NVGPU_USE_TEGRA_ALLOC_FD
@@ -40,6 +42,11 @@ bool nvgpu_addr_is_vidmem_page_alloc(u64 addr)
 	return !!(addr & 1ULL);
 }
 
+/* This constant string is used to determine if the dmabuf belongs
+ * to nvgpu.
+ */
+static const char exporter_name[] = "nvgpu";
+
 void nvgpu_vidmem_set_page_alloc(struct scatterlist *sgl, u64 addr)
 {
 	/* set bit 0 to indicate vidmem allocation */
@@ -74,24 +81,6 @@ static void gk20a_vidbuf_unmap_dma_buf(struct dma_buf_attachment *attach,
 {
 }
 
-static void gk20a_vidbuf_release(struct dma_buf *dmabuf)
-{
-	struct nvgpu_vidmem_buf *buf = dmabuf->priv;
-	struct nvgpu_vidmem_linux *linux_buf = buf->priv;
-	struct gk20a *g = buf->g;
-
-	vidmem_dbg(g, "Releasing Linux VIDMEM buf: dmabuf=0x%p size=%zuKB",
-		   dmabuf, buf->mem->size >> 10);
-
-	if (linux_buf && linux_buf->dmabuf_priv_delete)
-		linux_buf->dmabuf_priv_delete(linux_buf->dmabuf_priv);
-
-	nvgpu_kfree(g, linux_buf);
-	nvgpu_vidmem_buf_free(g, buf);
-
-	nvgpu_put(g);
-}
-
 #if LINUX_VERSION_CODE <= KERNEL_VERSION(5, 5, 0)
 static void *gk20a_vidbuf_kmap(struct dma_buf *dmabuf, unsigned long page_num)
 {
@@ -114,30 +103,22 @@ static int gk20a_vidbuf_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma)
 	return -EINVAL;
 }
 
-#ifdef CONFIG_NVGPU_DMABUF_HAS_DRVDATA
+static void gk20a_vidbuf_release(struct dma_buf *dmabuf)
-static int gk20a_vidbuf_set_private(struct dma_buf *dmabuf,
-		struct device *dev, void *priv, void (*delete)(void *priv))
 {
 	struct nvgpu_vidmem_buf *buf = dmabuf->priv;
 	struct nvgpu_vidmem_linux *linux_buf = buf->priv;
+	struct gk20a *g = buf->g;
 
-	linux_buf->dmabuf_priv = priv;
+	vidmem_dbg(g, "Releasing Linux VIDMEM buf: dmabuf=0x%p size=%zuKB",
-	linux_buf->dmabuf_priv_delete = delete;
+		   dmabuf, buf->mem->size >> 10);
 
-	return 0;
+	nvgpu_kfree(g, linux_buf);
+	nvgpu_vidmem_buf_free(g, buf);
+
+	nvgpu_put(g);
 }
 
-static void *gk20a_vidbuf_get_private(struct dma_buf *dmabuf,
+static struct dma_buf_ops gk20a_vidbuf_ops = {
-		struct device *dev)
-{
-	struct nvgpu_vidmem_buf *buf = dmabuf->priv;
-	struct nvgpu_vidmem_linux *linux_buf = buf->priv;
-
-	return linux_buf->dmabuf_priv;
-}
-#endif
-
-static const struct dma_buf_ops gk20a_vidbuf_ops = {
 	.map_dma_buf      = gk20a_vidbuf_map_dma_buf,
 	.unmap_dma_buf    = gk20a_vidbuf_unmap_dma_buf,
 	.release          = gk20a_vidbuf_release,
@@ -153,10 +134,6 @@ static const struct dma_buf_ops gk20a_vidbuf_ops = {
 	.kmap             = gk20a_vidbuf_kmap,
 #endif
 	.mmap             = gk20a_vidbuf_mmap,
-#ifdef CONFIG_NVGPU_DMABUF_HAS_DRVDATA
-	.set_drvdata      = gk20a_vidbuf_set_private,
-	.get_drvdata      = gk20a_vidbuf_get_private,
-#endif
 };
 
 static struct dma_buf *gk20a_vidbuf_export(struct nvgpu_vidmem_buf *buf)
@@ -167,6 +144,7 @@ static struct dma_buf *gk20a_vidbuf_export(struct nvgpu_vidmem_buf *buf)
 	exp_info.ops = &gk20a_vidbuf_ops;
 	exp_info.size = buf->mem->size;
 	exp_info.flags = O_RDWR;
+	exp_info.exp_name = exporter_name;
 
 	return dma_buf_export(&exp_info);
 }
@@ -175,8 +153,9 @@ struct gk20a *nvgpu_vidmem_buf_owner(struct dma_buf *dmabuf)
 {
 	struct nvgpu_vidmem_buf *buf = dmabuf->priv;
 
-	if (dmabuf->ops != &gk20a_vidbuf_ops)
+	if (dmabuf->exp_name != exporter_name) {
 		return NULL;
+	}
 
 	return buf->g;
 }

drivers/gpu/nvgpu/os/linux/ioctl_ctrl.c

@@ -61,7 +61,6 @@
 
 #include "platform_gk20a.h"
 #include "os_linux.h"
-#include "dmabuf.h"
 #include "channel.h"
 #include "dmabuf_vidmem.h"
 #include "fecs_trace_linux.h"

drivers/gpu/nvgpu/os/linux/linux-channel.c

@@ -33,7 +33,6 @@
 #include "channel.h"
 #include "ioctl_channel.h"
 #include "os_linux.h"
-#include "dmabuf.h"
 #include "dmabuf_priv.h"
 
 #include <nvgpu/hw/gk20a/hw_pbdma_gk20a.h>
@@ -434,14 +433,12 @@ int nvgpu_usermode_buf_from_dmabuf(struct gk20a *g, int dmabuf_fd,
 		goto put_dmabuf;
 	}
 
-#ifdef CONFIG_NVGPU_DMABUF_HAS_DRVDATA
 	err = gk20a_dmabuf_alloc_drvdata(dmabuf, dev);
 	if (err != 0) {
 		goto put_dmabuf;
 	}
-#endif
 
-	sgt = gk20a_mm_pin(dev, dmabuf, &attachment);
+	sgt = nvgpu_mm_pin_privdata(dev, dmabuf, &attachment);
 	if (IS_ERR(sgt)) {
 		nvgpu_warn(g, "Failed to pin dma_buf!");
 		err = PTR_ERR(sgt);
@@ -479,7 +476,7 @@ void nvgpu_os_channel_free_usermode_buffers(struct nvgpu_channel *c)
 	struct device *dev = dev_from_gk20a(g);
 
 	if (priv->usermode.gpfifo.dmabuf != NULL) {
-		gk20a_mm_unpin(dev, priv->usermode.gpfifo.dmabuf,
+		nvgpu_mm_unpin_privdata(dev, priv->usermode.gpfifo.dmabuf,
 			       priv->usermode.gpfifo.attachment,
 			       priv->usermode.gpfifo.sgt);
 		dma_buf_put(priv->usermode.gpfifo.dmabuf);
@@ -487,7 +484,7 @@ void nvgpu_os_channel_free_usermode_buffers(struct nvgpu_channel *c)
 	}
 
 	if (priv->usermode.userd.dmabuf != NULL) {
-		gk20a_mm_unpin(dev, priv->usermode.userd.dmabuf,
+		nvgpu_mm_unpin_privdata(dev, priv->usermode.userd.dmabuf,
 		       priv->usermode.userd.attachment,
 		       priv->usermode.userd.sgt);
 		dma_buf_put(priv->usermode.userd.dmabuf);
@@ -550,7 +547,7 @@ static int nvgpu_channel_alloc_usermode_buffers(struct nvgpu_channel *c,
 unmap_free_gpfifo:
 	nvgpu_dma_unmap_free(c->vm, &c->usermode_gpfifo);
 free_gpfifo:
-	gk20a_mm_unpin(dev, priv->usermode.gpfifo.dmabuf,
+	nvgpu_mm_unpin_privdata(dev, priv->usermode.gpfifo.dmabuf,
 		       priv->usermode.gpfifo.attachment,
 		       priv->usermode.gpfifo.sgt);
 	dma_buf_put(priv->usermode.gpfifo.dmabuf);

drivers/gpu/nvgpu/os/linux/module.c

@@ -78,6 +78,7 @@
 #include "driver_common.h"
 #include "channel.h"
 #include "debug_pmgr.h"
+#include "dmabuf_priv.h"
 
 #ifdef CONFIG_NVGPU_SUPPORT_CDE
 #include "cde.h"
@@ -1713,6 +1714,9 @@ static int gk20a_probe(struct platform_device *dev)
 	if (err)
 		goto return_err;
 
+	nvgpu_mutex_init(&l->dmabuf_priv_list_lock);
+	nvgpu_init_list_node(&l->dmabuf_priv_list);
+
 	return 0;
 
 return_err:
@@ -1793,6 +1797,9 @@ static int __exit gk20a_remove(struct platform_device *pdev)
 
 	err = nvgpu_remove(dev, &nvgpu_class);
 
+	gk20a_dma_buf_priv_list_clear(l);
+	nvgpu_mutex_destroy(&l->dmabuf_priv_list_lock);
+
 	unregister_reboot_notifier(&l->nvgpu_reboot_nb);
 
 	set_gk20a(pdev, NULL);

drivers/gpu/nvgpu/os/linux/os_linux.h

@@ -170,6 +170,9 @@ struct nvgpu_os_linux {
 
 	struct rw_semaphore busy_lock;
 
+	struct nvgpu_mutex dmabuf_priv_list_lock;
+	struct nvgpu_list_node dmabuf_priv_list;
+
 	bool init_done;
 
 	/** Debugfs knob for forcing syncpt support off in runtime. */

drivers/gpu/nvgpu/os/linux/pci.c

@@ -43,6 +43,7 @@
 #include "pci_power.h"
 
 #include "driver_common.h"
+#include "dmabuf_priv.h"
 
 #define BOOT_GPC2CLK_MHZ	2581U
 #define PCI_INTERFACE_NAME	"card-%s%%s"
@@ -684,6 +685,9 @@ static int nvgpu_pci_probe(struct pci_dev *pdev,
 		goto err_free_irq;
 	}
 
+	nvgpu_mutex_init(&l->dmabuf_priv_list_lock);
+	nvgpu_init_list_node(&l->dmabuf_priv_list);
+
 	return 0;
 
 err_free_irq:
@@ -719,11 +723,15 @@ static void nvgpu_pci_remove(struct pci_dev *pdev)
 	struct gk20a *g = get_gk20a(&pdev->dev);
 	struct device *dev = dev_from_gk20a(g);
 	int err;
+	struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
 
 	/* no support yet for unbind if DGPU is in VGPU mode */
 	if (gk20a_gpu_is_virtual(dev))
 		return;
 
+	gk20a_dma_buf_priv_list_clear(l);
+	nvgpu_mutex_destroy(&l->dmabuf_priv_list_lock);
+
 	err = nvgpu_pci_clear_pci_power(dev_name(dev));
 	WARN(err, "gpu failed to clear pci power");
 

drivers/gpu/nvgpu/os/linux/vgpu/vgpu_linux.c

@@ -60,6 +60,7 @@
 #include "os/linux/driver_common.h"
 #include "os/linux/platform_gk20a.h"
 #include "os/linux/vgpu/platform_vgpu_tegra.h"
+#include "os/linux/dmabuf_priv.h"
 
 struct vgpu_priv_data *vgpu_get_priv_data(struct gk20a *g)
 {
@@ -487,6 +488,9 @@ int vgpu_probe(struct platform_device *pdev)
 #endif
 	gk20a->max_comptag_mem = totalram_size_in_mb;
 
+	nvgpu_mutex_init(&l->dmabuf_priv_list_lock);
+	nvgpu_init_list_node(&l->dmabuf_priv_list);
+
 	nvgpu_ref_init(&gk20a->refcount);
 
 	return 0;
@@ -496,9 +500,13 @@ int vgpu_remove(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct gk20a *g = get_gk20a(dev);
+	struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
 
 	nvgpu_log_fn(g, " ");
 
+	gk20a_dma_buf_priv_list_clear(l);
+	nvgpu_mutex_destroy(&l->dmabuf_priv_list_lock);
+
 	vgpu_pm_qos_remove(dev);
 	if (g->remove_support)
 		g->remove_support(g);

drivers/gpu/nvgpu/os/linux/vm.c

@@ -35,7 +35,6 @@
 
 #include "platform_gk20a.h"
 #include "os_linux.h"
-#include "dmabuf.h"
 #include "dmabuf_priv.h"
 #include "dmabuf_vidmem.h"
 
@@ -174,7 +173,7 @@ struct nvgpu_mapped_buf *nvgpu_vm_find_mapping(struct vm_gk20a *vm,
 	 * the dmabuf doesn't support drvdata, prior SGT is unpinned as the
 	 * new SGT was pinned at the beginning of the current map call.
 	 */
-	gk20a_mm_unpin(os_buf->dev, os_buf->dmabuf,
+	nvgpu_mm_unpin_privdata(os_buf->dev, os_buf->dmabuf,
 		       mapped_buffer->os_priv.attachment,
 		       mapped_buffer->os_priv.sgt);
 	dma_buf_put(os_buf->dmabuf);
@@ -204,7 +203,7 @@ int nvgpu_vm_map_linux(struct vm_gk20a *vm,
 	struct dma_buf_attachment *attachment;
 	int err = 0;
 
-	sgt = gk20a_mm_pin(dev, dmabuf, &attachment);
+	sgt = nvgpu_mm_pin_privdata(dev, dmabuf, &attachment);
 	if (IS_ERR(sgt)) {
 		nvgpu_warn(g, "Failed to pin dma_buf!");
 		return PTR_ERR(sgt);
@@ -252,7 +251,7 @@ int nvgpu_vm_map_linux(struct vm_gk20a *vm,
 	return 0;
 
 clean_up:
-	gk20a_mm_unpin(dev, dmabuf, attachment, sgt);
+	nvgpu_mm_unpin_privdata(dev, dmabuf, attachment, sgt);
 
 	return err;
 }
@@ -319,13 +318,11 @@ int nvgpu_vm_map_buffer(struct vm_gk20a *vm,
 		return -EINVAL;
 	}
 
-#ifdef CONFIG_NVGPU_DMABUF_HAS_DRVDATA
 	err = gk20a_dmabuf_alloc_drvdata(dmabuf, dev_from_vm(vm));
 	if (err) {
 		dma_buf_put(dmabuf);
 		return err;
 	}
-#endif
 
 	err = nvgpu_vm_map_linux(vm, dmabuf, *map_addr,
 				 nvgpu_vm_translate_linux_flags(g, flags),
@@ -356,7 +353,7 @@ void nvgpu_vm_unmap_system(struct nvgpu_mapped_buf *mapped_buffer)
 {
 	struct vm_gk20a *vm = mapped_buffer->vm;
 
-	gk20a_mm_unpin(dev_from_vm(vm), mapped_buffer->os_priv.dmabuf,
+	nvgpu_mm_unpin_privdata(dev_from_vm(vm), mapped_buffer->os_priv.dmabuf,
 		       mapped_buffer->os_priv.attachment,
 		       mapped_buffer->os_priv.sgt);