drm/tegra: Implement correct DMA-BUF semantics

DMA-BUF requires that each device that accesses a DMA-BUF attaches to it
separately. To do so the host1x_bo_pin() and host1x_bo_unpin() functions
need to be reimplemented so that they can return a mapping, which either
represents an attachment or a map of the driver's own GEM object.

Bug 200768479

Signed-off-by: Thierry Reding <treding@nvidia.com>
Change-Id: Ia380b7dcc371ce47f5f35d44a60fbd6b4ab9d636
Reviewed-on: https://git-master.nvidia.com/r/c/linux-nvidia/+/2596398
(cherry picked from commit 28960586000fca025689edfd45645ab28e497bca)
Reviewed-on: https://git-master.nvidia.com/r/c/linux-nvidia/+/2620137
Tested-by: Jonathan Hunter <jonathanh@nvidia.com>
Tested-by: mobile promotions <svcmobile_promotions@nvidia.com>
Reviewed-by: Jonathan Hunter <jonathanh@nvidia.com>
Reviewed-by: svc_kernel_abi <svc_kernel_abi@nvidia.com>
Reviewed-by: Mikko Perttunen <mperttunen@nvidia.com>
Reviewed-by: mobile promotions <svcmobile_promotions@nvidia.com>
GVS: Gerrit_Virtual_Submit

drivers/gpu/drm/tegra/gem.c

@@ -21,6 +21,26 @@
 #include "drm.h"
 #include "gem.h"
 
+static unsigned int __sgt_dma_count_chunks(struct sg_table *sgt)
+{
+	dma_addr_t next = ~(dma_addr_t)0;
+	unsigned int count = 0, i;
+	struct scatterlist *s;
+
+	for_each_sg(sgt->sgl, s, sgt->nents, i) {
+		/* sg_dma_address(s) is only valid for entries that have sg_dma_len(s) != 0. */
+		if (!sg_dma_len(s))
+			continue;
+
+		if (sg_dma_address(s) != next) {
+			next = sg_dma_address(s) + sg_dma_len(s);
+			count++;
+		}
+	}
+
+	return count;
+}
+
 static void tegra_bo_put(struct host1x_bo *bo)
 {
 	struct tegra_bo *obj = host1x_to_tegra_bo(bo);
@@ -28,79 +48,64 @@ static void tegra_bo_put(struct host1x_bo *bo)
 	drm_gem_object_put(&obj->gem);
 }
 
-/* XXX move this into lib/scatterlist.c? */
+static struct host1x_bo_mapping *tegra_bo_pin(struct device *dev, struct host1x_bo *bo,
-static int sg_alloc_table_from_sg(struct sg_table *sgt, struct scatterlist *sg,
+					      enum dma_data_direction direction)
-				  unsigned int nents, gfp_t gfp_mask)
-{
-	struct scatterlist *dst;
-	unsigned int i;
-	int err;
-
-	err = sg_alloc_table(sgt, nents, gfp_mask);
-	if (err < 0)
-		return err;
-
-	dst = sgt->sgl;
-
-	for (i = 0; i < nents; i++) {
-		sg_set_page(dst, sg_page(sg), sg->length, 0);
-		dst = sg_next(dst);
-		sg = sg_next(sg);
-	}
-
-	return 0;
-}
-
-static struct sg_table *tegra_bo_pin(struct device *dev, struct host1x_bo *bo,
-				     dma_addr_t *phys)
 {
 	struct tegra_bo *obj = host1x_to_tegra_bo(bo);
-	struct sg_table *sgt;
+	struct drm_gem_object *gem = &obj->gem;
+	struct host1x_bo_mapping *map;
 	int err;
 
+	map = kzalloc(sizeof(*map), GFP_KERNEL);
+	if (!map)
+		return ERR_PTR(-ENOMEM);
+
+	map->bo = host1x_bo_get(bo);
+	map->direction = direction;
+	map->dev = dev;
+
 	/*
-	 * If we've manually mapped the buffer object through the IOMMU, make
+	 * Imported buffers need special treatment to satisfy the semantics of DMA-BUF.
-	 * sure to return the IOVA address of our mapping.
-	 *
-	 * Similarly, for buffers that have been allocated by the DMA API the
-	 * physical address can be used for devices that are not attached to
-	 * an IOMMU. For these devices, callers must pass a valid pointer via
-	 * the @phys argument.
-	 *
-	 * Imported buffers were also already mapped at import time, so the
-	 * existing mapping can be reused.
 	 */
-	if (phys) {
+	if (gem->import_attach) {
-		*phys = obj->iova;
+		struct dma_buf *buf = gem->import_attach->dmabuf;
-		return NULL;
+
+		map->attach = dma_buf_attach(buf, dev);
+		if (IS_ERR(map->attach)) {
+			err = PTR_ERR(map->attach);
+			goto free;
+		}
+
+		map->sgt = dma_buf_map_attachment(map->attach, direction);
+		if (IS_ERR(map->sgt)) {
+			dma_buf_detach(buf, map->attach);
+			err = PTR_ERR(map->sgt);
+			goto free;
+		}
+
+		err = __sgt_dma_count_chunks(map->sgt);
+		map->size = gem->size;
+
+		goto out;
 	}
 
 	/*
 	 * If we don't have a mapping for this buffer yet, return an SG table
 	 * so that host1x can do the mapping for us via the DMA API.
 	 */
-	sgt = kzalloc(sizeof(*sgt), GFP_KERNEL);
+	map->sgt = kzalloc(sizeof(*map->sgt), GFP_KERNEL);
-	if (!sgt)
+	if (!map->sgt) {
-		return ERR_PTR(-ENOMEM);
+		err = -ENOMEM;
+		goto free;
+	}
 
 	if (obj->pages) {
 		/*
 		 * If the buffer object was allocated from the explicit IOMMU
 		 * API code paths, construct an SG table from the pages.
 		 */
-		err = sg_alloc_table_from_pages(sgt, obj->pages, obj->num_pages,
+		err = sg_alloc_table_from_pages(map->sgt, obj->pages, obj->num_pages, 0, gem->size,
-						0, obj->gem.size, GFP_KERNEL);
+						GFP_KERNEL);
-		if (err < 0)
-			goto free;
-	} else if (obj->sgt) {
-		/*
-		 * If the buffer object already has an SG table but no pages
-		 * were allocated for it, it means the buffer was imported and
-		 * the SG table needs to be copied to avoid overwriting any
-		 * other potential users of the original SG table.
-		 */
-		err = sg_alloc_table_from_sg(sgt, obj->sgt->sgl,
-					     obj->sgt->orig_nents, GFP_KERNEL);
 		if (err < 0)
 			goto free;
 	} else {
@@ -109,25 +114,56 @@ static struct sg_table *tegra_bo_pin(struct device *dev, struct host1x_bo *bo,
 		 * not imported, it had to be allocated with the DMA API, so
 		 * the DMA API helper can be used.
 		 */
-		err = dma_get_sgtable(dev, sgt, obj->vaddr, obj->iova,
+		err = dma_get_sgtable(dev, map->sgt, obj->vaddr, obj->iova, gem->size);
-				      obj->gem.size);
 		if (err < 0)
 			goto free;
 	}
 
-	return sgt;
+	err = dma_map_sgtable(dev, map->sgt, direction, 0);
+	if (err)
+		goto free_sgt;
 
+out:
+	/*
+	 * If we've manually mapped the buffer object through the IOMMU, make sure to return the
+	 * existing IOVA address of our mapping.
+	 */
+	if (!obj->mm) {
+		map->phys = sg_dma_address(map->sgt->sgl);
+		map->chunks = err;
+	} else {
+		map->phys = obj->iova;
+		map->chunks = 1;
+	}
+
+	map->size = gem->size;
+
+	return map;
+
+free_sgt:
+	sg_free_table(map->sgt);
 free:
-	kfree(sgt);
+	kfree(map->sgt);
+	kfree(map);
 	return ERR_PTR(err);
 }
 
-static void tegra_bo_unpin(struct device *dev, struct sg_table *sgt)
+static void tegra_bo_unpin(struct host1x_bo_mapping *map)
 {
-	if (sgt) {
+	if (!map)
-		sg_free_table(sgt);
+		return;
-		kfree(sgt);
+
+	if (map->attach) {
+		dma_buf_unmap_attachment(map->attach, map->sgt, map->direction);
+		dma_buf_detach(map->attach->dmabuf, map->attach);
+	} else {
+		dma_unmap_sgtable(map->dev, map->sgt, map->direction, 0);
+		sg_free_table(map->sgt);
+		kfree(map->sgt);
 	}
+
+	host1x_bo_put(map->bo);
+	kfree(map);
 }
 
 static void *tegra_bo_mmap(struct host1x_bo *bo)

drivers/gpu/drm/tegra/plane.c

@@ -75,7 +75,7 @@ tegra_plane_atomic_duplicate_state(struct drm_plane *plane)
 
 	for (i = 0; i < 3; i++) {
 		copy->iova[i] = DMA_MAPPING_ERROR;
-		copy->sgt[i] = NULL;
+		copy->map[i] = NULL;
 	}
 
 	return &copy->base;
@@ -139,47 +139,37 @@ const struct drm_plane_funcs tegra_plane_funcs = {
 
 static int tegra_dc_pin(struct tegra_dc *dc, struct tegra_plane_state *state)
 {
-	struct iommu_domain *domain = iommu_get_domain_for_dev(dc->dev);
 	unsigned int i;
 	int err;
 
 	for (i = 0; i < state->base.fb->format->num_planes; i++) {
 		struct tegra_bo *bo = tegra_fb_get_plane(state->base.fb, i);
-		dma_addr_t phys_addr, *phys;
+		struct host1x_bo_mapping *map;
-		struct sg_table *sgt;
 
-		if (!domain || dc->client.group)
+		map = host1x_bo_pin(dc->dev, &bo->base, DMA_TO_DEVICE);
-			phys = &phys_addr;
+		if (IS_ERR(map)) {
-		else
+			err = PTR_ERR(map);
-			phys = NULL;
-
-		sgt = host1x_bo_pin(dc->dev, &bo->base, phys);
-		if (IS_ERR(sgt)) {
-			err = PTR_ERR(sgt);
 			goto unpin;
 		}
 
-		if (sgt) {
+		if (!dc->client.group) {
-			err = dma_map_sgtable(dc->dev, sgt, DMA_TO_DEVICE, 0);
-			if (err)
-				goto unpin;
-
 			/*
 			 * The display controller needs contiguous memory, so
 			 * fail if the buffer is discontiguous and we fail to
 			 * map its SG table to a single contiguous chunk of
 			 * I/O virtual memory.
 			 */
-			if (sgt->nents > 1) {
+			if (map->chunks > 1) {
 				err = -EINVAL;
 				goto unpin;
 			}
 
-			state->iova[i] = sg_dma_address(sgt->sgl);
+			state->iova[i] = map->phys;
-			state->sgt[i] = sgt;
 		} else {
-			state->iova[i] = phys_addr;
+			state->iova[i] = bo->iova;
 		}
+
+		state->map[i] = map;
 	}
 
 	return 0;
@@ -188,15 +178,9 @@ unpin:
 	dev_err(dc->dev, "failed to map plane %u: %d\n", i, err);
 
 	while (i--) {
-		struct tegra_bo *bo = tegra_fb_get_plane(state->base.fb, i);
+		host1x_bo_unpin(state->map[i]);
-		struct sg_table *sgt = state->sgt[i];
-
-		if (sgt)
-			dma_unmap_sgtable(dc->dev, sgt, DMA_TO_DEVICE, 0);
-
-		host1x_bo_unpin(dc->dev, &bo->base, sgt);
 		state->iova[i] = DMA_MAPPING_ERROR;
-		state->sgt[i] = NULL;
+		state->map[i] = NULL;
 	}
 
 	return err;
@@ -207,15 +191,9 @@ static void tegra_dc_unpin(struct tegra_dc *dc, struct tegra_plane_state *state)
 	unsigned int i;
 
 	for (i = 0; i < state->base.fb->format->num_planes; i++) {
-		struct tegra_bo *bo = tegra_fb_get_plane(state->base.fb, i);
+		host1x_bo_unpin(state->map[i]);
-		struct sg_table *sgt = state->sgt[i];
-
-		if (sgt)
-			dma_unmap_sgtable(dc->dev, sgt, DMA_TO_DEVICE, 0);
-
-		host1x_bo_unpin(dc->dev, &bo->base, sgt);
 		state->iova[i] = DMA_MAPPING_ERROR;
-		state->sgt[i] = NULL;
+		state->map[i] = NULL;
 	}
 }
 

drivers/gpu/drm/tegra/plane.h

@@ -39,7 +39,7 @@ struct tegra_plane_legacy_blending_state {
 struct tegra_plane_state {
 	struct drm_plane_state base;
 
-	struct sg_table *sgt[3];
+	struct host1x_bo_mapping *map[3];
 	dma_addr_t iova[3];
 
 	struct tegra_bo_tiling tiling;

drivers/gpu/drm/tegra/uapi.h

@@ -32,10 +32,9 @@ struct tegra_drm_channel_ctx {
 struct tegra_drm_mapping {
 	struct kref ref;
 
-	struct device *dev;
+	struct host1x_bo_mapping *map;
 	struct host1x_bo *bo;
-	struct sg_table *sgt;
+
-	enum dma_data_direction direction;
 	dma_addr_t iova;
 	dma_addr_t iova_end;
 };

drivers/gpu/drm/tegra/uapi/gather_bo.c

@@ -32,34 +32,61 @@ void gather_bo_put(struct host1x_bo *host_bo)
 	kref_put(&bo->ref, gather_bo_release);
 }
 
-static struct sg_table *
+static struct host1x_bo_mapping *
-gather_bo_pin(struct device *dev, struct host1x_bo *host_bo, dma_addr_t *phys)
+gather_bo_pin(struct device *dev, struct host1x_bo *bo, enum dma_data_direction direction)
 {
-	struct gather_bo *bo = container_of(host_bo, struct gather_bo, base);
+	struct gather_bo *gather = container_of(bo, struct gather_bo, base);
-	struct sg_table *sgt;
+	struct host1x_bo_mapping *map;
 	int err;
 
-	sgt = kzalloc(sizeof(*sgt), GFP_KERNEL);
+	map = kzalloc(sizeof(*map), GFP_KERNEL);
-	if (!sgt)
+	if (!map)
 		return ERR_PTR(-ENOMEM);
 
-	err = sg_alloc_table(sgt, 1, GFP_KERNEL);
+	map->bo = host1x_bo_get(bo);
-	if (err) {
+	map->direction = direction;
-		kfree(sgt);
+	map->dev = dev;
-		return ERR_PTR(err);
+
+	map->sgt = kzalloc(sizeof(*map->sgt), GFP_KERNEL);
+	if (!map->sgt) {
+		err = -ENOMEM;
+		goto free;
 	}
 
-	sg_init_one(sgt->sgl, bo->gather_data, bo->gather_data_words*4);
+	err = dma_get_sgtable(gather->dev, map->sgt, gather->gather_data, gather->gather_data_dma,
+			      gather->gather_data_words * 4);
+	if (err)
+		goto free_sgt;
 
-	return sgt;
+	err = dma_map_sgtable(dev, map->sgt, direction, 0);
+	if (err)
+		goto free_sgt;
+
+	map->phys = sg_dma_address(map->sgt->sgl);
+	map->size = gather->gather_data_words * 4;
+	map->chunks = err;
+
+	return map;
+
+free_sgt:
+	sg_free_table(map->sgt);
+	kfree(map->sgt);
+free:
+	kfree(map);
+	return ERR_PTR(err);
 }
 
-static void gather_bo_unpin(struct device *dev, struct sg_table *sgt)
+static void gather_bo_unpin(struct host1x_bo_mapping *map)
 {
-	if (sgt) {
+	if (!map)
-		sg_free_table(sgt);
+		return;
-		kfree(sgt);
+
-	}
+	dma_unmap_sgtable(map->dev, map->sgt, map->direction, 0);
+	sg_free_table(map->sgt);
+	kfree(map->sgt);
+	host1x_bo_put(map->bo);
+
+	kfree(map);
 }
 
 static void *gather_bo_mmap(struct host1x_bo *host_bo)

drivers/gpu/drm/tegra/uapi/uapi.c

@@ -29,11 +29,7 @@ static void tegra_drm_mapping_release(struct kref *ref)
 	struct tegra_drm_mapping *mapping =
 		container_of(ref, struct tegra_drm_mapping, ref);
 
-	if (mapping->sgt)
+	host1x_bo_unpin(mapping->map);
-		dma_unmap_sgtable(mapping->dev, mapping->sgt,
-				  mapping->direction, DMA_ATTR_SKIP_CPU_SYNC);
-
-	host1x_bo_unpin(mapping->dev, mapping->bo, mapping->sgt);
 	host1x_bo_put(mapping->bo);
 
 	kfree(mapping);
@@ -163,6 +159,7 @@ int tegra_drm_ioctl_channel_map(struct drm_device *drm, void *data,
 	struct drm_tegra_channel_map *args = data;
 	struct tegra_drm_channel_ctx *ctx;
 	struct tegra_drm_mapping *mapping;
+	enum dma_data_direction direction;
 	struct drm_gem_object *gem;
 	u32 mapping_id;
 	int err = 0;
@@ -185,39 +182,23 @@ int tegra_drm_ioctl_channel_map(struct drm_device *drm, void *data,
 	gem = drm_gem_object_lookup(file, args->handle);
 	if (!gem) {
 		err = -EINVAL;
-		goto unlock;
+		goto free;
 	}
 
-	mapping->dev = ctx->client->base.dev;
 	mapping->bo = &container_of(gem, struct tegra_bo, gem)->base;
 
-	if (ctx->client->base.group) {
-		/* Domain managed directly */
-		host1x_bo_pin(mapping->dev, mapping->bo,
-			      &mapping->iova);
-	} else {
-		/* No IOMMU or DMA API managed domain */
-		mapping->direction = DMA_TO_DEVICE;
 	if (args->flags & DRM_TEGRA_CHANNEL_MAP_READWRITE)
-			mapping->direction = DMA_BIDIRECTIONAL;
+		direction = DMA_BIDIRECTIONAL;
+	else
+		direction = DMA_TO_DEVICE;
 
-		mapping->sgt =
+	mapping->map = host1x_bo_pin(ctx->client->base.dev, mapping->bo, direction);
-			host1x_bo_pin(mapping->dev, mapping->bo, NULL);
+	if (IS_ERR(mapping->map)) {
-		if (IS_ERR(mapping->sgt)) {
+		err = PTR_ERR(mapping->map);
-			err = PTR_ERR(mapping->sgt);
 		goto put_gem;
 	}
 
-		err = dma_map_sgtable(mapping->dev, mapping->sgt,
+	mapping->iova = mapping->map->phys;
-				      mapping->direction,
-				      DMA_ATTR_SKIP_CPU_SYNC);
-		if (err)
-			goto unpin;
-
-		/* TODO only map the requested part */
-		mapping->iova = sg_dma_address(mapping->sgt->sgl);
-	}
-
 	mapping->iova_end = mapping->iova + gem->size;
 
 	mutex_unlock(&fpriv->lock);
@@ -225,21 +206,17 @@ int tegra_drm_ioctl_channel_map(struct drm_device *drm, void *data,
 	err = xa_alloc(&ctx->mappings, &mapping_id, mapping,
 		       XA_LIMIT(1, U32_MAX), GFP_KERNEL);
 	if (err < 0)
-		goto unmap;
+		goto unpin;
 
 	args->mapping_id = mapping_id;
 
 	return 0;
 
-unmap:
-	if (mapping->sgt) {
-		dma_unmap_sgtable(mapping->dev, mapping->sgt,
-				  mapping->direction, DMA_ATTR_SKIP_CPU_SYNC);
-	}
 unpin:
-	host1x_bo_unpin(mapping->dev, mapping->bo, mapping->sgt);
+	host1x_bo_unpin(mapping->map);
 put_gem:
 	drm_gem_object_put(gem);
+free:
 	kfree(mapping);
 unlock:
 	mutex_unlock(&fpriv->lock);