From 9207ceb10a0c661c31534f52279f20e9fc1720f8 Mon Sep 17 00:00:00 2001 From: Ashish Mhetre Date: Fri, 23 Jun 2023 17:34:21 +0530 Subject: [PATCH] video: tegra: nvmap: Add support for sgt caching Deferred dmabuf unmapping is being removed from kernel. So, add similar support to cache sgt in NvMap. During map_dma_buf() call, NvMap will create a mapping and an sgt corresponding to it. It will also cache this sgt. When unmap_dma_buf() is called for same sgt, NvMap will not unmap the mappings. It will simply return from there. Next time when the mapping request comes for same dmabuf, it will look for existing sgt in cache and return it. This significantly reduces mapping overhead for same buffer when it's mapped and unmapped multiple times. Free the sgt and unmap only when corresponding buffer is freed. When all references from a buffer are removed, dmabuf_release() will be called where sgt will be freed. Bug 4064339 Change-Id: I7ed767ecaaac7aa44e6576e701b28537b84986ec Signed-off-by: Ashish Mhetre Reviewed-on: https://git-master.nvidia.com/r/c/linux-nv-oot/+/2925224 Reviewed-by: svcacv Reviewed-by: Sachin Nikam GVS: Gerrit_Virtual_Submit --- drivers/video/tegra/nvmap/nvmap_dev.c | 5 + drivers/video/tegra/nvmap/nvmap_dmabuf.c | 124 ++++++++++++++++++++--- drivers/video/tegra/nvmap/nvmap_priv.h | 1 + 3 files changed, 117 insertions(+), 13 deletions(-) diff --git a/drivers/video/tegra/nvmap/nvmap_dev.c b/drivers/video/tegra/nvmap/nvmap_dev.c index 26dc96dd..57feeda4 100644 --- a/drivers/video/tegra/nvmap/nvmap_dev.c +++ b/drivers/video/tegra/nvmap/nvmap_dev.c @@ -1464,6 +1464,10 @@ int __init nvmap_probe(struct platform_device *pdev) nvmap_stats_init(nvmap_debug_root); platform_set_drvdata(pdev, dev); + e = nvmap_dmabuf_stash_init(); + if (e) + goto fail_heaps; + for (i = 0; i < dev->nr_carveouts; i++) if (dev->heaps[i].heap_bit & NVMAP_HEAP_CARVEOUT_GENERIC) generic_carveout_present = 1; @@ -1526,6 +1530,7 @@ int nvmap_remove(struct platform_device *pdev) #ifdef NVMAP_CONFIG_SCIIPC nvmap_sci_ipc_exit(); #endif + nvmap_dmabuf_stash_deinit(); debugfs_remove_recursive(dev->debug_root); misc_deregister(&dev->dev_user); #ifdef NVMAP_CONFIG_PAGE_POOLS diff --git a/drivers/video/tegra/nvmap/nvmap_dmabuf.c b/drivers/video/tegra/nvmap/nvmap_dmabuf.c index a12b5c61..8159ba20 100644 --- a/drivers/video/tegra/nvmap/nvmap_dmabuf.c +++ b/drivers/video/tegra/nvmap/nvmap_dmabuf.c @@ -37,6 +37,35 @@ #define NVMAP_DMABUF_ATTACH __nvmap_dmabuf_attach #endif +struct nvmap_handle_sgt { + enum dma_data_direction dir; + struct sg_table *sgt; + struct device *dev; + struct list_head maps_entry; + struct nvmap_handle_info *owner; +} ____cacheline_aligned_in_smp; + +static struct kmem_cache *handle_sgt_cache; + +/* + * Initialize a kmem cache for allocating nvmap_handle_sgt's. + */ +int nvmap_dmabuf_stash_init(void) +{ + handle_sgt_cache = KMEM_CACHE(nvmap_handle_sgt, 0); + if (IS_ERR_OR_NULL(handle_sgt_cache)) { + pr_err("Failed to make kmem cache for nvmap_handle_sgt.\n"); + return -ENOMEM; + } + + return 0; +} + +void nvmap_dmabuf_stash_deinit(void) +{ + kmem_cache_destroy(handle_sgt_cache); +} + static int __nvmap_dmabuf_attach(struct dma_buf *dmabuf, struct device *dev, struct dma_buf_attachment *attach) { @@ -81,12 +110,53 @@ static inline bool access_vpr_phys(struct device *dev) return !!of_find_property(dev->of_node, "access-vpr-phys", NULL); } +static int nvmap_dmabuf_stash_sgt_locked(struct dma_buf_attachment *attach, + enum dma_data_direction dir, + struct sg_table *sgt) +{ + struct nvmap_handle_sgt *nvmap_sgt; + struct nvmap_handle_info *info = attach->dmabuf->priv; + + nvmap_sgt = kmem_cache_alloc(handle_sgt_cache, GFP_KERNEL); + if (IS_ERR_OR_NULL(nvmap_sgt)) { + pr_err("Stashing SGT failed.\n"); + return -ENOMEM; + } + + nvmap_sgt->dir = dir; + nvmap_sgt->sgt = sgt; + nvmap_sgt->dev = attach->dev; + nvmap_sgt->owner = info; + list_add(&nvmap_sgt->maps_entry, &info->maps); + + return 0; +} + +static struct sg_table *nvmap_dmabuf_get_sgt_from_stash(struct dma_buf_attachment *attach, + enum dma_data_direction dir) +{ + struct nvmap_handle_info *info = attach->dmabuf->priv; + struct nvmap_handle_sgt *nvmap_sgt; + struct sg_table *sgt = NULL; + + list_for_each_entry(nvmap_sgt, &info->maps, maps_entry) { + if (nvmap_sgt->dir != dir || nvmap_sgt->dev != attach->dev) + continue; + + /* found sgt in stash */ + sgt = nvmap_sgt->sgt; + break; + } + + return sgt; +} + static struct sg_table *nvmap_dmabuf_map_dma_buf(struct dma_buf_attachment *attach, enum dma_data_direction dir) { struct nvmap_handle_info *info = attach->dmabuf->priv; int ents = 0; - struct sg_table *sgt; + struct sg_table *sgt = NULL; #ifdef NVMAP_CONFIG_DEBUG_MAPS char *device_name = NULL; u32 heap_type; @@ -110,6 +180,10 @@ static struct sg_table *nvmap_dmabuf_map_dma_buf(struct dma_buf_attachment *atta atomic_inc(&info->handle->pin); + sgt = nvmap_dmabuf_get_sgt_from_stash(attach, dir); + if (sgt) + goto cache_hit; + sgt = __nvmap_sg_table(NULL, info->handle); if (IS_ERR(sgt)) { atomic_dec(&info->handle->pin); @@ -133,6 +207,10 @@ static struct sg_table *nvmap_dmabuf_map_dma_buf(struct dma_buf_attachment *atta goto err_map; } + if (nvmap_dmabuf_stash_sgt_locked(attach, dir, sgt)) + WARN(1, "No mem to prep sgt.\n"); + +cache_hit: attach->priv = sgt; #ifdef NVMAP_CONFIG_DEBUG_MAPS @@ -155,6 +233,26 @@ err_map: return ERR_PTR(-ENOMEM); } +static void __nvmap_dmabuf_unmap_dma_buf(struct nvmap_handle_sgt *nvmap_sgt) +{ + struct nvmap_handle_info *info = nvmap_sgt->owner; + enum dma_data_direction dir = nvmap_sgt->dir; + struct sg_table *sgt = nvmap_sgt->sgt; + struct device *dev = nvmap_sgt->dev; + + if (!(nvmap_dev->dynamic_dma_map_mask & info->handle->heap_type)) { + sg_dma_address(sgt->sgl) = 0; + } else if (info->handle->heap_type == NVMAP_HEAP_CARVEOUT_VPR && + access_vpr_phys(dev)) { + sg_dma_address(sgt->sgl) = 0; + } else { + dma_unmap_sg_attrs(dev, + sgt->sgl, sgt->nents, + dir, DMA_ATTR_SKIP_CPU_SYNC); + } + __nvmap_free_sg_table(NULL, info->handle, sgt); +} + static void nvmap_dmabuf_unmap_dma_buf(struct dma_buf_attachment *attach, struct sg_table *sgt, enum dma_data_direction dir) @@ -174,18 +272,6 @@ static void nvmap_dmabuf_unmap_dma_buf(struct dma_buf_attachment *attach, return; } - if (!(nvmap_dev->dynamic_dma_map_mask & info->handle->heap_type)) { - sg_dma_address(sgt->sgl) = 0; - } else if (info->handle->heap_type == NVMAP_HEAP_CARVEOUT_VPR && - access_vpr_phys(attach->dev)) { - sg_dma_address(sgt->sgl) = 0; - } else { - dma_unmap_sg_attrs(attach->dev, - sgt->sgl, sgt->nents, - dir, DMA_ATTR_SKIP_CPU_SYNC); - } - __nvmap_free_sg_table(NULL, info->handle, sgt); - #ifdef NVMAP_CONFIG_DEBUG_MAPS /* Remove the device name from the list of carveout accessing devices */ heap_type = info->handle->heap_type; @@ -199,12 +285,24 @@ static void nvmap_dmabuf_unmap_dma_buf(struct dma_buf_attachment *attach, static void nvmap_dmabuf_release(struct dma_buf *dmabuf) { struct nvmap_handle_info *info = dmabuf->priv; + struct nvmap_handle_sgt *nvmap_sgt; trace_nvmap_dmabuf_release(info->handle->owner ? info->handle->owner->name : "unknown", info->handle, dmabuf); + mutex_lock(&info->maps_lock); + while (!list_empty(&info->maps)) { + nvmap_sgt = list_first_entry(&info->maps, + struct nvmap_handle_sgt, + maps_entry); + __nvmap_dmabuf_unmap_dma_buf(nvmap_sgt); + list_del(&nvmap_sgt->maps_entry); + kmem_cache_free(handle_sgt_cache, nvmap_sgt); + } + mutex_unlock(&info->maps_lock); + mutex_lock(&info->handle->lock); if (info->is_ro) { BUG_ON(dmabuf != info->handle->dmabuf_ro); diff --git a/drivers/video/tegra/nvmap/nvmap_priv.h b/drivers/video/tegra/nvmap/nvmap_priv.h index f524113d..7c577e1b 100644 --- a/drivers/video/tegra/nvmap/nvmap_priv.h +++ b/drivers/video/tegra/nvmap/nvmap_priv.h @@ -621,6 +621,7 @@ int __nvmap_dmabuf_fd(struct nvmap_client *client, struct dma_buf *dmabuf, int flags); int nvmap_dmabuf_stash_init(void); +void nvmap_dmabuf_stash_deinit(void); void *nvmap_altalloc(size_t len); void nvmap_altfree(void *ptr, size_t len);