diff --git a/drivers/video/tegra/nvmap/nvmap_alloc.c b/drivers/video/tegra/nvmap/nvmap_alloc.c index daf38d6b..fc590000 100644 --- a/drivers/video/tegra/nvmap/nvmap_alloc.c +++ b/drivers/video/tegra/nvmap/nvmap_alloc.c @@ -3,7 +3,7 @@ * * Handle allocation and freeing routines for nvmap * - * Copyright (c) 2011-2022, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, @@ -718,16 +718,42 @@ static void alloc_handle(struct nvmap_client *client, if (nvmap_cpu_map_is_allowed(h)) { void *cpu_addr; - cpu_addr = memremap(b->base, h->size, - MEMREMAP_WB); - if (cpu_addr != NULL) { - memset(cpu_addr, 0, h->size); + if (h->pgalloc.pages && + h->heap_type == NVMAP_HEAP_CARVEOUT_CBC) { + unsigned long page_count; + int i; + + page_count = h->size >> PAGE_SHIFT; + /* Iterate over 2MB chunks */ + for (i = 0; i < page_count; i += PAGES_PER_2MB) { + cpu_addr = memremap(page_to_phys( + h->pgalloc.pages[i]), + SIZE_2MB, MEMREMAP_WB); + if (cpu_addr != NULL) { + memset(cpu_addr, 0, SIZE_2MB); #ifdef NVMAP_UPSTREAM_KERNEL - arch_invalidate_pmem(cpu_addr, h->size); + arch_invalidate_pmem(cpu_addr, + SIZE_2MB); #else - __dma_flush_area(cpu_addr, h->size); + __dma_flush_area(cpu_addr, + SIZE_2MB); #endif - memunmap(cpu_addr); + memunmap(cpu_addr); + } + + } + } else { + cpu_addr = memremap(b->base, h->size, + MEMREMAP_WB); + if (cpu_addr != NULL) { + memset(cpu_addr, 0, h->size); +#ifdef NVMAP_UPSTREAM_KERNEL + arch_invalidate_pmem(cpu_addr, h->size); +#else + __dma_flush_area(cpu_addr, h->size); +#endif + memunmap(cpu_addr); + } } } } @@ -1006,14 +1032,19 @@ void _nvmap_handle_free(struct nvmap_handle *h) if (h->vaddr) { void *addr = h->vaddr; - addr -= (h->carveout->base & ~PAGE_MASK); - iounmap((void __iomem *)addr); + if (h->pgalloc.pages) { + vunmap(h->vaddr); + } else { + addr -= (h->carveout->base & ~PAGE_MASK); + iounmap((void __iomem *)addr); + } } nvmap_heap_free(h->carveout); nvmap_kmaps_dec(h); h->carveout = NULL; h->vaddr = NULL; + h->pgalloc.pages = NULL; goto out; } else { int ret = nvmap_heap_pgfree(h); diff --git a/drivers/video/tegra/nvmap/nvmap_cache.c b/drivers/video/tegra/nvmap/nvmap_cache.c index 453de77d..95702fdf 100644 --- a/drivers/video/tegra/nvmap/nvmap_cache.c +++ b/drivers/video/tegra/nvmap/nvmap_cache.c @@ -1,7 +1,7 @@ /* * drivers/video/tegra/nvmap/nvmap_cache.c * - * Copyright (c) 2011-2022, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, @@ -197,7 +197,7 @@ static int do_cache_maint(struct cache_maint_op *cache_work) goto out; } - if (h->heap_pgalloc) { + if (h->pgalloc.pages) { heap_page_cache_maint(h, pstart, pend, op, true, (h->flags == NVMAP_HANDLE_INNER_CACHEABLE) ? false : true, cache_work->clean_only_dirty); diff --git a/drivers/video/tegra/nvmap/nvmap_core.c b/drivers/video/tegra/nvmap/nvmap_core.c index c34afeea..458e6c8c 100644 --- a/drivers/video/tegra/nvmap/nvmap_core.c +++ b/drivers/video/tegra/nvmap/nvmap_core.c @@ -3,7 +3,7 @@ * * Memory manager for Tegra GPU * - * Copyright (c) 2009-2022, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2009-2023, NVIDIA CORPORATION. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, @@ -170,7 +170,7 @@ void *__nvmap_mmap(struct nvmap_handle *h) nvmap_kmaps_inc(h); prot = nvmap_pgprot(h, PG_PROT_KERNEL); - if (h->heap_pgalloc) { + if (h->pgalloc.pages) { pages = nvmap_pages(h->pgalloc.pages, h->size >> PAGE_SHIFT); if (!pages) goto out; @@ -327,7 +327,7 @@ struct sg_table *__nvmap_sg_table(struct nvmap_client *client, goto err; } - if (!h->heap_pgalloc) { + if (!h->pgalloc.pages) { phys_addr_t paddr = handle_phys(h); struct page *page = phys_to_page(paddr); diff --git a/drivers/video/tegra/nvmap/nvmap_fault.c b/drivers/video/tegra/nvmap/nvmap_fault.c index dacf46db..7a0ead22 100644 --- a/drivers/video/tegra/nvmap/nvmap_fault.c +++ b/drivers/video/tegra/nvmap/nvmap_fault.c @@ -1,7 +1,7 @@ /* * drivers/video/tegra/nvmap/nvmap_fault.c * - * Copyright (c) 2011-2022, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, @@ -207,8 +207,9 @@ static int nvmap_vma_fault(struct vm_area_struct *vma, struct vm_fault *vmf) if (offs >= priv->handle->size) return VM_FAULT_SIGBUS; - if (!priv->handle->heap_pgalloc) { + if (!priv->handle->pgalloc.pages) { unsigned long pfn; + BUG_ON(priv->handle->carveout->base & ~PAGE_MASK); pfn = ((priv->handle->carveout->base + offs) >> PAGE_SHIFT); if (!pfn_valid(pfn)) { @@ -220,38 +221,51 @@ static int nvmap_vma_fault(struct vm_area_struct *vma, struct vm_fault *vmf) page = pfn_to_page(pfn); } else { void *kaddr; + unsigned long pfn; - offs >>= PAGE_SHIFT; - if (atomic_read(&priv->handle->pgalloc.reserved)) - return VM_FAULT_SIGBUS; - page = nvmap_to_page(priv->handle->pgalloc.pages[offs]); + if (priv->handle->heap_type != NVMAP_HEAP_IOVMM) { + offs >>= PAGE_SHIFT; + page = priv->handle->pgalloc.pages[offs]; + pfn = page_to_pfn(page); + if (!pfn_valid(pfn)) { + vm_insert_pfn(vma, + (unsigned long)vmf_address, pfn); + return VM_FAULT_NOPAGE; + } + } else { + offs >>= PAGE_SHIFT; + if (atomic_read(&priv->handle->pgalloc.reserved)) + return VM_FAULT_SIGBUS; + page = nvmap_to_page(priv->handle->pgalloc.pages[offs]); - if (PageAnon(page) && (vma->vm_flags & VM_SHARED)) - return VM_FAULT_SIGSEGV; + if (PageAnon(page)) { + if (vma->vm_flags & VM_SHARED) + return VM_FAULT_SIGSEGV; + } - if (!nvmap_handle_track_dirty(priv->handle)) - goto finish; - mutex_lock(&priv->handle->lock); - if (nvmap_page_dirty(priv->handle->pgalloc.pages[offs])) { - mutex_unlock(&priv->handle->lock); - goto finish; - } + if (!nvmap_handle_track_dirty(priv->handle)) + goto finish; + mutex_lock(&priv->handle->lock); + if (nvmap_page_dirty(priv->handle->pgalloc.pages[offs])) { + mutex_unlock(&priv->handle->lock); + goto finish; + } - /* inner cache maint */ - kaddr = kmap(page); - BUG_ON(!kaddr); - inner_cache_maint(NVMAP_CACHE_OP_WB_INV, kaddr, PAGE_SIZE); - kunmap(page); + /* inner cache maint */ + kaddr = kmap(page); + BUG_ON(!kaddr); + inner_cache_maint(NVMAP_CACHE_OP_WB_INV, kaddr, PAGE_SIZE); + kunmap(page); - if (priv->handle->flags & NVMAP_HANDLE_INNER_CACHEABLE) - goto make_dirty; + if (priv->handle->flags & NVMAP_HANDLE_INNER_CACHEABLE) + goto make_dirty; make_dirty: - nvmap_page_mkdirty(&priv->handle->pgalloc.pages[offs]); - atomic_inc(&priv->handle->pgalloc.ndirty); - mutex_unlock(&priv->handle->lock); + nvmap_page_mkdirty(&priv->handle->pgalloc.pages[offs]); + atomic_inc(&priv->handle->pgalloc.ndirty); + mutex_unlock(&priv->handle->lock); + } } - finish: if (page) get_page(page); diff --git a/drivers/video/tegra/nvmap/nvmap_heap.c b/drivers/video/tegra/nvmap/nvmap_heap.c index f8899b1e..f345d507 100644 --- a/drivers/video/tegra/nvmap/nvmap_heap.c +++ b/drivers/video/tegra/nvmap/nvmap_heap.c @@ -128,7 +128,7 @@ void nvmap_heap_debugfs_init(struct dentry *heap_root, struct nvmap_heap *heap) } static phys_addr_t nvmap_alloc_mem(struct nvmap_heap *h, size_t len, - phys_addr_t *start) + phys_addr_t *start, struct nvmap_handle *handle) { #if LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0) phys_addr_t pa = DMA_ERROR_CODE; @@ -136,6 +136,9 @@ static phys_addr_t nvmap_alloc_mem(struct nvmap_heap *h, size_t len, phys_addr_t pa = DMA_MAPPING_ERROR; #endif struct device *dev = h->dma_dev; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0) + void *err = NULL; +#endif if (len > UINT_MAX) { dev_err(dev, "%s: %d alloc size is out of range\n", @@ -165,8 +168,26 @@ static phys_addr_t nvmap_alloc_mem(struct nvmap_heap *h, size_t len, (void)dma_alloc_attrs(dev, len, &pa, GFP_KERNEL, DMA_ATTR_ALLOC_EXACT_SIZE); #else - (void)nvmap_dma_alloc_attrs(dev, len, &pa, + err = nvmap_dma_alloc_attrs(dev, len, &pa, GFP_KERNEL, DMA_ATTR_ALLOC_EXACT_SIZE); + /* + * In case of CBC carveout, try to allocate the entire chunk in physically + * contiguous manner. If it returns error, then try to allocate the memory in + * 2MB chunks. + */ + if (h->is_cbc && IS_ERR(err)) { + err = nvmap_dma_alloc_attrs(dev, len, &pa, + GFP_KERNEL, DMA_ATTR_ALLOC_EXACT_SIZE | + DMA_ATTR_ALLOC_SINGLE_PAGES); + + if (!IS_ERR_OR_NULL(err)) { + /* + * Need to keep track of pages, so that only those pages + * can be freed while freeing the buffer. + */ + handle->pgalloc.pages = (struct page **)err; + } + } #endif if (!dma_mapping_error(dev, pa)) { #ifdef NVMAP_CONFIG_VPR_RESIZE @@ -194,7 +215,7 @@ static phys_addr_t nvmap_alloc_mem(struct nvmap_heap *h, size_t len, } static void nvmap_free_mem(struct nvmap_heap *h, phys_addr_t base, - size_t len) + size_t len, struct nvmap_handle *handle) { struct device *dev = h->dma_dev; @@ -222,10 +243,18 @@ static void nvmap_free_mem(struct nvmap_heap *h, phys_addr_t base, (void *)(uintptr_t)base, (dma_addr_t)base, DMA_ATTR_ALLOC_EXACT_SIZE); #else - nvmap_dma_free_attrs(dev, len, + if (h->is_cbc && handle->pgalloc.pages) { + /* In case of pages, we need to pass pointer to array of pages */ + nvmap_dma_free_attrs(dev, len, + (void *)handle->pgalloc.pages, + (dma_addr_t)base, + DMA_ATTR_ALLOC_EXACT_SIZE | DMA_ATTR_ALLOC_SINGLE_PAGES); + } else { + nvmap_dma_free_attrs(dev, len, (void *)(uintptr_t)base, (dma_addr_t)base, DMA_ATTR_ALLOC_EXACT_SIZE); + } #endif } } @@ -238,7 +267,8 @@ static struct nvmap_heap_block *do_heap_alloc(struct nvmap_heap *heap, size_t len, size_t align, unsigned int mem_prot, phys_addr_t base_max, - phys_addr_t *start) + phys_addr_t *start, + struct nvmap_handle *handle) { struct list_block *heap_block = NULL; dma_addr_t dev_base; @@ -265,7 +295,7 @@ static struct nvmap_heap_block *do_heap_alloc(struct nvmap_heap *heap, goto fail_heap_block_alloc; } - dev_base = nvmap_alloc_mem(heap, len, start); + dev_base = nvmap_alloc_mem(heap, len, start, handle); if (dma_mapping_error(dev, dev_base)) { dev_err(dev, "failed to alloc mem of size (%zu)\n", len); @@ -305,8 +335,8 @@ static void do_heap_free(struct nvmap_heap_block *block) list_del(&b->all_list); + nvmap_free_mem(heap, block->base, b->size, block->handle); heap->free_size += b->size; - nvmap_free_mem(heap, block->base, b->size); kmem_cache_free(heap_block_cache, b); } @@ -359,7 +389,7 @@ struct nvmap_heap_block *nvmap_heap_alloc(struct nvmap_heap *h, } align = max_t(size_t, align, L1_CACHE_BYTES); - b = do_heap_alloc(h, len, align, prot, 0, start); + b = do_heap_alloc(h, len, align, prot, 0, start, handle); if (b) { b->handle = handle; handle->carveout = b; @@ -473,7 +503,7 @@ struct nvmap_heap *nvmap_heap_create(struct device *parent, DMA_MEMORY_NOMAP); #else err = nvmap_dma_declare_coherent_memory(h->dma_dev, 0, base, len, - DMA_MEMORY_NOMAP); + DMA_MEMORY_NOMAP, co->is_cbc); #endif #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0) if (!err) { @@ -496,6 +526,7 @@ struct nvmap_heap *nvmap_heap_create(struct device *parent, h->base = base; h->can_alloc = !!co->can_alloc; h->is_ivm = co->is_ivm; + h->is_cbc = co->is_cbc; h->len = len; h->free_size = len; h->peer = co->peer; @@ -616,12 +647,32 @@ int nvmap_flush_heap_block(struct nvmap_client *client, phys_addr_t phys = block->base; phys_addr_t end = block->base + len; int ret = 0; + struct nvmap_handle *h; if (prot == NVMAP_HANDLE_UNCACHEABLE || prot == NVMAP_HANDLE_WRITE_COMBINE) goto out; - ret = nvmap_cache_maint_phys_range(NVMAP_CACHE_OP_WB_INV, phys, end, + h = block->handle; + if (h->pgalloc.pages) { + unsigned long page_count, i; + + /* + * For CBC carveout with physically discontiguous 2MB chunks, + * iterate over 2MB chunks and do cache maint for it. + */ + page_count = h->size >> PAGE_SHIFT; + for (i = 0; i < page_count; i += PAGES_PER_2MB) { + phys = page_to_phys(h->pgalloc.pages[i]); + end = phys + SIZE_2MB; + ret = nvmap_cache_maint_phys_range(NVMAP_CACHE_OP_WB_INV, phys, end, + true, prot != NVMAP_HANDLE_INNER_CACHEABLE); + if (ret) + goto out; + } + } else + ret = nvmap_cache_maint_phys_range(NVMAP_CACHE_OP_WB_INV, phys, end, true, prot != NVMAP_HANDLE_INNER_CACHEABLE); + if (ret) goto out; out: diff --git a/drivers/video/tegra/nvmap/nvmap_heap.h b/drivers/video/tegra/nvmap/nvmap_heap.h index 1527b6b2..cb909f63 100644 --- a/drivers/video/tegra/nvmap/nvmap_heap.h +++ b/drivers/video/tegra/nvmap/nvmap_heap.h @@ -3,7 +3,7 @@ * * GPU heap allocator. * - * Copyright (c) 2010-2022, NVIDIA Corporation. All rights reserved. + * Copyright (c) 2010-2023, NVIDIA Corporation. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, @@ -41,6 +41,7 @@ struct nvmap_heap { struct device *cma_dev; struct device *dma_dev; bool is_ivm; + bool is_cbc; bool can_alloc; /* Used only if is_ivm == true */ unsigned int peer; /* Used only if is_ivm == true */ unsigned int vm_id; /* Used only if is_ivm == true */ diff --git a/drivers/video/tegra/nvmap/nvmap_init.c b/drivers/video/tegra/nvmap/nvmap_init.c index 9f7b4b07..3d9fd43b 100644 --- a/drivers/video/tegra/nvmap/nvmap_init.c +++ b/drivers/video/tegra/nvmap/nvmap_init.c @@ -141,11 +141,13 @@ static struct nvmap_platform_carveout nvmap_carveouts[] = { .base = 0, .size = 0, }, - /* Need uninitialized entries for IVM carveouts */ [4] = { - .name = NULL, - .usage_mask = NVMAP_HEAP_CARVEOUT_IVM, + .name = "cbc", + .usage_mask = NVMAP_HEAP_CARVEOUT_CBC, + .base = 0, + .size = 0, }, + /* Need uninitialized entries for IVM carveouts */ [5] = { .name = NULL, .usage_mask = NVMAP_HEAP_CARVEOUT_IVM, @@ -158,11 +160,15 @@ static struct nvmap_platform_carveout nvmap_carveouts[] = { .name = NULL, .usage_mask = NVMAP_HEAP_CARVEOUT_IVM, }, + [8] = { + .name = NULL, + .usage_mask = NVMAP_HEAP_CARVEOUT_IVM, + }, }; static struct nvmap_platform_data nvmap_data = { .carveouts = nvmap_carveouts, - .nr_carveouts = 4, + .nr_carveouts = 5, }; static struct nvmap_platform_carveout *nvmap_get_carveout_pdata(const char *name) @@ -359,24 +365,39 @@ static void *__nvmap_dma_alloc_from_coherent(struct device *dev, { int order = get_order(size); unsigned long flags; - unsigned int count, i = 0, j = 0; + unsigned int count = 0, i = 0, j = 0, k = 0; unsigned int alloc_size; - unsigned long align, pageno, page_count; + unsigned long align, pageno, page_count, first_pageno; void *addr = NULL; struct page **pages = NULL; int do_memset = 0; int *bitmap_nos = NULL; + const char *device_name; + bool is_cbc = false; - if (dma_get_attr(DMA_ATTR_ALLOC_EXACT_SIZE, attrs)) { - page_count = PAGE_ALIGN(size) >> PAGE_SHIFT; - if (page_count > UINT_MAX) { - dev_err(dev, "Page count more than max value\n"); - return NULL; - } - count = (unsigned int)page_count; + device_name = dev_name(dev); + if (!device_name) { + pr_err("Could not get device_name\n"); + return NULL; + } + + if (!strncmp(device_name, "cbc", 3)) + is_cbc = true; + + if (is_cbc) { + /* Calculation for CBC should consider 2MB chunks */ + count = size >> PAGE_SHIFT_2MB; + } else { + if (dma_get_attr(DMA_ATTR_ALLOC_EXACT_SIZE, attrs)) { + page_count = PAGE_ALIGN(size) >> PAGE_SHIFT; + if (page_count > UINT_MAX) { + dev_err(dev, "Page count more than max value\n"); + return NULL; + } + count = (unsigned int)page_count; + } else + count = 1 << order; } - else - count = 1 << order; if (!count) return NULL; @@ -389,20 +410,30 @@ static void *__nvmap_dma_alloc_from_coherent(struct device *dev, if ((mem->flags & DMA_MEMORY_NOMAP) && dma_get_attr(DMA_ATTR_ALLOC_SINGLE_PAGES, attrs)) { alloc_size = 1; - pages = nvmap_kvzalloc_pages(count); - if (!pages) + /* pages contain the array of pages of kernel PAGE_SIZE */ + if (!is_cbc) + pages = nvmap_kvzalloc_pages(count); + else + pages = nvmap_kvzalloc_pages(count * PAGES_PER_2MB); + + if (!pages) { + kvfree(bitmap_nos); return NULL; + } } else { alloc_size = count; } spin_lock_irqsave(&mem->spinlock, flags); - if (unlikely(size > ((u64)mem->size << PAGE_SHIFT))) + if (!is_cbc && unlikely(size > ((u64)mem->size << PAGE_SHIFT))) + goto err; + else if (is_cbc && unlikely(size > ((u64)mem->size << PAGE_SHIFT_2MB))) goto err; - if ((mem->flags & DMA_MEMORY_NOMAP) && - dma_get_attr(DMA_ATTR_ALLOC_SINGLE_PAGES, attrs)) { + if (((mem->flags & DMA_MEMORY_NOMAP) && + dma_get_attr(DMA_ATTR_ALLOC_SINGLE_PAGES, attrs)) || + is_cbc) { align = 0; } else { if (order > DMA_BUF_ALIGNMENT) @@ -418,9 +449,21 @@ static void *__nvmap_dma_alloc_from_coherent(struct device *dev, if (pageno >= mem->size) goto err; + if (!i) + first_pageno = pageno; + count -= alloc_size; - if (pages) - pages[i++] = pfn_to_page(mem->pfn_base + pageno); + if (pages) { + if (!is_cbc) + pages[i++] = pfn_to_page(mem->pfn_base + pageno); + else { + /* Handle 2MB chunks */ + for (k = 0; k < (alloc_size * PAGES_PER_2MB); k++) + pages[i++] = pfn_to_page(mem->pfn_base + + pageno * PAGES_PER_2MB + k); + } + } + bitmap_set(mem->bitmap, pageno, alloc_size); bitmap_nos[j++] = pageno; } @@ -428,9 +471,13 @@ static void *__nvmap_dma_alloc_from_coherent(struct device *dev, /* * Memory was found in the coherent area. */ - *dma_handle = mem->device_base + (pageno << PAGE_SHIFT); + if (!is_cbc) + *dma_handle = mem->device_base + (first_pageno << PAGE_SHIFT); + else + *dma_handle = mem->device_base + (first_pageno << PAGE_SHIFT_2MB); + if (!(mem->flags & DMA_MEMORY_NOMAP)) { - addr = mem->virt_base + (pageno << PAGE_SHIFT); + addr = mem->virt_base + (first_pageno << PAGE_SHIFT); do_memset = 1; } else if (dma_get_attr(DMA_ATTR_ALLOC_SINGLE_PAGES, attrs)) { addr = pages; @@ -450,7 +497,7 @@ err: spin_unlock_irqrestore(&mem->spinlock, flags); kvfree(pages); kvfree(bitmap_nos); - return NULL; + return ERR_PTR(-ENOMEM); } void *nvmap_dma_alloc_attrs(struct device *dev, size_t size, @@ -476,12 +523,23 @@ void nvmap_dma_free_attrs(struct device *dev, size_t size, void *cpu_addr, { void *mem_addr; unsigned long flags; - unsigned int pageno; + unsigned int pageno, page_shift_val; struct dma_coherent_mem_replica *mem; + bool is_cbc = false; + const char *device_name; if (!dev || !dev->dma_mem) return; + device_name = dev_name(dev); + if (!device_name) { + pr_err("Could not get device_name\n"); + return; + } + + if (!strncmp(device_name, "cbc", 3)) + is_cbc = true; + mem = (struct dma_coherent_mem_replica *)(dev->dma_mem); if ((mem->flags & DMA_MEMORY_NOMAP) && dma_get_attr(DMA_ATTR_ALLOC_SINGLE_PAGES, attrs)) { @@ -489,12 +547,22 @@ void nvmap_dma_free_attrs(struct device *dev, size_t size, void *cpu_addr, int i; spin_lock_irqsave(&mem->spinlock, flags); - for (i = 0; i < (size >> PAGE_SHIFT); i++) { - pageno = page_to_pfn(pages[i]) - mem->pfn_base; - if (WARN_ONCE(pageno > mem->size, + if (!is_cbc) { + for (i = 0; i < (size >> PAGE_SHIFT); i++) { + pageno = page_to_pfn(pages[i]) - mem->pfn_base; + if (WARN_ONCE(pageno > mem->size, "invalid pageno:%d\n", pageno)) - continue; - bitmap_clear(mem->bitmap, pageno, 1); + continue; + bitmap_clear(mem->bitmap, pageno, 1); + } + } else { + for (i = 0; i < (size >> PAGE_SHIFT); i += PAGES_PER_2MB) { + pageno = (page_to_pfn(pages[i]) - mem->pfn_base) / PAGES_PER_2MB; + if (WARN_ONCE(pageno > mem->size, + "invalid pageno:%d\n", pageno)) + continue; + bitmap_clear(mem->bitmap, pageno, 1); + } } spin_unlock_irqrestore(&mem->spinlock, flags); kvfree(pages); @@ -506,14 +574,19 @@ void nvmap_dma_free_attrs(struct device *dev, size_t size, void *cpu_addr, else mem_addr = mem->virt_base; + page_shift_val = is_cbc ? PAGE_SHIFT_2MB : PAGE_SHIFT; if (mem && cpu_addr >= mem_addr && - cpu_addr - mem_addr < (u64)mem->size << PAGE_SHIFT) { - unsigned int page = (cpu_addr - mem_addr) >> PAGE_SHIFT; + cpu_addr - mem_addr < (u64)mem->size << page_shift_val) { + unsigned int page = (cpu_addr - mem_addr) >> page_shift_val; unsigned long flags; unsigned int count; - if (DMA_ATTR_ALLOC_EXACT_SIZE & attrs) - count = PAGE_ALIGN(size) >> PAGE_SHIFT; + if (DMA_ATTR_ALLOC_EXACT_SIZE & attrs) { + if (is_cbc) + count = ALIGN_2MB(size) >> page_shift_val; + else + count = PAGE_ALIGN(size) >> page_shift_val; + } else count = 1 << get_order(size); @@ -601,17 +674,24 @@ static int nvmap_dma_assign_coherent_memory(struct device *dev, static int nvmap_dma_init_coherent_memory( phys_addr_t phys_addr, dma_addr_t device_addr, size_t size, int flags, - struct dma_coherent_mem_replica **mem) + struct dma_coherent_mem_replica **mem, bool is_cbc) { struct dma_coherent_mem_replica *dma_mem = NULL; void *mem_base = NULL; - int pages = size >> PAGE_SHIFT; - int bitmap_size = BITS_TO_LONGS(pages) * sizeof(long); + int pages; + int bitmap_size; int ret; if (!size) return -EINVAL; + if (is_cbc) + pages = size >> PAGE_SHIFT_2MB; + else + pages = size >> PAGE_SHIFT; + + bitmap_size = BITS_TO_LONGS(pages) * sizeof(long); + if (!(flags & DMA_MEMORY_NOMAP)) { mem_base = memremap(phys_addr, size, MEMREMAP_WC); if (!mem_base) @@ -632,7 +712,7 @@ static int nvmap_dma_init_coherent_memory( dma_mem->virt_base = mem_base; dma_mem->device_base = device_addr; - dma_mem->pfn_base = PFN_DOWN(phys_addr); + dma_mem->pfn_base = PFN_DOWN(device_addr); dma_mem->size = pages; dma_mem->flags = flags; spin_lock_init(&dma_mem->spinlock); @@ -649,12 +729,12 @@ err_memunmap: } int nvmap_dma_declare_coherent_memory(struct device *dev, phys_addr_t phys_addr, - dma_addr_t device_addr, size_t size, int flags) + dma_addr_t device_addr, size_t size, int flags, bool is_cbc) { struct dma_coherent_mem_replica *mem; int ret; - ret = nvmap_dma_init_coherent_memory(phys_addr, device_addr, size, flags, &mem); + ret = nvmap_dma_init_coherent_memory(phys_addr, device_addr, size, flags, &mem, is_cbc); if (ret) return ret; @@ -686,7 +766,7 @@ static int __init nvmap_co_device_init(struct reserved_mem *rmem, #else err = nvmap_dma_declare_coherent_memory(co->dma_dev, 0, co->base, co->size, - DMA_MEMORY_NOMAP); + DMA_MEMORY_NOMAP, co->is_cbc); #endif #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0) if (!err) { @@ -816,6 +896,9 @@ int __init nvmap_co_setup(struct reserved_mem *rmem) co->base = rmem->base; co->size = rmem->size; co->cma_dev = NULL; + if (!strncmp(co->name, "cbc", 3)) + co->is_cbc = true; + nvmap_init_time += sched_clock() - start; return ret; } diff --git a/drivers/video/tegra/nvmap/nvmap_ioctl.c b/drivers/video/tegra/nvmap/nvmap_ioctl.c index 771870b0..11a9dc09 100644 --- a/drivers/video/tegra/nvmap/nvmap_ioctl.c +++ b/drivers/video/tegra/nvmap/nvmap_ioctl.c @@ -192,6 +192,7 @@ int nvmap_ioctl_alloc(struct file *filp, void __user *arg) struct dma_buf *dmabuf = NULL; bool is_ro; int err; + unsigned int page_sz = PAGE_SIZE; if (copy_from_user(&op, arg, sizeof(op))) return -EFAULT; @@ -206,6 +207,14 @@ int nvmap_ioctl_alloc(struct file *filp, void __user *arg) if (IS_ERR_OR_NULL(handle)) return -EINVAL; + /* + * In case of CBC carveout, the handle size needs to be aligned to 2MB. + */ + if (op.heap_mask & NVMAP_HEAP_CARVEOUT_CBC) { + handle->size = ALIGN_2MB(handle->size); + page_sz = SIZE_2MB; + } + if (!is_nvmap_memory_available(handle->size, op.heap_mask)) { nvmap_handle_put(handle); return -ENOMEM; @@ -213,7 +222,7 @@ int nvmap_ioctl_alloc(struct file *filp, void __user *arg) /* user-space handles are aligned to page boundaries, to prevent * data leakage. */ - op.align = max_t(size_t, op.align, PAGE_SIZE); + op.align = max_t(size_t, op.align, page_sz); err = nvmap_alloc_handle(client, handle, op.heap_mask, op.align, 0, /* no kind */ @@ -1134,13 +1143,15 @@ int nvmap_ioctl_get_handle_parameters(struct file *filp, void __user *arg) /* * Check handle is allocated or not while setting contig. * If heap type is IOVMM, check if it has flag set for contiguous memory - * allocation request. Otherwise, if handle belongs to any carveout then - * all allocations are contiguous, hence set contig flag to true. + * allocation request. Otherwise, if handle belongs to any carveout except cbc + * then all allocations are contiguous, hence set contig flag to true. + * In case of cbc, if allocation is page based then set contig flag to false + * otherwise true. */ if (handle->alloc && - ((handle->heap_type == NVMAP_HEAP_IOVMM && - handle->userflags & NVMAP_HANDLE_PHYS_CONTIG) || - handle->heap_type != NVMAP_HEAP_IOVMM)) { + ((handle->heap_type == NVMAP_HEAP_IOVMM && + handle->userflags & NVMAP_HANDLE_PHYS_CONTIG) || + (handle->heap_type != NVMAP_HEAP_IOVMM && !handle->pgalloc.pages))) { op.contig = 1U; } else { op.contig = 0U; diff --git a/drivers/video/tegra/nvmap/nvmap_priv.h b/drivers/video/tegra/nvmap/nvmap_priv.h index fea05f75..423bdfc2 100644 --- a/drivers/video/tegra/nvmap/nvmap_priv.h +++ b/drivers/video/tegra/nvmap/nvmap_priv.h @@ -51,6 +51,11 @@ #include +#define SIZE_2MB (2*1024*1024) +#define ALIGN_2MB(size) ((size + SIZE_2MB - 1) & ~(SIZE_2MB - 1)) +#define PAGE_SHIFT_2MB 21 +#define PAGES_PER_2MB (SIZE_2MB / PAGE_SIZE) + #define DMA_ERROR_CODE (~(dma_addr_t)0) #define __DMA_ATTR(attrs) attrs @@ -188,8 +193,7 @@ struct nvmap_carveout_node { size_t size; }; -/* handles allocated using shared system memory (either IOVMM- or high-order - * page allocations */ +/* handles allocated as collection of pages */ struct nvmap_pgalloc { struct page **pages; bool contig; /* contiguous system memory */ @@ -238,10 +242,8 @@ struct nvmap_handle { struct nvmap_client *owner; struct dma_buf *dmabuf; struct dma_buf *dmabuf_ro; - union { - struct nvmap_pgalloc pgalloc; - struct nvmap_heap_block *carveout; - }; + struct nvmap_pgalloc pgalloc; + struct nvmap_heap_block *carveout; bool heap_pgalloc; /* handle is page allocated (sysmem / iovmm) */ bool alloc; /* handle has memory allocated */ bool from_va; /* handle memory is from VA */ @@ -494,7 +496,7 @@ struct dma_coherent_mem_replica { }; int nvmap_dma_declare_coherent_memory(struct device *dev, phys_addr_t phys_addr, - dma_addr_t device_addr, size_t size, int flags); + dma_addr_t device_addr, size_t size, int flags, bool is_cbc); #endif int nvmap_probe(struct platform_device *pdev); int nvmap_remove(struct platform_device *pdev); diff --git a/include/linux/nvmap.h b/include/linux/nvmap.h index 6710b52a..c3964926 100644 --- a/include/linux/nvmap.h +++ b/include/linux/nvmap.h @@ -3,7 +3,7 @@ * * structure declarations for nvmem and nvmap user-space ioctls * - * Copyright (c) 2009-2022, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2009-2023, NVIDIA CORPORATION. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, @@ -31,6 +31,7 @@ #define NVMAP_HEAP_CARVEOUT_VPR (1ul<<28) #define NVMAP_HEAP_CARVEOUT_TSEC (1ul<<27) #define NVMAP_HEAP_CARVEOUT_VIDMEM (1ul<<26) +#define NVMAP_HEAP_CARVEOUT_CBC (1ul << 3) #define NVMAP_HEAP_CARVEOUT_FSI (1ul<<2) #define NVMAP_HEAP_CARVEOUT_IVM (1ul<<1) #define NVMAP_HEAP_CARVEOUT_GENERIC (1ul<<0) @@ -103,6 +104,7 @@ struct nvmap_platform_carveout { bool no_cpu_access; /* carveout can't be accessed from cpu at all */ bool init_done; /* FIXME: remove once all caveouts use reserved-memory */ struct nvmap_pm_ops pm_ops; + bool is_cbc; /* cbc carveout is treated differently */ }; struct nvmap_platform_data {