diff --git a/drivers/gpu/nvgpu/common/mm/nvgpu_mem.c b/drivers/gpu/nvgpu/common/mm/nvgpu_mem.c index 855d455d3..9f3b6cfa8 100644 --- a/drivers/gpu/nvgpu/common/mm/nvgpu_mem.c +++ b/drivers/gpu/nvgpu/common/mm/nvgpu_mem.c @@ -177,3 +177,112 @@ u64 nvgpu_sgt_alignment(struct gk20a *g, struct nvgpu_sgt *sgt) return align; } + +u32 nvgpu_mem_rd32(struct gk20a *g, struct nvgpu_mem *mem, u32 w) +{ + u32 data = 0; + + if (mem->aperture == APERTURE_SYSMEM) { + u32 *ptr = mem->cpu_va; + + WARN_ON(!ptr); + data = ptr[w]; + } else if (mem->aperture == APERTURE_VIDMEM) { + nvgpu_pramin_rd_n(g, mem, w * sizeof(u32), sizeof(u32), &data); + } else { + WARN_ON("Accessing unallocated nvgpu_mem"); + } + + return data; +} + +u32 nvgpu_mem_rd(struct gk20a *g, struct nvgpu_mem *mem, u32 offset) +{ + WARN_ON(offset & 3); + return nvgpu_mem_rd32(g, mem, offset / sizeof(u32)); +} + +void nvgpu_mem_rd_n(struct gk20a *g, struct nvgpu_mem *mem, + u32 offset, void *dest, u32 size) +{ + WARN_ON(offset & 3); + WARN_ON(size & 3); + + if (mem->aperture == APERTURE_SYSMEM) { + u8 *src = (u8 *)mem->cpu_va + offset; + + WARN_ON(!mem->cpu_va); + memcpy(dest, src, size); + } else if (mem->aperture == APERTURE_VIDMEM) { + nvgpu_pramin_rd_n(g, mem, offset, size, dest); + } else { + WARN_ON("Accessing unallocated nvgpu_mem"); + } +} + +void nvgpu_mem_wr32(struct gk20a *g, struct nvgpu_mem *mem, u32 w, u32 data) +{ + if (mem->aperture == APERTURE_SYSMEM) { + u32 *ptr = mem->cpu_va; + + WARN_ON(!ptr); + ptr[w] = data; + } else if (mem->aperture == APERTURE_VIDMEM) { + nvgpu_pramin_wr_n(g, mem, w * sizeof(u32), sizeof(u32), &data); + if (!mem->skip_wmb) + nvgpu_wmb(); + } else { + WARN_ON("Accessing unallocated nvgpu_mem"); + } +} + +void nvgpu_mem_wr(struct gk20a *g, struct nvgpu_mem *mem, u32 offset, u32 data) +{ + WARN_ON(offset & 3); + nvgpu_mem_wr32(g, mem, offset / sizeof(u32), data); +} + +void nvgpu_mem_wr_n(struct gk20a *g, struct nvgpu_mem *mem, u32 offset, + void *src, u32 size) +{ + WARN_ON(offset & 3); + WARN_ON(size & 3); + + if (mem->aperture == APERTURE_SYSMEM) { + u8 *dest = (u8 *)mem->cpu_va + offset; + + WARN_ON(!mem->cpu_va); + memcpy(dest, src, size); + } else if (mem->aperture == APERTURE_VIDMEM) { + nvgpu_pramin_wr_n(g, mem, offset, size, src); + if (!mem->skip_wmb) + nvgpu_wmb(); + } else { + WARN_ON("Accessing unallocated nvgpu_mem"); + } +} + +void nvgpu_memset(struct gk20a *g, struct nvgpu_mem *mem, u32 offset, + u32 c, u32 size) +{ + WARN_ON(offset & 3); + WARN_ON(size & 3); + WARN_ON(c & ~0xff); + + c &= 0xff; + + if (mem->aperture == APERTURE_SYSMEM) { + u8 *dest = (u8 *)mem->cpu_va + offset; + + WARN_ON(!mem->cpu_va); + memset(dest, c, size); + } else if (mem->aperture == APERTURE_VIDMEM) { + u32 repeat_value = c | (c << 8) | (c << 16) | (c << 24); + + nvgpu_pramin_memset(g, mem, offset, size, repeat_value); + if (!mem->skip_wmb) + nvgpu_wmb(); + } else { + WARN_ON("Accessing unallocated nvgpu_mem"); + } +} diff --git a/drivers/gpu/nvgpu/common/posix/io.c b/drivers/gpu/nvgpu/common/posix/io.c index ce0189405..dc32c20e7 100644 --- a/drivers/gpu/nvgpu/common/posix/io.c +++ b/drivers/gpu/nvgpu/common/posix/io.c @@ -35,6 +35,11 @@ void nvgpu_writel(struct gk20a *g, u32 r, u32 v) BUG(); } +void nvgpu_writel_relaxed(struct gk20a *g, u32 r, u32 v) +{ + BUG(); +} + u32 nvgpu_readl(struct gk20a *g, u32 r) { BUG(); diff --git a/drivers/gpu/nvgpu/common/posix/posix-nvgpu_mem.c b/drivers/gpu/nvgpu/common/posix/posix-nvgpu_mem.c index 7f3bf9f12..fa92a7c6d 100644 --- a/drivers/gpu/nvgpu/common/posix/posix-nvgpu_mem.c +++ b/drivers/gpu/nvgpu/common/posix/posix-nvgpu_mem.c @@ -26,65 +26,6 @@ #include #include -/* - * DMA memory buffers - obviously we don't really have DMA in userspace but we - * can emulate a lot of the DMA mem functionality for unit testing purposes. - */ - -u32 nvgpu_mem_rd32(struct gk20a *g, struct nvgpu_mem *mem, u32 w) -{ - u32 *mem_ptr = (u32 *)mem->cpu_va; - - return mem_ptr[w]; -} - -u32 nvgpu_mem_rd(struct gk20a *g, struct nvgpu_mem *mem, u32 offset) -{ - if (offset & 0x3) - BUG(); - - return nvgpu_mem_rd32(g, mem, offset >> 2); -} - -void nvgpu_mem_rd_n(struct gk20a *g, struct nvgpu_mem *mem, u32 offset, - void *dest, u32 size) -{ - if (offset & 0x3 || size & 0x3) - BUG(); - - memcpy(dest, ((char *)mem->cpu_va) + offset, size); -} - -void nvgpu_mem_wr32(struct gk20a *g, struct nvgpu_mem *mem, u32 w, u32 data) -{ - u32 *mem_ptr = (u32 *)mem->cpu_va; - - mem_ptr[w] = data; -} - -void nvgpu_mem_wr(struct gk20a *g, struct nvgpu_mem *mem, u32 offset, u32 data) -{ - if (offset & 0x3) - BUG(); - - nvgpu_mem_wr32(g, mem, offset >> 2, data); -} - -void nvgpu_mem_wr_n(struct gk20a *g, struct nvgpu_mem *mem, u32 offset, - void *src, u32 size) -{ - if (offset & 0x3 || size & 0x3) - BUG(); - - memcpy(((char *)mem->cpu_va) + offset, src, size); -} - -void nvgpu_memset(struct gk20a *g, struct nvgpu_mem *mem, u32 offset, - u32 c, u32 size) -{ - memset(((char *)mem->cpu_va) + offset, c, size); -} - /* * These functions are somewhat meaningless. */ diff --git a/drivers/gpu/nvgpu/common/pramin.c b/drivers/gpu/nvgpu/common/pramin.c index 4c6a4a0d4..99d588aac 100644 --- a/drivers/gpu/nvgpu/common/pramin.c +++ b/drivers/gpu/nvgpu/common/pramin.c @@ -27,12 +27,19 @@ #include "gk20a/gk20a.h" +/* + * This typedef is for functions that get called during the access_batched() + * operation. + */ +typedef void (*pramin_access_batch_fn)(struct gk20a *g, u32 start, u32 words, + u32 **arg); + /* * The PRAMIN range is 1 MB, must change base addr if a buffer crosses that. * This same loop is used for read/write/memset. Offset and size in bytes. * One call to "loop" is done per range, with "arg" supplied. */ -void nvgpu_pramin_access_batched(struct gk20a *g, struct nvgpu_mem *mem, +static void nvgpu_pramin_access_batched(struct gk20a *g, struct nvgpu_mem *mem, u32 offset, u32 size, pramin_access_batch_fn loop, u32 **arg) { struct nvgpu_page_alloc *alloc = NULL; @@ -87,6 +94,69 @@ void nvgpu_pramin_access_batched(struct gk20a *g, struct nvgpu_mem *mem, } } +static void nvgpu_pramin_access_batch_rd_n(struct gk20a *g, + u32 start, u32 words, u32 **arg) +{ + u32 r = start, *dest_u32 = *arg; + + while (words--) { + *dest_u32++ = nvgpu_readl(g, r); + r += sizeof(u32); + } + + *arg = dest_u32; +} + +void nvgpu_pramin_rd_n(struct gk20a *g, struct nvgpu_mem *mem, + u32 start, u32 size, void *dest) +{ + u32 *dest_u32 = dest; + + return nvgpu_pramin_access_batched(g, mem, start, size, + nvgpu_pramin_access_batch_rd_n, &dest_u32); +} + +static void nvgpu_pramin_access_batch_wr_n(struct gk20a *g, + u32 start, u32 words, u32 **arg) +{ + u32 r = start, *src_u32 = *arg; + + while (words--) { + nvgpu_writel_relaxed(g, r, *src_u32++); + r += sizeof(u32); + } + + *arg = src_u32; +} + +void nvgpu_pramin_wr_n(struct gk20a *g, struct nvgpu_mem *mem, + u32 start, u32 size, void *src) +{ + u32 *src_u32 = src; + + return nvgpu_pramin_access_batched(g, mem, start, size, + nvgpu_pramin_access_batch_wr_n, &src_u32); +} + +static void nvgpu_pramin_access_batch_set(struct gk20a *g, + u32 start, u32 words, u32 **arg) +{ + u32 r = start, repeat = **arg; + + while (words--) { + nvgpu_writel_relaxed(g, r, repeat); + r += sizeof(u32); + } +} + +void nvgpu_pramin_memset(struct gk20a *g, struct nvgpu_mem *mem, + u32 start, u32 size, u32 w) +{ + u32 *p = &w; + + return nvgpu_pramin_access_batched(g, mem, start, size, + nvgpu_pramin_access_batch_set, &p); +} void nvgpu_init_pramin(struct mm_gk20a *mm) { mm->pramin_window = 0; diff --git a/drivers/gpu/nvgpu/include/nvgpu/io.h b/drivers/gpu/nvgpu/include/nvgpu/io.h index 28011e041..8504829c8 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/io.h +++ b/drivers/gpu/nvgpu/include/nvgpu/io.h @@ -36,6 +36,7 @@ struct gk20a; void nvgpu_writel(struct gk20a *g, u32 r, u32 v); +void nvgpu_writel_relaxed(struct gk20a *g, u32 r, u32 v); u32 nvgpu_readl(struct gk20a *g, u32 r); u32 __nvgpu_readl(struct gk20a *g, u32 r); void nvgpu_writel_check(struct gk20a *g, u32 r, u32 v); diff --git a/drivers/gpu/nvgpu/include/nvgpu/pramin.h b/drivers/gpu/nvgpu/include/nvgpu/pramin.h index 33702bc86..c9f54a6f0 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/pramin.h +++ b/drivers/gpu/nvgpu/include/nvgpu/pramin.h @@ -29,16 +29,10 @@ struct gk20a; struct mm_gk20a; struct nvgpu_mem; -/* - * This typedef is for functions that get called during the access_batched() - * operation. - */ -typedef void (*pramin_access_batch_fn)(struct gk20a *g, u32 start, u32 words, - u32 **arg); -void nvgpu_pramin_access_batched(struct gk20a *g, struct nvgpu_mem *mem, - u32 offset, u32 size, - pramin_access_batch_fn loop, u32 **arg); +void nvgpu_pramin_rd_n(struct gk20a *g, struct nvgpu_mem *mem, u32 start, u32 words, void *dest); +void nvgpu_pramin_wr_n(struct gk20a *g, struct nvgpu_mem *mem, u32 start, u32 words, void *src); +void nvgpu_pramin_memset(struct gk20a *g, struct nvgpu_mem *mem, u32 start, u32 words, u32 w); void nvgpu_init_pramin(struct mm_gk20a *mm); diff --git a/drivers/gpu/nvgpu/os/linux/io.c b/drivers/gpu/nvgpu/os/linux/io.c index c06512a5e..9a0e29d76 100644 --- a/drivers/gpu/nvgpu/os/linux/io.c +++ b/drivers/gpu/nvgpu/os/linux/io.c @@ -31,6 +31,18 @@ void nvgpu_writel(struct gk20a *g, u32 r, u32 v) } } +void nvgpu_writel_relaxed(struct gk20a *g, u32 r, u32 v) +{ + struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); + + if (unlikely(!l->regs)) { + __gk20a_warn_on_no_regs(); + nvgpu_log(g, gpu_dbg_reg, "r=0x%x v=0x%x (failed)", r, v); + } else { + writel_relaxed(v, l->regs + r); + } +} + u32 nvgpu_readl(struct gk20a *g, u32 r) { u32 v = __nvgpu_readl(g, r); diff --git a/drivers/gpu/nvgpu/os/linux/nvgpu_mem.c b/drivers/gpu/nvgpu/os/linux/nvgpu_mem.c index 04b2afa74..aa8fcd84e 100644 --- a/drivers/gpu/nvgpu/os/linux/nvgpu_mem.c +++ b/drivers/gpu/nvgpu/os/linux/nvgpu_mem.c @@ -48,207 +48,6 @@ static u64 __nvgpu_sgl_phys(struct gk20a *g, struct nvgpu_sgl *sgl) return ipa; } -static void pramin_access_batch_rd_n(struct gk20a *g, u32 start, u32 words, u32 **arg) -{ - struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); - u32 r = start, *dest_u32 = *arg; - - if (!l->regs) { - __gk20a_warn_on_no_regs(); - return; - } - - while (words--) { - *dest_u32++ = gk20a_readl(g, r); - r += sizeof(u32); - } - - *arg = dest_u32; -} - -u32 nvgpu_mem_rd32(struct gk20a *g, struct nvgpu_mem *mem, u32 w) -{ - u32 data = 0; - - if (mem->aperture == APERTURE_SYSMEM) { - u32 *ptr = mem->cpu_va; - - WARN_ON(!ptr); - data = ptr[w]; -#ifdef CONFIG_TEGRA_SIMULATION_PLATFORM - nvgpu_log(g, gpu_dbg_mem, " %p = 0x%x", ptr + w, data); -#endif - } else if (mem->aperture == APERTURE_VIDMEM) { - u32 value; - u32 *p = &value; - - nvgpu_pramin_access_batched(g, mem, w * sizeof(u32), - sizeof(u32), pramin_access_batch_rd_n, &p); - - data = value; - - } else { - WARN_ON("Accessing unallocated nvgpu_mem"); - } - - return data; -} - -u32 nvgpu_mem_rd(struct gk20a *g, struct nvgpu_mem *mem, u32 offset) -{ - WARN_ON(offset & 3); - return nvgpu_mem_rd32(g, mem, offset / sizeof(u32)); -} - -void nvgpu_mem_rd_n(struct gk20a *g, struct nvgpu_mem *mem, - u32 offset, void *dest, u32 size) -{ - WARN_ON(offset & 3); - WARN_ON(size & 3); - - if (mem->aperture == APERTURE_SYSMEM) { - u8 *src = (u8 *)mem->cpu_va + offset; - - WARN_ON(!mem->cpu_va); - memcpy(dest, src, size); -#ifdef CONFIG_TEGRA_SIMULATION_PLATFORM - if (size) - nvgpu_log(g, gpu_dbg_mem, " %p = 0x%x ... [%d bytes]", - src, *dest, size); -#endif - } else if (mem->aperture == APERTURE_VIDMEM) { - u32 *dest_u32 = dest; - - nvgpu_pramin_access_batched(g, mem, offset, size, - pramin_access_batch_rd_n, &dest_u32); - } else { - WARN_ON("Accessing unallocated nvgpu_mem"); - } -} - -static void pramin_access_batch_wr_n(struct gk20a *g, u32 start, u32 words, u32 **arg) -{ - struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); - u32 r = start, *src_u32 = *arg; - - if (!l->regs) { - __gk20a_warn_on_no_regs(); - return; - } - - while (words--) { - writel_relaxed(*src_u32++, l->regs + r); - r += sizeof(u32); - } - - *arg = src_u32; -} - -void nvgpu_mem_wr32(struct gk20a *g, struct nvgpu_mem *mem, u32 w, u32 data) -{ - if (mem->aperture == APERTURE_SYSMEM) { - u32 *ptr = mem->cpu_va; - - WARN_ON(!ptr); -#ifdef CONFIG_TEGRA_SIMULATION_PLATFORM - nvgpu_log(g, gpu_dbg_mem, " %p = 0x%x", ptr + w, data); -#endif - ptr[w] = data; - } else if (mem->aperture == APERTURE_VIDMEM) { - u32 value = data; - u32 *p = &value; - - nvgpu_pramin_access_batched(g, mem, w * sizeof(u32), - sizeof(u32), pramin_access_batch_wr_n, &p); - if (!mem->skip_wmb) - wmb(); - } else { - WARN_ON("Accessing unallocated nvgpu_mem"); - } -} - -void nvgpu_mem_wr(struct gk20a *g, struct nvgpu_mem *mem, u32 offset, u32 data) -{ - WARN_ON(offset & 3); - nvgpu_mem_wr32(g, mem, offset / sizeof(u32), data); -} - -void nvgpu_mem_wr_n(struct gk20a *g, struct nvgpu_mem *mem, u32 offset, - void *src, u32 size) -{ - WARN_ON(offset & 3); - WARN_ON(size & 3); - - if (mem->aperture == APERTURE_SYSMEM) { - u8 *dest = (u8 *)mem->cpu_va + offset; - - WARN_ON(!mem->cpu_va); -#ifdef CONFIG_TEGRA_SIMULATION_PLATFORM - if (size) - nvgpu_log(g, gpu_dbg_mem, " %p = 0x%x ... [%d bytes]", - dest, *src, size); -#endif - memcpy(dest, src, size); - } else if (mem->aperture == APERTURE_VIDMEM) { - u32 *src_u32 = src; - - nvgpu_pramin_access_batched(g, mem, offset, size, - pramin_access_batch_wr_n, &src_u32); - if (!mem->skip_wmb) - wmb(); - } else { - WARN_ON("Accessing unallocated nvgpu_mem"); - } -} - -static void pramin_access_batch_set(struct gk20a *g, u32 start, u32 words, u32 **arg) -{ - struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); - u32 r = start, repeat = **arg; - - if (!l->regs) { - __gk20a_warn_on_no_regs(); - return; - } - - while (words--) { - writel_relaxed(repeat, l->regs + r); - r += sizeof(u32); - } -} - -void nvgpu_memset(struct gk20a *g, struct nvgpu_mem *mem, u32 offset, - u32 c, u32 size) -{ - WARN_ON(offset & 3); - WARN_ON(size & 3); - WARN_ON(c & ~0xff); - - c &= 0xff; - - if (mem->aperture == APERTURE_SYSMEM) { - u8 *dest = (u8 *)mem->cpu_va + offset; - - WARN_ON(!mem->cpu_va); -#ifdef CONFIG_TEGRA_SIMULATION_PLATFORM - if (size) - nvgpu_log(g, gpu_dbg_mem, " %p = 0x%x [times %d]", - dest, c, size); -#endif - memset(dest, c, size); - } else if (mem->aperture == APERTURE_VIDMEM) { - u32 repeat_value = c | (c << 8) | (c << 16) | (c << 24); - u32 *p = &repeat_value; - - nvgpu_pramin_access_batched(g, mem, offset, size, - pramin_access_batch_set, &p); - if (!mem->skip_wmb) - wmb(); - } else { - WARN_ON("Accessing unallocated nvgpu_mem"); - } -} - /* * Obtain a SYSMEM address from a Linux SGL. This should eventually go away * and/or become private to this file once all bad usages of Linux SGLs are