gpu: nvgpu: Split VIDMEM support from mm_gk20a.c

Split VIDMEM support into its own code files organized as such:

  common/mm/vidmem.c     - Base vidmem support
  common/linux/vidmem.c  - Linux specific user-space interaction
  include/nvgpu/vidmem.h - Vidmem API definitions

Also use the config to enable/disable VIDMEM support in the makefile
and remove as many CONFIG_GK20A_VIDMEM preprocessor checks as possible
from the source code.

And lastly update a while-loop that iterated over an SGT to use the
new for_each construct for iterating over SGTs.

Currently this organization is not perfectly adhered to. More patches
will fix that.

JIRA NVGPU-30
JIRA NVGPU-138

Change-Id: Ic0f4d2cf38b65849c7dc350a69b175421477069c
Signed-off-by: Alex Waterman <alexw@nvidia.com>
Reviewed-on: https://git-master.nvidia.com/r/1540705
Reviewed-by: mobile promotions <svcmobile_promotions@nvidia.com>
Tested-by: mobile promotions <svcmobile_promotions@nvidia.com>

drivers/gpu/nvgpu/common/linux/dma.c

@@ -25,6 +25,7 @@
 #include <nvgpu/gmmu.h>
 #include <nvgpu/kmem.h>
 #include <nvgpu/enabled.h>
+#include <nvgpu/vidmem.h>
 
 #include <nvgpu/linux/dma.h>
 

drivers/gpu/nvgpu/common/linux/ioctl_ctrl.c

@@ -25,6 +25,7 @@
 #include <nvgpu/kmem.h>
 #include <nvgpu/bug.h>
 #include <nvgpu/bus.h>
+#include <nvgpu/vidmem.h>
 
 #include "ioctl_ctrl.h"
 #include "ioctl_tsg.h"

drivers/gpu/nvgpu/common/linux/nvgpu_mem.c

@@ -22,6 +22,7 @@
 #include <nvgpu/bug.h>
 #include <nvgpu/enabled.h>
 #include <nvgpu/kmem.h>
+#include <nvgpu/vidmem.h>
 
 #include <nvgpu/linux/dma.h>
 

drivers/gpu/nvgpu/common/linux/vidmem.c

@@ -0,0 +1,268 @@
+/*
+ * Copyright (c) 2017, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/dma-buf.h>
+
+#include <linux/platform/tegra/tegra_fd.h>
+
+#include <nvgpu/dma.h>
+#include <nvgpu/enabled.h>
+#include <nvgpu/vidmem.h>
+#include <nvgpu/nvgpu_mem.h>
+#include <nvgpu/page_allocator.h>
+
+#include <nvgpu/linux/dma.h>
+
+#include "gk20a/gk20a.h"
+#include "gk20a/mm_gk20a.h"
+
+#include "vm_priv.h"
+
+void set_vidmem_page_alloc(struct scatterlist *sgl, u64 addr)
+{
+	/* set bit 0 to indicate vidmem allocation */
+	sg_dma_address(sgl) = (addr | 1ULL);
+}
+
+bool is_vidmem_page_alloc(u64 addr)
+{
+	return !!(addr & 1ULL);
+}
+
+struct nvgpu_page_alloc *get_vidmem_page_alloc(struct scatterlist *sgl)
+{
+	u64 addr;
+
+	addr = sg_dma_address(sgl);
+
+	if (is_vidmem_page_alloc(addr))
+		addr = addr & ~1ULL;
+	else
+		WARN_ON(1);
+
+	return (struct nvgpu_page_alloc *)(uintptr_t)addr;
+}
+
+static struct sg_table *gk20a_vidbuf_map_dma_buf(
+	struct dma_buf_attachment *attach, enum dma_data_direction dir)
+{
+	struct gk20a_vidmem_buf *buf = attach->dmabuf->priv;
+
+	return buf->mem->priv.sgt;
+}
+
+static void gk20a_vidbuf_unmap_dma_buf(struct dma_buf_attachment *attach,
+				       struct sg_table *sgt,
+				       enum dma_data_direction dir)
+{
+}
+
+static void gk20a_vidbuf_release(struct dma_buf *dmabuf)
+{
+	struct gk20a_vidmem_buf *buf = dmabuf->priv;
+
+	gk20a_dbg_fn("");
+
+	if (buf->dmabuf_priv)
+		buf->dmabuf_priv_delete(buf->dmabuf_priv);
+
+	nvgpu_dma_free(buf->g, buf->mem);
+	nvgpu_kfree(buf->g, buf);
+}
+
+static void *gk20a_vidbuf_kmap(struct dma_buf *dmabuf, unsigned long page_num)
+{
+	WARN_ON("Not supported");
+	return NULL;
+}
+
+static void *gk20a_vidbuf_kmap_atomic(struct dma_buf *dmabuf,
+				      unsigned long page_num)
+{
+	WARN_ON("Not supported");
+	return NULL;
+}
+
+static int gk20a_vidbuf_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma)
+{
+	return -EINVAL;
+}
+
+static int gk20a_vidbuf_set_private(struct dma_buf *dmabuf,
+		struct device *dev, void *priv, void (*delete)(void *priv))
+{
+	struct gk20a_vidmem_buf *buf = dmabuf->priv;
+
+	buf->dmabuf_priv = priv;
+	buf->dmabuf_priv_delete = delete;
+
+	return 0;
+}
+
+static void *gk20a_vidbuf_get_private(struct dma_buf *dmabuf,
+		struct device *dev)
+{
+	struct gk20a_vidmem_buf *buf = dmabuf->priv;
+
+	return buf->dmabuf_priv;
+}
+
+static const struct dma_buf_ops gk20a_vidbuf_ops = {
+	.map_dma_buf      = gk20a_vidbuf_map_dma_buf,
+	.unmap_dma_buf    = gk20a_vidbuf_unmap_dma_buf,
+	.release          = gk20a_vidbuf_release,
+	.kmap_atomic      = gk20a_vidbuf_kmap_atomic,
+	.kmap             = gk20a_vidbuf_kmap,
+	.mmap             = gk20a_vidbuf_mmap,
+	.set_drvdata      = gk20a_vidbuf_set_private,
+	.get_drvdata      = gk20a_vidbuf_get_private,
+};
+
+static struct dma_buf *gk20a_vidbuf_export(struct gk20a_vidmem_buf *buf)
+{
+	DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
+
+	exp_info.priv = buf;
+	exp_info.ops = &gk20a_vidbuf_ops;
+	exp_info.size = buf->mem->size;
+	exp_info.flags = O_RDWR;
+
+	return dma_buf_export(&exp_info);
+}
+
+struct gk20a *gk20a_vidmem_buf_owner(struct dma_buf *dmabuf)
+{
+	struct gk20a_vidmem_buf *buf = dmabuf->priv;
+
+	if (dmabuf->ops != &gk20a_vidbuf_ops)
+		return NULL;
+
+	return buf->g;
+}
+
+int gk20a_vidmem_buf_alloc(struct gk20a *g, size_t bytes)
+{
+	struct gk20a_vidmem_buf *buf;
+	int err = 0, fd;
+
+	gk20a_dbg_fn("");
+
+	buf = nvgpu_kzalloc(g, sizeof(*buf));
+	if (!buf)
+		return -ENOMEM;
+
+	buf->g = g;
+
+	if (!g->mm.vidmem.cleared) {
+		nvgpu_mutex_acquire(&g->mm.vidmem.first_clear_mutex);
+		if (!g->mm.vidmem.cleared) {
+			err = gk20a_vidmem_clear_all(g);
+			if (err) {
+				nvgpu_err(g,
+				          "failed to clear whole vidmem");
+				goto err_kfree;
+			}
+		}
+		nvgpu_mutex_release(&g->mm.vidmem.first_clear_mutex);
+	}
+
+	buf->mem = nvgpu_kzalloc(g, sizeof(struct nvgpu_mem));
+	if (!buf->mem)
+		goto err_kfree;
+
+	buf->mem->mem_flags |= NVGPU_MEM_FLAG_USER_MEM;
+
+	err = nvgpu_dma_alloc_vid(g, bytes, buf->mem);
+	if (err)
+		goto err_memfree;
+
+	buf->dmabuf = gk20a_vidbuf_export(buf);
+	if (IS_ERR(buf->dmabuf)) {
+		err = PTR_ERR(buf->dmabuf);
+		goto err_bfree;
+	}
+
+	fd = tegra_alloc_fd(current->files, 1024, O_RDWR);
+	if (fd < 0) {
+		/* ->release frees what we have done */
+		dma_buf_put(buf->dmabuf);
+		return fd;
+	}
+
+	/* fclose() on this drops one ref, freeing the dma buf */
+	fd_install(fd, buf->dmabuf->file);
+
+	return fd;
+
+err_bfree:
+	nvgpu_dma_free(g, buf->mem);
+err_memfree:
+	nvgpu_kfree(g, buf->mem);
+err_kfree:
+	nvgpu_kfree(g, buf);
+	return err;
+}
+
+int gk20a_vidbuf_access_memory(struct gk20a *g, struct dma_buf *dmabuf,
+		void *buffer, u64 offset, u64 size, u32 cmd)
+{
+	struct gk20a_vidmem_buf *vidmem_buf;
+	struct nvgpu_mem *mem;
+	int err = 0;
+
+	if (gk20a_dmabuf_aperture(g, dmabuf) != APERTURE_VIDMEM)
+		return -EINVAL;
+
+	vidmem_buf = dmabuf->priv;
+	mem = vidmem_buf->mem;
+
+	switch (cmd) {
+	case NVGPU_DBG_GPU_IOCTL_ACCESS_FB_MEMORY_CMD_READ:
+		nvgpu_mem_rd_n(g, mem, offset, buffer, size);
+		break;
+
+	case NVGPU_DBG_GPU_IOCTL_ACCESS_FB_MEMORY_CMD_WRITE:
+		nvgpu_mem_wr_n(g, mem, offset, buffer, size);
+		break;
+
+	default:
+		err = -EINVAL;
+	}
+
+	return err;
+}
+
+void gk20a_vidmem_clear_mem_worker(struct work_struct *work)
+{
+	struct mm_gk20a *mm = container_of(work, struct mm_gk20a,
+					vidmem.clear_mem_worker);
+	struct gk20a *g = mm->g;
+	struct nvgpu_mem *mem;
+
+	while ((mem = get_pending_mem_desc(mm)) != NULL) {
+		gk20a_gmmu_clear_vidmem_mem(g, mem);
+		nvgpu_free(mem->allocator,
+			   (u64)get_vidmem_page_alloc(mem->priv.sgt->sgl));
+		nvgpu_free_sgtable(g, &mem->priv.sgt);
+
+		WARN_ON(nvgpu_atomic64_sub_return(mem->aligned_size,
+					&g->mm.vidmem.bytes_pending) < 0);
+		mem->size = 0;
+		mem->aperture = APERTURE_INVALID;
+
+		nvgpu_kfree(g, mem);
+	}
+}

drivers/gpu/nvgpu/common/linux/vm.c

@@ -23,6 +23,8 @@
 #include <nvgpu/vm_area.h>
 #include <nvgpu/nvgpu_mem.h>
 #include <nvgpu/page_allocator.h>
+#include <nvgpu/vidmem.h>
+#include <nvgpu/enabled.h>
 
 #include <nvgpu/linux/nvgpu_mem.h>
 
@@ -34,6 +36,33 @@
 #include "vm_priv.h"
 #include "os_linux.h"
 
+/*
+ * Temporary location for this code until a dmabuf.c file exists.
+ */
+enum nvgpu_aperture gk20a_dmabuf_aperture(struct gk20a *g,
+					  struct dma_buf *dmabuf)
+{
+	struct gk20a *buf_owner = gk20a_vidmem_buf_owner(dmabuf);
+	bool unified_memory = nvgpu_is_enabled(g, NVGPU_MM_UNIFIED_MEMORY);
+
+	if (buf_owner == NULL) {
+		/* Not nvgpu-allocated, assume system memory */
+		return APERTURE_SYSMEM;
+	} else if (WARN_ON(buf_owner == g && unified_memory)) {
+		/* Looks like our video memory, but this gpu doesn't support
+		 * it. Warn about a bug and bail out */
+		nvgpu_warn(g,
+			"dmabuf is our vidmem but we don't have local vidmem");
+		return APERTURE_INVALID;
+	} else if (buf_owner != g) {
+		/* Someone else's vidmem */
+		return APERTURE_INVALID;
+	} else {
+		/* Yay, buf_owner == g */
+		return APERTURE_VIDMEM;
+	}
+}
+
 static struct nvgpu_mapped_buf *__nvgpu_vm_find_mapped_buf_reverse(
 	struct vm_gk20a *vm, struct dma_buf *dmabuf, u32 kind)
 {