gpu: nvgpu: Get coherency on gv100 + NVLINK working

This patch does a couple of things. First it renames
NVGPU_DMA_COHERENT to NVGPU_USE_COHERENT_SYSMEM since the former
is somewhat ambiguous in meaning. The latter clearly states what
must happen: nvgpu needs to treat sysmem as coherent. This flag
does simply follow the state of the DMA API but there's no reason
to expect a casual reader of the code to know that when the DMA
API is coherent nvgpu must treat sysmem as coherent.

One thing to note though: when the dGPU is using PCIe and the
PCIe controller is coherent, it doesn't actually matter what we
do. However, we use this flag for determining how to make CPU
mappings in nvgpu_mem_begin() so this flag is still relevant for
the CPU side of things.

Next this patch adds a check in the core kernel GMMU mapping
routine to make sure that when the NVGPU_USE_COHERENT_SYSMEM flag
is set that the IO coherent flag is passed into the mapping code.
This is the primary fix that made NVLINK start working.

Finally the setting of the USE_COHERENT_SYSMEM flag and the
NVGPU_SUPPORT_IO_COHERENCE flag were set both for PCIe and for
iGPUs. The iGPU also must correctly match it's CPU mappings and
GPU mappings for proper operation.

JIRA EVLR-2333

Change-Id: Icd5f07167c9f48a0a2e8493e34c9cc6238e56907
Signed-off-by: Alex Waterman <alexw@nvidia.com>
Reviewed-on: https://git-master.nvidia.com/r/1654519
Reviewed-by: mobile promotions <svcmobile_promotions@nvidia.com>
Tested-by: mobile promotions <svcmobile_promotions@nvidia.com>

drivers/gpu/nvgpu/common/linux/module.c

@@ -20,6 +20,7 @@
 #include <linux/of.h>
 #include <linux/of_device.h>
 #include <linux/of_platform.h>
+#include <linux/of_address.h>
 #include <linux/interrupt.h>
 #include <linux/pm_runtime.h>
 #include <linux/reset.h>
@@ -1107,6 +1108,7 @@ static int gk20a_probe(struct platform_device *dev)
 	struct gk20a *gk20a;
 	int err;
 	struct gk20a_platform *platform = NULL;
+	struct device_node *np;
 
 	if (dev->dev.of_node) {
 		const struct of_device_id *match;
@@ -1206,6 +1208,12 @@ static int gk20a_probe(struct platform_device *dev)
 
 	gk20a->mm.has_physical_mode = !nvgpu_is_hypervisor_mode(gk20a);
 
+	np = nvgpu_get_node(gk20a);
+	if (of_dma_is_coherent(np)) {
+		__nvgpu_set_enabled(gk20a, NVGPU_USE_COHERENT_SYSMEM, true);
+		__nvgpu_set_enabled(gk20a, NVGPU_SUPPORT_IO_COHERENCE, true);
+	}
+
 	return 0;
 
 return_err:

drivers/gpu/nvgpu/common/linux/nvgpu_mem.c

@@ -61,7 +61,8 @@ u32 nvgpu_aperture_mask(struct gk20a *g, struct nvgpu_mem *mem,
 int nvgpu_mem_begin(struct gk20a *g, struct nvgpu_mem *mem)
 {
 	void *cpu_va;
-	pgprot_t prot = nvgpu_is_enabled(g, NVGPU_DMA_COHERENT) ? PAGE_KERNEL :
+	pgprot_t prot = nvgpu_is_enabled(g, NVGPU_USE_COHERENT_SYSMEM) ?
+		PAGE_KERNEL :
 		pgprot_writecombine(PAGE_KERNEL);
 
 	if (mem->aperture != APERTURE_SYSMEM || g->mm.force_pramin)

drivers/gpu/nvgpu/common/linux/pci.c

@@ -17,13 +17,13 @@
 #include <linux/pci.h>
 #include <linux/interrupt.h>
 #include <linux/pm_runtime.h>
+#include <linux/of_platform.h>
+#include <linux/of_address.h>
 
 #include <nvgpu/nvgpu_common.h>
 #include <nvgpu/kmem.h>
 #include <nvgpu/enabled.h>
 #include <nvgpu/nvlink.h>
-#include <linux/of_platform.h>
-#include <linux/of_address.h>
 
 #include "gk20a/gk20a.h"
 #include "clk/clk.h"
@@ -647,7 +647,7 @@ static int nvgpu_pci_probe(struct pci_dev *pdev,
 	np = nvgpu_get_node(g);
 
 	if (of_dma_is_coherent(np)) {
-		__nvgpu_set_enabled(g, NVGPU_DMA_COHERENT, true);
+		__nvgpu_set_enabled(g, NVGPU_USE_COHERENT_SYSMEM, true);
 		__nvgpu_set_enabled(g, NVGPU_SUPPORT_IO_COHERENCE, true);
 	}
 

drivers/gpu/nvgpu/common/mm/gmmu.c

@@ -79,6 +79,13 @@ static u64 __nvgpu_gmmu_map(struct vm_gk20a *vm,
 	if (!sgt)
 		return -ENOMEM;
 
+	/*
+	 * If the GPU is IO coherent and the DMA API is giving us IO coherent
+	 * CPU mappings then we gotta make sure we use the IO coherent aperture.
+	 */
+	if (nvgpu_is_enabled(g, NVGPU_USE_COHERENT_SYSMEM))
+		flags |= NVGPU_VM_MAP_IO_COHERENT;
+
 	nvgpu_mutex_acquire(&vm->update_gmmu_lock);
 	vaddr = g->ops.mm.gmmu_map(vm, addr,
 				   sgt,    /* sg list */

drivers/gpu/nvgpu/include/nvgpu/enabled.h

@@ -75,8 +75,8 @@ struct gk20a;
 #define NVGPU_SUPPORT_MAP_DIRECT_KIND_CTRL	24
 /* Support batch mapping */
 #define NVGPU_SUPPORT_MAP_BUFFER_BATCH		25
-/* Support DMA coherence */
+/* Use coherent aperture for sysmem. */
-#define NVGPU_DMA_COHERENT			26
+#define NVGPU_USE_COHERENT_SYSMEM		26
 /* Use physical scatter tables instead of IOMMU */
 #define NVGPU_MM_USE_PHYSICAL_SG		27