gpu: nvgpu: gv100: add IPA to PA translation

Add IPA to PA translation for GV100 nvlink / pass-through mode
- define platform->phys_addr(g, ipa) method
- call nvgpu_init_soc_vars from nvgpu_tegra_pci_probe
- in nvgpu_init_soc_vars, define set platform->phys_addr to
  nvgpu_tegra_hv_ipa_pa, if hypervisor is present.
- in __nvgpu_sgl_phys, use sg_phys, then apply platform->phys_addr
  if defined.
- implement IPA to PA translation in nvgpu_tegra_hv_ipa_pa

Bug 200392719

Change-Id: I622049ddc62c2a57a665dd259c1bb4ed3843a537
Signed-off-by: Thomas Fleury <tfleury@nvidia.com>
Reviewed-on: https://git-master.nvidia.com/r/1673582
Reviewed-by: svc-mobile-coverity <svc-mobile-coverity@nvidia.com>
Reviewed-by: Richard Zhao <rizhao@nvidia.com>
GVS: Gerrit_Virtual_Submit
Reviewed-by: Vijayakumar Subbu <vsubbu@nvidia.com>
Reviewed-by: mobile promotions <svcmobile_promotions@nvidia.com>
Tested-by: mobile promotions <svcmobile_promotions@nvidia.com>

drivers/gpu/nvgpu/common/linux/nvgpu_mem.c

@@ -33,10 +33,18 @@
 
 #include "gk20a/gk20a.h"
 #include "gk20a/mm_gk20a.h"
+#include "platform_gk20a.h"
 
-static inline u64 __nvgpu_sgl_phys(struct gk20a *g, struct nvgpu_sgl *sgl)
+static u64 __nvgpu_sgl_phys(struct gk20a *g, struct nvgpu_sgl *sgl)
 {
-	return sg_phys((struct scatterlist *)sgl);
+	struct device *dev = dev_from_gk20a(g);
+	struct gk20a_platform *platform = gk20a_get_platform(dev);
+	u64 ipa = sg_phys((struct scatterlist *)sgl);
+
+	if (platform->phys_addr)
+		return platform->phys_addr(g, ipa);
+
+	return ipa;
 }
 
 int nvgpu_mem_begin(struct gk20a *g, struct nvgpu_mem *mem)

drivers/gpu/nvgpu/common/linux/pci.c

@@ -24,6 +24,7 @@
 #include <nvgpu/kmem.h>
 #include <nvgpu/enabled.h>
 #include <nvgpu/nvlink.h>
+#include <nvgpu/soc.h>
 
 #include "gk20a/gk20a.h"
 #include "clk/clk.h"
@@ -44,7 +45,7 @@
 
 static int nvgpu_pci_tegra_probe(struct device *dev)
 {
-	return 0;
+	return nvgpu_init_soc_vars(get_gk20a(dev));
 }
 
 static int nvgpu_pci_tegra_remove(struct device *dev)

drivers/gpu/nvgpu/common/linux/platform_gk20a.h

@@ -121,6 +121,9 @@ struct gk20a_platform {
 	 */
 	u32 default_pri_timeout;
 
+	/* guest/vm id, needed for IPA to PA transation */
+	int vmid;
+
 	/* Initialize the platform interface of the gk20a driver.
 	 *
 	 * The platform implementation of this function must
@@ -204,6 +207,12 @@ struct gk20a_platform {
 	 */
 	void (*dump_platform_dependencies)(struct device *dev);
 
+	/* Defined when SMMU stage-2 is enabled, and we need to use physical
+	 * addresses (not IPA). This is the case for GV100 nvlink in HV+L
+	 * configuration, when dGPU is in pass-through mode.
+	 */
+	u64 (*phys_addr)(struct gk20a *g, u64 ipa);
+
 	/* Callbacks to assert/deassert GPU reset */
 	int (*reset_assert)(struct device *dev);
 	int (*reset_deassert)(struct device *dev);

drivers/gpu/nvgpu/common/linux/soc.c

@@ -13,8 +13,13 @@
 
 #include <soc/tegra/chip-id.h>
 #include <soc/tegra/tegra_bpmp.h>
+#ifdef CONFIG_TEGRA_HV_MANAGER
+#include <soc/tegra/virt/syscalls.h>
+#endif
 
 #include <nvgpu/soc.h>
+#include "os_linux.h"
+#include "platform_gk20a.h"
 
 bool nvgpu_platform_is_silicon(struct gk20a *g)
 {
@@ -40,3 +45,57 @@ bool nvgpu_is_bpmp_running(struct gk20a *g)
 {
 	return tegra_bpmp_running();
 }
+
+#ifdef CONFIG_TEGRA_HV_MANAGER
+/* When nvlink is enabled on dGPU, we need to use physical memory addresses.
+ * There is no SMMU translation. However, the device initially enumerates as a
+ * PCIe device. As such, when allocation memory for this PCIe device, the DMA
+ * framework ends up allocating memory using SMMU (if enabled in device tree).
+ * As a result, when we switch to nvlink, we need to use underlying physical
+ * addresses, even if memory mappings exist in SMMU.
+ * In addition, when stage-2 SMMU translation is enabled (for instance when HV
+ * is enabled), the addresses we get from dma_alloc are IPAs. We need to
+ * convert them to PA.
+ */
+static u64 nvgpu_tegra_hv_ipa_pa(struct gk20a *g, u64 ipa)
+{
+	struct device *dev = dev_from_gk20a(g);
+	struct gk20a_platform *platform = gk20a_get_platform(dev);
+	struct hyp_ipa_pa_info info;
+	int err;
+	u64 pa = 0ULL;
+
+	err = hyp_read_ipa_pa_info(&info, platform->vmid, ipa);
+	if (err < 0) {
+		nvgpu_err(g, "ipa=%llx translation failed vmid=%u err=%d",
+				ipa, platform->vmid, err);
+	} else {
+		pa = info.base + info.offset;
+		nvgpu_log(g, gpu_dbg_map_v,
+				"ipa=%llx vmid=%d -> pa=%llx "
+				"base=%llx offset=%llx size=%llx\n",
+				ipa, platform->vmid, pa, info.base,
+				info.offset, info.size);
+	}
+	return pa;
+}
+#endif
+
+int nvgpu_init_soc_vars(struct gk20a *g)
+{
+#ifdef CONFIG_TEGRA_HV_MANAGER
+	struct device *dev = dev_from_gk20a(g);
+	struct gk20a_platform *platform = gk20a_get_platform(dev);
+	int err;
+
+	if (nvgpu_is_hypervisor_mode(g)) {
+		err = hyp_read_gid(&platform->vmid);
+		if (err) {
+			nvgpu_err(g, "failed to read vmid");
+			return err;
+		}
+		platform->phys_addr = nvgpu_tegra_hv_ipa_pa;
+	}
+#endif
+	return 0;
+}

drivers/gpu/nvgpu/include/nvgpu/soc.h

@@ -29,5 +29,6 @@ bool nvgpu_platform_is_simulation(struct gk20a *g);
 bool nvgpu_platform_is_fpga(struct gk20a *g);
 bool nvgpu_is_hypervisor_mode(struct gk20a *g);
 bool nvgpu_is_bpmp_running(struct gk20a *g);
+int nvgpu_init_soc_vars(struct gk20a *g);
 
 #endif