Open source GPL/LGPL release

2025-12-24 10:34:43 +03:00 · 2022-07-21 16:03:29 -07:00
commit f338182221
2260 changed files with 576813 additions and 0 deletions
--- a/drivers/gpu/nvgpu/common/nvlink/init/device_reginit.c
+++ b/drivers/gpu/nvgpu/common/nvlink/init/device_reginit.c
@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/gk20a.h>
+#include <nvgpu/nvlink_device_reginit.h>
+
+#ifdef CONFIG_NVGPU_NVLINK
+int nvgpu_nvlink_reg_init(struct gk20a *g)
+{
+	int err;
+
+	err = g->ops.nvlink.reg_init(g);
+
+	return err;
+}
+#endif
--- a/drivers/gpu/nvgpu/common/nvlink/init/device_reginit_gv100.c
+++ b/drivers/gpu/nvgpu/common/nvlink/init/device_reginit_gv100.c
@@ -0,0 +1,174 @@
+/*
+ * Copyright (c) 2019-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/nvlink.h>
+#include <nvgpu/io.h>
+#include <nvgpu/gk20a.h>
+#include "device_reginit_gv100.h"
+
+#ifdef CONFIG_NVGPU_NVLINK
+struct nvlink_reginit {
+	u32 addr;
+	u32 value;
+};
+
+static const struct nvlink_reginit  nvlink_reginit_per_link_tegra[] = {
+	/* NVTLC when connected to Tegra */
+	{ 0x300U, 0x00800040U },
+	{ 0x304U, 0x00000000U },
+	{ 0x308U, 0x00000000U },
+	{ 0x30CU, 0x00000000U },
+	{ 0x310U, 0x00000000U },
+	{ 0x314U, 0x00800040U },
+	{ 0x318U, 0x00000000U },
+	{ 0x31CU, 0x00000000U },
+	{ 0x200U, 0x007F003FU },
+	{ 0x204U, 0x007F003FU },
+	{ 0x208U, 0x007F003FU },
+	{ 0x20CU, 0x007F003FU },
+	{ 0x210U, 0x007F003FU },
+	{ 0x214U, 0x00FF007FU },
+	{ 0x218U, 0x00FF007FU },
+	{ 0x21CU, 0x00FF007FU },
+	{ 0xB00U, 0x010000C0U },
+	{ 0xB04U, 0x00000000U },
+	{ 0xB08U, 0x00000000U },
+	{ 0xB0CU, 0x00000000U },
+	{ 0xB10U, 0x00000000U },
+	{ 0xB14U, 0x010000C0U },
+	{ 0xB18U, 0x00000000U },
+	{ 0xB1CU, 0x00000000U },
+	{ 0xA00U, 0x00FF00BFU },
+	{ 0xA04U, 0x00FF00BFU },
+	{ 0xA08U, 0x00FF00BFU },
+	{ 0xA0CU, 0x00FF00BFU },
+	{ 0xA10U, 0x00FF00BFU },
+	{ 0xA14U, 0x01FF017FU },
+	{ 0xA18U, 0x01FF017FU },
+	{ 0xA1CU, 0x01FF017FU },
+	{ 0x400U, 0x00000001U },
+	{ 0xC00U, 0x00000001U },
+};
+
+static const struct nvlink_reginit nvlink_reginit_per_link_gpu[] = {
+	/* NVTLC for PEER GPU */
+	{ 0x300U, 0x00800040U },
+	{ 0x304U, 0x00000000U },
+	{ 0x308U, 0x00000000U },
+	{ 0x30CU, 0x00000000U },
+	{ 0x310U, 0x00000000U },
+	{ 0x314U, 0x00800040U },
+	{ 0x318U, 0x00000000U },
+	{ 0x31CU, 0x00000000U },
+	{ 0x200U, 0x007F003FU },
+	{ 0x204U, 0x007F003FU },
+	{ 0x208U, 0x007F003FU },
+	{ 0x20CU, 0x007F003FU },
+	{ 0x210U, 0x007F003FU },
+	{ 0x214U, 0x00FF007FU },
+	{ 0x218U, 0x00FF007FU },
+	{ 0x21CU, 0x00FF007FU },
+	{ 0xB00U, 0x010000C0U },
+	{ 0xB04U, 0x00000000U },
+	{ 0xB08U, 0x00000000U },
+	{ 0xB0CU, 0x00000000U },
+	{ 0xB10U, 0x00000000U },
+	{ 0xB14U, 0x010000C0U },
+	{ 0xB18U, 0x00000000U },
+	{ 0xB1CU, 0x00000000U },
+	{ 0xA00U, 0x00FF00BFU },
+	{ 0xA04U, 0x00FF00BFU },
+	{ 0xA08U, 0x00FF00BFU },
+	{ 0xA0CU, 0x00FF00BFU },
+	{ 0xA10U, 0x00FF00BFU },
+	{ 0xA14U, 0x01FF017FU },
+	{ 0xA18U, 0x01FF017FU },
+	{ 0xA1CU, 0x01FF017FU },
+	{ 0xF04U, 0x00FFFFFFU },
+	{ 0xF0CU, 0x00FFFFFFU },
+	{ 0xF1CU, 0x003FFFFFU },
+	{ 0xF24U, 0x003FFFFFU },
+	{ 0x704U, 0x003FFFFFU },
+	{ 0x70CU, 0x003FFFFFU },
+	{ 0x400U, 0x00000001U },
+	{ 0xC00U, 0x00000001U },
+};
+
+static int gv100_nvlink_get_tlc_reginit(enum nvgpu_nvlink_endp endp,
+		const struct nvlink_reginit **reg, u32 *count)
+{
+	int ret = 0;
+
+	switch(endp) {
+	case nvgpu_nvlink_endp_tegra:
+		*reg = (const struct nvlink_reginit *)
+			nvlink_reginit_per_link_tegra;
+		*count = (u32)ARRAY_SIZE(nvlink_reginit_per_link_tegra);
+		break;
+	case nvgpu_nvlink_endp_gpu:
+		*reg = (const struct nvlink_reginit *)
+			nvlink_reginit_per_link_gpu;
+		*count = (u32)ARRAY_SIZE(nvlink_reginit_per_link_gpu);
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+
+	return ret;
+}
+
+int gv100_nvlink_reg_init(struct gk20a *g)
+{
+	u32 i = 0;
+	u32 count = 0;
+	const struct nvlink_reginit *reg;
+	enum nvgpu_nvlink_endp endp;
+	int err;
+	u32 link_id;
+	unsigned long mask = g->nvlink.enabled_links;
+	struct nvgpu_nvlink_link *link;
+	unsigned long bit;
+
+	/* Apply automated reg init flow for PROD settings */
+	for_each_set_bit(bit, &mask, NVLINK_MAX_LINKS_SW) {
+		link_id = (u32)bit;
+		link = &g->nvlink.links[link_id];
+		if (!link->remote_info.is_connected) {
+			continue;
+		}
+
+		endp = link->remote_info.device_type;
+		err = gv100_nvlink_get_tlc_reginit(endp, &reg, &count);
+		if (err != 0) {
+			nvgpu_err(g, "no reginit for endp=%u", endp);
+			continue;
+		}
+
+		for (i = 0; i < count; i++) {
+			TLC_REG_WR32(g, link_id, reg->addr, reg->value);
+			reg++;
+		}
+	}
+	return 0;
+}
+#endif /* CONFIG_NVGPU_NVLINK */
--- a/drivers/gpu/nvgpu/common/nvlink/init/device_reginit_gv100.h
+++ b/drivers/gpu/nvgpu/common/nvlink/init/device_reginit_gv100.h
@@ -0,0 +1,30 @@
+/*
+ * Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef DEVICE_REGINIT_GV100_H
+#define DEVICE_REGINIT_GV100_H
+
+struct gk20a;
+
+int gv100_nvlink_reg_init(struct gk20a *g);
+
+#endif /* DEVICE_REGINIT_GV100_H */
--- a/drivers/gpu/nvgpu/common/nvlink/link_mode_transitions.c
+++ b/drivers/gpu/nvgpu/common/nvlink/link_mode_transitions.c
@@ -0,0 +1,151 @@
+/*
+ * Copyright (c) 2019-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/gk20a.h>
+#include <nvgpu/nvlink.h>
+#include <nvgpu/nvlink_link_mode_transitions.h>
+
+#ifdef CONFIG_NVGPU_NVLINK
+/*
+ * Fix: use this function to find detault link, as only one is supported
+ * on the library for now
+ * Returns NVLINK_MAX_LINKS_SW on failure
+ */
+static u32 nvgpu_nvlink_get_link(struct gk20a *g)
+{
+	u32 link_id;
+
+	if (g == NULL) {
+		return NVLINK_MAX_LINKS_SW;
+	}
+
+	/* Lets find the detected link */
+	if (g->nvlink.initialized_links != 0U) {
+		link_id = (u32)(nvgpu_ffs(g->nvlink.initialized_links) - 1UL);
+	} else {
+		return NVLINK_MAX_LINKS_SW;
+	}
+
+	if (g->nvlink.links[link_id].remote_info.is_connected) {
+		return link_id;
+	}
+
+	return NVLINK_MAX_LINKS_SW;
+}
+
+enum nvgpu_nvlink_link_mode nvgpu_nvlink_get_link_mode(struct gk20a *g)
+{
+	u32 link_id;
+
+	link_id = nvgpu_nvlink_get_link(g);
+	if (link_id == NVLINK_MAX_LINKS_SW) {
+		return nvgpu_nvlink_link__last;
+	}
+
+	return g->ops.nvlink.link_mode_transitions.get_link_mode(g, link_id);
+}
+
+u32 nvgpu_nvlink_get_link_state(struct gk20a *g)
+{
+	u32 link_id;
+
+	link_id = nvgpu_nvlink_get_link(g);
+	if (link_id == NVLINK_MAX_LINKS_SW) {
+		/* 0xff is an undefined link_state */
+		return U32_MAX;
+	}
+
+	return g->ops.nvlink.link_mode_transitions.get_link_state(g, link_id);
+}
+
+int nvgpu_nvlink_set_link_mode(struct gk20a *g,
+					enum nvgpu_nvlink_link_mode mode)
+{
+
+	u32 link_id;
+
+	link_id = nvgpu_nvlink_get_link(g);
+	if (link_id == NVLINK_MAX_LINKS_SW) {
+		return -EINVAL;
+	}
+
+	return g->ops.nvlink.link_mode_transitions.set_link_mode(g, link_id,
+									mode);
+}
+
+void nvgpu_nvlink_get_tx_sublink_state(struct gk20a *g, u32 *state)
+{
+	u32 link_id;
+
+	link_id = nvgpu_nvlink_get_link(g);
+	if (link_id == NVLINK_MAX_LINKS_SW) {
+		return;
+	}
+	if (state != NULL) {
+		*state = g->ops.nvlink.link_mode_transitions.
+					get_tx_sublink_state(g, link_id);
+	}
+}
+
+void nvgpu_nvlink_get_rx_sublink_state(struct gk20a *g, u32 *state)
+{
+	u32 link_id;
+
+	link_id = nvgpu_nvlink_get_link(g);
+	if (link_id == NVLINK_MAX_LINKS_SW) {
+		return;
+	}
+	if (state != NULL) {
+		*state = g->ops.nvlink.link_mode_transitions.
+					get_rx_sublink_state(g, link_id);
+	}
+}
+
+enum nvgpu_nvlink_sublink_mode nvgpu_nvlink_get_sublink_mode(struct gk20a *g,
+							bool is_rx_sublink)
+{
+	u32 link_id;
+
+	link_id = nvgpu_nvlink_get_link(g);
+	if (link_id == NVLINK_MAX_LINKS_SW) {
+		return nvgpu_nvlink_sublink_rx__last;
+	}
+
+	return g->ops.nvlink.link_mode_transitions.get_sublink_mode(g,
+							link_id, is_rx_sublink);
+}
+
+int nvgpu_nvlink_set_sublink_mode(struct gk20a *g,
+			bool is_rx_sublink, enum nvgpu_nvlink_sublink_mode mode)
+{
+	u32 link_id;
+
+	link_id = nvgpu_nvlink_get_link(g);
+	if (link_id == NVLINK_MAX_LINKS_SW) {
+		return -EINVAL;
+	}
+
+	return g->ops.nvlink.link_mode_transitions.set_sublink_mode(g, link_id,
+							is_rx_sublink, mode);
+}
+
+#endif
--- a/drivers/gpu/nvgpu/common/nvlink/minion.c
+++ b/drivers/gpu/nvgpu/common/nvlink/minion.c
@@ -0,0 +1,143 @@
+/*
+ * Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/gk20a.h>
+#include <nvgpu/firmware.h>
+#include <nvgpu/timers.h>
+#include <nvgpu/nvlink_minion.h>
+
+#ifdef CONFIG_NVGPU_NVLINK
+
+/* Extract a WORD from the MINION ucode */
+u32 nvgpu_nvlink_minion_extract_word(struct nvgpu_firmware *fw, u32 idx)
+{
+	u32 out_data = 0U;
+	u8 byte = 0U;
+	u32 i = 0U;
+
+	for (i = 0U; i < 4U; i++) {
+		byte = fw->data[idx + i];
+		out_data |= ((u32)byte) << (8U * i);
+	}
+
+	return out_data;
+}
+
+/*
+ * Load minion FW and set up bootstrap
+ */
+int nvgpu_nvlink_minion_load(struct gk20a *g)
+{
+	int err = 0;
+	struct nvgpu_firmware *nvgpu_minion_fw = NULL;
+	struct nvgpu_timeout timeout;
+	u32 delay = POLL_DELAY_MIN_US;
+	bool boot_cmplte;
+
+	nvgpu_log_fn(g, " ");
+
+	if (g->ops.nvlink.minion.is_running(g)) {
+		return 0;
+	}
+
+	/* Get minion ucode binary */
+	if (g->ops.nvlink.minion.is_debug_mode != NULL) {
+		if (g->ops.nvlink.minion.is_debug_mode(g)) {
+			nvgpu_minion_fw = nvgpu_request_firmware(g,
+							"dgpu_minion_debug.bin", 0);
+		} else {
+			nvgpu_minion_fw = nvgpu_request_firmware(g,
+							"dgpu_minion_prod.bin", 0);
+		}
+	}
+
+	if (nvgpu_minion_fw == NULL) {
+		nvgpu_err(g, "minion ucode get fail");
+		err = -ENOENT;
+		goto exit;
+	}
+
+	/* Minion reset */
+	err = nvgpu_falcon_reset(&g->minion_flcn);
+	if (err != 0) {
+		nvgpu_err(g, "Minion reset failed");
+		goto exit;
+	}
+
+	/* Clear interrupts */
+	g->ops.nvlink.minion.clear_intr(g);
+
+	err = nvgpu_nvlink_minion_load_ucode(g, nvgpu_minion_fw);
+	if (err != 0) {
+		goto exit;
+	}
+
+	/* set BOOTVEC to start of non-secure code */
+	err = nvgpu_falcon_bootstrap(&g->minion_flcn, 0x0);
+	if (err != 0) {
+		nvgpu_err(g, "Minion bootstrap failed");
+		goto exit;
+	}
+
+	err = nvgpu_timeout_init(g, &timeout, nvgpu_get_poll_timeout(g),
+		NVGPU_TIMER_CPU_TIMER);
+	if (err != 0) {
+		nvgpu_err(g, "Minion boot timeout init failed");
+		goto exit;
+	}
+
+	do {
+		err = g->ops.nvlink.minion.is_boot_complete(g, &boot_cmplte);
+		if (err != 0) {
+			goto exit;
+		}
+		if (boot_cmplte) {
+			nvgpu_log(g, gpu_dbg_nvlink,"MINION boot successful");
+			break;
+		}
+		nvgpu_usleep_range(delay, delay * 2U);
+		delay = min_t(unsigned int,
+				delay << 1, POLL_DELAY_MAX_US);
+	} while (nvgpu_timeout_expired_msg(&timeout,
+						"minion boot timeout") == 0);
+
+	/* Service interrupts */
+	g->ops.nvlink.minion.falcon_isr(g);
+
+	if (nvgpu_timeout_peek_expired(&timeout)) {
+		err = -ETIMEDOUT;
+		goto exit;
+	}
+
+	g->ops.nvlink.minion.init_intr(g);
+
+	nvgpu_release_firmware(g, nvgpu_minion_fw);
+
+	return err;
+
+exit:
+	nvgpu_nvlink_free_minion_used_mem(g, nvgpu_minion_fw);
+	return err;
+}
+
+#endif
+
--- a/drivers/gpu/nvgpu/common/nvlink/nvlink.c
+++ b/drivers/gpu/nvgpu/common/nvlink/nvlink.c
@@ -0,0 +1,391 @@
+/*
+ * Copyright (c) 2018-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/gk20a.h>
+#include <nvgpu/nvlink.h>
+#include <nvgpu/nvlink_probe.h>
+#include <nvgpu/enabled.h>
+#include <nvgpu/device.h>
+#include <nvgpu/nvlink_bios.h>
+#include <nvgpu/device.h>
+#include <nvgpu/cic.h>
+#include <nvgpu/errata.h>
+
+#ifdef CONFIG_NVGPU_NVLINK
+
+static int nvgpu_nvlink_enable_links_pre_top(struct gk20a *g,
+					unsigned long links)
+{
+	u32 link_id;
+	int err;
+	unsigned long bit;
+
+	nvgpu_log(g, gpu_dbg_nvlink, " enabling 0x%lx links", links);
+	for_each_set_bit(bit, &links, NVLINK_MAX_LINKS_SW) {
+		link_id = (u32)bit;
+
+		/* Take links out of reset */
+		g->ops.nvlink.clear_link_reset(g, link_id);
+
+		/* Before  doing any link initialization, run RXDET to check
+		 * if link is connected on  other end.
+		 */
+		if (g->ops.nvlink.rxdet != NULL) {
+			err = g->ops.nvlink.rxdet(g, link_id);
+			if (err != 0) {
+				return err;
+			}
+		}
+
+		/* Enable Link DLPL for AN0 */
+		g->ops.nvlink.enable_link_an0(g, link_id);
+
+		/* This should be done by the NVLINK API */
+		err = g->ops.nvlink.link_mode_transitions.set_sublink_mode(g,
+				link_id, false, nvgpu_nvlink_sublink_tx_common);
+		if (err != 0) {
+			nvgpu_err(g, "Failed to init phy of link: %u", link_id);
+			return err;
+		}
+
+		err = g->ops.nvlink.link_mode_transitions.set_sublink_mode(g,
+			link_id, true, nvgpu_nvlink_sublink_rx_rxcal);
+		if (err != 0) {
+			nvgpu_err(g, "Failed to RXcal on link: %u", link_id);
+			return err;
+		}
+
+		err = g->ops.nvlink.link_mode_transitions.set_sublink_mode(g,
+			link_id, false, nvgpu_nvlink_sublink_tx_data_ready);
+		if (err != 0) {
+			nvgpu_err(g, "Failed to set data ready link:%u",
+				link_id);
+			return err;
+		}
+
+		g->nvlink.enabled_links |= BIT32(link_id);
+	}
+
+	nvgpu_log(g, gpu_dbg_nvlink, "enabled_links=0x%08x",
+		g->nvlink.enabled_links);
+
+	if (g->nvlink.enabled_links != 0U) {
+		return 0;
+	}
+
+	nvgpu_err(g, "No links were enabled");
+	return -EINVAL;
+}
+
+static int nvgpu_nvlink_enable_links_post_top(struct gk20a *g,
+					unsigned long links)
+{
+	u32 link_id;
+	unsigned long bit;
+	unsigned long enabled_links = (links & g->nvlink.enabled_links) &
+			~g->nvlink.initialized_links;
+
+	for_each_set_bit(bit, &enabled_links, NVLINK_MAX_LINKS_SW) {
+		link_id = (u32)bit;
+		if (nvgpu_is_errata_present(g, NVGPU_ERRATA_1888034)) {
+			g->ops.nvlink.set_sw_errata(g, link_id);
+		}
+		g->ops.nvlink.intr.init_link_err_intr(g, link_id);
+		g->ops.nvlink.intr.enable_link_err_intr(g, link_id, true);
+
+		g->nvlink.initialized_links |= BIT32(link_id);
+	};
+
+	return 0;
+}
+
+/*
+ * Main Nvlink init function. Calls into the Nvlink core API
+ */
+int nvgpu_nvlink_init(struct gk20a *g)
+{
+	int err = 0;
+
+	if (!nvgpu_is_enabled(g, NVGPU_SUPPORT_NVLINK)) {
+		return -ENODEV;
+	}
+
+	err = nvgpu_nvlink_enumerate(g);
+	if (err != 0) {
+		nvgpu_err(g, "failed to enumerate nvlink");
+		goto fail;
+	}
+
+	/* Set HSHUB and SG_PHY */
+	nvgpu_set_enabled(g, NVGPU_MM_USE_PHYSICAL_SG, true);
+
+	err = g->ops.fb.enable_nvlink(g);
+	if (err != 0) {
+		nvgpu_err(g, "failed switch to nvlink sysmem");
+		goto fail;
+	}
+
+	return err;
+
+fail:
+	nvgpu_set_enabled(g, NVGPU_MM_USE_PHYSICAL_SG, false);
+	nvgpu_set_enabled(g, NVGPU_SUPPORT_NVLINK, false);
+	return err;
+}
+
+
+/*
+ * Query IOCTRL for device discovery
+ */
+static int nvgpu_nvlink_discover_ioctrl(struct gk20a *g)
+{
+	u32 i;
+	u32 ioctrl_num_entries = 0U;
+	struct nvgpu_nvlink_ioctrl_list *ioctrl_table;
+
+	ioctrl_num_entries = nvgpu_device_count(g, NVGPU_DEVTYPE_IOCTRL);
+	nvgpu_log_info(g, "ioctrl_num_entries: %d", ioctrl_num_entries);
+
+	if (ioctrl_num_entries == 0U) {
+		nvgpu_err(g, "No NVLINK IOCTRL entry found in dev_info table");
+		return -EINVAL;
+	}
+
+	ioctrl_table = nvgpu_kzalloc(g, ioctrl_num_entries *
+				sizeof(struct nvgpu_nvlink_ioctrl_list));
+	if (ioctrl_table == NULL) {
+		nvgpu_err(g, "Failed to allocate memory for nvlink io table");
+		return -ENOMEM;
+	}
+
+	for (i = 0U; i < ioctrl_num_entries; i++) {
+		const struct nvgpu_device *dev;
+
+		dev = nvgpu_device_get(g, NVGPU_DEVTYPE_IOCTRL, i);
+		if (dev == NULL) {
+			nvgpu_err(g, "Failed to parse dev_info table IOCTRL dev (%d)",
+				  NVGPU_DEVTYPE_IOCTRL);
+			nvgpu_kfree(g, ioctrl_table);
+			return -EINVAL;
+		}
+
+		ioctrl_table[i].valid = true;
+		ioctrl_table[i].intr_enum = dev->intr_id;
+		ioctrl_table[i].reset_enum = dev->reset_id;
+		ioctrl_table[i].pri_base_addr = dev->pri_base;
+		nvgpu_log(g, gpu_dbg_nvlink,
+			"Dev %d: Pri_Base = 0x%0x Intr = %d Reset = %d",
+			i, ioctrl_table[i].pri_base_addr,
+			ioctrl_table[i].intr_enum,
+			ioctrl_table[i].reset_enum);
+	}
+	g->nvlink.ioctrl_table = ioctrl_table;
+	g->nvlink.io_num_entries = ioctrl_num_entries;
+
+	return 0;
+}
+
+
+/*
+ * Performs nvlink device level initialization by discovering the topology
+ * taking device out of reset, boot minion, set clocks up and common interrupts
+ */
+int nvgpu_nvlink_early_init(struct gk20a *g)
+{
+	int err = 0;
+
+	if (!nvgpu_is_enabled(g, NVGPU_SUPPORT_NVLINK)) {
+		return -EINVAL;
+	}
+
+	err = nvgpu_bios_get_nvlink_config_data(g);
+	if (err != 0) {
+		nvgpu_err(g, "failed to read nvlink vbios data");
+		goto exit;
+	}
+
+	err = nvgpu_nvlink_discover_ioctrl(g);
+	if (err != 0) {
+		goto exit;
+	}
+
+	/* Enable NVLINK in MC */
+	nvgpu_log(g, gpu_dbg_nvlink, "mc_reset_nvlink_mask: 0x%x",
+			BIT32(g->nvlink.ioctrl_table[0].reset_enum));
+	err = nvgpu_mc_reset_units(g, NVGPU_UNIT_NVLINK);
+	if (err != 0) {
+		nvgpu_err(g, "Failed to reset NVLINK unit");
+	}
+
+	nvgpu_cic_intr_stall_unit_config(g, NVGPU_CIC_INTR_UNIT_NVLINK,
+					NVGPU_CIC_INTR_ENABLE);
+
+	err = g->ops.nvlink.discover_link(g);
+	if ((err != 0) || (g->nvlink.discovered_links == 0U)) {
+		nvgpu_err(g, "No links available");
+		goto exit;
+	}
+
+	err = nvgpu_falcon_sw_init(g, FALCON_ID_MINION);
+	if (err != 0) {
+		nvgpu_err(g, "failed to sw init FALCON_ID_MINION");
+		goto exit;
+	}
+
+	g->nvlink.discovered_links &= ~g->nvlink.link_disable_mask;
+	nvgpu_log(g, gpu_dbg_nvlink, "link_disable_mask = 0x%08x (from VBIOS)",
+		g->nvlink.link_disable_mask);
+
+	/* Links in reset should be removed from initialized link sw state */
+	g->nvlink.initialized_links &= g->ops.nvlink.get_link_reset_mask(g);
+
+	/* VBIOS link_disable_mask should be sufficient to find the connected
+	 * links. As VBIOS is not updated with correct mask, we parse the DT
+	 * node where we hardcode the link_id. DT method is not scalable as same
+	 * DT node is used for different dGPUs connected over PCIE.
+	 * Remove the DT parsing of link id and use HAL to get link_mask based
+	 * on the GPU. This is temporary fix while we get the VBIOS updated with
+	 * correct mask.
+	 */
+	if (nvgpu_is_errata_present(g, NVGPU_ERRATA_VBIOS_NVLINK_MASK)) {
+		g->ops.nvlink.get_connected_link_mask(
+			&(g->nvlink.connected_links));
+	}
+
+	nvgpu_log(g, gpu_dbg_nvlink, "connected_links = 0x%08x",
+						g->nvlink.connected_links);
+
+	/* Track only connected links */
+	g->nvlink.discovered_links &= g->nvlink.connected_links;
+
+	nvgpu_log(g, gpu_dbg_nvlink, "discovered_links = 0x%08x (combination)",
+		g->nvlink.discovered_links);
+
+	if (hweight32(g->nvlink.discovered_links) > 1U) {
+		nvgpu_err(g, "more than one link enabled");
+		err = -EINVAL;
+		goto nvlink_init_exit;
+	}
+
+	g->nvlink.speed = nvgpu_nvlink_speed_20G;
+	err = nvgpu_nvlink_minion_load(g);
+	if (err != 0) {
+		nvgpu_err(g, "Failed Nvlink state load");
+		goto nvlink_init_exit;
+	}
+	err = g->ops.nvlink.configure_ac_coupling(g,
+					g->nvlink.ac_coupling_mask, true);
+	if (err != 0) {
+		nvgpu_err(g, "Failed AC coupling configuration");
+		goto nvlink_init_exit;
+	}
+
+	/* Program clocks */
+	g->ops.nvlink.prog_alt_clk(g);
+
+nvlink_init_exit:
+	nvgpu_falcon_sw_free(g, FALCON_ID_MINION);
+exit:
+	return err;
+}
+
+int nvgpu_nvlink_link_early_init(struct gk20a *g)
+{
+	u32 link_id;
+	int ret = 0;
+	/*
+	 * First check the topology and setup connectivity
+	 * HACK: we are only enabling one link for now!!!
+	 */
+	link_id = (u32)(nvgpu_ffs(g->nvlink.discovered_links) - 1UL);
+	g->nvlink.links[link_id].remote_info.is_connected = true;
+	g->nvlink.links[link_id].remote_info.device_type =
+							nvgpu_nvlink_endp_tegra;
+
+	ret = nvgpu_nvlink_enable_links_pre_top(g, BIT32(link_id));
+	if (ret != 0) {
+		nvgpu_err(g, "Pre topology failed for link");
+		return ret;
+	}
+	ret = nvgpu_nvlink_enable_links_post_top(g, BIT32(link_id));
+	if (ret != 0) {
+		nvgpu_err(g, "Post topology failed for link");
+		return ret;
+	}
+	return ret;
+}
+
+int nvgpu_nvlink_interface_init(struct gk20a *g)
+{
+	int err;
+
+	err = g->ops.fb.init_nvlink(g);
+	if (err != 0) {
+		nvgpu_err(g, "failed to setup nvlinks for sysmem");
+		return err;
+	}
+
+	return 0;
+}
+
+int nvgpu_nvlink_interface_disable(struct gk20a *g)
+{
+	return 0;
+}
+
+int nvgpu_nvlink_dev_shutdown(struct gk20a *g)
+{
+	nvgpu_falcon_sw_free(g, FALCON_ID_MINION);
+	return 0;
+}
+#endif
+
+int nvgpu_nvlink_remove(struct gk20a *g)
+{
+#ifdef CONFIG_NVGPU_NVLINK
+	int err;
+
+	if (!nvgpu_is_enabled(g, NVGPU_SUPPORT_NVLINK)) {
+		return -ENODEV;
+	}
+
+	nvgpu_set_enabled(g, NVGPU_SUPPORT_NVLINK, false);
+
+	err = nvgpu_nvlink_unregister_link(g);
+	if (err != 0) {
+		nvgpu_err(g, "failed on nvlink link unregistration");
+		return err;
+	}
+
+	err = nvgpu_nvlink_unregister_device(g);
+	if (err != 0) {
+		nvgpu_err(g, "failed on nvlink device unregistration");
+		return err;
+	}
+
+	nvgpu_kfree(g, g->nvlink.priv);
+
+	return 0;
+#else
+	return -ENODEV;
+#endif
+}
--- a/drivers/gpu/nvgpu/common/nvlink/probe.c
+++ b/drivers/gpu/nvgpu/common/nvlink/probe.c
@@ -0,0 +1,76 @@
+/*
+ * Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/gk20a.h>
+#include <nvgpu/enabled.h>
+#include <nvgpu/nvlink_probe.h>
+
+int nvgpu_nvlink_probe(struct gk20a *g)
+{
+#ifdef CONFIG_NVGPU_NVLINK
+	int err;
+
+	err = nvgpu_nvlink_setup_ndev(g);
+	if (err != 0) {
+		return err;
+	}
+
+	err = nvgpu_nvlink_read_dt_props(g);
+	if (err != 0) {
+		goto free_ndev;
+	}
+
+	err = nvgpu_nvlink_init_ops(g);
+	if (err != 0) {
+		goto free_ndev;
+	}
+
+	/* Register device with core driver*/
+	err = nvgpu_nvlink_register_device(g);
+	if (err != 0) {
+		nvgpu_err(g, "failed on nvlink device registration");
+		goto free_ndev;
+	}
+
+	/* Register link with core driver */
+	err = nvgpu_nvlink_register_link(g);
+	if (err != 0) {
+		nvgpu_err(g, "failed on nvlink link registration");
+		goto unregister_ndev;
+	}
+
+	/* Enable NVLINK support */
+	nvgpu_set_enabled(g, NVGPU_SUPPORT_NVLINK, true);
+	return 0;
+
+unregister_ndev:
+	err = nvgpu_nvlink_unregister_device(g);
+
+free_ndev:
+	nvgpu_kfree(g, g->nvlink.priv);
+	g->nvlink.priv = NULL;
+	return err;
+#else
+	return -ENODEV;
+#endif
+}
+