nv-tegra-mirror/linux-nvgpu
1
0
Fork 0
mirror of git://nv-tegra.nvidia.com/linux-nvgpu.git synced 2025-12-24 02:22:34 +03:00
Code Issues Packages Projects Releases Wiki Activity
Files
3b75042842348a72d2e47d1d612cb1582ad7124e
linux-nvgpu/drivers/gpu/nvgpu/common/nvlink/nvlink_gv100.c
Tejal Kudav a307b6eb77 gpu: nvgpu: Move nvlink HAL files to common/nvlink 
Move the nvlink HAL code to unit specific directory as part
of nvgpu restructing.
This move is done after removing usage of other unit's hardware
headers from nvlink. Also confirmed that no other unit files are
including nvlink hardware headers.

JIRA NVGPU-966

Change-Id: I301e3f8de37c5792a3e1e799b97e5fdfc131f058
Signed-off-by: Tejal Kudav <tkudav@nvidia.com>
Reviewed-on: https://git-master.nvidia.com/r/1975259
Reviewed-by: mobile promotions <svcmobile_promotions@nvidia.com>
Tested-by: mobile promotions <svcmobile_promotions@nvidia.com>
2018-12-21 13:24:19 -08:00

2759 lines
79 KiB
C
Raw Blame History

/*
* Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
#ifdef CONFIG_TEGRA_NVLINK
#include <nvgpu/nvgpu_common.h>
#include <nvgpu/bios.h>
#include <nvgpu/firmware.h>
#include <nvgpu/bitops.h>
#include <nvgpu/nvlink.h>
#include <nvgpu/enabled.h>
#include <nvgpu/io.h>
#include <nvgpu/utils.h>
#include <nvgpu/timers.h>
#include <nvgpu/gk20a.h>
#include <nvgpu/top.h>
#include "nvlink_gv100.h"
#include <nvgpu/hw/gv100/hw_nvlinkip_discovery_gv100.h>
#include <nvgpu/hw/gv100/hw_nvlipt_gv100.h>
#include <nvgpu/hw/gv100/hw_ioctrl_gv100.h>
#include <nvgpu/hw/gv100/hw_minion_gv100.h>
#include <nvgpu/hw/gv100/hw_nvl_gv100.h>
#include <nvgpu/hw/gv100/hw_ioctrlmif_gv100.h>
#include <nvgpu/hw/gv100/hw_trim_gv100.h>
#include <nvgpu/hw/gv100/hw_nvtlc_gv100.h>
#define NVLINK_PLL_ON_TIMEOUT_MS 30
#define NVLINK_SUBLINK_TIMEOUT_MS 200
/*
* The manuals are missing some useful defines
* we add them for now
*/
#define IPT_INTR_CONTROL_LINK(i) (nvlipt_intr_control_link0_r() + (i)*4)
#define IPT_ERR_UC_STATUS_LINK(i) (nvlipt_err_uc_status_link0_r() + (i)*36)
#define IPT_ERR_UC_MASK_LINK(i) (nvlipt_err_uc_mask_link0_r() + (i)*36)
#define IPT_ERR_UC_SEVERITY_LINK(i) (nvlipt_err_uc_severity_link0_r() + (i)*36)
#define IPT_ERR_UC_FIRST_LINK(i) (nvlipt_err_uc_first_link0_r() + (i)*36)
#define IPT_ERR_UC_ADVISORY_LINK(i) (nvlipt_err_uc_advisory_link0_r() + (i)*36)
#define IPT_ERR_C_STATUS_LINK(i) (nvlipt_err_c_status_link0_r() + (i)*36)
#define IPT_ERR_C_MASK_LINK(i) (nvlipt_err_c_mask_link0_r() + (i)*36)
#define IPT_ERR_C_FIRST_LINK(i) (nvlipt_err_c_first_link0_r() + (i)*36)
#define IPT_ERR_CONTROL_LINK(i) (nvlipt_err_control_link0_r() + (i)*4)
#define IPT_ERR_UC_ACTIVE_BITS (nvlipt_err_uc_status_link0_dlprotocol_f(1) | \
nvlipt_err_uc_status_link0_datapoisoned_f(1) | \
nvlipt_err_uc_status_link0_flowcontrol_f(1) | \
nvlipt_err_uc_status_link0_responsetimeout_f(1) | \
nvlipt_err_uc_status_link0_targeterror_f(1) | \
nvlipt_err_uc_status_link0_unexpectedresponse_f(1) | \
nvlipt_err_uc_status_link0_receiveroverflow_f(1) | \
nvlipt_err_uc_status_link0_malformedpacket_f(1) | \
nvlipt_err_uc_status_link0_stompedpacketreceived_f(1) | \
nvlipt_err_uc_status_link0_unsupportedrequest_f(1) | \
nvlipt_err_uc_status_link0_ucinternal_f(1))
#define MINION_FALCON_INTR_MASK (minion_falcon_irqmset_wdtmr_set_f() | \
minion_falcon_irqmset_halt_set_f() | \
minion_falcon_irqmset_exterr_set_f()| \
minion_falcon_irqmset_swgen0_set_f()| \
minion_falcon_irqmset_swgen1_set_f())
#define MINION_FALCON_INTR_DEST ( \
minion_falcon_irqdest_host_wdtmr_host_f() | \
minion_falcon_irqdest_host_halt_host_f() | \
minion_falcon_irqdest_host_exterr_host_f() | \
minion_falcon_irqdest_host_swgen0_host_f() | \
minion_falcon_irqdest_host_swgen1_host_f() | \
minion_falcon_irqdest_target_wdtmr_host_normal_f() | \
minion_falcon_irqdest_target_halt_host_normal_f() | \
minion_falcon_irqdest_target_exterr_host_normal_f() | \
minion_falcon_irqdest_target_swgen0_host_normal_f() | \
minion_falcon_irqdest_target_swgen1_host_normal_f())
struct __nvlink_reginit {
u32 addr;
u32 value;
};
static const struct __nvlink_reginit __nvlink_reginit_per_link_tegra[] = {
/* NVTLC when connected to Tegra */
{ 0x300U, 0x00800040U },
{ 0x304U, 0x00000000U },
{ 0x308U, 0x00000000U },
{ 0x30CU, 0x00000000U },
{ 0x310U, 0x00000000U },
{ 0x314U, 0x00800040U },
{ 0x318U, 0x00000000U },
{ 0x31CU, 0x00000000U },
{ 0x200U, 0x007F003FU },
{ 0x204U, 0x007F003FU },
{ 0x208U, 0x007F003FU },
{ 0x20CU, 0x007F003FU },
{ 0x210U, 0x007F003FU },
{ 0x214U, 0x00FF007FU },
{ 0x218U, 0x00FF007FU },
{ 0x21CU, 0x00FF007FU },
{ 0xB00U, 0x010000C0U },
{ 0xB04U, 0x00000000U },
{ 0xB08U, 0x00000000U },
{ 0xB0CU, 0x00000000U },
{ 0xB10U, 0x00000000U },
{ 0xB14U, 0x010000C0U },
{ 0xB18U, 0x00000000U },
{ 0xB1CU, 0x00000000U },
{ 0xA00U, 0x00FF00BFU },
{ 0xA04U, 0x00FF00BFU },
{ 0xA08U, 0x00FF00BFU },
{ 0xA0CU, 0x00FF00BFU },
{ 0xA10U, 0x00FF00BFU },
{ 0xA14U, 0x01FF017FU },
{ 0xA18U, 0x01FF017FU },
{ 0xA1CU, 0x01FF017FU },
{ 0xF04U, 0x00FFFFFFU },
{ 0xF0CU, 0x00FFFFFFU },
{ 0xF1CU, 0x003FFFFFU },
{ 0xF24U, 0x003FFFFFU },
{ 0x704U, 0x003FFFFFU },
{ 0x70CU, 0x003FFFFFU },
{ 0x400U, 0x00000001U },
{ 0xC00U, 0x00000001U },
};
static const struct __nvlink_reginit __nvlink_reginit_per_link_gpu[] = {
/* NVTLC for PEER GPU */
{ 0x300U, 0x00800040U },
{ 0x304U, 0x00000000U },
{ 0x308U, 0x00000000U },
{ 0x30CU, 0x00000000U },
{ 0x310U, 0x00000000U },
{ 0x314U, 0x00800040U },
{ 0x318U, 0x00000000U },
{ 0x31CU, 0x00000000U },
{ 0x200U, 0x007F003FU },
{ 0x204U, 0x007F003FU },
{ 0x208U, 0x007F003FU },
{ 0x20CU, 0x007F003FU },
{ 0x210U, 0x007F003FU },
{ 0x214U, 0x00FF007FU },
{ 0x218U, 0x00FF007FU },
{ 0x21CU, 0x00FF007FU },
{ 0xB00U, 0x010000C0U },
{ 0xB04U, 0x00000000U },
{ 0xB08U, 0x00000000U },
{ 0xB0CU, 0x00000000U },
{ 0xB10U, 0x00000000U },
{ 0xB14U, 0x010000C0U },
{ 0xB18U, 0x00000000U },
{ 0xB1CU, 0x00000000U },
{ 0xA00U, 0x00FF00BFU },
{ 0xA04U, 0x00FF00BFU },
{ 0xA08U, 0x00FF00BFU },
{ 0xA0CU, 0x00FF00BFU },
{ 0xA10U, 0x00FF00BFU },
{ 0xA14U, 0x01FF017FU },
{ 0xA18U, 0x01FF017FU },
{ 0xA1CU, 0x01FF017FU },
{ 0xF04U, 0x00FFFFFFU },
{ 0xF0CU, 0x00FFFFFFU },
{ 0xF1CU, 0x003FFFFFU },
{ 0xF24U, 0x003FFFFFU },
{ 0x704U, 0x003FFFFFU },
{ 0x70CU, 0x003FFFFFU },
{ 0x400U, 0x00000001U },
{ 0xC00U, 0x00000001U },
};
#define NVL_DEVICE(str) nvlinkip_discovery_common_device_##str##_v()
static const char *__gv100_device_type_to_str(u32 type)
{
if (type == NVL_DEVICE(ioctrl))
return "IOCTRL";
if (type == NVL_DEVICE(dlpl))
return "DL/PL";
if (type == NVL_DEVICE(nvltlc))
return "NVLTLC";
if (type == NVL_DEVICE(ioctrlmif))
return "IOCTRLMIF";
if (type == NVL_DEVICE(nvlipt))
return "NVLIPT";
if (type == NVL_DEVICE(minion))
return "MINION";
if (type == NVL_DEVICE(dlpl_multicast))
return "DL/PL MULTICAST";
if (type == NVL_DEVICE(nvltlc_multicast))
return "NVLTLC MULTICAST";
if (type == NVL_DEVICE(ioctrlmif_multicast))
return "IOCTRLMIF MULTICAST";
if (type == NVL_DEVICE(nvltlc_multicast))
return "NVLTLC MULTICAST";
return "UNKNOWN";
}
/*
* Function prototypes
*/
static u32 __gv100_nvlink_get_link_reset_mask(struct gk20a *g);
static int gv100_nvlink_rxcal_en(struct gk20a *g, unsigned long mask);
/*
*******************************************************************************
* IP specific functions *
*******************************************************************************
*/
/*
*-----------------------------------------------------------------------------*
* MINION API
*-----------------------------------------------------------------------------*
*/
/*
* Initialization of link specific interrupts
*/
static void gv100_nvlink_minion_link_intr_enable(struct gk20a *g, u32 link_id,
bool enable)
{
u32 intr, links;
/* Only stall interrupts for now */
intr = MINION_REG_RD32(g, minion_minion_intr_stall_en_r());
links = minion_minion_intr_stall_en_link_v(intr);
if (enable)
links |= BIT(link_id);
else
links &= ~BIT(link_id);
intr = set_field(intr, minion_minion_intr_stall_en_link_m(),
minion_minion_intr_stall_en_link_f(links));
MINION_REG_WR32(g, minion_minion_intr_stall_en_r(), intr);
}
/*
* Initialization of falcon interrupts
*/
static void gv100_nvlink_minion_falcon_intr_enable(struct gk20a *g, bool enable)
{
u32 reg;
reg = MINION_REG_RD32(g, minion_minion_intr_stall_en_r());
if (enable) {
reg = set_field(reg, minion_minion_intr_stall_en_fatal_m(),
minion_minion_intr_stall_en_fatal_enable_f());
reg = set_field(reg, minion_minion_intr_stall_en_nonfatal_m(),
minion_minion_intr_stall_en_nonfatal_enable_f());
reg = set_field(reg, minion_minion_intr_stall_en_falcon_stall_m(),
minion_minion_intr_stall_en_falcon_stall_enable_f());
reg = set_field(reg, minion_minion_intr_stall_en_falcon_nostall_m(),
minion_minion_intr_stall_en_falcon_nostall_enable_f());
} else {
reg = set_field(reg, minion_minion_intr_stall_en_fatal_m(),
minion_minion_intr_stall_en_fatal_disable_f());
reg = set_field(reg, minion_minion_intr_stall_en_nonfatal_m(),
minion_minion_intr_stall_en_nonfatal_disable_f());
reg = set_field(reg, minion_minion_intr_stall_en_falcon_stall_m(),
minion_minion_intr_stall_en_falcon_stall_disable_f());
reg = set_field(reg, minion_minion_intr_stall_en_falcon_nostall_m(),
minion_minion_intr_stall_en_falcon_nostall_disable_f());
}
MINION_REG_WR32(g, minion_minion_intr_stall_en_r(), reg);
}
/*
* Initialize minion IP interrupts
*/
static void gv100_nvlink_initialize_minion(struct gk20a *g)
{
/* Disable non-stall tree */
MINION_REG_WR32(g, minion_minion_intr_nonstall_en_r(), 0x0);
gv100_nvlink_minion_falcon_intr_enable(g, true);
}
/*
* Check if minion is up
*/
static bool __gv100_nvlink_minion_is_running(struct gk20a *g)
{
/* if minion is booted and not halted, it is running */
if ((MINION_REG_RD32(g, minion_minion_status_r()) &
minion_minion_status_status_f(1)) &&
(!minion_falcon_irqstat_halt_v(
MINION_REG_RD32(g, minion_falcon_irqstat_r()))))
return true;
return false;
}
/*
* Falcon specific ISR handling
*/
static bool gv100_nvlink_minion_falcon_isr(struct gk20a *g)
{
u32 intr;
intr = MINION_REG_RD32(g, minion_falcon_irqstat_r()) &
MINION_REG_RD32(g, minion_falcon_irqmask_r());
if (!intr)
return true;
if (intr & minion_falcon_irqstat_exterr_true_f()) {
nvgpu_err(g, "FALCON EXT ADDR: 0x%x 0x%x 0x%x",
MINION_REG_RD32(g, 0x244),
MINION_REG_RD32(g, 0x248),
MINION_REG_RD32(g, 0x24c));
}
MINION_REG_WR32(g, minion_falcon_irqsclr_r(), intr);
nvgpu_err(g, "FATAL minion IRQ: 0x%08x", intr);
intr = MINION_REG_RD32(g, minion_falcon_irqstat_r()) &
MINION_REG_RD32(g, minion_falcon_irqmask_r());
return (intr == 0);
}
/*
* Link Specific ISR
*/
static bool gv100_nvlink_minion_link_isr(struct gk20a *g, u32 link_id)
{
u32 intr, code;
bool fatal = false;
intr = MINION_REG_RD32(g, minion_nvlink_link_intr_r(link_id));
code = minion_nvlink_link_intr_code_v(intr);
if (code == minion_nvlink_link_intr_code_swreq_v()) {
nvgpu_err(g, " Intr SWREQ, link: %d subcode: %x",
link_id, minion_nvlink_link_intr_subcode_v(intr));
} else if (code == minion_nvlink_link_intr_code_pmdisabled_v()) {
nvgpu_err(g, " Fatal Intr PMDISABLED, link: %d subcode: %x",
link_id, minion_nvlink_link_intr_subcode_v(intr));
fatal = true;
} else if (code == minion_nvlink_link_intr_code_na_v()) {
nvgpu_err(g, " Fatal Intr NA, link: %d subcode: %x",
link_id, minion_nvlink_link_intr_subcode_v(intr));
fatal = true;
} else if (code == minion_nvlink_link_intr_code_dlreq_v()) {
nvgpu_err(g, " Fatal Intr DLREQ, link: %d subcode: %x",
link_id, minion_nvlink_link_intr_subcode_v(intr));
fatal = true;
} else {
nvgpu_err(g, " Fatal Intr UNKN:%x, link: %d subcode: %x", code,
link_id, minion_nvlink_link_intr_subcode_v(intr));
fatal = true;
}
if (fatal)
gv100_nvlink_minion_link_intr_enable(g, link_id, false);
intr = set_field(intr, minion_nvlink_link_intr_state_m(),
minion_nvlink_link_intr_state_f(1));
MINION_REG_WR32(g, minion_nvlink_link_intr_r(link_id), intr);
return true;
}
/*
* Global minion routine to service interrupts
*/
static bool gv100_nvlink_minion_isr(struct gk20a *g) {
u32 intr, i;
unsigned long links;
intr = MINION_REG_RD32(g, minion_minion_intr_r()) &
MINION_REG_RD32(g, minion_minion_intr_stall_en_r());
if (minion_minion_intr_falcon_stall_v(intr) ||
minion_minion_intr_falcon_nostall_v(intr))
gv100_nvlink_minion_falcon_isr(g);
if (minion_minion_intr_fatal_v(intr)) {
gv100_nvlink_minion_falcon_intr_enable(g, false);
MINION_REG_WR32(g, minion_minion_intr_r(),
minion_minion_intr_fatal_f(1));
}
if (minion_minion_intr_nonfatal_v(intr))
MINION_REG_WR32(g, minion_minion_intr_r(),
minion_minion_intr_nonfatal_f(1));
links = minion_minion_intr_link_v(intr) & g->nvlink.enabled_links;
if (links)
for_each_set_bit(i, &links, 32)
gv100_nvlink_minion_link_isr(g, i);
/* Re-test interrupt status */
intr = MINION_REG_RD32(g, minion_minion_intr_r()) &
MINION_REG_RD32(g, minion_minion_intr_stall_en_r());
return (intr == 0);
}
/* Extract a WORD from the MINION ucode */
static inline u32 minion_extract_word(struct nvgpu_firmware *fw, u32 idx)
{
u32 out_data = 0;
u8 byte = 0;
u32 i = 0;
for (i = 0; i < 4; i++) {
byte = fw->data[idx + i];
out_data |= ((u32)byte) << (8 * i);
}
return out_data;
}
/*
* Load minion FW and set up bootstrap
*/
static u32 gv100_nvlink_minion_load(struct gk20a *g)
{
u32 err = 0;
struct nvlink_device *ndev = (struct nvlink_device *) g->nvlink.priv;
struct nvgpu_firmware *nvgpu_minion_fw = NULL;
struct minion_hdr *minion_hdr = &ndev->minion_hdr;
u32 data_idx = 0;
u32 app = 0;
struct nvgpu_timeout timeout;
u32 delay = GR_IDLE_CHECK_DEFAULT;
u32 reg;
nvgpu_log_fn(g, " ");
if (__gv100_nvlink_minion_is_running(g))
return 0;
/* get mem unlock ucode binary */
nvgpu_minion_fw = nvgpu_request_firmware(g, "minion.bin", 0);
if (!nvgpu_minion_fw) {
nvgpu_err(g, "minion ucode get fail");
err = -ENOENT;
goto exit;
}
/* nvdec falcon reset */
nvgpu_falcon_reset(&g->minion_flcn);
/* Read ucode header */
minion_hdr->os_code_offset = minion_extract_word(nvgpu_minion_fw,
data_idx);
data_idx += 4;
minion_hdr->os_code_size = minion_extract_word(nvgpu_minion_fw,
data_idx);
data_idx += 4;
minion_hdr->os_data_offset = minion_extract_word(nvgpu_minion_fw,
data_idx);
data_idx += 4;
minion_hdr->os_data_size = minion_extract_word(nvgpu_minion_fw,
data_idx);
data_idx += 4;
minion_hdr->num_apps = minion_extract_word(nvgpu_minion_fw,
data_idx);
data_idx += 4;
nvgpu_log(g, gpu_dbg_nvlink,
"MINION Ucode Header Info:");
nvgpu_log(g, gpu_dbg_nvlink,
"-------------------------");
nvgpu_log(g, gpu_dbg_nvlink,
" - OS Code Offset = %u", minion_hdr->os_code_offset);
nvgpu_log(g, gpu_dbg_nvlink,
" - OS Code Size = %u", minion_hdr->os_code_size);
nvgpu_log(g, gpu_dbg_nvlink,
" - OS Data Offset = %u", minion_hdr->os_data_offset);
nvgpu_log(g, gpu_dbg_nvlink,
" - OS Data Size = %u", minion_hdr->os_data_size);
nvgpu_log(g, gpu_dbg_nvlink,
" - Num Apps = %u", minion_hdr->num_apps);
/* Allocate offset/size arrays for all the ucode apps */
minion_hdr->app_code_offsets = nvgpu_kcalloc(g,
minion_hdr->num_apps,
sizeof(u32));
if (!minion_hdr->app_code_offsets) {
nvgpu_err(g, "Couldn't allocate MINION app_code_offsets array");
err = -ENOMEM;
goto exit;
}
minion_hdr->app_code_sizes = nvgpu_kcalloc(g,
minion_hdr->num_apps,
sizeof(u32));
if (!minion_hdr->app_code_sizes) {
nvgpu_err(g, "Couldn't allocate MINION app_code_sizes array");
err = -ENOMEM;
goto exit;
}
minion_hdr->app_data_offsets = nvgpu_kcalloc(g,
minion_hdr->num_apps,
sizeof(u32));
if (!minion_hdr->app_data_offsets) {
nvgpu_err(g, "Couldn't allocate MINION app_data_offsets array");
err = -ENOMEM;
goto exit;
}
minion_hdr->app_data_sizes = nvgpu_kcalloc(g,
minion_hdr->num_apps,
sizeof(u32));
if (!minion_hdr->app_data_sizes) {
nvgpu_err(g, "Couldn't allocate MINION app_data_sizes array");
err = -ENOMEM;
goto exit;
}
/* Get app code offsets and sizes */
for (app = 0; app < minion_hdr->num_apps; app++) {
minion_hdr->app_code_offsets[app] =
minion_extract_word(nvgpu_minion_fw, data_idx);
data_idx += 4;
minion_hdr->app_code_sizes[app] =
minion_extract_word(nvgpu_minion_fw, data_idx);
data_idx += 4;
nvgpu_log(g, gpu_dbg_nvlink,
" - App Code:");
nvgpu_log(g, gpu_dbg_nvlink,
" - App #%d: Code Offset = %u, Code Size = %u",
app,
minion_hdr->app_code_offsets[app],
minion_hdr->app_code_sizes[app]);
}
/* Get app data offsets and sizes */
for (app = 0; app < minion_hdr->num_apps; app++) {
minion_hdr->app_data_offsets[app] =
minion_extract_word(nvgpu_minion_fw, data_idx);
data_idx += 4;
minion_hdr->app_data_sizes[app] =
minion_extract_word(nvgpu_minion_fw, data_idx);
data_idx += 4;
nvgpu_log(g, gpu_dbg_nvlink,
" - App Data:");
nvgpu_log(g, gpu_dbg_nvlink,
" - App #%d: Data Offset = %u, Data Size = %u",
app,
minion_hdr->app_data_offsets[app],
minion_hdr->app_data_sizes[app]);
}
minion_hdr->ovl_offset = minion_extract_word(nvgpu_minion_fw, data_idx);
data_idx += 4;
minion_hdr->ovl_size = minion_extract_word(nvgpu_minion_fw, data_idx);
data_idx += 4;
ndev->minion_img = &(nvgpu_minion_fw->data[data_idx]);
minion_hdr->ucode_data_size = nvgpu_minion_fw->size - data_idx;
nvgpu_log(g, gpu_dbg_nvlink,
" - Overlay Offset = %u", minion_hdr->ovl_offset);
nvgpu_log(g, gpu_dbg_nvlink,
" - Overlay Size = %u", minion_hdr->ovl_size);
nvgpu_log(g, gpu_dbg_nvlink,
" - Ucode Data Size = %u", minion_hdr->ucode_data_size);
/* Clear interrupts */
nvgpu_falcon_set_irq(&g->minion_flcn, true, MINION_FALCON_INTR_MASK,
MINION_FALCON_INTR_DEST);
/* Copy Non Secure IMEM code */
nvgpu_falcon_copy_to_imem(&g->minion_flcn, 0,
(u8 *)&ndev->minion_img[minion_hdr->os_code_offset],
minion_hdr->os_code_size, 0, false,
GET_IMEM_TAG(minion_hdr->os_code_offset));
/* Copy Non Secure DMEM code */
nvgpu_falcon_copy_to_dmem(&g->minion_flcn, 0,
(u8 *)&ndev->minion_img[minion_hdr->os_data_offset],
minion_hdr->os_data_size, 0);
/* Load the apps securely */
for (app = 0; app < minion_hdr->num_apps; app++) {
u32 app_code_start = minion_hdr->app_code_offsets[app];
u32 app_code_size = minion_hdr->app_code_sizes[app];
u32 app_data_start = minion_hdr->app_data_offsets[app];
u32 app_data_size = minion_hdr->app_data_sizes[app];
if (app_code_size)
nvgpu_falcon_copy_to_imem(&g->minion_flcn,
app_code_start,
(u8 *)&ndev->minion_img[app_code_start],
app_code_size, 0, true,
GET_IMEM_TAG(app_code_start));
if (app_data_size)
nvgpu_falcon_copy_to_dmem(&g->minion_flcn,
app_data_start,
(u8 *)&ndev->minion_img[app_data_start],
app_data_size, 0);
}
/* set BOOTVEC to start of non-secure code */
nvgpu_falcon_bootstrap(&g->minion_flcn, 0x0);
nvgpu_timeout_init(g, &timeout, gk20a_get_gr_idle_timeout(g),
NVGPU_TIMER_CPU_TIMER);
do {
reg = MINION_REG_RD32(g, minion_minion_status_r());
if (minion_minion_status_status_v(reg)) {
/* Minion sequence completed, check status */
if (minion_minion_status_status_v(reg) !=
minion_minion_status_status_boot_v()) {
nvgpu_err(g, "MINION init sequence failed: 0x%x",
minion_minion_status_status_v(reg));
err = -EINVAL;
goto exit;
}
nvgpu_log(g, gpu_dbg_nvlink,
"MINION boot successful: 0x%x", reg);
err = 0;
break;
}
nvgpu_usleep_range(delay, delay * 2);
delay = min_t(unsigned long,
delay << 1, GR_IDLE_CHECK_MAX);
} while (!nvgpu_timeout_expired_msg(&timeout, " minion boot timeout"));
/* Service interrupts */
gv100_nvlink_minion_falcon_isr(g);
if (nvgpu_timeout_peek_expired(&timeout)) {
err = -ETIMEDOUT;
goto exit;
}
gv100_nvlink_initialize_minion(g);
exit:
nvgpu_kfree(g, minion_hdr->app_code_offsets);
nvgpu_kfree(g, minion_hdr->app_code_sizes);
nvgpu_kfree(g, minion_hdr->app_data_offsets);
nvgpu_kfree(g, minion_hdr->app_data_sizes);
if (nvgpu_minion_fw) {
nvgpu_release_firmware(g, nvgpu_minion_fw);
ndev->minion_img = NULL;
}
return err;
}
/*
* Check if MINION command is complete
*/
static u32 gv100_nvlink_minion_command_complete(struct gk20a *g, u32 link_id)
{
u32 reg;
struct nvgpu_timeout timeout;
u32 delay = GR_IDLE_CHECK_DEFAULT;
nvgpu_timeout_init(g, &timeout, gk20a_get_gr_idle_timeout(g),
NVGPU_TIMER_CPU_TIMER);
do {
reg = MINION_REG_RD32(g, minion_nvlink_dl_cmd_r(link_id));
if (minion_nvlink_dl_cmd_ready_v(reg) == 1) {
/* Command completed, check sucess */
if (minion_nvlink_dl_cmd_fault_v(reg) == 1) {
nvgpu_err(g, "minion cmd(%d) error: 0x%x",
link_id, reg);
reg = minion_nvlink_dl_cmd_fault_f(1);
MINION_REG_WR32(g,
minion_nvlink_dl_cmd_r(link_id), reg);
return -EINVAL;
}
/* Commnand success */
break;
}
nvgpu_usleep_range(delay, delay * 2);
delay = min_t(unsigned long,
delay << 1, GR_IDLE_CHECK_MAX);
} while (!nvgpu_timeout_expired_msg(&timeout, " minion cmd timeout"));
if (nvgpu_timeout_peek_expired(&timeout))
return -ETIMEDOUT;
nvgpu_log(g, gpu_dbg_nvlink, "minion cmd Complete");
return 0;
}
/*
* Send Minion command (can be async)
*/
int gv100_nvlink_minion_send_command(struct gk20a *g, u32 link_id,
u32 command, u32 scratch_0, bool sync)
{
int err = 0;
/* Check last command succeded */
err = gv100_nvlink_minion_command_complete(g, link_id);
if (err != 0)
return -EINVAL;
nvgpu_log(g, gpu_dbg_nvlink,
"sending MINION command 0x%x to link %d", command, link_id);
if (command == minion_nvlink_dl_cmd_command_configeom_v())
MINION_REG_WR32(g, minion_misc_0_r(),
minion_misc_0_scratch_swrw_0_f(scratch_0));
MINION_REG_WR32(g, minion_nvlink_dl_cmd_r(link_id),
minion_nvlink_dl_cmd_command_f(command) |
minion_nvlink_dl_cmd_fault_f(1));
if (sync)
err = gv100_nvlink_minion_command_complete(g, link_id);
return err;
}
/* MINION API COMMANDS */
/*
* Init UPHY
*/
static int gv100_nvlink_minion_init_uphy(struct gk20a *g, unsigned long mask,
bool sync)
{
int err = 0;
u32 link_id, master_pll, slave_pll;
u32 master_state, slave_state;
unsigned long link_enable;
link_enable = __gv100_nvlink_get_link_reset_mask(g);
for_each_set_bit(link_id, &mask, 32) {
master_pll = g->nvlink.links[link_id].pll_master_link_id;
slave_pll = g->nvlink.links[link_id].pll_slave_link_id;
master_state = nvl_link_state_state_init_v();
slave_state = nvl_link_state_state_init_v();
if (BIT(master_pll) & link_enable)
master_state = nvl_link_state_state_v(
g->ops.nvlink.link_get_state(g, master_pll));
if (BIT(slave_pll) & link_enable)
slave_state = nvl_link_state_state_v(
g->ops.nvlink.link_get_state(g, slave_pll));
if ((slave_state != nvl_link_state_state_init_v()) ||
(master_state != nvl_link_state_state_init_v())) {
nvgpu_err(g, "INIT PLL can only be executed when both "
"master and slave links are in init state");
return -EINVAL;
}
/* Check if INIT PLL is done on link */
if (!(BIT(master_pll) & g->nvlink.init_pll_done)) {
err = gv100_nvlink_minion_send_command(g, master_pll,
g->nvlink.initpll_cmd, 0, sync);
if (err != 0) {
nvgpu_err(g, " Error sending INITPLL to minion");
return err;
}
g->nvlink.init_pll_done |= BIT(master_pll);
}
}
err = g->ops.nvlink.setup_pll(g, mask);
if (err != 0) {
nvgpu_err(g, "Error setting up PLL");
return err;
}
/* INITPHY commands */
for_each_set_bit(link_id, &mask, 32) {
err = gv100_nvlink_minion_send_command(g, link_id,
minion_nvlink_dl_cmd_command_initphy_v(), 0, sync);
if (err != 0) {
nvgpu_err(g, "Error on INITPHY minion DL command %u",
link_id);
return err;
}
}
return 0;
}
/*
* Configure AC coupling
*/
static int gv100_nvlink_minion_configure_ac_coupling(struct gk20a *g,
unsigned long mask, bool sync)
{
int err = 0;
u32 i;
u32 temp;
for_each_set_bit(i, &mask, 32) {
temp = DLPL_REG_RD32(g, i, nvl_link_config_r());
temp &= ~nvl_link_config_ac_safe_en_m();
temp |= nvl_link_config_ac_safe_en_on_f();
DLPL_REG_WR32(g, i, nvl_link_config_r(), temp);
err = gv100_nvlink_minion_send_command(g, i,
minion_nvlink_dl_cmd_command_setacmode_v(), 0, sync);
if (err != 0)
return err;
}
return err;
}
/*
* Set Data ready
*/
int gv100_nvlink_minion_data_ready_en(struct gk20a *g,
unsigned long link_mask, bool sync)
{
int ret = 0;
u32 link_id;
for_each_set_bit(link_id, &link_mask, 32) {
ret = gv100_nvlink_minion_send_command(g, link_id,
minion_nvlink_dl_cmd_command_initlaneenable_v(), 0,
sync);
if (ret != 0) {
nvgpu_err(g, "Failed initlaneenable on link %u",
link_id);
return ret;
}
}
for_each_set_bit(link_id, &link_mask, 32) {
ret = gv100_nvlink_minion_send_command(g, link_id,
minion_nvlink_dl_cmd_command_initdlpl_v(), 0, sync);
if (ret != 0) {
nvgpu_err(g, "Failed initdlpl on link %u", link_id);
return ret;
}
}
return ret;
}
/*
* Request that minion disable the lane
*/
static int gv100_nvlink_minion_lane_disable(struct gk20a *g, u32 link_id,
bool sync)
{
int err = 0;
err = gv100_nvlink_minion_send_command(g, link_id,
minion_nvlink_dl_cmd_command_lanedisable_v(), 0, sync);
if (err != 0)
nvgpu_err(g, " failed to disable lane on %d", link_id);
return err;
}
/*
* Request that minion shutdown the lane
*/
static int gv100_nvlink_minion_lane_shutdown(struct gk20a *g, u32 link_id,
bool sync)
{
int err = 0;
err = gv100_nvlink_minion_send_command(g, link_id,
minion_nvlink_dl_cmd_command_laneshutdown_v(), 0, sync);
if (err != 0)
nvgpu_err(g, " failed to shutdown lane on %d", link_id);
return err;
}
/*
*-----------------------------------------------------------------------------*
* TLC API
*-----------------------------------------------------------------------------*
*/
static int gv100_nvlink_get_tlc_reginit(enum nvgpu_nvlink_endp endp,
struct __nvlink_reginit **reg, u32 *count)
{
switch(endp) {
case nvgpu_nvlink_endp_tegra:
*reg = (struct __nvlink_reginit *)
__nvlink_reginit_per_link_tegra;
*count = ARRAY_SIZE(__nvlink_reginit_per_link_tegra);
break;
case nvgpu_nvlink_endp_gpu:
*reg = (struct __nvlink_reginit *)
__nvlink_reginit_per_link_gpu;
*count = ARRAY_SIZE(__nvlink_reginit_per_link_gpu);
break;
default:
return -EINVAL;
}
return 0;
}
/*
* Init TLC IP and prod regs
*/
static void gv100_nvlink_initialize_tlc(struct gk20a *g, u32 link_id)
{
}
/*
* Init TLC per link interrupts
*/
static void gv100_nvlink_tlc_intr_enable(struct gk20a *g, u32 link_id,
bool enable)
{
u32 reg_rx0 = 0, reg_rx1 = 0, reg_tx = 0;
if (enable) {
/* Set PROD values */
reg_rx0 = 0x0FFFFFF;
reg_rx1 = 0x03FFFFF;
reg_tx = 0x1FFFFFF;
}
TLC_REG_WR32(g, link_id, nvtlc_rx_err_report_en_0_r(), reg_rx0);
TLC_REG_WR32(g, link_id, nvtlc_rx_err_report_en_1_r(), reg_rx1);
TLC_REG_WR32(g, link_id, nvtlc_tx_err_report_en_0_r(), reg_tx);
}
/*
* helper function to get TLC intr status in common structure
*/
static void gv100_nvlink_tlc_get_intr(struct gk20a *g, u32 link_id)
{
g->nvlink.tlc_rx_err_status_0[link_id] =
TLC_REG_RD32(g, link_id, nvtlc_rx_err_status_0_r());
g->nvlink.tlc_rx_err_status_1[link_id] =
TLC_REG_RD32(g, link_id, nvtlc_rx_err_status_1_r());
g->nvlink.tlc_tx_err_status_0[link_id] =
TLC_REG_RD32(g, link_id, nvtlc_tx_err_status_0_r());
}
/*
* Interrupt routine handler for TLC
*/
static void gv100_nvlink_tlc_isr(struct gk20a *g, u32 link_id)
{
if (g->nvlink.tlc_rx_err_status_0[link_id]) {
/* All TLC RX 0 errors are fatal. Notify and disable */
nvgpu_err(g, "Fatal TLC RX 0 interrupt on link %d mask: %x",
link_id, g->nvlink.tlc_rx_err_status_0[link_id]);
TLC_REG_WR32(g, link_id, nvtlc_rx_err_first_0_r(),
g->nvlink.tlc_rx_err_status_0[link_id]);
TLC_REG_WR32(g, link_id, nvtlc_rx_err_status_0_r(),
g->nvlink.tlc_rx_err_status_0[link_id]);
}
if (g->nvlink.tlc_rx_err_status_1[link_id]) {
/* All TLC RX 1 errors are fatal. Notify and disable */
nvgpu_err(g, "Fatal TLC RX 1 interrupt on link %d mask: %x",
link_id, g->nvlink.tlc_rx_err_status_1[link_id]);
TLC_REG_WR32(g, link_id, nvtlc_rx_err_first_1_r(),
g->nvlink.tlc_rx_err_status_1[link_id]);
TLC_REG_WR32(g, link_id, nvtlc_rx_err_status_1_r(),
g->nvlink.tlc_rx_err_status_1[link_id]);
}
if (g->nvlink.tlc_tx_err_status_0[link_id]) {
/* All TLC TX 0 errors are fatal. Notify and disable */
nvgpu_err(g, "Fatal TLC TX 0 interrupt on link %d mask: %x",
link_id, g->nvlink.tlc_tx_err_status_0[link_id]);
TLC_REG_WR32(g, link_id, nvtlc_tx_err_first_0_r(),
g->nvlink.tlc_tx_err_status_0[link_id]);
TLC_REG_WR32(g, link_id, nvtlc_tx_err_status_0_r(),
g->nvlink.tlc_tx_err_status_0[link_id]);
}
}
/*
*-----------------------------------------------------------------------------*
* DLPL API
*-----------------------------------------------------------------------------*
*/
/*
* DLPL interrupt enable helper
*/
static void gv100_nvlink_dlpl_intr_enable(struct gk20a *g, u32 link_id,
bool enable)
{
u32 reg = 0;
/* Always disable nonstall tree */
DLPL_REG_WR32(g, link_id, nvl_intr_nonstall_en_r(), 0);
if (!enable)
{
DLPL_REG_WR32(g, link_id, nvl_intr_stall_en_r(), 0);
return;
}
/* Clear interrupt register to get rid of stale state (W1C) */
DLPL_REG_WR32(g, link_id, nvl_intr_r(), 0xffffffff);
DLPL_REG_WR32(g, link_id, nvl_intr_sw2_r(), 0xffffffff);
reg = nvl_intr_stall_en_tx_recovery_long_enable_f() |
nvl_intr_stall_en_tx_fault_ram_enable_f() |
nvl_intr_stall_en_tx_fault_interface_enable_f() |
nvl_intr_stall_en_tx_fault_sublink_change_enable_f() |
nvl_intr_stall_en_rx_fault_sublink_change_enable_f() |
nvl_intr_stall_en_rx_fault_dl_protocol_enable_f() |
nvl_intr_stall_en_ltssm_fault_enable_f();
DLPL_REG_WR32(g, link_id, nvl_intr_stall_en_r(), reg);
/* Configure error threshold */
reg = DLPL_REG_RD32(g, link_id, nvl_sl1_error_rate_ctrl_r());
reg = set_field(reg, nvl_sl1_error_rate_ctrl_short_threshold_man_m(),
nvl_sl1_error_rate_ctrl_short_threshold_man_f(0x2));
reg = set_field(reg, nvl_sl1_error_rate_ctrl_long_threshold_man_m(),
nvl_sl1_error_rate_ctrl_long_threshold_man_f(0x2));
DLPL_REG_WR32(g, link_id, nvl_sl1_error_rate_ctrl_r(), reg);
}
/*
* DLPL per-link isr
*/
#define DLPL_NON_FATAL_INTR_MASK (nvl_intr_tx_replay_f(1) | \
nvl_intr_tx_recovery_short_f(1) | \
nvl_intr_tx_recovery_long_f(1) | \
nvl_intr_rx_short_error_rate_f(1) | \
nvl_intr_rx_long_error_rate_f(1) | \
nvl_intr_rx_ila_trigger_f(1) | \
nvl_intr_ltssm_protocol_f(1))
#define DLPL_FATAL_INTR_MASK ( nvl_intr_ltssm_fault_f(1) | \
nvl_intr_rx_fault_dl_protocol_f(1) | \
nvl_intr_rx_fault_sublink_change_f(1) | \
nvl_intr_tx_fault_sublink_change_f(1) | \
nvl_intr_tx_fault_interface_f(1) | \
nvl_intr_tx_fault_ram_f(1))
static void gv100_nvlink_dlpl_isr(struct gk20a *g, u32 link_id)
{
u32 non_fatal_mask = 0;
u32 fatal_mask = 0;
u32 intr = 0;
bool retrain = false;
int err;
intr = DLPL_REG_RD32(g, link_id, nvl_intr_r()) &
DLPL_REG_RD32(g, link_id, nvl_intr_stall_en_r());
if (!intr)
return;
fatal_mask = intr & DLPL_FATAL_INTR_MASK;
non_fatal_mask = intr & DLPL_NON_FATAL_INTR_MASK;
nvgpu_err(g, " handling DLPL %d isr. Fatal: %x non-Fatal: %x",
link_id, fatal_mask, non_fatal_mask);
/* Check if we are not handling an interupt */
if ((fatal_mask | non_fatal_mask) & ~intr)
nvgpu_err(g, "Unable to service DLPL intr on link %d", link_id);
if (non_fatal_mask & nvl_intr_tx_recovery_long_f(1))
retrain = true;
if (fatal_mask)
retrain = false;
if (retrain) {
err = nvgpu_nvlink_train(g, link_id, false);
if (err != 0)
nvgpu_err(g, "failed to retrain link %d", link_id);
}
/* Clear interrupts */
DLPL_REG_WR32(g, link_id, nvl_intr_r(), (non_fatal_mask | fatal_mask));
DLPL_REG_WR32(g, link_id, nvl_intr_sw2_r(), 0xffffffff);
}
/*
*-----------------------------------------------------------------------------*
* MIF API
*-----------------------------------------------------------------------------*
*/
/*
* Initialize MIF API with PROD settings
*/
static void gv100_nvlink_initialize_mif(struct gk20a *g, u32 link_id)
{
u32 tmp;
/* Enable MIF RX error */
/* Containment (make fatal) */
tmp = 0;
tmp = set_field(tmp,
ioctrlmif_rx_err_contain_en_0_rxramdataparityerr_m(),
ioctrlmif_rx_err_contain_en_0_rxramdataparityerr__prod_f());
tmp = set_field(tmp,
ioctrlmif_rx_err_contain_en_0_rxramhdrparityerr_m(),
ioctrlmif_rx_err_contain_en_0_rxramhdrparityerr__prod_f());
MIF_REG_WR32(g, link_id, ioctrlmif_rx_err_contain_en_0_r(), tmp);
/* Logging (do not ignore) */
tmp = 0;
tmp = set_field(tmp,
ioctrlmif_rx_err_log_en_0_rxramdataparityerr_m(),
ioctrlmif_rx_err_log_en_0_rxramdataparityerr_f(1));
tmp = set_field(tmp,
ioctrlmif_rx_err_log_en_0_rxramhdrparityerr_m(),
ioctrlmif_rx_err_log_en_0_rxramhdrparityerr_f(1));
MIF_REG_WR32(g, link_id, ioctrlmif_rx_err_log_en_0_r(), tmp);
/* Tx Error */
/* Containment (make fatal) */
tmp = 0;
tmp = set_field(tmp,
ioctrlmif_tx_err_contain_en_0_txramdataparityerr_m(),
ioctrlmif_tx_err_contain_en_0_txramdataparityerr__prod_f());
tmp = set_field(tmp,
ioctrlmif_tx_err_contain_en_0_txramhdrparityerr_m(),
ioctrlmif_tx_err_contain_en_0_txramhdrparityerr__prod_f());
MIF_REG_WR32(g, link_id, ioctrlmif_tx_err_contain_en_0_r(), tmp);
/* Logging (do not ignore) */
tmp = 0;
tmp = set_field(tmp, ioctrlmif_tx_err_log_en_0_txramdataparityerr_m(),
ioctrlmif_tx_err_log_en_0_txramdataparityerr_f(1));
tmp = set_field(tmp, ioctrlmif_tx_err_log_en_0_txramhdrparityerr_m(),
ioctrlmif_tx_err_log_en_0_txramhdrparityerr_f(1));
MIF_REG_WR32(g, link_id, ioctrlmif_tx_err_log_en_0_r(), tmp);
/* Credit release */
MIF_REG_WR32(g, link_id, ioctrlmif_rx_ctrl_buffer_ready_r(), 0x1);
MIF_REG_WR32(g, link_id, ioctrlmif_tx_ctrl_buffer_ready_r(), 0x1);
}
/*
* Enable per-link MIF interrupts
*/
static void gv100_nvlink_mif_intr_enable(struct gk20a *g, u32 link_id,
bool enable)
{
u32 reg0 = 0, reg1 = 0;
if (enable) {
reg0 = set_field(reg0,
ioctrlmif_rx_err_report_en_0_rxramdataparityerr_m(),
ioctrlmif_rx_err_report_en_0_rxramdataparityerr_f(1));
reg0 = set_field(reg0,
ioctrlmif_rx_err_report_en_0_rxramhdrparityerr_m(),
ioctrlmif_rx_err_report_en_0_rxramhdrparityerr_f(1));
reg1 = set_field(reg1,
ioctrlmif_tx_err_report_en_0_txramdataparityerr_m(),
ioctrlmif_tx_err_report_en_0_txramdataparityerr_f(1));
reg1 = set_field(reg1,
ioctrlmif_tx_err_report_en_0_txramhdrparityerr_m(),
ioctrlmif_tx_err_report_en_0_txramhdrparityerr_f(1));
}
MIF_REG_WR32(g, link_id, ioctrlmif_rx_err_report_en_0_r(), reg0);
MIF_REG_WR32(g, link_id, ioctrlmif_tx_err_report_en_0_r(), reg1);
}
/*
* Handle per-link MIF interrupts
*/
static void gv100_nvlink_mif_isr(struct gk20a *g, u32 link_id)
{
u32 intr, fatal_mask = 0;
/* RX Errors */
intr = MIF_REG_RD32(g, link_id, ioctrlmif_rx_err_status_0_r());
if (intr) {
if (intr & ioctrlmif_rx_err_status_0_rxramdataparityerr_m()) {
nvgpu_err(g, "Fatal MIF RX interrupt hit on link %d: RAM_DATA_PARITY",
link_id);
fatal_mask |= ioctrlmif_rx_err_status_0_rxramdataparityerr_f(1);
}
if (intr & ioctrlmif_rx_err_status_0_rxramhdrparityerr_m()) {
nvgpu_err(g, "Fatal MIF RX interrupt hit on link %d: RAM_HDR_PARITY",
link_id);
fatal_mask |= ioctrlmif_rx_err_status_0_rxramhdrparityerr_f(1);
}
if (fatal_mask) {
MIF_REG_WR32(g, link_id, ioctrlmif_rx_err_first_0_r(),
fatal_mask);
MIF_REG_WR32(g, link_id, ioctrlmif_rx_err_status_0_r(),
fatal_mask);
}
}
/* TX Errors */
fatal_mask = 0;
intr = MIF_REG_RD32(g, link_id, ioctrlmif_tx_err_status_0_r());
if (intr) {
if (intr & ioctrlmif_tx_err_status_0_txramdataparityerr_m()) {
nvgpu_err(g, "Fatal MIF TX interrupt hit on link %d: RAM_DATA_PARITY",
link_id);
fatal_mask |= ioctrlmif_tx_err_status_0_txramdataparityerr_f(1);
}
if (intr & ioctrlmif_tx_err_status_0_txramhdrparityerr_m()) {
nvgpu_err(g, "Fatal MIF TX interrupt hit on link %d: RAM_HDR_PARITY",
link_id);
fatal_mask |= ioctrlmif_tx_err_status_0_txramhdrparityerr_f(1);
}
if (fatal_mask) {
MIF_REG_WR32(g, link_id, ioctrlmif_tx_err_first_0_r(),
fatal_mask);
MIF_REG_WR32(g, link_id, ioctrlmif_tx_err_status_0_r(),
fatal_mask);
}
}
}
/*
*-----------------------------------------------------------------------------*
* NVLIPT API
*-----------------------------------------------------------------------------*
*/
/*
* NVLIPT IP initialization (per-link)
*/
static void gv100_nvlink_initialize_nvlipt(struct gk20a *g, u32 link_id)
{
/* init persistent scratch registers */
IPT_REG_WR32(g, nvlipt_scratch_cold_r(),
nvlipt_scratch_cold_data_init_v());
/*
* AErr settings (top level)
*/
/* UC first and status reg (W1C) need to be cleared byt arch */
IPT_REG_WR32(g, IPT_ERR_UC_FIRST_LINK(link_id), IPT_ERR_UC_ACTIVE_BITS);
IPT_REG_WR32(g, IPT_ERR_UC_STATUS_LINK(link_id), IPT_ERR_UC_ACTIVE_BITS);
/* AErr Severity */
IPT_REG_WR32(g, IPT_ERR_UC_SEVERITY_LINK(link_id), IPT_ERR_UC_ACTIVE_BITS);
/* AErr Control settings */
IPT_REG_WR32(g, IPT_ERR_CONTROL_LINK(link_id),
nvlipt_err_control_link0_fatalenable_f(1) |
nvlipt_err_control_link0_nonfatalenable_f(1));
}
/*
* Enable NVLIPT interrupts
*/
static void gv100_nvlink_nvlipt_intr_enable(struct gk20a *g, u32 link_id,
bool enable)
{
u32 val = 0;
u32 reg;
if (enable)
val = 1;
reg = IPT_REG_RD32(g, IPT_INTR_CONTROL_LINK(link_id));
reg = set_field(reg, nvlipt_intr_control_link0_stallenable_m(),
nvlipt_intr_control_link0_stallenable_f(val));
reg = set_field(reg, nvlipt_intr_control_link0_nostallenable_m(),
nvlipt_intr_control_link0_nostallenable_f(val));
IPT_REG_WR32(g, IPT_INTR_CONTROL_LINK(link_id), reg);
}
/*
* Per-link NVLIPT ISR handler
*/
static bool gv100_nvlink_nvlipt_isr(struct gk20a *g, u32 link_id)
{
/*
* Interrupt handling happens in leaf handlers. Assume all interrupts
* were handled and clear roll ups/
*/
IPT_REG_WR32(g, IPT_ERR_UC_FIRST_LINK(link_id), IPT_ERR_UC_ACTIVE_BITS);
IPT_REG_WR32(g, IPT_ERR_UC_STATUS_LINK(link_id), IPT_ERR_UC_ACTIVE_BITS);
return true;
}
/*
*******************************************************************************
* Interrupt handling functions *
*******************************************************************************
*/
/*
* Enable common interrupts
*/
static void gv100_nvlink_common_intr_enable(struct gk20a *g,
unsigned long mask)
{
u32 reg, i;
/* Init IOCTRL */
for_each_set_bit(i, &mask, 32) {
reg = IOCTRL_REG_RD32(g, ioctrl_link_intr_0_mask_r(i));
reg |= (ioctrl_link_intr_0_mask_fatal_f(1) |
ioctrl_link_intr_0_mask_nonfatal_f(1) |
ioctrl_link_intr_0_mask_correctable_f(1) |
ioctrl_link_intr_0_mask_intra_f(1));
IOCTRL_REG_WR32(g, ioctrl_link_intr_0_mask_r(i), reg);
}
reg = IOCTRL_REG_RD32(g, ioctrl_common_intr_0_mask_r());
reg |= (ioctrl_common_intr_0_mask_fatal_f(1) |
ioctrl_common_intr_0_mask_nonfatal_f(1) |
ioctrl_common_intr_0_mask_correctable_f(1) |
ioctrl_common_intr_0_mask_intra_f(1));
IOCTRL_REG_WR32(g, ioctrl_common_intr_0_mask_r(), reg);
/* Init NVLIPT */
IPT_REG_WR32(g, nvlipt_intr_control_common_r(),
nvlipt_intr_control_common_stallenable_f(1) |
nvlipt_intr_control_common_nonstallenable_f(1));
}
/*
* Enable link specific interrupts (top-level)
*/
static void gv100_nvlink_enable_link_intr(struct gk20a *g, u32 link_id,
bool enable)
{
gv100_nvlink_minion_link_intr_enable(g, link_id, enable);
gv100_nvlink_dlpl_intr_enable(g, link_id, enable);
gv100_nvlink_tlc_intr_enable(g, link_id, enable);
gv100_nvlink_mif_intr_enable(g, link_id, enable);
gv100_nvlink_nvlipt_intr_enable(g, link_id, enable);
}
/*
* Top level interrupt handler
*/
int gv100_nvlink_isr(struct gk20a *g)
{
unsigned long links;
u32 link_id;
links = ioctrl_top_intr_0_status_link_v(
IOCTRL_REG_RD32(g, ioctrl_top_intr_0_status_r()));
links &= g->nvlink.enabled_links;
/* As per ARCH minion must be serviced first */
gv100_nvlink_minion_isr(g);
for_each_set_bit(link_id, &links, 32) {
/* Cache error logs from TLC, DL handler may clear them */
gv100_nvlink_tlc_get_intr(g, link_id);
gv100_nvlink_dlpl_isr(g, link_id);
gv100_nvlink_tlc_isr(g, link_id);
gv100_nvlink_mif_isr(g, link_id);
/* NVLIPT is top-level. Do it last */
gv100_nvlink_nvlipt_isr(g, link_id);
}
return 0;
}
/*******************************************************************************
* Helper functions *
*******************************************************************************
*/
static u32 __gv100_nvlink_get_link_reset_mask(struct gk20a *g)
{
u32 reg_data;
reg_data = IOCTRL_REG_RD32(g, ioctrl_reset_r());
return ioctrl_reset_linkreset_v(reg_data);
}
static u32 __gv100_nvlink_state_load_hal(struct gk20a *g)
{
unsigned long discovered = g->nvlink.discovered_links;
gv100_nvlink_common_intr_enable(g, discovered);
return gv100_nvlink_minion_load(g);
}
#define TRIM_SYS_NVLINK_CTRL(i) (trim_sys_nvlink0_ctrl_r() + 16*i)
#define TRIM_SYS_NVLINK_STATUS(i) (trim_sys_nvlink0_status_r() + 16*i)
int gv100_nvlink_setup_pll(struct gk20a *g, unsigned long link_mask)
{
u32 reg;
u32 i;
u32 links_off;
struct nvgpu_timeout timeout;
u32 pad_ctrl = 0;
u32 swap_ctrl = 0;
u32 pll_id;
reg = gk20a_readl(g, trim_sys_nvlink_uphy_cfg_r());
reg = set_field(reg, trim_sys_nvlink_uphy_cfg_phy2clks_use_lockdet_m(),
trim_sys_nvlink_uphy_cfg_phy2clks_use_lockdet_f(1));
gk20a_writel(g, trim_sys_nvlink_uphy_cfg_r(), reg);
if (g->ops.top.get_nvhsclk_ctrl_e_clk_nvl) {
pad_ctrl = g->ops.top.get_nvhsclk_ctrl_e_clk_nvl(g);
}
if (g->ops.top.get_nvhsclk_ctrl_swap_clk_nvl) {
swap_ctrl = g->ops.top.get_nvhsclk_ctrl_swap_clk_nvl(g);
}
for_each_set_bit(i, &link_mask, 32) {
/* There are 3 PLLs for 6 links. We have 3 bits for each PLL.
* The PLL bit corresponding to a link is /2 of its master link.
*/
pll_id = g->nvlink.links[i].pll_master_link_id >> 1;
pad_ctrl |= BIT(pll_id);
swap_ctrl |= BIT(pll_id);
}
if (g->ops.top.set_nvhsclk_ctrl_e_clk_nvl) {
g->ops.top.set_nvhsclk_ctrl_e_clk_nvl(g, pad_ctrl);
}
if (g->ops.top.set_nvhsclk_ctrl_swap_clk_nvl) {
g->ops.top.set_nvhsclk_ctrl_swap_clk_nvl(g, swap_ctrl);
}
for_each_set_bit(i, &link_mask, 32) {
reg = gk20a_readl(g, TRIM_SYS_NVLINK_CTRL(i));
reg = set_field(reg,
trim_sys_nvlink0_ctrl_unit2clks_pll_turn_off_m(),
trim_sys_nvlink0_ctrl_unit2clks_pll_turn_off_f(0));
gk20a_writel(g, TRIM_SYS_NVLINK_CTRL(i), reg);
}
/* Poll for links to go up */
links_off = link_mask;
nvgpu_timeout_init(g, &timeout,
NVLINK_PLL_ON_TIMEOUT_MS, NVGPU_TIMER_CPU_TIMER);
do {
for_each_set_bit(i, &link_mask, 32) {
reg = gk20a_readl(g, TRIM_SYS_NVLINK_STATUS(i));
if (trim_sys_nvlink0_status_pll_off_v(reg) == 0)
links_off &= ~BIT(i);
}
nvgpu_udelay(5);
} while((!nvgpu_timeout_expired_msg(&timeout, "timeout on pll on")) &&
links_off);
if (nvgpu_timeout_peek_expired(&timeout))
return -ETIMEDOUT;
return 0;
}
static void gv100_nvlink_prog_alt_clk(struct gk20a *g)
{
u32 tmp;
/* RMW registers need to be separate */
tmp = gk20a_readl(g, trim_sys_nvl_common_clk_alt_switch_r());
tmp &= ~trim_sys_nvl_common_clk_alt_switch_slowclk_m();
tmp |= trim_sys_nvl_common_clk_alt_switch_slowclk_xtal4x_f();
gk20a_writel(g, trim_sys_nvl_common_clk_alt_switch_r(), tmp);
}
static int gv100_nvlink_enable_links_pre_top(struct gk20a *g, u32 links)
{
u32 link_id;
unsigned long enabled_links = links;
u32 tmp;
u32 reg;
u32 delay = ioctrl_reset_sw_post_reset_delay_microseconds_v();
int err;
nvgpu_log(g, gpu_dbg_nvlink, " enabling 0x%lx links", enabled_links);
/* Take links out of reset */
for_each_set_bit(link_id, &enabled_links, 32) {
reg = IOCTRL_REG_RD32(g, ioctrl_reset_r());
tmp = (BIT(link_id) |
BIT(g->nvlink.links[link_id].pll_master_link_id));
reg = set_field(reg, ioctrl_reset_linkreset_m(),
ioctrl_reset_linkreset_f( ioctrl_reset_linkreset_v(reg) |
tmp));
IOCTRL_REG_WR32(g, ioctrl_reset_r(), reg);
nvgpu_udelay(delay);
reg = IOCTRL_REG_RD32(g, ioctrl_debug_reset_r());
reg &= ~ioctrl_debug_reset_link_f(BIT(link_id));
IOCTRL_REG_WR32(g, ioctrl_debug_reset_r(), reg);
nvgpu_udelay(delay);
reg |= ioctrl_debug_reset_link_f(BIT(link_id));
IOCTRL_REG_WR32(g, ioctrl_debug_reset_r(), reg);
nvgpu_udelay(delay);
/* Before doing any link initialization, run RXDET to check
* if link is connected on other end.
*/
if (g->ops.nvlink.rxdet) {
err = g->ops.nvlink.rxdet(g, link_id);
if (err != 0)
return err;
}
/* Enable Link DLPL for AN0 */
reg = DLPL_REG_RD32(g, link_id, nvl_link_config_r());
reg = set_field(reg, nvl_link_config_link_en_m(),
nvl_link_config_link_en_f(1));
DLPL_REG_WR32(g, link_id, nvl_link_config_r(), reg);
/* This should be done by the NVLINK API */
err = gv100_nvlink_minion_init_uphy(g, BIT(link_id), true);
if (err != 0) {
nvgpu_err(g, "Failed to init phy of link: %u", link_id);
return err;
}
err = gv100_nvlink_rxcal_en(g, BIT(link_id));
if (err != 0) {
nvgpu_err(g, "Failed to RXcal on link: %u", link_id);
return err;
}
err = gv100_nvlink_minion_data_ready_en(g, BIT(link_id), true);
if (err != 0) {
nvgpu_err(g, "Failed to set data ready link:%u",
link_id);
return err;
}
g->nvlink.enabled_links |= BIT(link_id);
}
nvgpu_log(g, gpu_dbg_nvlink, "enabled_links=0x%08x",
g->nvlink.enabled_links);
if (g->nvlink.enabled_links)
return 0;
nvgpu_err(g, " No links were enabled");
return -EINVAL;
}
void gv100_nvlink_set_sw_war(struct gk20a *g, u32 link_id)
{
u32 reg;
/* WAR for HW bug 1888034 */
reg = DLPL_REG_RD32(g, link_id, nvl_sl0_safe_ctrl2_tx_r());
reg = set_field(reg, nvl_sl0_safe_ctrl2_tx_ctr_init_m(),
nvl_sl0_safe_ctrl2_tx_ctr_init_init_f());
reg = set_field(reg, nvl_sl0_safe_ctrl2_tx_ctr_initscl_m(),
nvl_sl0_safe_ctrl2_tx_ctr_initscl_init_f());
DLPL_REG_WR32(g, link_id, nvl_sl0_safe_ctrl2_tx_r(), reg);
}
static int gv100_nvlink_enable_links_post_top(struct gk20a *g, u32 links)
{
u32 link_id;
unsigned long enabled_links = (links & g->nvlink.enabled_links) &
~g->nvlink.initialized_links;
for_each_set_bit(link_id, &enabled_links, 32) {
if (g->ops.nvlink.set_sw_war)
g->ops.nvlink.set_sw_war(g, link_id);
gv100_nvlink_initialize_tlc(g, link_id);
gv100_nvlink_initialize_nvlipt(g, link_id);
gv100_nvlink_enable_link_intr(g, link_id, true);
g->nvlink.initialized_links |= BIT(link_id);
};
return 0;
}
static u32 gv100_nvlink_prbs_gen_en(struct gk20a *g, unsigned long mask)
{
u32 reg;
u32 link_id;
for_each_set_bit(link_id, &mask, 32) {
/* Write is required as part of HW sequence */
DLPL_REG_WR32(g, link_id, nvl_sl1_rxslsm_timeout_2_r(), 0);
reg = DLPL_REG_RD32(g, link_id, nvl_txiobist_config_r());
reg = set_field(reg, nvl_txiobist_config_dpg_prbsseedld_m(),
nvl_txiobist_config_dpg_prbsseedld_f(0x1));
DLPL_REG_WR32(g, link_id, nvl_txiobist_config_r(), reg);
reg = DLPL_REG_RD32(g, link_id, nvl_txiobist_config_r());
reg = set_field(reg, nvl_txiobist_config_dpg_prbsseedld_m(),
nvl_txiobist_config_dpg_prbsseedld_f(0x0));
DLPL_REG_WR32(g, link_id, nvl_txiobist_config_r(), reg);
}
return 0;
}
static int gv100_nvlink_rxcal_en(struct gk20a *g, unsigned long mask)
{
u32 link_id;
struct nvgpu_timeout timeout;
u32 reg;
for_each_set_bit(link_id, &mask, 32) {
/* Timeout from HW specs */
nvgpu_timeout_init(g, &timeout,
8*NVLINK_SUBLINK_TIMEOUT_MS, NVGPU_TIMER_CPU_TIMER);
reg = DLPL_REG_RD32(g, link_id, nvl_br0_cfg_cal_r());
reg = set_field(reg, nvl_br0_cfg_cal_rxcal_m(),
nvl_br0_cfg_cal_rxcal_on_f());
DLPL_REG_WR32(g, link_id, nvl_br0_cfg_cal_r(), reg);
do {
reg = DLPL_REG_RD32(g, link_id,
nvl_br0_cfg_status_cal_r());
if (nvl_br0_cfg_status_cal_rxcal_done_v(reg) == 1)
break;
nvgpu_udelay(5);
} while(!nvgpu_timeout_expired_msg(&timeout,
"timeout on rxcal"));
if (nvgpu_timeout_peek_expired(&timeout))
return -ETIMEDOUT;
}
return 0;
}
/*
*******************************************************************************
* Internal "ops" functions *
*******************************************************************************
*/
/*
* Main Nvlink init function. Calls into the Nvlink core API
*/
int gv100_nvlink_init(struct gk20a *g)
{
int err = 0;
if (!nvgpu_is_enabled(g, NVGPU_SUPPORT_NVLINK))
return -ENODEV;
err = nvgpu_nvlink_enumerate(g);
if (err != 0) {
nvgpu_err(g, "failed to enumerate nvlink");
goto fail;
}
/* Set HSHUB and SG_PHY */
__nvgpu_set_enabled(g, NVGPU_MM_USE_PHYSICAL_SG, true);
err = g->ops.fb.enable_nvlink(g);
if (err != 0) {
nvgpu_err(g, "failed switch to nvlink sysmem");
goto fail;
}
return err;
fail:
__nvgpu_set_enabled(g, NVGPU_MM_USE_PHYSICAL_SG, false);
__nvgpu_set_enabled(g, NVGPU_SUPPORT_NVLINK, false);
return err;
}
/*
* Query internal device topology and discover devices in nvlink local
* infrastructure. Initialize register base and offsets
*/
int gv100_nvlink_discover_link(struct gk20a *g)
{
u32 i;
u32 ioctrl_entry_addr;
u8 ioctrl_device_type;
u32 table_entry;
u32 ioctrl_info_entry_type;
u8 ioctrl_discovery_size;
bool is_chain = false;
u8 nvlink_num_devices = 0;
unsigned long available_links = 0;
struct nvgpu_nvlink_device_list *device_table;
u32 err = 0;
/*
* Process Entry 0 & 1 of IOCTRL table to find table size
*/
if (g->nvlink.ioctrl_table && g->nvlink.ioctrl_table[0].pri_base_addr) {
ioctrl_entry_addr = g->nvlink.ioctrl_table[0].pri_base_addr;
table_entry = gk20a_readl(g, ioctrl_entry_addr);
ioctrl_info_entry_type = nvlinkip_discovery_common_device_v(table_entry);
} else {
nvgpu_err(g, " Bad IOCTRL PRI Base addr");
return -EINVAL;
}
if (ioctrl_info_entry_type == NVL_DEVICE(ioctrl)) {
ioctrl_entry_addr = g->nvlink.ioctrl_table[0].pri_base_addr + 4;
table_entry = gk20a_readl(g, ioctrl_entry_addr);
ioctrl_discovery_size = nvlinkip_discovery_common_ioctrl_length_v(table_entry);
nvgpu_log(g, gpu_dbg_nvlink, "IOCTRL size: %d", ioctrl_discovery_size);
} else {
nvgpu_err(g, " First entry of IOCTRL_DISCOVERY invalid");
return -EINVAL;
}
device_table = nvgpu_kzalloc(g, ioctrl_discovery_size *
sizeof(struct nvgpu_nvlink_device_list));
if (!device_table) {
nvgpu_err(g, " Unable to allocate nvlink device table");
return -ENOMEM;
}
for (i = 0; i < ioctrl_discovery_size; i++) {
ioctrl_entry_addr =
g->nvlink.ioctrl_table[0].pri_base_addr + 4*i;
table_entry = gk20a_readl(g, ioctrl_entry_addr);
nvgpu_log(g, gpu_dbg_nvlink, "parsing ioctrl %d: 0x%08x", i, table_entry);
ioctrl_info_entry_type = nvlinkip_discovery_common_entry_v(table_entry);
if (ioctrl_info_entry_type ==
nvlinkip_discovery_common_entry_invalid_v())
continue;
if (ioctrl_info_entry_type ==
nvlinkip_discovery_common_entry_enum_v()) {
nvgpu_log(g, gpu_dbg_nvlink, "IOCTRL entry %d is ENUM", i);
ioctrl_device_type =
nvlinkip_discovery_common_device_v(table_entry);
if (nvlinkip_discovery_common_chain_v(table_entry) !=
nvlinkip_discovery_common_chain_enable_v()) {
nvgpu_log(g, gpu_dbg_nvlink,
"IOCTRL entry %d is ENUM but no chain",
i);
err = -EINVAL;
break;
}
is_chain = true;
device_table[nvlink_num_devices].valid = true;
device_table[nvlink_num_devices].device_type =
ioctrl_device_type;
device_table[nvlink_num_devices].device_id =
nvlinkip_discovery_common_id_v(table_entry);
device_table[nvlink_num_devices].device_version =
nvlinkip_discovery_common_version_v(
table_entry);
continue;
}
if (ioctrl_info_entry_type ==
nvlinkip_discovery_common_entry_data1_v()) {
nvgpu_log(g, gpu_dbg_nvlink, "IOCTRL entry %d is DATA1", i);
if (is_chain) {
device_table[nvlink_num_devices].pri_base_addr =
nvlinkip_discovery_common_pri_base_v(
table_entry) << 12;
device_table[nvlink_num_devices].intr_enum =
nvlinkip_discovery_common_intr_v(
table_entry);
device_table[nvlink_num_devices].reset_enum =
nvlinkip_discovery_common_reset_v(
table_entry);
nvgpu_log(g, gpu_dbg_nvlink, "IOCTRL entry %d type = %d base: 0x%08x intr: %d reset: %d",
i,
device_table[nvlink_num_devices].device_type,
device_table[nvlink_num_devices].pri_base_addr,
device_table[nvlink_num_devices].intr_enum,
device_table[nvlink_num_devices].reset_enum);
if (device_table[nvlink_num_devices].device_type ==
NVL_DEVICE(dlpl)) {
device_table[nvlink_num_devices].num_tx =
nvlinkip_discovery_common_dlpl_num_tx_v(table_entry);
device_table[nvlink_num_devices].num_rx =
nvlinkip_discovery_common_dlpl_num_rx_v(table_entry);
nvgpu_log(g, gpu_dbg_nvlink, "DLPL tx: %d rx: %d",
device_table[nvlink_num_devices].num_tx,
device_table[nvlink_num_devices].num_rx);
}
if (nvlinkip_discovery_common_chain_v(table_entry) !=
nvlinkip_discovery_common_chain_enable_v()) {
is_chain = false;
nvlink_num_devices++;
}
}
continue;
}
if (ioctrl_info_entry_type ==
nvlinkip_discovery_common_entry_data2_v()) {
nvgpu_log(g, gpu_dbg_nvlink, "IOCTRL entry %d is DATA2", i);
if (is_chain) {
if (nvlinkip_discovery_common_dlpl_data2_type_v(table_entry)) {
device_table[nvlink_num_devices].pll_master =
nvlinkip_discovery_common_dlpl_data2_master_v(table_entry);
device_table[nvlink_num_devices].pll_master_id =
nvlinkip_discovery_common_dlpl_data2_masterid_v(table_entry);
nvgpu_log(g, gpu_dbg_nvlink, "PLL info: Master: %d, Master ID: %d",
device_table[nvlink_num_devices].pll_master,
device_table[nvlink_num_devices].pll_master_id);
}
if (nvlinkip_discovery_common_chain_v(table_entry) !=
nvlinkip_discovery_common_chain_enable_v()) {
is_chain = false;
nvlink_num_devices++;
}
}
continue;
}
}
g->nvlink.device_table = device_table;
g->nvlink.num_devices = nvlink_num_devices;
/*
* Print table
*/
for (i = 0; i < nvlink_num_devices; i++) {
if (device_table[i].valid) {
nvgpu_log(g, gpu_dbg_nvlink, "Device %d - %s", i,
__gv100_device_type_to_str(
device_table[i].device_type));
nvgpu_log(g, gpu_dbg_nvlink, "+Link/Device Id: %d", device_table[i].device_id);
nvgpu_log(g, gpu_dbg_nvlink, "+Version: %d", device_table[i].device_version);
nvgpu_log(g, gpu_dbg_nvlink, "+Base Addr: 0x%08x", device_table[i].pri_base_addr);
nvgpu_log(g, gpu_dbg_nvlink, "+Intr Enum: %d", device_table[i].intr_enum);
nvgpu_log(g, gpu_dbg_nvlink, "+Reset Enum: %d", device_table[i].reset_enum);
if ((device_table[i].device_type == NVL_DEVICE(dlpl)) ||
(device_table[i].device_type == NVL_DEVICE(nvlink))) {
nvgpu_log(g, gpu_dbg_nvlink, "+TX: %d", device_table[i].num_tx);
nvgpu_log(g, gpu_dbg_nvlink, "+RX: %d", device_table[i].num_rx);
nvgpu_log(g, gpu_dbg_nvlink, "+PLL Master: %d", device_table[i].pll_master);
nvgpu_log(g, gpu_dbg_nvlink, "+PLL Master ID: %d", device_table[i].pll_master_id);
}
}
}
for (i = 0; i < nvlink_num_devices; i++) {
if (device_table[i].valid) {
if (device_table[i].device_type == NVL_DEVICE(ioctrl)) {
g->nvlink.ioctrl_type =
device_table[i].device_type;
g->nvlink.ioctrl_base =
device_table[i].pri_base_addr;
continue;
}
if (device_table[i].device_type == NVL_DEVICE(dlpl)) {
g->nvlink.dlpl_type =
device_table[i].device_type;
g->nvlink.dlpl_base[device_table[i].device_id] =
device_table[i].pri_base_addr;
g->nvlink.links[device_table[i].device_id].valid = true;
g->nvlink.links[device_table[i].device_id].g = g;
g->nvlink.links[device_table[i].device_id].dlpl_version =
device_table[i].device_version;
g->nvlink.links[device_table[i].device_id].dlpl_base =
device_table[i].pri_base_addr;
g->nvlink.links[device_table[i].device_id].intr_enum =
device_table[i].intr_enum;
g->nvlink.links[device_table[i].device_id].reset_enum =
device_table[i].reset_enum;
g->nvlink.links[device_table[i].device_id].link_id =
device_table[i].device_id;
/* initiate the PLL master and slave link id to max */
g->nvlink.links[device_table[i].device_id].pll_master_link_id =
NVLINK_MAX_LINKS_SW;
g->nvlink.links[device_table[i].device_id].pll_slave_link_id =
NVLINK_MAX_LINKS_SW;
/* Update Pll master */
if (device_table[i].pll_master)
g->nvlink.links[device_table[i].device_id].pll_master_link_id =
g->nvlink.links[device_table[i].device_id].link_id;
else {
g->nvlink.links[device_table[i].device_id].pll_master_link_id =
device_table[i].pll_master_id;
g->nvlink.links[device_table[i].device_id].pll_slave_link_id =
g->nvlink.links[device_table[i].device_id].link_id;
g->nvlink.links[device_table[i].pll_master_id].pll_slave_link_id =
g->nvlink.links[device_table[i].device_id].link_id;
}
available_links |= BIT(device_table[i].device_id);
continue;
}
if (device_table[i].device_type == NVL_DEVICE(nvltlc)) {
g->nvlink.tl_type = device_table[i].device_type;
g->nvlink.tl_base[device_table[i].device_id] =
device_table[i].pri_base_addr;
g->nvlink.links[device_table[i].device_id].tl_base =
device_table[i].pri_base_addr;
g->nvlink.links[device_table[i].device_id].tl_version =
device_table[i].device_version;
continue;
}
if (device_table[i].device_type == NVL_DEVICE(nvltlc)) {
g->nvlink.tl_type = device_table[i].device_type;
g->nvlink.tl_base[device_table[i].device_id] =
device_table[i].pri_base_addr;
g->nvlink.links[device_table[i].device_id].tl_base =
device_table[i].pri_base_addr;
g->nvlink.links[device_table[i].device_id].tl_version =
device_table[i].device_version;
continue;
}
if (device_table[i].device_type == NVL_DEVICE(ioctrlmif)) {
g->nvlink.mif_type = device_table[i].device_type;
g->nvlink.mif_base[device_table[i].device_id] =
device_table[i].pri_base_addr;
g->nvlink.links[device_table[i].device_id].mif_base =
device_table[i].pri_base_addr;
g->nvlink.links[device_table[i].device_id].mif_version =
device_table[i].device_version;
continue;
}
if (device_table[i].device_type == NVL_DEVICE(nvlipt)) {
g->nvlink.ipt_type =
device_table[i].device_type;
g->nvlink.ipt_base =
device_table[i].pri_base_addr;
g->nvlink.ipt_version =
device_table[i].device_version;
continue;
}
if (device_table[i].device_type == NVL_DEVICE(minion)) {
g->nvlink.minion_type =
device_table[i].device_type;
g->nvlink.minion_base =
device_table[i].pri_base_addr;
g->nvlink.minion_version =
device_table[i].device_version;
continue;
}
if (device_table[i].device_type == NVL_DEVICE(dlpl_multicast)) {
g->nvlink.dlpl_multicast_type =
device_table[i].device_type;
g->nvlink.dlpl_multicast_base =
device_table[i].pri_base_addr;
g->nvlink.dlpl_multicast_version =
device_table[i].device_version;
continue;
}
if (device_table[i].device_type == NVL_DEVICE(nvltlc_multicast)) {
g->nvlink.tl_multicast_type =
device_table[i].device_type;
g->nvlink.tl_multicast_base =
device_table[i].pri_base_addr;
g->nvlink.tl_multicast_version =
device_table[i].device_version;
continue;
}
if (device_table[i].device_type == NVL_DEVICE(ioctrlmif_multicast)) {
g->nvlink.mif_multicast_type =
device_table[i].device_type;
g->nvlink.mif_multicast_base =
device_table[i].pri_base_addr;
g->nvlink.mif_multicast_version =
device_table[i].device_version;
continue;
}
}
}
g->nvlink.discovered_links = (u32) available_links;
nvgpu_log(g, gpu_dbg_nvlink, "Nvlink Tree:");
nvgpu_log(g, gpu_dbg_nvlink, "+ Available Links: 0x%08lx", available_links);
nvgpu_log(g, gpu_dbg_nvlink, "+ Per-Link Devices:");
for_each_set_bit(i, &available_links, 32) {
nvgpu_log(g, gpu_dbg_nvlink, "-- Link %d Dl/Pl Base: 0x%08x TLC Base: 0x%08x MIF Base: 0x%08x",
i, g->nvlink.dlpl_base[i], g->nvlink.tl_base[i], g->nvlink.mif_base[i]);
}
nvgpu_log(g, gpu_dbg_nvlink, "+ IOCTRL Base: 0x%08x", g->nvlink.ioctrl_base);
nvgpu_log(g, gpu_dbg_nvlink, "+ NVLIPT Base: 0x%08x", g->nvlink.ipt_base);
nvgpu_log(g, gpu_dbg_nvlink, "+ MINION Base: 0x%08x", g->nvlink.minion_base);
nvgpu_log(g, gpu_dbg_nvlink, "+ DLPL MCAST Base: 0x%08x", g->nvlink.dlpl_multicast_base);
nvgpu_log(g, gpu_dbg_nvlink, "+ TLC MCAST Base: 0x%08x", g->nvlink.tl_multicast_base);
nvgpu_log(g, gpu_dbg_nvlink, "+ MIF MCAST Base: 0x%08x", g->nvlink.mif_multicast_base);
if (!g->nvlink.minion_version) {
nvgpu_err(g, "Unsupported MINION version");
nvgpu_kfree(g, device_table);
g->nvlink.device_table = NULL;
g->nvlink.num_devices = 0;
return -EINVAL;
}
return err;
}
/*
* Query IOCTRL for device discovery
*/
int gv100_nvlink_discover_ioctrl(struct gk20a *g)
{
int ret = 0;
u32 i;
struct nvgpu_nvlink_ioctrl_list *ioctrl_table;
u32 ioctrl_num_entries = 0;
if (g->ops.top.get_num_engine_type_entries) {
ioctrl_num_entries = g->ops.top.get_num_engine_type_entries(g,
NVGPU_ENGINE_IOCTRL);
nvgpu_info(g, "ioctrl_num_entries: %d", ioctrl_num_entries);
}
if (ioctrl_num_entries == 0) {
nvgpu_err(g, "No NVLINK IOCTRL entry found in dev_info table");
return -EINVAL;
}
ioctrl_table = nvgpu_kzalloc(g, ioctrl_num_entries *
sizeof(struct nvgpu_nvlink_ioctrl_list));
if (!ioctrl_table) {
nvgpu_err(g, "Failed to allocate memory for nvlink io table");
return -ENOMEM;
}
for (i = 0; i < ioctrl_num_entries; i++) {
struct nvgpu_device_info dev_info;
ret = g->ops.top.get_device_info(g, &dev_info,
NVGPU_ENGINE_IOCTRL, i);
if (ret) {
nvgpu_err(g, "Failed to parse dev_info table"
"for engine %d",
NVGPU_ENGINE_IOCTRL);
nvgpu_kfree(g, ioctrl_table);
return -EINVAL;
}
ioctrl_table[i].valid = true;
ioctrl_table[i].intr_enum = dev_info.intr_id;
ioctrl_table[i].reset_enum = dev_info.reset_id;
ioctrl_table[i].pri_base_addr = dev_info.pri_base;
nvgpu_log(g, gpu_dbg_nvlink,
"Dev %d: Pri_Base = 0x%0x Intr = %d Reset = %d",
i, ioctrl_table[i].pri_base_addr,
ioctrl_table[i].intr_enum,
ioctrl_table[i].reset_enum);
}
g->nvlink.ioctrl_table = ioctrl_table;
g->nvlink.io_num_entries = ioctrl_num_entries;
return 0;
}
/*
*******************************************************************************
* NVLINK API FUNCTIONS *
*******************************************************************************
*/
/*
* Performs link level initialization like phy inits, AN0 and interrupts
*/
int gv100_nvlink_link_early_init(struct gk20a *g, unsigned long mask)
{
int err;
err = gv100_nvlink_enable_links_pre_top(g, mask);
if (err != 0) {
nvgpu_err(g, "Pre topology failed for links %lx", mask);
return err;
}
nvgpu_log(g, gpu_dbg_nvlink, "pretopology enabled: 0x%lx",
mask & g->nvlink.enabled_links);
err = gv100_nvlink_enable_links_post_top(g, mask);
return err;
}
/*
* Performs memory interface initialization
*/
int gv100_nvlink_interface_init(struct gk20a *g)
{
unsigned long mask = g->nvlink.enabled_links;
u32 link_id;
int err;
for_each_set_bit(link_id, &mask, 32) {
gv100_nvlink_initialize_mif(g, link_id);
gv100_nvlink_mif_intr_enable(g, link_id, true);
}
err = g->ops.fb.init_nvlink(g);
if (err != 0) {
nvgpu_err(g, "failed to setup nvlinks for sysmem");
return err;
}
return 0;
}
int gv100_nvlink_interface_disable(struct gk20a *g)
{
return 0;
}
int gv100_nvlink_reg_init(struct gk20a *g)
{
u32 i = 0;
u32 count = 0;
struct __nvlink_reginit *reg;
enum nvgpu_nvlink_endp endp;
int err;
u32 link_id;
unsigned long mask = g->nvlink.enabled_links;
struct nvgpu_nvlink_link *link;
/* Apply automated reg init flow for PROD settings */
for_each_set_bit(link_id, &mask, 32) {
link = &g->nvlink.links[link_id];
if (!link->remote_info.is_connected)
continue;
endp = link->remote_info.device_type;
err = gv100_nvlink_get_tlc_reginit(endp, &reg, &count);
if (err != 0) {
nvgpu_err(g, "no reginit for endp=%u", endp);
continue;
}
for (i = 0; i < count; i++) {
TLC_REG_WR32(g, link_id, reg->addr, reg->value);
reg++;
}
}
return 0;
}
/*
* Shutdown device. This should tear down Nvlink connection.
* For now return.
*/
int gv100_nvlink_shutdown(struct gk20a *g)
{
return 0;
}
/*
* Get link state
*/
u32 gv100_nvlink_link_get_state(struct gk20a *g, u32 link_id)
{
return DLPL_REG_RD32(g, link_id, nvl_link_state_r()) &
nvl_link_state_state_m();
}
/* Get link mode */
u32 gv100_nvlink_link_get_mode(struct gk20a *g, u32 link_id)
{
u32 state;
if (!(BIT(link_id) & g->nvlink.discovered_links))
return nvgpu_nvlink_link__last;
state = nvl_link_state_state_v(
g->ops.nvlink.link_get_state(g, link_id));
if (state == nvl_link_state_state_init_v())
return nvgpu_nvlink_link_off;
if (state == nvl_link_state_state_hwcfg_v())
return nvgpu_nvlink_link_detect;
if (state == nvl_link_state_state_swcfg_v())
return nvgpu_nvlink_link_safe;
if (state == nvl_link_state_state_active_v())
return nvgpu_nvlink_link_hs;
if (state == nvl_link_state_state_fault_v())
return nvgpu_nvlink_link_fault;
if (state == nvl_link_state_state_rcvy_ac_v())
return nvgpu_nvlink_link_rcvy_ac;
if (state == nvl_link_state_state_rcvy_sw_v())
return nvgpu_nvlink_link_rcvy_sw;
if (state == nvl_link_state_state_rcvy_rx_v())
return nvgpu_nvlink_link_rcvy_rx;
return nvgpu_nvlink_link_off;
}
/* Set Link mode */
int gv100_nvlink_link_set_mode(struct gk20a *g, u32 link_id, u32 mode)
{
u32 state;
u32 reg;
int err = 0;
nvgpu_log(g, gpu_dbg_nvlink, "link :%d, mode:%u", link_id, mode);
if (!(BIT(link_id) & g->nvlink.enabled_links))
return -EINVAL;
state = nvl_link_state_state_v(
g->ops.nvlink.link_get_state(g, link_id));
switch (mode) {
case nvgpu_nvlink_link_safe:
if (state == nvl_link_state_state_swcfg_v()) {
nvgpu_warn(g, "link is already in safe mode");
break;
}
if (state == nvl_link_state_state_hwcfg_v()) {
nvgpu_warn(g, "link is transitioning to safe mode");
break;
}
if (state == nvl_link_state_state_init_v()) {
/* Off to Safe transition */
reg = DLPL_REG_RD32(g, link_id, nvl_link_change_r());
reg = set_field(reg, nvl_link_change_newstate_m(),
nvl_link_change_newstate_hwcfg_f());
reg = set_field(reg, nvl_link_change_oldstate_mask_m(),
nvl_link_change_oldstate_mask_dontcare_f());
reg = set_field(reg, nvl_link_change_action_m(),
nvl_link_change_action_ltssm_change_f());
DLPL_REG_WR32(g, link_id, nvl_link_change_r(), reg);
} else if (state == nvl_link_state_state_active_v()) {
/* TODO:
* Disable PM first since we are moving out active
* state
*/
reg = DLPL_REG_RD32(g, link_id, nvl_link_change_r());
reg = set_field(reg, nvl_link_change_newstate_m(),
nvl_link_change_newstate_swcfg_f());
reg = set_field(reg, nvl_link_change_oldstate_mask_m(),
nvl_link_change_oldstate_mask_dontcare_f());
reg = set_field(reg, nvl_link_change_action_m(),
nvl_link_change_action_ltssm_change_f());
DLPL_REG_WR32(g, link_id, nvl_link_change_r(), reg);
}
break;
case nvgpu_nvlink_link_hs:
if (state == nvl_link_state_state_active_v()) {
nvgpu_err(g, "link is already in active mode");
break;
}
if (state == nvl_link_state_state_init_v()) {
nvgpu_err(g, "link cannot be taken from init state");
return -EPERM;
}
reg = DLPL_REG_RD32(g, link_id, nvl_link_change_r());
reg = set_field(reg, nvl_link_change_newstate_m(),
nvl_link_change_newstate_active_f());
reg = set_field(reg, nvl_link_change_oldstate_mask_m(),
nvl_link_change_oldstate_mask_dontcare_f());
reg = set_field(reg, nvl_link_change_action_m(),
nvl_link_change_action_ltssm_change_f());
DLPL_REG_WR32(g, link_id, nvl_link_change_r(), reg);
break;
case nvgpu_nvlink_link_off:
if (state == nvl_link_state_state_active_v()) {
nvgpu_err(g, "link cannot be taken from active to init");
return -EPERM;
}
if (state == nvl_link_state_state_init_v()) {
nvgpu_err(g, "link already in init state");
}
/* GV100 UPHY is handled by MINION */
break;
/* 1/8 th mode not supported */
case nvgpu_nvlink_link_enable_pm:
case nvgpu_nvlink_link_disable_pm:
return -EPERM;
case nvgpu_nvlink_link_disable_err_detect:
/* Disable Link interrupts */
gv100_nvlink_dlpl_intr_enable(g, link_id, false);
break;
case nvgpu_nvlink_link_lane_disable:
err = gv100_nvlink_minion_lane_disable(g, link_id, true);
break;
case nvgpu_nvlink_link_lane_shutdown:
err = gv100_nvlink_minion_lane_shutdown(g, link_id, true);
break;
default:
nvgpu_err(g, "Unhandled mode %x", mode);
break;
}
return err;
}
static u32 gv100_nvlink_link_sublink_check_change(struct gk20a *g, u32 link_id)
{
struct nvgpu_timeout timeout;
u32 reg;
nvgpu_timeout_init(g, &timeout,
NVLINK_SUBLINK_TIMEOUT_MS, NVGPU_TIMER_CPU_TIMER);
/* Poll for sublink status */
do {
reg = DLPL_REG_RD32(g, link_id, nvl_sublink_change_r());
if (nvl_sublink_change_status_v(reg) ==
nvl_sublink_change_status_done_v())
break;
if (nvl_sublink_change_status_v(reg) ==
nvl_sublink_change_status_fault_v()) {
nvgpu_err(g, "Fault detected in sublink change");
return -EFAULT;
}
nvgpu_udelay(5);
} while(!nvgpu_timeout_expired_msg(&timeout, "timeout on sublink rdy"));
if (nvgpu_timeout_peek_expired(&timeout))
return -ETIMEDOUT;
return-0;
}
int gv100_nvlink_link_set_sublink_mode(struct gk20a *g, u32 link_id,
bool is_rx_sublink, u32 mode)
{
int err = 0;
u32 rx_sublink_state = nvgpu_nvlink_sublink_rx__last;
u32 tx_sublink_state = nvgpu_nvlink_sublink_tx__last;
u32 reg;
if (!(BIT(link_id) & g->nvlink.enabled_links))
return -EINVAL;
err = gv100_nvlink_link_sublink_check_change(g, link_id);
if (err != 0)
return err;
if (is_rx_sublink)
rx_sublink_state = g->ops.nvlink.get_rx_sublink_state(g,
link_id);
else
tx_sublink_state = g->ops.nvlink.get_tx_sublink_state(g,
link_id);
switch (mode) {
case nvgpu_nvlink_sublink_tx_hs:
if (tx_sublink_state ==
nvl_sl0_slsm_status_tx_primary_state_hs_v()) {
nvgpu_err(g, " TX already in HS");
break;
} else if (tx_sublink_state ==
nvl_sl0_slsm_status_tx_primary_state_off_v()) {
nvgpu_err(g, "TX cannot be do from OFF to HS");
return -EPERM;
}
reg = DLPL_REG_RD32(g, link_id, nvl_sublink_change_r());
reg = set_field(reg, nvl_sublink_change_newstate_m(),
nvl_sublink_change_newstate_hs_f());
reg = set_field(reg, nvl_sublink_change_sublink_m(),
nvl_sublink_change_sublink_tx_f());
reg = set_field(reg, nvl_sublink_change_action_m(),
nvl_sublink_change_action_slsm_change_f());
DLPL_REG_WR32(g, link_id, nvl_sublink_change_r(), reg);
err = gv100_nvlink_link_sublink_check_change(g, link_id);
if (err != 0) {
nvgpu_err(g, "Error in TX to HS");
return err;
}
break;
case nvgpu_nvlink_sublink_tx_common:
err = gv100_nvlink_minion_init_uphy(g, BIT(link_id), true);
break;
case nvgpu_nvlink_sublink_tx_common_disable:
/* NOP */
break;
case nvgpu_nvlink_sublink_tx_data_ready:
err = gv100_nvlink_minion_data_ready_en(g, BIT(link_id), true);
break;
case nvgpu_nvlink_sublink_tx_prbs_en:
err = gv100_nvlink_prbs_gen_en(g, BIT(link_id));
break;
case nvgpu_nvlink_sublink_tx_safe:
if (tx_sublink_state ==
nvl_sl0_slsm_status_tx_primary_state_safe_v()) {
nvgpu_err(g, "TX already SAFE: %d", link_id);
break;
}
reg = DLPL_REG_RD32(g, link_id, nvl_sublink_change_r());
reg = set_field(reg, nvl_sublink_change_newstate_m(),
nvl_sublink_change_newstate_safe_f());
reg = set_field(reg, nvl_sublink_change_sublink_m(),
nvl_sublink_change_sublink_tx_f());
reg = set_field(reg, nvl_sublink_change_action_m(),
nvl_sublink_change_action_slsm_change_f());
DLPL_REG_WR32(g, link_id, nvl_sublink_change_r(), reg);
err = gv100_nvlink_link_sublink_check_change(g, link_id);
if (err != 0) {
nvgpu_err(g, "Error in TX to SAFE");
return err;
}
break;
case nvgpu_nvlink_sublink_tx_off:
if (tx_sublink_state ==
nvl_sl0_slsm_status_tx_primary_state_off_v()) {
nvgpu_err(g, "TX already OFF: %d", link_id);
break;
} else if (tx_sublink_state ==
nvl_sl0_slsm_status_tx_primary_state_hs_v()) {
nvgpu_err(g, " TX cannot go off from HS %d", link_id);
return -EPERM;
}
reg = DLPL_REG_RD32(g, link_id, nvl_sublink_change_r());
reg = set_field(reg, nvl_sublink_change_newstate_m(),
nvl_sublink_change_newstate_off_f());
reg = set_field(reg, nvl_sublink_change_sublink_m(),
nvl_sublink_change_sublink_tx_f());
reg = set_field(reg, nvl_sublink_change_action_m(),
nvl_sublink_change_action_slsm_change_f());
DLPL_REG_WR32(g, link_id, nvl_sublink_change_r(), reg);
err = gv100_nvlink_link_sublink_check_change(g, link_id);
if (err != 0) {
nvgpu_err(g, "Error in TX to OFF");
return err;
}
break;
/* RX modes */
case nvgpu_nvlink_sublink_rx_hs:
case nvgpu_nvlink_sublink_rx_safe:
break;
case nvgpu_nvlink_sublink_rx_off:
if (rx_sublink_state ==
nvl_sl1_slsm_status_rx_primary_state_off_v()) {
nvgpu_err(g, "RX already OFF: %d", link_id);
break;
} else if (rx_sublink_state ==
nvl_sl1_slsm_status_rx_primary_state_hs_v()) {
nvgpu_err(g, " RX cannot go off from HS %d", link_id);
return -EPERM;
}
reg = DLPL_REG_RD32(g, link_id, nvl_sublink_change_r());
reg = set_field(reg, nvl_sublink_change_newstate_m(),
nvl_sublink_change_newstate_off_f());
reg = set_field(reg, nvl_sublink_change_sublink_m(),
nvl_sublink_change_sublink_rx_f());
reg = set_field(reg, nvl_sublink_change_action_m(),
nvl_sublink_change_action_slsm_change_f());
DLPL_REG_WR32(g, link_id, nvl_sublink_change_r(), reg);
err = gv100_nvlink_link_sublink_check_change(g, link_id);
if (err != 0) {
nvgpu_err(g, "Error in RX to OFF");
return err;
}
break;
case nvgpu_nvlink_sublink_rx_rxcal:
err = gv100_nvlink_rxcal_en(g, BIT(link_id));
break;
default:
if ((is_rx_sublink) && ((mode < nvgpu_nvlink_sublink_rx_hs) ||
(mode >= nvgpu_nvlink_sublink_rx__last))) {
nvgpu_err(g, "Unsupported RX mode %u", mode);
return -EINVAL;
}
if (mode >= nvgpu_nvlink_sublink_tx__last) {
nvgpu_err(g, "Unsupported TX mode %u", mode);
return -EINVAL;
}
nvgpu_err(g, "MODE %u", mode);
}
if (err != 0)
nvgpu_err(g, " failed on set_sublink_mode");
return err;
}
u32 gv100_nvlink_link_get_sublink_mode(struct gk20a *g, u32 link_id,
bool is_rx_sublink)
{
u32 state;
if (!(BIT(link_id) & g->nvlink.discovered_links)) {
if (!is_rx_sublink)
return nvgpu_nvlink_sublink_tx__last;
return nvgpu_nvlink_sublink_rx__last;
}
if (!is_rx_sublink) {
state = g->ops.nvlink.get_tx_sublink_state(g, link_id);
if (state == nvl_sl0_slsm_status_tx_primary_state_hs_v())
return nvgpu_nvlink_sublink_tx_hs;
if (state == nvl_sl0_slsm_status_tx_primary_state_eighth_v())
return nvgpu_nvlink_sublink_tx_single_lane;
if (state == nvl_sl0_slsm_status_tx_primary_state_safe_v())
return nvgpu_nvlink_sublink_tx_safe;
if (state == nvl_sl0_slsm_status_tx_primary_state_off_v())
return nvgpu_nvlink_sublink_tx_off;
return nvgpu_nvlink_sublink_tx__last;
} else {
state = g->ops.nvlink.get_rx_sublink_state(g, link_id);
if (state == nvl_sl1_slsm_status_rx_primary_state_hs_v())
return nvgpu_nvlink_sublink_rx_hs;
if (state == nvl_sl1_slsm_status_rx_primary_state_eighth_v())
return nvgpu_nvlink_sublink_rx_single_lane;
if (state == nvl_sl1_slsm_status_rx_primary_state_safe_v())
return nvgpu_nvlink_sublink_rx_safe;
if (state == nvl_sl1_slsm_status_rx_primary_state_off_v())
return nvgpu_nvlink_sublink_rx_off;
return nvgpu_nvlink_sublink_rx__last;
}
return nvgpu_nvlink_sublink_tx__last;
}
/*
* Get TX sublink state
*/
u32 gv100_nvlink_link_get_tx_sublink_state(struct gk20a *g, u32 link_id)
{
u32 reg = DLPL_REG_RD32(g, link_id, nvl_sl0_slsm_status_tx_r());
return nvl_sl0_slsm_status_tx_primary_state_v(reg);
}
/*
* Get RX sublink state
*/
u32 gv100_nvlink_link_get_rx_sublink_state(struct gk20a *g, u32 link_id)
{
u32 reg = DLPL_REG_RD32(g, link_id, nvl_sl1_slsm_status_rx_r());
return nvl_sl1_slsm_status_rx_primary_state_v(reg);
}
/* Hardcode the link_mask while we wait for VBIOS link_disable_mask field
* to be updated.
*/
void gv100_nvlink_get_connected_link_mask(u32 *link_mask)
{
*link_mask = GV100_CONNECTED_LINK_MASK;
}
/*
* Performs nvlink device level initialization by discovering the topology
* taking device out of reset, boot minion, set clocks up and common interrupts
*/
int gv100_nvlink_early_init(struct gk20a *g)
{
int err = 0;
u32 mc_reset_nvlink_mask;
if (!nvgpu_is_enabled(g, NVGPU_SUPPORT_NVLINK))
return -EINVAL;
err = nvgpu_bios_get_lpwr_nvlink_table_hdr(g);
if (err != 0) {
nvgpu_err(g, "Failed to read LWPR_NVLINK_TABLE header\n");
goto nvlink_init_exit;
}
err = nvgpu_bios_get_nvlink_config_data(g);
if (err != 0) {
nvgpu_err(g, "failed to read nvlink vbios data");
goto nvlink_init_exit;
}
err = g->ops.nvlink.discover_ioctrl(g);
if (err != 0)
goto nvlink_init_exit;
/* Enable NVLINK in MC */
mc_reset_nvlink_mask = BIT32(g->nvlink.ioctrl_table[0].reset_enum);
nvgpu_log(g, gpu_dbg_nvlink, "mc_reset_nvlink_mask: 0x%x",
mc_reset_nvlink_mask);
g->ops.mc.reset(g, mc_reset_nvlink_mask);
err = g->ops.nvlink.discover_link(g);
if ((err != 0) || (g->nvlink.discovered_links == 0)) {
nvgpu_err(g, "No links available");
goto nvlink_init_exit;
}
err = nvgpu_falcon_sw_init(g, FALCON_ID_MINION);
if (err != 0) {
nvgpu_err(g, "failed to sw init FALCON_ID_MINION");
goto nvlink_init_exit;
}
g->nvlink.discovered_links &= ~g->nvlink.link_disable_mask;
nvgpu_log(g, gpu_dbg_nvlink, "link_disable_mask = 0x%08x (from VBIOS)",
g->nvlink.link_disable_mask);
/* Links in reset should be removed from initialized link sw state */
g->nvlink.initialized_links &= __gv100_nvlink_get_link_reset_mask(g);
/* VBIOS link_disable_mask should be sufficient to find the connected
* links. As VBIOS is not updated with correct mask, we parse the DT
* node where we hardcode the link_id. DT method is not scalable as same
* DT node is used for different dGPUs connected over PCIE.
* Remove the DT parsing of link id and use HAL to get link_mask based
* on the GPU. This is temporary WAR while we get the VBIOS updated with
* correct mask.
*/
g->ops.nvlink.get_connected_link_mask(&(g->nvlink.connected_links));
nvgpu_log(g, gpu_dbg_nvlink, "connected_links = 0x%08x",
g->nvlink.connected_links);
/* Track only connected links */
g->nvlink.discovered_links &= g->nvlink.connected_links;
nvgpu_log(g, gpu_dbg_nvlink, "discovered_links = 0x%08x (combination)",
g->nvlink.discovered_links);
if (hweight32(g->nvlink.discovered_links) > 1) {
nvgpu_err(g, "more than one link enabled");
err = -EINVAL;
goto nvlink_init_exit;
}
err = __gv100_nvlink_state_load_hal(g);
if (err != 0) {
nvgpu_err(g, " failed Nvlink state load");
goto nvlink_init_exit;
}
err = gv100_nvlink_minion_configure_ac_coupling(g,
g->nvlink.ac_coupling_mask, true);
if (err != 0) {
nvgpu_err(g, " failed Nvlink state load");
goto nvlink_init_exit;
}
/* Program clocks */
gv100_nvlink_prog_alt_clk(g);
nvlink_init_exit:
return err;
}
int gv100_nvlink_speed_config(struct gk20a *g)
{
g->nvlink.speed = nvgpu_nvlink_speed_20G;
g->nvlink.initpll_ordinal = INITPLL_1;
g->nvlink.initpll_cmd = minion_nvlink_dl_cmd_command_initpll_1_v();
return 0;
}
u32 gv100_nvlink_falcon_base_addr(struct gk20a *g)
{
return g->nvlink.minion_base;
}
#endif /* CONFIG_TEGRA_NVLINK */
Reference in New Issue View Git Blame Copy Permalink