nv-tegra-mirror/linux-nvgpu
1
0
Fork 0
mirror of git://nv-tegra.nvidia.com/linux-nvgpu.git synced 2025-12-22 09:12:24 +03:00
Code Issues Packages Projects Releases Wiki Activity

gpu: nvgpu: Move nvlink HAL code to /hal

Browse Source 
Remove the nvlink register read/write code from /common.
Move the register handling code to /hal and add
HALs to to expose this functionality to common code.

JIRA NVGPU-2964

Change-Id: Iafba9f4e29cc0f1130dbf5dd14fbbf8b6b5bb8ec
Signed-off-by: Tejal Kudav <tkudav@nvidia.com>
Reviewed-on: https://git-master.nvidia.com/r/c/linux-nvgpu/+/2329195
Tested-by: mobile promotions <svcmobile_promotions@nvidia.com>
Reviewed-by: mobile promotions <svcmobile_promotions@nvidia.com>
This commit is contained in:
Tejal Kudav
2020-04-15 12:18:25 +00:00
committed by Alex Waterman
parent 3378fbbb49
commit 25461c7621
12 changed files with 401 additions and 465 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
arch/nvgpu-hal-new.yaml
View File

@@ -684,6 +684,10 @@ nvlink:
hal/nvlink/link_mode_transitions_tu104.h,
hal/nvlink/intr_and_err_handling_tu104.c,
hal/nvlink/intr_and_err_handling_tu104.h,
hal/nvlink/nvlink_gv100.c,
hal/nvlink/nvlink_gv100.h,
hal/nvlink/nvlink_tu104.c,
hal/nvlink/nvlink_tu104.h,
hal/nvlink/minion_gv100.c,
hal/nvlink/minion_gv100.h,
hal/nvlink/minion_tu104.c,

10
arch/nvgpu-hal.yaml
View File

@@ -1,4 +1,4 @@
# Copyright (c) 2019, NVIDIA CORPORATION. All Rights Reserved.
# Copyright (c) 2019-2020, NVIDIA CORPORATION. All Rights Reserved.
#
# HAL units. These are the units that have access to HW.
#
@@ -8,11 +8,3 @@ init:
owner: Terje B
sources: [ include/nvgpu/hal_init.h ]
nvlink:
safe: yes
owner: Tejal K
gpu: dgpu
sources: [ common/nvlink/nvlink_gv100.c,
common/nvlink/nvlink_gv100.h,
common/nvlink/nvlink_tu104.c,
common/nvlink/nvlink_tu104.h ]

4
drivers/gpu/nvgpu/Makefile
View File

@@ -160,13 +160,13 @@ nvgpu-y += \
common/nvlink/init/device_reginit_gv100.o \
common/nvlink/minion.o \
common/nvlink/link_mode_transitions.o \
common/nvlink/nvlink_gv100.o \
common/nvlink/nvlink_tu104.o \
common/nvlink/nvlink.o \
os/linux/nvlink_probe.o \
os/linux/nvlink.o \
hal/nvlink/minion_gv100.o \
hal/nvlink/minion_tu104.o \
hal/nvlink/nvlink_gv100.o \
hal/nvlink/nvlink_tu104.o \
hal/nvlink/intr_and_err_handling_tu104.o \
hal/nvlink/link_mode_transitions_gv100.o \
hal/nvlink/link_mode_transitions_tu104.o

4
drivers/gpu/nvgpu/Makefile.sources
View File

@@ -574,11 +574,11 @@ srcs += common/vbios/nvlink_bios.c \
common/nvlink/init/device_reginit_gv100.c \
common/nvlink/minion.c \
common/nvlink/link_mode_transitions.c \
common/nvlink/nvlink_gv100.c \
common/nvlink/nvlink_tu104.c \
common/nvlink/nvlink.c \
hal/nvlink/minion_gv100.c \
hal/nvlink/minion_tu104.c \
hal/nvlink/nvlink_gv100.c \
hal/nvlink/nvlink_tu104.c \
hal/nvlink/intr_and_err_handling_tu104.c \
hal/nvlink/link_mode_transitions_gv100.c \
hal/nvlink/link_mode_transitions_tu104.c

313
drivers/gpu/nvgpu/common/nvlink/nvlink.c
View File

@@ -24,18 +24,293 @@
#include <nvgpu/nvlink.h>
#include <nvgpu/nvlink_probe.h>
#include <nvgpu/enabled.h>
#include <nvgpu/top.h>
#include <nvgpu/nvlink_bios.h>
#ifdef CONFIG_NVGPU_NVLINK
static int nvgpu_nvlink_enable_links_pre_top(struct gk20a *g,
unsigned long links)
{
u32 link_id;
int err;
unsigned long bit;
nvgpu_log(g, gpu_dbg_nvlink, " enabling 0x%lx links", links);
for_each_set_bit(bit, &links, NVLINK_MAX_LINKS_SW) {
link_id = (u32)bit;
/* Take links out of reset */
g->ops.nvlink.clear_link_reset(g, link_id);
/* Before doing any link initialization, run RXDET to check
* if link is connected on other end.
*/
if (g->ops.nvlink.rxdet != NULL) {
err = g->ops.nvlink.rxdet(g, link_id);
if (err != 0) {
return err;
}
}
/* Enable Link DLPL for AN0 */
g->ops.nvlink.enable_link_an0(g, link_id);
/* This should be done by the NVLINK API */
err = g->ops.nvlink.link_mode_transitions.set_sublink_mode(g,
link_id, false, nvgpu_nvlink_sublink_tx_common);
if (err != 0) {
nvgpu_err(g, "Failed to init phy of link: %u", link_id);
return err;
}
err = g->ops.nvlink.link_mode_transitions.set_sublink_mode(g,
link_id, true, nvgpu_nvlink_sublink_rx_rxcal);
if (err != 0) {
nvgpu_err(g, "Failed to RXcal on link: %u", link_id);
return err;
}
err = g->ops.nvlink.link_mode_transitions.set_sublink_mode(g,
link_id, false, nvgpu_nvlink_sublink_tx_data_ready);
if (err != 0) {
nvgpu_err(g, "Failed to set data ready link:%u",
link_id);
return err;
}
g->nvlink.enabled_links |= BIT32(link_id);
}
nvgpu_log(g, gpu_dbg_nvlink, "enabled_links=0x%08x",
g->nvlink.enabled_links);
if (g->nvlink.enabled_links != 0U) {
return 0;
}
nvgpu_err(g, "No links were enabled");
return -EINVAL;
}
static int nvgpu_nvlink_enable_links_post_top(struct gk20a *g,
unsigned long links)
{
u32 link_id;
unsigned long bit;
unsigned long enabled_links = (links & g->nvlink.enabled_links) &
~g->nvlink.initialized_links;
for_each_set_bit(bit, &enabled_links, NVLINK_MAX_LINKS_SW) {
link_id = (u32)bit;
if (g->ops.nvlink.set_sw_war != NULL) {
g->ops.nvlink.set_sw_war(g, link_id);
}
g->ops.nvlink.intr.init_link_err_intr(g, link_id);
g->ops.nvlink.intr.enable_link_err_intr(g, link_id, true);
g->nvlink.initialized_links |= BIT32(link_id);
};
return 0;
}
/*
* Main Nvlink init function. Calls into the Nvlink core API
*/
int nvgpu_nvlink_init(struct gk20a *g)
{
int err = 0;
if (!nvgpu_is_enabled(g, NVGPU_SUPPORT_NVLINK)) {
return -ENODEV;
}
err = nvgpu_nvlink_enumerate(g);
if (err != 0) {
nvgpu_err(g, "failed to enumerate nvlink");
goto fail;
}
/* Set HSHUB and SG_PHY */
nvgpu_set_enabled(g, NVGPU_MM_USE_PHYSICAL_SG, true);
err = g->ops.fb.enable_nvlink(g);
if (err != 0) {
nvgpu_err(g, "failed switch to nvlink sysmem");
goto fail;
}
return err;
fail:
nvgpu_set_enabled(g, NVGPU_MM_USE_PHYSICAL_SG, false);
nvgpu_set_enabled(g, NVGPU_SUPPORT_NVLINK, false);
return err;
}
/*
* Query IOCTRL for device discovery
*/
static int nvgpu_nvlink_discover_ioctrl(struct gk20a *g)
{
int ret = 0;
u32 i;
struct nvgpu_nvlink_ioctrl_list *ioctrl_table;
u32 ioctrl_num_entries = 0U;
if (g->ops.top.get_num_engine_type_entries != NULL) {
ioctrl_num_entries = g->ops.top.get_num_engine_type_entries(g,
NVGPU_ENGINE_IOCTRL);
nvgpu_log_info(g, "ioctrl_num_entries: %d", ioctrl_num_entries);
}
if (ioctrl_num_entries == 0U) {
nvgpu_err(g, "No NVLINK IOCTRL entry found in dev_info table");
return -EINVAL;
}
ioctrl_table = nvgpu_kzalloc(g, ioctrl_num_entries *
sizeof(struct nvgpu_nvlink_ioctrl_list));
if (ioctrl_table == NULL) {
nvgpu_err(g, "Failed to allocate memory for nvlink io table");
return -ENOMEM;
}
for (i = 0U; i < ioctrl_num_entries; i++) {
struct nvgpu_device_info dev_info;
ret = g->ops.top.get_device_info(g, &dev_info,
NVGPU_ENGINE_IOCTRL, i);
if (ret != 0) {
nvgpu_err(g, "Failed to parse dev_info table"
"for engine %d",
NVGPU_ENGINE_IOCTRL);
nvgpu_kfree(g, ioctrl_table);
return -EINVAL;
}
ioctrl_table[i].valid = true;
ioctrl_table[i].intr_enum = dev_info.intr_id;
ioctrl_table[i].reset_enum = dev_info.reset_id;
ioctrl_table[i].pri_base_addr = dev_info.pri_base;
nvgpu_log(g, gpu_dbg_nvlink,
"Dev %d: Pri_Base = 0x%0x Intr = %d Reset = %d",
i, ioctrl_table[i].pri_base_addr,
ioctrl_table[i].intr_enum,
ioctrl_table[i].reset_enum);
}
g->nvlink.ioctrl_table = ioctrl_table;
g->nvlink.io_num_entries = ioctrl_num_entries;
return 0;
}
/*
* Performs nvlink device level initialization by discovering the topology
* taking device out of reset, boot minion, set clocks up and common interrupts
*/
int nvgpu_nvlink_early_init(struct gk20a *g)
{
return g->ops.nvlink.early_init(g);
int err = 0;
u32 mc_reset_nvlink_mask;
if (!nvgpu_is_enabled(g, NVGPU_SUPPORT_NVLINK)) {
return -EINVAL;
}
err = nvgpu_bios_get_nvlink_config_data(g);
if (err != 0) {
nvgpu_err(g, "failed to read nvlink vbios data");
goto exit;
}
err = nvgpu_nvlink_discover_ioctrl(g);
if (err != 0) {
goto exit;
}
/* Enable NVLINK in MC */
mc_reset_nvlink_mask = BIT32(g->nvlink.ioctrl_table[0].reset_enum);
nvgpu_log(g, gpu_dbg_nvlink, "mc_reset_nvlink_mask: 0x%x",
mc_reset_nvlink_mask);
g->ops.mc.reset(g, mc_reset_nvlink_mask);
nvgpu_mc_intr_stall_unit_config(g, MC_INTR_UNIT_NVLINK,
MC_INTR_ENABLE);
err = g->ops.nvlink.discover_link(g);
if ((err != 0) || (g->nvlink.discovered_links == 0U)) {
nvgpu_err(g, "No links available");
goto exit;
}
err = nvgpu_falcon_sw_init(g, FALCON_ID_MINION);
if (err != 0) {
nvgpu_err(g, "failed to sw init FALCON_ID_MINION");
goto exit;
}
g->nvlink.discovered_links &= ~g->nvlink.link_disable_mask;
nvgpu_log(g, gpu_dbg_nvlink, "link_disable_mask = 0x%08x (from VBIOS)",
g->nvlink.link_disable_mask);
/* Links in reset should be removed from initialized link sw state */
g->nvlink.initialized_links &= g->ops.nvlink.get_link_reset_mask(g);
/* VBIOS link_disable_mask should be sufficient to find the connected
* links. As VBIOS is not updated with correct mask, we parse the DT
* node where we hardcode the link_id. DT method is not scalable as same
* DT node is used for different dGPUs connected over PCIE.
* Remove the DT parsing of link id and use HAL to get link_mask based
* on the GPU. This is temporary WAR while we get the VBIOS updated with
* correct mask.
*/
g->ops.nvlink.get_connected_link_mask(&(g->nvlink.connected_links));
nvgpu_log(g, gpu_dbg_nvlink, "connected_links = 0x%08x",
g->nvlink.connected_links);
/* Track only connected links */
g->nvlink.discovered_links &= g->nvlink.connected_links;
nvgpu_log(g, gpu_dbg_nvlink, "discovered_links = 0x%08x (combination)",
g->nvlink.discovered_links);
if (hweight32(g->nvlink.discovered_links) > 1U) {
nvgpu_err(g, "more than one link enabled");
err = -EINVAL;
goto nvlink_init_exit;
}
g->nvlink.speed = nvgpu_nvlink_speed_20G;
err = nvgpu_nvlink_minion_load(g);
if (err != 0) {
nvgpu_err(g, "Failed Nvlink state load");
goto nvlink_init_exit;
}
err = g->ops.nvlink.configure_ac_coupling(g,
g->nvlink.ac_coupling_mask, true);
if (err != 0) {
nvgpu_err(g, "Failed AC coupling configuration");
goto nvlink_init_exit;
}
/* Program clocks */
g->ops.nvlink.prog_alt_clk(g);
nvlink_init_exit:
nvgpu_falcon_sw_free(g, FALCON_ID_MINION);
exit:
return err;
}
int nvgpu_nvlink_link_early_init(struct gk20a *g)
{
u32 link_id;
int ret = 0;
/*
* First check the topology and setup connectivity
* HACK: we are only enabling one link for now!!!
@@ -44,34 +319,42 @@ int nvgpu_nvlink_link_early_init(struct gk20a *g)
g->nvlink.links[link_id].remote_info.is_connected = true;
g->nvlink.links[link_id].remote_info.device_type =
nvgpu_nvlink_endp_tegra;
return g->ops.nvlink.link_early_init(g, BIT32(link_id));
ret = nvgpu_nvlink_enable_links_pre_top(g, BIT32(link_id));
if (ret != 0) {
nvgpu_err(g, "Pre topology failed for link");
return ret;
}
ret = nvgpu_nvlink_enable_links_post_top(g, BIT32(link_id));
if (ret != 0) {
nvgpu_err(g, "Post topology failed for link");
return ret;
}
return ret;
}
int nvgpu_nvlink_interface_init(struct gk20a *g)
{
int err;
err = g->ops.nvlink.interface_init(g);
return err;
err = g->ops.fb.init_nvlink(g);
if (err != 0) {
nvgpu_err(g, "failed to setup nvlinks for sysmem");
return err;
}
return 0;
}
int nvgpu_nvlink_interface_disable(struct gk20a *g)
{
int err = 0;
if (g->ops.nvlink.interface_disable != NULL) {
err = g->ops.nvlink.interface_disable(g);
}
return err;
return 0;
}
int nvgpu_nvlink_dev_shutdown(struct gk20a *g)
{
int err;
err = g->ops.nvlink.shutdown(g);
return err;
nvgpu_falcon_sw_free(g, FALCON_ID_MINION);
return 0;
}
#endif

16
drivers/gpu/nvgpu/hal/init/hal_tu104.c
View File

@@ -185,8 +185,8 @@
#include "hal/nvlink/minion_tu104.h"
#include "hal/nvlink/link_mode_transitions_gv100.h"
#include "hal/nvlink/link_mode_transitions_tu104.h"
#include "common/nvlink/nvlink_gv100.h"
#include "common/nvlink/nvlink_tu104.h"
#include "hal/nvlink/nvlink_gv100.h"
#include "hal/nvlink/nvlink_tu104.h"
#include "hal/fifo/channel_gk20a.h"
#include "hal/fifo/channel_gm20b.h"
#include "hal/fifo/channel_gv11b.h"
@@ -236,6 +236,7 @@
#include <nvgpu/nvgpu_init.h>
#include <nvgpu/sbr.h>
#include <nvgpu/nvhost.h>
#include <nvgpu/nvlink.h>
#include <nvgpu/hw/tu104/hw_pwr_tu104.h>
@@ -1514,14 +1515,16 @@ static const struct gpu_ops tu104_ops = {
},
#if defined(CONFIG_NVGPU_NVLINK)
.nvlink = {
.init = nvgpu_nvlink_init,
.get_link_reset_mask = gv100_nvlink_get_link_reset_mask,
.discover_ioctrl = gv100_nvlink_discover_ioctrl,
.discover_link = gv100_nvlink_discover_link,
.init = gv100_nvlink_init,
.rxdet = tu104_nvlink_rxdet,
.get_connected_link_mask = tu104_nvlink_get_connected_link_mask,
.set_sw_war = NULL,
.link_early_init = gv100_nvlink_link_early_init,
.configure_ac_coupling = gv100_nvlink_configure_ac_coupling,
.prog_alt_clk = gv100_nvlink_prog_alt_clk,
.clear_link_reset = gv100_nvlink_clear_link_reset,
.enable_link_an0 = gv100_nvlink_enable_link_an0,
/* API */
.link_mode_transitions = {
.setup_pll = tu104_nvlink_setup_pll,
@@ -1534,10 +1537,7 @@ static const struct gpu_ops tu104_ops = {
.get_sublink_mode = gv100_nvlink_link_get_sublink_mode,
.set_sublink_mode = gv100_nvlink_link_set_sublink_mode,
},
.interface_init = gv100_nvlink_interface_init,
.reg_init = gv100_nvlink_reg_init,
.shutdown = gv100_nvlink_shutdown,
.early_init = gv100_nvlink_early_init,
.minion = {
.base_addr = gv100_nvlink_minion_base_addr,
.is_running = gv100_nvlink_minion_is_running,

484
drivers/gpu/nvgpu/common/nvlink/nvlink_gv100.c → drivers/gpu/nvgpu/hal/nvlink/nvlink_gv100.c
View File

@@ -22,18 +22,11 @@
#ifdef CONFIG_NVGPU_NVLINK
#include <nvgpu/nvgpu_common.h>
#include <nvgpu/nvlink_bios.h>
#include <nvgpu/bitops.h>
#include <nvgpu/nvlink.h>
#include <nvgpu/enabled.h>
#include <nvgpu/io.h>
#include <nvgpu/utils.h>
#include <nvgpu/timers.h>
#include <nvgpu/gk20a.h>
#include <nvgpu/top.h>
#include <nvgpu/nvlink_minion.h>
#include <nvgpu/nvlink_link_mode_transitions.h>
#include <nvgpu/gops_mc.h>
#include <nvgpu/mc.h>
@@ -55,11 +48,6 @@ u32 gv100_nvlink_get_link_reset_mask(struct gk20a *g)
return ioctrl_reset_linkreset_v(reg_data);
}
static int gv100_nvlink_state_load_hal(struct gk20a *g)
{
return nvgpu_nvlink_minion_load(g);
}
static const char *gv100_device_type_to_str(u32 type)
{
if (type == NVL_DEVICE(ioctrl)) {
@@ -95,215 +83,6 @@ static const char *gv100_device_type_to_str(u32 type)
return "UNKNOWN";
}
/*
* Configure AC coupling
*/
static int gv100_nvlink_minion_configure_ac_coupling(struct gk20a *g,
unsigned long mask, bool sync)
{
int err = 0;
u32 link_id;
u32 temp;
unsigned long bit;
for_each_set_bit(bit, &mask, NVLINK_MAX_LINKS_SW) {
link_id = (u32)bit;
temp = DLPL_REG_RD32(g, link_id, nvl_link_config_r());
temp &= ~nvl_link_config_ac_safe_en_m();
temp |= nvl_link_config_ac_safe_en_on_f();
DLPL_REG_WR32(g, link_id, nvl_link_config_r(), temp);
err = g->ops.nvlink.minion.send_dlcmd(g, link_id,
NVGPU_NVLINK_MINION_DLCMD_SETACMODE, sync);
if (err != 0) {
return err;
}
}
return err;
}
static void gv100_nvlink_prog_alt_clk(struct gk20a *g)
{
u32 tmp;
/* RMW registers need to be separate */
tmp = gk20a_readl(g, trim_sys_nvl_common_clk_alt_switch_r());
tmp &= ~trim_sys_nvl_common_clk_alt_switch_slowclk_m();
tmp |= trim_sys_nvl_common_clk_alt_switch_slowclk_xtal4x_f();
gk20a_writel(g, trim_sys_nvl_common_clk_alt_switch_r(), tmp);
}
static int gv100_nvlink_enable_links_pre_top(struct gk20a *g,
unsigned long links)
{
u32 link_id;
u32 tmp;
u32 reg;
u32 delay = ioctrl_reset_sw_post_reset_delay_microseconds_v();
int err;
unsigned long bit;
nvgpu_log(g, gpu_dbg_nvlink, " enabling 0x%lx links", links);
/* Take links out of reset */
for_each_set_bit(bit, &links, NVLINK_MAX_LINKS_SW) {
link_id = (u32)bit;
reg = IOCTRL_REG_RD32(g, ioctrl_reset_r());
tmp = (BIT32(link_id) |
BIT32(g->nvlink.links[link_id].pll_master_link_id));
reg = set_field(reg, ioctrl_reset_linkreset_m(),
ioctrl_reset_linkreset_f(ioctrl_reset_linkreset_v(reg) |
tmp));
IOCTRL_REG_WR32(g, ioctrl_reset_r(), reg);
nvgpu_udelay(delay);
/* Clear warm reset persistent state */
reg = IOCTRL_REG_RD32(g, ioctrl_debug_reset_r());
reg &= ~(ioctrl_debug_reset_link_f(1U) |
ioctrl_debug_reset_common_f(1U));
IOCTRL_REG_WR32(g, ioctrl_debug_reset_r(), reg);
nvgpu_udelay(delay);
reg |= (ioctrl_debug_reset_link_f(1U) |
ioctrl_debug_reset_common_f(1U));
IOCTRL_REG_WR32(g, ioctrl_debug_reset_r(), reg);
nvgpu_udelay(delay);
/* Before doing any link initialization, run RXDET to check
* if link is connected on other end.
*/
if (g->ops.nvlink.rxdet != NULL) {
err = g->ops.nvlink.rxdet(g, link_id);
if (err != 0) {
return err;
}
}
/* Enable Link DLPL for AN0 */
reg = DLPL_REG_RD32(g, link_id, nvl_link_config_r());
reg = set_field(reg, nvl_link_config_link_en_m(),
nvl_link_config_link_en_f(1));
DLPL_REG_WR32(g, link_id, nvl_link_config_r(), reg);
/* This should be done by the NVLINK API */
err = g->ops.nvlink.link_mode_transitions.set_sublink_mode(g,
link_id, false, nvgpu_nvlink_sublink_tx_common);
if (err != 0) {
nvgpu_err(g, "Failed to init phy of link: %u", link_id);
return err;
}
err = g->ops.nvlink.link_mode_transitions.set_sublink_mode(g,
link_id, true, nvgpu_nvlink_sublink_rx_rxcal);
if (err != 0) {
nvgpu_err(g, "Failed to RXcal on link: %u", link_id);
return err;
}
err = g->ops.nvlink.link_mode_transitions.set_sublink_mode(g,
link_id, false, nvgpu_nvlink_sublink_tx_data_ready);
if (err != 0) {
nvgpu_err(g, "Failed to set data ready link:%u",
link_id);
return err;
}
g->nvlink.enabled_links |= BIT32(link_id);
}
nvgpu_log(g, gpu_dbg_nvlink, "enabled_links=0x%08x",
g->nvlink.enabled_links);
if (g->nvlink.enabled_links != 0U) {
return 0;
}
nvgpu_err(g, " No links were enabled");
return -EINVAL;
}
void gv100_nvlink_set_sw_war(struct gk20a *g, u32 link_id)
{
u32 reg;
/* WAR for HW bug 1888034 */
reg = DLPL_REG_RD32(g, link_id, nvl_sl0_safe_ctrl2_tx_r());
reg = set_field(reg, nvl_sl0_safe_ctrl2_tx_ctr_init_m(),
nvl_sl0_safe_ctrl2_tx_ctr_init_init_f());
reg = set_field(reg, nvl_sl0_safe_ctrl2_tx_ctr_initscl_m(),
nvl_sl0_safe_ctrl2_tx_ctr_initscl_init_f());
DLPL_REG_WR32(g, link_id, nvl_sl0_safe_ctrl2_tx_r(), reg);
}
static int gv100_nvlink_enable_links_post_top(struct gk20a *g,
unsigned long links)
{
u32 link_id;
unsigned long bit;
unsigned long enabled_links = (links & g->nvlink.enabled_links) &
~g->nvlink.initialized_links;
for_each_set_bit(bit, &enabled_links, NVLINK_MAX_LINKS_SW) {
link_id = (u32)bit;
if (g->ops.nvlink.set_sw_war != NULL) {
g->ops.nvlink.set_sw_war(g, link_id);
}
g->ops.nvlink.intr.init_link_err_intr(g, link_id);
g->ops.nvlink.intr.enable_link_err_intr(g, link_id, true);
g->nvlink.initialized_links |= BIT32(link_id);
};
return 0;
}
/*
*******************************************************************************
* Internal "ops" functions *
*******************************************************************************
*/
/*
* Main Nvlink init function. Calls into the Nvlink core API
*/
int gv100_nvlink_init(struct gk20a *g)
{
int err = 0;
if (!nvgpu_is_enabled(g, NVGPU_SUPPORT_NVLINK)) {
return -ENODEV;
}
err = nvgpu_nvlink_enumerate(g);
if (err != 0) {
nvgpu_err(g, "failed to enumerate nvlink");
goto fail;
}
/* Set HSHUB and SG_PHY */
nvgpu_set_enabled(g, NVGPU_MM_USE_PHYSICAL_SG, true);
err = g->ops.fb.enable_nvlink(g);
if (err != 0) {
nvgpu_err(g, "failed switch to nvlink sysmem");
goto fail;
}
return err;
fail:
nvgpu_set_enabled(g, NVGPU_MM_USE_PHYSICAL_SG, false);
nvgpu_set_enabled(g, NVGPU_SUPPORT_NVLINK, false);
return err;
}
/*
* Query internal device topology and discover devices in nvlink local
* infrastructure. Initialize register base and offsets
@@ -676,121 +455,100 @@ int gv100_nvlink_discover_link(struct gk20a *g)
return err;
}
/*
* Query IOCTRL for device discovery
* Configure AC coupling
*/
int gv100_nvlink_discover_ioctrl(struct gk20a *g)
int gv100_nvlink_configure_ac_coupling(struct gk20a *g,
unsigned long mask, bool sync)
{
int ret = 0;
u32 i;
struct nvgpu_nvlink_ioctrl_list *ioctrl_table;
u32 ioctrl_num_entries = 0U;
int err = 0;
u32 link_id;
u32 temp;
unsigned long bit;
if (g->ops.top.get_num_engine_type_entries != NULL) {
ioctrl_num_entries = g->ops.top.get_num_engine_type_entries(g,
NVGPU_ENGINE_IOCTRL);
nvgpu_log_info(g, "ioctrl_num_entries: %d", ioctrl_num_entries);
}
for_each_set_bit(bit, &mask, NVLINK_MAX_LINKS_SW) {
link_id = (u32)bit;
temp = DLPL_REG_RD32(g, link_id, nvl_link_config_r());
temp &= ~nvl_link_config_ac_safe_en_m();
temp |= nvl_link_config_ac_safe_en_on_f();
if (ioctrl_num_entries == 0U) {
nvgpu_err(g, "No NVLINK IOCTRL entry found in dev_info table");
return -EINVAL;
}
DLPL_REG_WR32(g, link_id, nvl_link_config_r(), temp);
ioctrl_table = nvgpu_kzalloc(g, ioctrl_num_entries *
sizeof(struct nvgpu_nvlink_ioctrl_list));
if (ioctrl_table == NULL) {
nvgpu_err(g, "Failed to allocate memory for nvlink io table");
return -ENOMEM;
}
err = g->ops.nvlink.minion.send_dlcmd(g, link_id,
NVGPU_NVLINK_MINION_DLCMD_SETACMODE, sync);
for (i = 0U; i < ioctrl_num_entries; i++) {
struct nvgpu_device_info dev_info;
ret = g->ops.top.get_device_info(g, &dev_info,
NVGPU_ENGINE_IOCTRL, i);
if (ret != 0) {
nvgpu_err(g, "Failed to parse dev_info table"
"for engine %d",
NVGPU_ENGINE_IOCTRL);
nvgpu_kfree(g, ioctrl_table);
return -EINVAL;
if (err != 0) {
return err;
}
ioctrl_table[i].valid = true;
ioctrl_table[i].intr_enum = (u8)dev_info.intr_id;
ioctrl_table[i].reset_enum = (u8)dev_info.reset_id;
ioctrl_table[i].pri_base_addr = dev_info.pri_base;
nvgpu_log(g, gpu_dbg_nvlink,
"Dev %d: Pri_Base = 0x%0x Intr = %d Reset = %d",
i, ioctrl_table[i].pri_base_addr,
ioctrl_table[i].intr_enum,
ioctrl_table[i].reset_enum);
}
g->nvlink.ioctrl_table = ioctrl_table;
g->nvlink.io_num_entries = ioctrl_num_entries;
return 0;
}
/*
*******************************************************************************
* NVLINK API FUNCTIONS *
*******************************************************************************
*/
/*
* Performs link level initialization like phy inits, AN0 and interrupts
*/
int gv100_nvlink_link_early_init(struct gk20a *g, unsigned long mask)
{
int err;
err = gv100_nvlink_enable_links_pre_top(g, mask);
if (err != 0) {
nvgpu_err(g, "Pre topology failed for links 0x%lx", mask);
return err;
}
nvgpu_log(g, gpu_dbg_nvlink, "pretopology enabled: 0x%lx",
mask & g->nvlink.enabled_links);
err = gv100_nvlink_enable_links_post_top(g, mask);
return err;
}
/*
* Performs memory interface initialization
*/
int gv100_nvlink_interface_init(struct gk20a *g)
void gv100_nvlink_prog_alt_clk(struct gk20a *g)
{
int err;
u32 tmp;
err = g->ops.fb.init_nvlink(g);
if (err != 0) {
nvgpu_err(g, "failed to setup nvlinks for sysmem");
return err;
}
return 0;
/* RMW registers need to be separate */
tmp = gk20a_readl(g, trim_sys_nvl_common_clk_alt_switch_r());
tmp &= ~trim_sys_nvl_common_clk_alt_switch_slowclk_m();
tmp |= trim_sys_nvl_common_clk_alt_switch_slowclk_xtal4x_f();
gk20a_writel(g, trim_sys_nvl_common_clk_alt_switch_r(), tmp);
}
int gv100_nvlink_interface_disable(struct gk20a *g)
void gv100_nvlink_clear_link_reset(struct gk20a *g, u32 link_id)
{
return 0;
u32 reg;
u32 tmp;
u32 delay = ioctrl_reset_sw_post_reset_delay_microseconds_v();
reg = IOCTRL_REG_RD32(g, ioctrl_reset_r());
tmp = (BIT32(link_id) |
BIT32(g->nvlink.links[link_id].pll_master_link_id));
reg = set_field(reg, ioctrl_reset_linkreset_m(),
ioctrl_reset_linkreset_f(ioctrl_reset_linkreset_v(reg) |
tmp));
IOCTRL_REG_WR32(g, ioctrl_reset_r(), reg);
nvgpu_udelay(delay);
/* Clear warm reset persistent state */
reg = IOCTRL_REG_RD32(g, ioctrl_debug_reset_r());
reg &= ~(ioctrl_debug_reset_link_f(1U) |
ioctrl_debug_reset_common_f(1U));
IOCTRL_REG_WR32(g, ioctrl_debug_reset_r(), reg);
nvgpu_udelay(delay);
reg |= (ioctrl_debug_reset_link_f(1U) |
ioctrl_debug_reset_common_f(1U));
IOCTRL_REG_WR32(g, ioctrl_debug_reset_r(), reg);
nvgpu_udelay(delay);
}
/*
* Shutdown device. This should tear down Nvlink connection.
* For now return.
*/
int gv100_nvlink_shutdown(struct gk20a *g)
void gv100_nvlink_enable_link_an0(struct gk20a *g, u32 link_id)
{
nvgpu_falcon_sw_free(g, FALCON_ID_MINION);
u32 reg;
return 0;
reg = DLPL_REG_RD32(g, link_id, nvl_link_config_r());
reg = set_field(reg, nvl_link_config_link_en_m(),
nvl_link_config_link_en_f(1));
DLPL_REG_WR32(g, link_id, nvl_link_config_r(), reg);
}
void gv100_nvlink_set_sw_war(struct gk20a *g, u32 link_id)
{
u32 reg;
/* WAR for HW bug 1888034 */
reg = DLPL_REG_RD32(g, link_id, nvl_sl0_safe_ctrl2_tx_r());
reg = set_field(reg, nvl_sl0_safe_ctrl2_tx_ctr_init_m(),
nvl_sl0_safe_ctrl2_tx_ctr_init_init_f());
reg = set_field(reg, nvl_sl0_safe_ctrl2_tx_ctr_initscl_m(),
nvl_sl0_safe_ctrl2_tx_ctr_initscl_init_f());
DLPL_REG_WR32(g, link_id, nvl_sl0_safe_ctrl2_tx_r(), reg);
}
/* Hardcode the link_mask while we wait for VBIOS link_disable_mask field
@@ -800,103 +558,5 @@ void gv100_nvlink_get_connected_link_mask(u32 *link_mask)
{
*link_mask = GV100_CONNECTED_LINK_MASK;
}
/*
* Performs nvlink device level initialization by discovering the topology
* taking device out of reset, boot minion, set clocks up and common interrupts
*/
int gv100_nvlink_early_init(struct gk20a *g)
{
int err = 0;
u32 mc_reset_nvlink_mask;
if (!nvgpu_is_enabled(g, NVGPU_SUPPORT_NVLINK)) {
return -EINVAL;
}
err = nvgpu_bios_get_nvlink_config_data(g);
if (err != 0) {
nvgpu_err(g, "failed to read nvlink vbios data");
goto exit;
}
err = g->ops.nvlink.discover_ioctrl(g);
if (err != 0) {
goto exit;
}
/* Enable NVLINK in MC */
mc_reset_nvlink_mask = BIT32(g->nvlink.ioctrl_table[0].reset_enum);
nvgpu_log(g, gpu_dbg_nvlink, "mc_reset_nvlink_mask: 0x%x",
mc_reset_nvlink_mask);
g->ops.mc.reset(g, mc_reset_nvlink_mask);
nvgpu_mc_intr_stall_unit_config(g, MC_INTR_UNIT_NVLINK,
MC_INTR_ENABLE);
err = g->ops.nvlink.discover_link(g);
if ((err != 0) || (g->nvlink.discovered_links == 0U)) {
nvgpu_err(g, "No links available");
goto exit;
}
err = nvgpu_falcon_sw_init(g, FALCON_ID_MINION);
if (err != 0) {
nvgpu_err(g, "failed to sw init FALCON_ID_MINION");
goto exit;
}
g->nvlink.discovered_links &= ~g->nvlink.link_disable_mask;
nvgpu_log(g, gpu_dbg_nvlink, "link_disable_mask = 0x%08x (from VBIOS)",
g->nvlink.link_disable_mask);
/* Links in reset should be removed from initialized link sw state */
g->nvlink.initialized_links &= g->ops.nvlink.get_link_reset_mask(g);
/* VBIOS link_disable_mask should be sufficient to find the connected
* links. As VBIOS is not updated with correct mask, we parse the DT
* node where we hardcode the link_id. DT method is not scalable as same
* DT node is used for different dGPUs connected over PCIE.
* Remove the DT parsing of link id and use HAL to get link_mask based
* on the GPU. This is temporary WAR while we get the VBIOS updated with
* correct mask.
*/
g->ops.nvlink.get_connected_link_mask(&(g->nvlink.connected_links));
nvgpu_log(g, gpu_dbg_nvlink, "connected_links = 0x%08x",
g->nvlink.connected_links);
/* Track only connected links */
g->nvlink.discovered_links &= g->nvlink.connected_links;
nvgpu_log(g, gpu_dbg_nvlink, "discovered_links = 0x%08x (combination)",
g->nvlink.discovered_links);
if (hweight32(g->nvlink.discovered_links) > 1U) {
nvgpu_err(g, "more than one link enabled");
err = -EINVAL;
goto nvlink_init_exit;
}
g->nvlink.speed = nvgpu_nvlink_speed_20G;
err = gv100_nvlink_state_load_hal(g);
if (err != 0) {
nvgpu_err(g, " failed Nvlink state load");
goto nvlink_init_exit;
}
err = gv100_nvlink_minion_configure_ac_coupling(g,
g->nvlink.ac_coupling_mask, true);
if (err != 0) {
nvgpu_err(g, " failed Nvlink state load");
goto nvlink_init_exit;
}
/* Program clocks */
gv100_nvlink_prog_alt_clk(g);
nvlink_init_exit:
nvgpu_falcon_sw_free(g, FALCON_ID_MINION);
exit:
return err;
}
#endif /* CONFIG_NVGPU_NVLINK */

12
drivers/gpu/nvgpu/common/nvlink/nvlink_gv100.h → drivers/gpu/nvgpu/hal/nvlink/nvlink_gv100.h
View File

@@ -28,15 +28,13 @@
struct gk20a;
u32 gv100_nvlink_get_link_reset_mask(struct gk20a *g);
int gv100_nvlink_discover_ioctrl(struct gk20a *g);
int gv100_nvlink_discover_link(struct gk20a *g);
int gv100_nvlink_init(struct gk20a *g);
void gv100_nvlink_get_connected_link_mask(u32 *link_mask);
void gv100_nvlink_set_sw_war(struct gk20a *g, u32 link_id);
/* API */
int gv100_nvlink_link_early_init(struct gk20a *g, unsigned long mask);
int gv100_nvlink_interface_init(struct gk20a *g);
int gv100_nvlink_interface_disable(struct gk20a *g);
int gv100_nvlink_shutdown(struct gk20a *g);
int gv100_nvlink_early_init(struct gk20a *g);
int gv100_nvlink_configure_ac_coupling(struct gk20a *g,
unsigned long mask, bool sync);
void gv100_nvlink_prog_alt_clk(struct gk20a *g);
void gv100_nvlink_clear_link_reset(struct gk20a *g, u32 link_id);
void gv100_nvlink_enable_link_an0(struct gk20a *g, u32 link_id);
#endif

0
drivers/gpu/nvgpu/common/nvlink/nvlink_tu104.c → drivers/gpu/nvgpu/hal/nvlink/nvlink_tu104.c
View File

0
drivers/gpu/nvgpu/common/nvlink/nvlink_tu104.h → drivers/gpu/nvgpu/hal/nvlink/nvlink_tu104.h
View File

13
drivers/gpu/nvgpu/include/nvgpu/gk20a.h
View File

@@ -520,15 +520,18 @@ struct gpu_ops {
} fbp;
struct gops_priv_ring priv_ring;
struct {
u32 (*get_link_reset_mask)(struct gk20a *g);
int (*init)(struct gk20a *g);
int (*discover_ioctrl)(struct gk20a *g);
u32 (*get_link_reset_mask)(struct gk20a *g);
int (*discover_link)(struct gk20a *g);
int (*rxdet)(struct gk20a *g, u32 link_id);
void (*get_connected_link_mask)(u32 *link_mask);
void (*set_sw_war)(struct gk20a *g, u32 link_id);
int (*configure_ac_coupling)(struct gk20a *g,
unsigned long mask, bool sync);
void (*prog_alt_clk)(struct gk20a *g);
void (*clear_link_reset)(struct gk20a *g, u32 link_id);
void (*enable_link_an0)(struct gk20a *g, u32 link_id);
/* API */
int (*link_early_init)(struct gk20a *g, unsigned long mask);
struct {
int (*setup_pll)(struct gk20a *g,
unsigned long link_mask);
@@ -551,11 +554,7 @@ struct gpu_ops {
bool is_rx_sublink,
enum nvgpu_nvlink_sublink_mode mode);
} link_mode_transitions;
int (*interface_init)(struct gk20a *g);
int (*interface_disable)(struct gk20a *g);
int (*reg_init)(struct gk20a *g);
int (*shutdown)(struct gk20a *g);
int (*early_init)(struct gk20a *g);
struct {
u32 (*base_addr)(struct gk20a *g);
bool (*is_running)(struct gk20a *g);

6
drivers/gpu/nvgpu/include/nvgpu/nvlink.h
View File

@@ -47,8 +47,8 @@ struct gk20a;
struct nvgpu_nvlink_ioctrl_list {
bool valid;
u32 pri_base_addr;
u8 intr_enum;
u8 reset_enum;
u32 intr_enum;
u32 reset_enum;
};
struct nvgpu_nvlink_device_list {
@@ -175,7 +175,7 @@ struct nvgpu_nvlink_dev {
/* priv struct */
void *priv;
};
int nvgpu_nvlink_init(struct gk20a *g);
int nvgpu_nvlink_early_init(struct gk20a *g);
int nvgpu_nvlink_link_early_init(struct gk20a *g);
int nvgpu_nvlink_interface_init(struct gk20a *g);