diff --git a/arch/nvgpu-hal-new.yaml b/arch/nvgpu-hal-new.yaml index a92973090..d77f4be84 100644 --- a/arch/nvgpu-hal-new.yaml +++ b/arch/nvgpu-hal-new.yaml @@ -684,6 +684,10 @@ nvlink: hal/nvlink/link_mode_transitions_tu104.h, hal/nvlink/intr_and_err_handling_tu104.c, hal/nvlink/intr_and_err_handling_tu104.h, + hal/nvlink/nvlink_gv100.c, + hal/nvlink/nvlink_gv100.h, + hal/nvlink/nvlink_tu104.c, + hal/nvlink/nvlink_tu104.h, hal/nvlink/minion_gv100.c, hal/nvlink/minion_gv100.h, hal/nvlink/minion_tu104.c, diff --git a/arch/nvgpu-hal.yaml b/arch/nvgpu-hal.yaml index 50d165a41..d2753fd19 100644 --- a/arch/nvgpu-hal.yaml +++ b/arch/nvgpu-hal.yaml @@ -1,4 +1,4 @@ -# Copyright (c) 2019, NVIDIA CORPORATION. All Rights Reserved. +# Copyright (c) 2019-2020, NVIDIA CORPORATION. All Rights Reserved. # # HAL units. These are the units that have access to HW. # @@ -8,11 +8,3 @@ init: owner: Terje B sources: [ include/nvgpu/hal_init.h ] -nvlink: - safe: yes - owner: Tejal K - gpu: dgpu - sources: [ common/nvlink/nvlink_gv100.c, - common/nvlink/nvlink_gv100.h, - common/nvlink/nvlink_tu104.c, - common/nvlink/nvlink_tu104.h ] diff --git a/drivers/gpu/nvgpu/Makefile b/drivers/gpu/nvgpu/Makefile index 8b5ae90ff..01de0c2f0 100644 --- a/drivers/gpu/nvgpu/Makefile +++ b/drivers/gpu/nvgpu/Makefile @@ -160,13 +160,13 @@ nvgpu-y += \ common/nvlink/init/device_reginit_gv100.o \ common/nvlink/minion.o \ common/nvlink/link_mode_transitions.o \ - common/nvlink/nvlink_gv100.o \ - common/nvlink/nvlink_tu104.o \ common/nvlink/nvlink.o \ os/linux/nvlink_probe.o \ os/linux/nvlink.o \ hal/nvlink/minion_gv100.o \ hal/nvlink/minion_tu104.o \ + hal/nvlink/nvlink_gv100.o \ + hal/nvlink/nvlink_tu104.o \ hal/nvlink/intr_and_err_handling_tu104.o \ hal/nvlink/link_mode_transitions_gv100.o \ hal/nvlink/link_mode_transitions_tu104.o diff --git a/drivers/gpu/nvgpu/Makefile.sources b/drivers/gpu/nvgpu/Makefile.sources index c01cfd51f..75ce9e521 100644 --- a/drivers/gpu/nvgpu/Makefile.sources +++ b/drivers/gpu/nvgpu/Makefile.sources @@ -574,11 +574,11 @@ srcs += common/vbios/nvlink_bios.c \ common/nvlink/init/device_reginit_gv100.c \ common/nvlink/minion.c \ common/nvlink/link_mode_transitions.c \ - common/nvlink/nvlink_gv100.c \ - common/nvlink/nvlink_tu104.c \ common/nvlink/nvlink.c \ hal/nvlink/minion_gv100.c \ hal/nvlink/minion_tu104.c \ + hal/nvlink/nvlink_gv100.c \ + hal/nvlink/nvlink_tu104.c \ hal/nvlink/intr_and_err_handling_tu104.c \ hal/nvlink/link_mode_transitions_gv100.c \ hal/nvlink/link_mode_transitions_tu104.c diff --git a/drivers/gpu/nvgpu/common/nvlink/nvlink.c b/drivers/gpu/nvgpu/common/nvlink/nvlink.c index a81ea8034..62af785bc 100644 --- a/drivers/gpu/nvgpu/common/nvlink/nvlink.c +++ b/drivers/gpu/nvgpu/common/nvlink/nvlink.c @@ -24,18 +24,293 @@ #include #include #include +#include +#include #ifdef CONFIG_NVGPU_NVLINK +static int nvgpu_nvlink_enable_links_pre_top(struct gk20a *g, + unsigned long links) +{ + u32 link_id; + int err; + unsigned long bit; + + nvgpu_log(g, gpu_dbg_nvlink, " enabling 0x%lx links", links); + for_each_set_bit(bit, &links, NVLINK_MAX_LINKS_SW) { + link_id = (u32)bit; + + /* Take links out of reset */ + g->ops.nvlink.clear_link_reset(g, link_id); + + /* Before doing any link initialization, run RXDET to check + * if link is connected on other end. + */ + if (g->ops.nvlink.rxdet != NULL) { + err = g->ops.nvlink.rxdet(g, link_id); + if (err != 0) { + return err; + } + } + + /* Enable Link DLPL for AN0 */ + g->ops.nvlink.enable_link_an0(g, link_id); + + /* This should be done by the NVLINK API */ + err = g->ops.nvlink.link_mode_transitions.set_sublink_mode(g, + link_id, false, nvgpu_nvlink_sublink_tx_common); + if (err != 0) { + nvgpu_err(g, "Failed to init phy of link: %u", link_id); + return err; + } + + err = g->ops.nvlink.link_mode_transitions.set_sublink_mode(g, + link_id, true, nvgpu_nvlink_sublink_rx_rxcal); + if (err != 0) { + nvgpu_err(g, "Failed to RXcal on link: %u", link_id); + return err; + } + + err = g->ops.nvlink.link_mode_transitions.set_sublink_mode(g, + link_id, false, nvgpu_nvlink_sublink_tx_data_ready); + if (err != 0) { + nvgpu_err(g, "Failed to set data ready link:%u", + link_id); + return err; + } + + g->nvlink.enabled_links |= BIT32(link_id); + } + + nvgpu_log(g, gpu_dbg_nvlink, "enabled_links=0x%08x", + g->nvlink.enabled_links); + + if (g->nvlink.enabled_links != 0U) { + return 0; + } + + nvgpu_err(g, "No links were enabled"); + return -EINVAL; +} + +static int nvgpu_nvlink_enable_links_post_top(struct gk20a *g, + unsigned long links) +{ + u32 link_id; + unsigned long bit; + unsigned long enabled_links = (links & g->nvlink.enabled_links) & + ~g->nvlink.initialized_links; + + for_each_set_bit(bit, &enabled_links, NVLINK_MAX_LINKS_SW) { + link_id = (u32)bit; + if (g->ops.nvlink.set_sw_war != NULL) { + g->ops.nvlink.set_sw_war(g, link_id); + } + g->ops.nvlink.intr.init_link_err_intr(g, link_id); + g->ops.nvlink.intr.enable_link_err_intr(g, link_id, true); + + g->nvlink.initialized_links |= BIT32(link_id); + }; + + return 0; +} + +/* + * Main Nvlink init function. Calls into the Nvlink core API + */ +int nvgpu_nvlink_init(struct gk20a *g) +{ + int err = 0; + + if (!nvgpu_is_enabled(g, NVGPU_SUPPORT_NVLINK)) { + return -ENODEV; + } + + err = nvgpu_nvlink_enumerate(g); + if (err != 0) { + nvgpu_err(g, "failed to enumerate nvlink"); + goto fail; + } + + /* Set HSHUB and SG_PHY */ + nvgpu_set_enabled(g, NVGPU_MM_USE_PHYSICAL_SG, true); + + err = g->ops.fb.enable_nvlink(g); + if (err != 0) { + nvgpu_err(g, "failed switch to nvlink sysmem"); + goto fail; + } + + return err; + +fail: + nvgpu_set_enabled(g, NVGPU_MM_USE_PHYSICAL_SG, false); + nvgpu_set_enabled(g, NVGPU_SUPPORT_NVLINK, false); + return err; +} + + +/* + * Query IOCTRL for device discovery + */ +static int nvgpu_nvlink_discover_ioctrl(struct gk20a *g) +{ + int ret = 0; + u32 i; + struct nvgpu_nvlink_ioctrl_list *ioctrl_table; + u32 ioctrl_num_entries = 0U; + + if (g->ops.top.get_num_engine_type_entries != NULL) { + ioctrl_num_entries = g->ops.top.get_num_engine_type_entries(g, + NVGPU_ENGINE_IOCTRL); + nvgpu_log_info(g, "ioctrl_num_entries: %d", ioctrl_num_entries); + } + + if (ioctrl_num_entries == 0U) { + nvgpu_err(g, "No NVLINK IOCTRL entry found in dev_info table"); + return -EINVAL; + } + + ioctrl_table = nvgpu_kzalloc(g, ioctrl_num_entries * + sizeof(struct nvgpu_nvlink_ioctrl_list)); + if (ioctrl_table == NULL) { + nvgpu_err(g, "Failed to allocate memory for nvlink io table"); + return -ENOMEM; + } + + for (i = 0U; i < ioctrl_num_entries; i++) { + struct nvgpu_device_info dev_info; + + ret = g->ops.top.get_device_info(g, &dev_info, + NVGPU_ENGINE_IOCTRL, i); + if (ret != 0) { + nvgpu_err(g, "Failed to parse dev_info table" + "for engine %d", + NVGPU_ENGINE_IOCTRL); + nvgpu_kfree(g, ioctrl_table); + return -EINVAL; + } + + ioctrl_table[i].valid = true; + ioctrl_table[i].intr_enum = dev_info.intr_id; + ioctrl_table[i].reset_enum = dev_info.reset_id; + ioctrl_table[i].pri_base_addr = dev_info.pri_base; + nvgpu_log(g, gpu_dbg_nvlink, + "Dev %d: Pri_Base = 0x%0x Intr = %d Reset = %d", + i, ioctrl_table[i].pri_base_addr, + ioctrl_table[i].intr_enum, + ioctrl_table[i].reset_enum); + } + g->nvlink.ioctrl_table = ioctrl_table; + g->nvlink.io_num_entries = ioctrl_num_entries; + + return 0; +} + + +/* + * Performs nvlink device level initialization by discovering the topology + * taking device out of reset, boot minion, set clocks up and common interrupts + */ int nvgpu_nvlink_early_init(struct gk20a *g) { - return g->ops.nvlink.early_init(g); + int err = 0; + u32 mc_reset_nvlink_mask; + + if (!nvgpu_is_enabled(g, NVGPU_SUPPORT_NVLINK)) { + return -EINVAL; + } + + err = nvgpu_bios_get_nvlink_config_data(g); + if (err != 0) { + nvgpu_err(g, "failed to read nvlink vbios data"); + goto exit; + } + + err = nvgpu_nvlink_discover_ioctrl(g); + if (err != 0) { + goto exit; + } + + /* Enable NVLINK in MC */ + mc_reset_nvlink_mask = BIT32(g->nvlink.ioctrl_table[0].reset_enum); + nvgpu_log(g, gpu_dbg_nvlink, "mc_reset_nvlink_mask: 0x%x", + mc_reset_nvlink_mask); + g->ops.mc.reset(g, mc_reset_nvlink_mask); + + nvgpu_mc_intr_stall_unit_config(g, MC_INTR_UNIT_NVLINK, + MC_INTR_ENABLE); + + err = g->ops.nvlink.discover_link(g); + if ((err != 0) || (g->nvlink.discovered_links == 0U)) { + nvgpu_err(g, "No links available"); + goto exit; + } + + err = nvgpu_falcon_sw_init(g, FALCON_ID_MINION); + if (err != 0) { + nvgpu_err(g, "failed to sw init FALCON_ID_MINION"); + goto exit; + } + + g->nvlink.discovered_links &= ~g->nvlink.link_disable_mask; + nvgpu_log(g, gpu_dbg_nvlink, "link_disable_mask = 0x%08x (from VBIOS)", + g->nvlink.link_disable_mask); + + /* Links in reset should be removed from initialized link sw state */ + g->nvlink.initialized_links &= g->ops.nvlink.get_link_reset_mask(g); + + /* VBIOS link_disable_mask should be sufficient to find the connected + * links. As VBIOS is not updated with correct mask, we parse the DT + * node where we hardcode the link_id. DT method is not scalable as same + * DT node is used for different dGPUs connected over PCIE. + * Remove the DT parsing of link id and use HAL to get link_mask based + * on the GPU. This is temporary WAR while we get the VBIOS updated with + * correct mask. + */ + g->ops.nvlink.get_connected_link_mask(&(g->nvlink.connected_links)); + + nvgpu_log(g, gpu_dbg_nvlink, "connected_links = 0x%08x", + g->nvlink.connected_links); + + /* Track only connected links */ + g->nvlink.discovered_links &= g->nvlink.connected_links; + + nvgpu_log(g, gpu_dbg_nvlink, "discovered_links = 0x%08x (combination)", + g->nvlink.discovered_links); + + if (hweight32(g->nvlink.discovered_links) > 1U) { + nvgpu_err(g, "more than one link enabled"); + err = -EINVAL; + goto nvlink_init_exit; + } + + g->nvlink.speed = nvgpu_nvlink_speed_20G; + err = nvgpu_nvlink_minion_load(g); + if (err != 0) { + nvgpu_err(g, "Failed Nvlink state load"); + goto nvlink_init_exit; + } + err = g->ops.nvlink.configure_ac_coupling(g, + g->nvlink.ac_coupling_mask, true); + if (err != 0) { + nvgpu_err(g, "Failed AC coupling configuration"); + goto nvlink_init_exit; + } + + /* Program clocks */ + g->ops.nvlink.prog_alt_clk(g); + +nvlink_init_exit: + nvgpu_falcon_sw_free(g, FALCON_ID_MINION); +exit: + return err; } int nvgpu_nvlink_link_early_init(struct gk20a *g) { u32 link_id; - + int ret = 0; /* * First check the topology and setup connectivity * HACK: we are only enabling one link for now!!! @@ -44,34 +319,42 @@ int nvgpu_nvlink_link_early_init(struct gk20a *g) g->nvlink.links[link_id].remote_info.is_connected = true; g->nvlink.links[link_id].remote_info.device_type = nvgpu_nvlink_endp_tegra; - return g->ops.nvlink.link_early_init(g, BIT32(link_id)); + ret = nvgpu_nvlink_enable_links_pre_top(g, BIT32(link_id)); + if (ret != 0) { + nvgpu_err(g, "Pre topology failed for link"); + return ret; + } + ret = nvgpu_nvlink_enable_links_post_top(g, BIT32(link_id)); + if (ret != 0) { + nvgpu_err(g, "Post topology failed for link"); + return ret; + } + return ret; } int nvgpu_nvlink_interface_init(struct gk20a *g) { int err; - err = g->ops.nvlink.interface_init(g); - return err; + err = g->ops.fb.init_nvlink(g); + if (err != 0) { + nvgpu_err(g, "failed to setup nvlinks for sysmem"); + return err; + } + + return 0; } int nvgpu_nvlink_interface_disable(struct gk20a *g) { - int err = 0; - - if (g->ops.nvlink.interface_disable != NULL) { - err = g->ops.nvlink.interface_disable(g); - } - return err; + return 0; } int nvgpu_nvlink_dev_shutdown(struct gk20a *g) { - int err; - - err = g->ops.nvlink.shutdown(g); - return err; + nvgpu_falcon_sw_free(g, FALCON_ID_MINION); + return 0; } #endif diff --git a/drivers/gpu/nvgpu/hal/init/hal_tu104.c b/drivers/gpu/nvgpu/hal/init/hal_tu104.c index f0f30b988..c78413bfb 100644 --- a/drivers/gpu/nvgpu/hal/init/hal_tu104.c +++ b/drivers/gpu/nvgpu/hal/init/hal_tu104.c @@ -185,8 +185,8 @@ #include "hal/nvlink/minion_tu104.h" #include "hal/nvlink/link_mode_transitions_gv100.h" #include "hal/nvlink/link_mode_transitions_tu104.h" -#include "common/nvlink/nvlink_gv100.h" -#include "common/nvlink/nvlink_tu104.h" +#include "hal/nvlink/nvlink_gv100.h" +#include "hal/nvlink/nvlink_tu104.h" #include "hal/fifo/channel_gk20a.h" #include "hal/fifo/channel_gm20b.h" #include "hal/fifo/channel_gv11b.h" @@ -236,6 +236,7 @@ #include #include #include +#include #include @@ -1514,14 +1515,16 @@ static const struct gpu_ops tu104_ops = { }, #if defined(CONFIG_NVGPU_NVLINK) .nvlink = { + .init = nvgpu_nvlink_init, .get_link_reset_mask = gv100_nvlink_get_link_reset_mask, - .discover_ioctrl = gv100_nvlink_discover_ioctrl, .discover_link = gv100_nvlink_discover_link, - .init = gv100_nvlink_init, .rxdet = tu104_nvlink_rxdet, .get_connected_link_mask = tu104_nvlink_get_connected_link_mask, .set_sw_war = NULL, - .link_early_init = gv100_nvlink_link_early_init, + .configure_ac_coupling = gv100_nvlink_configure_ac_coupling, + .prog_alt_clk = gv100_nvlink_prog_alt_clk, + .clear_link_reset = gv100_nvlink_clear_link_reset, + .enable_link_an0 = gv100_nvlink_enable_link_an0, /* API */ .link_mode_transitions = { .setup_pll = tu104_nvlink_setup_pll, @@ -1534,10 +1537,7 @@ static const struct gpu_ops tu104_ops = { .get_sublink_mode = gv100_nvlink_link_get_sublink_mode, .set_sublink_mode = gv100_nvlink_link_set_sublink_mode, }, - .interface_init = gv100_nvlink_interface_init, .reg_init = gv100_nvlink_reg_init, - .shutdown = gv100_nvlink_shutdown, - .early_init = gv100_nvlink_early_init, .minion = { .base_addr = gv100_nvlink_minion_base_addr, .is_running = gv100_nvlink_minion_is_running, diff --git a/drivers/gpu/nvgpu/common/nvlink/nvlink_gv100.c b/drivers/gpu/nvgpu/hal/nvlink/nvlink_gv100.c similarity index 62% rename from drivers/gpu/nvgpu/common/nvlink/nvlink_gv100.c rename to drivers/gpu/nvgpu/hal/nvlink/nvlink_gv100.c index 1688dffe1..1d08afa83 100644 --- a/drivers/gpu/nvgpu/common/nvlink/nvlink_gv100.c +++ b/drivers/gpu/nvgpu/hal/nvlink/nvlink_gv100.c @@ -22,18 +22,11 @@ #ifdef CONFIG_NVGPU_NVLINK -#include -#include -#include -#include -#include #include #include #include #include -#include #include -#include #include #include @@ -55,11 +48,6 @@ u32 gv100_nvlink_get_link_reset_mask(struct gk20a *g) return ioctrl_reset_linkreset_v(reg_data); } -static int gv100_nvlink_state_load_hal(struct gk20a *g) -{ - return nvgpu_nvlink_minion_load(g); -} - static const char *gv100_device_type_to_str(u32 type) { if (type == NVL_DEVICE(ioctrl)) { @@ -95,215 +83,6 @@ static const char *gv100_device_type_to_str(u32 type) return "UNKNOWN"; } -/* - * Configure AC coupling - */ -static int gv100_nvlink_minion_configure_ac_coupling(struct gk20a *g, - unsigned long mask, bool sync) -{ - int err = 0; - u32 link_id; - u32 temp; - unsigned long bit; - - for_each_set_bit(bit, &mask, NVLINK_MAX_LINKS_SW) { - link_id = (u32)bit; - temp = DLPL_REG_RD32(g, link_id, nvl_link_config_r()); - temp &= ~nvl_link_config_ac_safe_en_m(); - temp |= nvl_link_config_ac_safe_en_on_f(); - - DLPL_REG_WR32(g, link_id, nvl_link_config_r(), temp); - - err = g->ops.nvlink.minion.send_dlcmd(g, link_id, - NVGPU_NVLINK_MINION_DLCMD_SETACMODE, sync); - - if (err != 0) { - return err; - } - } - - return err; -} - -static void gv100_nvlink_prog_alt_clk(struct gk20a *g) -{ - u32 tmp; - - /* RMW registers need to be separate */ - tmp = gk20a_readl(g, trim_sys_nvl_common_clk_alt_switch_r()); - tmp &= ~trim_sys_nvl_common_clk_alt_switch_slowclk_m(); - tmp |= trim_sys_nvl_common_clk_alt_switch_slowclk_xtal4x_f(); - gk20a_writel(g, trim_sys_nvl_common_clk_alt_switch_r(), tmp); -} - -static int gv100_nvlink_enable_links_pre_top(struct gk20a *g, - unsigned long links) -{ - u32 link_id; - u32 tmp; - u32 reg; - u32 delay = ioctrl_reset_sw_post_reset_delay_microseconds_v(); - int err; - unsigned long bit; - - nvgpu_log(g, gpu_dbg_nvlink, " enabling 0x%lx links", links); - /* Take links out of reset */ - for_each_set_bit(bit, &links, NVLINK_MAX_LINKS_SW) { - link_id = (u32)bit; - reg = IOCTRL_REG_RD32(g, ioctrl_reset_r()); - - tmp = (BIT32(link_id) | - BIT32(g->nvlink.links[link_id].pll_master_link_id)); - - reg = set_field(reg, ioctrl_reset_linkreset_m(), - ioctrl_reset_linkreset_f(ioctrl_reset_linkreset_v(reg) | - tmp)); - - IOCTRL_REG_WR32(g, ioctrl_reset_r(), reg); - nvgpu_udelay(delay); - - /* Clear warm reset persistent state */ - reg = IOCTRL_REG_RD32(g, ioctrl_debug_reset_r()); - - reg &= ~(ioctrl_debug_reset_link_f(1U) | - ioctrl_debug_reset_common_f(1U)); - IOCTRL_REG_WR32(g, ioctrl_debug_reset_r(), reg); - nvgpu_udelay(delay); - - reg |= (ioctrl_debug_reset_link_f(1U) | - ioctrl_debug_reset_common_f(1U)); - IOCTRL_REG_WR32(g, ioctrl_debug_reset_r(), reg); - nvgpu_udelay(delay); - - /* Before doing any link initialization, run RXDET to check - * if link is connected on other end. - */ - if (g->ops.nvlink.rxdet != NULL) { - err = g->ops.nvlink.rxdet(g, link_id); - if (err != 0) { - return err; - } - } - - /* Enable Link DLPL for AN0 */ - reg = DLPL_REG_RD32(g, link_id, nvl_link_config_r()); - reg = set_field(reg, nvl_link_config_link_en_m(), - nvl_link_config_link_en_f(1)); - DLPL_REG_WR32(g, link_id, nvl_link_config_r(), reg); - - /* This should be done by the NVLINK API */ - err = g->ops.nvlink.link_mode_transitions.set_sublink_mode(g, - link_id, false, nvgpu_nvlink_sublink_tx_common); - if (err != 0) { - nvgpu_err(g, "Failed to init phy of link: %u", link_id); - return err; - } - - err = g->ops.nvlink.link_mode_transitions.set_sublink_mode(g, - link_id, true, nvgpu_nvlink_sublink_rx_rxcal); - if (err != 0) { - nvgpu_err(g, "Failed to RXcal on link: %u", link_id); - return err; - } - - err = g->ops.nvlink.link_mode_transitions.set_sublink_mode(g, - link_id, false, nvgpu_nvlink_sublink_tx_data_ready); - if (err != 0) { - nvgpu_err(g, "Failed to set data ready link:%u", - link_id); - return err; - } - - g->nvlink.enabled_links |= BIT32(link_id); - } - - nvgpu_log(g, gpu_dbg_nvlink, "enabled_links=0x%08x", - g->nvlink.enabled_links); - - if (g->nvlink.enabled_links != 0U) { - return 0; - } - - nvgpu_err(g, " No links were enabled"); - return -EINVAL; -} - -void gv100_nvlink_set_sw_war(struct gk20a *g, u32 link_id) -{ - u32 reg; - - /* WAR for HW bug 1888034 */ - reg = DLPL_REG_RD32(g, link_id, nvl_sl0_safe_ctrl2_tx_r()); - reg = set_field(reg, nvl_sl0_safe_ctrl2_tx_ctr_init_m(), - nvl_sl0_safe_ctrl2_tx_ctr_init_init_f()); - reg = set_field(reg, nvl_sl0_safe_ctrl2_tx_ctr_initscl_m(), - nvl_sl0_safe_ctrl2_tx_ctr_initscl_init_f()); - DLPL_REG_WR32(g, link_id, nvl_sl0_safe_ctrl2_tx_r(), reg); -} - -static int gv100_nvlink_enable_links_post_top(struct gk20a *g, - unsigned long links) -{ - u32 link_id; - unsigned long bit; - unsigned long enabled_links = (links & g->nvlink.enabled_links) & - ~g->nvlink.initialized_links; - - for_each_set_bit(bit, &enabled_links, NVLINK_MAX_LINKS_SW) { - link_id = (u32)bit; - if (g->ops.nvlink.set_sw_war != NULL) { - g->ops.nvlink.set_sw_war(g, link_id); - } - g->ops.nvlink.intr.init_link_err_intr(g, link_id); - g->ops.nvlink.intr.enable_link_err_intr(g, link_id, true); - - g->nvlink.initialized_links |= BIT32(link_id); - }; - - return 0; -} - -/* - ******************************************************************************* - * Internal "ops" functions * - ******************************************************************************* - */ - - -/* - * Main Nvlink init function. Calls into the Nvlink core API - */ -int gv100_nvlink_init(struct gk20a *g) -{ - int err = 0; - - if (!nvgpu_is_enabled(g, NVGPU_SUPPORT_NVLINK)) { - return -ENODEV; - } - - err = nvgpu_nvlink_enumerate(g); - if (err != 0) { - nvgpu_err(g, "failed to enumerate nvlink"); - goto fail; - } - - /* Set HSHUB and SG_PHY */ - nvgpu_set_enabled(g, NVGPU_MM_USE_PHYSICAL_SG, true); - - err = g->ops.fb.enable_nvlink(g); - if (err != 0) { - nvgpu_err(g, "failed switch to nvlink sysmem"); - goto fail; - } - - return err; - -fail: - nvgpu_set_enabled(g, NVGPU_MM_USE_PHYSICAL_SG, false); - nvgpu_set_enabled(g, NVGPU_SUPPORT_NVLINK, false); - return err; -} - /* * Query internal device topology and discover devices in nvlink local * infrastructure. Initialize register base and offsets @@ -676,121 +455,100 @@ int gv100_nvlink_discover_link(struct gk20a *g) return err; } + /* - * Query IOCTRL for device discovery + * Configure AC coupling */ -int gv100_nvlink_discover_ioctrl(struct gk20a *g) +int gv100_nvlink_configure_ac_coupling(struct gk20a *g, + unsigned long mask, bool sync) { - int ret = 0; - u32 i; - struct nvgpu_nvlink_ioctrl_list *ioctrl_table; - u32 ioctrl_num_entries = 0U; + int err = 0; + u32 link_id; + u32 temp; + unsigned long bit; - if (g->ops.top.get_num_engine_type_entries != NULL) { - ioctrl_num_entries = g->ops.top.get_num_engine_type_entries(g, - NVGPU_ENGINE_IOCTRL); - nvgpu_log_info(g, "ioctrl_num_entries: %d", ioctrl_num_entries); - } + for_each_set_bit(bit, &mask, NVLINK_MAX_LINKS_SW) { + link_id = (u32)bit; + temp = DLPL_REG_RD32(g, link_id, nvl_link_config_r()); + temp &= ~nvl_link_config_ac_safe_en_m(); + temp |= nvl_link_config_ac_safe_en_on_f(); - if (ioctrl_num_entries == 0U) { - nvgpu_err(g, "No NVLINK IOCTRL entry found in dev_info table"); - return -EINVAL; - } + DLPL_REG_WR32(g, link_id, nvl_link_config_r(), temp); - ioctrl_table = nvgpu_kzalloc(g, ioctrl_num_entries * - sizeof(struct nvgpu_nvlink_ioctrl_list)); - if (ioctrl_table == NULL) { - nvgpu_err(g, "Failed to allocate memory for nvlink io table"); - return -ENOMEM; - } + err = g->ops.nvlink.minion.send_dlcmd(g, link_id, + NVGPU_NVLINK_MINION_DLCMD_SETACMODE, sync); - for (i = 0U; i < ioctrl_num_entries; i++) { - struct nvgpu_device_info dev_info; - - ret = g->ops.top.get_device_info(g, &dev_info, - NVGPU_ENGINE_IOCTRL, i); - if (ret != 0) { - nvgpu_err(g, "Failed to parse dev_info table" - "for engine %d", - NVGPU_ENGINE_IOCTRL); - nvgpu_kfree(g, ioctrl_table); - return -EINVAL; + if (err != 0) { + return err; } - - ioctrl_table[i].valid = true; - ioctrl_table[i].intr_enum = (u8)dev_info.intr_id; - ioctrl_table[i].reset_enum = (u8)dev_info.reset_id; - ioctrl_table[i].pri_base_addr = dev_info.pri_base; - nvgpu_log(g, gpu_dbg_nvlink, - "Dev %d: Pri_Base = 0x%0x Intr = %d Reset = %d", - i, ioctrl_table[i].pri_base_addr, - ioctrl_table[i].intr_enum, - ioctrl_table[i].reset_enum); } - g->nvlink.ioctrl_table = ioctrl_table; - g->nvlink.io_num_entries = ioctrl_num_entries; - - return 0; -} - -/* - ******************************************************************************* - * NVLINK API FUNCTIONS * - ******************************************************************************* - */ - -/* - * Performs link level initialization like phy inits, AN0 and interrupts - */ - -int gv100_nvlink_link_early_init(struct gk20a *g, unsigned long mask) -{ - int err; - - err = gv100_nvlink_enable_links_pre_top(g, mask); - if (err != 0) { - nvgpu_err(g, "Pre topology failed for links 0x%lx", mask); - return err; - } - - nvgpu_log(g, gpu_dbg_nvlink, "pretopology enabled: 0x%lx", - mask & g->nvlink.enabled_links); - err = gv100_nvlink_enable_links_post_top(g, mask); return err; } -/* - * Performs memory interface initialization - */ - -int gv100_nvlink_interface_init(struct gk20a *g) +void gv100_nvlink_prog_alt_clk(struct gk20a *g) { - int err; + u32 tmp; - err = g->ops.fb.init_nvlink(g); - if (err != 0) { - nvgpu_err(g, "failed to setup nvlinks for sysmem"); - return err; - } - - return 0; + /* RMW registers need to be separate */ + tmp = gk20a_readl(g, trim_sys_nvl_common_clk_alt_switch_r()); + tmp &= ~trim_sys_nvl_common_clk_alt_switch_slowclk_m(); + tmp |= trim_sys_nvl_common_clk_alt_switch_slowclk_xtal4x_f(); + gk20a_writel(g, trim_sys_nvl_common_clk_alt_switch_r(), tmp); } -int gv100_nvlink_interface_disable(struct gk20a *g) +void gv100_nvlink_clear_link_reset(struct gk20a *g, u32 link_id) { - return 0; + u32 reg; + u32 tmp; + u32 delay = ioctrl_reset_sw_post_reset_delay_microseconds_v(); + + reg = IOCTRL_REG_RD32(g, ioctrl_reset_r()); + tmp = (BIT32(link_id) | + BIT32(g->nvlink.links[link_id].pll_master_link_id)); + reg = set_field(reg, ioctrl_reset_linkreset_m(), + ioctrl_reset_linkreset_f(ioctrl_reset_linkreset_v(reg) | + tmp)); + IOCTRL_REG_WR32(g, ioctrl_reset_r(), reg); + + nvgpu_udelay(delay); + + /* Clear warm reset persistent state */ + reg = IOCTRL_REG_RD32(g, ioctrl_debug_reset_r()); + + reg &= ~(ioctrl_debug_reset_link_f(1U) | + ioctrl_debug_reset_common_f(1U)); + IOCTRL_REG_WR32(g, ioctrl_debug_reset_r(), reg); + nvgpu_udelay(delay); + + reg |= (ioctrl_debug_reset_link_f(1U) | + ioctrl_debug_reset_common_f(1U)); + IOCTRL_REG_WR32(g, ioctrl_debug_reset_r(), reg); + nvgpu_udelay(delay); } -/* - * Shutdown device. This should tear down Nvlink connection. - * For now return. - */ -int gv100_nvlink_shutdown(struct gk20a *g) +void gv100_nvlink_enable_link_an0(struct gk20a *g, u32 link_id) { - nvgpu_falcon_sw_free(g, FALCON_ID_MINION); + u32 reg; - return 0; + reg = DLPL_REG_RD32(g, link_id, nvl_link_config_r()); + reg = set_field(reg, nvl_link_config_link_en_m(), + nvl_link_config_link_en_f(1)); + DLPL_REG_WR32(g, link_id, nvl_link_config_r(), reg); +} + + +void gv100_nvlink_set_sw_war(struct gk20a *g, u32 link_id) +{ + u32 reg; + + /* WAR for HW bug 1888034 */ + reg = DLPL_REG_RD32(g, link_id, nvl_sl0_safe_ctrl2_tx_r()); + reg = set_field(reg, nvl_sl0_safe_ctrl2_tx_ctr_init_m(), + nvl_sl0_safe_ctrl2_tx_ctr_init_init_f()); + reg = set_field(reg, nvl_sl0_safe_ctrl2_tx_ctr_initscl_m(), + nvl_sl0_safe_ctrl2_tx_ctr_initscl_init_f()); + DLPL_REG_WR32(g, link_id, nvl_sl0_safe_ctrl2_tx_r(), reg); } /* Hardcode the link_mask while we wait for VBIOS link_disable_mask field @@ -800,103 +558,5 @@ void gv100_nvlink_get_connected_link_mask(u32 *link_mask) { *link_mask = GV100_CONNECTED_LINK_MASK; } -/* - * Performs nvlink device level initialization by discovering the topology - * taking device out of reset, boot minion, set clocks up and common interrupts - */ -int gv100_nvlink_early_init(struct gk20a *g) -{ - int err = 0; - u32 mc_reset_nvlink_mask; - - if (!nvgpu_is_enabled(g, NVGPU_SUPPORT_NVLINK)) { - return -EINVAL; - } - - err = nvgpu_bios_get_nvlink_config_data(g); - if (err != 0) { - nvgpu_err(g, "failed to read nvlink vbios data"); - goto exit; - } - - err = g->ops.nvlink.discover_ioctrl(g); - if (err != 0) { - goto exit; - } - - /* Enable NVLINK in MC */ - mc_reset_nvlink_mask = BIT32(g->nvlink.ioctrl_table[0].reset_enum); - nvgpu_log(g, gpu_dbg_nvlink, "mc_reset_nvlink_mask: 0x%x", - mc_reset_nvlink_mask); - g->ops.mc.reset(g, mc_reset_nvlink_mask); - - nvgpu_mc_intr_stall_unit_config(g, MC_INTR_UNIT_NVLINK, - MC_INTR_ENABLE); - - err = g->ops.nvlink.discover_link(g); - if ((err != 0) || (g->nvlink.discovered_links == 0U)) { - nvgpu_err(g, "No links available"); - goto exit; - } - - err = nvgpu_falcon_sw_init(g, FALCON_ID_MINION); - if (err != 0) { - nvgpu_err(g, "failed to sw init FALCON_ID_MINION"); - goto exit; - } - - g->nvlink.discovered_links &= ~g->nvlink.link_disable_mask; - nvgpu_log(g, gpu_dbg_nvlink, "link_disable_mask = 0x%08x (from VBIOS)", - g->nvlink.link_disable_mask); - - /* Links in reset should be removed from initialized link sw state */ - g->nvlink.initialized_links &= g->ops.nvlink.get_link_reset_mask(g); - - /* VBIOS link_disable_mask should be sufficient to find the connected - * links. As VBIOS is not updated with correct mask, we parse the DT - * node where we hardcode the link_id. DT method is not scalable as same - * DT node is used for different dGPUs connected over PCIE. - * Remove the DT parsing of link id and use HAL to get link_mask based - * on the GPU. This is temporary WAR while we get the VBIOS updated with - * correct mask. - */ - g->ops.nvlink.get_connected_link_mask(&(g->nvlink.connected_links)); - - nvgpu_log(g, gpu_dbg_nvlink, "connected_links = 0x%08x", - g->nvlink.connected_links); - - /* Track only connected links */ - g->nvlink.discovered_links &= g->nvlink.connected_links; - - nvgpu_log(g, gpu_dbg_nvlink, "discovered_links = 0x%08x (combination)", - g->nvlink.discovered_links); - - if (hweight32(g->nvlink.discovered_links) > 1U) { - nvgpu_err(g, "more than one link enabled"); - err = -EINVAL; - goto nvlink_init_exit; - } - - g->nvlink.speed = nvgpu_nvlink_speed_20G; - err = gv100_nvlink_state_load_hal(g); - if (err != 0) { - nvgpu_err(g, " failed Nvlink state load"); - goto nvlink_init_exit; - } - err = gv100_nvlink_minion_configure_ac_coupling(g, - g->nvlink.ac_coupling_mask, true); - if (err != 0) { - nvgpu_err(g, " failed Nvlink state load"); - goto nvlink_init_exit; - } - - /* Program clocks */ - gv100_nvlink_prog_alt_clk(g); - -nvlink_init_exit: - nvgpu_falcon_sw_free(g, FALCON_ID_MINION); -exit: - return err; -} #endif /* CONFIG_NVGPU_NVLINK */ diff --git a/drivers/gpu/nvgpu/common/nvlink/nvlink_gv100.h b/drivers/gpu/nvgpu/hal/nvlink/nvlink_gv100.h similarity index 82% rename from drivers/gpu/nvgpu/common/nvlink/nvlink_gv100.h rename to drivers/gpu/nvgpu/hal/nvlink/nvlink_gv100.h index c8e7bbf46..64bd7edb7 100644 --- a/drivers/gpu/nvgpu/common/nvlink/nvlink_gv100.h +++ b/drivers/gpu/nvgpu/hal/nvlink/nvlink_gv100.h @@ -28,15 +28,13 @@ struct gk20a; u32 gv100_nvlink_get_link_reset_mask(struct gk20a *g); -int gv100_nvlink_discover_ioctrl(struct gk20a *g); int gv100_nvlink_discover_link(struct gk20a *g); int gv100_nvlink_init(struct gk20a *g); void gv100_nvlink_get_connected_link_mask(u32 *link_mask); void gv100_nvlink_set_sw_war(struct gk20a *g, u32 link_id); -/* API */ -int gv100_nvlink_link_early_init(struct gk20a *g, unsigned long mask); -int gv100_nvlink_interface_init(struct gk20a *g); -int gv100_nvlink_interface_disable(struct gk20a *g); -int gv100_nvlink_shutdown(struct gk20a *g); -int gv100_nvlink_early_init(struct gk20a *g); +int gv100_nvlink_configure_ac_coupling(struct gk20a *g, + unsigned long mask, bool sync); +void gv100_nvlink_prog_alt_clk(struct gk20a *g); +void gv100_nvlink_clear_link_reset(struct gk20a *g, u32 link_id); +void gv100_nvlink_enable_link_an0(struct gk20a *g, u32 link_id); #endif diff --git a/drivers/gpu/nvgpu/common/nvlink/nvlink_tu104.c b/drivers/gpu/nvgpu/hal/nvlink/nvlink_tu104.c similarity index 100% rename from drivers/gpu/nvgpu/common/nvlink/nvlink_tu104.c rename to drivers/gpu/nvgpu/hal/nvlink/nvlink_tu104.c diff --git a/drivers/gpu/nvgpu/common/nvlink/nvlink_tu104.h b/drivers/gpu/nvgpu/hal/nvlink/nvlink_tu104.h similarity index 100% rename from drivers/gpu/nvgpu/common/nvlink/nvlink_tu104.h rename to drivers/gpu/nvgpu/hal/nvlink/nvlink_tu104.h diff --git a/drivers/gpu/nvgpu/include/nvgpu/gk20a.h b/drivers/gpu/nvgpu/include/nvgpu/gk20a.h index b560567d9..786d15eb2 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/gk20a.h +++ b/drivers/gpu/nvgpu/include/nvgpu/gk20a.h @@ -520,15 +520,18 @@ struct gpu_ops { } fbp; struct gops_priv_ring priv_ring; struct { - u32 (*get_link_reset_mask)(struct gk20a *g); int (*init)(struct gk20a *g); - int (*discover_ioctrl)(struct gk20a *g); + u32 (*get_link_reset_mask)(struct gk20a *g); int (*discover_link)(struct gk20a *g); int (*rxdet)(struct gk20a *g, u32 link_id); void (*get_connected_link_mask)(u32 *link_mask); void (*set_sw_war)(struct gk20a *g, u32 link_id); + int (*configure_ac_coupling)(struct gk20a *g, + unsigned long mask, bool sync); + void (*prog_alt_clk)(struct gk20a *g); + void (*clear_link_reset)(struct gk20a *g, u32 link_id); + void (*enable_link_an0)(struct gk20a *g, u32 link_id); /* API */ - int (*link_early_init)(struct gk20a *g, unsigned long mask); struct { int (*setup_pll)(struct gk20a *g, unsigned long link_mask); @@ -551,11 +554,7 @@ struct gpu_ops { bool is_rx_sublink, enum nvgpu_nvlink_sublink_mode mode); } link_mode_transitions; - int (*interface_init)(struct gk20a *g); - int (*interface_disable)(struct gk20a *g); int (*reg_init)(struct gk20a *g); - int (*shutdown)(struct gk20a *g); - int (*early_init)(struct gk20a *g); struct { u32 (*base_addr)(struct gk20a *g); bool (*is_running)(struct gk20a *g); diff --git a/drivers/gpu/nvgpu/include/nvgpu/nvlink.h b/drivers/gpu/nvgpu/include/nvgpu/nvlink.h index ab639fe9d..53889f236 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/nvlink.h +++ b/drivers/gpu/nvgpu/include/nvgpu/nvlink.h @@ -47,8 +47,8 @@ struct gk20a; struct nvgpu_nvlink_ioctrl_list { bool valid; u32 pri_base_addr; - u8 intr_enum; - u8 reset_enum; + u32 intr_enum; + u32 reset_enum; }; struct nvgpu_nvlink_device_list { @@ -175,7 +175,7 @@ struct nvgpu_nvlink_dev { /* priv struct */ void *priv; }; - +int nvgpu_nvlink_init(struct gk20a *g); int nvgpu_nvlink_early_init(struct gk20a *g); int nvgpu_nvlink_link_early_init(struct gk20a *g); int nvgpu_nvlink_interface_init(struct gk20a *g);