diff --git a/drivers/gpu/nvgpu/Makefile b/drivers/gpu/nvgpu/Makefile index d70c44e8a..626fea248 100644 --- a/drivers/gpu/nvgpu/Makefile +++ b/drivers/gpu/nvgpu/Makefile @@ -55,6 +55,7 @@ nvgpu-y := \ common/linux/ce2.o \ common/linux/sim.o \ common/linux/os_sched.o \ + common/linux/nvlink.o \ common/mm/nvgpu_allocator.o \ common/mm/bitmap_allocator.o \ common/mm/buddy_allocator.o \ @@ -291,7 +292,6 @@ nvgpu-$(CONFIG_TEGRA_GR_VIRTUALIZATION) += \ common/linux/vgpu/gp10b/vgpu_fuse_gp10b.o \ common/linux/vgpu/gp10b/vgpu_mm_gp10b.o - nvgpu-$(CONFIG_TEGRA_GR_VIRTUALIZATION) += \ common/linux/vgpu/gv11b/platform_gv11b_vgpu_tegra.o \ common/linux/vgpu/gv11b/vgpu_gv11b.o \ diff --git a/drivers/gpu/nvgpu/common/falcon/falcon.c b/drivers/gpu/nvgpu/common/falcon/falcon.c index 42b33c270..41e394f9b 100644 --- a/drivers/gpu/nvgpu/common/falcon/falcon.c +++ b/drivers/gpu/nvgpu/common/falcon/falcon.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -385,6 +385,10 @@ void nvgpu_flcn_sw_init(struct gk20a *g, u32 flcn_id) flcn = &g->nvdec_flcn; flcn->flcn_id = flcn_id; break; + case FALCON_ID_MINION: + flcn = &g->minion_flcn; + flcn->flcn_id = flcn_id; + break; default: nvgpu_err(g, "Invalid/Unsupported falcon ID %x", flcn_id); break; diff --git a/drivers/gpu/nvgpu/common/linux/nvlink.c b/drivers/gpu/nvgpu/common/linux/nvlink.c new file mode 100644 index 000000000..eea6eda77 --- /dev/null +++ b/drivers/gpu/nvgpu/common/linux/nvlink.c @@ -0,0 +1,531 @@ +/* + * Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include "module.h" + +#ifdef CONFIG_TEGRA_NVLINK +#include +#endif + +#ifdef CONFIG_TEGRA_NVLINK + +/* + * WAR: use this function to find detault link, as only one is supported + * on the library for now + * Returns NVLINK_MAX_LINKS_SW on failure + */ +static u32 __nvgpu_nvlink_get_link(struct nvlink_device *ndev) +{ + u32 link_id; + struct gk20a *g = (struct gk20a *) ndev->priv; + + if (!g) + return NVLINK_MAX_LINKS_SW; + + /* Lets find the detected link */ + if (g->nvlink.initialized_links) + link_id = fls(g->nvlink.initialized_links); + else + return NVLINK_MAX_LINKS_SW; + + if (g->nvlink.links[link_id].remote_info.is_connected) + return link_id; + + return NVLINK_MAX_LINKS_SW; +} +static int nvgpu_nvlink_early_init(struct nvlink_device *ndev) +{ + struct gk20a *g = (struct gk20a *) ndev->priv; + int err; + + /* For now master topology is the only one supported */ + if (!ndev->is_master) { + nvgpu_log(g, gpu_dbg_info | gpu_dbg_nvlink, + "dGPU is not master of Nvlink link"); + return -EINVAL; + } + + err = g->ops.nvlink.early_init(g); + return err; +} + +static int nvgpu_nvlink_link_early_init(struct nvlink_device *ndev) +{ + struct gk20a *g = (struct gk20a *) ndev->priv; + int err; + u32 link_id; + + /* + * First check the topology and setup connectivity + * HACK: we are only enabling one link for now!!! + */ + link_id = fls(g->nvlink.discovered_links); + g->nvlink.links[link_id].remote_info.is_connected = true; + + err = g->ops.nvlink.link_early_init(g, BIT(link_id)); + + if (err == 0) { + g->nvlink.links[link_id].priv = (void *) &(ndev->link); + ndev->link.priv = (void *) g; + } + return err; +} + +static int nvgpu_nvlink_interface_init(struct nvlink_device *ndev) +{ + int err; + struct gk20a *g = (struct gk20a *) ndev->priv; + + err = g->ops.nvlink.interface_init(g); + return err; +} + +static int nvgpu_nvlink_shutdown(struct nvlink_device *ndev) +{ + int err; + struct gk20a *g = (struct gk20a *) ndev->priv; + + err = g->ops.nvlink.shutdown(g); + return 0; +} + +static int nvgpu_nvlink_reg_init(struct nvlink_device *ndev) +{ + struct gk20a *g = (struct gk20a *) ndev->priv; + int err; + + err = g->ops.nvlink.reg_init(g); + + return err; +} + +static u32 nvgpu_nvlink_get_link_mode(struct nvlink_device *ndev) +{ + struct gk20a *g = (struct gk20a *) ndev->priv; + u32 link_id; + u32 mode; + + link_id = __nvgpu_nvlink_get_link(ndev); + if (link_id == NVLINK_MAX_LINKS_SW) + return -EINVAL; + + mode = g->ops.nvlink.link_get_mode(g, link_id); + + switch (mode) { + case nvgpu_nvlink_link_off: + return NVLINK_LINK_OFF; + case nvgpu_nvlink_link_hs: + return NVLINK_LINK_HS; + case nvgpu_nvlink_link_safe: + return NVLINK_LINK_SAFE; + case nvgpu_nvlink_link_fault: + return NVLINK_LINK_FAULT; + case nvgpu_nvlink_link_recovery: + return NVLINK_LINK_RECOVERY; + case nvgpu_nvlink_link_detect: + return NVLINK_LINK_DETECT; + case nvgpu_nvlink_link_reset: + return NVLINK_LINK_RESET; + case nvgpu_nvlink_link_enable_pm: + return NVLINK_LINK_ENABLE_PM; + case nvgpu_nvlink_link_disable_pm: + return NVLINK_LINK_DISABLE_PM; + case nvgpu_nvlink_link_disable_err_detect: + return NVLINK_LINK_DISABLE_ERR_DETECT; + case nvgpu_nvlink_link_lane_disable: + return NVLINK_LINK_LANE_DISABLE; + case nvgpu_nvlink_link_lane_shutdown: + return NVLINK_LINK_LANE_SHUTDOWN; + default: + nvgpu_log(g, gpu_dbg_info | gpu_dbg_nvlink, + "unsupported mode %u", mode); + } + + return NVLINK_LINK_OFF; +} + +static u32 nvgpu_nvlink_get_link_state(struct nvlink_device *ndev) +{ + struct gk20a *g = (struct gk20a *) ndev->priv; + u32 link_id; + + link_id = __nvgpu_nvlink_get_link(ndev); + if (link_id == NVLINK_MAX_LINKS_SW) + return -EINVAL; + + return g->ops.nvlink.link_get_state(g, link_id); +} + +static int nvgpu_nvlink_set_link_mode(struct nvlink_device *ndev, u32 mode) +{ + + struct gk20a *g = (struct gk20a *) ndev->priv; + u32 link_id; + u32 mode_sw; + + link_id = __nvgpu_nvlink_get_link(ndev); + if (link_id == NVLINK_MAX_LINKS_SW) + return -EINVAL; + + switch (mode) { + case NVLINK_LINK_OFF: + mode_sw = nvgpu_nvlink_link_off; + break; + case NVLINK_LINK_HS: + mode_sw = nvgpu_nvlink_link_hs; + break; + case NVLINK_LINK_SAFE: + mode_sw = nvgpu_nvlink_link_safe; + break; + case NVLINK_LINK_FAULT: + mode_sw = nvgpu_nvlink_link_fault; + break; + case NVLINK_LINK_RECOVERY: + mode_sw = nvgpu_nvlink_link_recovery; + break; + case NVLINK_LINK_DETECT: + mode_sw = nvgpu_nvlink_link_detect; + break; + case NVLINK_LINK_RESET: + mode_sw = nvgpu_nvlink_link_reset; + break; + case NVLINK_LINK_ENABLE_PM: + mode_sw = nvgpu_nvlink_link_enable_pm; + break; + case NVLINK_LINK_DISABLE_PM: + mode_sw = nvgpu_nvlink_link_disable_pm; + break; + case NVLINK_LINK_LANE_DISABLE: + mode_sw = nvgpu_nvlink_link_lane_disable; + break; + case NVLINK_LINK_LANE_SHUTDOWN: + mode_sw = nvgpu_nvlink_link_lane_shutdown; + break; + default: + mode_sw = nvgpu_nvlink_link_off; + } + + return g->ops.nvlink.link_set_mode(g, link_id, mode_sw); +} + +static void nvgpu_nvlink_get_tx_sublink_state(struct nvlink_device *ndev, + u32 *state) +{ + struct gk20a *g = (struct gk20a *) ndev->priv; + u32 link_id; + + link_id = __nvgpu_nvlink_get_link(ndev); + if (link_id == NVLINK_MAX_LINKS_SW) + return; + if (state) + *state = g->ops.nvlink.get_tx_sublink_state(g, link_id); +} + +static void nvgpu_nvlink_get_rx_sublink_state(struct nvlink_device *ndev, + u32 *state) +{ + struct gk20a *g = (struct gk20a *) ndev->priv; + u32 link_id; + + link_id = __nvgpu_nvlink_get_link(ndev); + if (link_id == NVLINK_MAX_LINKS_SW) + return; + if (state) + *state = g->ops.nvlink.get_rx_sublink_state(g, link_id); +} + +static u32 nvgpu_nvlink_get_sublink_mode(struct nvlink_device *ndev, + bool is_rx_sublink) +{ + struct gk20a *g = (struct gk20a *) ndev->priv; + u32 link_id; + u32 mode; + + link_id = __nvgpu_nvlink_get_link(ndev); + if (link_id == NVLINK_MAX_LINKS_SW) + return -EINVAL; + + mode = g->ops.nvlink.get_sublink_mode(g, link_id, is_rx_sublink); + switch (mode) { + case nvgpu_nvlink_sublink_tx_hs: + return NVLINK_TX_HS; + case nvgpu_nvlink_sublink_tx_off: + return NVLINK_TX_OFF; + case nvgpu_nvlink_sublink_tx_single_lane: + return NVLINK_TX_SINGLE_LANE; + case nvgpu_nvlink_sublink_tx_safe: + return NVLINK_TX_SAFE; + case nvgpu_nvlink_sublink_tx_enable_pm: + return NVLINK_TX_ENABLE_PM; + case nvgpu_nvlink_sublink_tx_disable_pm: + return NVLINK_TX_DISABLE_PM; + case nvgpu_nvlink_sublink_tx_common: + return NVLINK_TX_COMMON; + case nvgpu_nvlink_sublink_tx_common_disable: + return NVLINK_TX_COMMON_DISABLE; + case nvgpu_nvlink_sublink_tx_data_ready: + return NVLINK_TX_DATA_READY; + case nvgpu_nvlink_sublink_tx_prbs_en: + return NVLINK_TX_PRBS_EN; + case nvgpu_nvlink_sublink_rx_hs: + return NVLINK_RX_HS; + case nvgpu_nvlink_sublink_rx_enable_pm: + return NVLINK_RX_ENABLE_PM; + case nvgpu_nvlink_sublink_rx_disable_pm: + return NVLINK_RX_DISABLE_PM; + case nvgpu_nvlink_sublink_rx_single_lane: + return NVLINK_RX_SINGLE_LANE; + case nvgpu_nvlink_sublink_rx_safe: + return NVLINK_RX_SAFE; + case nvgpu_nvlink_sublink_rx_off: + return NVLINK_RX_OFF; + case nvgpu_nvlink_sublink_rx_rxcal: + return NVLINK_RX_RXCAL; + default: + nvgpu_log(g, gpu_dbg_nvlink, "Unsupported mode: %u", mode); + break; + } + + if (is_rx_sublink) + return NVLINK_RX_OFF; + return NVLINK_TX_OFF; +} + +static int nvgpu_nvlink_set_sublink_mode(struct nvlink_device *ndev, + bool is_rx_sublink, u32 mode) +{ + struct gk20a *g = (struct gk20a *) ndev->priv; + u32 link_id; + u32 mode_sw; + + link_id = __nvgpu_nvlink_get_link(ndev); + if (link_id == NVLINK_MAX_LINKS_SW) + return -EINVAL; + + if (!is_rx_sublink) { + switch (mode) { + case NVLINK_TX_HS: + mode_sw = nvgpu_nvlink_sublink_tx_hs; + break; + case NVLINK_TX_ENABLE_PM: + mode_sw = nvgpu_nvlink_sublink_tx_enable_pm; + break; + case NVLINK_TX_DISABLE_PM: + mode_sw = nvgpu_nvlink_sublink_tx_disable_pm; + break; + case NVLINK_TX_SINGLE_LANE: + mode_sw = nvgpu_nvlink_sublink_tx_single_lane; + break; + case NVLINK_TX_SAFE: + mode_sw = nvgpu_nvlink_sublink_tx_safe; + break; + case NVLINK_TX_OFF: + mode_sw = nvgpu_nvlink_sublink_tx_off; + break; + case NVLINK_TX_COMMON: + mode_sw = nvgpu_nvlink_sublink_tx_common; + break; + case NVLINK_TX_COMMON_DISABLE: + mode_sw = nvgpu_nvlink_sublink_tx_common_disable; + break; + case NVLINK_TX_DATA_READY: + mode_sw = nvgpu_nvlink_sublink_tx_data_ready; + break; + case NVLINK_TX_PRBS_EN: + mode_sw = nvgpu_nvlink_sublink_tx_prbs_en; + break; + default: + return -EINVAL; + } + } else { + switch (mode) { + case NVLINK_RX_HS: + mode_sw = nvgpu_nvlink_sublink_rx_hs; + break; + case NVLINK_RX_ENABLE_PM: + mode_sw = nvgpu_nvlink_sublink_rx_enable_pm; + break; + case NVLINK_RX_DISABLE_PM: + mode_sw = nvgpu_nvlink_sublink_rx_disable_pm; + break; + case NVLINK_RX_SINGLE_LANE: + mode_sw = nvgpu_nvlink_sublink_rx_single_lane; + break; + case NVLINK_RX_SAFE: + mode_sw = nvgpu_nvlink_sublink_rx_safe; + break; + case NVLINK_RX_OFF: + mode_sw = nvgpu_nvlink_sublink_rx_off; + break; + case NVLINK_RX_RXCAL: + mode_sw = nvgpu_nvlink_sublink_rx_rxcal; + break; + default: + return -EINVAL; + } + } + + return g->ops.nvlink.set_sublink_mode(g, link_id, is_rx_sublink, + mode_sw); +} +#endif + +u32 nvgpu_nvlink_enumerate(struct gk20a *g) +{ + u32 err = -ENODEV; +#ifdef CONFIG_TEGRA_NVLINK + struct nvlink_device *ndev; + + ndev = (struct nvlink_device *) g->nvlink.priv; + + if (ndev) + err = nvlink_enumerate(ndev); +#endif + return err; +} + +u32 nvgpu_nvlink_train(struct gk20a *g, u32 link_id, bool from_off) +{ + u32 err = -ENODEV; +#ifdef CONFIG_TEGRA_NVLINK + struct nvlink_device *ndev; + + ndev = (struct nvlink_device *) g->nvlink.priv; + + if (!ndev) + return -ENODEV; + + /* Check if the link is connected */ + if (!g->nvlink.links[link_id].remote_info.is_connected) + return -ENODEV; + + if (from_off) + return nvlink_transition_intranode_conn_off_to_safe(ndev); + return nvlink_train_intranode_conn_safe_to_hs(ndev); + +#endif + return err; +} + +u32 nvgpu_nvlink_probe(struct gk20a *g) +{ +#ifdef CONFIG_TEGRA_NVLINK + u32 err = 0; + struct device_node *np = nvgpu_get_node(g); + struct device_node *nvlink_np = NULL, *endp_np = NULL; + struct nvlink_device *ndev; + u32 phys_link_id; + + /* Parse DT */ + if (np) { + nvlink_np = of_get_child_by_name(np, "nvidia,nvlink"); + if (nvlink_np) + endp_np = of_get_child_by_name(np, "endpoint"); + } + + if (!endp_np) { + nvgpu_log(g, gpu_dbg_info | gpu_dbg_nvlink, + "No Nvlink DT detected"); + return -ENODEV; + } + + /* Allocating structures */ + ndev = nvgpu_kzalloc(g, sizeof(struct nvlink_device)); + if (!ndev) { + nvgpu_err(g, "OOM while allocating nvlink device struct"); + return -ENOMEM; + } + + ndev->priv = (void *) g; + g->nvlink.priv = (void *) ndev; + + /* Parse DT structure to detect endpoint topology */ + of_property_read_u32(endp_np, "local_dev_id", &ndev->device_id); + of_property_read_u32(endp_np, "local_link_id", &ndev->link.link_id); + ndev->is_master = of_property_read_bool(endp_np, "is_master"); + of_property_read_u32(endp_np, "remote_dev_id", + &ndev->link.remote_dev_info.device_id); + of_property_read_u32(endp_np, "remote_link_id", + &ndev->link.remote_dev_info.link_id); + of_property_read_u32(endp_np, "physical_link", + &phys_link_id); + + g->nvlink.topology_connected_links = BIT(phys_link_id); + + mutex_init(&ndev->init_state_mutex); + + /* Check that we are in dGPU mode */ + if (ndev->device_id != NVLINK_ENDPT_GV100) { + nvgpu_err(g, "Local nvlink device is not dGPU"); + err = -EINVAL; + goto free_nvlink; + } + + err = nvlink_set_init_state(ndev, NVLINK_DEV_OFF); + if (err) { + nvgpu_err(g, "Error initalizing device state to OFF"); + goto free_nvlink; + } + + /* Fill in device struct */ + ndev->dev_ops.dev_early_init = nvgpu_nvlink_early_init; + ndev->dev_ops.dev_interface_init = nvgpu_nvlink_interface_init; + ndev->dev_ops.dev_reg_init = nvgpu_nvlink_reg_init; + ndev->dev_ops.dev_shutdown = nvgpu_nvlink_shutdown; + + /* Fill in the link struct */ + ndev->link.device_id = ndev->device_id; + ndev->link.mode = NVLINK_LINK_OFF; + ndev->link.link_ops.get_link_mode = nvgpu_nvlink_get_link_mode; + ndev->link.link_ops.set_link_mode = nvgpu_nvlink_set_link_mode; + ndev->link.link_ops.get_sublink_mode = nvgpu_nvlink_get_sublink_mode; + ndev->link.link_ops.set_sublink_mode = nvgpu_nvlink_set_sublink_mode; + ndev->link.link_ops.get_link_state = nvgpu_nvlink_get_link_state; + ndev->link.link_ops.get_tx_sublink_state = + nvgpu_nvlink_get_tx_sublink_state; + ndev->link.link_ops.get_rx_sublink_state = + nvgpu_nvlink_get_rx_sublink_state; + ndev->link.link_ops.link_early_init = + nvgpu_nvlink_link_early_init; + + /* Register device with core driver*/ + err = nvlink_register_device(ndev); + if (err) { + nvgpu_err(g, "failed on nvlink device registration"); + goto free_nvlink; + } + + /* Register link with core driver */ + err = nvlink_register_link(&ndev->link); + if (err) { + nvgpu_err(g, "failed on nvlink link registration"); + goto free_nvlink; + } + + /* Enable NVLINK support */ + __nvgpu_set_enabled(g, NVGPU_SUPPORT_NVLINK, true); +free_nvlink: + nvgpu_kfree(g, ndev); + return err; + +#else + return -ENODEV; +#endif +} + diff --git a/drivers/gpu/nvgpu/common/linux/pci.c b/drivers/gpu/nvgpu/common/linux/pci.c index 9c18fbc9f..905a3d394 100644 --- a/drivers/gpu/nvgpu/common/linux/pci.c +++ b/drivers/gpu/nvgpu/common/linux/pci.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -629,6 +630,18 @@ static int nvgpu_pci_probe(struct pci_dev *pdev, return err; } + err = nvgpu_nvlink_probe(g); + /* + * ENODEV is a legal error which means there is no NVLINK + * any other error is fatal + */ + if (err) { + if (err != -ENODEV) { + nvgpu_err(g, "fatal error probing nvlink, bailing out"); + return err; + } + } + g->mm.has_physical_mode = false; np = nvgpu_get_node(g); diff --git a/drivers/gpu/nvgpu/common/vbios/bios.c b/drivers/gpu/nvgpu/common/vbios/bios.c index fa700a667..52c0a7982 100644 --- a/drivers/gpu/nvgpu/common/vbios/bios.c +++ b/drivers/gpu/nvgpu/common/vbios/bios.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2017, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2015-2018, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -74,6 +74,22 @@ struct bit { #define TOKEN_ID_VIRT_PTRS 0x56 #define TOKEN_ID_MEMORY_PTRS 0x4D +#define NVLINK_CONFIG_DATA_HDR_VER_10 0x1 +#define NVLINK_CONFIG_DATA_HDR_10_SIZE 16 +#define NVLINK_CONFIG_DATA_HDR_11_SIZE 17 +#define NVLINK_CONFIG_DATA_HDR_12_SIZE 21 + +struct nvlink_config_data_hdr_v1 { + u8 version; + u8 hdr_size; + u16 rsvd0; + u32 link_disable_mask; + u32 link_mode_mask; + u32 link_refclk_mask; + u8 train_at_boot; + u32 ac_coupling_mask; +} __packed; + #define MEMORY_PTRS_V1 1 #define MEMORY_PTRS_V2 2 @@ -369,6 +385,41 @@ static void nvgpu_bios_parse_nvinit_ptrs(struct gk20a *g, int offset) g->bios.bootscripts = &g->bios.data[nvinit_ptrs.bootscripts_ptr]; g->bios.bootscripts_size = nvinit_ptrs.bootscripts_size; g->bios.condition_table_ptr = nvinit_ptrs.condition_table_ptr; + g->bios.nvlink_config_data_offset = nvinit_ptrs.nvlink_config_data_ptr; +} + +u32 nvgpu_bios_get_nvlink_config_data(struct gk20a *g) +{ + struct nvlink_config_data_hdr_v1 config; + + if (g->bios.nvlink_config_data_offset == 0) + return -EINVAL; + + memcpy(&config, &g->bios.data[g->bios.nvlink_config_data_offset], + sizeof(config)); + + if (config.version != NVLINK_CONFIG_DATA_HDR_VER_10) { + nvgpu_err(g, "unsupported nvlink bios version: 0x%x", + config.version); + return -EINVAL; + } + + switch (config.hdr_size) { + case NVLINK_CONFIG_DATA_HDR_12_SIZE: + g->nvlink.ac_coupling_mask = config.ac_coupling_mask; + case NVLINK_CONFIG_DATA_HDR_11_SIZE: + g->nvlink.train_at_boot = config.train_at_boot; + case NVLINK_CONFIG_DATA_HDR_10_SIZE: + g->nvlink.link_disable_mask = config.link_disable_mask; + g->nvlink.link_mode_mask = config.link_mode_mask; + g->nvlink.link_refclk_mask = config.link_refclk_mask; + break; + default: + nvgpu_err(g, "invalid nvlink bios config size"); + return -EINVAL; + } + + return 0; } static void nvgpu_bios_parse_memory_ptrs(struct gk20a *g, int offset, u8 version) diff --git a/drivers/gpu/nvgpu/gk20a/gk20a.c b/drivers/gpu/nvgpu/gk20a/gk20a.c index b4886e314..868792c0b 100644 --- a/drivers/gpu/nvgpu/gk20a/gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/gk20a.c @@ -186,6 +186,18 @@ int gk20a_finalize_poweron(struct gk20a *g) } } + if (nvgpu_is_enabled(g, NVGPU_SUPPORT_NVLINK)) { + if (g->ops.nvlink.init) { + err = g->ops.nvlink.init(g); + if (err) { + nvgpu_err(g, "failed to init nvlink"); + __nvgpu_set_enabled(g, NVGPU_SUPPORT_NVLINK, + false); + } + } else + __nvgpu_set_enabled(g, NVGPU_SUPPORT_NVLINK, false); + } + if (g->ops.fb.mem_unlock) { err = g->ops.fb.mem_unlock(g); if (err) { diff --git a/drivers/gpu/nvgpu/gk20a/gk20a.h b/drivers/gpu/nvgpu/gk20a/gk20a.h index cc62865c3..f187f730c 100644 --- a/drivers/gpu/nvgpu/gk20a/gk20a.h +++ b/drivers/gpu/nvgpu/gk20a/gk20a.h @@ -62,6 +62,7 @@ struct nvgpu_ctxsw_trace_filter; #include #include #include +#include #include "clk_gk20a.h" #include "ce2_gk20a.h" @@ -961,6 +962,7 @@ struct gpu_ops { bool enable, bool is_stalling, u32 unit); void (*isr_stall)(struct gk20a *g); bool (*is_intr_hub_pending)(struct gk20a *g, u32 mc_intr); + bool (*is_intr_nvlink_pending)(struct gk20a *g, u32 mc_intr); u32 (*intr_stall)(struct gk20a *g); void (*intr_stall_pause)(struct gk20a *g); void (*intr_stall_resume)(struct gk20a *g); @@ -1057,7 +1059,27 @@ struct gpu_ops { struct { int (*check_priv_security)(struct gk20a *g); } fuse; - + struct { + u32 (*init)(struct gk20a *g); + u32 (*discover_ioctrl)(struct gk20a *g); + u32 (*discover_link)(struct gk20a *g); + u32 (*isr)(struct gk20a *g); + /* API */ + int (*link_early_init)(struct gk20a *g, unsigned long mask); + u32 (*link_get_mode)(struct gk20a *g, u32 link_id); + u32 (*link_get_state)(struct gk20a *g, u32 link_id); + int (*link_set_mode)(struct gk20a *g, u32 link_id, u32 mode); + u32 (*get_sublink_mode)(struct gk20a *g, u32 link_id, + bool is_rx_sublink); + u32 (*get_rx_sublink_state)(struct gk20a *g, u32 link_id); + u32 (*get_tx_sublink_state)(struct gk20a *g, u32 link_id); + int (*set_sublink_mode)(struct gk20a *g, u32 link_id, + bool is_rx_sublink, u32 mode); + int (*interface_init)(struct gk20a *g); + int (*reg_init)(struct gk20a *g); + int (*shutdown)(struct gk20a *g); + int (*early_init)(struct gk20a *g); + } nvlink; }; struct nvgpu_bios_ucode { @@ -1100,6 +1122,8 @@ struct nvgpu_bios { struct bit_token *clock_token; struct bit_token *virt_token; u32 expansion_rom_offset; + + u32 nvlink_config_data_offset; }; struct nvgpu_gpu_params { @@ -1153,8 +1177,10 @@ struct gk20a { struct nvgpu_falcon fecs_flcn; struct nvgpu_falcon gpccs_flcn; struct nvgpu_falcon nvdec_flcn; + struct nvgpu_falcon minion_flcn; struct clk_gk20a clk; struct fifo_gk20a fifo; + struct nvgpu_nvlink_dev nvlink; struct gr_gk20a gr; struct sim_gk20a *sim; struct mm_gk20a mm; diff --git a/drivers/gpu/nvgpu/gp10b/mc_gp10b.c b/drivers/gpu/nvgpu/gp10b/mc_gp10b.c index 9aea76f9a..dde128544 100644 --- a/drivers/gpu/nvgpu/gp10b/mc_gp10b.c +++ b/drivers/gpu/nvgpu/gp10b/mc_gp10b.c @@ -1,7 +1,7 @@ /* * GP10B master * - * Copyright (c) 2014-2017, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2014-2018, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -122,6 +122,9 @@ void mc_gp10b_isr_stall(struct gk20a *g) g->ops.ltc.isr(g); if (mc_intr_0 & mc_intr_pbus_pending_f()) g->ops.bus.isr(g); + if (g->ops.mc.is_intr_nvlink_pending && + g->ops.mc.is_intr_nvlink_pending(g, mc_intr_0)) + g->ops.nvlink.isr(g); gk20a_dbg(gpu_dbg_intr, "stall intr done 0x%08x\n", mc_intr_0); diff --git a/drivers/gpu/nvgpu/include/nvgpu/bios.h b/drivers/gpu/nvgpu/include/nvgpu/bios.h index a4823caa1..fb0a313fe 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/bios.h +++ b/drivers/gpu/nvgpu/include/nvgpu/bios.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2017, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2014-2018, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -1051,5 +1051,5 @@ u32 nvgpu_bios_read_u32(struct gk20a *g, u32 offset); void *nvgpu_bios_get_perf_table_ptrs(struct gk20a *g, struct bit_token *ptoken, u8 table_id); int nvgpu_bios_execute_script(struct gk20a *g, u32 offset); - +u32 nvgpu_bios_get_nvlink_config_data(struct gk20a *g); #endif diff --git a/drivers/gpu/nvgpu/include/nvgpu/enabled.h b/drivers/gpu/nvgpu/include/nvgpu/enabled.h index c614ce4da..4d8dbb08f 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/enabled.h +++ b/drivers/gpu/nvgpu/include/nvgpu/enabled.h @@ -77,7 +77,8 @@ struct gk20a; #define NVGPU_SUPPORT_MAP_BUFFER_BATCH 25 /* Support DMA coherence */ #define NVGPU_DMA_COHERENT 26 - +/* Use physical scatter tables instead of IOMMU */ +#define NVGPU_MM_USE_PHYSICAL_SG 27 /* * Host flags @@ -113,6 +114,11 @@ struct gk20a; #define NVGPU_SEC_SECUREGPCCS 41 #define NVGPU_SEC_PRIVSECURITY 42 +/* + * Nvlink flags + */ + +#define NVGPU_SUPPORT_NVLINK 45 /* * PMU flags. */ diff --git a/drivers/gpu/nvgpu/include/nvgpu/falcon.h b/drivers/gpu/nvgpu/include/nvgpu/falcon.h index 1f104fa18..31587ee7b 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/falcon.h +++ b/drivers/gpu/nvgpu/include/nvgpu/falcon.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -34,6 +34,7 @@ #define FALCON_ID_GPCCS (3) #define FALCON_ID_NVDEC (4) #define FALCON_ID_SEC2 (7) +#define FALCON_ID_MINION (10) /* * Falcon Base address Defines diff --git a/drivers/gpu/nvgpu/include/nvgpu/log.h b/drivers/gpu/nvgpu/include/nvgpu/log.h index 65f861988..7d7a41e33 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/log.h +++ b/drivers/gpu/nvgpu/include/nvgpu/log.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -79,6 +79,7 @@ enum nvgpu_log_categories { gpu_dbg_dma = BIT(22), /* DMA allocation prints. */ gpu_dbg_sgl = BIT(23), /* SGL related traces. */ gpu_dbg_vidmem = BIT(24), /* VIDMEM tracing. */ + gpu_dbg_nvlink = BIT(25), /* nvlink Operation tracing. */ gpu_dbg_mem = BIT(31), /* memory accesses; very verbose. */ }; diff --git a/drivers/gpu/nvgpu/include/nvgpu/nvlink.h b/drivers/gpu/nvgpu/include/nvgpu/nvlink.h new file mode 100644 index 000000000..48851ff16 --- /dev/null +++ b/drivers/gpu/nvgpu/include/nvgpu/nvlink.h @@ -0,0 +1,206 @@ +/* + * Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef __NVGPU_NVLINK_H__ +#define __NVGPU_NVLINK_H__ + +#include + +struct gk20a; + +struct nvgpu_nvlink_ioctrl_list { + bool valid; + u32 pri_base_addr; + u8 intr_enum; + u8 reset_enum; +}; + +struct nvgpu_nvlink_device_list { + bool valid; + u8 device_type; + u8 device_id; + u8 device_version; + u32 pri_base_addr; + u8 intr_enum; + u8 reset_enum; + u8 num_tx; + u8 num_rx; + u8 pll_master; + u8 pll_master_id; +}; + +enum nvgpu_nvlink_endp { + nvgpu_nvlink_endp_gpu, + nvgpu_nvlink_endp_tegra, + nvgpu_nvlink_endp__last, +}; + +enum nvgpu_nvlink_link_mode { + nvgpu_nvlink_link_off, + nvgpu_nvlink_link_hs, + nvgpu_nvlink_link_safe, + nvgpu_nvlink_link_fault, + nvgpu_nvlink_link_recovery, + nvgpu_nvlink_link_detect, + nvgpu_nvlink_link_reset, + nvgpu_nvlink_link_enable_pm, + nvgpu_nvlink_link_disable_pm, + nvgpu_nvlink_link_disable_err_detect, + nvgpu_nvlink_link_lane_disable, + nvgpu_nvlink_link_lane_shutdown, + nvgpu_nvlink_link__last, +}; + +enum nvgpu_nvlink_sublink_mode { + nvgpu_nvlink_sublink_tx_hs, + nvgpu_nvlink_sublink_tx_enable_pm, + nvgpu_nvlink_sublink_tx_disable_pm, + nvgpu_nvlink_sublink_tx_single_lane, + nvgpu_nvlink_sublink_tx_safe, + nvgpu_nvlink_sublink_tx_off, + nvgpu_nvlink_sublink_tx_common, + nvgpu_nvlink_sublink_tx_common_disable, + nvgpu_nvlink_sublink_tx_data_ready, + nvgpu_nvlink_sublink_tx_prbs_en, + nvgpu_nvlink_sublink_tx__last, + /* RX */ + nvgpu_nvlink_sublink_rx_hs, + nvgpu_nvlink_sublink_rx_enable_pm, + nvgpu_nvlink_sublink_rx_disable_pm, + nvgpu_nvlink_sublink_rx_single_lane, + nvgpu_nvlink_sublink_rx_safe, + nvgpu_nvlink_sublink_rx_off, + nvgpu_nvlink_sublink_rx_rxcal, + nvgpu_nvlink_sublink_rx__last, +}; + +struct nvgpu_nvlink_conn_info { + enum nvgpu_nvlink_endp device_type; + u32 link_number; + bool is_connected; +}; + +struct nvgpu_nvlink_link { + bool valid; + struct gk20a *g; + u8 link_id; + + u32 dlpl_base; + u8 dlpl_version; + + u32 tl_base; + u8 tl_version; + + u32 mif_base; + u8 mif_version; + + u8 intr_enum; + u8 reset_enum; + + bool dl_init_done; + + u8 pll_master_link_id; + u8 pll_slave_link_id; + + struct nvgpu_nvlink_conn_info remote_info; + void *priv; +}; + +#define NVLINK_MAX_LINKS_SW 6 + +enum nvgpu_nvlink_speed { + nvgpu_nvlink_speed_default, + nvgpu_nvlink_speed__last, +}; + +struct nvgpu_nvlink_dev { + struct nvgpu_nvlink_ioctrl_list *ioctrl_table; + u32 io_num_entries; + + struct nvgpu_nvlink_device_list *device_table; + u32 num_devices; + + struct nvgpu_nvlink_link links[NVLINK_MAX_LINKS_SW]; + + u8 dlpl_type; + u32 dlpl_base[NVLINK_MAX_LINKS_SW]; + + u8 tl_type; + u32 tl_base[NVLINK_MAX_LINKS_SW]; + + u8 mif_type; + u32 mif_base[NVLINK_MAX_LINKS_SW]; + + u8 ipt_type; + u32 ipt_base; + u8 ipt_version; + + u8 dlpl_multicast_type; + u8 dlpl_multicast_version; + u32 dlpl_multicast_base; + + u8 tl_multicast_type; + u8 tl_multicast_version; + u32 tl_multicast_base; + + u8 mif_multicast_type; + u8 mif_multicast_version; + u32 mif_multicast_base; + + u8 ioctrl_type; + u32 ioctrl_base; + + u8 minion_type; + u32 minion_base; + u8 minion_version; + + u32 discovered_links; + + /* VBIOS settings */ + u32 link_disable_mask; + u32 link_mode_mask; + u32 link_refclk_mask; + u8 train_at_boot; + u32 ac_coupling_mask; + + u32 init_disabled_links; + u32 connected_links; + u32 initialized_links; + u32 enabled_links; + u32 topology_connected_links; + + enum nvgpu_nvlink_speed speed; + + /* hshub settings */ + u32 hshub_config0; + u32 hshub_config1; + u32 hshub_config2; + u32 hshub_config6; + /* priv struct */ + void *priv; +}; + + +u32 nvgpu_nvlink_enumerate(struct gk20a *g); +u32 nvgpu_nvlink_train(struct gk20a *g, u32 link_id, bool from_off); +u32 nvgpu_nvlink_probe(struct gk20a *g); +#endif