diff --git a/drivers/gpu/nvgpu/Makefile b/drivers/gpu/nvgpu/Makefile index 3a9bf7042..8d142566f 100644 --- a/drivers/gpu/nvgpu/Makefile +++ b/drivers/gpu/nvgpu/Makefile @@ -178,6 +178,7 @@ nvgpu-y += \ os/linux/sim.o \ os/linux/sim_pci.o \ os/linux/os_sched.o \ + os/linux/nvlink_probe.o \ os/linux/nvlink.o \ os/linux/dt.o \ os/linux/ecc_sysfs.o \ @@ -328,6 +329,7 @@ nvgpu-y += \ common/ecc.o \ common/ce2.o \ common/debugger.o \ + common/nvlink/probe.o \ common/nvlink/nvlink.o \ common/nvlink/nvlink_gv100.o \ common/nvlink/nvlink_tu104.o \ diff --git a/drivers/gpu/nvgpu/Makefile.sources b/drivers/gpu/nvgpu/Makefile.sources index 07ce0c0c8..ba1994016 100644 --- a/drivers/gpu/nvgpu/Makefile.sources +++ b/drivers/gpu/nvgpu/Makefile.sources @@ -265,6 +265,7 @@ srcs += common/sim.c \ gv100/bios_gv100.c \ gv100/fifo_gv100.c \ gv100/gr_gv100.c \ + common/nvlink/probe.c \ common/nvlink/nvlink_gv100.c \ common/nvlink/nvlink_tu104.c \ gv100/hal_gv100.c \ diff --git a/drivers/gpu/nvgpu/common/nvlink/nvlink.c b/drivers/gpu/nvgpu/common/nvlink/nvlink.c index 5beeb2e2f..354b2c16d 100644 --- a/drivers/gpu/nvgpu/common/nvlink/nvlink.c +++ b/drivers/gpu/nvgpu/common/nvlink/nvlink.c @@ -22,6 +22,7 @@ #include #include +#include #include #include @@ -209,53 +210,6 @@ u32 nvgpu_nvlink_minion_extract_word(struct nvgpu_firmware *fw, u32 idx) #endif -int nvgpu_nvlink_probe(struct gk20a *g) -{ -#ifdef CONFIG_TEGRA_NVLINK - int err; - - err = nvgpu_nvlink_setup_ndev(g); - if (err != 0) - return err; - - err = nvgpu_nvlink_read_dt_props(g); - if (err != 0) - goto free_ndev; - - err = nvgpu_nvlink_init_ops(g); - if (err != 0) - goto free_ndev; - - /* Register device with core driver*/ - err = nvgpu_nvlink_register_device(g); - if (err != 0) { - nvgpu_err(g, "failed on nvlink device registration"); - goto free_ndev; - } - - /* Register link with core driver */ - err = nvgpu_nvlink_register_link(g); - if (err != 0) { - nvgpu_err(g, "failed on nvlink link registration"); - goto unregister_ndev; - } - - /* Enable NVLINK support */ - nvgpu_set_enabled(g, NVGPU_SUPPORT_NVLINK, true); - return 0; - -unregister_ndev: - nvgpu_nvlink_unregister_device(g); - -free_ndev: - nvgpu_kfree(g, g->nvlink.priv); - g->nvlink.priv = NULL; - return err; -#else - return -ENODEV; -#endif -} - int nvgpu_nvlink_remove(struct gk20a *g) { #ifdef CONFIG_TEGRA_NVLINK diff --git a/drivers/gpu/nvgpu/common/nvlink/probe.c b/drivers/gpu/nvgpu/common/nvlink/probe.c new file mode 100644 index 000000000..7e9d74e49 --- /dev/null +++ b/drivers/gpu/nvgpu/common/nvlink/probe.c @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include + +int nvgpu_nvlink_probe(struct gk20a *g) +{ +#ifdef CONFIG_TEGRA_NVLINK + int err; + + err = nvgpu_nvlink_setup_ndev(g); + if (err != 0) + return err; + + err = nvgpu_nvlink_read_dt_props(g); + if (err != 0) + goto free_ndev; + + err = nvgpu_nvlink_init_ops(g); + if (err != 0) + goto free_ndev; + + /* Register device with core driver*/ + err = nvgpu_nvlink_register_device(g); + if (err != 0) { + nvgpu_err(g, "failed on nvlink device registration"); + goto free_ndev; + } + + /* Register link with core driver */ + err = nvgpu_nvlink_register_link(g); + if (err != 0) { + nvgpu_err(g, "failed on nvlink link registration"); + goto unregister_ndev; + } + + /* Enable NVLINK support */ + nvgpu_set_enabled(g, NVGPU_SUPPORT_NVLINK, true); + return 0; + +unregister_ndev: + nvgpu_nvlink_unregister_device(g); + +free_ndev: + nvgpu_kfree(g, g->nvlink.priv); + g->nvlink.priv = NULL; + return err; +#else + return -ENODEV; +#endif +} + diff --git a/drivers/gpu/nvgpu/include/nvgpu/nvlink.h b/drivers/gpu/nvgpu/include/nvgpu/nvlink.h index 3d798c01a..881cffb86 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/nvlink.h +++ b/drivers/gpu/nvgpu/include/nvgpu/nvlink.h @@ -227,7 +227,6 @@ struct nvgpu_nvlink_dev { void nvgpu_nvlink_free_minion_used_mem(struct gk20a *g, struct nvgpu_firmware *nvgpu_minion_fw); u32 nvgpu_nvlink_minion_extract_word(struct nvgpu_firmware *fw, u32 idx); - int nvgpu_nvlink_speed_config(struct gk20a *g); int nvgpu_nvlink_early_init(struct gk20a *g); int nvgpu_nvlink_link_early_init(struct gk20a *g); @@ -243,18 +242,9 @@ void nvgpu_nvlink_get_rx_sublink_state(struct gk20a *g, u32 *state); u32 nvgpu_nvlink_get_sublink_mode(struct gk20a *g, bool is_rx_sublink); int nvgpu_nvlink_set_sublink_mode(struct gk20a *g, bool is_rx_sublink, u32 mode); -int nvgpu_nvlink_setup_ndev(struct gk20a *g); -int nvgpu_nvlink_init_ops(struct gk20a *g); int nvgpu_nvlink_enumerate(struct gk20a *g); int nvgpu_nvlink_train(struct gk20a *g, u32 link_id, bool from_off); -int nvgpu_nvlink_read_dt_props(struct gk20a *g); - -int nvgpu_nvlink_probe(struct gk20a *g); int nvgpu_nvlink_remove(struct gk20a *g); -int nvgpu_nvlink_register_device(struct gk20a *g); -int nvgpu_nvlink_unregister_device(struct gk20a *g); -int nvgpu_nvlink_register_link(struct gk20a *g); -int nvgpu_nvlink_unregister_link(struct gk20a *g); u32 nvgpu_nvlink_minion_load_ucode(struct gk20a *g, struct nvgpu_firmware *nvgpu_minion_fw); void nvgpu_mss_nvlink_init_credits(struct gk20a *g); diff --git a/drivers/gpu/nvgpu/include/nvgpu/nvlink_probe.h b/drivers/gpu/nvgpu/include/nvgpu/nvlink_probe.h new file mode 100644 index 000000000..a93ad980a --- /dev/null +++ b/drivers/gpu/nvgpu/include/nvgpu/nvlink_probe.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef NVGPU_NVLINK_PROBE_H +#define NVGPU_NVLINK_PROBE_H + +struct gk20a; + +int nvgpu_nvlink_setup_ndev(struct gk20a *g); +int nvgpu_nvlink_init_ops(struct gk20a *g); +int nvgpu_nvlink_read_dt_props(struct gk20a *g); +int nvgpu_nvlink_probe(struct gk20a *g); +int nvgpu_nvlink_register_device(struct gk20a *g); +int nvgpu_nvlink_unregister_device(struct gk20a *g); +int nvgpu_nvlink_register_link(struct gk20a *g); +int nvgpu_nvlink_unregister_link(struct gk20a *g); + +#endif /* NVGPU_NVLINK_PROBE_H */ diff --git a/drivers/gpu/nvgpu/os/linux/nvlink.c b/drivers/gpu/nvgpu/os/linux/nvlink.c index cfe81f535..bf230df59 100644 --- a/drivers/gpu/nvgpu/os/linux/nvlink.c +++ b/drivers/gpu/nvgpu/os/linux/nvlink.c @@ -23,457 +23,8 @@ #include #include #include -#include "module.h" #ifdef CONFIG_TEGRA_NVLINK -int nvgpu_nvlink_read_dt_props(struct gk20a *g) -{ - struct device_node *np; - struct nvlink_device *ndev = g->nvlink.priv; - u32 local_dev_id; - u32 local_link_id; - u32 remote_dev_id; - u32 remote_link_id; - bool is_master; - - /* Parse DT */ - np = nvgpu_get_node(g); - if (!np) - goto fail; - - np = of_get_child_by_name(np, "nvidia,nvlink"); - if (!np) - goto fail; - - np = of_get_child_by_name(np, "endpoint"); - if (!np) - goto fail; - - /* Parse DT structure to detect endpoint topology */ - of_property_read_u32(np, "local_dev_id", &local_dev_id); - of_property_read_u32(np, "local_link_id", &local_link_id); - of_property_read_u32(np, "remote_dev_id", &remote_dev_id); - of_property_read_u32(np, "remote_link_id", &remote_link_id); - is_master = of_property_read_bool(np, "is_master"); - - /* Check that we are in dGPU mode */ - if (local_dev_id != NVLINK_ENDPT_GV100) { - nvgpu_err(g, "Local nvlink device is not dGPU"); - return -EINVAL; - } - - ndev->is_master = is_master; - ndev->device_id = local_dev_id; - ndev->link.link_id = local_link_id; - ndev->link.remote_dev_info.device_id = remote_dev_id; - ndev->link.remote_dev_info.link_id = remote_link_id; - - return 0; - -fail: - nvgpu_info(g, "nvlink endpoint not found or invaling in DT"); - return -ENODEV; -} - -static int nvgpu_nvlink_ops_speed_config(struct nvlink_device *ndev) -{ - struct gk20a *g = (struct gk20a *) ndev->priv; - int err; - - err = nvgpu_nvlink_speed_config(g); - if (err != 0) { - nvgpu_err(g, "Nvlink speed config failed.\n"); - } else { - ndev->speed = g->nvlink.speed; - nvgpu_log(g, gpu_dbg_nvlink, "Nvlink default speed set to %d\n", - ndev->speed); - } - - return err; -} - -static int nvgpu_nvlink_ops_early_init(struct nvlink_device *ndev) -{ - struct gk20a *g = (struct gk20a *) ndev->priv; - - return nvgpu_nvlink_early_init(g); -} - -static int nvgpu_nvlink_ops_link_early_init(struct nvlink_device *ndev) -{ - struct gk20a *g = (struct gk20a *) ndev->priv; - - return nvgpu_nvlink_link_early_init(g); -} - -static int nvgpu_nvlink_ops_interface_init(struct nvlink_device *ndev) -{ - struct gk20a *g = (struct gk20a *) ndev->priv; - - return nvgpu_nvlink_interface_init(g); -} - -static int nvgpu_nvlink_ops_interface_disable(struct nvlink_device *ndev) -{ - struct gk20a *g = (struct gk20a *) ndev->priv; - - return nvgpu_nvlink_interface_disable(g); -} - -static int nvgpu_nvlink_ops_dev_shutdown(struct nvlink_device *ndev) -{ - struct gk20a *g = (struct gk20a *) ndev->priv; - - return nvgpu_nvlink_dev_shutdown(g); -} - -static int nvgpu_nvlink_ops_reg_init(struct nvlink_device *ndev) -{ - struct gk20a *g = (struct gk20a *) ndev->priv; - - return nvgpu_nvlink_reg_init(g); -} - -static u32 nvgpu_nvlink_ops_get_link_mode(struct nvlink_device *ndev) -{ - struct gk20a *g = (struct gk20a *) ndev->priv; - u32 mode; - - mode = nvgpu_nvlink_get_link_mode(g); - - switch (mode) { - case nvgpu_nvlink_link_off: - return NVLINK_LINK_OFF; - case nvgpu_nvlink_link_hs: - return NVLINK_LINK_HS; - case nvgpu_nvlink_link_safe: - return NVLINK_LINK_SAFE; - case nvgpu_nvlink_link_fault: - return NVLINK_LINK_FAULT; - case nvgpu_nvlink_link_rcvy_ac: - return NVLINK_LINK_RCVY_AC; - case nvgpu_nvlink_link_rcvy_sw: - return NVLINK_LINK_RCVY_SW; - case nvgpu_nvlink_link_rcvy_rx: - return NVLINK_LINK_RCVY_RX; - case nvgpu_nvlink_link_detect: - return NVLINK_LINK_DETECT; - case nvgpu_nvlink_link_reset: - return NVLINK_LINK_RESET; - case nvgpu_nvlink_link_enable_pm: - return NVLINK_LINK_ENABLE_PM; - case nvgpu_nvlink_link_disable_pm: - return NVLINK_LINK_DISABLE_PM; - case nvgpu_nvlink_link_disable_err_detect: - return NVLINK_LINK_DISABLE_ERR_DETECT; - case nvgpu_nvlink_link_lane_disable: - return NVLINK_LINK_LANE_DISABLE; - case nvgpu_nvlink_link_lane_shutdown: - return NVLINK_LINK_LANE_SHUTDOWN; - default: - nvgpu_log(g, gpu_dbg_info | gpu_dbg_nvlink, - "unsupported mode %u", mode); - } - - return NVLINK_LINK_OFF; -} - -static u32 nvgpu_nvlink_ops_get_link_state(struct nvlink_device *ndev) -{ - struct gk20a *g = (struct gk20a *) ndev->priv; - - return nvgpu_nvlink_get_link_state(g); -} - -static int nvgpu_nvlink_ops_set_link_mode(struct nvlink_device *ndev, u32 mode) -{ - struct gk20a *g = (struct gk20a *) ndev->priv; - u32 mode_sw; - - switch (mode) { - case NVLINK_LINK_OFF: - mode_sw = nvgpu_nvlink_link_off; - break; - case NVLINK_LINK_HS: - mode_sw = nvgpu_nvlink_link_hs; - break; - case NVLINK_LINK_SAFE: - mode_sw = nvgpu_nvlink_link_safe; - break; - case NVLINK_LINK_FAULT: - mode_sw = nvgpu_nvlink_link_fault; - break; - case NVLINK_LINK_RCVY_AC: - mode_sw = nvgpu_nvlink_link_rcvy_ac; - break; - case NVLINK_LINK_RCVY_SW: - mode_sw = nvgpu_nvlink_link_rcvy_sw; - break; - case NVLINK_LINK_RCVY_RX: - mode_sw = nvgpu_nvlink_link_rcvy_rx; - break; - case NVLINK_LINK_DETECT: - mode_sw = nvgpu_nvlink_link_detect; - break; - case NVLINK_LINK_RESET: - mode_sw = nvgpu_nvlink_link_reset; - break; - case NVLINK_LINK_ENABLE_PM: - mode_sw = nvgpu_nvlink_link_enable_pm; - break; - case NVLINK_LINK_DISABLE_PM: - mode_sw = nvgpu_nvlink_link_disable_pm; - break; - case NVLINK_LINK_DISABLE_ERR_DETECT: - mode_sw = nvgpu_nvlink_link_disable_err_detect; - break; - case NVLINK_LINK_LANE_DISABLE: - mode_sw = nvgpu_nvlink_link_lane_disable; - break; - case NVLINK_LINK_LANE_SHUTDOWN: - mode_sw = nvgpu_nvlink_link_lane_shutdown; - break; - default: - mode_sw = nvgpu_nvlink_link_off; - } - - return nvgpu_nvlink_set_link_mode(g, mode_sw); -} - -static void nvgpu_nvlink_ops_get_tx_sublink_state(struct nvlink_device *ndev, - u32 *tx_sublink_state) -{ - struct gk20a *g = (struct gk20a *) ndev->priv; - - return nvgpu_nvlink_get_tx_sublink_state(g, tx_sublink_state); -} - -static void nvgpu_nvlink_ops_get_rx_sublink_state(struct nvlink_device *ndev, - u32 *rx_sublink_state) -{ - struct gk20a *g = (struct gk20a *) ndev->priv; - - return nvgpu_nvlink_get_rx_sublink_state(g, rx_sublink_state); -} - -static u32 nvgpu_nvlink_ops_get_sublink_mode(struct nvlink_device *ndev, - bool is_rx_sublink) -{ - struct gk20a *g = (struct gk20a *) ndev->priv; - u32 mode; - - mode = nvgpu_nvlink_get_sublink_mode(g, is_rx_sublink); - - switch (mode) { - case nvgpu_nvlink_sublink_tx_hs: - return NVLINK_TX_HS; - case nvgpu_nvlink_sublink_tx_off: - return NVLINK_TX_OFF; - case nvgpu_nvlink_sublink_tx_single_lane: - return NVLINK_TX_SINGLE_LANE; - case nvgpu_nvlink_sublink_tx_safe: - return NVLINK_TX_SAFE; - case nvgpu_nvlink_sublink_tx_enable_pm: - return NVLINK_TX_ENABLE_PM; - case nvgpu_nvlink_sublink_tx_disable_pm: - return NVLINK_TX_DISABLE_PM; - case nvgpu_nvlink_sublink_tx_common: - return NVLINK_TX_COMMON; - case nvgpu_nvlink_sublink_tx_common_disable: - return NVLINK_TX_COMMON_DISABLE; - case nvgpu_nvlink_sublink_tx_data_ready: - return NVLINK_TX_DATA_READY; - case nvgpu_nvlink_sublink_tx_prbs_en: - return NVLINK_TX_PRBS_EN; - case nvgpu_nvlink_sublink_rx_hs: - return NVLINK_RX_HS; - case nvgpu_nvlink_sublink_rx_enable_pm: - return NVLINK_RX_ENABLE_PM; - case nvgpu_nvlink_sublink_rx_disable_pm: - return NVLINK_RX_DISABLE_PM; - case nvgpu_nvlink_sublink_rx_single_lane: - return NVLINK_RX_SINGLE_LANE; - case nvgpu_nvlink_sublink_rx_safe: - return NVLINK_RX_SAFE; - case nvgpu_nvlink_sublink_rx_off: - return NVLINK_RX_OFF; - case nvgpu_nvlink_sublink_rx_rxcal: - return NVLINK_RX_RXCAL; - default: - nvgpu_log(g, gpu_dbg_nvlink, "Unsupported mode: %u", mode); - break; - } - - if (is_rx_sublink) - return NVLINK_RX_OFF; - return NVLINK_TX_OFF; -} - -static int nvgpu_nvlink_ops_set_sublink_mode(struct nvlink_device *ndev, - bool is_rx_sublink, u32 mode) -{ - struct gk20a *g = (struct gk20a *) ndev->priv; - u32 mode_sw; - - if (!is_rx_sublink) { - switch (mode) { - case NVLINK_TX_HS: - mode_sw = nvgpu_nvlink_sublink_tx_hs; - break; - case NVLINK_TX_ENABLE_PM: - mode_sw = nvgpu_nvlink_sublink_tx_enable_pm; - break; - case NVLINK_TX_DISABLE_PM: - mode_sw = nvgpu_nvlink_sublink_tx_disable_pm; - break; - case NVLINK_TX_SINGLE_LANE: - mode_sw = nvgpu_nvlink_sublink_tx_single_lane; - break; - case NVLINK_TX_SAFE: - mode_sw = nvgpu_nvlink_sublink_tx_safe; - break; - case NVLINK_TX_OFF: - mode_sw = nvgpu_nvlink_sublink_tx_off; - break; - case NVLINK_TX_COMMON: - mode_sw = nvgpu_nvlink_sublink_tx_common; - break; - case NVLINK_TX_COMMON_DISABLE: - mode_sw = nvgpu_nvlink_sublink_tx_common_disable; - break; - case NVLINK_TX_DATA_READY: - mode_sw = nvgpu_nvlink_sublink_tx_data_ready; - break; - case NVLINK_TX_PRBS_EN: - mode_sw = nvgpu_nvlink_sublink_tx_prbs_en; - break; - default: - return -EINVAL; - } - } else { - switch (mode) { - case NVLINK_RX_HS: - mode_sw = nvgpu_nvlink_sublink_rx_hs; - break; - case NVLINK_RX_ENABLE_PM: - mode_sw = nvgpu_nvlink_sublink_rx_enable_pm; - break; - case NVLINK_RX_DISABLE_PM: - mode_sw = nvgpu_nvlink_sublink_rx_disable_pm; - break; - case NVLINK_RX_SINGLE_LANE: - mode_sw = nvgpu_nvlink_sublink_rx_single_lane; - break; - case NVLINK_RX_SAFE: - mode_sw = nvgpu_nvlink_sublink_rx_safe; - break; - case NVLINK_RX_OFF: - mode_sw = nvgpu_nvlink_sublink_rx_off; - break; - case NVLINK_RX_RXCAL: - mode_sw = nvgpu_nvlink_sublink_rx_rxcal; - break; - default: - return -EINVAL; - } - } - - return nvgpu_nvlink_set_sublink_mode(g, is_rx_sublink, mode_sw); -} - -int nvgpu_nvlink_setup_ndev(struct gk20a *g) -{ - struct nvlink_device *ndev; - - /* Allocating structures */ - ndev = nvgpu_kzalloc(g, sizeof(struct nvlink_device)); - if (!ndev) { - nvgpu_err(g, "OOM while allocating nvlink device struct"); - return -ENOMEM; - } - ndev->priv = (void *) g; - g->nvlink.priv = (void *) ndev; - - return 0; -} - -int nvgpu_nvlink_init_ops(struct gk20a *g) -{ - struct nvlink_device *ndev = (struct nvlink_device *) g->nvlink.priv; - - if (!ndev) - return -EINVAL; - - /* Fill in device struct */ - ndev->dev_ops.dev_early_init = nvgpu_nvlink_ops_early_init; - ndev->dev_ops.dev_interface_init = nvgpu_nvlink_ops_interface_init; - ndev->dev_ops.dev_reg_init = nvgpu_nvlink_ops_reg_init; - ndev->dev_ops.dev_interface_disable = - nvgpu_nvlink_ops_interface_disable; - ndev->dev_ops.dev_shutdown = nvgpu_nvlink_ops_dev_shutdown; - ndev->dev_ops.dev_speed_config = nvgpu_nvlink_ops_speed_config; - - /* Fill in the link struct */ - ndev->link.device_id = ndev->device_id; - ndev->link.mode = NVLINK_LINK_OFF; - ndev->link.is_sl_supported = false; - ndev->link.link_ops.get_link_mode = nvgpu_nvlink_ops_get_link_mode; - ndev->link.link_ops.set_link_mode = nvgpu_nvlink_ops_set_link_mode; - ndev->link.link_ops.get_sublink_mode = - nvgpu_nvlink_ops_get_sublink_mode; - ndev->link.link_ops.set_sublink_mode = - nvgpu_nvlink_ops_set_sublink_mode; - ndev->link.link_ops.get_link_state = nvgpu_nvlink_ops_get_link_state; - ndev->link.link_ops.get_tx_sublink_state = - nvgpu_nvlink_ops_get_tx_sublink_state; - ndev->link.link_ops.get_rx_sublink_state = - nvgpu_nvlink_ops_get_rx_sublink_state; - ndev->link.link_ops.link_early_init = - nvgpu_nvlink_ops_link_early_init; - - return 0; -} - -int nvgpu_nvlink_register_device(struct gk20a *g) -{ - struct nvlink_device *ndev = (struct nvlink_device *) g->nvlink.priv; - - if (!ndev) - return -ENODEV; - - return nvlink_register_device(ndev); -} - -int nvgpu_nvlink_unregister_device(struct gk20a *g) -{ - struct nvlink_device *ndev = (struct nvlink_device *) g->nvlink.priv; - - if (!ndev) - return -ENODEV; - - return nvlink_unregister_device(ndev); -} - -int nvgpu_nvlink_register_link(struct gk20a *g) -{ - struct nvlink_device *ndev = (struct nvlink_device *) g->nvlink.priv; - - if (!ndev) - return -ENODEV; - - return nvlink_register_link(&ndev->link); -} - -int nvgpu_nvlink_unregister_link(struct gk20a *g) -{ - struct nvlink_device *ndev = (struct nvlink_device *) g->nvlink.priv; - - if (!ndev) - return -ENODEV; - - return nvlink_unregister_link(&ndev->link); -} - int nvgpu_nvlink_enumerate(struct gk20a *g) { struct nvlink_device *ndev = (struct nvlink_device *) g->nvlink.priv; @@ -699,7 +250,6 @@ u32 nvgpu_nvlink_minion_load_ucode(struct gk20a *g, return err; } - #endif /* CONFIG_TEGRA_NVLINK */ void nvgpu_mss_nvlink_init_credits(struct gk20a *g) diff --git a/drivers/gpu/nvgpu/os/linux/nvlink_probe.c b/drivers/gpu/nvgpu/os/linux/nvlink_probe.c new file mode 100644 index 000000000..c02b6207e --- /dev/null +++ b/drivers/gpu/nvgpu/os/linux/nvlink_probe.c @@ -0,0 +1,478 @@ +/* + * Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include +#ifdef CONFIG_TEGRA_NVLINK +#include +#endif + +#include +#include +#include +#include "module.h" +#include + +#ifdef CONFIG_TEGRA_NVLINK +int nvgpu_nvlink_read_dt_props(struct gk20a *g) +{ + struct device_node *np; + struct nvlink_device *ndev = g->nvlink.priv; + u32 local_dev_id; + u32 local_link_id; + u32 remote_dev_id; + u32 remote_link_id; + bool is_master; + + /* Parse DT */ + np = nvgpu_get_node(g); + if (!np) + goto fail; + + np = of_get_child_by_name(np, "nvidia,nvlink"); + if (!np) + goto fail; + + np = of_get_child_by_name(np, "endpoint"); + if (!np) + goto fail; + + /* Parse DT structure to detect endpoint topology */ + of_property_read_u32(np, "local_dev_id", &local_dev_id); + of_property_read_u32(np, "local_link_id", &local_link_id); + of_property_read_u32(np, "remote_dev_id", &remote_dev_id); + of_property_read_u32(np, "remote_link_id", &remote_link_id); + is_master = of_property_read_bool(np, "is_master"); + + /* Check that we are in dGPU mode */ + if (local_dev_id != NVLINK_ENDPT_GV100) { + nvgpu_err(g, "Local nvlink device is not dGPU"); + return -EINVAL; + } + + ndev->is_master = is_master; + ndev->device_id = local_dev_id; + ndev->link.link_id = local_link_id; + ndev->link.remote_dev_info.device_id = remote_dev_id; + ndev->link.remote_dev_info.link_id = remote_link_id; + + return 0; + +fail: + nvgpu_info(g, "nvlink endpoint not found or invaling in DT"); + return -ENODEV; +} + +static int nvgpu_nvlink_ops_speed_config(struct nvlink_device *ndev) +{ + struct gk20a *g = (struct gk20a *) ndev->priv; + int err; + + err = nvgpu_nvlink_speed_config(g); + if (err != 0) { + nvgpu_err(g, "Nvlink speed config failed.\n"); + } else { + ndev->speed = g->nvlink.speed; + nvgpu_log(g, gpu_dbg_nvlink, "Nvlink default speed set to %d\n", + ndev->speed); + } + + return err; +} + +static int nvgpu_nvlink_ops_early_init(struct nvlink_device *ndev) +{ + struct gk20a *g = (struct gk20a *) ndev->priv; + + return nvgpu_nvlink_early_init(g); +} + +static int nvgpu_nvlink_ops_link_early_init(struct nvlink_device *ndev) +{ + struct gk20a *g = (struct gk20a *) ndev->priv; + + return nvgpu_nvlink_link_early_init(g); +} + +static int nvgpu_nvlink_ops_interface_init(struct nvlink_device *ndev) +{ + struct gk20a *g = (struct gk20a *) ndev->priv; + + return nvgpu_nvlink_interface_init(g); +} + +static int nvgpu_nvlink_ops_interface_disable(struct nvlink_device *ndev) +{ + struct gk20a *g = (struct gk20a *) ndev->priv; + + return nvgpu_nvlink_interface_disable(g); +} + +static int nvgpu_nvlink_ops_dev_shutdown(struct nvlink_device *ndev) +{ + struct gk20a *g = (struct gk20a *) ndev->priv; + + return nvgpu_nvlink_dev_shutdown(g); +} + +static int nvgpu_nvlink_ops_reg_init(struct nvlink_device *ndev) +{ + struct gk20a *g = (struct gk20a *) ndev->priv; + + return nvgpu_nvlink_reg_init(g); +} + +static u32 nvgpu_nvlink_ops_get_link_mode(struct nvlink_device *ndev) +{ + struct gk20a *g = (struct gk20a *) ndev->priv; + u32 mode; + + mode = nvgpu_nvlink_get_link_mode(g); + + switch (mode) { + case nvgpu_nvlink_link_off: + return NVLINK_LINK_OFF; + case nvgpu_nvlink_link_hs: + return NVLINK_LINK_HS; + case nvgpu_nvlink_link_safe: + return NVLINK_LINK_SAFE; + case nvgpu_nvlink_link_fault: + return NVLINK_LINK_FAULT; + case nvgpu_nvlink_link_rcvy_ac: + return NVLINK_LINK_RCVY_AC; + case nvgpu_nvlink_link_rcvy_sw: + return NVLINK_LINK_RCVY_SW; + case nvgpu_nvlink_link_rcvy_rx: + return NVLINK_LINK_RCVY_RX; + case nvgpu_nvlink_link_detect: + return NVLINK_LINK_DETECT; + case nvgpu_nvlink_link_reset: + return NVLINK_LINK_RESET; + case nvgpu_nvlink_link_enable_pm: + return NVLINK_LINK_ENABLE_PM; + case nvgpu_nvlink_link_disable_pm: + return NVLINK_LINK_DISABLE_PM; + case nvgpu_nvlink_link_disable_err_detect: + return NVLINK_LINK_DISABLE_ERR_DETECT; + case nvgpu_nvlink_link_lane_disable: + return NVLINK_LINK_LANE_DISABLE; + case nvgpu_nvlink_link_lane_shutdown: + return NVLINK_LINK_LANE_SHUTDOWN; + default: + nvgpu_log(g, gpu_dbg_info | gpu_dbg_nvlink, + "unsupported mode %u", mode); + } + + return NVLINK_LINK_OFF; +} + +static u32 nvgpu_nvlink_ops_get_link_state(struct nvlink_device *ndev) +{ + struct gk20a *g = (struct gk20a *) ndev->priv; + + return nvgpu_nvlink_get_link_state(g); +} + +static int nvgpu_nvlink_ops_set_link_mode(struct nvlink_device *ndev, u32 mode) +{ + struct gk20a *g = (struct gk20a *) ndev->priv; + u32 mode_sw; + + switch (mode) { + case NVLINK_LINK_OFF: + mode_sw = nvgpu_nvlink_link_off; + break; + case NVLINK_LINK_HS: + mode_sw = nvgpu_nvlink_link_hs; + break; + case NVLINK_LINK_SAFE: + mode_sw = nvgpu_nvlink_link_safe; + break; + case NVLINK_LINK_FAULT: + mode_sw = nvgpu_nvlink_link_fault; + break; + case NVLINK_LINK_RCVY_AC: + mode_sw = nvgpu_nvlink_link_rcvy_ac; + break; + case NVLINK_LINK_RCVY_SW: + mode_sw = nvgpu_nvlink_link_rcvy_sw; + break; + case NVLINK_LINK_RCVY_RX: + mode_sw = nvgpu_nvlink_link_rcvy_rx; + break; + case NVLINK_LINK_DETECT: + mode_sw = nvgpu_nvlink_link_detect; + break; + case NVLINK_LINK_RESET: + mode_sw = nvgpu_nvlink_link_reset; + break; + case NVLINK_LINK_ENABLE_PM: + mode_sw = nvgpu_nvlink_link_enable_pm; + break; + case NVLINK_LINK_DISABLE_PM: + mode_sw = nvgpu_nvlink_link_disable_pm; + break; + case NVLINK_LINK_DISABLE_ERR_DETECT: + mode_sw = nvgpu_nvlink_link_disable_err_detect; + break; + case NVLINK_LINK_LANE_DISABLE: + mode_sw = nvgpu_nvlink_link_lane_disable; + break; + case NVLINK_LINK_LANE_SHUTDOWN: + mode_sw = nvgpu_nvlink_link_lane_shutdown; + break; + default: + mode_sw = nvgpu_nvlink_link_off; + } + + return nvgpu_nvlink_set_link_mode(g, mode_sw); +} + +static void nvgpu_nvlink_ops_get_tx_sublink_state(struct nvlink_device *ndev, + u32 *tx_sublink_state) +{ + struct gk20a *g = (struct gk20a *) ndev->priv; + + return nvgpu_nvlink_get_tx_sublink_state(g, tx_sublink_state); +} + +static void nvgpu_nvlink_ops_get_rx_sublink_state(struct nvlink_device *ndev, + u32 *rx_sublink_state) +{ + struct gk20a *g = (struct gk20a *) ndev->priv; + + return nvgpu_nvlink_get_rx_sublink_state(g, rx_sublink_state); +} + +static u32 nvgpu_nvlink_ops_get_sublink_mode(struct nvlink_device *ndev, + bool is_rx_sublink) +{ + struct gk20a *g = (struct gk20a *) ndev->priv; + u32 mode; + + mode = nvgpu_nvlink_get_sublink_mode(g, is_rx_sublink); + + switch (mode) { + case nvgpu_nvlink_sublink_tx_hs: + return NVLINK_TX_HS; + case nvgpu_nvlink_sublink_tx_off: + return NVLINK_TX_OFF; + case nvgpu_nvlink_sublink_tx_single_lane: + return NVLINK_TX_SINGLE_LANE; + case nvgpu_nvlink_sublink_tx_safe: + return NVLINK_TX_SAFE; + case nvgpu_nvlink_sublink_tx_enable_pm: + return NVLINK_TX_ENABLE_PM; + case nvgpu_nvlink_sublink_tx_disable_pm: + return NVLINK_TX_DISABLE_PM; + case nvgpu_nvlink_sublink_tx_common: + return NVLINK_TX_COMMON; + case nvgpu_nvlink_sublink_tx_common_disable: + return NVLINK_TX_COMMON_DISABLE; + case nvgpu_nvlink_sublink_tx_data_ready: + return NVLINK_TX_DATA_READY; + case nvgpu_nvlink_sublink_tx_prbs_en: + return NVLINK_TX_PRBS_EN; + case nvgpu_nvlink_sublink_rx_hs: + return NVLINK_RX_HS; + case nvgpu_nvlink_sublink_rx_enable_pm: + return NVLINK_RX_ENABLE_PM; + case nvgpu_nvlink_sublink_rx_disable_pm: + return NVLINK_RX_DISABLE_PM; + case nvgpu_nvlink_sublink_rx_single_lane: + return NVLINK_RX_SINGLE_LANE; + case nvgpu_nvlink_sublink_rx_safe: + return NVLINK_RX_SAFE; + case nvgpu_nvlink_sublink_rx_off: + return NVLINK_RX_OFF; + case nvgpu_nvlink_sublink_rx_rxcal: + return NVLINK_RX_RXCAL; + default: + nvgpu_log(g, gpu_dbg_nvlink, "Unsupported mode: %u", mode); + break; + } + + if (is_rx_sublink) + return NVLINK_RX_OFF; + return NVLINK_TX_OFF; +} + +static int nvgpu_nvlink_ops_set_sublink_mode(struct nvlink_device *ndev, + bool is_rx_sublink, u32 mode) +{ + struct gk20a *g = (struct gk20a *) ndev->priv; + u32 mode_sw; + + if (!is_rx_sublink) { + switch (mode) { + case NVLINK_TX_HS: + mode_sw = nvgpu_nvlink_sublink_tx_hs; + break; + case NVLINK_TX_ENABLE_PM: + mode_sw = nvgpu_nvlink_sublink_tx_enable_pm; + break; + case NVLINK_TX_DISABLE_PM: + mode_sw = nvgpu_nvlink_sublink_tx_disable_pm; + break; + case NVLINK_TX_SINGLE_LANE: + mode_sw = nvgpu_nvlink_sublink_tx_single_lane; + break; + case NVLINK_TX_SAFE: + mode_sw = nvgpu_nvlink_sublink_tx_safe; + break; + case NVLINK_TX_OFF: + mode_sw = nvgpu_nvlink_sublink_tx_off; + break; + case NVLINK_TX_COMMON: + mode_sw = nvgpu_nvlink_sublink_tx_common; + break; + case NVLINK_TX_COMMON_DISABLE: + mode_sw = nvgpu_nvlink_sublink_tx_common_disable; + break; + case NVLINK_TX_DATA_READY: + mode_sw = nvgpu_nvlink_sublink_tx_data_ready; + break; + case NVLINK_TX_PRBS_EN: + mode_sw = nvgpu_nvlink_sublink_tx_prbs_en; + break; + default: + return -EINVAL; + } + } else { + switch (mode) { + case NVLINK_RX_HS: + mode_sw = nvgpu_nvlink_sublink_rx_hs; + break; + case NVLINK_RX_ENABLE_PM: + mode_sw = nvgpu_nvlink_sublink_rx_enable_pm; + break; + case NVLINK_RX_DISABLE_PM: + mode_sw = nvgpu_nvlink_sublink_rx_disable_pm; + break; + case NVLINK_RX_SINGLE_LANE: + mode_sw = nvgpu_nvlink_sublink_rx_single_lane; + break; + case NVLINK_RX_SAFE: + mode_sw = nvgpu_nvlink_sublink_rx_safe; + break; + case NVLINK_RX_OFF: + mode_sw = nvgpu_nvlink_sublink_rx_off; + break; + case NVLINK_RX_RXCAL: + mode_sw = nvgpu_nvlink_sublink_rx_rxcal; + break; + default: + return -EINVAL; + } + } + + return nvgpu_nvlink_set_sublink_mode(g, is_rx_sublink, mode_sw); +} + +int nvgpu_nvlink_setup_ndev(struct gk20a *g) +{ + struct nvlink_device *ndev; + + /* Allocating structures */ + ndev = nvgpu_kzalloc(g, sizeof(struct nvlink_device)); + if (!ndev) { + nvgpu_err(g, "OOM while allocating nvlink device struct"); + return -ENOMEM; + } + ndev->priv = (void *) g; + g->nvlink.priv = (void *) ndev; + + return 0; +} + +int nvgpu_nvlink_init_ops(struct gk20a *g) +{ + struct nvlink_device *ndev = (struct nvlink_device *) g->nvlink.priv; + + if (!ndev) + return -EINVAL; + + /* Fill in device struct */ + ndev->dev_ops.dev_early_init = nvgpu_nvlink_ops_early_init; + ndev->dev_ops.dev_interface_init = nvgpu_nvlink_ops_interface_init; + ndev->dev_ops.dev_reg_init = nvgpu_nvlink_ops_reg_init; + ndev->dev_ops.dev_interface_disable = + nvgpu_nvlink_ops_interface_disable; + ndev->dev_ops.dev_shutdown = nvgpu_nvlink_ops_dev_shutdown; + ndev->dev_ops.dev_speed_config = nvgpu_nvlink_ops_speed_config; + + /* Fill in the link struct */ + ndev->link.device_id = ndev->device_id; + ndev->link.mode = NVLINK_LINK_OFF; + ndev->link.is_sl_supported = false; + ndev->link.link_ops.get_link_mode = nvgpu_nvlink_ops_get_link_mode; + ndev->link.link_ops.set_link_mode = nvgpu_nvlink_ops_set_link_mode; + ndev->link.link_ops.get_sublink_mode = + nvgpu_nvlink_ops_get_sublink_mode; + ndev->link.link_ops.set_sublink_mode = + nvgpu_nvlink_ops_set_sublink_mode; + ndev->link.link_ops.get_link_state = nvgpu_nvlink_ops_get_link_state; + ndev->link.link_ops.get_tx_sublink_state = + nvgpu_nvlink_ops_get_tx_sublink_state; + ndev->link.link_ops.get_rx_sublink_state = + nvgpu_nvlink_ops_get_rx_sublink_state; + ndev->link.link_ops.link_early_init = + nvgpu_nvlink_ops_link_early_init; + + return 0; +} + +int nvgpu_nvlink_register_device(struct gk20a *g) +{ + struct nvlink_device *ndev = (struct nvlink_device *) g->nvlink.priv; + + if (!ndev) + return -ENODEV; + + return nvlink_register_device(ndev); +} + +int nvgpu_nvlink_unregister_device(struct gk20a *g) +{ + struct nvlink_device *ndev = (struct nvlink_device *) g->nvlink.priv; + + if (!ndev) + return -ENODEV; + + return nvlink_unregister_device(ndev); +} + +int nvgpu_nvlink_register_link(struct gk20a *g) +{ + struct nvlink_device *ndev = (struct nvlink_device *) g->nvlink.priv; + + if (!ndev) + return -ENODEV; + + return nvlink_register_link(&ndev->link); +} + +int nvgpu_nvlink_unregister_link(struct gk20a *g) +{ + struct nvlink_device *ndev = (struct nvlink_device *) g->nvlink.priv; + + if (!ndev) + return -ENODEV; + + return nvlink_unregister_link(&ndev->link); +} + +#endif /* CONFIG_TEGRA_NVLINK */ + diff --git a/drivers/gpu/nvgpu/os/linux/pci.c b/drivers/gpu/nvgpu/os/linux/pci.c index b663fa0bb..9e6c0491b 100644 --- a/drivers/gpu/nvgpu/os/linux/pci.c +++ b/drivers/gpu/nvgpu/os/linux/pci.c @@ -24,7 +24,7 @@ #include #include #include -#include +#include #include #include #include