diff --git a/drivers/gpu/nvgpu/Makefile b/drivers/gpu/nvgpu/Makefile index 4169bc9b5..d9d1da951 100644 --- a/drivers/gpu/nvgpu/Makefile +++ b/drivers/gpu/nvgpu/Makefile @@ -246,6 +246,9 @@ nvgpu-y += \ gv100/fifo_gv100.o \ gv100/gr_gv100.o \ gv100/regops_gv100.o \ + gv100/flcn_gv100.o \ + gv100/mc_gv100.o \ + gv100/nvlink_gv100.o \ gv100/hal_gv100.o \ gv100/pmu_gv100.o \ pstate/pstate.o \ diff --git a/drivers/gpu/nvgpu/common/linux/nvgpu_mem.c b/drivers/gpu/nvgpu/common/linux/nvgpu_mem.c index e4b4be064..206b83e15 100644 --- a/drivers/gpu/nvgpu/common/linux/nvgpu_mem.c +++ b/drivers/gpu/nvgpu/common/linux/nvgpu_mem.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, @@ -536,6 +536,8 @@ static u64 nvgpu_mem_linux_sgl_gpu_addr(struct gk20a *g, void *sgl, static bool nvgpu_mem_linux_sgt_iommuable(struct gk20a *g, struct nvgpu_sgt *sgt) { + if (nvgpu_is_enabled(g, NVGPU_MM_USE_PHYSICAL_SG)) + return false; return true; } diff --git a/drivers/gpu/nvgpu/common/linux/nvlink.c b/drivers/gpu/nvgpu/common/linux/nvlink.c index 6cf74048f..61dc8bc74 100644 --- a/drivers/gpu/nvgpu/common/linux/nvlink.c +++ b/drivers/gpu/nvgpu/common/linux/nvlink.c @@ -40,7 +40,7 @@ static u32 __nvgpu_nvlink_get_link(struct nvlink_device *ndev) /* Lets find the detected link */ if (g->nvlink.initialized_links) - link_id = fls(g->nvlink.initialized_links); + link_id = ffs(g->nvlink.initialized_links) - 1; else return NVLINK_MAX_LINKS_SW; @@ -49,6 +49,7 @@ static u32 __nvgpu_nvlink_get_link(struct nvlink_device *ndev) return NVLINK_MAX_LINKS_SW; } + static int nvgpu_nvlink_early_init(struct nvlink_device *ndev) { struct gk20a *g = (struct gk20a *) ndev->priv; @@ -75,9 +76,10 @@ static int nvgpu_nvlink_link_early_init(struct nvlink_device *ndev) * First check the topology and setup connectivity * HACK: we are only enabling one link for now!!! */ - link_id = fls(g->nvlink.discovered_links); + link_id = ffs(g->nvlink.discovered_links) - 1; g->nvlink.links[link_id].remote_info.is_connected = true; - + g->nvlink.links[link_id].remote_info.device_type = + nvgpu_nvlink_endp_tegra; err = g->ops.nvlink.link_early_init(g, BIT(link_id)); if (err == 0) { @@ -262,6 +264,7 @@ static u32 nvgpu_nvlink_get_sublink_mode(struct nvlink_device *ndev, return -EINVAL; mode = g->ops.nvlink.get_sublink_mode(g, link_id, is_rx_sublink); + switch (mode) { case nvgpu_nvlink_sublink_tx_hs: return NVLINK_TX_HS; @@ -386,9 +389,9 @@ static int nvgpu_nvlink_set_sublink_mode(struct nvlink_device *ndev, } #endif -u32 nvgpu_nvlink_enumerate(struct gk20a *g) +int nvgpu_nvlink_enumerate(struct gk20a *g) { - u32 err = -ENODEV; + int err = -ENODEV; #ifdef CONFIG_TEGRA_NVLINK struct nvlink_device *ndev; @@ -400,9 +403,9 @@ u32 nvgpu_nvlink_enumerate(struct gk20a *g) return err; } -u32 nvgpu_nvlink_train(struct gk20a *g, u32 link_id, bool from_off) +int nvgpu_nvlink_train(struct gk20a *g, u32 link_id, bool from_off) { - u32 err = -ENODEV; + int err = -ENODEV; #ifdef CONFIG_TEGRA_NVLINK struct nvlink_device *ndev; @@ -423,10 +426,10 @@ u32 nvgpu_nvlink_train(struct gk20a *g, u32 link_id, bool from_off) return err; } -u32 nvgpu_nvlink_probe(struct gk20a *g) +int nvgpu_nvlink_probe(struct gk20a *g) { #ifdef CONFIG_TEGRA_NVLINK - u32 err = 0; + int err = 0; struct device_node *np = nvgpu_get_node(g); struct device_node *nvlink_np = NULL, *endp_np = NULL; struct nvlink_device *ndev; @@ -435,13 +438,13 @@ u32 nvgpu_nvlink_probe(struct gk20a *g) /* Parse DT */ if (np) { nvlink_np = of_get_child_by_name(np, "nvidia,nvlink"); - if (nvlink_np) - endp_np = of_get_child_by_name(np, "endpoint"); + if (nvlink_np) { + endp_np = of_get_child_by_name(nvlink_np, "endpoint"); + } } if (!endp_np) { - nvgpu_log(g, gpu_dbg_info | gpu_dbg_nvlink, - "No Nvlink DT detected"); + nvgpu_info(g, "no nvlink DT detected"); return -ENODEV; } @@ -466,13 +469,13 @@ u32 nvgpu_nvlink_probe(struct gk20a *g) of_property_read_u32(endp_np, "physical_link", &phys_link_id); - g->nvlink.topology_connected_links = BIT(phys_link_id); + g->nvlink.connected_links = BIT(phys_link_id); /* Check that we are in dGPU mode */ if (ndev->device_id != NVLINK_ENDPT_GV100) { nvgpu_err(g, "Local nvlink device is not dGPU"); err = -EINVAL; - goto free_nvlink; + goto free_ndev; } /* Fill in device struct */ @@ -500,22 +503,27 @@ u32 nvgpu_nvlink_probe(struct gk20a *g) err = nvlink_register_device(ndev); if (err) { nvgpu_err(g, "failed on nvlink device registration"); - goto free_nvlink; + goto free_ndev; } /* Register link with core driver */ err = nvlink_register_link(&ndev->link); if (err) { nvgpu_err(g, "failed on nvlink link registration"); - goto free_nvlink; + goto unregister_ndev; } /* Enable NVLINK support */ __nvgpu_set_enabled(g, NVGPU_SUPPORT_NVLINK, true); -free_nvlink: - nvgpu_kfree(g, ndev); - return err; + return 0; +unregister_ndev: + nvlink_unregister_device(ndev); + +free_ndev: + nvgpu_kfree(g, ndev); + g->nvlink.priv = NULL; + return err; #else return -ENODEV; #endif diff --git a/drivers/gpu/nvgpu/gk20a/gk20a.h b/drivers/gpu/nvgpu/gk20a/gk20a.h index 02c7d0d9d..40656eddd 100644 --- a/drivers/gpu/nvgpu/gk20a/gk20a.h +++ b/drivers/gpu/nvgpu/gk20a/gk20a.h @@ -967,6 +967,8 @@ struct gpu_ops { void (*isr_stall)(struct gk20a *g); bool (*is_intr_hub_pending)(struct gk20a *g, u32 mc_intr); bool (*is_intr_nvlink_pending)(struct gk20a *g, u32 mc_intr); + bool (*is_stall_and_eng_intr_pending)(struct gk20a *g, + u32 act_eng_id); u32 (*intr_stall)(struct gk20a *g); void (*intr_stall_pause)(struct gk20a *g); void (*intr_stall_resume)(struct gk20a *g); @@ -1064,10 +1066,10 @@ struct gpu_ops { int (*check_priv_security)(struct gk20a *g); } fuse; struct { - u32 (*init)(struct gk20a *g); - u32 (*discover_ioctrl)(struct gk20a *g); - u32 (*discover_link)(struct gk20a *g); - u32 (*isr)(struct gk20a *g); + int (*init)(struct gk20a *g); + int (*discover_ioctrl)(struct gk20a *g); + int (*discover_link)(struct gk20a *g); + int (*isr)(struct gk20a *g); /* API */ int (*link_early_init)(struct gk20a *g, unsigned long mask); u32 (*link_get_mode)(struct gk20a *g, u32 link_id); diff --git a/drivers/gpu/nvgpu/gv100/flcn_gv100.c b/drivers/gpu/nvgpu/gv100/flcn_gv100.c new file mode 100644 index 000000000..5820d6c9a --- /dev/null +++ b/drivers/gpu/nvgpu/gv100/flcn_gv100.c @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ +#include +#include + +#include "gk20a/gk20a.h" +#include "gk20a/flcn_gk20a.h" +#include "gp106/flcn_gp106.h" +#include "gv100/flcn_gv100.h" + +#include + +void gv100_falcon_hal_sw_init(struct nvgpu_falcon *flcn) +{ + struct gk20a *g = flcn->g; + + switch (flcn->flcn_id) { + case FALCON_ID_MINION: + flcn->flcn_base = g->nvlink.minion_base; + flcn->is_falcon_supported = true; + flcn->is_interrupt_enabled = true; + break; + default: + break; + } + + if (flcn->is_falcon_supported) { + nvgpu_mutex_init(&flcn->copy_lock); + gk20a_falcon_ops(flcn); + } else { + /* + * Fall back + */ + gp106_falcon_hal_sw_init(flcn); + } +} diff --git a/drivers/gpu/nvgpu/gv100/flcn_gv100.h b/drivers/gpu/nvgpu/gv100/flcn_gv100.h new file mode 100644 index 000000000..869e74c8b --- /dev/null +++ b/drivers/gpu/nvgpu/gv100/flcn_gv100.h @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ +#ifndef __FLCN_GV100_H__ +#define __FLCN_GV100_H__ + +void gv100_falcon_hal_sw_init(struct nvgpu_falcon *flcn); + +#endif /* __FLCN_GV100_H__ */ diff --git a/drivers/gpu/nvgpu/gv100/hal_gv100.c b/drivers/gpu/nvgpu/gv100/hal_gv100.c index 6f4ab8759..53d61bfb0 100644 --- a/drivers/gpu/nvgpu/gv100/hal_gv100.c +++ b/drivers/gpu/nvgpu/gv100/hal_gv100.c @@ -51,20 +51,19 @@ #include "gm20b/pmu_gm20b.h" #include "gm20b/acr_gm20b.h" -#include "gp10b/fb_gp10b.h" -#include "gp10b/gr_gp10b.h" - #include "gp106/clk_gp106.h" #include "gp106/clk_arb_gp106.h" #include "gp106/pmu_gp106.h" #include "gp106/acr_gp106.h" #include "gp106/sec2_gp106.h" #include "gp106/bios_gp106.h" -#include "gv100/bios_gv100.h" #include "gp106/therm_gp106.h" #include "gp106/xve_gp106.h" #include "gp106/clk_gp106.h" #include "gp106/flcn_gp106.h" + +#include "gp10b/fb_gp10b.h" +#include "gp10b/gr_gp10b.h" #include "gp10b/ltc_gp10b.h" #include "gp10b/therm_gp10b.h" #include "gp10b/mc_gp10b.h" @@ -78,32 +77,33 @@ #include "gv11b/css_gr_gv11b.h" #include "gv11b/dbg_gpu_gv11b.h" #include "gv11b/hal_gv11b.h" -#include "gv100/gr_gv100.h" #include "gv11b/gr_gv11b.h" #include "gv11b/mc_gv11b.h" #include "gv11b/ltc_gv11b.h" #include "gv11b/gv11b.h" #include "gv11b/ce_gv11b.h" -#include "gv100/gr_ctx_gv100.h" #include "gv11b/mm_gv11b.h" #include "gv11b/pmu_gv11b.h" #include "gv11b/fb_gv11b.h" -#include "gv100/mm_gv100.h" #include "gv11b/pmu_gv11b.h" -#include "gv100/fb_gv100.h" -#include "gv100/fifo_gv100.h" #include "gv11b/fifo_gv11b.h" #include "gv11b/regops_gv11b.h" - #include "gv11b/gv11b_gating_reglist.h" -#include "gv100/regops_gv100.h" #include "gv11b/subctx_gv11b.h" #include "gv100.h" #include "hal_gv100.h" +#include "gv100/bios_gv100.h" #include "gv100/fb_gv100.h" +#include "gv100/fifo_gv100.h" +#include "gv100/flcn_gv100.h" +#include "gv100/gr_ctx_gv100.h" +#include "gv100/gr_gv100.h" +#include "gv100/mc_gv100.h" #include "gv100/mm_gv100.h" #include "gv100/pmu_gv100.h" +#include "gv100/nvlink_gv100.h" +#include "gv100/regops_gv100.h" #include #include @@ -651,7 +651,7 @@ static const struct gpu_ops gv100_ops = { .apply_smpc_war = gv100_apply_smpc_war, }, .mc = { - .intr_enable = mc_gv11b_intr_enable, + .intr_enable = mc_gv100_intr_enable, .intr_unit_config = mc_gp10b_intr_unit_config, .isr_stall = mc_gp10b_isr_stall, .intr_stall = mc_gp10b_intr_stall, @@ -666,6 +666,9 @@ static const struct gpu_ops gv100_ops = { .boot_0 = gk20a_mc_boot_0, .is_intr1_pending = mc_gp10b_is_intr1_pending, .is_intr_hub_pending = gv11b_mc_is_intr_hub_pending, + .is_intr_nvlink_pending = gv100_mc_is_intr_nvlink_pending, + .is_stall_and_eng_intr_pending = + gv100_mc_is_stall_and_eng_intr_pending, }, .debug = { .show_dump = gk20a_debug_show_dump, @@ -712,11 +715,30 @@ static const struct gpu_ops gv100_ops = { .disable_shadow_rom = xve_disable_shadow_rom_gp106, }, .falcon = { - .falcon_hal_sw_init = gp106_falcon_hal_sw_init, + .falcon_hal_sw_init = gv100_falcon_hal_sw_init, }, .priv_ring = { .isr = gp10b_priv_ring_isr, }, + .nvlink = { + .discover_ioctrl = gv100_nvlink_discover_ioctrl, + .discover_link = gv100_nvlink_discover_link, + .init = gv100_nvlink_init, + .isr = gv100_nvlink_isr, + /* API */ + .link_early_init = gv100_nvlink_link_early_init, + .link_get_state = gv100_nvlink_link_get_state, + .link_set_mode = gv100_nvlink_link_set_mode, + .link_get_mode = gv100_nvlink_link_get_mode, + .get_sublink_mode = gv100_nvlink_link_get_sublink_mode, + .get_tx_sublink_state = gv100_nvlink_link_get_tx_sublink_state, + .get_rx_sublink_state = gv100_nvlink_link_get_rx_sublink_state, + .set_sublink_mode = gv100_nvlink_link_set_sublink_mode, + .interface_init = gv100_nvlink_interface_init, + .reg_init = gv100_nvlink_reg_init, + .shutdown = gv100_nvlink_shutdown, + .early_init = gv100_nvlink_early_init, + }, .chip_init_gpu_characteristics = gv100_init_gpu_characteristics, .get_litter_value = gv100_get_litter_value, }; @@ -751,6 +773,7 @@ int gv100_init_hal(struct gk20a *g) gops->xve = gv100_ops.xve; gops->falcon = gv100_ops.falcon; gops->priv_ring = gv100_ops.priv_ring; + gops->nvlink = gv100_ops.nvlink; /* clocks */ gops->clk.init_clk_support = gv100_ops.clk.init_clk_support; diff --git a/drivers/gpu/nvgpu/gv100/mc_gv100.c b/drivers/gpu/nvgpu/gv100/mc_gv100.c new file mode 100644 index 000000000..a3f8ab065 --- /dev/null +++ b/drivers/gpu/nvgpu/gv100/mc_gv100.c @@ -0,0 +1,94 @@ +/* + * GV100 master + * + * Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include + +#include "gk20a/gk20a.h" + +#include "gp10b/mc_gp10b.h" + +#include "mc_gv100.h" +#include "gv11b/fb_gv11b.h" + +#include + +void mc_gv100_intr_enable(struct gk20a *g) +{ + u32 eng_intr_mask = gk20a_fifo_engine_interrupt_mask(g); + + gk20a_writel(g, mc_intr_en_clear_r(NVGPU_MC_INTR_STALLING), + 0xffffffff); + gk20a_writel(g, mc_intr_en_clear_r(NVGPU_MC_INTR_NONSTALLING), + 0xffffffff); + gv11b_fb_disable_hub_intr(g, STALL_REG_INDEX, HUB_INTR_TYPE_ALL); + + g->mc_intr_mask_restore[NVGPU_MC_INTR_STALLING] = + mc_intr_pfifo_pending_f() | + mc_intr_hub_pending_f() | + mc_intr_priv_ring_pending_f() | + mc_intr_pbus_pending_f() | + mc_intr_ltc_pending_f() | + mc_intr_nvlink_pending_f() | + eng_intr_mask; + + g->mc_intr_mask_restore[NVGPU_MC_INTR_NONSTALLING] = + mc_intr_pfifo_pending_f() + | eng_intr_mask; + + /* TODO: Enable PRI faults for HUB ECC err intr */ + gv11b_fb_enable_hub_intr(g, STALL_REG_INDEX, g->mm.hub_intr_types); + + gk20a_writel(g, mc_intr_en_set_r(NVGPU_MC_INTR_STALLING), + g->mc_intr_mask_restore[NVGPU_MC_INTR_STALLING]); + + gk20a_writel(g, mc_intr_en_set_r(NVGPU_MC_INTR_NONSTALLING), + g->mc_intr_mask_restore[NVGPU_MC_INTR_NONSTALLING]); + +} + +bool gv100_mc_is_intr_nvlink_pending(struct gk20a *g, u32 mc_intr_0) +{ + return ((mc_intr_0 & mc_intr_nvlink_pending_f()) ? true : false); +} + +bool gv100_mc_is_stall_and_eng_intr_pending(struct gk20a *g, u32 act_eng_id) +{ + u32 mc_intr_0 = gk20a_readl(g, mc_intr_r(0)); + u32 stall_intr, eng_intr_mask; + + eng_intr_mask = gk20a_fifo_act_eng_interrupt_mask(g, act_eng_id); + if (mc_intr_0 & eng_intr_mask) + return true; + + stall_intr = mc_intr_pfifo_pending_f() | + mc_intr_hub_pending_f() | + mc_intr_priv_ring_pending_f() | + mc_intr_pbus_pending_f() | + mc_intr_ltc_pending_f() | + mc_intr_nvlink_pending_f(); + if (mc_intr_0 & stall_intr) + return true; + + return false; +} diff --git a/drivers/gpu/nvgpu/gv100/mc_gv100.h b/drivers/gpu/nvgpu/gv100/mc_gv100.h new file mode 100644 index 000000000..4aff4a366 --- /dev/null +++ b/drivers/gpu/nvgpu/gv100/mc_gv100.h @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef MC_GV100_H +#define MC_GV100_H +struct gk20a; + +void mc_gv100_intr_enable(struct gk20a *g); +bool gv100_mc_is_intr_nvlink_pending(struct gk20a *g, u32 mc_intr_0); +bool gv100_mc_is_stall_and_eng_intr_pending(struct gk20a *g, u32 act_eng_id); +#endif diff --git a/drivers/gpu/nvgpu/gv100/nvlink_gv100.c b/drivers/gpu/nvgpu/gv100/nvlink_gv100.c new file mode 100644 index 000000000..e5e65c51e --- /dev/null +++ b/drivers/gpu/nvgpu/gv100/nvlink_gv100.c @@ -0,0 +1,2654 @@ +/* + * Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include + +#include "gk20a/gk20a.h" +#include "nvlink_gv100.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#define NVLINK_PLL_ON_TIMEOUT_MS 30 +#define NVLINK_SUBLINK_TIMEOUT_MS 200 +/* + * The manuals are missing some useful defines + * we add them for now + */ + +#define IPT_INTR_CONTROL_LINK(i) (nvlipt_intr_control_link0_r() + (i)*4) +#define IPT_ERR_UC_STATUS_LINK(i) (nvlipt_err_uc_status_link0_r() + (i)*36) +#define IPT_ERR_UC_MASK_LINK(i) (nvlipt_err_uc_mask_link0_r() + (i)*36) +#define IPT_ERR_UC_SEVERITY_LINK(i) (nvlipt_err_uc_severity_link0_r() + (i)*36) +#define IPT_ERR_UC_FIRST_LINK(i) (nvlipt_err_uc_first_link0_r() + (i)*36) +#define IPT_ERR_UC_ADVISORY_LINK(i) (nvlipt_err_uc_advisory_link0_r() + (i)*36) +#define IPT_ERR_C_STATUS_LINK(i) (nvlipt_err_c_status_link0_r() + (i)*36) +#define IPT_ERR_C_MASK_LINK(i) (nvlipt_err_c_mask_link0_r() + (i)*36) +#define IPT_ERR_C_FIRST_LINK(i) (nvlipt_err_c_first_link0_r() + (i)*36) +#define IPT_ERR_CONTROL_LINK(i) (nvlipt_err_control_link0_r() + (i)*4) + +#define IPT_ERR_UC_ACTIVE_BITS (nvlipt_err_uc_status_link0_dlprotocol_f(1) | \ + nvlipt_err_uc_status_link0_datapoisoned_f(1) | \ + nvlipt_err_uc_status_link0_flowcontrol_f(1) | \ + nvlipt_err_uc_status_link0_responsetimeout_f(1) | \ + nvlipt_err_uc_status_link0_targeterror_f(1) | \ + nvlipt_err_uc_status_link0_unexpectedresponse_f(1) | \ + nvlipt_err_uc_status_link0_receiveroverflow_f(1) | \ + nvlipt_err_uc_status_link0_malformedpacket_f(1) | \ + nvlipt_err_uc_status_link0_stompedpacketreceived_f(1) | \ + nvlipt_err_uc_status_link0_unsupportedrequest_f(1) | \ + nvlipt_err_uc_status_link0_ucinternal_f(1)) + + +#define MINION_FALCON_INTR_MASK (minion_falcon_irqmset_wdtmr_set_f() | \ + minion_falcon_irqmset_halt_set_f() | \ + minion_falcon_irqmset_exterr_set_f()| \ + minion_falcon_irqmset_swgen0_set_f()| \ + minion_falcon_irqmset_swgen1_set_f()) + +#define MINION_FALCON_INTR_DEST ( \ + minion_falcon_irqdest_host_wdtmr_host_f() | \ + minion_falcon_irqdest_host_halt_host_f() | \ + minion_falcon_irqdest_host_exterr_host_f() | \ + minion_falcon_irqdest_host_swgen0_host_f() | \ + minion_falcon_irqdest_host_swgen1_host_f() | \ + minion_falcon_irqdest_target_wdtmr_host_normal_f() | \ + minion_falcon_irqdest_target_halt_host_normal_f() | \ + minion_falcon_irqdest_target_exterr_host_normal_f() | \ + minion_falcon_irqdest_target_swgen0_host_normal_f() | \ + minion_falcon_irqdest_target_swgen1_host_normal_f()) + +/* Minion FW header format */ +union gv100_minion_hdr { + struct { + u32 os_code_offset; + u32 os_code_size; + u32 os_data_offset; + u32 os_data_size; + u32 num_apps; + u32 app_0_code_start; + }; + u8 raw_data[256]; +}; + +struct __nvlink_reginit { + u32 addr; + u32 value; +}; + +static const struct __nvlink_reginit __nvlink_reginit_per_link_tegra[] = { + /* NVTLC when connected to Tegra */ + { 0x300U, 0x00800040U }, + { 0x304U, 0x00000020U }, + { 0x308U, 0x00000020U }, + { 0x30CU, 0x00000020U }, + { 0x310U, 0x00000020U }, + { 0x314U, 0x00800040U }, + { 0x318U, 0x00000000U }, + { 0x31CU, 0x00000000U }, + { 0x200U, 0x007F003FU }, + { 0x204U, 0x007F005FU }, + { 0x208U, 0x007F007FU }, + { 0x20CU, 0x007F009FU }, + { 0x210U, 0x007F00BFU }, + { 0x214U, 0x00FF003FU }, + { 0x218U, 0x00FF003FU }, + { 0x21CU, 0x00FF003FU }, + { 0xB00U, 0x010000BEU }, + { 0xB04U, 0x00000064U }, + { 0xB08U, 0x00000000U }, + { 0xB0CU, 0x00000020U }, + { 0xB10U, 0x00000000U }, + { 0xB14U, 0x010000BEU }, + { 0xB18U, 0x00000000U }, + { 0xB1CU, 0x00000000U }, + { 0xA00U, 0x00FF00BDU }, + { 0xA04U, 0x00FF0121U }, + { 0xA08U, 0x00FF0121U }, + { 0xA0CU, 0x00FF0141U }, + { 0xA10U, 0x00FF0141U }, + { 0xA14U, 0x01FF01FFU }, + { 0xA18U, 0x01FF01FFU }, + { 0xA1CU, 0x01FF01FFU }, + { 0xF04U, 0x00FFFFFFU }, + { 0xF0CU, 0x00FFFFFFU }, + { 0xF1CU, 0x003FFFFFU }, + { 0xF24U, 0x003FFFFFU }, + { 0x704U, 0x003FFFFFU }, + { 0x70CU, 0x003FFFFFU }, + { 0x400U, 0x00000001U }, + { 0xC00U, 0x00000001U }, +}; + +static const struct __nvlink_reginit __nvlink_reginit_per_link_gpu[] = { + /* NVTLC for PEER GPU */ + { 0x300U, 0x00800040U }, + { 0x304U, 0x00000000U }, + { 0x308U, 0x00000000U }, + { 0x30CU, 0x00000000U }, + { 0x310U, 0x00000000U }, + { 0x314U, 0x00800040U }, + { 0x318U, 0x00000000U }, + { 0x31CU, 0x00000000U }, + { 0x200U, 0x007F003FU }, + { 0x204U, 0x007F003FU }, + { 0x208U, 0x007F003FU }, + { 0x20CU, 0x007F003FU }, + { 0x210U, 0x007F003FU }, + { 0x214U, 0x00FF007FU }, + { 0x218U, 0x00FF007FU }, + { 0x21CU, 0x00FF007FU }, + { 0xB00U, 0x010000C0U }, + { 0xB04U, 0x00000000U }, + { 0xB08U, 0x00000000U }, + { 0xB0CU, 0x00000000U }, + { 0xB10U, 0x00000000U }, + { 0xB14U, 0x010000C0U }, + { 0xB18U, 0x00000000U }, + { 0xB1CU, 0x00000000U }, + { 0xA00U, 0x00FF00BFU }, + { 0xA04U, 0x00FF00BFU }, + { 0xA08U, 0x00FF00BFU }, + { 0xA0CU, 0x00FF00BFU }, + { 0xA10U, 0x00FF00BFU }, + { 0xA14U, 0x01FF017FU }, + { 0xA18U, 0x01FF017FU }, + { 0xA1CU, 0x01FF017FU }, + { 0xF04U, 0x00FFFFFFU }, + { 0xF0CU, 0x00FFFFFFU }, + { 0xF1CU, 0x003FFFFFU }, + { 0xF24U, 0x003FFFFFU }, + { 0x704U, 0x003FFFFFU }, + { 0x70CU, 0x003FFFFFU }, + { 0x400U, 0x00000001U }, + { 0xC00U, 0x00000001U }, +}; + + +#define NVL_DEVICE(str) nvlinkip_discovery_common_device_##str##_v() + +const char * __gv100_device_type_to_str(u32 type) +{ + if (type == NVL_DEVICE(ioctrl)) + return "IOCTRL"; + if (type == NVL_DEVICE(dlpl)) + return "DL/PL"; + if (type == NVL_DEVICE(nvltlc)) + return "NVLTLC"; + if (type == NVL_DEVICE(ioctrlmif)) + return "IOCTRLMIF"; + if (type == NVL_DEVICE(nvlipt)) + return "NVLIPT"; + if (type == NVL_DEVICE(minion)) + return "MINION"; + if (type == NVL_DEVICE(dlpl_multicast)) + return "DL/PL MULTICAST"; + if (type == NVL_DEVICE(nvltlc_multicast)) + return "NVLTLC MULTICAST"; + if (type == NVL_DEVICE(ioctrlmif_multicast)) + return "IOCTRLMIF MULTICAST"; + if (type == NVL_DEVICE(nvltlc_multicast)) + return "NVLTLC MULTICAST"; + return "UNKNOWN"; +} + +/* + * Function prototypes + */ +static int gv100_nvlink_setup_pll(struct gk20a *g, unsigned long mask); +static u32 __gv100_nvlink_get_link_reset_mask(struct gk20a *g); +static u32 gv100_nvlink_rxcal_en(struct gk20a *g, unsigned long mask); +static u32 gv100_nvlink_minion_data_ready_en(struct gk20a *g, + unsigned long mask, bool sync); + + +/* + ******************************************************************************* + * IP specific functions * + ******************************************************************************* + */ + +/* + *-----------------------------------------------------------------------------* + * MINION API + *-----------------------------------------------------------------------------* + */ + +/* + * Initialization of link specific interrupts + */ +static void gv100_nvlink_minion_link_intr_enable(struct gk20a *g, u32 link_id, + bool enable) +{ + u32 intr, links; + + /* Only stall interrupts for now */ + intr = MINION_REG_RD32(g, minion_minion_intr_stall_en_r()); + links = minion_minion_intr_stall_en_link_v(intr); + + if (enable) + links |= BIT(link_id); + else + links &= ~BIT(link_id); + + intr = set_field(intr, minion_minion_intr_stall_en_link_m(), + minion_minion_intr_stall_en_link_f(links)); + MINION_REG_WR32(g, minion_minion_intr_stall_en_r(), intr); +} + +/* + * Initialization of falcon interrupts + */ +static void gv100_nvlink_minion_falcon_intr_enable(struct gk20a *g, bool enable) +{ + u32 reg; + + reg = MINION_REG_RD32(g, minion_minion_intr_stall_en_r()); + if (enable) { + reg = set_field(reg, minion_minion_intr_stall_en_fatal_m(), + minion_minion_intr_stall_en_fatal_enable_f()); + reg = set_field(reg, minion_minion_intr_stall_en_nonfatal_m(), + minion_minion_intr_stall_en_nonfatal_enable_f()); + reg = set_field(reg, minion_minion_intr_stall_en_falcon_stall_m(), + minion_minion_intr_stall_en_falcon_stall_enable_f()); + reg = set_field(reg, minion_minion_intr_stall_en_falcon_nostall_m(), + minion_minion_intr_stall_en_falcon_nostall_enable_f()); + } else { + reg = set_field(reg, minion_minion_intr_stall_en_fatal_m(), + minion_minion_intr_stall_en_fatal_disable_f()); + reg = set_field(reg, minion_minion_intr_stall_en_nonfatal_m(), + minion_minion_intr_stall_en_nonfatal_disable_f()); + reg = set_field(reg, minion_minion_intr_stall_en_falcon_stall_m(), + minion_minion_intr_stall_en_falcon_stall_disable_f()); + reg = set_field(reg, minion_minion_intr_stall_en_falcon_nostall_m(), + minion_minion_intr_stall_en_falcon_nostall_disable_f()); + } + + MINION_REG_WR32(g, minion_minion_intr_stall_en_r(), reg); +} + +/* + * Initialize minion IP interrupts + */ +static void gv100_nvlink_initialize_minion(struct gk20a *g) +{ + /* Disable non-stall tree */ + MINION_REG_WR32(g, minion_minion_intr_nonstall_en_r(), 0x0); + + gv100_nvlink_minion_falcon_intr_enable(g, true); +} + +/* + * Check if minion is up + */ +static bool __gv100_nvlink_minion_is_running(struct gk20a *g) +{ + + /* if minion is booted and not halted, it is running */ + if ((MINION_REG_RD32(g, minion_minion_status_r()) & + minion_minion_status_status_f(1)) && + (!minion_falcon_irqstat_halt_v( + MINION_REG_RD32(g, minion_falcon_irqstat_r())))) + return true; + + return false; +} + +/* + * Falcon specific ISR handling + */ +static bool gv100_nvlink_minion_falcon_isr(struct gk20a *g) +{ + u32 intr; + + intr = MINION_REG_RD32(g, minion_falcon_irqstat_r()) & + MINION_REG_RD32(g, minion_falcon_irqmask_r()); + + if (!intr) + return true; + + if (intr & falcon_falcon_irqstat_exterr_true_f()) { + nvgpu_err(g, "FALCON EXT ADDR: 0x%x 0x%x 0x%x", + MINION_REG_RD32(g, 0x244), + MINION_REG_RD32(g, 0x248), + MINION_REG_RD32(g, 0x24c)); + } + + MINION_REG_WR32(g, minion_falcon_irqsclr_r(), intr); + + nvgpu_err(g, "FATAL minion IRQ: 0x%08x", intr); + + + intr = MINION_REG_RD32(g, minion_falcon_irqstat_r()) & + MINION_REG_RD32(g, minion_falcon_irqmask_r()); + + return (intr == 0); +} + +/* + * Link Specific ISR + */ + +static bool gv100_nvlink_minion_link_isr(struct gk20a *g, u32 link_id) +{ + u32 intr, code; + bool fatal = false; + + intr = MINION_REG_RD32(g, minion_nvlink_link_intr_r(link_id)); + code = minion_nvlink_link_intr_code_v(intr); + + if (code == minion_nvlink_link_intr_code_swreq_v()) { + nvgpu_err(g, " Intr SWREQ, link: %d subcode: %x", + link_id, minion_nvlink_link_intr_subcode_v(intr)); + } else if (code == minion_nvlink_link_intr_code_pmdisabled_v()) { + nvgpu_err(g, " Fatal Intr PMDISABLED, link: %d subcode: %x", + link_id, minion_nvlink_link_intr_subcode_v(intr)); + fatal = true; + } else if (code == minion_nvlink_link_intr_code_na_v()) { + nvgpu_err(g, " Fatal Intr NA, link: %d subcode: %x", + link_id, minion_nvlink_link_intr_subcode_v(intr)); + fatal = true; + } else if (code == minion_nvlink_link_intr_code_dlreq_v()) { + nvgpu_err(g, " Fatal Intr DLREQ, link: %d subcode: %x", + link_id, minion_nvlink_link_intr_subcode_v(intr)); + fatal = true; + } else { + nvgpu_err(g, " Fatal Intr UNKN:%x, link: %d subcode: %x", code, + link_id, minion_nvlink_link_intr_subcode_v(intr)); + fatal = true; + } + + if (fatal) + gv100_nvlink_minion_link_intr_enable(g, link_id, false); + + intr = set_field(intr, minion_nvlink_link_intr_state_m(), + minion_nvlink_link_intr_state_f(1)); + MINION_REG_WR32(g, minion_nvlink_link_intr_r(link_id), intr); + + return true; +} + +/* + * Global minion routine to service interrupts + */ +static bool gv100_nvlink_minion_isr(struct gk20a *g) { + + u32 intr, i; + unsigned long links; + + intr = MINION_REG_RD32(g, minion_minion_intr_r()) & + MINION_REG_RD32(g, minion_minion_intr_stall_en_r()); + + if (minion_minion_intr_falcon_stall_v(intr) || + minion_minion_intr_falcon_nostall_v(intr)) + gv100_nvlink_minion_falcon_isr(g); + + if (minion_minion_intr_fatal_v(intr)) { + gv100_nvlink_minion_falcon_intr_enable(g, false); + MINION_REG_WR32(g, minion_minion_intr_r(), + minion_minion_intr_fatal_f(1)); + } + + if (minion_minion_intr_nonfatal_v(intr)) + MINION_REG_WR32(g, minion_minion_intr_r(), + minion_minion_intr_nonfatal_f(1)); + + links = minion_minion_intr_link_v(intr) & g->nvlink.enabled_links; + + if (links) + for_each_set_bit(i, &links, 32) + gv100_nvlink_minion_link_isr(g, i); + + /* Re-test interrupt status */ + intr = MINION_REG_RD32(g, minion_minion_intr_r()) & + MINION_REG_RD32(g, minion_minion_intr_stall_en_r()); + + return (intr == 0); +} + +/* + * Load minion FW and set up bootstrap + */ +static u32 gv100_nvlink_minion_load(struct gk20a *g) +{ + struct bin_hdr *hdr = NULL; + struct nvgpu_firmware *minion_fw = NULL; + union gv100_minion_hdr *minion_hdr; + u32 *minion_ucode = NULL; + u32 err = 0; + struct nvgpu_timeout timeout; + u32 delay = GR_IDLE_CHECK_DEFAULT; + u32 reg; + u32 app; + + nvgpu_log_fn(g, " "); + + if (__gv100_nvlink_minion_is_running(g)) + return 0; + + /* get mem unlock ucode binary */ + minion_fw = nvgpu_request_firmware(g, "minion.bin", 0); + if (!minion_fw) { + nvgpu_err(g, "minion ucode get fail"); + err = -ENOENT; + goto exit; + } + + /* nvdec falcon reset */ + nvgpu_flcn_reset(&g->minion_flcn); + + hdr = (struct bin_hdr *) minion_fw->data; + + minion_hdr = (union gv100_minion_hdr *) (minion_fw->data + + hdr->header_offset); + minion_ucode = (u32 *) (minion_fw->data + hdr->data_offset); + + nvgpu_log(g, gpu_dbg_nvlink, + "os_code_offset: 0x%x", minion_hdr->os_code_offset); + nvgpu_log(g, gpu_dbg_nvlink, + "os_code_size: 0x%x", minion_hdr->os_code_size); + nvgpu_log(g, gpu_dbg_nvlink, + "os_data_offset: 0x%x", minion_hdr->os_data_offset); + nvgpu_log(g, gpu_dbg_nvlink, + "os_data_size: 0x%x", minion_hdr->os_data_size); + nvgpu_log(g, gpu_dbg_nvlink, + "num_apps: 0x%x", minion_hdr->num_apps); + nvgpu_log(g, gpu_dbg_nvlink, + "app_0_code_start: 0x%x", minion_hdr->app_0_code_start); + + /* Clear interrupts */ + nvgpu_flcn_set_irq(&g->minion_flcn, true, MINION_FALCON_INTR_MASK, + MINION_FALCON_INTR_DEST); + + /* Copy Non Secure IMEM code */ + nvgpu_flcn_copy_to_imem(&g->minion_flcn, 0, + (u8 *)&minion_ucode[minion_hdr->os_code_offset >> 2], + minion_hdr->os_code_size, 0, false, + GET_IMEM_TAG(minion_hdr->os_code_offset)); + + /* Copy Non Secure DMEM code */ + nvgpu_flcn_copy_to_dmem(&g->minion_flcn, 0, + (u8 *)&minion_ucode[minion_hdr->os_data_offset >> 2], + minion_hdr->os_data_size, 0); + + /* If apps are there load them securely */ + if (minion_hdr->num_apps) { + for (app = 0; app < minion_hdr->num_apps; app++) { + u32 app_code_start, app_code_size; + u32 app_data_start, app_data_size; + + app_code_start = + *((u32 *) &minion_hdr->app_0_code_start + + 2*app); + app_code_size = + *((u32 *) &minion_hdr->app_0_code_start + + 2*app + 1); + app_data_start = + *((u32 *) &minion_hdr->app_0_code_start + + 2*minion_hdr->num_apps + 2*app); + app_data_size = + *((u32 *) &minion_hdr->app_0_code_start + + 2*minion_hdr->num_apps + 2*app + 1); + + nvgpu_log(g, gpu_dbg_nvlink, + " app %d code_offset 0x%x", app, app_code_start); + nvgpu_log(g, gpu_dbg_nvlink, + " app %d code_size 0x%x", app, app_code_size); + nvgpu_log(g, gpu_dbg_nvlink, + " app %d data_offset 0x%x", app, app_data_start); + nvgpu_log(g, gpu_dbg_nvlink, + " app %d data_size 0x%x", app, app_data_size); + + if (app_code_size) + nvgpu_flcn_copy_to_imem(&g->minion_flcn, + app_code_start, + (u8 *)&minion_ucode[ + app_code_start >> 2], + app_code_size, 0, true, + GET_IMEM_TAG(app_code_start)); + + if (app_data_size) + nvgpu_flcn_copy_to_dmem(&g->minion_flcn, + app_data_start, + (u8 *)&minion_ucode[ + app_data_start >> 2], + app_data_size, 0); + } + } + + /* set BOOTVEC to start of non-secure code */ + nvgpu_flcn_bootstrap(&g->minion_flcn, 0x0); + + nvgpu_timeout_init(g, &timeout, gk20a_get_gr_idle_timeout(g), + NVGPU_TIMER_CPU_TIMER); + + do { + reg = MINION_REG_RD32(g, minion_minion_status_r()); + + if (minion_minion_status_status_v(reg)) { + /* Minion sequence completed, check status */ + if (minion_minion_status_status_v(reg) != + minion_minion_status_status_boot_v()) { + nvgpu_err(g, "MINION init sequence failed: 0x%x", + minion_minion_status_status_v(reg)); + err = -EINVAL; + + goto exit; + } + + nvgpu_log(g, gpu_dbg_nvlink, + "MINION boot successful: 0x%x", reg); + err = 0; + break; + } + + nvgpu_usleep_range(delay, delay * 2); + delay = min_t(unsigned long, + delay << 1, GR_IDLE_CHECK_MAX); + } while (!nvgpu_timeout_expired_msg(&timeout, " minion boot timeout")); + + /* Service interrupts */ + gv100_nvlink_minion_falcon_isr(g); + + if (nvgpu_timeout_peek_expired(&timeout)) { + err = -ETIMEDOUT; + goto exit; + } + + gv100_nvlink_initialize_minion(g); + +exit: + if (minion_fw) + nvgpu_release_firmware(g, minion_fw); + + return err; +} + +/* + * Check if MINION command is complete + */ +static u32 gv100_nvlink_minion_command_complete(struct gk20a *g, u32 link_id) +{ + u32 reg; + struct nvgpu_timeout timeout; + u32 delay = GR_IDLE_CHECK_DEFAULT; + + + nvgpu_timeout_init(g, &timeout, gk20a_get_gr_idle_timeout(g), + NVGPU_TIMER_CPU_TIMER); + + do { + reg = MINION_REG_RD32(g, minion_nvlink_dl_cmd_r(link_id)); + + if (minion_nvlink_dl_cmd_ready_v(reg) == 1) { + /* Command completed, check sucess */ + if (minion_nvlink_dl_cmd_fault_v(reg) == 1) { + nvgpu_err(g, "minion cmd(%d) error: 0x%x", + link_id, reg); + + reg = minion_nvlink_dl_cmd_fault_f(1); + MINION_REG_WR32(g, + minion_nvlink_dl_cmd_r(link_id), reg); + + return -EINVAL; + } + + /* Commnand success */ + break; + } + nvgpu_usleep_range(delay, delay * 2); + delay = min_t(unsigned long, + delay << 1, GR_IDLE_CHECK_MAX); + + } while (!nvgpu_timeout_expired_msg(&timeout, " minion cmd timeout")); + + if (nvgpu_timeout_peek_expired(&timeout)) + return -ETIMEDOUT; + + nvgpu_log(g, gpu_dbg_nvlink, "minion cmd Complete"); + return 0; +} + +/* + * Send Minion command (can be async) + */ +static u32 gv100_nvlink_minion_send_command(struct gk20a *g, + u32 link_id, u32 command, u32 scratch_0, bool sync) +{ + u32 err = 0; + + /* Check last command succeded */ + err = gv100_nvlink_minion_command_complete(g, link_id); + if (err) + return -EINVAL; + + nvgpu_log(g, gpu_dbg_nvlink, + "sending MINION command 0x%x to link %d", command, link_id); + + if (command == minion_nvlink_dl_cmd_command_configeom_v()) + MINION_REG_WR32(g, minion_misc_0_r(), + minion_misc_0_scratch_swrw_0_f(scratch_0)); + + MINION_REG_WR32(g, minion_nvlink_dl_cmd_r(link_id), + minion_nvlink_dl_cmd_command_f(command) | + minion_nvlink_dl_cmd_fault_f(1)); + + if (sync) + err = gv100_nvlink_minion_command_complete(g, link_id); + + return err; +} + +/* MINION API COMMANDS */ + +/* + * Init UPHY + */ +static u32 gv100_nvlink_minion_init_uphy(struct gk20a *g, unsigned long mask, + bool sync) +{ + u32 err = 0; + u32 init_pll_cmd; + u32 link_id, master_pll, slave_pll; + u32 master_state, slave_state; + + unsigned long link_enable; + + switch(g->nvlink.speed) { + case nvgpu_nvlink_speed_default: + init_pll_cmd = minion_nvlink_dl_cmd_command_initpll_0_v(); + break; + default: + nvgpu_err(g, "Unsupported UPHY speed"); + return -EINVAL; + } + + link_enable = __gv100_nvlink_get_link_reset_mask(g); + + /* Cannot use links 0/1 without refclk buffer */ + if (mask & (BIT(1)|BIT(0))) { + nvgpu_err(g, "links 0/1 not supported on GV100"); + return -EINVAL; + } + + for_each_set_bit(link_id, &mask, 32) { + master_pll = g->nvlink.links[link_id].pll_master_link_id; + slave_pll = g->nvlink.links[link_id].pll_slave_link_id; + + master_state = nvl_link_state_state_init_v(); + slave_state = nvl_link_state_state_init_v(); + + if (BIT(master_pll) & link_enable) + master_state = nvl_link_state_state_v( + g->ops.nvlink.link_get_state(g, master_pll)); + + if (BIT(slave_pll) & link_enable) + slave_state = nvl_link_state_state_v( + g->ops.nvlink.link_get_state(g, slave_pll)); + + if ((slave_state != nvl_link_state_state_init_v()) || + (master_state != nvl_link_state_state_init_v())) { + nvgpu_err(g, "INIT PLL can only be executed when both " + "master and slave links are in init state"); + return -EINVAL; + } + + /* Check if INIT PLL is done on link */ + if (!(BIT(master_pll) & g->nvlink.init_pll_done)) { + err = gv100_nvlink_minion_send_command(g, master_pll, + init_pll_cmd, 0, sync); + if (err) { + nvgpu_err(g, " Error sending INITPLL to minion"); + return err; + } + + g->nvlink.init_pll_done |= BIT(master_pll); + } + } + + err = gv100_nvlink_setup_pll(g, mask); + if (err) { + nvgpu_err(g, "Error setting up PLL"); + return err; + } + + /* INITPHY commands */ + for_each_set_bit(link_id, &mask, 32) { + err = gv100_nvlink_minion_send_command(g, link_id, + minion_nvlink_dl_cmd_command_initphy_v(), 0, sync); + if (err) { + nvgpu_err(g, "Error on INITPHY minion DL command %u", + link_id); + return err; + } + } + + return 0; +} + +/* + * Configure AC coupling + */ +static u32 gv100_nvlink_minion_configure_ac_coupling(struct gk20a *g, + unsigned long mask, bool sync) +{ + u32 err = 0; + u32 i; + u32 temp; + + for_each_set_bit(i, &mask, 32) { + + temp = DLPL_REG_RD32(g, i, nvl_link_config_r()); + temp &= ~nvl_link_config_ac_safe_en_m(); + temp |= nvl_link_config_ac_safe_en_on_f(); + + DLPL_REG_WR32(g, i, nvl_link_config_r(), temp); + + err = gv100_nvlink_minion_send_command(g, i, + minion_nvlink_dl_cmd_command_setacmode_v(), 0, sync); + + if (err) + return err; + } + + return err; +} + +/* + * Set Data ready + */ +static u32 gv100_nvlink_minion_data_ready_en(struct gk20a *g, + unsigned long mask, bool sync) +{ + u32 err = 0; + u32 i; + + for_each_set_bit(i, &mask, 32) { + err = gv100_nvlink_minion_send_command(g, i, + minion_nvlink_dl_cmd_command_initlaneenable_v(), 0, + sync); + if (err) { + nvgpu_err(g, "Failed init lane enable on minion"); + return err; + } + } + + for_each_set_bit(i, &mask, 32) { + err = gv100_nvlink_minion_send_command(g, i, + minion_nvlink_dl_cmd_command_initdlpl_v(), 0, sync); + if (err) { + nvgpu_err(g, "Failed init DLPL on minion"); + return err; + } + } + return err; +} + +/* + * Request that minion disable the lane + */ +static u32 gv100_nvlink_minion_lane_disable(struct gk20a *g, u32 link_id, + bool sync) +{ + u32 err = 0; + + err = gv100_nvlink_minion_send_command(g, link_id, + minion_nvlink_dl_cmd_command_lanedisable_v(), 0, sync); + + if (err) + nvgpu_err(g, " failed to disable lane on %d", link_id); + + return err; +} + +/* + * Request that minion shutdown the lane + */ +static u32 gv100_nvlink_minion_lane_shutdown(struct gk20a *g, u32 link_id, + bool sync) +{ + u32 err = 0; + + err = gv100_nvlink_minion_send_command(g, link_id, + minion_nvlink_dl_cmd_command_laneshutdown_v(), 0, sync); + + if (err) + nvgpu_err(g, " failed to shutdown lane on %d", link_id); + + return err; +} + + + +/* + *-----------------------------------------------------------------------------* + * TLC API + *-----------------------------------------------------------------------------* + */ + +static int gv100_nvlink_get_tlc_reginit(enum nvgpu_nvlink_endp endp, + struct __nvlink_reginit **reg, u32 *count) +{ + switch(endp) { + case nvgpu_nvlink_endp_tegra: + *reg = (struct __nvlink_reginit *) + __nvlink_reginit_per_link_tegra; + *count = ARRAY_SIZE(__nvlink_reginit_per_link_tegra); + break; + case nvgpu_nvlink_endp_gpu: + *reg = (struct __nvlink_reginit *) + __nvlink_reginit_per_link_gpu; + *count = ARRAY_SIZE(__nvlink_reginit_per_link_gpu); + break; + default: + return -EINVAL; + } + + return 0; +} + +/* + * Init TLC IP and prod regs + */ +static void gv100_nvlink_initialize_tlc(struct gk20a *g, u32 link_id) +{ +} + +/* + * Init TLC per link interrupts + */ +static void gv100_nvlink_tlc_intr_enable(struct gk20a *g, u32 link_id, + bool enable) +{ + u32 reg_rx0 = 0, reg_rx1 = 0, reg_tx = 0; + + if (enable) { + /* Set PROD values */ + reg_rx0 = 0x0FFFFFF; + reg_rx1 = 0x03FFFFF; + reg_tx = 0x1FFFFFF; + } + + TLC_REG_WR32(g, link_id, nvtlc_rx_err_report_en_0_r(), reg_rx0); + TLC_REG_WR32(g, link_id, nvtlc_rx_err_report_en_1_r(), reg_rx1); + TLC_REG_WR32(g, link_id, nvtlc_tx_err_report_en_0_r(), reg_tx); +} + +/* + * helper function to get TLC intr status in common structure + */ +static void gv100_nvlink_tlc_get_intr(struct gk20a *g, u32 link_id) +{ + g->nvlink.tlc_rx_err_status_0[link_id] = + TLC_REG_RD32(g, link_id, nvtlc_rx_err_status_0_r()); + g->nvlink.tlc_rx_err_status_1[link_id] = + TLC_REG_RD32(g, link_id, nvtlc_rx_err_status_1_r()); + g->nvlink.tlc_tx_err_status_0[link_id] = + TLC_REG_RD32(g, link_id, nvtlc_tx_err_status_0_r()); +} + +/* + * Interrupt routine handler for TLC + */ +static void gv100_nvlink_tlc_isr(struct gk20a *g, u32 link_id) +{ + + if (g->nvlink.tlc_rx_err_status_0[link_id]) { + /* All TLC RX 0 errors are fatal. Notify and disable */ + nvgpu_err(g, "Fatal TLC RX 0 interrupt on link %d mask: %x", + link_id, g->nvlink.tlc_rx_err_status_0[link_id]); + TLC_REG_WR32(g, link_id, nvtlc_rx_err_first_0_r(), + g->nvlink.tlc_rx_err_status_0[link_id]); + TLC_REG_WR32(g, link_id, nvtlc_rx_err_status_0_r(), + g->nvlink.tlc_rx_err_status_0[link_id]); + } + if (g->nvlink.tlc_rx_err_status_1[link_id]) { + /* All TLC RX 1 errors are fatal. Notify and disable */ + nvgpu_err(g, "Fatal TLC RX 1 interrupt on link %d mask: %x", + link_id, g->nvlink.tlc_rx_err_status_1[link_id]); + TLC_REG_WR32(g, link_id, nvtlc_rx_err_first_1_r(), + g->nvlink.tlc_rx_err_status_1[link_id]); + TLC_REG_WR32(g, link_id, nvtlc_rx_err_status_1_r(), + g->nvlink.tlc_rx_err_status_1[link_id]); + } + if (g->nvlink.tlc_tx_err_status_0[link_id]) { + /* All TLC TX 0 errors are fatal. Notify and disable */ + nvgpu_err(g, "Fatal TLC TX 0 interrupt on link %d mask: %x", + link_id, g->nvlink.tlc_tx_err_status_0[link_id]); + TLC_REG_WR32(g, link_id, nvtlc_tx_err_first_0_r(), + g->nvlink.tlc_tx_err_status_0[link_id]); + TLC_REG_WR32(g, link_id, nvtlc_tx_err_status_0_r(), + g->nvlink.tlc_tx_err_status_0[link_id]); + } +} + +/* + *-----------------------------------------------------------------------------* + * DLPL API + *-----------------------------------------------------------------------------* + */ + +/* + * DLPL interrupt enable helper + */ +static void gv100_nvlink_dlpl_intr_enable(struct gk20a *g, u32 link_id, + bool enable) +{ + u32 reg = 0; + + /* Always disable nonstall tree */ + DLPL_REG_WR32(g, link_id, nvl_intr_nonstall_en_r(), 0); + + if (!enable) + { + DLPL_REG_WR32(g, link_id, nvl_intr_stall_en_r(), 0); + return; + } + + /* Clear interrupt register to get rid of stale state (W1C) */ + DLPL_REG_WR32(g, link_id, nvl_intr_r(), 0xffffffff); + DLPL_REG_WR32(g, link_id, nvl_intr_sw2_r(), 0xffffffff); + + reg = nvl_intr_stall_en_tx_recovery_long_enable_f() | + nvl_intr_stall_en_tx_fault_ram_enable_f() | + nvl_intr_stall_en_tx_fault_interface_enable_f() | + nvl_intr_stall_en_tx_fault_sublink_change_enable_f() | + nvl_intr_stall_en_rx_fault_sublink_change_enable_f() | + nvl_intr_stall_en_rx_fault_dl_protocol_enable_f() | + nvl_intr_stall_en_ltssm_fault_enable_f(); + + DLPL_REG_WR32(g, link_id, nvl_intr_stall_en_r(), reg); + + /* Configure error threshold */ + reg = DLPL_REG_RD32(g, link_id, nvl_sl1_error_rate_ctrl_r()); + reg = set_field(reg, nvl_sl1_error_rate_ctrl_short_threshold_man_m(), + nvl_sl1_error_rate_ctrl_short_threshold_man_f(0x2)); + reg = set_field(reg, nvl_sl1_error_rate_ctrl_long_threshold_man_m(), + nvl_sl1_error_rate_ctrl_long_threshold_man_f(0x2)); + DLPL_REG_WR32(g, link_id, nvl_sl1_error_rate_ctrl_r(), reg); +} + +/* + * DLPL per-link isr + */ + +#define DLPL_NON_FATAL_INTR_MASK (nvl_intr_tx_replay_f(1) | \ + nvl_intr_tx_recovery_short_f(1) | \ + nvl_intr_tx_recovery_long_f(1) | \ + nvl_intr_rx_short_error_rate_f(1) | \ + nvl_intr_rx_long_error_rate_f(1) | \ + nvl_intr_rx_ila_trigger_f(1) | \ + nvl_intr_ltssm_protocol_f(1)) + +#define DLPL_FATAL_INTR_MASK ( nvl_intr_ltssm_fault_f(1) | \ + nvl_intr_rx_fault_dl_protocol_f(1) | \ + nvl_intr_rx_fault_sublink_change_f(1) | \ + nvl_intr_tx_fault_sublink_change_f(1) | \ + nvl_intr_tx_fault_interface_f(1) | \ + nvl_intr_tx_fault_ram_f(1)) + +static void gv100_nvlink_dlpl_isr(struct gk20a *g, u32 link_id) +{ + u32 non_fatal_mask = 0; + u32 fatal_mask = 0; + u32 intr = 0; + bool retrain = false; + u32 err; + + intr = DLPL_REG_RD32(g, link_id, nvl_intr_r()) & + DLPL_REG_RD32(g, link_id, nvl_intr_stall_en_r()); + + if (!intr) + return; + + fatal_mask = intr & DLPL_FATAL_INTR_MASK; + non_fatal_mask = intr & DLPL_NON_FATAL_INTR_MASK; + + nvgpu_err(g, " handling DLPL %d isr. Fatal: %x non-Fatal: %x", + link_id, fatal_mask, non_fatal_mask); + + /* Check if we are not handling an interupt */ + if ((fatal_mask | non_fatal_mask) & ~intr) + nvgpu_err(g, "Unable to service DLPL intr on link %d", link_id); + + if (non_fatal_mask & nvl_intr_tx_recovery_long_f(1)) + retrain = true; + if (fatal_mask) + retrain = false; + + if (retrain) { + err = nvgpu_nvlink_train(g, link_id, false); + if (err) + nvgpu_err(g, "failed to retrain link %d", link_id); + } + + /* Clear interrupts */ + DLPL_REG_WR32(g, link_id, nvl_intr_r(), (non_fatal_mask | fatal_mask)); + DLPL_REG_WR32(g, link_id, nvl_intr_sw2_r(), 0xffffffff); +} + +/* + *-----------------------------------------------------------------------------* + * MIF API + *-----------------------------------------------------------------------------* + */ + +/* + * Initialize MIF API with PROD settings + */ +static void gv100_nvlink_initialize_mif(struct gk20a *g, u32 link_id) +{ + u32 tmp; + + /* Enable MIF RX error */ + + /* Containment (make fatal) */ + tmp = 0; + tmp = set_field(tmp, + ioctrlmif_rx_err_contain_en_0_rxramdataparityerr_m(), + ioctrlmif_rx_err_contain_en_0_rxramdataparityerr__prod_f()); + tmp = set_field(tmp, + ioctrlmif_rx_err_contain_en_0_rxramhdrparityerr_m(), + ioctrlmif_rx_err_contain_en_0_rxramhdrparityerr__prod_f()); + MIF_REG_WR32(g, link_id, ioctrlmif_rx_err_contain_en_0_r(), tmp); + + /* Logging (do not ignore) */ + tmp = 0; + tmp = set_field(tmp, + ioctrlmif_rx_err_log_en_0_rxramdataparityerr_m(), + ioctrlmif_rx_err_log_en_0_rxramdataparityerr_f(1)); + tmp = set_field(tmp, + ioctrlmif_rx_err_log_en_0_rxramhdrparityerr_m(), + ioctrlmif_rx_err_log_en_0_rxramhdrparityerr_f(1)); + MIF_REG_WR32(g, link_id, ioctrlmif_rx_err_log_en_0_r(), tmp); + + /* Tx Error */ + /* Containment (make fatal) */ + tmp = 0; + tmp = set_field(tmp, + ioctrlmif_tx_err_contain_en_0_txramdataparityerr_m(), + ioctrlmif_tx_err_contain_en_0_txramdataparityerr__prod_f()); + tmp = set_field(tmp, + ioctrlmif_tx_err_contain_en_0_txramhdrparityerr_m(), + ioctrlmif_tx_err_contain_en_0_txramhdrparityerr__prod_f()); + MIF_REG_WR32(g, link_id, ioctrlmif_tx_err_contain_en_0_r(), tmp); + + /* Logging (do not ignore) */ + tmp = 0; + tmp = set_field(tmp, ioctrlmif_tx_err_log_en_0_txramdataparityerr_m(), + ioctrlmif_tx_err_log_en_0_txramdataparityerr_f(1)); + tmp = set_field(tmp, ioctrlmif_tx_err_log_en_0_txramhdrparityerr_m(), + ioctrlmif_tx_err_log_en_0_txramhdrparityerr_f(1)); + MIF_REG_WR32(g, link_id, ioctrlmif_tx_err_log_en_0_r(), tmp); + + /* Credit release */ + MIF_REG_WR32(g, link_id, ioctrlmif_rx_ctrl_buffer_ready_r(), 0x1); + MIF_REG_WR32(g, link_id, ioctrlmif_tx_ctrl_buffer_ready_r(), 0x1); +} + +/* + * Enable per-link MIF interrupts + */ +static void gv100_nvlink_mif_intr_enable(struct gk20a *g, u32 link_id, + bool enable) +{ + u32 reg0 = 0, reg1 = 0; + + if (enable) { + reg0 = set_field(reg0, + ioctrlmif_rx_err_report_en_0_rxramdataparityerr_m(), + ioctrlmif_rx_err_report_en_0_rxramdataparityerr_f(1)); + reg0 = set_field(reg0, + ioctrlmif_rx_err_report_en_0_rxramhdrparityerr_m(), + ioctrlmif_rx_err_report_en_0_rxramhdrparityerr_f(1)); + reg1 = set_field(reg1, + ioctrlmif_tx_err_report_en_0_txramdataparityerr_m(), + ioctrlmif_tx_err_report_en_0_txramdataparityerr_f(1)); + reg1 = set_field(reg1, + ioctrlmif_tx_err_report_en_0_txramhdrparityerr_m(), + ioctrlmif_tx_err_report_en_0_txramhdrparityerr_f(1)); + } + + MIF_REG_WR32(g, link_id, ioctrlmif_rx_err_report_en_0_r(), reg0); + MIF_REG_WR32(g, link_id, ioctrlmif_tx_err_report_en_0_r(), reg1); +} + +/* + * Handle per-link MIF interrupts + */ +static void gv100_nvlink_mif_isr(struct gk20a *g, u32 link_id) +{ + u32 intr, fatal_mask = 0; + + /* RX Errors */ + intr = MIF_REG_RD32(g, link_id, ioctrlmif_rx_err_status_0_r()); + if (intr) { + if (intr & ioctrlmif_rx_err_status_0_rxramdataparityerr_m()) { + nvgpu_err(g, "Fatal MIF RX interrupt hit on link %d: RAM_DATA_PARITY", + link_id); + fatal_mask |= ioctrlmif_rx_err_status_0_rxramdataparityerr_f(1); + } + if (intr & ioctrlmif_rx_err_status_0_rxramhdrparityerr_m()) { + nvgpu_err(g, "Fatal MIF RX interrupt hit on link %d: RAM_HDR_PARITY", + link_id); + fatal_mask |= ioctrlmif_rx_err_status_0_rxramhdrparityerr_f(1); + } + + if (fatal_mask) { + MIF_REG_WR32(g, link_id, ioctrlmif_rx_err_first_0_r(), + fatal_mask); + MIF_REG_WR32(g, link_id, ioctrlmif_rx_err_status_0_r(), + fatal_mask); + } + } + + /* TX Errors */ + fatal_mask = 0; + intr = MIF_REG_RD32(g, link_id, ioctrlmif_tx_err_status_0_r()); + if (intr) { + if (intr & ioctrlmif_tx_err_status_0_txramdataparityerr_m()) { + nvgpu_err(g, "Fatal MIF TX interrupt hit on link %d: RAM_DATA_PARITY", + link_id); + fatal_mask |= ioctrlmif_tx_err_status_0_txramdataparityerr_f(1); + } + if (intr & ioctrlmif_tx_err_status_0_txramhdrparityerr_m()) { + nvgpu_err(g, "Fatal MIF TX interrupt hit on link %d: RAM_HDR_PARITY", + link_id); + fatal_mask |= ioctrlmif_tx_err_status_0_txramhdrparityerr_f(1); + } + + if (fatal_mask) { + MIF_REG_WR32(g, link_id, ioctrlmif_tx_err_first_0_r(), + fatal_mask); + MIF_REG_WR32(g, link_id, ioctrlmif_tx_err_status_0_r(), + fatal_mask); + } + } +} + +/* + *-----------------------------------------------------------------------------* + * NVLIPT API + *-----------------------------------------------------------------------------* + */ + +/* + * NVLIPT IP initialization (per-link) + */ +static void gv100_nvlink_initialize_nvlipt(struct gk20a *g, u32 link_id) +{ + /* init persistent scratch registers */ + IPT_REG_WR32(g, nvlipt_scratch_cold_r(), + nvlipt_scratch_cold_data_init_v()); + + /* + * AErr settings (top level) + */ + + /* UC first and status reg (W1C) need to be cleared byt arch */ + IPT_REG_WR32(g, IPT_ERR_UC_FIRST_LINK(link_id), IPT_ERR_UC_ACTIVE_BITS); + IPT_REG_WR32(g, IPT_ERR_UC_STATUS_LINK(link_id), IPT_ERR_UC_ACTIVE_BITS); + + /* AErr Severity */ + IPT_REG_WR32(g, IPT_ERR_UC_SEVERITY_LINK(link_id), IPT_ERR_UC_ACTIVE_BITS); + + /* AErr Control settings */ + IPT_REG_WR32(g, IPT_ERR_CONTROL_LINK(link_id), + nvlipt_err_control_link0_fatalenable_f(1) | + nvlipt_err_control_link0_nonfatalenable_f(1)); +} + +/* + * Enable NVLIPT interrupts + */ +static void gv100_nvlink_nvlipt_intr_enable(struct gk20a *g, u32 link_id, + bool enable) +{ + u32 val = 0; + u32 reg; + + if (enable) + val = 1; + + reg = IPT_REG_RD32(g, IPT_INTR_CONTROL_LINK(link_id)); + reg = set_field(reg, nvlipt_intr_control_link0_stallenable_m(), + nvlipt_intr_control_link0_stallenable_f(val)); + reg = set_field(reg, nvlipt_intr_control_link0_nostallenable_m(), + nvlipt_intr_control_link0_nostallenable_f(val)); + IPT_REG_WR32(g, IPT_INTR_CONTROL_LINK(link_id), reg); +} + +/* + * Per-link NVLIPT ISR handler + */ +bool gv100_nvlink_nvlipt_isr(struct gk20a *g, u32 link_id) +{ + /* + * Interrupt handling happens in leaf handlers. Assume all interrupts + * were handled and clear roll ups/ + */ + IPT_REG_WR32(g, IPT_ERR_UC_FIRST_LINK(link_id), IPT_ERR_UC_ACTIVE_BITS); + IPT_REG_WR32(g, IPT_ERR_UC_STATUS_LINK(link_id), IPT_ERR_UC_ACTIVE_BITS); + + return true; +} + +/* + ******************************************************************************* + * Interrupt handling functions * + ******************************************************************************* + */ + +/* + * Enable common interrupts + */ +static void gv100_nvlink_common_intr_enable(struct gk20a *g, + unsigned long mask) +{ + u32 reg, i; + + /* Init HS HUB SW state */ + g->nvlink.hshub_config0 = gk20a_readl(g, fb_hshub_config0_r()); + g->nvlink.hshub_config1 = gk20a_readl(g, fb_hshub_config1_r()); + g->nvlink.hshub_config2 = gk20a_readl(g, fb_hshub_config2_r()); + g->nvlink.hshub_config6 = gk20a_readl(g, fb_hshub_config6_r()); + + /* Init IOCTRL */ + for_each_set_bit(i, &mask, 32) { + reg = IOCTRL_REG_RD32(g, ioctrl_link_intr_0_mask_r(i)); + reg |= (ioctrl_link_intr_0_mask_fatal_f(1) | + ioctrl_link_intr_0_mask_nonfatal_f(1) | + ioctrl_link_intr_0_mask_correctable_f(1) | + ioctrl_link_intr_0_mask_intra_f(1)); + IOCTRL_REG_WR32(g, ioctrl_link_intr_0_mask_r(i), reg); + } + + reg = IOCTRL_REG_RD32(g, ioctrl_common_intr_0_mask_r()); + reg |= (ioctrl_common_intr_0_mask_fatal_f(1) | + ioctrl_common_intr_0_mask_nonfatal_f(1) | + ioctrl_common_intr_0_mask_correctable_f(1) | + ioctrl_common_intr_0_mask_intra_f(1)); + IOCTRL_REG_WR32(g, ioctrl_common_intr_0_mask_r(), reg); + + /* Init NVLIPT */ + IPT_REG_WR32(g, nvlipt_intr_control_common_r(), + nvlipt_intr_control_common_stallenable_f(1) | + nvlipt_intr_control_common_nonstallenable_f(1)); +} + +/* + * Enable link specific interrupts (top-level) + */ +static void gv100_nvlink_enable_link_intr(struct gk20a *g, u32 link_id, + bool enable) +{ + gv100_nvlink_minion_link_intr_enable(g, link_id, enable); + gv100_nvlink_dlpl_intr_enable(g, link_id, enable); + gv100_nvlink_tlc_intr_enable(g, link_id, enable); + gv100_nvlink_mif_intr_enable(g, link_id, enable); + gv100_nvlink_nvlipt_intr_enable(g, link_id, enable); +} + +/* + * Top level interrupt handler + */ +int gv100_nvlink_isr(struct gk20a *g) +{ + unsigned long links; + u32 link_id; + + links = ioctrl_top_intr_0_status_link_v( + IOCTRL_REG_RD32(g, ioctrl_top_intr_0_status_r())); + + links &= g->nvlink.enabled_links; + /* As per ARCH minion must be serviced first */ + gv100_nvlink_minion_isr(g); + + for_each_set_bit(link_id, &links, 32) { + /* Cache error logs from TLC, DL handler may clear them */ + gv100_nvlink_tlc_get_intr(g, link_id); + gv100_nvlink_dlpl_isr(g, link_id); + gv100_nvlink_tlc_isr(g, link_id); + gv100_nvlink_mif_isr(g, link_id); + + /* NVLIPT is top-level. Do it last */ + gv100_nvlink_nvlipt_isr(g, link_id); + } + return 0; +} + +/******************************************************************************* + * Helper functions * + ******************************************************************************* + */ + +static u32 __gv100_nvlink_get_link_reset_mask(struct gk20a *g) +{ + u32 reg_data; + + reg_data = IOCTRL_REG_RD32(g, ioctrl_reset_r()); + + return ioctrl_reset_linkreset_v(reg_data); +} + +static u32 __gv100_nvlink_state_load_hal(struct gk20a *g) +{ + unsigned long discovered = g->nvlink.discovered_links; + + gv100_nvlink_common_intr_enable(g, discovered); + return gv100_nvlink_minion_load(g); +} + +#define TRIM_SYS_NVLINK_CTRL(i) (trim_sys_nvlink0_ctrl_r() + 16*i) +#define TRIM_SYS_NVLINK_STATUS(i) (trim_sys_nvlink0_status_r() + 16*i) + +static int gv100_nvlink_setup_pll(struct gk20a *g, unsigned long mask) +{ + u32 reg; + u32 i; + u32 links_off; + struct nvgpu_timeout timeout; + u32 pad_ctrl, swap_ctrl; + u32 pll_id; + + reg = gk20a_readl(g, trim_sys_nvlink_uphy_cfg_r()); + reg = set_field(reg, trim_sys_nvlink_uphy_cfg_phy2clks_use_lockdet_m(), + trim_sys_nvlink_uphy_cfg_phy2clks_use_lockdet_f(1)); + gk20a_writel(g, trim_sys_nvlink_uphy_cfg_r(), reg); + + reg = gk20a_readl(g, top_nvhsclk_ctrl_r()); + + pad_ctrl = top_nvhsclk_ctrl_e_clk_nvl_v(reg); + swap_ctrl = top_nvhsclk_ctrl_swap_clk_nvl_v(reg); + + for_each_set_bit(i, &mask, 32) { + /* There are 3 PLLs for 6 links. We have 3 bits for each PLL. + * The PLL bit corresponding to a link is /2 of its master link. + */ + pll_id = g->nvlink.links[i].pll_master_link_id >> 1; + pad_ctrl |= BIT(pll_id); + swap_ctrl |= BIT(pll_id); + } + + reg = set_field(reg, top_nvhsclk_ctrl_e_clk_nvl_m(), + top_nvhsclk_ctrl_e_clk_nvl_f(pad_ctrl)); + reg = set_field(reg, top_nvhsclk_ctrl_e_clk_core_m(), + top_nvhsclk_ctrl_e_clk_core_f(0x1)); + reg = set_field(reg, top_nvhsclk_ctrl_swap_clk_nvl_m(), + top_nvhsclk_ctrl_swap_clk_nvl_f(swap_ctrl)); + reg = set_field(reg, top_nvhsclk_ctrl_swap_clk_core_m(), + top_nvhsclk_ctrl_swap_clk_core_f(0x0)); + + gk20a_writel(g, top_nvhsclk_ctrl_r(), reg); + + for_each_set_bit(i, &mask, 32) { + reg = gk20a_readl(g, TRIM_SYS_NVLINK_CTRL(i)); + reg = set_field(reg, + trim_sys_nvlink0_ctrl_unit2clks_pll_turn_off_m(), + trim_sys_nvlink0_ctrl_unit2clks_pll_turn_off_f(0)); + gk20a_writel(g, TRIM_SYS_NVLINK_CTRL(i), reg); + } + + /* Poll for links to go up */ + links_off = mask; + + nvgpu_timeout_init(g, &timeout, + NVLINK_PLL_ON_TIMEOUT_MS, NVGPU_TIMER_CPU_TIMER); + do { + for_each_set_bit(i, &mask, 32) { + reg = gk20a_readl(g, TRIM_SYS_NVLINK_STATUS(i)); + if (trim_sys_nvlink0_status_pll_off_v(reg) == 0) + links_off &= ~BIT(i); + } + nvgpu_udelay(5); + + } while((!nvgpu_timeout_expired_msg(&timeout, "timeout on pll on")) && + links_off); + + if (nvgpu_timeout_peek_expired(&timeout)) + return -ETIMEDOUT; + + return 0; +} + +static void gv100_nvlink_prog_alt_clk(struct gk20a *g) +{ + u32 tmp; + + /* RMW registers need to be separate */ + tmp = gk20a_readl(g, trim_sys_nvl_common_clk_alt_switch_r()); + tmp &= ~trim_sys_nvl_common_clk_alt_switch_slowclk_m(); + tmp |= trim_sys_nvl_common_clk_alt_switch_slowclk_xtal4x_f(); + gk20a_writel(g, trim_sys_nvl_common_clk_alt_switch_r(), tmp); + + tmp &= ~trim_sys_nvl_common_clk_alt_switch_finalsel_m(); + tmp |= trim_sys_nvl_common_clk_alt_switch_finalsel_slowclk_f(); +} + +static int gv100_nvlink_enable_links_pre_top(struct gk20a *g, u32 links) +{ + u32 link_id; + unsigned long enabled_links = links; + u32 tmp; + u32 reg; + u32 delay = ioctrl_reset_sw_post_reset_delay_microseconds_v(); + u32 err; + + nvgpu_log(g, gpu_dbg_nvlink, " enabling 0x%lx links", enabled_links); + /* Take links out of reset */ + for_each_set_bit(link_id, &enabled_links, 32) { + reg = IOCTRL_REG_RD32(g, ioctrl_reset_r()); + + tmp = (BIT(link_id) | + BIT(g->nvlink.links[link_id].pll_master_link_id)); + + reg = set_field(reg, ioctrl_reset_linkreset_m(), + ioctrl_reset_linkreset_f( ioctrl_reset_linkreset_v(reg) | + tmp)); + + IOCTRL_REG_WR32(g, ioctrl_reset_r(), reg); + nvgpu_udelay(delay); + + reg = IOCTRL_REG_RD32(g, ioctrl_debug_reset_r()); + + reg &= ~ioctrl_debug_reset_link_f(BIT(link_id)); + IOCTRL_REG_WR32(g, ioctrl_debug_reset_r(), reg); + nvgpu_udelay(delay); + + reg |= ioctrl_debug_reset_link_f(BIT(link_id)); + IOCTRL_REG_WR32(g, ioctrl_debug_reset_r(), reg); + nvgpu_udelay(delay); + + /* Enable Link DLPL for AN0 */ + reg = DLPL_REG_RD32(g, link_id, nvl_link_config_r()); + reg = set_field(reg, nvl_link_config_link_en_m(), + nvl_link_config_link_en_f(1)); + DLPL_REG_WR32(g, link_id, nvl_link_config_r(), reg); + + /* This should be done by the NVLINK API */ + err = gv100_nvlink_minion_init_uphy(g, BIT(link_id), true); + if (err) { + nvgpu_err(g, "Failed to init phy of link: %u", link_id); + return err; + } + + err = gv100_nvlink_rxcal_en(g, BIT(link_id)); + if (err) { + nvgpu_err(g, "Failed to RXcal on link: %u", link_id); + return err; + } + + err = gv100_nvlink_minion_data_ready_en(g, BIT(link_id), true); + if (err) { + nvgpu_err(g, "Failed to set data ready link:%u", + link_id); + return err; + } + + g->nvlink.enabled_links |= BIT(link_id); + } + + nvgpu_log(g, gpu_dbg_nvlink, "enabled_links=0x%08x", + g->nvlink.enabled_links); + + if (g->nvlink.enabled_links) + return 0; + + nvgpu_err(g, " No links were enabled"); + return -EINVAL; +} + +static int gv100_nvlink_enable_links_post_top(struct gk20a *g, u32 links) +{ + u32 link_id; + unsigned long enabled_links = (links & g->nvlink.enabled_links) & + ~g->nvlink.initialized_links; + u32 reg; + + for_each_set_bit(link_id, &enabled_links, 32) { + + /* WAR for HW bug 1888034 */ + reg = DLPL_REG_RD32(g, link_id, nvl_sl0_safe_ctrl2_tx_r()); + reg = set_field(reg, nvl_sl0_safe_ctrl2_tx_ctr_init_m(), + nvl_sl0_safe_ctrl2_tx_ctr_init_init_f()); + reg = set_field(reg, nvl_sl0_safe_ctrl2_tx_ctr_initscl_m(), + nvl_sl0_safe_ctrl2_tx_ctr_initscl_init_f()); + DLPL_REG_WR32(g, link_id, nvl_sl0_safe_ctrl2_tx_r(), reg); + + gv100_nvlink_initialize_tlc(g, link_id); + gv100_nvlink_initialize_nvlipt(g, link_id); + gv100_nvlink_enable_link_intr(g, link_id, true); + + g->nvlink.initialized_links |= BIT(link_id); + }; + + return 0; +} + +static u32 gv100_nvlink_prbs_gen_en(struct gk20a *g, unsigned long mask) +{ + u32 reg; + u32 link_id; + + for_each_set_bit(link_id, &mask, 32) { + /* Write is required as part of HW sequence */ + DLPL_REG_WR32(g, link_id, nvl_sl1_rxslsm_timeout_2_r(), 0); + + reg = DLPL_REG_RD32(g, link_id, nvl_txiobist_config_r()); + reg = set_field(reg, nvl_txiobist_config_dpg_prbsseedld_m(), + nvl_txiobist_config_dpg_prbsseedld_f(0x1)); + DLPL_REG_WR32(g, link_id, nvl_txiobist_config_r(), reg); + + reg = DLPL_REG_RD32(g, link_id, nvl_txiobist_config_r()); + reg = set_field(reg, nvl_txiobist_config_dpg_prbsseedld_m(), + nvl_txiobist_config_dpg_prbsseedld_f(0x0)); + DLPL_REG_WR32(g, link_id, nvl_txiobist_config_r(), reg); + } + + return 0; +} + +static u32 gv100_nvlink_rxcal_en(struct gk20a *g, unsigned long mask) +{ + u32 link_id; + struct nvgpu_timeout timeout; + u32 reg; + + for_each_set_bit(link_id, &mask, 32) { + /* Timeout from HW specs */ + nvgpu_timeout_init(g, &timeout, + 8*NVLINK_SUBLINK_TIMEOUT_MS, NVGPU_TIMER_CPU_TIMER); + reg = DLPL_REG_RD32(g, link_id, nvl_br0_cfg_cal_r()); + reg = set_field(reg, nvl_br0_cfg_cal_rxcal_m(), + nvl_br0_cfg_cal_rxcal_on_f()); + DLPL_REG_WR32(g, link_id, nvl_br0_cfg_cal_r(), reg); + + do { + reg = DLPL_REG_RD32(g, link_id, + nvl_br0_cfg_status_cal_r()); + + if (nvl_br0_cfg_status_cal_rxcal_done_v(reg) == 1) + break; + nvgpu_udelay(5); + } while(!nvgpu_timeout_expired_msg(&timeout, + "timeout on rxcal")); + + if (nvgpu_timeout_peek_expired(&timeout)) + return -ETIMEDOUT; + } + + return 0; +} + +/* + ******************************************************************************* + * Internal "ops" functions * + ******************************************************************************* + */ + + +/* + * Main Nvlink init function. Calls into the Nvlink core API + */ +int gv100_nvlink_init(struct gk20a *g) +{ + int err = 0; + + if (!nvgpu_is_enabled(g, NVGPU_SUPPORT_NVLINK)) + return -ENODEV; + + err = nvgpu_nvlink_enumerate(g); + if (err) + return err; + + /* Set HSHUB and SG_PHY */ + __nvgpu_set_enabled(g, NVGPU_MM_USE_PHYSICAL_SG, true); + return err; +} + +/* + * Query internal device topology and discover devices in nvlink local + * infrastructure. Initialize register base and offsets + */ +int gv100_nvlink_discover_link(struct gk20a *g) +{ + u32 i; + u32 ioctrl_entry_addr; + u8 ioctrl_device_type; + u32 table_entry; + u32 ioctrl_info_entry_type; + u8 ioctrl_discovery_size; + bool is_chain = false; + u8 nvlink_num_devices = 0; + unsigned long available_links = 0; + struct nvgpu_nvlink_device_list *device_table; + u32 err = 0; + + /* + * Process Entry 0 & 1 of IOCTRL table to find table size + */ + if (g->nvlink.ioctrl_table && g->nvlink.ioctrl_table[0].pri_base_addr) { + ioctrl_entry_addr = g->nvlink.ioctrl_table[0].pri_base_addr; + table_entry = gk20a_readl(g, ioctrl_entry_addr); + ioctrl_info_entry_type = nvlinkip_discovery_common_device_v(table_entry); + } else { + nvgpu_err(g, " Bad IOCTRL PRI Base addr"); + return -EINVAL; + } + + if (ioctrl_info_entry_type == NVL_DEVICE(ioctrl)) { + ioctrl_entry_addr = g->nvlink.ioctrl_table[0].pri_base_addr + 4; + table_entry = gk20a_readl(g, ioctrl_entry_addr); + ioctrl_discovery_size = nvlinkip_discovery_common_ioctrl_length_v(table_entry); + nvgpu_log(g, gpu_dbg_nvlink, "IOCTRL size: %d", ioctrl_discovery_size); + } else { + nvgpu_err(g, " First entry of IOCTRL_DISCOVERY invalid"); + return -EINVAL; + } + + device_table = nvgpu_kzalloc(g, ioctrl_discovery_size * + sizeof(struct nvgpu_nvlink_device_list)); + if (!device_table) { + nvgpu_err(g, " Unable to allocate nvlink device table"); + return -ENOMEM; + } + + for (i = 0; i < ioctrl_discovery_size; i++) { + ioctrl_entry_addr = + g->nvlink.ioctrl_table[0].pri_base_addr + 4*i; + table_entry = gk20a_readl(g, ioctrl_entry_addr); + + nvgpu_log(g, gpu_dbg_nvlink, "parsing ioctrl %d: 0x%08x", i, table_entry); + + ioctrl_info_entry_type = nvlinkip_discovery_common_entry_v(table_entry); + + if (ioctrl_info_entry_type == + nvlinkip_discovery_common_entry_invalid_v()) + continue; + + if (ioctrl_info_entry_type == + nvlinkip_discovery_common_entry_enum_v()) { + + nvgpu_log(g, gpu_dbg_nvlink, "IOCTRL entry %d is ENUM", i); + + ioctrl_device_type = + nvlinkip_discovery_common_device_v(table_entry); + + if (nvlinkip_discovery_common_chain_v(table_entry) != + nvlinkip_discovery_common_chain_enable_v()) { + + nvgpu_log(g, gpu_dbg_nvlink, + "IOCTRL entry %d is ENUM but no chain", + i); + err = -EINVAL; + break; + } + + is_chain = true; + device_table[nvlink_num_devices].valid = true; + device_table[nvlink_num_devices].device_type = + ioctrl_device_type; + device_table[nvlink_num_devices].device_id = + nvlinkip_discovery_common_id_v(table_entry); + device_table[nvlink_num_devices].device_version = + nvlinkip_discovery_common_version_v( + table_entry); + continue; + } + + if (ioctrl_info_entry_type == + nvlinkip_discovery_common_entry_data1_v()) { + nvgpu_log(g, gpu_dbg_nvlink, "IOCTRL entry %d is DATA1", i); + + if (is_chain) { + device_table[nvlink_num_devices].pri_base_addr = + nvlinkip_discovery_common_pri_base_v( + table_entry) << 12; + + device_table[nvlink_num_devices].intr_enum = + nvlinkip_discovery_common_intr_v( + table_entry); + + device_table[nvlink_num_devices].reset_enum = + nvlinkip_discovery_common_reset_v( + table_entry); + + nvgpu_log(g, gpu_dbg_nvlink, "IOCTRL entry %d type = %d base: 0x%08x intr: %d reset: %d", + i, + device_table[nvlink_num_devices].device_type, + device_table[nvlink_num_devices].pri_base_addr, + device_table[nvlink_num_devices].intr_enum, + device_table[nvlink_num_devices].reset_enum); + + if (device_table[nvlink_num_devices].device_type == + NVL_DEVICE(dlpl)) { + device_table[nvlink_num_devices].num_tx = + nvlinkip_discovery_common_dlpl_num_tx_v(table_entry); + device_table[nvlink_num_devices].num_rx = + nvlinkip_discovery_common_dlpl_num_rx_v(table_entry); + + nvgpu_log(g, gpu_dbg_nvlink, "DLPL tx: %d rx: %d", + device_table[nvlink_num_devices].num_tx, + device_table[nvlink_num_devices].num_rx); + } + + if (nvlinkip_discovery_common_chain_v(table_entry) != + nvlinkip_discovery_common_chain_enable_v()) { + + is_chain = false; + nvlink_num_devices++; + } + } + continue; + } + + if (ioctrl_info_entry_type == + nvlinkip_discovery_common_entry_data2_v()) { + + nvgpu_log(g, gpu_dbg_nvlink, "IOCTRL entry %d is DATA2", i); + + if (is_chain) { + if (nvlinkip_discovery_common_dlpl_data2_type_v(table_entry)) { + device_table[nvlink_num_devices].pll_master = + nvlinkip_discovery_common_dlpl_data2_master_v(table_entry); + device_table[nvlink_num_devices].pll_master_id = + nvlinkip_discovery_common_dlpl_data2_masterid_v(table_entry); + nvgpu_log(g, gpu_dbg_nvlink, "PLL info: Master: %d, Master ID: %d", + device_table[nvlink_num_devices].pll_master, + device_table[nvlink_num_devices].pll_master_id); + } + + if (nvlinkip_discovery_common_chain_v(table_entry) != + nvlinkip_discovery_common_chain_enable_v()) { + + is_chain = false; + nvlink_num_devices++; + } + } + continue; + } + } + + g->nvlink.device_table = device_table; + g->nvlink.num_devices = nvlink_num_devices; + + /* + * Print table + */ + for (i = 0; i < nvlink_num_devices; i++) { + if (device_table[i].valid) { + nvgpu_log(g, gpu_dbg_nvlink, "Device %d - %s", i, + __gv100_device_type_to_str( + device_table[i].device_type)); + nvgpu_log(g, gpu_dbg_nvlink, "+Link/Device Id: %d", device_table[i].device_id); + nvgpu_log(g, gpu_dbg_nvlink, "+Version: %d", device_table[i].device_version); + nvgpu_log(g, gpu_dbg_nvlink, "+Base Addr: 0x%08x", device_table[i].pri_base_addr); + nvgpu_log(g, gpu_dbg_nvlink, "+Intr Enum: %d", device_table[i].intr_enum); + nvgpu_log(g, gpu_dbg_nvlink, "+Reset Enum: %d", device_table[i].reset_enum); + if ((device_table[i].device_type == NVL_DEVICE(dlpl)) || + (device_table[i].device_type == NVL_DEVICE(nvlink))) { + nvgpu_log(g, gpu_dbg_nvlink, "+TX: %d", device_table[i].num_tx); + nvgpu_log(g, gpu_dbg_nvlink, "+RX: %d", device_table[i].num_rx); + nvgpu_log(g, gpu_dbg_nvlink, "+PLL Master: %d", device_table[i].pll_master); + nvgpu_log(g, gpu_dbg_nvlink, "+PLL Master ID: %d", device_table[i].pll_master_id); + } + } + } + + for (i = 0; i < nvlink_num_devices; i++) { + if (device_table[i].valid) { + + if (device_table[i].device_type == NVL_DEVICE(ioctrl)) { + + g->nvlink.ioctrl_type = + device_table[i].device_type; + g->nvlink.ioctrl_base = + device_table[i].pri_base_addr; + continue; + } + + if (device_table[i].device_type == NVL_DEVICE(dlpl)) { + + g->nvlink.dlpl_type = + device_table[i].device_type; + g->nvlink.dlpl_base[device_table[i].device_id] = + device_table[i].pri_base_addr; + g->nvlink.links[device_table[i].device_id].valid = true; + g->nvlink.links[device_table[i].device_id].g = g; + g->nvlink.links[device_table[i].device_id].dlpl_version = + device_table[i].device_version; + g->nvlink.links[device_table[i].device_id].dlpl_base = + device_table[i].pri_base_addr; + g->nvlink.links[device_table[i].device_id].intr_enum = + device_table[i].intr_enum; + g->nvlink.links[device_table[i].device_id].reset_enum = + device_table[i].reset_enum; + g->nvlink.links[device_table[i].device_id].link_id = + device_table[i].device_id; + + /* initiate the PLL master and slave link id to max */ + g->nvlink.links[device_table[i].device_id].pll_master_link_id = + NVLINK_MAX_LINKS_SW; + g->nvlink.links[device_table[i].device_id].pll_slave_link_id = + NVLINK_MAX_LINKS_SW; + + /* Update Pll master */ + if (device_table[i].pll_master) + g->nvlink.links[device_table[i].device_id].pll_master_link_id = + g->nvlink.links[device_table[i].device_id].link_id; + else { + g->nvlink.links[device_table[i].device_id].pll_master_link_id = + device_table[i].pll_master_id; + g->nvlink.links[device_table[i].device_id].pll_slave_link_id = + g->nvlink.links[device_table[i].device_id].link_id; + g->nvlink.links[device_table[i].pll_master_id].pll_slave_link_id = + g->nvlink.links[device_table[i].device_id].link_id; + } + + available_links |= BIT(device_table[i].device_id); + continue; + } + + if (device_table[i].device_type == NVL_DEVICE(nvltlc)) { + + g->nvlink.tl_type = device_table[i].device_type; + g->nvlink.tl_base[device_table[i].device_id] = + device_table[i].pri_base_addr; + g->nvlink.links[device_table[i].device_id].tl_base = + device_table[i].pri_base_addr; + g->nvlink.links[device_table[i].device_id].tl_version = + device_table[i].device_version; + continue; + } + + if (device_table[i].device_type == NVL_DEVICE(nvltlc)) { + + g->nvlink.tl_type = device_table[i].device_type; + g->nvlink.tl_base[device_table[i].device_id] = + device_table[i].pri_base_addr; + g->nvlink.links[device_table[i].device_id].tl_base = + device_table[i].pri_base_addr; + g->nvlink.links[device_table[i].device_id].tl_version = + device_table[i].device_version; + continue; + } + + if (device_table[i].device_type == NVL_DEVICE(ioctrlmif)) { + + g->nvlink.mif_type = device_table[i].device_type; + g->nvlink.mif_base[device_table[i].device_id] = + device_table[i].pri_base_addr; + g->nvlink.links[device_table[i].device_id].mif_base = + device_table[i].pri_base_addr; + g->nvlink.links[device_table[i].device_id].mif_version = + device_table[i].device_version; + continue; + } + + if (device_table[i].device_type == NVL_DEVICE(nvlipt)) { + + g->nvlink.ipt_type = + device_table[i].device_type; + g->nvlink.ipt_base = + device_table[i].pri_base_addr; + g->nvlink.ipt_version = + device_table[i].device_version; + continue; + } + + if (device_table[i].device_type == NVL_DEVICE(minion)) { + + g->nvlink.minion_type = + device_table[i].device_type; + g->nvlink.minion_base = + device_table[i].pri_base_addr; + g->nvlink.minion_version = + device_table[i].device_version; + continue; + } + + if (device_table[i].device_type == NVL_DEVICE(dlpl_multicast)) { + + g->nvlink.dlpl_multicast_type = + device_table[i].device_type; + g->nvlink.dlpl_multicast_base = + device_table[i].pri_base_addr; + g->nvlink.dlpl_multicast_version = + device_table[i].device_version; + continue; + } + if (device_table[i].device_type == NVL_DEVICE(nvltlc_multicast)) { + + g->nvlink.tl_multicast_type = + device_table[i].device_type; + g->nvlink.tl_multicast_base = + device_table[i].pri_base_addr; + g->nvlink.tl_multicast_version = + device_table[i].device_version; + continue; + } + + if (device_table[i].device_type == NVL_DEVICE(ioctrlmif_multicast)) { + + g->nvlink.mif_multicast_type = + device_table[i].device_type; + g->nvlink.mif_multicast_base = + device_table[i].pri_base_addr; + g->nvlink.mif_multicast_version = + device_table[i].device_version; + continue; + } + + } + } + + g->nvlink.discovered_links = (u32) available_links; + + nvgpu_log(g, gpu_dbg_nvlink, "Nvlink Tree:"); + nvgpu_log(g, gpu_dbg_nvlink, "+ Available Links: 0x%08lx", available_links); + nvgpu_log(g, gpu_dbg_nvlink, "+ Per-Link Devices:"); + + for_each_set_bit(i, &available_links, 32) { + nvgpu_log(g, gpu_dbg_nvlink, "-- Link %d Dl/Pl Base: 0x%08x TLC Base: 0x%08x MIF Base: 0x%08x", + i, g->nvlink.dlpl_base[i], g->nvlink.tl_base[i], g->nvlink.mif_base[i]); + } + + nvgpu_log(g, gpu_dbg_nvlink, "+ IOCTRL Base: 0x%08x", g->nvlink.ioctrl_base); + nvgpu_log(g, gpu_dbg_nvlink, "+ NVLIPT Base: 0x%08x", g->nvlink.ipt_base); + nvgpu_log(g, gpu_dbg_nvlink, "+ MINION Base: 0x%08x", g->nvlink.minion_base); + nvgpu_log(g, gpu_dbg_nvlink, "+ DLPL MCAST Base: 0x%08x", g->nvlink.dlpl_multicast_base); + nvgpu_log(g, gpu_dbg_nvlink, "+ TLC MCAST Base: 0x%08x", g->nvlink.tl_multicast_base); + nvgpu_log(g, gpu_dbg_nvlink, "+ MIF MCAST Base: 0x%08x", g->nvlink.mif_multicast_base); + + if (!g->nvlink.minion_version) { + nvgpu_err(g, "Unsupported MINION version"); + + nvgpu_kfree(g, device_table); + g->nvlink.device_table = NULL; + g->nvlink.num_devices = 0; + return -EINVAL; + } + + return err; +} + +/* + * Query IOCTRL for device discovery + */ +int gv100_nvlink_discover_ioctrl(struct gk20a *g) +{ + u32 i; + struct nvgpu_nvlink_ioctrl_list *ioctrl_table; + u32 table_entry; + u32 devinfo_type; + bool is_ioctrl = false; + bool is_chain = false; + u32 io_num_entries = 0; + + ioctrl_table = nvgpu_kzalloc(g, top_device_info__size_1_v() * + sizeof(struct nvgpu_nvlink_ioctrl_list)); + + if (!ioctrl_table) { + nvgpu_err(g, "failed to allocate memory for nvlink io table"); + return -ENOMEM; + } + + for (i = 0; i < top_device_info__size_1_v(); i++) { + table_entry = gk20a_readl(g, top_device_info_r(i)); + + devinfo_type = top_device_info_entry_v(table_entry); + + if (devinfo_type == top_device_info_entry_not_valid_v()) + continue; + + if (devinfo_type == top_device_info_entry_engine_type_v()) { + if (top_device_info_type_enum_v(table_entry) == + top_device_info_type_enum_ioctrl_v()) + is_ioctrl = true; + else { + is_ioctrl = false; + continue; + } + + if (top_device_info_chain_v(table_entry) != + top_device_info_chain_enable_v()) + break; + + is_chain = true; + ioctrl_table[io_num_entries].valid = true; + continue; + } + + if (devinfo_type == top_device_info_entry_data_v()) { + if (is_ioctrl && is_chain) { + + ioctrl_table[io_num_entries].pri_base_addr = + top_device_info_data_pri_base_v(table_entry) << + top_device_info_data_pri_base_align_v(); + + if (top_device_info_chain_v(table_entry) != + top_device_info_chain_enable_v()) { + is_chain = false; + io_num_entries++; + } + } + continue; + } + + if (devinfo_type == top_device_info_entry_enum_v()) { + if (is_ioctrl && is_chain) { + + ioctrl_table[io_num_entries].intr_enum = + top_device_info_intr_v(table_entry); + ioctrl_table[io_num_entries].reset_enum = + top_device_info_reset_v(table_entry); + + if (top_device_info_chain_v(table_entry) != + top_device_info_chain_enable_v()) { + is_chain = false; + io_num_entries++; + } + } + continue; + } + } + + if (io_num_entries == 0 || !ioctrl_table[0].pri_base_addr) { + nvgpu_err(g, "No NVLINK io found"); + nvgpu_kfree(g, ioctrl_table); + return -EINVAL; + } + + g->nvlink.ioctrl_table = ioctrl_table; + g->nvlink.io_num_entries = io_num_entries; + + + for (i =0; i < io_num_entries; i++) + nvgpu_log(g, gpu_dbg_nvlink, + "Device %d : Pri Base Addr = 0x%0x Intr = %d Reset = %d", + i, ioctrl_table[i].pri_base_addr, ioctrl_table[i].intr_enum, + ioctrl_table[i].reset_enum); + + return 0; +} + +/* + ******************************************************************************* + * NVLINK API FUNCTIONS * + ******************************************************************************* + */ + +/* + * Performs link level initialization like phy inits, AN0 and interrupts + */ + +int gv100_nvlink_link_early_init(struct gk20a *g, unsigned long mask) +{ + int err; + + err = gv100_nvlink_enable_links_pre_top(g, mask); + if (err) { + nvgpu_err(g, "Pre topology failed for links %lx", mask); + return err; + } + + nvgpu_log(g, gpu_dbg_nvlink, "pretopology enabled: 0x%lx", + mask & g->nvlink.enabled_links); + err = gv100_nvlink_enable_links_post_top(g, mask); + + return err; +} + +/* + * Performs memory interface initialization + */ + +int gv100_nvlink_interface_init(struct gk20a *g) +{ + unsigned long mask = g->nvlink.enabled_links; + u32 link_id; + + for_each_set_bit(link_id, &mask, 32) { + gv100_nvlink_initialize_mif(g, link_id); + gv100_nvlink_mif_intr_enable(g, link_id, true); + } + + return 0; +} + +int gv100_nvlink_reg_init(struct gk20a *g) +{ + u32 i = 0; + u32 count = 0; + struct __nvlink_reginit *reg; + enum nvgpu_nvlink_endp endp; + int err; + u32 link_id; + unsigned long mask = g->nvlink.enabled_links; + struct nvgpu_nvlink_link *link; + + /* Apply automated reg init flow for PROD settings */ + for_each_set_bit(link_id, &mask, 32) { + + link = &g->nvlink.links[link_id]; + if (!link->remote_info.is_connected) + continue; + + endp = link->remote_info.device_type; + err = gv100_nvlink_get_tlc_reginit(endp, ®, &count); + if (err) { + nvgpu_err(g, "no reginit for endp=%u", endp); + continue; + } + + for (i = 0; i < count; i++) { + TLC_REG_WR32(g, link_id, reg->addr, reg->value); + reg++; + } + } + return 0; +} + +/* + * Shutdown device. This should tear down Nvlink connection. + * For now return. + */ +int gv100_nvlink_shutdown(struct gk20a *g) +{ + return 0; +} + +/* + * Get link state + */ +u32 gv100_nvlink_link_get_state(struct gk20a *g, u32 link_id) +{ + return DLPL_REG_RD32(g, link_id, nvl_link_state_r()) & + nvl_link_state_state_m(); +} + +/* Get link mode */ +u32 gv100_nvlink_link_get_mode(struct gk20a *g, u32 link_id) +{ + u32 state; + if (!(BIT(link_id) & g->nvlink.discovered_links)) + return nvgpu_nvlink_link__last; + + state = nvl_link_state_state_v( + g->ops.nvlink.link_get_state(g, link_id)); + + if (state == nvl_link_state_state_init_v()) + return nvgpu_nvlink_link_off; + if (state == nvl_link_state_state_hwcfg_v()) + return nvgpu_nvlink_link_detect; + if (state == nvl_link_state_state_swcfg_v()) + return nvgpu_nvlink_link_safe; + if (state == nvl_link_state_state_active_v()) + return nvgpu_nvlink_link_hs; + if (state == nvl_link_state_state_fault_v()) + return nvgpu_nvlink_link_fault; + if ((state == nvl_link_state_state_rcvy_ac_v()) || + (state == nvl_link_state_state_rcvy_sw_v()) || + (state == nvl_link_state_state_rcvy_rx_v())) + return nvgpu_nvlink_link_recovery; + + return nvgpu_nvlink_link_off; +} + +/* Set Link mode */ +int gv100_nvlink_link_set_mode(struct gk20a *g, u32 link_id, u32 mode) +{ + u32 state; + u32 reg; + u32 err = 0; + + nvgpu_log(g, gpu_dbg_nvlink, "link :%d, mode:%u", link_id, mode); + + if (!(BIT(link_id) & g->nvlink.enabled_links)) + return -EINVAL; + + state = nvl_link_state_state_v( + g->ops.nvlink.link_get_state(g, link_id)); + + switch (mode) { + case nvgpu_nvlink_link_safe: + if (state == nvl_link_state_state_swcfg_v()) { + nvgpu_warn(g, "link is already in safe mode"); + break; + } + if (state == nvl_link_state_state_hwcfg_v()) { + nvgpu_warn(g, "link is transitioning to safe mode"); + break; + } + + if (state == nvl_link_state_state_init_v()) { + /* Off to Safe transition */ + reg = DLPL_REG_RD32(g, link_id, nvl_link_change_r()); + reg = set_field(reg, nvl_link_change_newstate_m(), + nvl_link_change_newstate_hwcfg_f()); + reg = set_field(reg, nvl_link_change_oldstate_mask_m(), + nvl_link_change_oldstate_mask_dontcare_f()); + reg = set_field(reg, nvl_link_change_action_m(), + nvl_link_change_action_ltssm_change_f()); + DLPL_REG_WR32(g, link_id, nvl_link_change_r(), reg); + } else if (state == nvl_link_state_state_active_v()) { + /* TODO: + * Disable PM first since we are moving out active + * state + */ + reg = DLPL_REG_RD32(g, link_id, nvl_link_change_r()); + reg = set_field(reg, nvl_link_change_newstate_m(), + nvl_link_change_newstate_swcfg_f()); + reg = set_field(reg, nvl_link_change_oldstate_mask_m(), + nvl_link_change_oldstate_mask_dontcare_f()); + reg = set_field(reg, nvl_link_change_action_m(), + nvl_link_change_action_ltssm_change_f()); + DLPL_REG_WR32(g, link_id, nvl_link_change_r(), reg); + } + break; + + case nvgpu_nvlink_link_hs: + if (state == nvl_link_state_state_active_v()) { + nvgpu_err(g, "link is already in active mode"); + break; + } + if (state == nvl_link_state_state_init_v()) { + nvgpu_err(g, "link cannot be taken from init state"); + return -EPERM; + } + + reg = DLPL_REG_RD32(g, link_id, nvl_link_change_r()); + reg = set_field(reg, nvl_link_change_newstate_m(), + nvl_link_change_newstate_active_f()); + reg = set_field(reg, nvl_link_change_oldstate_mask_m(), + nvl_link_change_oldstate_mask_dontcare_f()); + reg = set_field(reg, nvl_link_change_action_m(), + nvl_link_change_action_ltssm_change_f()); + DLPL_REG_WR32(g, link_id, nvl_link_change_r(), reg); + break; + + case nvgpu_nvlink_link_off: + if (state == nvl_link_state_state_active_v()) { + nvgpu_err(g, "link cannot be taken from active to init"); + return -EPERM; + } + if (state == nvl_link_state_state_init_v()) { + nvgpu_err(g, "link already in init state"); + } + + /* GV100 UPHY is handled by MINION */ + break; + /* 1/8 th mode not supported */ + case nvgpu_nvlink_link_enable_pm: + case nvgpu_nvlink_link_disable_pm: + return -EPERM; + case nvgpu_nvlink_link_disable_err_detect: + /* Disable Link interrupts */ + gv100_nvlink_dlpl_intr_enable(g, link_id, false); + break; + case nvgpu_nvlink_link_lane_disable: + err = gv100_nvlink_minion_lane_disable(g, link_id, true); + break; + case nvgpu_nvlink_link_lane_shutdown: + err = gv100_nvlink_minion_lane_shutdown(g, link_id, true); + break; + default: + nvgpu_err(g, "Unhandled mode %x", mode); + break; + } + + return err; +} + +static u32 gv100_nvlink_link_sublink_check_change(struct gk20a *g, u32 link_id) +{ + struct nvgpu_timeout timeout; + u32 reg; + + nvgpu_timeout_init(g, &timeout, + NVLINK_SUBLINK_TIMEOUT_MS, NVGPU_TIMER_CPU_TIMER); + /* Poll for sublink status */ + do { + reg = DLPL_REG_RD32(g, link_id, nvl_sublink_change_r()); + + if (nvl_sublink_change_status_v(reg) == + nvl_sublink_change_status_done_v()) + break; + if (nvl_sublink_change_status_v(reg) == + nvl_sublink_change_status_fault_v()) { + nvgpu_err(g, "Fault detected in sublink change"); + return -EFAULT; + } + nvgpu_udelay(5); + } while(!nvgpu_timeout_expired_msg(&timeout, "timeout on sublink rdy")); + + if (nvgpu_timeout_peek_expired(&timeout)) + return -ETIMEDOUT; + return-0; +} + +int gv100_nvlink_link_set_sublink_mode(struct gk20a *g, u32 link_id, + bool is_rx_sublink, u32 mode) +{ + int err = 0; + u32 rx_sublink_state, tx_sublink_state; + u32 reg; + + if (!(BIT(link_id) & g->nvlink.enabled_links)) + return -EINVAL; + + err = gv100_nvlink_link_sublink_check_change(g, link_id); + if (err) + return err; + + rx_sublink_state = g->ops.nvlink.get_rx_sublink_state(g, link_id); + tx_sublink_state = g->ops.nvlink.get_tx_sublink_state(g, link_id); + + switch (mode) { + case nvgpu_nvlink_sublink_tx_hs: + if (tx_sublink_state == + nvl_sl0_slsm_status_tx_primary_state_hs_v()) { + nvgpu_err(g, " TX already in HS"); + break; + } else if (tx_sublink_state == + nvl_sl0_slsm_status_tx_primary_state_off_v()) { + nvgpu_err(g, "TX cannot be do from OFF to HS"); + return -EPERM; + } + + reg = DLPL_REG_RD32(g, link_id, nvl_sublink_change_r()); + reg = set_field(reg, nvl_sublink_change_newstate_m(), + nvl_sublink_change_newstate_hs_f()); + reg = set_field(reg, nvl_sublink_change_sublink_m(), + nvl_sublink_change_sublink_tx_f()); + reg = set_field(reg, nvl_sublink_change_action_m(), + nvl_sublink_change_action_slsm_change_f()); + DLPL_REG_WR32(g, link_id, nvl_sublink_change_r(), reg); + + err = gv100_nvlink_link_sublink_check_change(g, link_id); + if (err) { + nvgpu_err(g, "Error in TX to HS"); + return err; + } + break; + case nvgpu_nvlink_sublink_tx_common: + err = gv100_nvlink_minion_init_uphy(g, BIT(link_id), true); + break; + case nvgpu_nvlink_sublink_tx_common_disable: + /* NOP */ + break; + case nvgpu_nvlink_sublink_tx_data_ready: + err = gv100_nvlink_minion_data_ready_en(g, BIT(link_id), true); + break; + case nvgpu_nvlink_sublink_tx_prbs_en: + err = gv100_nvlink_prbs_gen_en(g, BIT(link_id)); + break; + case nvgpu_nvlink_sublink_tx_safe: + if (tx_sublink_state == + nvl_sl0_slsm_status_tx_primary_state_safe_v()) { + nvgpu_err(g, "TX already SAFE: %d", link_id); + break; + } + + reg = DLPL_REG_RD32(g, link_id, nvl_sublink_change_r()); + reg = set_field(reg, nvl_sublink_change_newstate_m(), + nvl_sublink_change_newstate_safe_f()); + reg = set_field(reg, nvl_sublink_change_sublink_m(), + nvl_sublink_change_sublink_tx_f()); + reg = set_field(reg, nvl_sublink_change_action_m(), + nvl_sublink_change_action_slsm_change_f()); + DLPL_REG_WR32(g, link_id, nvl_sublink_change_r(), reg); + + err = gv100_nvlink_link_sublink_check_change(g, link_id); + if (err) { + nvgpu_err(g, "Error in TX to SAFE"); + return err; + } + break; + case nvgpu_nvlink_sublink_tx_off: + if (tx_sublink_state == + nvl_sl0_slsm_status_tx_primary_state_off_v()) { + nvgpu_err(g, "TX already OFF: %d", link_id); + break; + } else if (tx_sublink_state == + nvl_sl0_slsm_status_tx_primary_state_hs_v()) { + nvgpu_err(g, " TX cannot go off from HS %d", link_id); + return -EPERM; + } + + reg = DLPL_REG_RD32(g, link_id, nvl_sublink_change_r()); + reg = set_field(reg, nvl_sublink_change_newstate_m(), + nvl_sublink_change_newstate_off_f()); + reg = set_field(reg, nvl_sublink_change_sublink_m(), + nvl_sublink_change_sublink_tx_f()); + reg = set_field(reg, nvl_sublink_change_action_m(), + nvl_sublink_change_action_slsm_change_f()); + DLPL_REG_WR32(g, link_id, nvl_sublink_change_r(), reg); + + err = gv100_nvlink_link_sublink_check_change(g, link_id); + if (err) { + nvgpu_err(g, "Error in TX to OFF"); + return err; + } + break; + + /* RX modes */ + case nvgpu_nvlink_sublink_rx_hs: + case nvgpu_nvlink_sublink_rx_safe: + break; + case nvgpu_nvlink_sublink_rx_off: + if (rx_sublink_state == + nvl_sl1_slsm_status_rx_primary_state_off_v()) { + nvgpu_err(g, "RX already OFF: %d", link_id); + break; + } else if (rx_sublink_state == + nvl_sl1_slsm_status_rx_primary_state_hs_v()) { + nvgpu_err(g, " RX cannot go off from HS %d", link_id); + return -EPERM; + } + + reg = DLPL_REG_RD32(g, link_id, nvl_sublink_change_r()); + reg = set_field(reg, nvl_sublink_change_newstate_m(), + nvl_sublink_change_newstate_off_f()); + reg = set_field(reg, nvl_sublink_change_sublink_m(), + nvl_sublink_change_sublink_rx_f()); + reg = set_field(reg, nvl_sublink_change_action_m(), + nvl_sublink_change_action_slsm_change_f()); + DLPL_REG_WR32(g, link_id, nvl_sublink_change_r(), reg); + + err = gv100_nvlink_link_sublink_check_change(g, link_id); + if (err) { + nvgpu_err(g, "Error in RX to OFF"); + return err; + } + break; + case nvgpu_nvlink_sublink_rx_rxcal: + err = gv100_nvlink_rxcal_en(g, BIT(link_id)); + break; + + default: + if ((is_rx_sublink) && ((mode < nvgpu_nvlink_sublink_rx_hs) || + (mode >= nvgpu_nvlink_sublink_rx__last))) { + nvgpu_err(g, "Unsupported RX mode %u", mode); + return -EINVAL; + } + if (mode >= nvgpu_nvlink_sublink_tx__last) { + nvgpu_err(g, "Unsupported TX mode %u", mode); + return -EINVAL; + } + nvgpu_err(g, "MODE %u", mode); + } + + if (err) + nvgpu_err(g, " failed on set_sublink_mode"); + return err; +} + +u32 gv100_nvlink_link_get_sublink_mode(struct gk20a *g, u32 link_id, + bool is_rx_sublink) +{ + u32 state; + + if (!(BIT(link_id) & g->nvlink.discovered_links)) { + if (!is_rx_sublink) + return nvgpu_nvlink_sublink_tx__last; + return nvgpu_nvlink_sublink_rx__last; + } + + if (!is_rx_sublink) { + state = g->ops.nvlink.get_tx_sublink_state(g, link_id); + if (state == nvl_sl0_slsm_status_tx_primary_state_hs_v()) + return nvgpu_nvlink_sublink_tx_hs; + if (state == nvl_sl0_slsm_status_tx_primary_state_eighth_v()) + return nvgpu_nvlink_sublink_tx_single_lane; + if (state == nvl_sl0_slsm_status_tx_primary_state_safe_v()) + return nvgpu_nvlink_sublink_tx_safe; + if (state == nvl_sl0_slsm_status_tx_primary_state_off_v()) + return nvgpu_nvlink_sublink_tx_off; + return nvgpu_nvlink_sublink_tx_off; + } else { + state = g->ops.nvlink.get_rx_sublink_state(g, link_id); + if (state == nvl_sl1_slsm_status_rx_primary_state_hs_v()) + return nvgpu_nvlink_sublink_rx_hs; + if (state == nvl_sl1_slsm_status_rx_primary_state_eighth_v()) + return nvgpu_nvlink_sublink_rx_single_lane; + if (state == nvl_sl1_slsm_status_rx_primary_state_safe_v()) + return nvgpu_nvlink_sublink_rx_safe; + if (state == nvl_sl1_slsm_status_rx_primary_state_off_v()) + return nvgpu_nvlink_sublink_rx_off; + return nvgpu_nvlink_sublink_rx_off; + } + return nvgpu_nvlink_sublink_tx__last; +} + +/* + * Get TX sublink state + */ +u32 gv100_nvlink_link_get_tx_sublink_state(struct gk20a *g, u32 link_id) +{ + u32 reg = DLPL_REG_RD32(g, link_id, nvl_sl0_slsm_status_tx_r()); + + return nvl_sl0_slsm_status_tx_primary_state_v(reg); +} + +/* + * Get RX sublink state + */ +u32 gv100_nvlink_link_get_rx_sublink_state(struct gk20a *g, u32 link_id) +{ + u32 reg = DLPL_REG_RD32(g, link_id, nvl_sl1_slsm_status_rx_r()); + + return nvl_sl1_slsm_status_rx_primary_state_v(reg); +} + +/* + * Performs nvlink device level initialization by discovering the topology + * taking device out of reset, boot minion, set clocks up and common interrupts + */ +int gv100_nvlink_early_init(struct gk20a *g) +{ + int err = 0; + + if (!nvgpu_is_enabled(g, NVGPU_SUPPORT_NVLINK)) + return -EINVAL; + + err = nvgpu_bios_get_nvlink_config_data(g); + if (err) { + nvgpu_err(g, "failed to read nvlink vbios data"); + goto nvlink_init_exit; + } + + err = g->ops.nvlink.discover_ioctrl(g); + if (err) + goto nvlink_init_exit; + + /* Enable NVLINK in MC */ + g->ops.mc.reset(g, mc_enable_nvlink_enabled_f()); + + err = g->ops.nvlink.discover_link(g); + if (err || g->nvlink.discovered_links == 0) { + nvgpu_err(g, "No links available"); + goto nvlink_init_exit; + } + + nvgpu_flcn_sw_init(g, FALCON_ID_MINION); + + g->nvlink.discovered_links &= ~g->nvlink.link_disable_mask; + nvgpu_log(g, gpu_dbg_nvlink, "link_disable_mask = 0x%08x (from VBIOS)", + g->nvlink.link_disable_mask); + + /* Links in reset should be removed from initialized link sw state */ + g->nvlink.initialized_links &= __gv100_nvlink_get_link_reset_mask(g); + + nvgpu_log(g, gpu_dbg_nvlink, "connected_links = 0x%08x (from DT)", + g->nvlink.connected_links); + + /* Track unconnected links */ + g->nvlink.discovered_links &= g->nvlink.connected_links; + + nvgpu_log(g, gpu_dbg_nvlink, "discovered_links = 0x%08x (combination)", + g->nvlink.discovered_links); + + if (hweight32(g->nvlink.discovered_links) > 1) { + nvgpu_err(g, "more than one link enabled"); + err = -EINVAL; + goto nvlink_init_exit; + } + + /* For now set default speed */ + g->nvlink.speed = nvgpu_nvlink_speed_default; + + err = __gv100_nvlink_state_load_hal(g); + if (err) { + nvgpu_err(g, " failed Nvlink state load"); + goto nvlink_init_exit; + } + err = gv100_nvlink_minion_configure_ac_coupling(g, + g->nvlink.ac_coupling_mask, true); + if (err) { + nvgpu_err(g, " failed Nvlink state load"); + goto nvlink_init_exit; + } + + /* Program clocks */ + gv100_nvlink_prog_alt_clk(g); + +nvlink_init_exit: + return err; +} + diff --git a/drivers/gpu/nvgpu/gv100/nvlink_gv100.h b/drivers/gpu/nvgpu/gv100/nvlink_gv100.h new file mode 100644 index 000000000..4ac8b9078 --- /dev/null +++ b/drivers/gpu/nvgpu/gv100/nvlink_gv100.h @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef NVGPU_NVLINK_GV100_H +#define NVGPU_NVLINK_GV100_H + +struct gk20a; + +#define MINION_REG_RD32(g, off) gk20a_readl(g, g->nvlink.minion_base + (off)) +#define MINION_REG_WR32(g, off, v) gk20a_writel(g, g->nvlink.minion_base + (off), (v)) +#define IOCTRL_REG_RD32(g, off) gk20a_readl(g, g->nvlink.ioctrl_base + (off)) +#define IOCTRL_REG_WR32(g, off, v) gk20a_writel(g, g->nvlink.ioctrl_base + (off), (v)); +#define MIF_REG_RD32(g, id, off) gk20a_readl(g, g->nvlink.links[(id)].mif_base + (off)) +#define MIF_REG_WR32(g, id, off, v) gk20a_writel(g, g->nvlink.links[(id)].mif_base + (off), (v)) +#define IPT_REG_RD32(g, off) gk20a_readl(g, g->nvlink.ipt_base + (off)) +#define IPT_REG_WR32(g, off, v) gk20a_writel(g, g->nvlink.ipt_base + (off), (v)) +#define TLC_REG_RD32(g, id, off) gk20a_readl(g, g->nvlink.links[(id)].tl_base + (off)) +#define TLC_REG_WR32(g, id, off, v) gk20a_writel(g, g->nvlink.links[(id)].tl_base + (off), (v)) +#define DLPL_REG_RD32(g, id, off) gk20a_readl(g, g->nvlink.links[(id)].dlpl_base + (off)) +#define DLPL_REG_WR32(g, id, off, v) gk20a_writel(g, g->nvlink.links[(id)].dlpl_base + (off), (v)) + +int gv100_nvlink_discover_ioctrl(struct gk20a *g); +int gv100_nvlink_discover_link(struct gk20a *g); +int gv100_nvlink_init(struct gk20a *g); +int gv100_nvlink_isr(struct gk20a *g); +/* API */ +int gv100_nvlink_link_early_init(struct gk20a *g, unsigned long mask); +u32 gv100_nvlink_link_get_mode(struct gk20a *g, u32 link_id); +u32 gv100_nvlink_link_get_state(struct gk20a *g, u32 link_id); +int gv100_nvlink_link_set_mode(struct gk20a *g, u32 link_id, u32 mode); +u32 gv100_nvlink_link_get_sublink_mode(struct gk20a *g, u32 link_id, + bool is_rx_sublink); +u32 gv100_nvlink_link_get_tx_sublink_state(struct gk20a *g, u32 link_id); +u32 gv100_nvlink_link_get_rx_sublink_state(struct gk20a *g, u32 link_id); +int gv100_nvlink_link_set_sublink_mode(struct gk20a *g, u32 link_id, + bool is_rx_sublink, u32 mode); +int gv100_nvlink_interface_init(struct gk20a *g); +int gv100_nvlink_reg_init(struct gk20a *g); +int gv100_nvlink_shutdown(struct gk20a *g); +int gv100_nvlink_early_init(struct gk20a *g); +#endif diff --git a/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c b/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c index 89f3f0b9b..97ab7aab9 100644 --- a/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c +++ b/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c @@ -482,7 +482,7 @@ static int gv11b_fifo_poll_eng_ctx_status(struct gk20a *g, u32 id, eng_stat = gk20a_readl(g, fifo_engine_status_r(act_eng_id)); ctx_stat = fifo_engine_status_ctx_status_v(eng_stat); - if (gv11b_mc_is_stall_and_eng_intr_pending(g, act_eng_id)) { + if (g->ops.mc.is_stall_and_eng_intr_pending(g, act_eng_id)) { stall_intr = true; nvgpu_log(g, gpu_dbg_info | gpu_dbg_intr, "stall intr set, " diff --git a/drivers/gpu/nvgpu/gv11b/hal_gv11b.c b/drivers/gpu/nvgpu/gv11b/hal_gv11b.c index 6fdac85ad..5282af055 100644 --- a/drivers/gpu/nvgpu/gv11b/hal_gv11b.c +++ b/drivers/gpu/nvgpu/gv11b/hal_gv11b.c @@ -658,6 +658,8 @@ static const struct gpu_ops gv11b_ops = { .boot_0 = gk20a_mc_boot_0, .is_intr1_pending = mc_gp10b_is_intr1_pending, .is_intr_hub_pending = gv11b_mc_is_intr_hub_pending, + .is_stall_and_eng_intr_pending = + gv11b_mc_is_stall_and_eng_intr_pending, }, .debug = { .show_dump = gk20a_debug_show_dump, diff --git a/drivers/gpu/nvgpu/include/nvgpu/hw/gv100/hw_minion_gv100.h b/drivers/gpu/nvgpu/include/nvgpu/hw/gv100/hw_minion_gv100.h index c59a6e6bb..a57ea97ad 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/hw/gv100/hw_minion_gv100.h +++ b/drivers/gpu/nvgpu/include/nvgpu/hw/gv100/hw_minion_gv100.h @@ -892,6 +892,14 @@ static inline u32 minion_nvlink_link_intr_code_dlreq_f(void) { return 0x2U; } +static inline u32 minion_nvlink_link_intr_code_pmdisabled_v(void) +{ + return 0x00000003U; +} +static inline u32 minion_nvlink_link_intr_code_pmdisabled_f(void) +{ + return 0x3U; +} static inline u32 minion_nvlink_link_intr_subcode_f(u32 v) { return (v & 0xffU) << 8U; diff --git a/drivers/gpu/nvgpu/include/nvgpu/nvlink.h b/drivers/gpu/nvgpu/include/nvgpu/nvlink.h index 48851ff16..18e0aab67 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/nvlink.h +++ b/drivers/gpu/nvgpu/include/nvgpu/nvlink.h @@ -182,11 +182,10 @@ struct nvgpu_nvlink_dev { u8 train_at_boot; u32 ac_coupling_mask; - u32 init_disabled_links; u32 connected_links; u32 initialized_links; u32 enabled_links; - u32 topology_connected_links; + u32 init_pll_done; enum nvgpu_nvlink_speed speed; @@ -195,12 +194,18 @@ struct nvgpu_nvlink_dev { u32 hshub_config1; u32 hshub_config2; u32 hshub_config6; + + /* tlc cached errors */ + u32 tlc_rx_err_status_0[NVLINK_MAX_LINKS_SW]; + u32 tlc_rx_err_status_1[NVLINK_MAX_LINKS_SW]; + u32 tlc_tx_err_status_0[NVLINK_MAX_LINKS_SW]; + /* priv struct */ void *priv; }; -u32 nvgpu_nvlink_enumerate(struct gk20a *g); -u32 nvgpu_nvlink_train(struct gk20a *g, u32 link_id, bool from_off); -u32 nvgpu_nvlink_probe(struct gk20a *g); +int nvgpu_nvlink_enumerate(struct gk20a *g); +int nvgpu_nvlink_train(struct gk20a *g, u32 link_id, bool from_off); +int nvgpu_nvlink_probe(struct gk20a *g); #endif