diff --git a/arch/nvgpu-hal-new.yaml b/arch/nvgpu-hal-new.yaml index 61d595877..7ccf1d097 100644 --- a/arch/nvgpu-hal-new.yaml +++ b/arch/nvgpu-hal-new.yaml @@ -618,7 +618,6 @@ mc: safe: no owner: Seema K sources: [ hal/mc/mc_gm20b.c, - hal/mc/mc_gp10b.c, hal/mc/mc_gv100.c, hal/mc/mc_gv100.h, hal/mc/mc_tu104.c, diff --git a/drivers/gpu/nvgpu/Makefile b/drivers/gpu/nvgpu/Makefile index 7ed5d8da8..60f51435d 100644 --- a/drivers/gpu/nvgpu/Makefile +++ b/drivers/gpu/nvgpu/Makefile @@ -195,7 +195,6 @@ nvgpu-y += \ common/nvlink/nvlink_gv100.o \ common/nvlink/nvlink_tu104.o \ common/rc/rc.o \ - hal/mc/mc_gp10b.o \ hal/mc/mc_gv100.o \ hal/mc/mc_tu104.o \ hal/bus/bus_gk20a.o \ diff --git a/drivers/gpu/nvgpu/Makefile.sources b/drivers/gpu/nvgpu/Makefile.sources index 8ceffa0c8..83d62735e 100644 --- a/drivers/gpu/nvgpu/Makefile.sources +++ b/drivers/gpu/nvgpu/Makefile.sources @@ -256,7 +256,6 @@ srcs += hal/init/hal_gp10b.c \ hal/mm/gmmu/gmmu_gk20a.c \ hal/mm/gmmu/gmmu_gm20b.c \ hal/mc/mc_gm20b.c \ - hal/mc/mc_gp10b.c \ hal/bus/bus_gk20a.c \ hal/class/class_gm20b.c \ hal/clk/clk_gm20b.c \ diff --git a/drivers/gpu/nvgpu/common/ce/ce.c b/drivers/gpu/nvgpu/common/ce/ce.c index 1e6f558fd..59ca73161 100644 --- a/drivers/gpu/nvgpu/common/ce/ce.c +++ b/drivers/gpu/nvgpu/common/ce/ce.c @@ -26,6 +26,7 @@ #include #include #include +#include int nvgpu_ce_init_support(struct gk20a *g) { @@ -47,5 +48,9 @@ int nvgpu_ce_init_support(struct gk20a *g) g->ops.ce.init_prod_values(g); } + /** Enable interrupts at MC level */ + nvgpu_mc_intr_stall_unit_config(g, MC_INTR_UNIT_CE, MC_INTR_ENABLE); + nvgpu_mc_intr_nonstall_unit_config(g, MC_INTR_UNIT_CE, MC_INTR_ENABLE); + return 0; } diff --git a/drivers/gpu/nvgpu/common/fifo/engines.c b/drivers/gpu/nvgpu/common/fifo/engines.c index 3d4ddb918..5db89c6d0 100644 --- a/drivers/gpu/nvgpu/common/fifo/engines.c +++ b/drivers/gpu/nvgpu/common/fifo/engines.c @@ -195,7 +195,7 @@ u32 nvgpu_engine_act_interrupt_mask(struct gk20a *g, u32 act_eng_id) return 0; } -u32 nvgpu_engine_interrupt_mask(struct gk20a *g) +u32 nvgpu_gr_engine_interrupt_mask(struct gk20a *g) { u32 eng_intr_mask = 0; unsigned int i; @@ -204,9 +204,35 @@ u32 nvgpu_engine_interrupt_mask(struct gk20a *g) for (i = 0; i < g->fifo.num_engines; i++) { u32 intr_mask; + active_engine_id = g->fifo.active_engines_list[i]; intr_mask = g->fifo.engine_info[active_engine_id].intr_mask; engine_enum = g->fifo.engine_info[active_engine_id].engine_enum; + + if (engine_enum != NVGPU_ENGINE_GR) { + continue; + } + + eng_intr_mask |= intr_mask; + } + + return eng_intr_mask; +} + +u32 nvgpu_ce_engine_interrupt_mask(struct gk20a *g) +{ + u32 eng_intr_mask = 0; + unsigned int i; + u32 active_engine_id = 0; + enum nvgpu_fifo_engine engine_enum; + + for (i = 0; i < g->fifo.num_engines; i++) { + u32 intr_mask; + + active_engine_id = g->fifo.active_engines_list[i]; + intr_mask = g->fifo.engine_info[active_engine_id].intr_mask; + engine_enum = g->fifo.engine_info[active_engine_id].engine_enum; + if (((engine_enum == NVGPU_ENGINE_GRCE) || (engine_enum == NVGPU_ENGINE_ASYNC_CE)) && ((g->ops.ce.isr_stall == NULL) || diff --git a/drivers/gpu/nvgpu/common/fifo/fifo.c b/drivers/gpu/nvgpu/common/fifo/fifo.c index cffb99990..8b232fe09 100644 --- a/drivers/gpu/nvgpu/common/fifo/fifo.c +++ b/drivers/gpu/nvgpu/common/fifo/fifo.c @@ -36,6 +36,7 @@ #include #include #include +#include void nvgpu_fifo_cleanup_sw_common(struct gk20a *g) { @@ -248,6 +249,18 @@ const char *nvgpu_fifo_decode_pbdma_ch_eng_status(u32 index) } } +static void disable_fifo_interrupts(struct gk20a *g) +{ + /** Disable fifo intr */ + g->ops.fifo.intr_0_enable(g, false); + g->ops.fifo.intr_1_enable(g, false); + + nvgpu_mc_intr_stall_unit_config(g, MC_INTR_UNIT_FIFO, + MC_INTR_DISABLE); + nvgpu_mc_intr_nonstall_unit_config(g, MC_INTR_UNIT_FIFO, + MC_INTR_DISABLE); +} + int nvgpu_fifo_suspend(struct gk20a *g) { nvgpu_log_fn(g, " "); @@ -256,9 +269,7 @@ int nvgpu_fifo_suspend(struct gk20a *g) g->ops.fifo.bar1_snooping_disable(g); } - /* disable fifo intr */ - g->ops.fifo.intr_0_enable(g, false); - g->ops.fifo.intr_1_enable(g, false); + disable_fifo_interrupts(g); nvgpu_log_fn(g, "done"); return 0; diff --git a/drivers/gpu/nvgpu/common/gr/gr.c b/drivers/gpu/nvgpu/common/gr/gr.c index ee831fc01..57fb6bdf2 100644 --- a/drivers/gpu/nvgpu/common/gr/gr.c +++ b/drivers/gpu/nvgpu/common/gr/gr.c @@ -151,6 +151,21 @@ void nvgpu_gr_init(struct gk20a *g) (void)nvgpu_cond_init(&g->gr->init_wq); } +static void disable_gr_interrupts(struct gk20a *g) +{ + /** Disable gr intr */ + g->ops.gr.intr.enable_interrupts(g, false); + + /** Disable all exceptions */ + g->ops.gr.intr.enable_exceptions(g, g->gr->config, false); + + /** Disable interrupts at MC level */ + nvgpu_mc_intr_stall_unit_config(g, MC_INTR_UNIT_GR, + MC_INTR_DISABLE); + nvgpu_mc_intr_nonstall_unit_config(g, MC_INTR_UNIT_GR, + MC_INTR_DISABLE); +} + int nvgpu_gr_suspend(struct gk20a *g) { int ret = 0; @@ -165,11 +180,7 @@ int nvgpu_gr_suspend(struct gk20a *g) /* Disable fifo access */ g->ops.gr.init.fifo_access(g, false); - /* disable gr intr */ - g->ops.gr.intr.enable_interrupts(g, false); - - /* disable all exceptions */ - g->ops.gr.intr.enable_exceptions(g, g->gr->config, false); + disable_gr_interrupts(g); g->ops.gr.intr.flush_channel_tlb(g); @@ -179,6 +190,16 @@ int nvgpu_gr_suspend(struct gk20a *g) return ret; } +static void enable_gr_interrupts(struct gk20a *g) +{ + /** Enable interrupts at MC level */ + nvgpu_mc_intr_stall_unit_config(g, MC_INTR_UNIT_GR, MC_INTR_ENABLE); + nvgpu_mc_intr_nonstall_unit_config(g, MC_INTR_UNIT_GR, MC_INTR_ENABLE); + + /** Enable interrupts */ + g->ops.gr.intr.enable_interrupts(g, true); +} + static int gr_init_setup_hw(struct gk20a *g) { struct nvgpu_gr *gr = g->gr; @@ -207,26 +228,25 @@ static int gr_init_setup_hw(struct gk20a *g) /* TBD: reload gr ucode when needed */ - /* enable interrupts */ - g->ops.gr.intr.enable_interrupts(g, true); + enable_gr_interrupts(g); - /* enable fecs error interrupts */ + /** Enable fecs error interrupts */ g->ops.gr.falcon.fecs_host_int_enable(g); g->ops.gr.intr.enable_hww_exceptions(g); g->ops.gr.intr.set_hww_esr_report_mask(g); - /* enable TPC exceptions per GPC */ + /** Enable TPC exceptions per GPC */ g->ops.gr.intr.enable_gpc_exceptions(g, gr->config); + /** TBD: enable per BE exceptions */ + /* enable ECC for L1/SM */ if (g->ops.gr.init.ecc_scrub_reg != NULL) { g->ops.gr.init.ecc_scrub_reg(g, gr->config); } - /* TBD: enable per BE exceptions */ - - /* reset and enable exceptions */ + /** Reset and enable exceptions */ g->ops.gr.intr.enable_exceptions(g, gr->config, true); #ifdef CONFIG_NVGPU_GRAPHICS @@ -515,8 +535,7 @@ static int gr_init_reset_enable_hw(struct gk20a *g) nvgpu_log_fn(g, " "); - /* enable interrupts */ - g->ops.gr.intr.enable_interrupts(g, true); + enable_gr_interrupts(g); /* load non_ctx init */ for (i = 0; i < sw_non_ctx_load->count; i++) { diff --git a/drivers/gpu/nvgpu/common/init/nvgpu_init.c b/drivers/gpu/nvgpu/common/init/nvgpu_init.c index 1182c139e..04eb86d67 100644 --- a/drivers/gpu/nvgpu/common/init/nvgpu_init.c +++ b/drivers/gpu/nvgpu/common/init/nvgpu_init.c @@ -65,13 +65,8 @@ void nvgpu_check_gpu_state(struct gk20a *g) static void gk20a_mask_interrupts(struct gk20a *g) { - if (g->ops.mc.intr_mask != NULL) { - g->ops.mc.intr_mask(g); - } - - if (g->ops.mc.log_pending_intrs != NULL) { - g->ops.mc.log_pending_intrs(g); - } + nvgpu_mc_intr_mask(g); + nvgpu_mc_log_pending_intrs(g); } #ifndef CONFIG_NVGPU_RECOVERY @@ -474,6 +469,22 @@ static int nvgpu_init_syncpt_mem(struct gk20a *g) return 0; } +static int nvgpu_init_interrupt_setup(struct gk20a *g) +{ + /** + * Disable all interrupts at the start. + */ + nvgpu_mc_intr_mask(g); + + /** + * For certain chips like gm20b, there is global interrupt control in + * registers mc_intr_en_*_r. Program them here upfront. + */ + nvgpu_mc_intr_enable(g); + + return 0; +} + typedef int (*nvgpu_init_func_t)(struct gk20a *g); struct nvgpu_init_table_t { nvgpu_init_func_t func; @@ -522,6 +533,7 @@ int nvgpu_finalize_poweron(struct gk20a *g) #ifdef CONFIG_NVGPU_DGPU NVGPU_INIT_TABLE_ENTRY(g->ops.bios.bios_sw_init, NO_FLAG), #endif + NVGPU_INIT_TABLE_ENTRY(&nvgpu_init_interrupt_setup, NO_FLAG), NVGPU_INIT_TABLE_ENTRY(g->ops.bus.init_hw, NO_FLAG), NVGPU_INIT_TABLE_ENTRY(g->ops.priv_ring.enable_priv_ring, NO_FLAG), @@ -548,7 +560,6 @@ int nvgpu_finalize_poweron(struct gk20a *g) NVGPU_INIT_TABLE_ENTRY(g->ops.fifo.fifo_init_support, NO_FLAG), NVGPU_INIT_TABLE_ENTRY(g->ops.therm.elcg_init_idle_filters, NO_FLAG), - NVGPU_INIT_TABLE_ENTRY(g->ops.mc.intr_enable, NO_FLAG), NVGPU_INIT_TABLE_ENTRY(&nvgpu_init_power_gate, NO_FLAG), NVGPU_INIT_TABLE_ENTRY(&nvgpu_init_acquire_tpc_pg_lock, NO_FLAG), #ifdef CONFIG_NVGPU_DEBUGGER diff --git a/drivers/gpu/nvgpu/common/ltc/ltc.c b/drivers/gpu/nvgpu/common/ltc/ltc.c index c16f59b96..212e12cd1 100644 --- a/drivers/gpu/nvgpu/common/ltc/ltc.c +++ b/drivers/gpu/nvgpu/common/ltc/ltc.c @@ -73,6 +73,12 @@ int nvgpu_init_ltc_support(struct gk20a *g) } } + if (g->ops.ltc.intr.configure != NULL) { + nvgpu_mc_intr_stall_unit_config(g, MC_INTR_UNIT_LTC, + MC_INTR_ENABLE); + g->ops.ltc.intr.configure(g); + } + return 0; } diff --git a/drivers/gpu/nvgpu/common/mc/mc.c b/drivers/gpu/nvgpu/common/mc/mc.c index 002ed49bb..7a1e4b17b 100644 --- a/drivers/gpu/nvgpu/common/mc/mc.c +++ b/drivers/gpu/nvgpu/common/mc/mc.c @@ -64,3 +64,86 @@ void nvgpu_wait_for_deferred_interrupts(struct gk20a *g) nvgpu_atomic_read(&g->mc.sw_irq_nonstall_last_handled)) <= 0, 0U); } + +void nvgpu_mc_intr_mask(struct gk20a *g) +{ + unsigned long flags = 0; + + if (g->ops.mc.intr_mask != NULL) { + nvgpu_spinlock_irqsave(&g->mc.intr_lock, flags); + g->ops.mc.intr_mask(g); + nvgpu_spinunlock_irqrestore(&g->mc.intr_lock, flags); + } +} + +void nvgpu_mc_log_pending_intrs(struct gk20a *g) +{ + if (g->ops.mc.log_pending_intrs != NULL) { + g->ops.mc.log_pending_intrs(g); + } +} + +void nvgpu_mc_intr_enable(struct gk20a *g) +{ + unsigned long flags = 0; + + if (g->ops.mc.intr_enable != NULL) { + nvgpu_spinlock_irqsave(&g->mc.intr_lock, flags); + g->ops.mc.intr_enable(g); + nvgpu_spinunlock_irqrestore(&g->mc.intr_lock, flags); + } +} + +void nvgpu_mc_intr_stall_unit_config(struct gk20a *g, u32 unit, bool enable) +{ + unsigned long flags = 0; + + nvgpu_spinlock_irqsave(&g->mc.intr_lock, flags); + g->ops.mc.intr_stall_unit_config(g, unit, enable); + nvgpu_spinunlock_irqrestore(&g->mc.intr_lock, flags); +} + +void nvgpu_mc_intr_nonstall_unit_config(struct gk20a *g, u32 unit, bool enable) +{ + unsigned long flags = 0; + + nvgpu_spinlock_irqsave(&g->mc.intr_lock, flags); + g->ops.mc.intr_nonstall_unit_config(g, unit, enable); + nvgpu_spinunlock_irqrestore(&g->mc.intr_lock, flags); +} + +void nvgpu_mc_intr_stall_pause(struct gk20a *g) +{ + unsigned long flags = 0; + + nvgpu_spinlock_irqsave(&g->mc.intr_lock, flags); + g->ops.mc.intr_stall_pause(g); + nvgpu_spinunlock_irqrestore(&g->mc.intr_lock, flags); +} + +void nvgpu_mc_intr_stall_resume(struct gk20a *g) +{ + unsigned long flags = 0; + + nvgpu_spinlock_irqsave(&g->mc.intr_lock, flags); + g->ops.mc.intr_stall_resume(g); + nvgpu_spinunlock_irqrestore(&g->mc.intr_lock, flags); +} + +void nvgpu_mc_intr_nonstall_pause(struct gk20a *g) +{ + unsigned long flags = 0; + + nvgpu_spinlock_irqsave(&g->mc.intr_lock, flags); + g->ops.mc.intr_nonstall_pause(g); + nvgpu_spinunlock_irqrestore(&g->mc.intr_lock, flags); +} + +void nvgpu_mc_intr_nonstall_resume(struct gk20a *g) +{ + unsigned long flags = 0; + + nvgpu_spinlock_irqsave(&g->mc.intr_lock, flags); + g->ops.mc.intr_nonstall_resume(g); + nvgpu_spinunlock_irqrestore(&g->mc.intr_lock, flags); +} diff --git a/drivers/gpu/nvgpu/common/nvlink/nvlink_gv100.c b/drivers/gpu/nvgpu/common/nvlink/nvlink_gv100.c index ddea0eb75..c00315011 100644 --- a/drivers/gpu/nvgpu/common/nvlink/nvlink_gv100.c +++ b/drivers/gpu/nvgpu/common/nvlink/nvlink_gv100.c @@ -35,6 +35,7 @@ #include #include #include +#include #include "nvlink_gv100.h" @@ -844,6 +845,9 @@ int gv100_nvlink_early_init(struct gk20a *g) mc_reset_nvlink_mask); g->ops.mc.reset(g, mc_reset_nvlink_mask); + nvgpu_mc_intr_stall_unit_config(g, MC_INTR_UNIT_NVLINK, + MC_INTR_ENABLE); + err = g->ops.nvlink.discover_link(g); if ((err != 0) || (g->nvlink.discovered_links == 0U)) { nvgpu_err(g, "No links available"); diff --git a/drivers/gpu/nvgpu/hal/bus/bus_gk20a_fusa.c b/drivers/gpu/nvgpu/hal/bus/bus_gk20a_fusa.c index 7d38ce660..e641e88b5 100644 --- a/drivers/gpu/nvgpu/hal/bus/bus_gk20a_fusa.c +++ b/drivers/gpu/nvgpu/hal/bus/bus_gk20a_fusa.c @@ -28,6 +28,7 @@ #include #include #include +#include #include "bus_gk20a.h" @@ -43,7 +44,9 @@ int gk20a_bus_init_hw(struct gk20a *g) bus_intr_en_0_pri_timeout_m(); } - gk20a_writel(g, bus_intr_en_0_r(), intr_en_mask); + nvgpu_mc_intr_stall_unit_config(g, MC_INTR_UNIT_BUS, MC_INTR_ENABLE); + + nvgpu_writel(g, bus_intr_en_0_r(), intr_en_mask); if (g->ops.bus.configure_debug_bus != NULL) { g->ops.bus.configure_debug_bus(g); @@ -57,7 +60,7 @@ void gk20a_bus_isr(struct gk20a *g) u32 val; u32 err_type = GPU_HOST_INVALID_ERROR; - val = gk20a_readl(g, bus_intr_0_r()); + val = nvgpu_readl(g, bus_intr_0_r()); if ((val & (bus_intr_0_pri_squash_m() | bus_intr_0_pri_fecserr_m() | @@ -83,5 +86,5 @@ void gk20a_bus_isr(struct gk20a *g) } nvgpu_report_host_err(g, NVGPU_ERR_MODULE_HOST, 0, err_type, val); - gk20a_writel(g, bus_intr_0_r(), val); + nvgpu_writel(g, bus_intr_0_r(), val); } diff --git a/drivers/gpu/nvgpu/hal/fb/intr/fb_intr_gv11b_fusa.c b/drivers/gpu/nvgpu/hal/fb/intr/fb_intr_gv11b_fusa.c index 687ed59cb..4d4575948 100644 --- a/drivers/gpu/nvgpu/hal/fb/intr/fb_intr_gv11b_fusa.c +++ b/drivers/gpu/nvgpu/hal/fb/intr/fb_intr_gv11b_fusa.c @@ -25,6 +25,7 @@ #include #include #include +#include #include "hal/fb/fb_gv11b.h" #include "hal/fb/fb_mmu_fault_gv11b.h" @@ -34,7 +35,6 @@ #include - void gv11b_fb_intr_enable(struct gk20a *g) { u32 mask; @@ -48,6 +48,8 @@ void gv11b_fb_intr_enable(struct gk20a *g) #endif fb_niso_intr_en_set_mmu_ecc_uncorrected_error_notify_m(); + nvgpu_mc_intr_stall_unit_config(g, MC_INTR_UNIT_HUB, MC_INTR_ENABLE); + nvgpu_writel(g, fb_niso_intr_en_set_r(0), mask); } @@ -65,6 +67,8 @@ void gv11b_fb_intr_disable(struct gk20a *g) fb_niso_intr_en_set_mmu_ecc_uncorrected_error_notify_m(); nvgpu_writel(g, fb_niso_intr_en_clr_r(0), mask); + + nvgpu_mc_intr_stall_unit_config(g, MC_INTR_UNIT_HUB, MC_INTR_DISABLE); } void gv11b_fb_intr_isr(struct gk20a *g) diff --git a/drivers/gpu/nvgpu/hal/fbpa/fbpa_tu104.c b/drivers/gpu/nvgpu/hal/fbpa/fbpa_tu104.c index 63312d791..4fffe243d 100644 --- a/drivers/gpu/nvgpu/hal/fbpa/fbpa_tu104.c +++ b/drivers/gpu/nvgpu/hal/fbpa/fbpa_tu104.c @@ -27,6 +27,7 @@ #include #include #include +#include #include "hal/fbpa/fbpa_tu104.h" @@ -34,12 +35,15 @@ int tu104_fbpa_init(struct gk20a *g) { u32 val; - val = gk20a_readl(g, fbpa_ecc_intr_ctrl_r()); + val = nvgpu_readl(g, fbpa_ecc_intr_ctrl_r()); val |= fbpa_ecc_intr_ctrl_sec_intr_en_enabled_f() | fbpa_ecc_intr_ctrl_ded_intr_en_enabled_f(); - gk20a_writel(g, fbpa_ecc_intr_ctrl_r(), val); + + nvgpu_mc_intr_stall_unit_config(g, MC_INTR_UNIT_FBPA, MC_INTR_ENABLE); + + nvgpu_writel(g, fbpa_ecc_intr_ctrl_r(), val); /* read back broadcast register */ - (void) gk20a_readl(g, fbpa_ecc_intr_ctrl_r()); + (void) nvgpu_readl(g, fbpa_ecc_intr_ctrl_r()); return 0; } @@ -51,7 +55,7 @@ static void tu104_fbpa_handle_ecc_intr(struct gk20a *g, u32 offset = nvgpu_get_litter_value(g, GPU_LIT_FBPA_STRIDE) * fbpa_id; u32 cnt_idx = fbpa_id * 2U + subp_id; - status = gk20a_readl(g, offset + fbpa_0_ecc_status_r(subp_id)); + status = nvgpu_readl(g, offset + fbpa_0_ecc_status_r(subp_id)); if ((status & fbpa_0_ecc_status_sec_counter_overflow_pending_f()) != 0U) { nvgpu_err(g, "fbpa %u subp %u ecc sec counter overflow", @@ -64,20 +68,20 @@ static void tu104_fbpa_handle_ecc_intr(struct gk20a *g, } if ((status & fbpa_0_ecc_status_sec_intr_pending_f()) != 0U) { - sec_cnt = gk20a_readl(g, + sec_cnt = nvgpu_readl(g, offset + fbpa_0_ecc_sec_count_r(subp_id)); - gk20a_writel(g, offset + fbpa_0_ecc_sec_count_r(subp_id), 0u); + nvgpu_writel(g, offset + fbpa_0_ecc_sec_count_r(subp_id), 0u); g->ecc.fbpa.fbpa_ecc_sec_err_count[cnt_idx].counter += sec_cnt; } if ((status & fbpa_0_ecc_status_ded_intr_pending_f()) != 0U) { - ded_cnt = gk20a_readl(g, + ded_cnt = nvgpu_readl(g, offset + fbpa_0_ecc_ded_count_r(subp_id)); - gk20a_writel(g, offset + fbpa_0_ecc_ded_count_r(subp_id), 0u); + nvgpu_writel(g, offset + fbpa_0_ecc_ded_count_r(subp_id), 0u); g->ecc.fbpa.fbpa_ecc_ded_err_count[cnt_idx].counter += ded_cnt; } - gk20a_writel(g, offset + fbpa_0_ecc_status_r(subp_id), status); + nvgpu_writel(g, offset + fbpa_0_ecc_status_r(subp_id), status); } void tu104_fbpa_handle_intr(struct gk20a *g, u32 fbpa_id) @@ -90,7 +94,7 @@ void tu104_fbpa_handle_intr(struct gk20a *g, u32 fbpa_id) offset = nvgpu_get_litter_value(g, GPU_LIT_FBPA_STRIDE) * fbpa_id; - status = gk20a_readl(g, offset + fbpa_0_intr_status_r()); + status = nvgpu_readl(g, offset + fbpa_0_intr_status_r()); if ((status & (ecc_subp0_mask | ecc_subp1_mask)) == 0U) { nvgpu_err(g, "unknown interrupt fbpa %u status %08x", fbpa_id, status); diff --git a/drivers/gpu/nvgpu/hal/fifo/fifo_gk20a.c b/drivers/gpu/nvgpu/hal/fifo/fifo_gk20a.c index 183b0dc9c..edaea4e94 100644 --- a/drivers/gpu/nvgpu/hal/fifo/fifo_gk20a.c +++ b/drivers/gpu/nvgpu/hal/fifo/fifo_gk20a.c @@ -34,6 +34,16 @@ #include +static void enable_fifo_interrupts(struct gk20a *g) +{ + nvgpu_mc_intr_stall_unit_config(g, MC_INTR_UNIT_FIFO, MC_INTR_ENABLE); + nvgpu_mc_intr_nonstall_unit_config(g, MC_INTR_UNIT_FIFO, + MC_INTR_ENABLE); + + g->ops.fifo.intr_0_enable(g, true); + g->ops.fifo.intr_1_enable(g, true); +} + int gk20a_init_fifo_reset_enable_hw(struct gk20a *g) { u32 timeout; @@ -55,8 +65,7 @@ int gk20a_init_fifo_reset_enable_hw(struct gk20a *g) g->ops.pbdma.setup_hw(g); - g->ops.fifo.intr_0_enable(g, true); - g->ops.fifo.intr_1_enable(g, true); + enable_fifo_interrupts(g); nvgpu_log_fn(g, "done"); diff --git a/drivers/gpu/nvgpu/hal/fifo/fifo_gv11b_fusa.c b/drivers/gpu/nvgpu/hal/fifo/fifo_gv11b_fusa.c index 721b869c2..b6b3e11c8 100644 --- a/drivers/gpu/nvgpu/hal/fifo/fifo_gv11b_fusa.c +++ b/drivers/gpu/nvgpu/hal/fifo/fifo_gv11b_fusa.c @@ -36,6 +36,17 @@ #include "fifo_gv11b.h" +static void enable_fifo_interrupts(struct gk20a *g) +{ + nvgpu_mc_intr_stall_unit_config(g, MC_INTR_UNIT_FIFO, + MC_INTR_ENABLE); + nvgpu_mc_intr_nonstall_unit_config(g, MC_INTR_UNIT_FIFO, + MC_INTR_ENABLE); + + g->ops.fifo.intr_0_enable(g, true); + g->ops.fifo.intr_1_enable(g, true); +} + int gv11b_init_fifo_reset_enable_hw(struct gk20a *g) { u32 timeout; @@ -65,8 +76,7 @@ int gv11b_init_fifo_reset_enable_hw(struct gk20a *g) g->ops.pbdma.setup_hw(g); - g->ops.fifo.intr_0_enable(g, true); - g->ops.fifo.intr_1_enable(g, true); + enable_fifo_interrupts(g); nvgpu_log_fn(g, "done"); diff --git a/drivers/gpu/nvgpu/hal/init/hal_gm20b.c b/drivers/gpu/nvgpu/hal/init/hal_gm20b.c index a24d3f95a..29b8bf086 100644 --- a/drivers/gpu/nvgpu/hal/init/hal_gm20b.c +++ b/drivers/gpu/nvgpu/hal/init/hal_gm20b.c @@ -963,9 +963,8 @@ static const struct gpu_ops gm20b_ops = { .get_chip_details = gm20b_get_chip_details, .intr_mask = gm20b_mc_intr_mask, .intr_enable = gm20b_mc_intr_enable, -#ifdef CONFIG_NVGPU_LS_PMU - .intr_pmu_unit_config = gm20b_mc_intr_pmu_unit_config, -#endif + .intr_stall_unit_config = gm20b_mc_intr_stall_unit_config, + .intr_nonstall_unit_config = gm20b_mc_intr_nonstall_unit_config, .isr_stall = gm20b_mc_isr_stall, .intr_stall = gm20b_mc_intr_stall, .intr_stall_pause = gm20b_mc_intr_stall_pause, diff --git a/drivers/gpu/nvgpu/hal/init/hal_gp10b.c b/drivers/gpu/nvgpu/hal/init/hal_gp10b.c index b3742d36c..af752beb4 100644 --- a/drivers/gpu/nvgpu/hal/init/hal_gp10b.c +++ b/drivers/gpu/nvgpu/hal/init/hal_gp10b.c @@ -1060,10 +1060,9 @@ static const struct gpu_ops gp10b_ops = { .mc = { .get_chip_details = gm20b_get_chip_details, .intr_mask = mc_gp10b_intr_mask, - .intr_enable = mc_gp10b_intr_enable, -#ifdef CONFIG_NVGPU_LS_PMU - .intr_pmu_unit_config = mc_gp10b_intr_pmu_unit_config, -#endif + .intr_enable = NULL, + .intr_stall_unit_config = mc_gp10b_intr_stall_unit_config, + .intr_nonstall_unit_config = mc_gp10b_intr_nonstall_unit_config, .isr_stall = mc_gp10b_isr_stall, .intr_stall = mc_gp10b_intr_stall, .intr_stall_pause = mc_gp10b_intr_stall_pause, diff --git a/drivers/gpu/nvgpu/hal/init/hal_gv11b.c b/drivers/gpu/nvgpu/hal/init/hal_gv11b.c index 061ea89c5..73cccac22 100644 --- a/drivers/gpu/nvgpu/hal/init/hal_gv11b.c +++ b/drivers/gpu/nvgpu/hal/init/hal_gv11b.c @@ -1269,10 +1269,9 @@ static const struct gpu_ops gv11b_ops = { .mc = { .get_chip_details = gm20b_get_chip_details, .intr_mask = mc_gp10b_intr_mask, - .intr_enable = mc_gv11b_intr_enable, -#ifdef CONFIG_NVGPU_LS_PMU - .intr_pmu_unit_config = mc_gp10b_intr_pmu_unit_config, -#endif + .intr_enable = NULL, + .intr_stall_unit_config = mc_gp10b_intr_stall_unit_config, + .intr_nonstall_unit_config = mc_gp10b_intr_nonstall_unit_config, .isr_stall = mc_gp10b_isr_stall, .intr_stall = mc_gp10b_intr_stall, .intr_stall_pause = mc_gp10b_intr_stall_pause, diff --git a/drivers/gpu/nvgpu/hal/init/hal_tu104.c b/drivers/gpu/nvgpu/hal/init/hal_tu104.c index 497f72c6c..4de385f90 100644 --- a/drivers/gpu/nvgpu/hal/init/hal_tu104.c +++ b/drivers/gpu/nvgpu/hal/init/hal_tu104.c @@ -1292,11 +1292,10 @@ static const struct gpu_ops tu104_ops = { #endif .mc = { .get_chip_details = gm20b_get_chip_details, - .intr_enable = intr_tu104_enable, .intr_mask = intr_tu104_mask, -#ifdef CONFIG_NVGPU_LS_PMU - .intr_pmu_unit_config = mc_gp10b_intr_pmu_unit_config, -#endif + .intr_enable = NULL, + .intr_stall_unit_config = intr_tu104_stall_unit_config, + .intr_nonstall_unit_config = intr_tu104_nonstall_unit_config, .isr_stall = mc_gp10b_isr_stall, .intr_stall = intr_tu104_stall, .intr_stall_pause = intr_tu104_stall_pause, diff --git a/drivers/gpu/nvgpu/hal/ltc/ltc_gm20b.c b/drivers/gpu/nvgpu/hal/ltc/ltc_gm20b.c index d5ff4d5f1..2a0b72330 100644 --- a/drivers/gpu/nvgpu/hal/ltc/ltc_gm20b.c +++ b/drivers/gpu/nvgpu/hal/ltc/ltc_gm20b.c @@ -35,6 +35,7 @@ #include #include #include +#include #include @@ -65,8 +66,6 @@ void gm20b_ltc_init_fs_state(struct gk20a *g) gk20a_writel(g, ltc_ltcs_ltss_dstg_cfg0_r(), gk20a_readl(g, ltc_ltc0_lts0_dstg_cfg0_r()) | ltc_ltcs_ltss_dstg_cfg0_vdc_4to2_disable_m()); - - g->ops.ltc.intr.configure(g); } /* diff --git a/drivers/gpu/nvgpu/hal/ltc/ltc_gv11b_fusa.c b/drivers/gpu/nvgpu/hal/ltc/ltc_gv11b_fusa.c index c2882fc07..698a90037 100644 --- a/drivers/gpu/nvgpu/hal/ltc/ltc_gv11b_fusa.c +++ b/drivers/gpu/nvgpu/hal/ltc/ltc_gv11b_fusa.c @@ -26,6 +26,7 @@ #include #include #include +#include #include "ltc_gv11b.h" @@ -53,9 +54,6 @@ void gv11b_ltc_init_fs_state(struct gk20a *g) g->ltc->slices_per_ltc = ltc_ltcs_ltss_cbc_param_slices_per_ltc_v(reg);; g->ltc->cacheline_size = line_size << ltc_ltcs_ltss_cbc_param_cache_line_size_v(reg); - - g->ops.ltc.intr.configure(g); - } int gv11b_lts_ecc_init(struct gk20a *g) diff --git a/drivers/gpu/nvgpu/hal/mc/mc_gm20b.c b/drivers/gpu/nvgpu/hal/mc/mc_gm20b.c index 8783362c6..d56cfcea3 100644 --- a/drivers/gpu/nvgpu/hal/mc/mc_gm20b.c +++ b/drivers/gpu/nvgpu/hal/mc/mc_gm20b.c @@ -96,43 +96,80 @@ void gm20b_mc_intr_mask(struct gk20a *g) mc_intr_en_0_inta_disabled_f()); nvgpu_writel(g, mc_intr_en_1_r(), mc_intr_en_1_inta_disabled_f()); + nvgpu_writel(g, mc_intr_mask_0_r(), 0); + nvgpu_writel(g, mc_intr_mask_1_r(), 0); } -int gm20b_mc_intr_enable(struct gk20a *g) +void gm20b_mc_intr_enable(struct gk20a *g) { - u32 eng_intr_mask = nvgpu_engine_interrupt_mask(g); - - nvgpu_writel(g, mc_intr_mask_1_r(), - mc_intr_pfifo_pending_f() - | eng_intr_mask); nvgpu_writel(g, mc_intr_en_1_r(), mc_intr_en_1_inta_hardware_f()); - nvgpu_writel(g, mc_intr_mask_0_r(), - mc_intr_pfifo_pending_f() - | mc_intr_priv_ring_pending_f() - | mc_intr_ltc_pending_f() - | mc_intr_pbus_pending_f() - | eng_intr_mask); nvgpu_writel(g, mc_intr_en_0_r(), mc_intr_en_0_inta_hardware_f()); - - return 0; } -void gm20b_mc_intr_pmu_unit_config(struct gk20a *g, bool enable) +static u32 gm20b_mc_intr_pending_f(struct gk20a *g, u32 unit) { + u32 intr_pending_f = 0; + + switch (unit) { + case MC_INTR_UNIT_BUS: + intr_pending_f = mc_intr_pbus_pending_f(); + break; + case MC_INTR_UNIT_PRIV_RING: + intr_pending_f = mc_intr_priv_ring_pending_f(); + break; + case MC_INTR_UNIT_FIFO: + intr_pending_f = mc_intr_pfifo_pending_f(); + break; + case MC_INTR_UNIT_LTC: + intr_pending_f = mc_intr_ltc_pending_f(); + break; + case MC_INTR_UNIT_GR: + intr_pending_f = nvgpu_gr_engine_interrupt_mask(g); + break; + case MC_INTR_UNIT_PMU: + intr_pending_f = mc_intr_mask_0_pmu_enabled_f(); + break; + case MC_INTR_UNIT_CE: + intr_pending_f = nvgpu_ce_engine_interrupt_mask(g); + break; + default: + nvgpu_err(g, "Invalid MC interrupt unit specified !!!"); + break; + } + + return intr_pending_f; +} + +void gm20b_mc_intr_stall_unit_config(struct gk20a *g, u32 unit, bool enable) +{ + u32 unit_pending_f = gm20b_mc_intr_pending_f(g, unit); + if (enable) { nvgpu_writel(g, mc_intr_mask_0_r(), nvgpu_readl(g, mc_intr_mask_0_r()) | - mc_intr_mask_0_pmu_enabled_f()); + unit_pending_f); } else { nvgpu_writel(g, mc_intr_mask_0_r(), nvgpu_readl(g, mc_intr_mask_0_r()) & - ~mc_intr_mask_0_pmu_enabled_f()); + ~unit_pending_f); + } +} + +void gm20b_mc_intr_nonstall_unit_config(struct gk20a *g, u32 unit, bool enable) +{ + u32 unit_pending_f = gm20b_mc_intr_pending_f(g, unit); + + if (enable) { + nvgpu_writel(g, mc_intr_mask_1_r(), + nvgpu_readl(g, mc_intr_mask_1_r()) | + unit_pending_f); + } else { nvgpu_writel(g, mc_intr_mask_1_r(), nvgpu_readl(g, mc_intr_mask_1_r()) & - ~mc_intr_mask_1_pmu_enabled_f()); + ~unit_pending_f); } } diff --git a/drivers/gpu/nvgpu/hal/mc/mc_gm20b.h b/drivers/gpu/nvgpu/hal/mc/mc_gm20b.h index bb0e701d7..709d67f66 100644 --- a/drivers/gpu/nvgpu/hal/mc/mc_gm20b.h +++ b/drivers/gpu/nvgpu/hal/mc/mc_gm20b.h @@ -43,8 +43,11 @@ bool gm20b_mc_is_enabled(struct gk20a *g, enum nvgpu_unit unit); #ifdef CONFIG_NVGPU_HAL_NON_FUSA void gm20b_mc_intr_mask(struct gk20a *g); -int gm20b_mc_intr_enable(struct gk20a *g); -void gm20b_mc_intr_pmu_unit_config(struct gk20a *g, bool enable); +void gm20b_mc_intr_enable(struct gk20a *g); +void gm20b_mc_intr_stall_unit_config(struct gk20a *g, u32 unit, + bool enable); +void gm20b_mc_intr_nonstall_unit_config(struct gk20a *g, u32 unit, + bool enable); void gm20b_mc_isr_stall(struct gk20a *g); u32 gm20b_mc_intr_stall(struct gk20a *g); void gm20b_mc_intr_stall_pause(struct gk20a *g); diff --git a/drivers/gpu/nvgpu/hal/mc/mc_gp10b.c b/drivers/gpu/nvgpu/hal/mc/mc_gp10b.c deleted file mode 100644 index e31114359..000000000 --- a/drivers/gpu/nvgpu/hal/mc/mc_gp10b.c +++ /dev/null @@ -1,86 +0,0 @@ -/* - * GP10B master - * - * Copyright (c) 2014-2019, NVIDIA CORPORATION. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - */ - -#include -#include -#include -#include -#include -#include - -#include "mc_gp10b.h" - -#include - -#include - -int mc_gp10b_intr_enable(struct gk20a *g) -{ - u32 eng_intr_mask = nvgpu_engine_interrupt_mask(g); - - nvgpu_writel(g, mc_intr_en_clear_r(NVGPU_MC_INTR_STALLING), - U32_MAX); - g->mc.intr_mask_restore[NVGPU_MC_INTR_STALLING] = - mc_intr_pfifo_pending_f() | - mc_intr_priv_ring_pending_f() | - mc_intr_pbus_pending_f() | - mc_intr_ltc_pending_f() | - mc_intr_replayable_fault_pending_f() | - eng_intr_mask; - nvgpu_writel(g, mc_intr_en_set_r(NVGPU_MC_INTR_STALLING), - g->mc.intr_mask_restore[NVGPU_MC_INTR_STALLING]); - - nvgpu_writel(g, mc_intr_en_clear_r(NVGPU_MC_INTR_NONSTALLING), - U32_MAX); - g->mc.intr_mask_restore[NVGPU_MC_INTR_NONSTALLING] = - mc_intr_pfifo_pending_f() | - eng_intr_mask; - nvgpu_writel(g, mc_intr_en_set_r(NVGPU_MC_INTR_NONSTALLING), - g->mc.intr_mask_restore[NVGPU_MC_INTR_NONSTALLING]); - - return 0; -} - -void mc_gp10b_intr_pmu_unit_config(struct gk20a *g, bool enable) -{ - u32 reg = 0U; - - if (enable) { - reg = mc_intr_en_set_r(NVGPU_MC_INTR_STALLING); - g->mc.intr_mask_restore[NVGPU_MC_INTR_STALLING] |= - mc_intr_pmu_pending_f(); - nvgpu_writel(g, reg, mc_intr_pmu_pending_f()); - - } else { - reg = mc_intr_en_clear_r(NVGPU_MC_INTR_STALLING); - g->mc.intr_mask_restore[NVGPU_MC_INTR_STALLING] &= - ~mc_intr_pmu_pending_f(); - nvgpu_writel(g, reg, mc_intr_pmu_pending_f()); - - reg = mc_intr_en_clear_r(NVGPU_MC_INTR_NONSTALLING); - g->mc.intr_mask_restore[NVGPU_MC_INTR_NONSTALLING] &= - ~mc_intr_pmu_pending_f(); - nvgpu_writel(g, reg, mc_intr_pmu_pending_f()); - } -} diff --git a/drivers/gpu/nvgpu/hal/mc/mc_gp10b.h b/drivers/gpu/nvgpu/hal/mc/mc_gp10b.h index ac50b3ec8..7cb47922a 100644 --- a/drivers/gpu/nvgpu/hal/mc/mc_gp10b.h +++ b/drivers/gpu/nvgpu/hal/mc/mc_gp10b.h @@ -31,6 +31,8 @@ struct gk20a; enum nvgpu_unit; void mc_gp10b_intr_mask(struct gk20a *g); +void mc_gp10b_intr_stall_unit_config(struct gk20a *g, u32 unit, bool enable); +void mc_gp10b_intr_nonstall_unit_config(struct gk20a *g, u32 unit, bool enable); void mc_gp10b_isr_stall(struct gk20a *g); bool mc_gp10b_is_intr1_pending(struct gk20a *g, enum nvgpu_unit unit, u32 mc_intr_1); @@ -44,12 +46,4 @@ void mc_gp10b_intr_nonstall_pause(struct gk20a *g); void mc_gp10b_intr_nonstall_resume(struct gk20a *g); void mc_gp10b_ltc_isr(struct gk20a *g); -#ifdef CONFIG_NVGPU_LS_PMU -void mc_gp10b_intr_pmu_unit_config(struct gk20a *g, bool enable); -#endif - -#ifdef CONFIG_NVGPU_HAL_NON_FUSA -int mc_gp10b_intr_enable(struct gk20a *g); -#endif - #endif diff --git a/drivers/gpu/nvgpu/hal/mc/mc_gp10b_fusa.c b/drivers/gpu/nvgpu/hal/mc/mc_gp10b_fusa.c index 28ec7a533..b52567553 100644 --- a/drivers/gpu/nvgpu/hal/mc/mc_gp10b_fusa.c +++ b/drivers/gpu/nvgpu/hal/mc/mc_gp10b_fusa.c @@ -40,9 +40,48 @@ void mc_gp10b_intr_mask(struct gk20a *g) { nvgpu_writel(g, mc_intr_en_clear_r(NVGPU_MC_INTR_STALLING), U32_MAX); + g->mc.intr_mask_restore[NVGPU_MC_INTR_STALLING] = 0; nvgpu_writel(g, mc_intr_en_clear_r(NVGPU_MC_INTR_NONSTALLING), U32_MAX); + g->mc.intr_mask_restore[NVGPU_MC_INTR_NONSTALLING] = 0; +} + +static u32 mc_gp10b_intr_pending_f(struct gk20a *g, u32 unit) +{ + u32 intr_pending_f = 0; + + switch (unit) { + case MC_INTR_UNIT_BUS: + intr_pending_f = mc_intr_pbus_pending_f(); + break; + case MC_INTR_UNIT_PRIV_RING: + intr_pending_f = mc_intr_priv_ring_pending_f(); + break; + case MC_INTR_UNIT_FIFO: + intr_pending_f = mc_intr_pfifo_pending_f(); + break; + case MC_INTR_UNIT_LTC: + intr_pending_f = mc_intr_ltc_pending_f(); + break; + case MC_INTR_UNIT_GR: + intr_pending_f = nvgpu_gr_engine_interrupt_mask(g); + break; + case MC_INTR_UNIT_PMU: + intr_pending_f = mc_intr_pmu_pending_f(); + break; + case MC_INTR_UNIT_HUB: + intr_pending_f = mc_intr_replayable_fault_pending_f(); + break; + case MC_INTR_UNIT_CE: + intr_pending_f = nvgpu_ce_engine_interrupt_mask(g); + break; + default: + nvgpu_err(g, "Invalid MC interrupt unit specified !!!"); + break; + } + + return intr_pending_f; } static void mc_gp10b_isr_stall_secondary_1(struct gk20a *g, u32 mc_intr_0) @@ -65,7 +104,6 @@ static void mc_gp10b_isr_stall_secondary_1(struct gk20a *g, u32 mc_intr_0) #endif } - static void mc_gp10b_isr_stall_secondary_0(struct gk20a *g, u32 mc_intr_0) { if ((g->ops.mc.is_intr_hub_pending != NULL) && @@ -107,6 +145,42 @@ static void mc_gp10b_isr_stall_engine(struct gk20a *g, } } +void mc_gp10b_intr_stall_unit_config(struct gk20a *g, u32 unit, bool enable) +{ + u32 unit_pending_f = mc_gp10b_intr_pending_f(g, unit); + u32 reg = 0U; + + if (enable) { + reg = mc_intr_en_set_r(NVGPU_MC_INTR_STALLING); + g->mc.intr_mask_restore[NVGPU_MC_INTR_STALLING] |= + unit_pending_f; + nvgpu_writel(g, reg, unit_pending_f); + } else { + reg = mc_intr_en_clear_r(NVGPU_MC_INTR_STALLING); + g->mc.intr_mask_restore[NVGPU_MC_INTR_STALLING] &= + ~unit_pending_f; + nvgpu_writel(g, reg, unit_pending_f); + } +} + +void mc_gp10b_intr_nonstall_unit_config(struct gk20a *g, u32 unit, bool enable) +{ + u32 unit_pending_f = mc_gp10b_intr_pending_f(g, unit); + u32 reg = 0U; + + if (enable) { + reg = mc_intr_en_set_r(NVGPU_MC_INTR_NONSTALLING); + g->mc.intr_mask_restore[NVGPU_MC_INTR_NONSTALLING] |= + unit_pending_f; + nvgpu_writel(g, reg, unit_pending_f); + } else { + reg = mc_intr_en_clear_r(NVGPU_MC_INTR_NONSTALLING); + g->mc.intr_mask_restore[NVGPU_MC_INTR_NONSTALLING] &= + ~unit_pending_f; + nvgpu_writel(g, reg, unit_pending_f); + } +} + void mc_gp10b_isr_stall(struct gk20a *g) { u32 mc_intr_0; diff --git a/drivers/gpu/nvgpu/hal/mc/mc_gv100.c b/drivers/gpu/nvgpu/hal/mc/mc_gv100.c index b00d3da66..f96f92634 100644 --- a/drivers/gpu/nvgpu/hal/mc/mc_gv100.c +++ b/drivers/gpu/nvgpu/hal/mc/mc_gv100.c @@ -34,36 +34,6 @@ #include -int mc_gv100_intr_enable(struct gk20a *g) -{ - u32 eng_intr_mask = nvgpu_engine_interrupt_mask(g); - - nvgpu_writel(g, mc_intr_en_clear_r(NVGPU_MC_INTR_STALLING), - U32_MAX); - nvgpu_writel(g, mc_intr_en_clear_r(NVGPU_MC_INTR_NONSTALLING), - U32_MAX); - g->mc.intr_mask_restore[NVGPU_MC_INTR_STALLING] = - mc_intr_pfifo_pending_f() | - mc_intr_hub_pending_f() | - mc_intr_priv_ring_pending_f() | - mc_intr_pbus_pending_f() | - mc_intr_ltc_pending_f() | - mc_intr_nvlink_pending_f() | - eng_intr_mask; - - g->mc.intr_mask_restore[NVGPU_MC_INTR_NONSTALLING] = - mc_intr_pfifo_pending_f() - | eng_intr_mask; - - nvgpu_writel(g, mc_intr_en_set_r(NVGPU_MC_INTR_STALLING), - g->mc.intr_mask_restore[NVGPU_MC_INTR_STALLING]); - - nvgpu_writel(g, mc_intr_en_set_r(NVGPU_MC_INTR_NONSTALLING), - g->mc.intr_mask_restore[NVGPU_MC_INTR_NONSTALLING]); - - return 0; -} - bool gv100_mc_is_intr_nvlink_pending(struct gk20a *g, u32 mc_intr_0) { return ((mc_intr_0 & mc_intr_nvlink_pending_f()) != 0U); diff --git a/drivers/gpu/nvgpu/hal/mc/mc_gv100.h b/drivers/gpu/nvgpu/hal/mc/mc_gv100.h index 501ffc93a..78ec9c214 100644 --- a/drivers/gpu/nvgpu/hal/mc/mc_gv100.h +++ b/drivers/gpu/nvgpu/hal/mc/mc_gv100.h @@ -27,7 +27,6 @@ struct gk20a; -int mc_gv100_intr_enable(struct gk20a *g); bool gv100_mc_is_intr_nvlink_pending(struct gk20a *g, u32 mc_intr_0); bool gv100_mc_is_stall_and_eng_intr_pending(struct gk20a *g, u32 act_eng_id, u32 *eng_intr_pending); diff --git a/drivers/gpu/nvgpu/hal/mc/mc_gv11b.h b/drivers/gpu/nvgpu/hal/mc/mc_gv11b.h index 1784ef5fa..2f1817070 100644 --- a/drivers/gpu/nvgpu/hal/mc/mc_gv11b.h +++ b/drivers/gpu/nvgpu/hal/mc/mc_gv11b.h @@ -27,7 +27,6 @@ struct gk20a; -int mc_gv11b_intr_enable(struct gk20a *g); bool gv11b_mc_is_intr_hub_pending(struct gk20a *g, u32 mc_intr_0); bool gv11b_mc_is_stall_and_eng_intr_pending(struct gk20a *g, u32 act_eng_id, u32 *eng_intr_pending); diff --git a/drivers/gpu/nvgpu/hal/mc/mc_gv11b_fusa.c b/drivers/gpu/nvgpu/hal/mc/mc_gv11b_fusa.c index 3820e64f5..686e1c39e 100644 --- a/drivers/gpu/nvgpu/hal/mc/mc_gv11b_fusa.c +++ b/drivers/gpu/nvgpu/hal/mc/mc_gv11b_fusa.c @@ -33,36 +33,6 @@ #include -int mc_gv11b_intr_enable(struct gk20a *g) -{ - u32 eng_intr_mask = nvgpu_engine_interrupt_mask(g); - - nvgpu_writel(g, mc_intr_en_clear_r(NVGPU_MC_INTR_STALLING), - U32_MAX); - nvgpu_writel(g, mc_intr_en_clear_r(NVGPU_MC_INTR_NONSTALLING), - U32_MAX); - - g->mc.intr_mask_restore[NVGPU_MC_INTR_STALLING] = - mc_intr_pfifo_pending_f() | - mc_intr_hub_pending_f() | - mc_intr_priv_ring_pending_f() | - mc_intr_pbus_pending_f() | - mc_intr_ltc_pending_f() | - eng_intr_mask; - - g->mc.intr_mask_restore[NVGPU_MC_INTR_NONSTALLING] = - mc_intr_pfifo_pending_f() - | eng_intr_mask; - - nvgpu_writel(g, mc_intr_en_set_r(NVGPU_MC_INTR_STALLING), - g->mc.intr_mask_restore[NVGPU_MC_INTR_STALLING]); - - nvgpu_writel(g, mc_intr_en_set_r(NVGPU_MC_INTR_NONSTALLING), - g->mc.intr_mask_restore[NVGPU_MC_INTR_NONSTALLING]); - - return 0; -} - bool gv11b_mc_is_intr_hub_pending(struct gk20a *g, u32 mc_intr_0) { return ((mc_intr_0 & mc_intr_hub_pending_f()) != 0U); diff --git a/drivers/gpu/nvgpu/hal/mc/mc_tu104.c b/drivers/gpu/nvgpu/hal/mc/mc_tu104.c index 710351d7e..40ec42024 100644 --- a/drivers/gpu/nvgpu/hal/mc/mc_tu104.c +++ b/drivers/gpu/nvgpu/hal/mc/mc_tu104.c @@ -136,25 +136,6 @@ bool intr_tu104_vector_intr_pending(struct gk20a *g, u32 intr_vector) BIT32(NV_CPU_INTR_GPU_VECTOR_TO_LEAF_BIT(intr_vector))) != 0U); } -static void intr_tu104_stall_enable(struct gk20a *g) -{ - u32 eng_intr_mask = nvgpu_engine_interrupt_mask(g); - - nvgpu_writel(g, mc_intr_en_clear_r(NVGPU_MC_INTR_STALLING), U32_MAX); - - g->mc.intr_mask_restore[NVGPU_MC_INTR_STALLING] = - mc_intr_pfifo_pending_f() | - mc_intr_priv_ring_pending_f() | - mc_intr_pbus_pending_f() | - mc_intr_ltc_pending_f() | - mc_intr_nvlink_pending_f() | - mc_intr_pfb_pending_f() | - eng_intr_mask; - - nvgpu_writel(g, mc_intr_en_set_r(NVGPU_MC_INTR_STALLING), - g->mc.intr_mask_restore[NVGPU_MC_INTR_STALLING]); -} - static void intr_tu104_nonstall_enable(struct gk20a *g) { u32 i; @@ -202,13 +183,78 @@ static void intr_tu104_nonstall_enable(struct gk20a *g) u64_hi32(nonstall_intr_mask)); } +static u32 intr_tu104_intr_pending_f(struct gk20a *g, u32 unit) +{ + u32 intr_pending_f = 0; + + switch (unit) { + case MC_INTR_UNIT_BUS: + intr_pending_f = mc_intr_pbus_pending_f(); + break; + case MC_INTR_UNIT_PRIV_RING: + intr_pending_f = mc_intr_priv_ring_pending_f(); + break; + case MC_INTR_UNIT_FIFO: + intr_pending_f = mc_intr_pfifo_pending_f(); + break; + case MC_INTR_UNIT_LTC: + intr_pending_f = mc_intr_ltc_pending_f(); + break; + case MC_INTR_UNIT_GR: + intr_pending_f = nvgpu_gr_engine_interrupt_mask(g); + break; + case MC_INTR_UNIT_PMU: + intr_pending_f = mc_intr_pmu_pending_f(); + break; + case MC_INTR_UNIT_CE: + intr_pending_f = nvgpu_ce_engine_interrupt_mask(g); + break; + case MC_INTR_UNIT_NVLINK: + intr_pending_f = mc_intr_nvlink_pending_f(); + break; + case MC_INTR_UNIT_FBPA: + intr_pending_f = mc_intr_pfb_pending_f(); + break; + default: + nvgpu_err(g, "Invalid MC interrupt unit specified !!!"); + break; + } + + return intr_pending_f; +} + +void intr_tu104_stall_unit_config(struct gk20a *g, u32 unit, bool enable) +{ + u32 unit_pending_f = intr_tu104_intr_pending_f(g, unit); + u32 reg = 0U; + + if (enable) { + reg = mc_intr_en_set_r(NVGPU_MC_INTR_STALLING); + g->mc.intr_mask_restore[NVGPU_MC_INTR_STALLING] |= + unit_pending_f; + nvgpu_writel(g, reg, unit_pending_f); + } else { + reg = mc_intr_en_clear_r(NVGPU_MC_INTR_STALLING); + g->mc.intr_mask_restore[NVGPU_MC_INTR_STALLING] &= + ~unit_pending_f; + nvgpu_writel(g, reg, unit_pending_f); + } +} + +void intr_tu104_nonstall_unit_config(struct gk20a *g, u32 unit, bool enable) +{ + intr_tu104_nonstall_enable(g); +} + void intr_tu104_mask(struct gk20a *g) { u32 size, reg, i; nvgpu_writel(g, mc_intr_en_clear_r(NVGPU_MC_INTR_STALLING), U32_MAX); + g->mc.intr_mask_restore[NVGPU_MC_INTR_STALLING] = 0; nvgpu_writel(g, mc_intr_en_clear_r(NVGPU_MC_INTR_NONSTALLING), U32_MAX); + g->mc.intr_mask_restore[NVGPU_MC_INTR_NONSTALLING] = 0; size = func_priv_cpu_intr_top_en_clear__size_1_v(); for (i = 0U; i < size; i++) { @@ -217,15 +263,6 @@ void intr_tu104_mask(struct gk20a *g) } } -/* Enable all required interrupts */ -int intr_tu104_enable(struct gk20a *g) -{ - intr_tu104_stall_enable(g); - intr_tu104_nonstall_enable(g); - - return 0; -} - /* Return non-zero if nonstall interrupts are pending */ u32 intr_tu104_nonstall(struct gk20a *g) { diff --git a/drivers/gpu/nvgpu/hal/mc/mc_tu104.h b/drivers/gpu/nvgpu/hal/mc/mc_tu104.h index 8b25190bb..00133b939 100644 --- a/drivers/gpu/nvgpu/hal/mc/mc_tu104.h +++ b/drivers/gpu/nvgpu/hal/mc/mc_tu104.h @@ -49,8 +49,9 @@ void intr_tu104_vector_en_clear(struct gk20a *g, u32 intr_vector); bool intr_tu104_vector_intr_pending(struct gk20a *g, u32 intr_vector); void intr_tu104_intr_clear_leaf_vector(struct gk20a *g, u32 intr_vector); +void intr_tu104_stall_unit_config(struct gk20a *g, u32 unit, bool enable); +void intr_tu104_nonstall_unit_config(struct gk20a *g, u32 unit, bool enable); void intr_tu104_mask(struct gk20a *g); -int intr_tu104_enable(struct gk20a *g); u32 intr_tu104_stall(struct gk20a *g); void intr_tu104_stall_pause(struct gk20a *g); void intr_tu104_stall_resume(struct gk20a *g); diff --git a/drivers/gpu/nvgpu/hal/pmu/pmu_gk20a.c b/drivers/gpu/nvgpu/hal/pmu/pmu_gk20a.c index 7922ac143..51da0b8c5 100644 --- a/drivers/gpu/nvgpu/hal/pmu/pmu_gk20a.c +++ b/drivers/gpu/nvgpu/hal/pmu/pmu_gk20a.c @@ -445,7 +445,7 @@ void gk20a_pmu_enable_irq(struct nvgpu_pmu *pmu, bool enable) nvgpu_log_fn(g, " "); - g->ops.mc.intr_pmu_unit_config(g, MC_INTR_UNIT_DISABLE); + nvgpu_mc_intr_stall_unit_config(g, MC_INTR_UNIT_PMU, MC_INTR_DISABLE); nvgpu_falcon_set_irq(pmu->flcn, false, 0x0, 0x0); @@ -461,9 +461,10 @@ void gk20a_pmu_enable_irq(struct nvgpu_pmu *pmu, bool enable) pwr_falcon_irqmset_swgen0_f(1) | pwr_falcon_irqmset_swgen1_f(1); - nvgpu_falcon_set_irq(pmu->flcn, true, intr_mask, intr_dest); + nvgpu_mc_intr_stall_unit_config(g, MC_INTR_UNIT_PMU, + MC_INTR_ENABLE); - g->ops.mc.intr_pmu_unit_config(g, MC_INTR_UNIT_ENABLE); + nvgpu_falcon_set_irq(pmu->flcn, true, intr_mask, intr_dest); } nvgpu_log_fn(g, "done"); diff --git a/drivers/gpu/nvgpu/hal/priv_ring/priv_ring_gm20b_fusa.c b/drivers/gpu/nvgpu/hal/priv_ring/priv_ring_gm20b_fusa.c index a75da304f..d811b5801 100644 --- a/drivers/gpu/nvgpu/hal/priv_ring/priv_ring_gm20b_fusa.c +++ b/drivers/gpu/nvgpu/hal/priv_ring/priv_ring_gm20b_fusa.c @@ -29,6 +29,7 @@ #include #include #include +#include #include "priv_ring_gm20b.h" @@ -55,6 +56,9 @@ int gm20b_priv_ring_enable(struct gk20a *g) (void) nvgpu_readl(g, pri_ringstation_sys_decode_config_r()); + nvgpu_mc_intr_stall_unit_config(g, MC_INTR_UNIT_PRIV_RING, + MC_INTR_ENABLE); + return 0; } diff --git a/drivers/gpu/nvgpu/hal/vgpu/init/vgpu_hal_gp10b.c b/drivers/gpu/nvgpu/hal/vgpu/init/vgpu_hal_gp10b.c index aedf045d2..0b1a2d212 100644 --- a/drivers/gpu/nvgpu/hal/vgpu/init/vgpu_hal_gp10b.c +++ b/drivers/gpu/nvgpu/hal/vgpu/init/vgpu_hal_gp10b.c @@ -754,9 +754,8 @@ static const struct gpu_ops vgpu_gp10b_ops = { .get_chip_details = NULL, .intr_mask = NULL, .intr_enable = NULL, -#ifdef CONFIG_NVGPU_LS_PMU - .intr_pmu_unit_config = NULL, -#endif + .intr_stall_unit_config = NULL, + .intr_nonstall_unit_config = NULL, .isr_stall = NULL, .intr_stall = NULL, .intr_stall_pause = NULL, diff --git a/drivers/gpu/nvgpu/hal/vgpu/init/vgpu_hal_gv11b.c b/drivers/gpu/nvgpu/hal/vgpu/init/vgpu_hal_gv11b.c index b6bcef594..219231ac3 100644 --- a/drivers/gpu/nvgpu/hal/vgpu/init/vgpu_hal_gv11b.c +++ b/drivers/gpu/nvgpu/hal/vgpu/init/vgpu_hal_gv11b.c @@ -876,9 +876,8 @@ static const struct gpu_ops vgpu_gv11b_ops = { .get_chip_details = NULL, .intr_mask = NULL, .intr_enable = NULL, -#ifdef CONFIG_NVGPU_LS_PMU - .intr_pmu_unit_config = NULL, -#endif + .intr_stall_unit_config = NULL, + .intr_nonstall_unit_config = NULL, .isr_stall = NULL, .intr_stall = NULL, .intr_stall_pause = NULL, diff --git a/drivers/gpu/nvgpu/include/nvgpu/engines.h b/drivers/gpu/nvgpu/include/nvgpu/engines.h index 03d4962be..a71785d2f 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/engines.h +++ b/drivers/gpu/nvgpu/include/nvgpu/engines.h @@ -185,7 +185,24 @@ bool nvgpu_engine_check_valid_id(struct gk20a *g, u32 engine_id); */ u32 nvgpu_engine_get_gr_id(struct gk20a *g); /** - * @brief Get intr mask for the engines supported by the chip. + * @brief Get intr mask for the GR engine supported by the chip. + * + * @param g[in] The GPU driver struct. + * + * For each of #nvgpu_fifo.num_engines, get pointer to + * #nvgpu_engine_info. Use this to get #nvgpu_engine_info.intr_mask. + * If #nvgpu_engine_info.engine_num type matches with + * #NVGPU_ENGINE_GR, local intr_mask variable is logically ORed with + * #nvgpu_engine_info.intr_mask. + * + * @return Interrupt mask for GR engine. + * @retval 0 if #nvgpu_fifo.num_engines is 0. + * @retval 0 if all of the supported engine enum types don't match with + * #NVGPU_ENGINE_GR. + */ +u32 nvgpu_gr_engine_interrupt_mask(struct gk20a *g); +/** + * @brief Get intr mask for the CE engines supported by the chip. * * @param g [in] The GPU driver struct. * @@ -193,16 +210,18 @@ u32 nvgpu_engine_get_gr_id(struct gk20a *g); * #nvgpu_engine_info. Use this to get #nvgpu_engine_info.intr_mask. * If #nvgpu_engine_info.engine_num type matches with * #NVGPU_ENGINE_GRCE or #NVGPU_ENGINE_ASYNC_CE but interrupt handlers - * are not supported, local intr_mask variable is not logically ORed with + * are not supported or engine_enum type matches with #NVGPU_ENGINE_GR + * , local intr_mask variable is not logically ORed with * #nvgpu_engine_info.intr_mask. * - * @return Interrupt mask for h/w engines that support interrupt handlers. + * @return Interrupt mask for CE engines that support interrupt handlers. * @retval 0 if #nvgpu_fifo.num_engines is 0. * @retval 0 if all of the supported engine enum types match with * #NVGPU_ENGINE_GRCE or #NVGPU_ENGINE_ASYNC_CE and does not - * support interrupt handler. + * support interrupt handler or engine enum type matches with + * #NVGPU_ENGINE_GR. */ -u32 nvgpu_engine_interrupt_mask(struct gk20a *g); +u32 nvgpu_ce_engine_interrupt_mask(struct gk20a *g); /** * @brief Get intr mask for the h/w engine id. * diff --git a/drivers/gpu/nvgpu/include/nvgpu/gk20a.h b/drivers/gpu/nvgpu/include/nvgpu/gk20a.h index ee9bf526a..5250a21d5 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/gk20a.h +++ b/drivers/gpu/nvgpu/include/nvgpu/gk20a.h @@ -184,9 +184,6 @@ struct railgate_stats { }; #endif -#define MC_INTR_UNIT_DISABLE false -#define MC_INTR_UNIT_ENABLE true - #define GPU_LIT_NUM_GPCS 0 #define GPU_LIT_NUM_PES_PER_GPC 1 #define GPU_LIT_NUM_ZCULL_BANKS 2 diff --git a/drivers/gpu/nvgpu/include/nvgpu/gops_mc.h b/drivers/gpu/nvgpu/include/nvgpu/gops_mc.h index caa3040b4..23936e2ac 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/gops_mc.h +++ b/drivers/gpu/nvgpu/include/nvgpu/gops_mc.h @@ -72,53 +72,6 @@ struct gops_mc { u32 (*get_chip_details)(struct gk20a *g, u32 *arch, u32 *impl, u32 *rev); - /** - * @brief Clear the GPU device interrupts at master level. - * - * @param g [in] The GPU driver struct. - * - * This function is invoked before powering off or finishing - * SW quiesce of nvgpu driver. - * - * Steps: - * - Write U32_MAX to the stalling interrupts enable clear register. - * mc_intr_en_clear_r are write only registers which clear - * the corresponding bit in INTR_EN whenever a 1 is written - * to it. - * - Write U32_MAX to the non-stalling interrupts enable clear register. - */ - void (*intr_mask)(struct gk20a *g); - - /** - * @brief Enable the applicable GPU device interrupts at master level. - * - * @param g [in] The GPU driver struct. - * - * This function is invoked during #nvgpu_finalize_poweron before - * enabling the individual HW units interrupts. - * - * Steps: - * - Get the engine interrupts mask for supported FIFO engines by - * calling #nvgpu_engine_interrupt_mask. - * - Clear the GPU device interrupts. - * - Set the master level interrupts masks to be used for - * enabling/disabling the interrupts at runtime in - * #intr_stall_pause, #intr_stall_resume, - * #intr_nonstall_pause and #intr_nonstall_resume. - * - Initialize the stalling interrupts bitmask - * #intr_mask_restore[#NVGPU_MC_INTR_STALLING] with various - * units (FIFO, HUB, PRIV_RING, PBUS, LTC) OR'ing with engine - * interrupts mask. - * - Initialize the non-stalling interrupts bitmask - * #intr_mask_restore[#NVGPU_MC_INTR_NONSTALLING] with FIFO - * unit OR'ing with engine interrupts mask. - * - Write the bitmasks to the stalling and the non-stalling interrupts - * enable registers respectively (mc_intr_en_set_r()). - * - * @return 0 in case of success, < 0 in case of failure. - */ - int (*intr_enable)(struct gk20a *g); - /** * @brief Read the the stalling interrupts status register. * @@ -135,20 +88,6 @@ struct gops_mc { */ u32 (*intr_stall)(struct gk20a *g); - /** - * @brief Disable/Pause the stalling interrupts. - * - * @param g [in] The GPU driver struct. - * - * This function is invoked to disable the stalling interrupts before - * the ISR is executed. - * - * Steps: - * - Write U32_MAX to the stalling interrupts enable clear register - * (mc_intr_en_clear_r(#NVGPU_MC_INTR_STALLING)). - */ - void (*intr_stall_pause)(struct gk20a *g); - /** * @brief Interrupt Service Routine (ISR) for handling the stalling * interrupts. @@ -184,22 +123,6 @@ struct gops_mc { */ void (*isr_stall)(struct gk20a *g); - /** - * @brief Enable/Resume the stalling interrupts. - * - * @param g [in] The GPU driver struct. - * - * This function is invoked to enable the stalling interrupts after - * the ISR is executed. - * - * Steps: - * - Enable the stalling interrupts as configured during #intr_enable. - * Write #intr_mask_restore[#NVGPU_MC_INTR_STALLING] to the - * stalling interrupts enable set register - * (mc_intr_en_set_r(#NVGPU_MC_INTR_STALLING)). - */ - void (*intr_stall_resume)(struct gk20a *g); - /** * @brief Read the non-stalling interrupts status register. * @@ -216,20 +139,6 @@ struct gops_mc { */ u32 (*intr_nonstall)(struct gk20a *g); - /** - * @brief Disable/Pause the non-stalling interrupts. - * - * @param g [in] The GPU driver struct. - * - * This function is invoked to disable the non-stalling interrupts - * before the ISR is executed. - * - * Steps: - * - Write U32_MAX to the non-stalling interrupts enable clear register - * (mc_intr_en_clear_r(#NVGPU_MC_INTR_NONSTALLING)). - */ - void (*intr_nonstall_pause)(struct gk20a *g); - /** * @brief Interrupt Service Routine (ISR) for handling the non-stalling * interrupts. @@ -256,23 +165,6 @@ struct gops_mc { */ u32 (*isr_nonstall)(struct gk20a *g); - /** - * @brief Enable/Resume the non-stalling interrupts. - * - * @param g [in] The GPU driver struct. - * - * This function is invoked to enable the non-stalling interrupts after - * the ISR is executed. - * - * Steps: - * - Enable the non-stalling interrupts as configured during - * #intr_enable. - * Write #intr_mask_restore[#NVGPU_MC_INTR_NONSTALLING] - * to the non-stalling interrupts enable set register - * (mc_intr_en_set_r(#NVGPU_MC_INTR_NONSTALLING)). - */ - void (*intr_nonstall_resume)(struct gk20a *g); - /** * @brief Check if stalling or engine interrupts are pending. * @@ -345,10 +237,10 @@ struct gops_mc { * @param g [in] The GPU driver struct. * @param unit [in] Value designating the GPU HW unit/engine * controlled by MC. Supported values are: - * - NVGPU_UNIT_FIFO - * - NVGPU_UNIT_PERFMON - * - NVGPU_UNIT_GRAPH - * - NVGPU_UNIT_BLG + * - #NVGPU_UNIT_FIFO + * - #NVGPU_UNIT_PERFMON + * - #NVGPU_UNIT_GRAPH + * - #NVGPU_UNIT_BLG * * This function is invoked to get the reset mask of the engines for * resetting CE, GR, FIFO during #nvgpu_finalize_poweron. @@ -385,6 +277,24 @@ struct gops_mc { /** @cond DOXYGEN_SHOULD_SKIP_THIS */ + void (*intr_mask)(struct gk20a *g); + + void (*intr_enable)(struct gk20a *g); + + void (*intr_stall_unit_config)(struct gk20a *g, u32 unit, + bool enable); + + void (*intr_nonstall_unit_config)(struct gk20a *g, u32 unit, + bool enable); + + void (*intr_stall_pause)(struct gk20a *g); + + void (*intr_stall_resume)(struct gk20a *g); + + void (*intr_nonstall_pause)(struct gk20a *g); + + void (*intr_nonstall_resume)(struct gk20a *g); + void (*enable)(struct gk20a *g, u32 units); void (*disable)(struct gk20a *g, u32 units); @@ -407,10 +317,6 @@ struct gops_mc { void (*fbpa_isr)(struct gk20a *g); #endif -#ifdef CONFIG_NVGPU_LS_PMU - void (*intr_pmu_unit_config)(struct gk20a *g, - bool enable); -#endif /** @endcond DOXYGEN_SHOULD_SKIP_THIS */ }; diff --git a/drivers/gpu/nvgpu/include/nvgpu/mc.h b/drivers/gpu/nvgpu/include/nvgpu/mc.h index ddd586038..e4a87efd3 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/mc.h +++ b/drivers/gpu/nvgpu/include/nvgpu/mc.h @@ -153,11 +153,39 @@ enum nvgpu_unit { #define NVGPU_NONSTALL_OPS_WAKEUP_SEMAPHORE BIT32(0) #define NVGPU_NONSTALL_OPS_POST_EVENTS BIT32(1) +/** MC interrupt for Bus unit. */ +#define MC_INTR_UNIT_BUS 0 +/** MC interrupt for PRIV_RING unit. */ +#define MC_INTR_UNIT_PRIV_RING 1 +/** MC interrupt for FIFO unit. */ +#define MC_INTR_UNIT_FIFO 2 +/** MC interrupt for LTC unit. */ +#define MC_INTR_UNIT_LTC 3 +/** MC interrupt for HUB unit. */ +#define MC_INTR_UNIT_HUB 4 +/** MC interrupt for GR unit. */ +#define MC_INTR_UNIT_GR 5 +/** MC interrupt for PMU unit. */ +#define MC_INTR_UNIT_PMU 6 +/** MC interrupt for CE unit. */ +#define MC_INTR_UNIT_CE 7 +/** MC interrupt for NVLINK unit. */ +#define MC_INTR_UNIT_NVLINK 8 +/** MC interrupt for FBPA unit. */ +#define MC_INTR_UNIT_FBPA 9 + +/** Value to be passed to mc.intr_*_unit_config to enable the interrupt. */ +#define MC_INTR_ENABLE true +/** Value to be passed to mc.intr_*_unit_config to disable the interrupt. */ +#define MC_INTR_DISABLE false + /** * This struct holds the variables needed to manage the configuration and * interrupt handling of the units/engines. */ struct nvgpu_mc { + struct nvgpu_spinlock intr_lock; + /** Lock to access the mc_enable_r */ struct nvgpu_spinlock enable_lock; @@ -246,4 +274,182 @@ struct nvgpu_mc { */ void nvgpu_wait_for_deferred_interrupts(struct gk20a *g); +/** + * @brief Clear the GPU device interrupts at master level. + * + * @param g [in] The GPU driver struct. + * + * This function is invoked before powering off or finishing + * SW quiesce of nvgpu driver. + * + * Steps: + * - Acquire the spinlock g->mc.intr_lock. + * - Write U32_MAX to the stalling interrupts enable clear register. + * mc_intr_en_clear_r are write only registers which clear + * the corresponding bit in INTR_EN whenever a 1 is written + * to it. + * - Set g->mc.intr_mask_restore[NVGPU_MC_INTR_STALLING] and + * g->mc.intr_mask_restore[NVGPU_MC_INTR_NONSTALLING] to 0. + * - Write U32_MAX to the non-stalling interrupts enable clear register. + * - Release the spinlock g->mc.intr_lock. + */ +void nvgpu_mc_intr_mask(struct gk20a *g); + +void nvgpu_mc_log_pending_intrs(struct gk20a *g); + +void nvgpu_mc_intr_enable(struct gk20a *g); + +/** + * @brief Enable the stalling interrupts for GPU unit at the master + * level. + * + * @param g [in] The GPU driver struct. + * @param unit [in] Value designating the GPU HW unit/engine + * controlled by MC. Supported values are: + * - #MC_INTR_UNIT_BUS + * - #MC_INTR_UNIT_PRIV_RING + * - #MC_INTR_UNIT_FIFO + * - #MC_INTR_UNIT_LTC + * - #MC_INTR_UNIT_HUB + * - #MC_INTR_UNIT_GR + * - #MC_INTR_UNIT_PMU + * - #MC_INTR_UNIT_CE + * - #MC_INTR_UNIT_NVLINK + * - #MC_INTR_UNIT_FBPA + * @param enable [in] Boolean control to enable/disable the stalling + * interrupt. Supported values are: + * - #MC_INTR_ENABLE + * - #MC_INTR_DISABLE + * + * This function is invoked during individual unit's init before + * enabling that unit's interrupts. + * + * Steps: + * - Get the interrupt bitmask for \a unit. + * - Acquire the spinlock g->mc.intr_lock. + * - If interrupt is to be enabled + * - Set interrupt bitmask in + * #intr_mask_restore[#NVGPU_MC_INTR_STALLING]. + * - Write the interrupt bitmask to the register + * mc_intr_en_set_r(#NVGPU_MC_INTR_STALLING). + * - Else + * - Clear interrupt bitmask in + * #intr_mask_restore[#NVGPU_MC_INTR_STALLING]. + * - Write the interrupt bitmask to the register + * mc_intr_en_clear_r(#NVGPU_MC_INTR_STALLING). + * - Release the spinlock g->mc.intr_lock. + */ +void nvgpu_mc_intr_stall_unit_config(struct gk20a *g, u32 unit, bool enable); + +/** + * @brief Enable the non-stalling interrupts for GPU unit at the master + * level. + * + * @param g [in] The GPU driver struct. + * @param unit [in] Value designating the GPU HW unit/engine + * controlled by MC. Supported values are: + * - #MC_INTR_UNIT_BUS + * - #MC_INTR_UNIT_PRIV_RING + * - #MC_INTR_UNIT_FIFO + * - #MC_INTR_UNIT_LTC + * - #MC_INTR_UNIT_HUB + * - #MC_INTR_UNIT_GR + * - #MC_INTR_UNIT_PMU + * - #MC_INTR_UNIT_CE + * - #MC_INTR_UNIT_NVLINK + * - #MC_INTR_UNIT_FBPA + * @param enable [in] Boolean control to enable/disable the stalling + * interrupt. Supported values are: + * - #MC_INTR_ENABLE + * - #MC_INTR_DISABLE + * + * This function is invoked during individual unit's init before + * enabling that unit's interrupts. + * + * Steps: + * - Get the interrupt bitmask for \a unit. + * - Acquire the spinlock g->mc.intr_lock. + * - If interrupt is to be enabled + * - Set interrupt bitmask in + * #intr_mask_restore[#NVGPU_MC_INTR_NONSTALLING]. + * - Write the interrupt bitmask to the register + * mc_intr_en_set_r(#NVGPU_MC_INTR_NONSTALLING). + * - Else + * - Clear interrupt bitmask in + * #intr_mask_restore[#NVGPU_MC_INTR_NONSTALLING]. + * - Write the interrupt bitmask to the register + * mc_intr_en_clear_r(#NVGPU_MC_INTR_NONSTALLING). + * - Release the spinlock g->mc.intr_lock. + */ +void nvgpu_mc_intr_nonstall_unit_config(struct gk20a *g, u32 unit, bool enable); + +/** + * @brief Disable/Pause the stalling interrupts. + * + * @param g [in] The GPU driver struct. + * + * This function is invoked to disable the stalling interrupts before + * the ISR is executed. + * + * Steps: + * - Acquire the spinlock g->mc.intr_lock. + * - Write U32_MAX to the stalling interrupts enable clear register + * (mc_intr_en_clear_r(#NVGPU_MC_INTR_STALLING)). + * - Release the spinlock g->mc.intr_lock. + */ +void nvgpu_mc_intr_stall_pause(struct gk20a *g); + +/** + * @brief Enable/Resume the stalling interrupts. + * + * @param g [in] The GPU driver struct. + * + * This function is invoked to enable the stalling interrupts after + * the ISR is executed. + * + * Steps: + * - Acquire the spinlock g->mc.intr_lock. + * - Enable the stalling interrupts as configured during #intr_enable. + * Write #intr_mask_restore[#NVGPU_MC_INTR_STALLING] to the + * stalling interrupts enable set register + * (mc_intr_en_set_r(#NVGPU_MC_INTR_STALLING)). + * - Release the spinlock g->mc.intr_lock. + */ +void nvgpu_mc_intr_stall_resume(struct gk20a *g); + +/** + * @brief Disable/Pause the non-stalling interrupts. + * + * @param g [in] The GPU driver struct. + * + * This function is invoked to disable the non-stalling interrupts + * before the ISR is executed. + * + * Steps: + * - Acquire the spinlock g->mc.intr_lock. + * - Write U32_MAX to the non-stalling interrupts enable clear register + * (mc_intr_en_clear_r(#NVGPU_MC_INTR_NONSTALLING)). + * - Release the spinlock g->mc.intr_lock. + */ +void nvgpu_mc_intr_nonstall_pause(struct gk20a *g); + +/** + * @brief Enable/Resume the non-stalling interrupts. + * + * @param g [in] The GPU driver struct. + * + * This function is invoked to enable the non-stalling interrupts after + * the ISR is executed. + * + * Steps: + * - Acquire the spinlock g->mc.intr_lock. + * - Enable the non-stalling interrupts as configured during + * #intr_enable. + * Write #intr_mask_restore[#NVGPU_MC_INTR_NONSTALLING] + * to the non-stalling interrupts enable set register + * (mc_intr_en_set_r(#NVGPU_MC_INTR_NONSTALLING)). + * - Release the spinlock g->mc.intr_lock. + */ +void nvgpu_mc_intr_nonstall_resume(struct gk20a *g); + #endif diff --git a/drivers/gpu/nvgpu/include/nvgpu/posix/lock.h b/drivers/gpu/nvgpu/include/nvgpu/posix/lock.h index 76e8c14e5..88e3873c9 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/posix/lock.h +++ b/drivers/gpu/nvgpu/include/nvgpu/posix/lock.h @@ -102,4 +102,16 @@ struct nvgpu_raw_spinlock { struct __nvgpu_posix_lock lock; }; +static inline void nvgpu_spinlock_irqsave(struct nvgpu_spinlock *mutex, + unsigned long flags) +{ + nvgpu_posix_lock_acquire(&mutex->lock); +} + +static inline void nvgpu_spinunlock_irqrestore(struct nvgpu_spinlock *mutex, + unsigned long flags) +{ + nvgpu_posix_lock_release(&mutex->lock); +} + #endif /* NVGPU_POSIX_LOCK_H */ diff --git a/drivers/gpu/nvgpu/libnvgpu-drv_safe.export b/drivers/gpu/nvgpu/libnvgpu-drv_safe.export index 57177d157..b70ed6111 100644 --- a/drivers/gpu/nvgpu/libnvgpu-drv_safe.export +++ b/drivers/gpu/nvgpu/libnvgpu-drv_safe.export @@ -111,6 +111,8 @@ gv11b_blcg_hshub_gating_prod_size gv11b_blcg_hshub_get_gating_prod gv11b_netlist_is_firmware_defined gv11b_top_get_num_lce +mc_gp10b_intr_stall_unit_config +mc_gp10b_intr_nonstall_unit_config nvgpu_acr_bootstrap_hs_acr nvgpu_acr_construct_execute nvgpu_acr_init diff --git a/drivers/gpu/nvgpu/os/linux/driver_common.c b/drivers/gpu/nvgpu/os/linux/driver_common.c index 9c6e65d11..9b617f01e 100644 --- a/drivers/gpu/nvgpu/os/linux/driver_common.c +++ b/drivers/gpu/nvgpu/os/linux/driver_common.c @@ -62,6 +62,8 @@ static void nvgpu_init_vars(struct gk20a *g) nvgpu_spinlock_init(&g->power_spinlock); + nvgpu_spinlock_init(&g->mc.intr_lock); + nvgpu_mutex_init(&platform->railgate_lock); nvgpu_mutex_init(&g->dbg_sessions_lock); nvgpu_mutex_init(&g->client_lock); diff --git a/drivers/gpu/nvgpu/os/linux/intr.c b/drivers/gpu/nvgpu/os/linux/intr.c index 7f3f7f072..cd1f45f4b 100644 --- a/drivers/gpu/nvgpu/os/linux/intr.c +++ b/drivers/gpu/nvgpu/os/linux/intr.c @@ -40,7 +40,7 @@ irqreturn_t nvgpu_intr_stall(struct gk20a *g) if (unlikely(!mc_intr_0)) return IRQ_NONE; - g->ops.mc.intr_stall_pause(g); + nvgpu_mc_intr_stall_pause(g); #ifndef CONFIG_NVGPU_RECOVERY if (g->sw_quiesce_pending) { return IRQ_NONE; @@ -68,7 +68,7 @@ irqreturn_t nvgpu_intr_thread_stall(struct gk20a *g) hw_irq_count = nvgpu_atomic_read(&g->mc.hw_irq_stall_count); g->ops.mc.isr_stall(g); - g->ops.mc.intr_stall_resume(g); + nvgpu_mc_intr_stall_resume(g); /* sync handled irq counter before re-enabling interrupts */ nvgpu_atomic_set(&g->mc.sw_irq_stall_last_handled, hw_irq_count); @@ -96,7 +96,7 @@ irqreturn_t nvgpu_intr_nonstall(struct gk20a *g) if (unlikely(!non_stall_intr_val)) return IRQ_NONE; - g->ops.mc.intr_nonstall_pause(g); + nvgpu_mc_intr_nonstall_pause(g); #ifndef CONFIG_NVGPU_RECOVERY if (g->sw_quiesce_pending) { return IRQ_NONE; @@ -119,7 +119,7 @@ irqreturn_t nvgpu_intr_nonstall(struct gk20a *g) /* sync handled irq counter before re-enabling interrupts */ nvgpu_atomic_set(&g->mc.sw_irq_nonstall_last_handled, hw_irq_count); - g->ops.mc.intr_nonstall_resume(g); + nvgpu_mc_intr_nonstall_resume(g); nvgpu_cond_broadcast(&g->mc.sw_irq_nonstall_last_handled_cond); diff --git a/drivers/gpu/nvgpu/os/linux/vgpu/vgpu_linux.c b/drivers/gpu/nvgpu/os/linux/vgpu/vgpu_linux.c index 5fb91e55b..96a40589b 100644 --- a/drivers/gpu/nvgpu/os/linux/vgpu/vgpu_linux.c +++ b/drivers/gpu/nvgpu/os/linux/vgpu/vgpu_linux.c @@ -387,6 +387,8 @@ int vgpu_probe(struct platform_device *pdev) nvgpu_spinlock_init(&gk20a->mc.enable_lock); + nvgpu_spinlock_init(&gk20a->mc.intr_lock); + gk20a->ch_wdt_init_limit_ms = platform->ch_wdt_init_limit_ms; /* Initialize the platform interface. */ diff --git a/userspace/units/init/nvgpu-init.c b/userspace/units/init/nvgpu-init.c index 96bfc4c1e..301e7b22f 100644 --- a/userspace/units/init/nvgpu-init.c +++ b/userspace/units/init/nvgpu-init.c @@ -514,7 +514,6 @@ static void set_poweron_funcs_success(struct gk20a *g) setup_simple_init_func_success(&g->ops.ce.ce_init_support, i++); setup_simple_init_func_success(&g->ops.bus.init_hw, i++); setup_simple_init_func_success(&g->ops.priv_ring.enable_priv_ring, i++); - setup_simple_init_func_success(&g->ops.mc.intr_enable, i++); setup_simple_init_func_success(&g->ops.channel.resume_all_serviceable_ch, i++); setup_simple_init_func_success(&g->ops.pmu.pmu_early_init, i++); setup_simple_init_func_success(&g->ops.acr.acr_init, i++); diff --git a/userspace/units/mm/mm/mm.c b/userspace/units/mm/mm/mm.c index 77e696c60..8373500be 100644 --- a/userspace/units/mm/mm/mm.c +++ b/userspace/units/mm/mm/mm.c @@ -31,6 +31,7 @@ #include "os/posix/os_posix.h" +#include "hal/mc/mc_gp10b.h" #include "hal/mm/mm_gp10b.h" #include "hal/mm/mm_gv11b.h" #include "hal/mm/cache/flush_gk20a.h" @@ -465,6 +466,10 @@ int test_mm_init_hal(struct unit_module *m, struct gk20a *g, void *args) p->mm_is_iommuable = true; g->has_cde = true; + g->ops.mc.intr_stall_unit_config = mc_gp10b_intr_stall_unit_config; + g->ops.mc.intr_nonstall_unit_config = + mc_gp10b_intr_nonstall_unit_config; + g->ops.mm.gmmu.get_default_big_page_size = gp10b_mm_get_default_big_page_size; g->ops.mm.gmmu.get_mmu_levels = gp10b_mm_get_mmu_levels; diff --git a/userspace/units/mm/page_table_faults/page_table_faults.c b/userspace/units/mm/page_table_faults/page_table_faults.c index 0c2a6dd7f..f6f62d790 100644 --- a/userspace/units/mm/page_table_faults/page_table_faults.c +++ b/userspace/units/mm/page_table_faults/page_table_faults.c @@ -39,6 +39,7 @@ #include "os/posix/os_posix.h" #include "hal/fifo/channel_gv11b.h" +#include "hal/mc/mc_gp10b.h" #include "hal/mm/mmu_fault/mmu_fault_gv11b.h" #include "hal/mm/mm_gv11b.h" #include "hal/mm/cache/flush_gk20a.h" @@ -120,6 +121,10 @@ static int init_mm(struct unit_module *m, struct gk20a *g) p->mm_is_iommuable = true; + g->ops.mc.intr_stall_unit_config = mc_gp10b_intr_stall_unit_config; + g->ops.mc.intr_nonstall_unit_config = + mc_gp10b_intr_nonstall_unit_config; + /* Minimum HALs for page_table */ g->ops.mm.gmmu.get_default_big_page_size = gp10b_mm_get_default_big_page_size;