diff --git a/drivers/gpu/nvgpu/common/linux/intr.c b/drivers/gpu/nvgpu/common/linux/intr.c index 77e44dd97..7d699dee4 100644 --- a/drivers/gpu/nvgpu/common/linux/intr.c +++ b/drivers/gpu/nvgpu/common/linux/intr.c @@ -12,10 +12,12 @@ */ #include +#include #include "gk20a/gk20a.h" #include +#include irqreturn_t nvgpu_intr_stall(struct gk20a *g) { @@ -56,3 +58,85 @@ irqreturn_t nvgpu_intr_thread_stall(struct gk20a *g) return IRQ_HANDLED; } +irqreturn_t nvgpu_intr_nonstall(struct gk20a *g) +{ + u32 mc_intr_1; + u32 hw_irq_count; + u32 engine_id_idx; + u32 active_engine_id = 0; + u32 engine_enum = ENGINE_INVAL_GK20A; + int ops_old, ops_new, ops = 0; + if (!g->power_on) + return IRQ_NONE; + + /* not from gpu when sharing irq with others */ + mc_intr_1 = g->ops.mc.intr_nonstall(g); + if (unlikely(!mc_intr_1)) + return IRQ_NONE; + + g->ops.mc.intr_nonstall_pause(g); + + if (g->ops.mc.is_intr1_pending(g, NVGPU_UNIT_FIFO, mc_intr_1)) + ops |= gk20a_fifo_nonstall_isr(g); + + for (engine_id_idx = 0; engine_id_idx < g->fifo.num_engines; + engine_id_idx++) { + struct fifo_engine_info_gk20a *engine_info; + + active_engine_id = g->fifo.active_engines_list[engine_id_idx]; + engine_info = &g->fifo.engine_info[active_engine_id]; + + if (mc_intr_1 & engine_info->intr_mask) { + engine_enum = engine_info->engine_enum; + /* GR Engine */ + if (engine_enum == ENGINE_GR_GK20A) + ops |= gk20a_gr_nonstall_isr(g); + + /* CE Engine */ + if (((engine_enum == ENGINE_GRCE_GK20A) || + (engine_enum == ENGINE_ASYNC_CE_GK20A)) && + g->ops.ce2.isr_nonstall) + ops |= g->ops.ce2.isr_nonstall(g, + engine_info->inst_id, + engine_info->pri_base); + } + } + if (ops) { + do { + ops_old = atomic_read(&g->nonstall_ops); + ops_new = ops_old | ops; + } while (ops_old != atomic_cmpxchg(&g->nonstall_ops, + ops_old, ops_new)); + + queue_work(g->nonstall_work_queue, &g->nonstall_fn_work); + } + + hw_irq_count = atomic_inc_return(&g->hw_irq_nonstall_count); + + /* sync handled irq counter before re-enabling interrupts */ + atomic_set(&g->sw_irq_nonstall_last_handled, hw_irq_count); + + g->ops.mc.intr_nonstall_resume(g); + + wake_up_all(&g->sw_irq_nonstall_last_handled_wq); + + return IRQ_HANDLED; +} + +void nvgpu_intr_nonstall_cb(struct work_struct *work) +{ + struct gk20a *g = container_of(work, struct gk20a, nonstall_fn_work); + u32 ops; + bool semaphore_wakeup, post_events; + + do { + ops = atomic_xchg(&g->nonstall_ops, 0); + + semaphore_wakeup = ops & gk20a_nonstall_ops_wakeup_semaphore; + post_events = ops & gk20a_nonstall_ops_post_events; + + if (semaphore_wakeup) + gk20a_channel_semaphore_wakeup(g, post_events); + + } while (atomic_read(&g->nonstall_ops) != 0); +} diff --git a/drivers/gpu/nvgpu/common/linux/intr.h b/drivers/gpu/nvgpu/common/linux/intr.h index 243d8f51a..d43cdccb8 100644 --- a/drivers/gpu/nvgpu/common/linux/intr.h +++ b/drivers/gpu/nvgpu/common/linux/intr.h @@ -17,4 +17,6 @@ struct gk20a; irqreturn_t nvgpu_intr_stall(struct gk20a *g); irqreturn_t nvgpu_intr_thread_stall(struct gk20a *g); +irqreturn_t nvgpu_intr_nonstall(struct gk20a *g); +void nvgpu_intr_nonstall_cb(struct work_struct *work); #endif diff --git a/drivers/gpu/nvgpu/common/linux/module.c b/drivers/gpu/nvgpu/common/linux/module.c index 4f7fc3fa9..34a0ded67 100644 --- a/drivers/gpu/nvgpu/common/linux/module.c +++ b/drivers/gpu/nvgpu/common/linux/module.c @@ -162,6 +162,13 @@ int gk20a_pm_finalize_poweron(struct device *dev) nice_value = task_nice(current); set_user_nice(current, -20); + /* Enable interrupt workqueue */ + if (!g->nonstall_work_queue) { + g->nonstall_work_queue = alloc_workqueue("%s", + WQ_HIGHPRI, 1, "mc_nonstall"); + INIT_WORK(&g->nonstall_fn_work, nvgpu_intr_nonstall_cb); + } + err = gk20a_finalize_poweron(g); set_user_nice(current, nice_value); if (err) @@ -492,7 +499,7 @@ static irqreturn_t gk20a_intr_isr_nonstall(int irq, void *dev_id) { struct gk20a *g = dev_id; - return g->ops.mc.isr_nonstall(g); + return nvgpu_intr_nonstall(g); } static irqreturn_t gk20a_intr_thread_stall(int irq, void *dev_id) diff --git a/drivers/gpu/nvgpu/common/linux/pci.c b/drivers/gpu/nvgpu/common/linux/pci.c index b66a6658a..4351ba5bd 100644 --- a/drivers/gpu/nvgpu/common/linux/pci.c +++ b/drivers/gpu/nvgpu/common/linux/pci.c @@ -235,7 +235,7 @@ static irqreturn_t nvgpu_pci_isr(int irq, void *dev_id) irqreturn_t ret_nonstall; ret_stall = nvgpu_intr_stall(g); - ret_nonstall = g->ops.mc.isr_nonstall(g); + ret_nonstall = nvgpu_intr_nonstall(g); #if defined(CONFIG_PCI_MSI) /* Send MSI EOI */ diff --git a/drivers/gpu/nvgpu/gk20a/gk20a.c b/drivers/gpu/nvgpu/gk20a/gk20a.c index e4d454fee..b038cd818 100644 --- a/drivers/gpu/nvgpu/gk20a/gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/gk20a.c @@ -187,13 +187,6 @@ int gk20a_finalize_poweron(struct gk20a *g) if (g->ops.clk.disable_slowboot) g->ops.clk.disable_slowboot(g); - /* Enable interrupt workqueue */ - if (!g->nonstall_work_queue) { - g->nonstall_work_queue = alloc_workqueue("%s", - WQ_HIGHPRI, 1, "mc_nonstall"); - INIT_WORK(&g->nonstall_fn_work, g->ops.mc.isr_nonstall_cb); - } - gk20a_enable_priv_ring(g); /* TBD: move this after graphics init in which blcg/slcg is enabled. diff --git a/drivers/gpu/nvgpu/gk20a/gk20a.h b/drivers/gpu/nvgpu/gk20a/gk20a.h index bd93cc33b..604940506 100644 --- a/drivers/gpu/nvgpu/gk20a/gk20a.h +++ b/drivers/gpu/nvgpu/gk20a/gk20a.h @@ -36,7 +36,6 @@ struct nvgpu_clk_pll_debug_data; #include #include #include -#include #include #include #ifdef CONFIG_DEBUG_FS @@ -841,14 +840,14 @@ struct gpu_ops { void (*intr_unit_config)(struct gk20a *g, bool enable, bool is_stalling, u32 unit); void (*isr_stall)(struct gk20a *g); - irqreturn_t (*isr_nonstall)(struct gk20a *g); - void (*isr_thread_nonstall)(struct gk20a *g, u32 intr); - void (*isr_nonstall_cb)(struct work_struct *work); bool (*is_intr_hub_pending)(struct gk20a *g, u32 mc_intr); u32 intr_mask_restore[4]; u32 (*intr_stall)(struct gk20a *g); void (*intr_stall_pause)(struct gk20a *g); void (*intr_stall_resume)(struct gk20a *g); + u32 (*intr_nonstall)(struct gk20a *g); + void (*intr_nonstall_pause)(struct gk20a *g); + void (*intr_nonstall_resume)(struct gk20a *g); void (*enable)(struct gk20a *g, u32 units); void (*disable)(struct gk20a *g, u32 units); void (*reset)(struct gk20a *g, u32 units); diff --git a/drivers/gpu/nvgpu/gk20a/mc_gk20a.c b/drivers/gpu/nvgpu/gk20a/mc_gk20a.c index 2cdcaaeb6..111872a20 100644 --- a/drivers/gpu/nvgpu/gk20a/mc_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/mc_gk20a.c @@ -24,62 +24,6 @@ #include -void mc_gk20a_nonstall_cb(struct work_struct *work) -{ - struct gk20a *g = container_of(work, struct gk20a, nonstall_fn_work); - u32 ops; - bool semaphore_wakeup, post_events; - - do { - ops = atomic_xchg(&g->nonstall_ops, 0); - - semaphore_wakeup = ops & gk20a_nonstall_ops_wakeup_semaphore; - post_events = ops & gk20a_nonstall_ops_post_events; - - if (semaphore_wakeup) - gk20a_channel_semaphore_wakeup(g, post_events); - - } while (atomic_read(&g->nonstall_ops) != 0); -} - -irqreturn_t mc_gk20a_isr_nonstall(struct gk20a *g) -{ - u32 mc_intr_1; - u32 hw_irq_count; - - if (!g->power_on) - return IRQ_NONE; - - /* not from gpu when sharing irq with others */ - mc_intr_1 = gk20a_readl(g, mc_intr_1_r()); - if (unlikely(!mc_intr_1)) - return IRQ_NONE; - - gk20a_writel(g, mc_intr_en_1_r(), - mc_intr_en_1_inta_disabled_f()); - - /* flush previous write */ - gk20a_readl(g, mc_intr_en_1_r()); - - if (g->ops.mc.isr_thread_nonstall) - g->ops.mc.isr_thread_nonstall(g, mc_intr_1); - - hw_irq_count = atomic_inc_return(&g->hw_irq_nonstall_count); - - /* sync handled irq counter before re-enabling interrupts */ - atomic_set(&g->sw_irq_nonstall_last_handled, hw_irq_count); - - gk20a_writel(g, mc_intr_en_1_r(), - mc_intr_en_1_inta_hardware_f()); - - /* flush previous write */ - gk20a_readl(g, mc_intr_en_1_r()); - - wake_up_all(&g->sw_irq_nonstall_last_handled_wq); - - return IRQ_HANDLED; -} - void mc_gk20a_isr_stall(struct gk20a *g) { u32 mc_intr_0; @@ -128,49 +72,6 @@ void mc_gk20a_isr_stall(struct gk20a *g) atomic_set(&g->sw_irq_stall_last_handled, hw_irq_count); } -void mc_gk20a_intr_thread_nonstall(struct gk20a *g, u32 mc_intr_1) -{ - u32 engine_id_idx; - u32 active_engine_id = 0; - u32 engine_enum = ENGINE_INVAL_GK20A; - int ops_old, ops_new, ops = 0; - - if (g->ops.mc.is_intr1_pending(g, NVGPU_UNIT_FIFO, mc_intr_1)) - ops |= gk20a_fifo_nonstall_isr(g); - - for (engine_id_idx = 0; engine_id_idx < g->fifo.num_engines; - engine_id_idx++) { - active_engine_id = g->fifo.active_engines_list[engine_id_idx]; - - if (mc_intr_1 & - g->fifo.engine_info[active_engine_id].intr_mask) { - engine_enum = g->fifo.engine_info[active_engine_id].engine_enum; - /* GR Engine */ - if (engine_enum == ENGINE_GR_GK20A) - ops |= gk20a_gr_nonstall_isr(g); - - /* CE Engine */ - if (((engine_enum == ENGINE_GRCE_GK20A) || - (engine_enum == ENGINE_ASYNC_CE_GK20A)) && - g->ops.ce2.isr_nonstall) - ops |= g->ops.ce2.isr_nonstall(g, - g->fifo.engine_info[active_engine_id]. - inst_id, - g->fifo.engine_info[active_engine_id]. - pri_base); - } - } - if (ops) { - do { - ops_old = atomic_read(&g->nonstall_ops); - ops_new = ops_old | ops; - } while (ops_old != atomic_cmpxchg(&g->nonstall_ops, - ops_old, ops_new)); - - queue_work(g->nonstall_work_queue, &g->nonstall_fn_work); - } -} - void mc_gk20a_intr_enable(struct gk20a *g) { u32 eng_intr_mask = gk20a_fifo_engine_interrupt_mask(g); @@ -226,11 +127,34 @@ void mc_gk20a_intr_stall_resume(struct gk20a *g) gk20a_readl(g, mc_intr_en_0_r()); } +void mc_gk20a_intr_nonstall_pause(struct gk20a *g) +{ + gk20a_writel(g, mc_intr_en_1_r(), + mc_intr_en_0_inta_disabled_f()); + + /* flush previous write */ + gk20a_readl(g, mc_intr_en_1_r()); +} + +void mc_gk20a_intr_nonstall_resume(struct gk20a *g) +{ + gk20a_writel(g, mc_intr_en_1_r(), + mc_intr_en_0_inta_hardware_f()); + + /* flush previous write */ + gk20a_readl(g, mc_intr_en_1_r()); +} + u32 mc_gk20a_intr_stall(struct gk20a *g) { return gk20a_readl(g, mc_intr_0_r()); } +u32 mc_gk20a_intr_nonstall(struct gk20a *g) +{ + return gk20a_readl(g, mc_intr_1_r()); +} + void gk20a_mc_disable(struct gk20a *g, u32 units) { u32 pmc; @@ -320,9 +244,9 @@ void gk20a_init_mc(struct gpu_ops *gops) gops->mc.intr_stall = mc_gk20a_intr_stall; gops->mc.intr_stall_pause = mc_gk20a_intr_stall_pause; gops->mc.intr_stall_resume = mc_gk20a_intr_stall_resume; - gops->mc.isr_nonstall = mc_gk20a_isr_nonstall; - gops->mc.isr_thread_nonstall = mc_gk20a_intr_thread_nonstall; - gops->mc.isr_nonstall_cb = mc_gk20a_nonstall_cb; + gops->mc.intr_nonstall = mc_gk20a_intr_nonstall; + gops->mc.intr_nonstall_pause = mc_gk20a_intr_nonstall_pause; + gops->mc.intr_nonstall_resume = mc_gk20a_intr_nonstall_resume; gops->mc.enable = gk20a_mc_enable; gops->mc.disable = gk20a_mc_disable; gops->mc.reset = gk20a_mc_reset; diff --git a/drivers/gpu/nvgpu/gk20a/mc_gk20a.h b/drivers/gpu/nvgpu/gk20a/mc_gk20a.h index 6c0d159a8..7ecd27bf5 100644 --- a/drivers/gpu/nvgpu/gk20a/mc_gk20a.h +++ b/drivers/gpu/nvgpu/gk20a/mc_gk20a.h @@ -23,10 +23,9 @@ void mc_gk20a_isr_stall(struct gk20a *g); u32 mc_gk20a_intr_stall(struct gk20a *g); void mc_gk20a_intr_stall_pause(struct gk20a *g); void mc_gk20a_intr_stall_resume(struct gk20a *g); -irqreturn_t mc_gk20a_isr_nonstall(struct gk20a *g); -irqreturn_t mc_gk20a_intr_thread_stall(struct gk20a *g); -void mc_gk20a_intr_thread_nonstall(struct gk20a *g, u32 intr); -void mc_gk20a_nonstall_cb(struct work_struct *work); +u32 mc_gk20a_intr_nonstall(struct gk20a *g); +void mc_gk20a_intr_nonstall_pause(struct gk20a *g); +void mc_gk20a_intr_nonstall_resume(struct gk20a *g); void gk20a_mc_enable(struct gk20a *g, u32 units); void gk20a_mc_disable(struct gk20a *g, u32 units); void gk20a_mc_reset(struct gk20a *g, u32 units); diff --git a/drivers/gpu/nvgpu/gm20b/mc_gm20b.c b/drivers/gpu/nvgpu/gm20b/mc_gm20b.c index 7330a0274..005ec729a 100644 --- a/drivers/gpu/nvgpu/gm20b/mc_gm20b.c +++ b/drivers/gpu/nvgpu/gm20b/mc_gm20b.c @@ -25,9 +25,9 @@ void gm20b_init_mc(struct gpu_ops *gops) gops->mc.intr_stall = mc_gk20a_intr_stall; gops->mc.intr_stall_pause = mc_gk20a_intr_stall_pause; gops->mc.intr_stall_resume = mc_gk20a_intr_stall_resume; - gops->mc.isr_nonstall = mc_gk20a_isr_nonstall; - gops->mc.isr_thread_nonstall = mc_gk20a_intr_thread_nonstall; - gops->mc.isr_nonstall_cb = mc_gk20a_nonstall_cb; + gops->mc.intr_nonstall = mc_gk20a_intr_nonstall; + gops->mc.intr_nonstall_pause = mc_gk20a_intr_nonstall_pause; + gops->mc.intr_nonstall_resume = mc_gk20a_intr_nonstall_resume; gops->mc.enable = gk20a_mc_enable; gops->mc.disable = gk20a_mc_disable; gops->mc.reset = gk20a_mc_reset; diff --git a/drivers/gpu/nvgpu/gp10b/mc_gp10b.c b/drivers/gpu/nvgpu/gp10b/mc_gp10b.c index 7ccea3705..39ad8f9b3 100644 --- a/drivers/gpu/nvgpu/gp10b/mc_gp10b.c +++ b/drivers/gpu/nvgpu/gp10b/mc_gp10b.c @@ -68,37 +68,6 @@ void mc_gp10b_intr_unit_config(struct gk20a *g, bool enable, gk20a_writel(g, reg, mask); } -irqreturn_t mc_gp10b_isr_nonstall(struct gk20a *g) -{ - u32 mc_intr_1; - u32 hw_irq_count; - - if (!g->power_on) - return IRQ_NONE; - - /* not from gpu when sharing irq with others */ - mc_intr_1 = gk20a_readl(g, mc_intr_r(1)); - if (unlikely(!mc_intr_1)) - return IRQ_NONE; - - gk20a_writel(g, mc_intr_en_clear_r(1), 0xffffffff); - - if (g->ops.mc.isr_thread_nonstall) - g->ops.mc.isr_thread_nonstall(g, mc_intr_1); - - hw_irq_count = atomic_inc_return(&g->hw_irq_nonstall_count); - - gk20a_writel(g, mc_intr_en_set_r(NVGPU_MC_INTR_NONSTALLING), - g->ops.mc.intr_mask_restore[NVGPU_MC_INTR_NONSTALLING]); - - /* sync handled irq counter before re-enabling interrupts */ - atomic_set(&g->sw_irq_nonstall_last_handled, hw_irq_count); - - wake_up_all(&g->sw_irq_nonstall_last_handled_wq); - - return IRQ_HANDLED; -} - void mc_gp10b_isr_stall(struct gk20a *g) { u32 mc_intr_0; @@ -170,6 +139,23 @@ void mc_gp10b_intr_stall_resume(struct gk20a *g) g->ops.mc.intr_mask_restore[NVGPU_MC_INTR_STALLING]); } +u32 mc_gp10b_intr_nonstall(struct gk20a *g) +{ + return gk20a_readl(g, mc_intr_r(NVGPU_MC_INTR_NONSTALLING)); +} + +void mc_gp10b_intr_nonstall_pause(struct gk20a *g) +{ + gk20a_writel(g, mc_intr_en_clear_r(NVGPU_MC_INTR_NONSTALLING), + 0xffffffff); +} + +void mc_gp10b_intr_nonstall_resume(struct gk20a *g) +{ + gk20a_writel(g, mc_intr_en_set_r(NVGPU_MC_INTR_NONSTALLING), + g->ops.mc.intr_mask_restore[NVGPU_MC_INTR_NONSTALLING]); +} + static bool mc_gp10b_is_intr1_pending(struct gk20a *g, enum nvgpu_unit unit, u32 mc_intr_1) { @@ -202,9 +188,9 @@ void gp10b_init_mc(struct gpu_ops *gops) gops->mc.intr_stall = mc_gp10b_intr_stall; gops->mc.intr_stall_pause = mc_gp10b_intr_stall_pause; gops->mc.intr_stall_resume = mc_gp10b_intr_stall_resume; - gops->mc.isr_nonstall = mc_gp10b_isr_nonstall; - gops->mc.isr_thread_nonstall = mc_gk20a_intr_thread_nonstall; - gops->mc.isr_nonstall_cb = mc_gk20a_nonstall_cb; + gops->mc.intr_nonstall = mc_gp10b_intr_nonstall; + gops->mc.intr_nonstall_pause = mc_gp10b_intr_nonstall_pause; + gops->mc.intr_nonstall_resume = mc_gp10b_intr_nonstall_resume; gops->mc.enable = gk20a_mc_enable; gops->mc.disable = gk20a_mc_disable; gops->mc.reset = gk20a_mc_reset; diff --git a/drivers/gpu/nvgpu/gp10b/mc_gp10b.h b/drivers/gpu/nvgpu/gp10b/mc_gp10b.h index 31867a888..ceba0b39f 100644 --- a/drivers/gpu/nvgpu/gp10b/mc_gp10b.h +++ b/drivers/gpu/nvgpu/gp10b/mc_gp10b.h @@ -25,6 +25,4 @@ void mc_gp10b_intr_enable(struct gk20a *g); void mc_gp10b_intr_unit_config(struct gk20a *g, bool enable, bool is_stalling, u32 mask); void mc_gp10b_isr_stall(struct gk20a *g); -irqreturn_t mc_gp10b_isr_nonstall(struct gk20a *g); -irqreturn_t mc_gp10b_intr_thread_nonstall(struct gk20a *g); #endif