diff --git a/drivers/gpu/nvgpu/common/linux/driver_common.c b/drivers/gpu/nvgpu/common/linux/driver_common.c index 4e2cb2b4e..bd1b7611b 100644 --- a/drivers/gpu/nvgpu/common/linux/driver_common.c +++ b/drivers/gpu/nvgpu/common/linux/driver_common.c @@ -38,8 +38,8 @@ static void nvgpu_init_vars(struct gk20a *g) struct device *dev = dev_from_gk20a(g); struct gk20a_platform *platform = dev_get_drvdata(dev); - init_waitqueue_head(&g->sw_irq_stall_last_handled_wq); - init_waitqueue_head(&g->sw_irq_nonstall_last_handled_wq); + init_waitqueue_head(&l->sw_irq_stall_last_handled_wq); + init_waitqueue_head(&l->sw_irq_nonstall_last_handled_wq); gk20a_init_gr(g); init_rwsem(&g->busy_lock); @@ -236,18 +236,19 @@ static int cyclic_delta(int a, int b) */ void nvgpu_wait_for_deferred_interrupts(struct gk20a *g) { - int stall_irq_threshold = atomic_read(&g->hw_irq_stall_count); - int nonstall_irq_threshold = atomic_read(&g->hw_irq_nonstall_count); + struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); + int stall_irq_threshold = atomic_read(&l->hw_irq_stall_count); + int nonstall_irq_threshold = atomic_read(&l->hw_irq_nonstall_count); /* wait until all stalling irqs are handled */ - wait_event(g->sw_irq_stall_last_handled_wq, + wait_event(l->sw_irq_stall_last_handled_wq, cyclic_delta(stall_irq_threshold, - atomic_read(&g->sw_irq_stall_last_handled)) + atomic_read(&l->sw_irq_stall_last_handled)) <= 0); /* wait until all non-stalling irqs are handled */ - wait_event(g->sw_irq_nonstall_last_handled_wq, + wait_event(l->sw_irq_nonstall_last_handled_wq, cyclic_delta(nonstall_irq_threshold, - atomic_read(&g->sw_irq_nonstall_last_handled)) + atomic_read(&l->sw_irq_nonstall_last_handled)) <= 0); } diff --git a/drivers/gpu/nvgpu/common/linux/intr.c b/drivers/gpu/nvgpu/common/linux/intr.c index 7d699dee4..da177b555 100644 --- a/drivers/gpu/nvgpu/common/linux/intr.c +++ b/drivers/gpu/nvgpu/common/linux/intr.c @@ -18,9 +18,11 @@ #include #include +#include "os_linux.h" irqreturn_t nvgpu_intr_stall(struct gk20a *g) { + struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); u32 mc_intr_0; trace_mc_gk20a_intr_stall(g->name); @@ -35,7 +37,7 @@ irqreturn_t nvgpu_intr_stall(struct gk20a *g) g->ops.mc.intr_stall_pause(g); - atomic_inc(&g->hw_irq_stall_count); + atomic_inc(&l->hw_irq_stall_count); trace_mc_gk20a_intr_stall_done(g->name); @@ -44,14 +46,20 @@ irqreturn_t nvgpu_intr_stall(struct gk20a *g) irqreturn_t nvgpu_intr_thread_stall(struct gk20a *g) { + struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); + int hw_irq_count; + gk20a_dbg(gpu_dbg_intr, "interrupt thread launched"); trace_mc_gk20a_intr_thread_stall(g->name); + hw_irq_count = atomic_read(&l->hw_irq_stall_count); g->ops.mc.isr_stall(g); g->ops.mc.intr_stall_resume(g); + /* sync handled irq counter before re-enabling interrupts */ + atomic_set(&l->sw_irq_stall_last_handled, hw_irq_count); - wake_up_all(&g->sw_irq_stall_last_handled_wq); + wake_up_all(&l->sw_irq_stall_last_handled_wq); trace_mc_gk20a_intr_thread_stall_done(g->name); @@ -66,6 +74,8 @@ irqreturn_t nvgpu_intr_nonstall(struct gk20a *g) u32 active_engine_id = 0; u32 engine_enum = ENGINE_INVAL_GK20A; int ops_old, ops_new, ops = 0; + struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); + if (!g->power_on) return IRQ_NONE; @@ -103,34 +113,36 @@ irqreturn_t nvgpu_intr_nonstall(struct gk20a *g) } if (ops) { do { - ops_old = atomic_read(&g->nonstall_ops); + ops_old = atomic_read(&l->nonstall_ops); ops_new = ops_old | ops; - } while (ops_old != atomic_cmpxchg(&g->nonstall_ops, + } while (ops_old != atomic_cmpxchg(&l->nonstall_ops, ops_old, ops_new)); - queue_work(g->nonstall_work_queue, &g->nonstall_fn_work); + queue_work(l->nonstall_work_queue, &l->nonstall_fn_work); } - hw_irq_count = atomic_inc_return(&g->hw_irq_nonstall_count); + hw_irq_count = atomic_inc_return(&l->hw_irq_nonstall_count); /* sync handled irq counter before re-enabling interrupts */ - atomic_set(&g->sw_irq_nonstall_last_handled, hw_irq_count); + atomic_set(&l->sw_irq_nonstall_last_handled, hw_irq_count); g->ops.mc.intr_nonstall_resume(g); - wake_up_all(&g->sw_irq_nonstall_last_handled_wq); + wake_up_all(&l->sw_irq_nonstall_last_handled_wq); return IRQ_HANDLED; } void nvgpu_intr_nonstall_cb(struct work_struct *work) { - struct gk20a *g = container_of(work, struct gk20a, nonstall_fn_work); + struct nvgpu_os_linux *l = + container_of(work, struct nvgpu_os_linux, nonstall_fn_work); + struct gk20a *g = &l->g; u32 ops; bool semaphore_wakeup, post_events; do { - ops = atomic_xchg(&g->nonstall_ops, 0); + ops = atomic_xchg(&l->nonstall_ops, 0); semaphore_wakeup = ops & gk20a_nonstall_ops_wakeup_semaphore; post_events = ops & gk20a_nonstall_ops_post_events; @@ -138,5 +150,5 @@ void nvgpu_intr_nonstall_cb(struct work_struct *work) if (semaphore_wakeup) gk20a_channel_semaphore_wakeup(g, post_events); - } while (atomic_read(&g->nonstall_ops) != 0); + } while (atomic_read(&l->nonstall_ops) != 0); } diff --git a/drivers/gpu/nvgpu/common/linux/module.c b/drivers/gpu/nvgpu/common/linux/module.c index d19a7a45e..cbfe6ad71 100644 --- a/drivers/gpu/nvgpu/common/linux/module.c +++ b/drivers/gpu/nvgpu/common/linux/module.c @@ -136,6 +136,7 @@ void gk20a_idle(struct gk20a *g) int gk20a_pm_finalize_poweron(struct device *dev) { struct gk20a *g = get_gk20a(dev); + struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); struct gk20a_platform *platform = gk20a_get_platform(dev); int err, nice_value; @@ -163,10 +164,10 @@ int gk20a_pm_finalize_poweron(struct device *dev) set_user_nice(current, -20); /* Enable interrupt workqueue */ - if (!g->nonstall_work_queue) { - g->nonstall_work_queue = alloc_workqueue("%s", + if (!l->nonstall_work_queue) { + l->nonstall_work_queue = alloc_workqueue("%s", WQ_HIGHPRI, 1, "mc_nonstall"); - INIT_WORK(&g->nonstall_fn_work, nvgpu_intr_nonstall_cb); + INIT_WORK(&l->nonstall_fn_work, nvgpu_intr_nonstall_cb); } err = gk20a_finalize_poweron(g); @@ -827,6 +828,34 @@ static int gk20a_pm_init(struct device *dev) return err; } +/* + * Start the process for unloading the driver. Set NVGPU_DRIVER_IS_DYING. + */ +void gk20a_driver_start_unload(struct gk20a *g) +{ + struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); + + gk20a_dbg(gpu_dbg_shutdown, "Driver is now going down!\n"); + + down_write(&g->busy_lock); + __nvgpu_set_enabled(g, NVGPU_DRIVER_IS_DYING, true); + up_write(&g->busy_lock); + + if (g->is_virtual) + return; + + gk20a_wait_for_idle(dev_from_gk20a(g)); + + nvgpu_wait_for_deferred_interrupts(g); + gk20a_channel_cancel_pending_sema_waits(g); + + if (l->nonstall_work_queue) { + cancel_work_sync(&l->nonstall_fn_work); + destroy_workqueue(l->nonstall_work_queue); + l->nonstall_work_queue = NULL; + } +} + static inline void set_gk20a(struct platform_device *pdev, struct gk20a *gk20a) { gk20a_get_platform(&pdev->dev)->g = gk20a; diff --git a/drivers/gpu/nvgpu/common/linux/module.h b/drivers/gpu/nvgpu/common/linux/module.h index 0fde0b416..cfbbc0c77 100644 --- a/drivers/gpu/nvgpu/common/linux/module.h +++ b/drivers/gpu/nvgpu/common/linux/module.h @@ -18,6 +18,7 @@ struct device; int gk20a_pm_finalize_poweron(struct device *dev); void gk20a_remove_support(struct gk20a *g); +void gk20a_driver_start_unload(struct gk20a *g); extern struct class nvgpu_class; diff --git a/drivers/gpu/nvgpu/common/linux/os_linux.h b/drivers/gpu/nvgpu/common/linux/os_linux.h index cf012acc3..8f304fe3e 100644 --- a/drivers/gpu/nvgpu/common/linux/os_linux.h +++ b/drivers/gpu/nvgpu/common/linux/os_linux.h @@ -69,6 +69,20 @@ struct nvgpu_os_linux { struct devfreq *devfreq; struct device_dma_parameters dma_parms; + + atomic_t hw_irq_stall_count; + atomic_t hw_irq_nonstall_count; + + wait_queue_head_t sw_irq_stall_last_handled_wq; + atomic_t sw_irq_stall_last_handled; + + atomic_t nonstall_ops; + + wait_queue_head_t sw_irq_nonstall_last_handled_wq; + atomic_t sw_irq_nonstall_last_handled; + + struct work_struct nonstall_fn_work; + struct workqueue_struct *nonstall_work_queue; }; static inline struct nvgpu_os_linux *nvgpu_os_linux_from_gk20a(struct gk20a *g) diff --git a/drivers/gpu/nvgpu/gk20a/gk20a.c b/drivers/gpu/nvgpu/gk20a/gk20a.c index 21e861feb..6350bcf57 100644 --- a/drivers/gpu/nvgpu/gk20a/gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/gk20a.c @@ -358,32 +358,6 @@ int gk20a_can_busy(struct gk20a *g) return 1; } -/* - * Start the process for unloading the driver. Set NVGPU_DRIVER_IS_DYING. - */ -void gk20a_driver_start_unload(struct gk20a *g) -{ - gk20a_dbg(gpu_dbg_shutdown, "Driver is now going down!\n"); - - down_write(&g->busy_lock); - __nvgpu_set_enabled(g, NVGPU_DRIVER_IS_DYING, true); - up_write(&g->busy_lock); - - if (g->is_virtual) - return; - - gk20a_wait_for_idle(dev_from_gk20a(g)); - - nvgpu_wait_for_deferred_interrupts(g); - gk20a_channel_cancel_pending_sema_waits(g); - - if (g->nonstall_work_queue) { - cancel_work_sync(&g->nonstall_fn_work); - destroy_workqueue(g->nonstall_work_queue); - g->nonstall_work_queue = NULL; - } -} - int gk20a_wait_for_idle(struct device *dev) { struct gk20a *g = get_gk20a(dev); diff --git a/drivers/gpu/nvgpu/gk20a/gk20a.h b/drivers/gpu/nvgpu/gk20a/gk20a.h index 90c35a7b2..8d9318b2f 100644 --- a/drivers/gpu/nvgpu/gk20a/gk20a.h +++ b/drivers/gpu/nvgpu/gk20a/gk20a.h @@ -1042,10 +1042,6 @@ struct gk20a { atomic_t usage_count; - atomic_t nonstall_ops; - struct work_struct nonstall_fn_work; - struct workqueue_struct *nonstall_work_queue; - struct kref refcount; struct resource *reg_mem; @@ -1224,15 +1220,6 @@ struct gk20a { u32 max_ltc_count; u32 ltc_count; - atomic_t hw_irq_stall_count; - atomic_t hw_irq_nonstall_count; - - atomic_t sw_irq_stall_last_handled; - wait_queue_head_t sw_irq_stall_last_handled_wq; - - atomic_t sw_irq_nonstall_last_handled; - wait_queue_head_t sw_irq_nonstall_last_handled_wq; - struct gk20a_channel_worker { struct nvgpu_thread poll_task; atomic_t put; @@ -1485,7 +1472,6 @@ int __gk20a_do_idle(struct gk20a *g, bool force_reset); int __gk20a_do_unidle(struct gk20a *g); int gk20a_can_busy(struct gk20a *g); -void gk20a_driver_start_unload(struct gk20a *g); int gk20a_wait_for_idle(struct device *dev); #define NVGPU_GPU_ARCHITECTURE_SHIFT 4 diff --git a/drivers/gpu/nvgpu/gk20a/mc_gk20a.c b/drivers/gpu/nvgpu/gk20a/mc_gk20a.c index accda9725..e25fcfc30 100644 --- a/drivers/gpu/nvgpu/gk20a/mc_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/mc_gk20a.c @@ -27,13 +27,11 @@ void mc_gk20a_isr_stall(struct gk20a *g) { u32 mc_intr_0; - int hw_irq_count; u32 engine_id_idx; u32 active_engine_id = 0; u32 engine_enum = ENGINE_INVAL_GK20A; mc_intr_0 = g->ops.mc.intr_stall(g); - hw_irq_count = atomic_read(&g->hw_irq_stall_count); gk20a_dbg(gpu_dbg_intr, "stall intr %08x\n", mc_intr_0); @@ -67,9 +65,6 @@ void mc_gk20a_isr_stall(struct gk20a *g) g->ops.ltc.isr(g); if (mc_intr_0 & mc_intr_0_pbus_pending_f()) g->ops.bus.isr(g); - - /* sync handled irq counter before re-enabling interrupts */ - atomic_set(&g->sw_irq_stall_last_handled, hw_irq_count); } void mc_gk20a_intr_enable(struct gk20a *g) diff --git a/drivers/gpu/nvgpu/gp10b/mc_gp10b.c b/drivers/gpu/nvgpu/gp10b/mc_gp10b.c index 718869f6f..b7a523496 100644 --- a/drivers/gpu/nvgpu/gp10b/mc_gp10b.c +++ b/drivers/gpu/nvgpu/gp10b/mc_gp10b.c @@ -71,14 +71,12 @@ void mc_gp10b_intr_unit_config(struct gk20a *g, bool enable, void mc_gp10b_isr_stall(struct gk20a *g) { u32 mc_intr_0; - int hw_irq_count; u32 engine_id_idx; u32 active_engine_id = 0; u32 engine_enum = ENGINE_INVAL_GK20A; mc_intr_0 = gk20a_readl(g, mc_intr_r(0)); - hw_irq_count = atomic_read(&g->hw_irq_stall_count); gk20a_dbg(gpu_dbg_intr, "stall intr 0x%08x\n", mc_intr_0); @@ -116,9 +114,6 @@ void mc_gp10b_isr_stall(struct gk20a *g) if (mc_intr_0 & mc_intr_pbus_pending_f()) g->ops.bus.isr(g); - /* sync handled irq counter before re-enabling interrupts */ - atomic_set(&g->sw_irq_stall_last_handled, hw_irq_count); - gk20a_dbg(gpu_dbg_intr, "stall intr done 0x%08x\n", mc_intr_0); }