From c6908922e55ea82f6dbaf31d8770ad90d093545d Mon Sep 17 00:00:00 2001 From: Vedashree Vidwans Date: Fri, 10 Apr 2020 08:59:46 -0700 Subject: [PATCH] gpu: nvgpu: move generic preempt hals to common - Move fifo.preempt_runlists_for_rc and fifo.preempt_tsg hals to common source file as nvgpu_fifo_preempt_runlists_for_rc and nvgpu_fifo_preempt_tsg. Jira NVGPU-4881 Change-Id: I31f7973276c075130d8a0ac684c6c99e35be6017 Signed-off-by: Vedashree Vidwans Reviewed-on: https://git-master.nvidia.com/r/c/linux-nvgpu/+/2323866 Tested-by: mobile promotions Reviewed-by: mobile promotions --- drivers/gpu/nvgpu/common/fifo/fifo.c | 3 +- drivers/gpu/nvgpu/common/fifo/preempt.c | 120 ++++++++++++++- drivers/gpu/nvgpu/hal/fifo/preempt_gv11b.h | 5 +- .../gpu/nvgpu/hal/fifo/preempt_gv11b_fusa.c | 138 +----------------- .../gpu/nvgpu/hal/fifo/runlist_fifo_gk20a.c | 14 +- .../gpu/nvgpu/hal/fifo/runlist_fifo_gk20a.h | 4 +- drivers/gpu/nvgpu/hal/init/hal_gv11b.c | 9 +- drivers/gpu/nvgpu/hal/init/hal_tu104.c | 4 +- drivers/gpu/nvgpu/hal/rc/rc_gv11b.c | 2 +- drivers/gpu/nvgpu/include/nvgpu/fifo.h | 4 + drivers/gpu/nvgpu/include/nvgpu/gops_fifo.h | 36 ++--- drivers/gpu/nvgpu/include/nvgpu/preempt.h | 46 +++++- drivers/gpu/nvgpu/libnvgpu-drv_safe.export | 4 +- .../fifo/preempt/gv11b/nvgpu-preempt-gv11b.c | 7 +- userspace/units/init/nvgpu-init.c | 5 - .../gv11b_fusa/mmu-fault-gv11b-fusa.c | 3 +- 16 files changed, 215 insertions(+), 189 deletions(-) diff --git a/drivers/gpu/nvgpu/common/fifo/fifo.c b/drivers/gpu/nvgpu/common/fifo/fifo.c index 343630b77..2b31eed32 100644 --- a/drivers/gpu/nvgpu/common/fifo/fifo.c +++ b/drivers/gpu/nvgpu/common/fifo/fifo.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -288,6 +289,6 @@ void nvgpu_fifo_sw_quiesce(struct gk20a *g) g->ops.runlist.write_state(g, runlist_mask, RUNLIST_DISABLED); /* Preempt all runlists */ - g->ops.fifo.preempt_runlists_for_rc(g, runlist_mask); + nvgpu_fifo_preempt_runlists_for_rc(g, runlist_mask); } #endif diff --git a/drivers/gpu/nvgpu/common/fifo/preempt.c b/drivers/gpu/nvgpu/common/fifo/preempt.c index 272f3ff9d..9046b3c24 100644 --- a/drivers/gpu/nvgpu/common/fifo/preempt.c +++ b/drivers/gpu/nvgpu/common/fifo/preempt.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2019, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2011-2020, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -20,6 +20,7 @@ * DEALINGS IN THE SOFTWARE. */ +#include #include #include #include @@ -27,13 +28,75 @@ #include #include #include - +#include +#ifdef CONFIG_NVGPU_LS_PMU +#include +#endif u32 nvgpu_preempt_get_timeout(struct gk20a *g) { return g->ctxsw_timeout_period_ms; } +int nvgpu_fifo_preempt_tsg(struct gk20a *g, struct nvgpu_tsg *tsg) +{ + struct nvgpu_fifo *f = &g->fifo; + int ret = 0; +#ifdef CONFIG_NVGPU_LS_PMU + u32 token = PMU_INVALID_MUTEX_OWNER_ID; + int mutex_ret = 0; +#endif + u32 runlist_id; + + nvgpu_log_fn(g, "tsgid: %d", tsg->tsgid); + + runlist_id = tsg->runlist_id; + if (runlist_id == NVGPU_INVALID_RUNLIST_ID) { + return 0; + } + + nvgpu_mutex_acquire(&f->runlist_info[runlist_id]->runlist_lock); + + /* WAR for Bug 2065990 */ + nvgpu_tsg_disable_sched(g, tsg); +#ifdef CONFIG_NVGPU_LS_PMU + mutex_ret = nvgpu_pmu_lock_acquire(g, g->pmu, + PMU_MUTEX_ID_FIFO, &token); +#endif + nvgpu_log_fn(g, "preempt id: %d", tsg->tsgid); + + g->ops.fifo.preempt_trigger(g, tsg->tsgid, ID_TYPE_TSG); + + /* poll for preempt done */ + ret = g->ops.fifo.is_preempt_pending(g, tsg->tsgid, ID_TYPE_TSG); + +#ifdef CONFIG_NVGPU_LS_PMU + if (mutex_ret == 0) { + int err = nvgpu_pmu_lock_release(g, g->pmu, PMU_MUTEX_ID_FIFO, + &token); + if (err != 0) { + nvgpu_err(g, "PMU_MUTEX_ID_FIFO not released err=%d", + err); + } + } +#endif + /* WAR for Bug 2065990 */ + nvgpu_tsg_enable_sched(g, tsg); + + nvgpu_mutex_release(&f->runlist_info[runlist_id]->runlist_lock); + + if (ret != 0) { + if (nvgpu_platform_is_silicon(g)) { + nvgpu_err(g, "preempt timed out for tsgid: %u, " + "ctxsw timeout will trigger recovery if needed", + tsg->tsgid); + } else { + nvgpu_rc_preempt_timeout(g, tsg); + } + } + return ret; +} + int nvgpu_preempt_channel(struct gk20a *g, struct nvgpu_channel *ch) { int err; @@ -85,3 +148,56 @@ void nvgpu_preempt_poll_tsg_on_pbdma(struct gk20a *g, } } } + +/* + * This should be called with runlist_lock held for all the + * runlists set in runlists_mask + */ +void nvgpu_fifo_preempt_runlists_for_rc(struct gk20a *g, u32 runlists_bitmask) +{ + struct nvgpu_fifo *f = &g->fifo; + u32 i; +#ifdef CONFIG_NVGPU_LS_PMU + u32 token = PMU_INVALID_MUTEX_OWNER_ID; + int mutex_ret = 0; +#endif + + /* runlist_lock are locked by teardown and sched are disabled too */ + nvgpu_log_fn(g, "preempt runlists_bitmask:0x%08x", runlists_bitmask); +#ifdef CONFIG_NVGPU_LS_PMU + mutex_ret = nvgpu_pmu_lock_acquire(g, g->pmu, + PMU_MUTEX_ID_FIFO, &token); +#endif + + for (i = 0U; i < f->num_runlists; i++) { + struct nvgpu_runlist_info *runlist; + + runlist = &f->active_runlist_info[i]; + + if ((BIT32(runlist->runlist_id) & runlists_bitmask) == 0U) { + continue; + } + /* issue runlist preempt */ + g->ops.fifo.preempt_trigger(g, runlist->runlist_id, + ID_TYPE_RUNLIST); +#ifdef CONFIG_NVGPU_RECOVERY + /* + * Preemption will never complete in RC due to some + * fatal condition. Do not poll for preemption to + * complete. Reset engines served by runlists. + */ + runlist->reset_eng_bitmask = runlist->eng_bitmask; +#endif + } + +#ifdef CONFIG_NVGPU_LS_PMU + if (mutex_ret == 0) { + int err = nvgpu_pmu_lock_release(g, g->pmu, PMU_MUTEX_ID_FIFO, + &token); + if (err != 0) { + nvgpu_err(g, "PMU_MUTEX_ID_FIFO not released err=%d", + err); + } + } +#endif +} diff --git a/drivers/gpu/nvgpu/hal/fifo/preempt_gv11b.h b/drivers/gpu/nvgpu/hal/fifo/preempt_gv11b.h index 869ef2e88..0d8a17130 100644 --- a/drivers/gpu/nvgpu/hal/fifo/preempt_gv11b.h +++ b/drivers/gpu/nvgpu/hal/fifo/preempt_gv11b.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2019, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2016-2020, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -32,11 +32,8 @@ struct nvgpu_tsg; void gv11b_fifo_preempt_trigger(struct gk20a *g, u32 id, unsigned int id_type); int gv11b_fifo_preempt_channel(struct gk20a *g, struct nvgpu_channel *ch); -int gv11b_fifo_preempt_tsg(struct gk20a *g, struct nvgpu_tsg *tsg); int gv11b_fifo_is_preempt_pending(struct gk20a *g, u32 id, unsigned int id_type); -void gv11b_fifo_preempt_runlists_for_rc(struct gk20a *g, u32 runlists_mask); - int gv11b_fifo_preempt_poll_pbdma(struct gk20a *g, u32 tsgid, u32 pbdma_id); #endif /* FIFO_PREEMPT_GV11B_H */ diff --git a/drivers/gpu/nvgpu/hal/fifo/preempt_gv11b_fusa.c b/drivers/gpu/nvgpu/hal/fifo/preempt_gv11b_fusa.c index a0b47be8d..67f5080ef 100644 --- a/drivers/gpu/nvgpu/hal/fifo/preempt_gv11b_fusa.c +++ b/drivers/gpu/nvgpu/hal/fifo/preempt_gv11b_fusa.c @@ -27,7 +27,6 @@ #include #include #include -#include #include #include #include @@ -52,87 +51,17 @@ void gv11b_fifo_preempt_trigger(struct gk20a *g, u32 id, unsigned int id_type) nvgpu_writel(g, fifo_preempt_r(), fifo_preempt_id_f(id) | fifo_preempt_type_tsg_f()); + } else if (id_type == ID_TYPE_RUNLIST) { + u32 reg_val; + + reg_val = nvgpu_readl(g, fifo_runlist_preempt_r()); + reg_val |= BIT32(id); + nvgpu_writel(g, fifo_runlist_preempt_r(), reg_val); } else { nvgpu_log_info(g, "channel preempt is noop"); } } -static void gv11b_fifo_issue_runlist_preempt(struct gk20a *g, - u32 runlists_mask) -{ - u32 reg_val; - - /* issue runlist preempt */ - reg_val = nvgpu_readl(g, fifo_runlist_preempt_r()); - reg_val |= runlists_mask; - nvgpu_writel(g, fifo_runlist_preempt_r(), reg_val); -} - -static int gv11b_fifo_preempt_locked(struct gk20a *g, u32 id, - unsigned int id_type) -{ - nvgpu_log_fn(g, "preempt id: %d id_type: %d", id, id_type); - - g->ops.fifo.preempt_trigger(g, id, id_type); - - /* poll for preempt done */ - return g->ops.fifo.is_preempt_pending(g, id, id_type); - -} - -/* - * This should be called with runlist_lock held for all the - * runlists set in runlists_mask - */ -void gv11b_fifo_preempt_runlists_for_rc(struct gk20a *g, u32 runlists_mask) -{ -#ifdef CONFIG_NVGPU_LS_PMU - u32 token = PMU_INVALID_MUTEX_OWNER_ID; - int mutex_ret = 0; -#endif -#ifdef CONFIG_NVGPU_RECOVERY - struct nvgpu_fifo *f = &g->fifo; - u32 i; -#endif - - /* runlist_lock are locked by teardown and sched are disabled too */ - nvgpu_log_fn(g, "preempt runlists_mask:0x%08x", runlists_mask); -#ifdef CONFIG_NVGPU_LS_PMU - mutex_ret = nvgpu_pmu_lock_acquire(g, g->pmu, - PMU_MUTEX_ID_FIFO, &token); -#endif - /* issue runlist preempt */ - gv11b_fifo_issue_runlist_preempt(g, runlists_mask); - -#ifdef CONFIG_NVGPU_RECOVERY - /* - * Preemption will never complete in RC due to some fatal condition. - * Do not poll for preemption to complete. Reset engines served by - * runlists. - */ - for (i = 0U; i < f->num_runlists; i++) { - struct nvgpu_runlist_info *runlist; - - runlist = &f->active_runlist_info[i]; - - if ((fifo_runlist_preempt_runlist_m(runlist->runlist_id) & - runlists_mask) != 0U) { - runlist->reset_eng_bitmask = runlist->eng_bitmask; - } - } -#endif -#ifdef CONFIG_NVGPU_LS_PMU - if (mutex_ret == 0) { - int err = nvgpu_pmu_lock_release(g, g->pmu, PMU_MUTEX_ID_FIFO, - &token); - if (err != 0) { - nvgpu_err(g, "PMU_MUTEX_ID_FIFO not released err=%d", - err); - } - } -#endif -} - static int fifo_preempt_check_tsg_on_pbdma(u32 tsgid, struct nvgpu_pbdma_status_info *pbdma_status) { @@ -446,58 +375,3 @@ int gv11b_fifo_preempt_channel(struct gk20a *g, struct nvgpu_channel *ch) /* Preempt tsg. Channel preempt is NOOP */ return g->ops.fifo.preempt_tsg(g, tsg); } - -int gv11b_fifo_preempt_tsg(struct gk20a *g, struct nvgpu_tsg *tsg) -{ - struct nvgpu_fifo *f = &g->fifo; - int ret = 0; -#ifdef CONFIG_NVGPU_LS_PMU - u32 token = PMU_INVALID_MUTEX_OWNER_ID; - int mutex_ret = 0; -#endif - u32 runlist_id; - - nvgpu_log_fn(g, "tsgid: %d", tsg->tsgid); - - runlist_id = tsg->runlist_id; - nvgpu_log_fn(g, "runlist_id: %d", runlist_id); - if (runlist_id == NVGPU_INVALID_RUNLIST_ID) { - return 0; - } - - nvgpu_mutex_acquire(&f->runlist_info[runlist_id]->runlist_lock); - - /* WAR for Bug 2065990 */ - nvgpu_tsg_disable_sched(g, tsg); -#ifdef CONFIG_NVGPU_LS_PMU - mutex_ret = nvgpu_pmu_lock_acquire(g, g->pmu, - PMU_MUTEX_ID_FIFO, &token); -#endif - ret = gv11b_fifo_preempt_locked(g, tsg->tsgid, ID_TYPE_TSG); -#ifdef CONFIG_NVGPU_LS_PMU - if (mutex_ret == 0) { - int err = nvgpu_pmu_lock_release(g, g->pmu, PMU_MUTEX_ID_FIFO, - &token); - if (err != 0) { - nvgpu_err(g, "PMU_MUTEX_ID_FIFO not released err=%d", - err); - } - } -#endif - /* WAR for Bug 2065990 */ - nvgpu_tsg_enable_sched(g, tsg); - - nvgpu_mutex_release(&f->runlist_info[runlist_id]->runlist_lock); - - if (ret != 0) { - if (nvgpu_platform_is_silicon(g)) { - nvgpu_err(g, "preempt timed out for tsgid: %u, " - "ctxsw timeout will trigger recovery if needed", - tsg->tsgid); - } else { - nvgpu_rc_preempt_timeout(g, tsg); - } - } - - return ret; -} diff --git a/drivers/gpu/nvgpu/hal/fifo/runlist_fifo_gk20a.c b/drivers/gpu/nvgpu/hal/fifo/runlist_fifo_gk20a.c index 7b6152b7a..6ec6bfa47 100644 --- a/drivers/gpu/nvgpu/hal/fifo/runlist_fifo_gk20a.c +++ b/drivers/gpu/nvgpu/hal/fifo/runlist_fifo_gk20a.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2019, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2011-2020, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -35,19 +35,17 @@ #define FECS_MAILBOX_0_ACK_RESTORE 0x4U -#ifdef CONFIG_NVGPU_CHANNEL_TSG_SCHEDULING -int gk20a_runlist_reschedule(struct nvgpu_channel *ch, bool preempt_next) -{ - return nvgpu_runlist_reschedule(ch, preempt_next, true); -} -#endif - u32 gk20a_runlist_count_max(void) { return fifo_eng_runlist_base__size_1_v(); } #ifdef CONFIG_NVGPU_CHANNEL_TSG_SCHEDULING +int gk20a_runlist_reschedule(struct nvgpu_channel *ch, bool preempt_next) +{ + return nvgpu_runlist_reschedule(ch, preempt_next, true); +} + /* trigger host preempt of GR pending load ctx if that ctx is not for ch */ int gk20a_fifo_reschedule_preempt_next(struct nvgpu_channel *ch, bool wait_preempt) diff --git a/drivers/gpu/nvgpu/hal/fifo/runlist_fifo_gk20a.h b/drivers/gpu/nvgpu/hal/fifo/runlist_fifo_gk20a.h index 066a0289c..5d255f652 100644 --- a/drivers/gpu/nvgpu/hal/fifo/runlist_fifo_gk20a.h +++ b/drivers/gpu/nvgpu/hal/fifo/runlist_fifo_gk20a.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2019, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2011-2020, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -30,11 +30,11 @@ struct nvgpu_tsg; struct gk20a; #ifdef CONFIG_NVGPU_CHANNEL_TSG_SCHEDULING +int gk20a_runlist_reschedule(struct nvgpu_channel *ch, bool preempt_next); int gk20a_fifo_reschedule_preempt_next(struct nvgpu_channel *ch, bool wait_preempt); #endif #ifdef CONFIG_NVGPU_HAL_NON_FUSA -int gk20a_runlist_reschedule(struct nvgpu_channel *ch, bool preempt_next); u32 gk20a_runlist_count_max(void); #endif diff --git a/drivers/gpu/nvgpu/hal/init/hal_gv11b.c b/drivers/gpu/nvgpu/hal/init/hal_gv11b.c index e240a8ed8..196fe6201 100644 --- a/drivers/gpu/nvgpu/hal/init/hal_gv11b.c +++ b/drivers/gpu/nvgpu/hal/init/hal_gv11b.c @@ -35,6 +35,7 @@ #include #include #include +#include #include #include #include @@ -936,9 +937,8 @@ NVGPU_COV_WHITELIST_BLOCK_END(NVGPU_MISRA(Rule, 8_7)) .fifo_suspend = nvgpu_fifo_suspend, .init_fifo_setup_hw = gv11b_init_fifo_setup_hw, .preempt_channel = gv11b_fifo_preempt_channel, - .preempt_tsg = gv11b_fifo_preempt_tsg, + .preempt_tsg = nvgpu_fifo_preempt_tsg, .preempt_trigger = gv11b_fifo_preempt_trigger, - .preempt_runlists_for_rc = gv11b_fifo_preempt_runlists_for_rc, .preempt_poll_pbdma = gv11b_fifo_preempt_poll_pbdma, .init_pbdma_map = gk20a_fifo_init_pbdma_map, .is_preempt_pending = gv11b_fifo_is_preempt_pending, @@ -1072,9 +1072,10 @@ NVGPU_COV_WHITELIST_BLOCK_END(NVGPU_MISRA(Rule, 8_7)) .set_eng_method_buffer = gv11b_ramin_set_eng_method_buffer, }, .runlist = { -#ifdef NVGPU_CHANNEL_TSG_SCHEULING +#ifdef CONFIG_NVGPU_CHANNEL_TSG_SCHEDULING .reschedule = gv11b_runlist_reschedule, - .reschedule_preempt_next_locked = gk20a_fifo_reschedule_preempt_next, + .reschedule_preempt_next_locked = + gk20a_fifo_reschedule_preempt_next, #endif .update_for_channel = nvgpu_runlist_update_for_channel, .reload = nvgpu_runlist_reload, diff --git a/drivers/gpu/nvgpu/hal/init/hal_tu104.c b/drivers/gpu/nvgpu/hal/init/hal_tu104.c index c6daed9c4..0ae11068f 100644 --- a/drivers/gpu/nvgpu/hal/init/hal_tu104.c +++ b/drivers/gpu/nvgpu/hal/init/hal_tu104.c @@ -21,6 +21,7 @@ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER * DEALINGS IN THE SOFTWARE. */ +#include #include "hal/mm/mm_gm20b.h" #include "hal/mm/mm_gp10b.h" @@ -961,9 +962,8 @@ static const struct gpu_ops tu104_ops = { .fifo_suspend = nvgpu_fifo_suspend, .init_fifo_setup_hw = tu104_init_fifo_setup_hw, .preempt_channel = gv11b_fifo_preempt_channel, - .preempt_tsg = gv11b_fifo_preempt_tsg, + .preempt_tsg = nvgpu_fifo_preempt_tsg, .preempt_trigger = gv11b_fifo_preempt_trigger, - .preempt_runlists_for_rc = gv11b_fifo_preempt_runlists_for_rc, .preempt_poll_pbdma = gv11b_fifo_preempt_poll_pbdma, .init_pbdma_map = gk20a_fifo_init_pbdma_map, .is_preempt_pending = gv11b_fifo_is_preempt_pending, diff --git a/drivers/gpu/nvgpu/hal/rc/rc_gv11b.c b/drivers/gpu/nvgpu/hal/rc/rc_gv11b.c index 3ecbf24b0..2800c54f5 100644 --- a/drivers/gpu/nvgpu/hal/rc/rc_gv11b.c +++ b/drivers/gpu/nvgpu/hal/rc/rc_gv11b.c @@ -218,7 +218,7 @@ void gv11b_fifo_recover(struct gk20a *g, u32 act_eng_bitmask, * that all PBDMAs serving the engine are not loaded when engine is * reset. */ - g->ops.fifo.preempt_runlists_for_rc(g, runlists_mask); + nvgpu_fifo_preempt_runlists_for_rc(g, runlists_mask); /* * For each PBDMA which serves the runlist, poll to verify the TSG is no * longer on the PBDMA and the engine phase of the preempt has started. diff --git a/drivers/gpu/nvgpu/include/nvgpu/fifo.h b/drivers/gpu/nvgpu/include/nvgpu/fifo.h index fc55cd80c..5cdfc2255 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/fifo.h +++ b/drivers/gpu/nvgpu/include/nvgpu/fifo.h @@ -205,6 +205,10 @@ * H/w defined value for Tsg ID type */ #define ID_TYPE_TSG 1U +/** + * S/w defined value for Runlist ID type + */ +#define ID_TYPE_RUNLIST 2U /** * S/w defined value for unknown ID type. */ diff --git a/drivers/gpu/nvgpu/include/nvgpu/gops_fifo.h b/drivers/gpu/nvgpu/include/nvgpu/gops_fifo.h index cfe2b5492..1da49298f 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/gops_fifo.h +++ b/drivers/gpu/nvgpu/include/nvgpu/gops_fifo.h @@ -92,25 +92,6 @@ struct gops_fifo { */ int (*preempt_tsg)(struct gk20a *g, struct nvgpu_tsg *tsg); - /** - * @brief Preempt a set of runlists. - * - * @param g [in] Pointer to GPU driver struct. - * @param runlists_mask [in] Bitmask of runlists to preempt. - * - * Preempt runlists in \a runlists_mask: - * - Write h/w register to trigger preempt on runlists. - * - All TSG in those runlists are preempted. - * - * @note This HAL is called in case of critical error, and does - * not poll PBDMAs or engines to wait for preempt completion. - * - * @note This HAL should be called with runlist lock held for all - * the runlists in \a runlists_mask. - */ - void (*preempt_runlists_for_rc)(struct gk20a *g, - u32 runlists_bitmask); - /** * @brief Enable and configure FIFO. * @@ -179,8 +160,21 @@ struct gops_fifo { void (*cleanup_sw)(struct gk20a *g); int (*init_fifo_setup_hw)(struct gk20a *g); int (*preempt_channel)(struct gk20a *g, struct nvgpu_channel *ch); - void (*preempt_trigger)(struct gk20a *g, - u32 id, unsigned int id_type); + /** + * @brief Preempt requested channel,tsg or runlist. + * + * @param g [in] Pointer to GPU driver struct. + * @param id [in] Tsg or channel or hardware runlist id + * @param id_type [in] channel,tsg or runlist type + * + * Depending on given \a id_type: + * - Preempt channel + * - Preempt tsg + * - Preempt runlist + * + * @return: None + */ + void (*preempt_trigger)(struct gk20a *g, u32 id, unsigned int id_type); int (*preempt_poll_pbdma)(struct gk20a *g, u32 tsgid, u32 pbdma_id); void (*init_pbdma_map)(struct gk20a *g, diff --git a/drivers/gpu/nvgpu/include/nvgpu/preempt.h b/drivers/gpu/nvgpu/include/nvgpu/preempt.h index 23724e105..f848a6f4f 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/preempt.h +++ b/drivers/gpu/nvgpu/include/nvgpu/preempt.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2019, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2011-2020, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -72,4 +72,48 @@ int nvgpu_preempt_channel(struct gk20a *g, struct nvgpu_channel *ch); */ void nvgpu_preempt_poll_tsg_on_pbdma(struct gk20a *g, struct nvgpu_tsg *tsg); +/** + * @brief Preempt a set of runlists. + * + * @param g [in] Pointer to GPU driver struct. + * @param runlists_bitmask [in] Bitmask of runlists to preempt. + * + * Preempt runlists in \a runlists_bitmask: + * - Write h/w register to trigger preempt on runlists. + * - All TSG in those runlists are preempted. + * + * @note This function is called in case of recovery for error, and does + * not poll PBDMAs or engines to wait for preempt completion. + * + * @note This function should be called with runlist lock held for all + * the runlists in \a runlists_bitmask. + */ +void nvgpu_fifo_preempt_runlists_for_rc(struct gk20a *g, u32 runlists_bitmask); + +/** + * @brief Preempt TSG. + * + * @param g [in] Pointer to GPU driver struct. + * @param tsg [in] Pointer to TSG struct. + * + * Preempt TSG: + * - Acquire lock for active runlist. + * - Write h/w register to trigger TSG preempt for \a tsg. + * - Preemption mode (e.g. CTA or WFI) depends on the preemption + * mode configured in the GR context. + * - Release lock acquired for active runlist. + * - Poll PBDMAs and engines status until preemption is complete, + * or poll timeout occurs. + * + * On some chips, it is also needed to disable scheduling + * before preempting TSG. + * + * @see nvgpu_preempt_get_timeout + * @see nvgpu_gr_ctx::compute_preempt_mode + * + * @return 0 in case preemption succeeded, < 0 in case of failure. + * @retval -ETIMEDOUT when preemption was triggered, but did not + * complete within preemption poll timeout. + */ +int nvgpu_fifo_preempt_tsg(struct gk20a *g, struct nvgpu_tsg *tsg); #endif /* NVGPU_PREEMPT_H */ diff --git a/drivers/gpu/nvgpu/libnvgpu-drv_safe.export b/drivers/gpu/nvgpu/libnvgpu-drv_safe.export index a7b18b56e..65c3e32b8 100644 --- a/drivers/gpu/nvgpu/libnvgpu-drv_safe.export +++ b/drivers/gpu/nvgpu/libnvgpu-drv_safe.export @@ -153,9 +153,7 @@ gv11b_fifo_is_preempt_pending gv11b_fifo_mmu_fault_id_to_pbdma_id gv11b_fifo_preempt_channel gv11b_fifo_preempt_poll_pbdma -gv11b_fifo_preempt_runlists_for_rc gv11b_fifo_preempt_trigger -gv11b_fifo_preempt_tsg gv11b_get_litter_value gv11b_gpu_phys_addr gv11b_init_fifo_reset_enable_hw @@ -411,6 +409,8 @@ nvgpu_fbp_remove_support nvgpu_fifo_cleanup_sw_common nvgpu_fifo_decode_pbdma_ch_eng_status nvgpu_fifo_init_support +nvgpu_fifo_preempt_runlists_for_rc +nvgpu_fifo_preempt_tsg nvgpu_fifo_suspend nvgpu_fifo_sw_quiesce nvgpu_finalize_poweron diff --git a/userspace/units/fifo/preempt/gv11b/nvgpu-preempt-gv11b.c b/userspace/units/fifo/preempt/gv11b/nvgpu-preempt-gv11b.c index c629b01dd..b673f06e1 100644 --- a/userspace/units/fifo/preempt/gv11b/nvgpu-preempt-gv11b.c +++ b/userspace/units/fifo/preempt/gv11b/nvgpu-preempt-gv11b.c @@ -32,6 +32,7 @@ #include #include #include +#include #include #include #include @@ -135,7 +136,7 @@ int test_gv11b_fifo_preempt_runlists_for_rc(struct unit_module *m, 0U, 0U); reg_val = nvgpu_readl(g, fifo_runlist_preempt_r()); - gv11b_fifo_preempt_runlists_for_rc(g, runlist_mask); + nvgpu_fifo_preempt_runlists_for_rc(g, runlist_mask); unit_assert(nvgpu_readl(g, fifo_runlist_preempt_r()) == (reg_val | runlist_mask), goto done); @@ -345,7 +346,7 @@ done: } static void stub_fifo_preempt_trigger(struct gk20a *g, u32 id, - unsigned int id_type) + unsigned int id_type) { } @@ -419,7 +420,7 @@ int test_gv11b_fifo_preempt_tsg(struct unit_module *m, struct gk20a *g, branches & F_PREEMPT_TSG_PLATFORM_SILICON ? true : false; - err = EXPECT_BUG(gv11b_fifo_preempt_tsg(g, tsg)); + err = EXPECT_BUG(nvgpu_fifo_preempt_tsg(g, tsg)); if (branches & F_PREEMPT_TSG_PREEMPT_LOCKED_FAIL) { if (branches & F_PREEMPT_TSG_PLATFORM_SILICON) { diff --git a/userspace/units/init/nvgpu-init.c b/userspace/units/init/nvgpu-init.c index 5377f3adc..2e8afc1d2 100644 --- a/userspace/units/init/nvgpu-init.c +++ b/userspace/units/init/nvgpu-init.c @@ -699,10 +699,6 @@ static void mock_runlist_write_state(struct gk20a *g, u32 runlist_mask, { } -static void mock_fifo_preempt_runlists_for_rc(struct gk20a *g, u32 runlist_mask) -{ -} - int test_quiesce(struct unit_module *m, struct gk20a *g, void *args) { int ret = UNIT_SUCCESS; @@ -733,7 +729,6 @@ int test_quiesce(struct unit_module *m, struct gk20a *g, void *args) /* mock out fifo HALs called during quiesce */ g->ops.runlist.write_state = mock_runlist_write_state; - g->ops.fifo.preempt_runlists_for_rc = mock_fifo_preempt_runlists_for_rc; nvgpu_sw_quiesce(g); /* wait for quiesce thread to complete */ diff --git a/userspace/units/mm/hal/mmu_fault/gv11b_fusa/mmu-fault-gv11b-fusa.c b/userspace/units/mm/hal/mmu_fault/gv11b_fusa/mmu-fault-gv11b-fusa.c index 5293cfce5..244eca622 100644 --- a/userspace/units/mm/hal/mmu_fault/gv11b_fusa/mmu-fault-gv11b-fusa.c +++ b/userspace/units/mm/hal/mmu_fault/gv11b_fusa/mmu-fault-gv11b-fusa.c @@ -36,6 +36,7 @@ #include #include #include +#include #include #include #include @@ -645,7 +646,7 @@ int test_handle_mmu_fault_common(struct unit_module *m, g->ops.channel.unbind = gv11b_channel_unbind; g->ops.channel.free_inst = nvgpu_channel_free_inst; g->ops.tsg.disable = nvgpu_tsg_disable; - g->ops.fifo.preempt_tsg = gv11b_fifo_preempt_tsg; + g->ops.fifo.preempt_tsg = nvgpu_fifo_preempt_tsg; g->aggressive_sync_destroy_thresh = 0U; g->fifo.g = g;