From bdfc26af8bb9adc3c6f10bb3704168faffbca82c Mon Sep 17 00:00:00 2001 From: Seema Khowala Date: Thu, 18 Apr 2019 12:02:35 -0700 Subject: [PATCH] gpu: nvgpu: move preempt code to common/fifo and hal/fifo Move chip specific preempt code to hal/fifo Move non-chip specific preempt code to common/fifo Remove fifo.get_preempt_timeout Rename gk20a_fifo_get_preempt_timeout -> nvgpu_preempt_get_timeout Rename gk20a_fifo_preempt -> nvgpu_preempt_channel Add fifo.preempt_trigger hal for issuing preempt Add fifo.preempt_runlists_for_rc hal for preempting runlists during rc Add fifo.preempt_poll_pbdma hal Add nvgpu_preempt_poll_tsg_on_pbdma to be called from rc JIRA NVGPU-3144 Change-Id: Idb089acaa0c6ca08de17487c3496459a61f0bcd4 Signed-off-by: Seema Khowala Reviewed-on: https://git-master.nvidia.com/r/2100819 Reviewed-by: mobile promotions Tested-by: mobile promotions --- drivers/gpu/nvgpu/Makefile | 3 + drivers/gpu/nvgpu/Makefile.sources | 3 + drivers/gpu/nvgpu/common/fifo/channel.c | 5 +- drivers/gpu/nvgpu/common/fifo/preempt.c | 86 ++++ drivers/gpu/nvgpu/common/gr/gr_setup.c | 5 +- .../nvgpu/common/vgpu/gv11b/vgpu_hal_gv11b.c | 1 + drivers/gpu/nvgpu/gk20a/fifo_gk20a.c | 182 ------- drivers/gpu/nvgpu/gk20a/fifo_gk20a.h | 9 - drivers/gpu/nvgpu/gk20a/gr_gk20a.c | 5 +- drivers/gpu/nvgpu/gp10b/gr_gp10b.c | 6 +- drivers/gpu/nvgpu/gv100/fifo_gv100.c | 6 - drivers/gpu/nvgpu/gv100/fifo_gv100.h | 1 - drivers/gpu/nvgpu/gv11b/fifo_gv11b.c | 428 +--------------- drivers/gpu/nvgpu/gv11b/fifo_gv11b.h | 9 - drivers/gpu/nvgpu/hal/fifo/preempt_gk20a.c | 171 +++++++ drivers/gpu/nvgpu/hal/fifo/preempt_gk20a.h | 37 ++ drivers/gpu/nvgpu/hal/fifo/preempt_gv11b.c | 468 ++++++++++++++++++ drivers/gpu/nvgpu/hal/fifo/preempt_gv11b.h | 42 ++ drivers/gpu/nvgpu/hal/fifo/runlist_gk20a.c | 2 +- drivers/gpu/nvgpu/hal/init/hal_gm20b.c | 2 + drivers/gpu/nvgpu/hal/init/hal_gp10b.c | 2 + drivers/gpu/nvgpu/hal/init/hal_gv100.c | 5 +- drivers/gpu/nvgpu/hal/init/hal_gv11b.c | 5 +- drivers/gpu/nvgpu/hal/init/hal_tu104.c | 5 +- drivers/gpu/nvgpu/include/nvgpu/gk20a.h | 7 +- drivers/gpu/nvgpu/include/nvgpu/preempt.h | 36 ++ drivers/gpu/nvgpu/os/linux/ioctl_channel.c | 3 +- 27 files changed, 889 insertions(+), 645 deletions(-) create mode 100644 drivers/gpu/nvgpu/common/fifo/preempt.c create mode 100644 drivers/gpu/nvgpu/hal/fifo/preempt_gk20a.c create mode 100644 drivers/gpu/nvgpu/hal/fifo/preempt_gk20a.h create mode 100644 drivers/gpu/nvgpu/hal/fifo/preempt_gv11b.c create mode 100644 drivers/gpu/nvgpu/hal/fifo/preempt_gv11b.h create mode 100644 drivers/gpu/nvgpu/include/nvgpu/preempt.h diff --git a/drivers/gpu/nvgpu/Makefile b/drivers/gpu/nvgpu/Makefile index 385baf3ec..788822b78 100644 --- a/drivers/gpu/nvgpu/Makefile +++ b/drivers/gpu/nvgpu/Makefile @@ -248,6 +248,8 @@ nvgpu-y += \ hal/fuse/fuse_gp10b.o \ hal/fuse/fuse_gp106.o \ hal/rc/rc_gk20a.o \ + hal/fifo/preempt_gk20a.o \ + hal/fifo/preempt_gv11b.o \ hal/fifo/usermode_gv11b.o \ hal/fifo/usermode_tu104.o \ hal/fifo/engines_gm20b.o \ @@ -488,6 +490,7 @@ nvgpu-y += \ common/sim/sim_netlist.o \ common/rc/rc.o \ common/fifo/fifo.o \ + common/fifo/preempt.o \ common/fifo/channel.o \ common/fifo/pbdma.o \ common/fifo/submit.o \ diff --git a/drivers/gpu/nvgpu/Makefile.sources b/drivers/gpu/nvgpu/Makefile.sources index be52e22c6..fda73990e 100644 --- a/drivers/gpu/nvgpu/Makefile.sources +++ b/drivers/gpu/nvgpu/Makefile.sources @@ -169,6 +169,7 @@ srcs += common/sim/sim.c \ common/power_features/power_features.c \ common/power_features/cg/cg.c \ common/power_features/pg/pg.c \ + common/fifo/preempt.c \ common/fifo/channel.c \ common/rc/rc.c \ common/fifo/fifo.c \ @@ -359,6 +360,8 @@ srcs += common/sim/sim.c \ hal/fuse/fuse_gp10b.c \ hal/fuse/fuse_gp106.c \ hal/rc/rc_gk20a.c \ + hal/fifo/preempt_gk20a.c \ + hal/fifo/preempt_gv11b.c \ hal/fifo/usermode_gv11b.c \ hal/fifo/usermode_tu104.c \ hal/fifo/engines_gm20b.c \ diff --git a/drivers/gpu/nvgpu/common/fifo/channel.c b/drivers/gpu/nvgpu/common/fifo/channel.c index 7a8d73a77..ef3c7dd26 100644 --- a/drivers/gpu/nvgpu/common/fifo/channel.c +++ b/drivers/gpu/nvgpu/common/fifo/channel.c @@ -51,6 +51,7 @@ #include #include #include +#include #include "common/gr/gr_priv.h" #include "gk20a/gr_gk20a.h" @@ -1175,7 +1176,7 @@ int nvgpu_channel_set_syncpt(struct channel_gk20a *ch) gk20a_disable_channel_tsg(g, ch); /* preempt the channel */ - WARN_ON(gk20a_fifo_preempt(g, ch) != 0); + WARN_ON(nvgpu_preempt_channel(g, ch) != 0); g->ops.ramfc.set_syncpt(ch, new_syncpt); } @@ -2516,7 +2517,7 @@ int nvgpu_channel_suspend_all_serviceable_ch(struct gk20a *g) /* disable channel */ gk20a_disable_channel_tsg(g, ch); /* preempt the channel */ - gk20a_fifo_preempt(g, ch); + nvgpu_preempt_channel(g, ch); /* wait for channel update notifiers */ if (g->os_channel.work_completion_cancel_sync != NULL) { g->os_channel.work_completion_cancel_sync(ch); diff --git a/drivers/gpu/nvgpu/common/fifo/preempt.c b/drivers/gpu/nvgpu/common/fifo/preempt.c new file mode 100644 index 000000000..e88bde793 --- /dev/null +++ b/drivers/gpu/nvgpu/common/fifo/preempt.c @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2011-2019, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include +#include +#include + + +u32 nvgpu_preempt_get_timeout(struct gk20a *g) +{ + return g->ctxsw_timeout_period_ms; +} + +int nvgpu_preempt_channel(struct gk20a *g, struct channel_gk20a *ch) +{ + int err; + struct tsg_gk20a *tsg = tsg_gk20a_from_ch(ch); + + if (tsg != NULL) { + err = g->ops.fifo.preempt_tsg(ch->g, tsg); + } else { + err = g->ops.fifo.preempt_channel(ch->g, ch); + } + + return err; +} + +/* called from rc */ +void nvgpu_preempt_poll_tsg_on_pbdma(struct gk20a *g, + struct tsg_gk20a *tsg) +{ + struct fifo_gk20a *f = &g->fifo; + u32 runlist_id; + unsigned long runlist_served_pbdmas; + unsigned long pbdma_id_bit; + u32 tsgid, pbdma_id; + + if (g->ops.fifo.preempt_poll_pbdma == NULL) { + return; + } + + if (tsg == NULL) { + return; + } + + tsgid = tsg->tsgid; + runlist_id = tsg->runlist_id; + runlist_served_pbdmas = f->runlist_info[runlist_id]->pbdma_bitmask; + + for_each_set_bit(pbdma_id_bit, &runlist_served_pbdmas, f->num_pbdma) { + pbdma_id = U32(pbdma_id_bit); + /* + * If pbdma preempt fails the only option is to reset + * GPU. Any sort of hang indicates the entire GPU’s + * memory system would be blocked. + */ + if (g->ops.fifo.preempt_poll_pbdma(g, tsgid, + pbdma_id) != 0) { + nvgpu_report_host_error(g, 0, + GPU_HOST_PBDMA_PREEMPT_ERROR, + pbdma_id); + nvgpu_err(g, "PBDMA preempt failed"); + } + } +} diff --git a/drivers/gpu/nvgpu/common/gr/gr_setup.c b/drivers/gpu/nvgpu/common/gr/gr_setup.c index 86aa7c86e..7b449f8ad 100644 --- a/drivers/gpu/nvgpu/common/gr/gr_setup.c +++ b/drivers/gpu/nvgpu/common/gr/gr_setup.c @@ -28,6 +28,7 @@ #include #include #include +#include #include "gr_priv.h" @@ -44,7 +45,7 @@ static int nvgpu_gr_setup_zcull(struct gk20a *g, struct channel_gk20a *c, return ret; } - ret = gk20a_fifo_preempt(g, c); + ret = nvgpu_preempt_channel(g, c); if (ret != 0) { if (gk20a_enable_channel_tsg(g, c) != 0) { nvgpu_err(g, "failed to re-enable channel/TSG"); @@ -267,7 +268,7 @@ int nvgpu_gr_setup_set_preemption_mode(struct channel_gk20a *ch, return err; } - err = gk20a_fifo_preempt(g, ch); + err = nvgpu_preempt_channel(g, ch); if (err != 0) { goto enable_ch; } diff --git a/drivers/gpu/nvgpu/common/vgpu/gv11b/vgpu_hal_gv11b.c b/drivers/gpu/nvgpu/common/vgpu/gv11b/vgpu_hal_gv11b.c index 30d83a274..1b3f64b99 100644 --- a/drivers/gpu/nvgpu/common/vgpu/gv11b/vgpu_hal_gv11b.c +++ b/drivers/gpu/nvgpu/common/vgpu/gv11b/vgpu_hal_gv11b.c @@ -24,6 +24,7 @@ #include "hal/bus/bus_gm20b.h" #include "hal/regops/regops_gv11b.h" #include "hal/class/class_gv11b.h" +#include "hal/fifo/preempt_gv11b.h" #include "hal/fifo/engines_gv11b.h" #include "hal/fifo/pbdma_gm20b.h" #include "hal/fifo/pbdma_gp10b.h" diff --git a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c index 7f90f8880..3c9b081e2 100644 --- a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c @@ -119,188 +119,6 @@ int gk20a_init_fifo_setup_hw(struct gk20a *g) return 0; } -void gk20a_fifo_issue_preempt(struct gk20a *g, u32 id, bool is_tsg) -{ - if (is_tsg) { - gk20a_writel(g, fifo_preempt_r(), - fifo_preempt_id_f(id) | - fifo_preempt_type_tsg_f()); - } else { - gk20a_writel(g, fifo_preempt_r(), - fifo_preempt_chid_f(id) | - fifo_preempt_type_channel_f()); - } -} - -static u32 gk20a_fifo_get_preempt_timeout(struct gk20a *g) -{ - /* Use fifo_eng_timeout converted to ms for preempt - * polling. gr_idle_timeout i.e 3000 ms is and not appropriate - * for polling preempt done as context switch timeout gets - * triggered every ctxsw_timeout_period_ms. - */ - - return g->ctxsw_timeout_period_ms; -} - -int gk20a_fifo_is_preempt_pending(struct gk20a *g, u32 id, - unsigned int id_type) -{ - struct nvgpu_timeout timeout; - u32 delay = POLL_DELAY_MIN_US; - int ret = 0; - - ret = nvgpu_timeout_init(g, &timeout, gk20a_fifo_get_preempt_timeout(g), - NVGPU_TIMER_CPU_TIMER); - if (ret != 0) { - nvgpu_err(g, "nvgpu_timeout_init failed err=%d ", ret); - return ret; - } - - ret = -EBUSY; - do { - if ((gk20a_readl(g, fifo_preempt_r()) & - fifo_preempt_pending_true_f()) == 0U) { - ret = 0; - break; - } - - nvgpu_usleep_range(delay, delay * 2U); - delay = min_t(u32, delay << 1, POLL_DELAY_MAX_US); - } while (nvgpu_timeout_expired(&timeout) == 0); - - if (ret != 0) { - nvgpu_err(g, "preempt timeout: id: %u id_type: %d ", - id, id_type); - } - return ret; -} - -int __locked_fifo_preempt(struct gk20a *g, u32 id, bool is_tsg) -{ - int ret; - unsigned int id_type; - - nvgpu_log_fn(g, "id: %d is_tsg: %d", id, is_tsg); - - /* issue preempt */ - gk20a_fifo_issue_preempt(g, id, is_tsg); - - id_type = is_tsg ? ID_TYPE_TSG : ID_TYPE_CHANNEL; - - /* wait for preempt */ - ret = g->ops.fifo.is_preempt_pending(g, id, id_type); - - return ret; -} - -int gk20a_fifo_preempt_channel(struct gk20a *g, struct channel_gk20a *ch) -{ - int ret = 0; - u32 token = PMU_INVALID_MUTEX_OWNER_ID; - int mutex_ret = 0; - int err = 0; - - nvgpu_log_fn(g, "chid: %d", ch->chid); - - /* we have no idea which runlist we are using. lock all */ - nvgpu_fifo_lock_active_runlists(g); - - mutex_ret = nvgpu_pmu_lock_acquire(g, &g->pmu, - PMU_MUTEX_ID_FIFO, &token); - - ret = __locked_fifo_preempt(g, ch->chid, false); - - if (mutex_ret == 0) { - err = nvgpu_pmu_lock_release(g, &g->pmu, PMU_MUTEX_ID_FIFO, - &token); - if (err != 0) { - nvgpu_err(g, "nvgpu_pmu_lock_release failed err=%d", - err); - } - } - - nvgpu_fifo_unlock_active_runlists(g); - - if (ret != 0) { - if (nvgpu_platform_is_silicon(g)) { - nvgpu_err(g, "preempt timed out for chid: %u, " - "ctxsw timeout will trigger recovery if needed", - ch->chid); - } else { - struct tsg_gk20a *tsg; - - nvgpu_err(g, "preempt channel %d timeout", ch->chid); - tsg = tsg_gk20a_from_ch(ch); - if (tsg != NULL) { - nvgpu_rc_preempt_timeout(g, tsg); - } else { - nvgpu_err(g, "chid: %d is not bound to tsg", - ch->chid); - } - - } - } - - return ret; -} - -int gk20a_fifo_preempt_tsg(struct gk20a *g, struct tsg_gk20a *tsg) -{ - int ret = 0; - u32 token = PMU_INVALID_MUTEX_OWNER_ID; - int mutex_ret = 0; - int err = 0; - - nvgpu_log_fn(g, "tsgid: %d", tsg->tsgid); - - /* we have no idea which runlist we are using. lock all */ - nvgpu_fifo_lock_active_runlists(g); - - mutex_ret = nvgpu_pmu_lock_acquire(g, &g->pmu, - PMU_MUTEX_ID_FIFO, &token); - - ret = __locked_fifo_preempt(g, tsg->tsgid, true); - - if (mutex_ret == 0) { - err = nvgpu_pmu_lock_release(g, &g->pmu, PMU_MUTEX_ID_FIFO, - &token); - if (err != 0) { - nvgpu_err(g, "nvgpu_pmu_lock_release failed err=%d", - err); - } - } - - nvgpu_fifo_unlock_active_runlists(g); - - if (ret != 0) { - if (nvgpu_platform_is_silicon(g)) { - nvgpu_err(g, "preempt timed out for tsgid: %u, " - "ctxsw timeout will trigger recovery if needed", - tsg->tsgid); - } else { - nvgpu_err(g, "preempt TSG %d timeout", tsg->tsgid); - nvgpu_rc_preempt_timeout(g, tsg); - } - } - - return ret; -} - -int gk20a_fifo_preempt(struct gk20a *g, struct channel_gk20a *ch) -{ - int err; - struct tsg_gk20a *tsg = tsg_gk20a_from_ch(ch); - - if (tsg != NULL) { - err = g->ops.fifo.preempt_tsg(ch->g, tsg); - } else { - err = g->ops.fifo.preempt_channel(ch->g, ch); - } - - return err; -} - u32 gk20a_fifo_default_timeslice_us(struct gk20a *g) { u64 slice = (((u64)(NVGPU_FIFO_DEFAULT_TIMESLICE_TIMEOUT << diff --git a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.h b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.h index b88f579c9..c33308892 100644 --- a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.h +++ b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.h @@ -210,10 +210,6 @@ struct fifo_gk20a { int gk20a_init_fifo_setup_hw(struct gk20a *g); -int gk20a_fifo_preempt_channel(struct gk20a *g, struct channel_gk20a *ch); -int gk20a_fifo_preempt_tsg(struct gk20a *g, struct tsg_gk20a *tsg); -int gk20a_fifo_preempt(struct gk20a *g, struct channel_gk20a *ch); - u32 gk20a_fifo_engines_on_ch(struct gk20a *g, u32 chid); int gk20a_fifo_suspend(struct gk20a *g); @@ -223,7 +219,6 @@ int gk20a_init_fifo_reset_enable_hw(struct gk20a *g); void fifo_gk20a_finish_mmu_fault_handling(struct gk20a *g, unsigned long fault_id); -void gk20a_fifo_issue_preempt(struct gk20a *g, u32 id, bool is_tsg); int gk20a_fifo_tsg_set_timeslice(struct tsg_gk20a *tsg, u32 timeslice); #ifdef CONFIG_DEBUG_FS @@ -247,10 +242,6 @@ static inline void gk20a_fifo_profile_snapshot( } #endif -int gk20a_fifo_is_preempt_pending(struct gk20a *g, u32 id, - unsigned int id_type); -int __locked_fifo_preempt(struct gk20a *g, u32 id, bool is_tsg); - u32 gk20a_fifo_default_timeslice_us(struct gk20a *g); int gk20a_fifo_init_pbdma_map(struct gk20a *g, u32 *pbdma_map, u32 num_pbdma); diff --git a/drivers/gpu/nvgpu/gk20a/gr_gk20a.c b/drivers/gpu/nvgpu/gk20a/gr_gk20a.c index 3d3b93940..78e054aa6 100644 --- a/drivers/gpu/nvgpu/gk20a/gr_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/gr_gk20a.c @@ -60,6 +60,7 @@ #include #include #include +#include #include "gr_gk20a.h" #include "gr_pri_gk20a.h" @@ -153,7 +154,7 @@ int gr_gk20a_update_smpc_ctxsw_mode(struct gk20a *g, nvgpu_err(g, "failed to disable channel/TSG"); goto out; } - ret = gk20a_fifo_preempt(g, c); + ret = nvgpu_preempt_channel(g, c); if (ret != 0) { gk20a_enable_channel_tsg(g, c); nvgpu_err(g, "failed to preempt channel/TSG"); @@ -220,7 +221,7 @@ int gr_gk20a_update_hwpm_ctxsw_mode(struct gk20a *g, return ret; } - ret = gk20a_fifo_preempt(g, c); + ret = nvgpu_preempt_channel(g, c); if (ret != 0) { gk20a_enable_channel_tsg(g, c); nvgpu_err(g, "failed to preempt channel/TSG"); diff --git a/drivers/gpu/nvgpu/gp10b/gr_gp10b.c b/drivers/gpu/nvgpu/gp10b/gr_gp10b.c index 1a3691c65..2d0621331 100644 --- a/drivers/gpu/nvgpu/gp10b/gr_gp10b.c +++ b/drivers/gpu/nvgpu/gp10b/gr_gp10b.c @@ -35,6 +35,7 @@ #include #include #include +#include #include #include #include @@ -46,6 +47,7 @@ #include #include #include +#include #include "gk20a/gr_gk20a.h" #include "gm20b/gr_gm20b.h" @@ -649,7 +651,7 @@ static int gr_gp10b_disable_channel_or_tsg(struct gk20a *g, struct channel_gk20a nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg | gpu_dbg_intr, "CILP: tsgid: 0x%x", tsg->tsgid); - gk20a_fifo_issue_preempt(g, tsg->tsgid, true); + g->ops.fifo.preempt_trigger(g, tsg->tsgid, ID_TYPE_TSG); nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg | gpu_dbg_intr, "CILP: preempted tsg"); return ret; @@ -1146,7 +1148,7 @@ int gr_gp10b_set_boosted_ctx(struct channel_gk20a *ch, return err; } - err = gk20a_fifo_preempt(g, ch); + err = nvgpu_preempt_channel(g, ch); if (err != 0) { goto enable_ch; } diff --git a/drivers/gpu/nvgpu/gv100/fifo_gv100.c b/drivers/gpu/nvgpu/gv100/fifo_gv100.c index 7601f4e7b..8d2382191 100644 --- a/drivers/gpu/nvgpu/gv100/fifo_gv100.c +++ b/drivers/gpu/nvgpu/gv100/fifo_gv100.c @@ -31,12 +31,6 @@ #include -#define DEFAULT_FIFO_PREEMPT_TIMEOUT 0x3FFFFFUL - -u32 gv100_fifo_get_preempt_timeout(struct gk20a *g) -{ - return g->ctxsw_timeout_period_ms; -} void gv100_fifo_intr_set_recover_mask(struct gk20a *g) { diff --git a/drivers/gpu/nvgpu/gv100/fifo_gv100.h b/drivers/gpu/nvgpu/gv100/fifo_gv100.h index fd9a582fc..eb9a21694 100644 --- a/drivers/gpu/nvgpu/gv100/fifo_gv100.h +++ b/drivers/gpu/nvgpu/gv100/fifo_gv100.h @@ -28,7 +28,6 @@ #include struct gk20a; -u32 gv100_fifo_get_preempt_timeout(struct gk20a *g); void gv100_fifo_intr_set_recover_mask(struct gk20a *g); void gv100_fifo_intr_unset_recover_mask(struct gk20a *g); #endif diff --git a/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c b/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c index 2cdf6beb0..deb685fe1 100644 --- a/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c +++ b/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c @@ -52,6 +52,7 @@ #include #include #include +#include #include "gk20a/fifo_gk20a.h" @@ -64,247 +65,6 @@ #include "fifo_gv11b.h" #include "gr_gv11b.h" -u32 gv11b_fifo_get_preempt_timeout(struct gk20a *g) -{ - /* using gr_idle_timeout for polling pdma/eng/runlist - * might kick in timeout handler in the cases where - * preempt is stuck. Use ctxsw_timeout_period_ms - * for preempt polling */ - - return g->ctxsw_timeout_period_ms; -} - -static int gv11b_fifo_poll_pbdma_chan_status(struct gk20a *g, u32 id, - u32 pbdma_id) -{ - struct nvgpu_timeout timeout; - u32 delay = POLL_DELAY_MIN_US; /* in micro seconds */ - int ret; - unsigned int loop_count = 0; - struct nvgpu_pbdma_status_info pbdma_status; - - /* timeout in milli seconds */ - ret = nvgpu_timeout_init(g, &timeout, - g->ops.fifo.get_preempt_timeout(g), - NVGPU_TIMER_CPU_TIMER); - if (ret != 0) { - nvgpu_err(g, "timeout_init failed: %d", ret); - return ret; - } - - /* Default return value */ - ret = -EBUSY; - - nvgpu_log(g, gpu_dbg_info, "wait preempt pbdma %d", pbdma_id); - /* Verify that ch/tsg is no longer on the pbdma */ - do { - if (!nvgpu_platform_is_silicon(g)) { - if (loop_count >= MAX_PRE_SI_RETRIES) { - nvgpu_err(g, "preempt pbdma retries: %u", - loop_count); - break; - } - loop_count++; - } - /* - * If the PBDMA has a stalling interrupt and receives a NACK, - * the PBDMA won't save out until the STALLING interrupt is - * cleared. Stalling interrupt need not be directly addressed, - * as simply clearing of the interrupt bit will be sufficient - * to allow the PBDMA to save out. If the stalling interrupt - * was due to a SW method or another deterministic failure, - * the PBDMA will assert it when the channel is reloaded - * or resumed. Note that the fault will still be - * reported to SW. - */ - - /* Ignore un-needed return value "recover" */ - (void)g->ops.pbdma.handle_intr(g, pbdma_id, NULL); - - g->ops.pbdma_status.read_pbdma_status_info(g, pbdma_id, - &pbdma_status); - - if (nvgpu_pbdma_status_is_chsw_valid(&pbdma_status) || - nvgpu_pbdma_status_is_chsw_save(&pbdma_status)) { - - if (id != pbdma_status.id) { - ret = 0; - break; - } - - } else if (nvgpu_pbdma_status_is_chsw_load(&pbdma_status)) { - - if (id != pbdma_status.next_id) { - ret = 0; - break; - } - - } else if (nvgpu_pbdma_status_is_chsw_switch(&pbdma_status)) { - - if ((id != pbdma_status.next_id) && - (id != pbdma_status.id)) { - ret = 0; - break; - } - } else { - /* pbdma status is invalid i.e. it is not loaded */ - ret = 0; - break; - } - - nvgpu_usleep_range(delay, delay * 2U); - delay = min_t(u32, delay << 1, POLL_DELAY_MAX_US); - } while (nvgpu_timeout_expired(&timeout) == 0); - - if (ret != 0) { - nvgpu_err(g, "preempt timeout pbdma: %u pbdma_stat: %u " - "tsgid: %u", pbdma_id, - pbdma_status.pbdma_reg_status, id); - } - return ret; -} - -static int gv11b_fifo_poll_eng_ctx_status(struct gk20a *g, u32 id, - u32 act_eng_id, u32 *reset_eng_bitmask) -{ - struct nvgpu_timeout timeout; - u32 delay = POLL_DELAY_MIN_US; /* in micro seconds */ - u32 eng_stat; - u32 ctx_stat; - int ret; - unsigned int loop_count = 0; - u32 eng_intr_pending; - - /* timeout in milli seconds */ - ret = nvgpu_timeout_init(g, &timeout, - g->ops.fifo.get_preempt_timeout(g), - NVGPU_TIMER_CPU_TIMER); - if (ret != 0) { - nvgpu_err(g, "timeout_init failed: %d", ret); - return ret; - } - - /* Default return value */ - ret = -EBUSY; - - nvgpu_log(g, gpu_dbg_info, "wait preempt act engine id: %u", - act_eng_id); - /* Check if ch/tsg has saved off the engine or if ctxsw is hung */ - do { - if (!nvgpu_platform_is_silicon(g)) { - if (loop_count >= MAX_PRE_SI_RETRIES) { - nvgpu_err(g, "preempt eng retries: %u", - loop_count); - break; - } - loop_count++; - } - eng_stat = gk20a_readl(g, fifo_engine_status_r(act_eng_id)); - ctx_stat = fifo_engine_status_ctx_status_v(eng_stat); - - if (g->ops.mc.is_stall_and_eng_intr_pending(g, act_eng_id, - &eng_intr_pending)) { - /* From h/w team - * Engine save can be blocked by eng stalling interrupts. - * FIFO interrupts shouldn’t block an engine save from - * finishing, but could block FIFO from reporting preempt done. - * No immediate reason to reset the engine if FIFO interrupt is - * pending. - * The hub, priv_ring, and ltc interrupts could block context - * switch (or memory), but doesn’t necessarily have to. - * For Hub interrupts they just report access counters and page - * faults. Neither of these necessarily block context switch - * or preemption, but they could. - * For example a page fault for graphics would prevent graphics - * from saving out. An access counter interrupt is a - * notification and has no effect. - * SW should handle page faults though for preempt to complete. - * PRI interrupt (due to a failed PRI transaction) will result - * in ctxsw failure reported to HOST. - * LTC interrupts are generally ECC related and if so, - * certainly don’t block preemption/ctxsw but they could. - * Bus interrupts shouldn’t have anything to do with preemption - * state as they are part of the Host EXT pipe, though they may - * exhibit a symptom that indicates that GPU is in a bad state. - * To be completely fair, when an engine is preempting SW - * really should just handle other interrupts as they come in. - * It’s generally bad to just poll and wait on a preempt - * to complete since there are many things in the GPU which may - * cause a system to hang/stop responding. - */ - nvgpu_log(g, gpu_dbg_info | gpu_dbg_intr, - "stall intr set, " - "preemption might not finish"); - } - if (ctx_stat == - fifo_engine_status_ctx_status_ctxsw_switch_v()) { - /* Eng save hasn't started yet. Continue polling */ - if (eng_intr_pending != 0U) { - /* if eng intr, stop polling */ - *reset_eng_bitmask |= BIT32(act_eng_id); - ret = 0; - break; - } - - } else if (ctx_stat == - fifo_engine_status_ctx_status_valid_v() || - ctx_stat == - fifo_engine_status_ctx_status_ctxsw_save_v()) { - - if (id == fifo_engine_status_id_v(eng_stat)) { - if (eng_intr_pending != 0U) { - /* preemption will not finish */ - *reset_eng_bitmask |= BIT32(act_eng_id); - ret = 0; - break; - } - } else { - /* context is not running on the engine */ - ret = 0; - break; - } - - } else if (ctx_stat == - fifo_engine_status_ctx_status_ctxsw_load_v()) { - - if (id == fifo_engine_status_next_id_v(eng_stat)) { - if (eng_intr_pending != 0U) { - /* preemption will not finish */ - *reset_eng_bitmask |= BIT32(act_eng_id); - ret = 0; - break; - } - } else { - /* context is not running on the engine */ - ret = 0; - break; - } - - } else { - /* Preempt should be finished */ - ret = 0; - break; - } - nvgpu_usleep_range(delay, delay * 2U); - delay = min_t(u32, delay << 1, POLL_DELAY_MAX_US); - } while (nvgpu_timeout_expired(&timeout) == 0); - - if (ret != 0) { - /* - * The reasons a preempt can fail are: - * 1.Some other stalling interrupt is asserted preventing - * channel or context save. - * 2.The memory system hangs. - * 3.The engine hangs during CTXSW. - */ - nvgpu_err(g, "preempt timeout eng: %u ctx_stat: %u tsgid: %u", - act_eng_id, ctx_stat, id); - *reset_eng_bitmask |= BIT32(act_eng_id); - } - - return ret; -} - static u32 gv11b_fifo_get_runlists_mask(struct gk20a *g, u32 act_eng_bitmask, u32 id, unsigned int id_type, unsigned int rc_type, struct mmu_fault_info *mmfault) @@ -358,168 +118,6 @@ static u32 gv11b_fifo_get_runlists_mask(struct gk20a *g, u32 act_eng_bitmask, return runlists_mask; } -static void gv11b_fifo_issue_runlist_preempt(struct gk20a *g, - u32 runlists_mask) -{ - u32 reg_val; - - /* issue runlist preempt */ - reg_val = gk20a_readl(g, fifo_runlist_preempt_r()); - reg_val |= runlists_mask; - gk20a_writel(g, fifo_runlist_preempt_r(), reg_val); -} - -int gv11b_fifo_is_preempt_pending(struct gk20a *g, u32 id, - unsigned int id_type) -{ - struct fifo_gk20a *f = &g->fifo; - unsigned long runlist_served_pbdmas; - unsigned long runlist_served_engines; - unsigned long bit; - u32 pbdma_id; - u32 act_eng_id; - u32 runlist_id; - int ret = 0; - u32 tsgid; - - if (id_type == ID_TYPE_TSG) { - runlist_id = f->tsg[id].runlist_id; - tsgid = id; - } else { - runlist_id = f->channel[id].runlist_id; - tsgid = f->channel[id].tsgid; - } - - nvgpu_log_info(g, "Check preempt pending for tsgid = %u", tsgid); - - runlist_served_pbdmas = f->runlist_info[runlist_id]->pbdma_bitmask; - runlist_served_engines = f->runlist_info[runlist_id]->eng_bitmask; - - for_each_set_bit(bit, &runlist_served_pbdmas, f->num_pbdma) { - pbdma_id = U32(bit); - ret |= gv11b_fifo_poll_pbdma_chan_status(g, tsgid, pbdma_id); - } - - f->runlist_info[runlist_id]->reset_eng_bitmask = 0; - - for_each_set_bit(bit, &runlist_served_engines, f->max_engines) { - act_eng_id = U32(bit); - ret |= gv11b_fifo_poll_eng_ctx_status(g, tsgid, act_eng_id, - &f->runlist_info[runlist_id]->reset_eng_bitmask); - } - return ret; -} - -int gv11b_fifo_preempt_channel(struct gk20a *g, struct channel_gk20a *ch) -{ - struct tsg_gk20a *tsg = NULL; - - tsg = tsg_gk20a_from_ch(ch); - - if (tsg == NULL) { - return 0; - } - - nvgpu_log_info(g, "chid:%d tsgid:%d", ch->chid, tsg->tsgid); - - /* Preempt tsg. Channel preempt is NOOP */ - return g->ops.fifo.preempt_tsg(g, tsg); -} - -int gv11b_fifo_preempt_tsg(struct gk20a *g, struct tsg_gk20a *tsg) -{ - struct fifo_gk20a *f = &g->fifo; - int ret = 0; - u32 token = PMU_INVALID_MUTEX_OWNER_ID; - int mutex_ret = 0; - u32 runlist_id; - - nvgpu_log_fn(g, "tsgid: %d", tsg->tsgid); - - runlist_id = tsg->runlist_id; - nvgpu_log_fn(g, "runlist_id: %d", runlist_id); - if (runlist_id == FIFO_INVAL_RUNLIST_ID) { - return 0; - } - - nvgpu_mutex_acquire(&f->runlist_info[runlist_id]->runlist_lock); - - /* WAR for Bug 2065990 */ - gk20a_tsg_disable_sched(g, tsg); - - mutex_ret = nvgpu_pmu_lock_acquire(g, &g->pmu, - PMU_MUTEX_ID_FIFO, &token); - - ret = __locked_fifo_preempt(g, tsg->tsgid, true); - - if (mutex_ret == 0) { - int err = nvgpu_pmu_lock_release(g, &g->pmu, PMU_MUTEX_ID_FIFO, - &token); - if (err != 0) { - nvgpu_err(g, "PMU_MUTEX_ID_FIFO not released err=%d", - err); - } - } - - /* WAR for Bug 2065990 */ - gk20a_tsg_enable_sched(g, tsg); - - nvgpu_mutex_release(&f->runlist_info[runlist_id]->runlist_lock); - - if (ret != 0) { - if (nvgpu_platform_is_silicon(g)) { - nvgpu_err(g, "preempt timed out for tsgid: %u, " - "ctxsw timeout will trigger recovery if needed", tsg->tsgid); - } else { - nvgpu_rc_preempt_timeout(g, tsg); - } - } - - return ret; -} - -static void gv11b_fifo_locked_preempt_runlists_rc(struct gk20a *g, - u32 runlists_mask) -{ - struct fifo_gk20a *f = &g->fifo; - struct fifo_runlist_info_gk20a *runlist; - u32 token = PMU_INVALID_MUTEX_OWNER_ID; - int mutex_ret = 0; - u32 i; - - /* runlist_lock are locked by teardown and sched are disabled too */ - nvgpu_log_fn(g, "preempt runlists_mask:0x%08x", runlists_mask); - - mutex_ret = nvgpu_pmu_lock_acquire(g, &g->pmu, - PMU_MUTEX_ID_FIFO, &token); - - /* issue runlist preempt */ - gv11b_fifo_issue_runlist_preempt(g, runlists_mask); - - /* - * Preemption will never complete in RC due to some fatal condition. - * Do not poll for preemption to complete. Reset engines served by - * runlists. - */ - for (i = 0U; i < f->num_runlists; i++) { - runlist = &f->active_runlist_info[i]; - - if ((fifo_runlist_preempt_runlist_m(runlist->runlist_id) & - runlists_mask) != 0U) { - runlist->reset_eng_bitmask = runlist->eng_bitmask; - } - } - - if (mutex_ret == 0) { - int err = nvgpu_pmu_lock_release(g, &g->pmu, PMU_MUTEX_ID_FIFO, - &token); - if (err != 0) { - nvgpu_err(g, "PMU_MUTEX_ID_FIFO not released err=%d", - err); - } - } -} - static void gv11b_fifo_locked_abort_runlist_active_tsgs(struct gk20a *g, unsigned int rc_type, u32 runlists_mask) @@ -609,16 +207,14 @@ void gv11b_fifo_teardown_ch_tsg(struct gk20a *g, u32 act_eng_bitmask, struct mmu_fault_info *mmfault) { struct tsg_gk20a *tsg = NULL; - u32 runlists_mask, rlid, i; + u32 runlists_mask, i; unsigned long bit; - u32 pbdma_id; struct fifo_runlist_info_gk20a *runlist = NULL; u32 engine_id; u32 client_type = ~U32(0U); struct fifo_gk20a *f = &g->fifo; u32 runlist_id = FIFO_INVAL_RUNLIST_ID; u32 num_runlists = 0U; - unsigned long runlist_served_pbdmas; bool deferred_reset_pending = false; nvgpu_log_info(g, "acquire engines_reset_mutex"); @@ -711,29 +307,13 @@ void gv11b_fifo_teardown_ch_tsg(struct gk20a *g, u32 act_eng_bitmask, * that all PBDMAs serving the engine are not loaded when engine is * reset. */ - gv11b_fifo_locked_preempt_runlists_rc(g, runlists_mask); + g->ops.fifo.preempt_runlists_for_rc(g, runlists_mask); /* * For each PBDMA which serves the runlist, poll to verify the TSG is no * longer on the PBDMA and the engine phase of the preempt has started. */ if (tsg != NULL) { - rlid = f->tsg[id].runlist_id; - runlist_served_pbdmas = f->runlist_info[rlid]->pbdma_bitmask; - for_each_set_bit(bit, &runlist_served_pbdmas, f->num_pbdma) { - pbdma_id = U32(bit); - /* - * If pbdma preempt fails the only option is to reset - * GPU. Any sort of hang indicates the entire GPU’s - * memory system would be blocked. - */ - if (gv11b_fifo_poll_pbdma_chan_status(g, id, - pbdma_id) != 0) { - nvgpu_report_host_error(g, 0, - GPU_HOST_PBDMA_PREEMPT_ERROR, - pbdma_id); - nvgpu_err(g, "PBDMA preempt failed"); - } - } + nvgpu_preempt_poll_tsg_on_pbdma(g, tsg); } nvgpu_mutex_acquire(&f->deferred_reset_mutex); diff --git a/drivers/gpu/nvgpu/gv11b/fifo_gv11b.h b/drivers/gpu/nvgpu/gv11b/fifo_gv11b.h index 09336e7d8..95fd41d4c 100644 --- a/drivers/gpu/nvgpu/gv11b/fifo_gv11b.h +++ b/drivers/gpu/nvgpu/gv11b/fifo_gv11b.h @@ -29,23 +29,14 @@ #define CHANNEL_INFO_VEID0 0U -#define MAX_PRE_SI_RETRIES 200000U /* 1G/500KHz * 100 */ - struct gpu_ops; u32 gv11b_fifo_mmu_fault_id_to_pbdma_id(struct gk20a *g, u32 mmu_fault_id); -int gv11b_fifo_is_preempt_pending(struct gk20a *g, u32 id, - unsigned int id_type); -int gv11b_fifo_preempt_channel(struct gk20a *g, struct channel_gk20a *ch); -int gv11b_fifo_preempt_tsg(struct gk20a *g, struct tsg_gk20a *tsg); void gv11b_fifo_teardown_ch_tsg(struct gk20a *g, u32 act_eng_bitmask, u32 id, unsigned int id_type, unsigned int rc_type, struct mmu_fault_info *mmfault); -void gv11b_fifo_init_pbdma_intr_descs(struct fifo_gk20a *f); int gv11b_init_fifo_reset_enable_hw(struct gk20a *g); int gv11b_init_fifo_setup_hw(struct gk20a *g); -u32 gv11b_fifo_get_preempt_timeout(struct gk20a *g); - #endif diff --git a/drivers/gpu/nvgpu/hal/fifo/preempt_gk20a.c b/drivers/gpu/nvgpu/hal/fifo/preempt_gk20a.c new file mode 100644 index 000000000..cca8ad84f --- /dev/null +++ b/drivers/gpu/nvgpu/hal/fifo/preempt_gk20a.c @@ -0,0 +1,171 @@ +/* + * Copyright (c) 2011-2019, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "preempt_gk20a.h" + +#include + +void gk20a_fifo_preempt_trigger(struct gk20a *g, u32 id, unsigned int id_type) +{ + if (id_type == ID_TYPE_TSG) { + nvgpu_writel(g, fifo_preempt_r(), + fifo_preempt_id_f(id) | + fifo_preempt_type_tsg_f()); + } else { + nvgpu_writel(g, fifo_preempt_r(), + fifo_preempt_chid_f(id) | + fifo_preempt_type_channel_f()); + } +} + +static int gk20a_fifo_preempt_locked(struct gk20a *g, u32 id, + unsigned int id_type) +{ + nvgpu_log_fn(g, "id: %d id_type: %d", id, id_type); + + /* issue preempt */ + g->ops.fifo.preempt_trigger(g, id, id_type); + + /* wait for preempt */ + return g->ops.fifo.is_preempt_pending(g, id, id_type); +} + +int gk20a_fifo_is_preempt_pending(struct gk20a *g, u32 id, + unsigned int id_type) +{ + struct nvgpu_timeout timeout; + u32 delay = POLL_DELAY_MIN_US; + int ret = -EBUSY; + + nvgpu_timeout_init(g, &timeout, nvgpu_preempt_get_timeout(g), + NVGPU_TIMER_CPU_TIMER); + do { + if ((nvgpu_readl(g, fifo_preempt_r()) & + fifo_preempt_pending_true_f()) == 0U) { + ret = 0; + break; + } + + nvgpu_usleep_range(delay, delay * 2U); + delay = min_t(u32, delay << 1, POLL_DELAY_MAX_US); + } while (nvgpu_timeout_expired(&timeout) == 0); + + if (ret != 0) { + nvgpu_err(g, "preempt timeout: id: %u id_type: %d ", + id, id_type); + } + return ret; +} + +int gk20a_fifo_preempt_channel(struct gk20a *g, struct channel_gk20a *ch) +{ + int ret = 0; + u32 token = PMU_INVALID_MUTEX_OWNER_ID; + int mutex_ret = 0; + + nvgpu_log_fn(g, "preempt chid: %d", ch->chid); + + /* we have no idea which runlist we are using. lock all */ + nvgpu_fifo_lock_active_runlists(g); + + mutex_ret = nvgpu_pmu_lock_acquire(g, &g->pmu, + PMU_MUTEX_ID_FIFO, &token); + + ret = gk20a_fifo_preempt_locked(g, ch->chid, ID_TYPE_CHANNEL); + + if (mutex_ret == 0) { + nvgpu_pmu_lock_release(g, &g->pmu, PMU_MUTEX_ID_FIFO, &token); + } + + nvgpu_fifo_unlock_active_runlists(g); + + if (ret != 0) { + if (nvgpu_platform_is_silicon(g)) { + nvgpu_err(g, "preempt timed out for chid: %u, " + "ctxsw timeout will trigger recovery if needed", + ch->chid); + } else { + struct tsg_gk20a *tsg; + + nvgpu_err(g, "preempt channel %d timeout", ch->chid); + tsg = tsg_gk20a_from_ch(ch); + if (tsg != NULL) { + nvgpu_rc_preempt_timeout(g, tsg); + } else { + nvgpu_err(g, "chid: %d is not bound to tsg", + ch->chid); + } + + } + } + + return ret; +} + +int gk20a_fifo_preempt_tsg(struct gk20a *g, struct tsg_gk20a *tsg) +{ + int ret = 0; + u32 token = PMU_INVALID_MUTEX_OWNER_ID; + int mutex_ret = 0; + + nvgpu_log_fn(g, "tsgid: %d", tsg->tsgid); + + /* we have no idea which runlist we are using. lock all */ + nvgpu_fifo_lock_active_runlists(g); + + mutex_ret = nvgpu_pmu_lock_acquire(g, &g->pmu, + PMU_MUTEX_ID_FIFO, &token); + + ret = gk20a_fifo_preempt_locked(g, tsg->tsgid, ID_TYPE_TSG); + + if (mutex_ret == 0) { + nvgpu_pmu_lock_release(g, &g->pmu, PMU_MUTEX_ID_FIFO, &token); + } + + nvgpu_fifo_unlock_active_runlists(g); + + if (ret != 0) { + if (nvgpu_platform_is_silicon(g)) { + nvgpu_err(g, "preempt timed out for tsgid: %u, " + "ctxsw timeout will trigger recovery if needed", + tsg->tsgid); + } else { + nvgpu_err(g, "preempt TSG %d timeout", tsg->tsgid); + nvgpu_rc_preempt_timeout(g, tsg); + } + } + + return ret; +} diff --git a/drivers/gpu/nvgpu/hal/fifo/preempt_gk20a.h b/drivers/gpu/nvgpu/hal/fifo/preempt_gk20a.h new file mode 100644 index 000000000..eeaa618ed --- /dev/null +++ b/drivers/gpu/nvgpu/hal/fifo/preempt_gk20a.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2011-2019, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ +#ifndef FIFO_PREEMPT_GK20A_H +#define FIFO_PREEMPT_GK20A_H + +#include + +struct gk20a; +struct channel_gk20a; +struct tsg_gk20a; + +void gk20a_fifo_preempt_trigger(struct gk20a *g, u32 id, unsigned int id_type); +int gk20a_fifo_preempt_channel(struct gk20a *g, struct channel_gk20a *ch); +int gk20a_fifo_preempt_tsg(struct gk20a *g, struct tsg_gk20a *tsg); +int gk20a_fifo_is_preempt_pending(struct gk20a *g, u32 id, + unsigned int id_type); + +#endif /* FIFO_PREEMPT_GK20A_H */ diff --git a/drivers/gpu/nvgpu/hal/fifo/preempt_gv11b.c b/drivers/gpu/nvgpu/hal/fifo/preempt_gv11b.c new file mode 100644 index 000000000..6af4ab77e --- /dev/null +++ b/drivers/gpu/nvgpu/hal/fifo/preempt_gv11b.c @@ -0,0 +1,468 @@ +/* + * Copyright (c) 2015-2019, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "preempt_gv11b.h" + +#include + + +void gv11b_fifo_preempt_trigger(struct gk20a *g, u32 id, unsigned int id_type) +{ + if (id_type == ID_TYPE_TSG) { + nvgpu_writel(g, fifo_preempt_r(), + fifo_preempt_id_f(id) | + fifo_preempt_type_tsg_f()); + } else { + nvgpu_log_info(g, "channel preempt is noop"); + } +} + +static void gv11b_fifo_issue_runlist_preempt(struct gk20a *g, + u32 runlists_mask) +{ + u32 reg_val; + + /* issue runlist preempt */ + reg_val = nvgpu_readl(g, fifo_runlist_preempt_r()); + reg_val |= runlists_mask; + nvgpu_writel(g, fifo_runlist_preempt_r(), reg_val); +} + +static int gv11b_fifo_preempt_locked(struct gk20a *g, u32 id, + unsigned int id_type) +{ + nvgpu_log_fn(g, "preempt id: %d id_type: %d", id, id_type); + + g->ops.fifo.preempt_trigger(g, id, id_type); + + /* poll for preempt done */ + return g->ops.fifo.is_preempt_pending(g, id, id_type); + +} + +/* + * This should be called with runlist_lock held for all the + * runlists set in runlists_mask + */ +void gv11b_fifo_preempt_runlists_for_rc(struct gk20a *g, u32 runlists_mask) +{ + struct fifo_gk20a *f = &g->fifo; + struct fifo_runlist_info_gk20a *runlist; + u32 token = PMU_INVALID_MUTEX_OWNER_ID; + int mutex_ret = 0; + u32 i; + + /* runlist_lock are locked by teardown and sched are disabled too */ + nvgpu_log_fn(g, "preempt runlists_mask:0x%08x", runlists_mask); + + mutex_ret = nvgpu_pmu_lock_acquire(g, &g->pmu, + PMU_MUTEX_ID_FIFO, &token); + + /* issue runlist preempt */ + gv11b_fifo_issue_runlist_preempt(g, runlists_mask); + + /* + * Preemption will never complete in RC due to some fatal condition. + * Do not poll for preemption to complete. Reset engines served by + * runlists. + */ + for (i = 0U; i < f->num_runlists; i++) { + runlist = &f->active_runlist_info[i]; + + if ((fifo_runlist_preempt_runlist_m(runlist->runlist_id) & + runlists_mask) != 0U) { + runlist->reset_eng_bitmask = runlist->eng_bitmask; + } + } + + if (mutex_ret == 0) { + int err = nvgpu_pmu_lock_release(g, &g->pmu, PMU_MUTEX_ID_FIFO, + &token); + if (err != 0) { + nvgpu_err(g, "PMU_MUTEX_ID_FIFO not released err=%d", + err); + } + } +} + +int gv11b_fifo_preempt_poll_pbdma(struct gk20a *g, u32 tsgid, + u32 pbdma_id) +{ + struct nvgpu_timeout timeout; + u32 delay = POLL_DELAY_MIN_US; + int ret; + unsigned int loop_count = 0; + struct nvgpu_pbdma_status_info pbdma_status; + + /* timeout in milli seconds */ + ret = nvgpu_timeout_init(g, &timeout, + nvgpu_preempt_get_timeout(g), + NVGPU_TIMER_CPU_TIMER); + if (ret != 0) { + nvgpu_err(g, "timeout_init failed: %d", ret); + return ret; + } + + /* Default return value */ + ret = -EBUSY; + + nvgpu_log(g, gpu_dbg_info, "wait preempt pbdma %d", pbdma_id); + /* Verify that ch/tsg is no longer on the pbdma */ + do { + if (!nvgpu_platform_is_silicon(g)) { + if (loop_count >= PREEMPT_PENDING_POLL_PRE_SI_RETRIES) { + nvgpu_err(g, "preempt pbdma retries: %u", + loop_count); + break; + } + loop_count++; + } + /* + * If the PBDMA has a stalling interrupt and receives a NACK, + * the PBDMA won't save out until the STALLING interrupt is + * cleared. Stalling interrupt need not be directly addressed, + * as simply clearing of the interrupt bit will be sufficient + * to allow the PBDMA to save out. If the stalling interrupt + * was due to a SW method or another deterministic failure, + * the PBDMA will assert it when the channel is reloaded + * or resumed. Note that the fault will still be + * reported to SW. + */ + + /* Ignore un-needed return value "recover" */ + (void)g->ops.pbdma.handle_intr(g, pbdma_id, NULL); + + g->ops.pbdma_status.read_pbdma_status_info(g, pbdma_id, + &pbdma_status); + + if (nvgpu_pbdma_status_is_chsw_valid(&pbdma_status) || + nvgpu_pbdma_status_is_chsw_save(&pbdma_status)) { + + if (tsgid != pbdma_status.id) { + ret = 0; + break; + } + + } else if (nvgpu_pbdma_status_is_chsw_load(&pbdma_status)) { + + if (tsgid != pbdma_status.next_id) { + ret = 0; + break; + } + + } else if (nvgpu_pbdma_status_is_chsw_switch(&pbdma_status)) { + + if ((tsgid != pbdma_status.next_id) && + (tsgid != pbdma_status.id)) { + ret = 0; + break; + } + } else { + /* pbdma status is invalid i.e. it is not loaded */ + ret = 0; + break; + } + + nvgpu_usleep_range(delay, delay * 2U); + delay = min_t(u32, delay << 1, POLL_DELAY_MAX_US); + } while (nvgpu_timeout_expired(&timeout) == 0); + + if (ret != 0) { + nvgpu_err(g, "preempt timeout pbdma: %u pbdma_stat: %u " + "tsgid: %u", pbdma_id, + pbdma_status.pbdma_reg_status, tsgid); + } + return ret; +} + +static int gv11b_fifo_preempt_poll_eng(struct gk20a *g, u32 id, + u32 act_eng_id, u32 *reset_eng_bitmask) +{ + struct nvgpu_timeout timeout; + u32 delay = POLL_DELAY_MIN_US; + u32 eng_stat; + u32 ctx_stat; + int ret; + unsigned int loop_count = 0; + u32 eng_intr_pending; + + /* timeout in milli seconds */ + ret = nvgpu_timeout_init(g, &timeout, + nvgpu_preempt_get_timeout(g), + NVGPU_TIMER_CPU_TIMER); + if (ret != 0) { + nvgpu_err(g, "timeout_init failed: %d", ret); + return ret; + } + + /* Default return value */ + ret = -EBUSY; + + nvgpu_log(g, gpu_dbg_info, "wait preempt act engine id: %u", + act_eng_id); + /* Check if ch/tsg has saved off the engine or if ctxsw is hung */ + do { + if (!nvgpu_platform_is_silicon(g)) { + if (loop_count >= PREEMPT_PENDING_POLL_PRE_SI_RETRIES) { + nvgpu_err(g, "preempt eng retries: %u", + loop_count); + break; + } + loop_count++; + } + eng_stat = nvgpu_readl(g, fifo_engine_status_r(act_eng_id)); + ctx_stat = fifo_engine_status_ctx_status_v(eng_stat); + + if (g->ops.mc.is_stall_and_eng_intr_pending(g, act_eng_id, + &eng_intr_pending)) { + /* + * From h/w team + * Engine save can be blocked by eng stalling interrupts. + * FIFO interrupts shouldn’t block an engine save from + * finishing, but could block FIFO from reporting preempt done. + * No immediate reason to reset the engine if FIFO interrupt is + * pending. + * The hub, priv_ring, and ltc interrupts could block context + * switch (or memory), but doesn’t necessarily have to. + * For Hub interrupts they just report access counters and page + * faults. Neither of these necessarily block context switch + * or preemption, but they could. + * For example a page fault for graphics would prevent graphics + * from saving out. An access counter interrupt is a + * notification and has no effect. + * SW should handle page faults though for preempt to complete. + * PRI interrupt (due to a failed PRI transaction) will result + * in ctxsw failure reported to HOST. + * LTC interrupts are generally ECC related and if so, + * certainly don’t block preemption/ctxsw but they could. + * Bus interrupts shouldn’t have anything to do with preemption + * state as they are part of the Host EXT pipe, though they may + * exhibit a symptom that indicates that GPU is in a bad state. + * To be completely fair, when an engine is preempting SW + * really should just handle other interrupts as they come in. + * It’s generally bad to just poll and wait on a preempt + * to complete since there are many things in the GPU which may + * cause a system to hang/stop responding. + */ + nvgpu_log(g, gpu_dbg_info | gpu_dbg_intr, + "stall intr set, " + "preemption might not finish"); + } + if (ctx_stat == + fifo_engine_status_ctx_status_ctxsw_switch_v()) { + /* Eng save hasn't started yet. Continue polling */ + if (eng_intr_pending != 0U) { + /* if eng intr, stop polling */ + *reset_eng_bitmask |= BIT32(act_eng_id); + ret = 0; + break; + } + + } else if (ctx_stat == + fifo_engine_status_ctx_status_valid_v() || + ctx_stat == + fifo_engine_status_ctx_status_ctxsw_save_v()) { + + if (id == fifo_engine_status_id_v(eng_stat)) { + if (eng_intr_pending != 0U) { + /* preemption will not finish */ + *reset_eng_bitmask |= BIT32(act_eng_id); + ret = 0; + break; + } + } else { + /* context is not running on the engine */ + ret = 0; + break; + } + + } else if (ctx_stat == + fifo_engine_status_ctx_status_ctxsw_load_v()) { + + if (id == fifo_engine_status_next_id_v(eng_stat)) { + if (eng_intr_pending != 0U) { + /* preemption will not finish */ + *reset_eng_bitmask |= BIT32(act_eng_id); + ret = 0; + break; + } + } else { + /* context is not running on the engine */ + ret = 0; + break; + } + + } else { + /* Preempt should be finished */ + ret = 0; + break; + } + nvgpu_usleep_range(delay, delay * 2U); + delay = min_t(u32, delay << 1, POLL_DELAY_MAX_US); + } while (nvgpu_timeout_expired(&timeout) == 0); + + if (ret != 0) { + /* + * The reasons a preempt can fail are: + * 1.Some other stalling interrupt is asserted preventing + * channel or context save. + * 2.The memory system hangs. + * 3.The engine hangs during CTXSW. + */ + nvgpu_err(g, "preempt timeout eng: %u ctx_stat: %u tsgid: %u", + act_eng_id, ctx_stat, id); + *reset_eng_bitmask |= BIT32(act_eng_id); + } + + return ret; +} + +int gv11b_fifo_is_preempt_pending(struct gk20a *g, u32 id, + unsigned int id_type) +{ + struct fifo_gk20a *f = &g->fifo; + unsigned long runlist_served_pbdmas; + unsigned long runlist_served_engines; + unsigned long bit; + u32 pbdma_id; + u32 act_eng_id; + u32 runlist_id; + int ret = 0; + u32 tsgid; + + if (id_type == ID_TYPE_TSG) { + runlist_id = f->tsg[id].runlist_id; + tsgid = id; + } else { + runlist_id = f->channel[id].runlist_id; + tsgid = f->channel[id].tsgid; + } + + nvgpu_log_info(g, "Check preempt pending for tsgid = %u", tsgid); + + runlist_served_pbdmas = f->runlist_info[runlist_id]->pbdma_bitmask; + runlist_served_engines = f->runlist_info[runlist_id]->eng_bitmask; + + for_each_set_bit(bit, &runlist_served_pbdmas, f->num_pbdma) { + pbdma_id = U32(bit); + ret |= gv11b_fifo_preempt_poll_pbdma(g, tsgid, + pbdma_id); + } + + f->runlist_info[runlist_id]->reset_eng_bitmask = 0U; + + for_each_set_bit(bit, &runlist_served_engines, f->max_engines) { + act_eng_id = U32(bit); + ret |= gv11b_fifo_preempt_poll_eng(g, + tsgid, act_eng_id, + &f->runlist_info[runlist_id]->reset_eng_bitmask); + } + return ret; +} + +int gv11b_fifo_preempt_channel(struct gk20a *g, struct channel_gk20a *ch) +{ + struct tsg_gk20a *tsg = NULL; + + tsg = tsg_gk20a_from_ch(ch); + + if (tsg == NULL) { + nvgpu_log_info(g, "chid: %d is not bound to tsg", ch->chid); + return 0; + } + + nvgpu_log_info(g, "chid:%d tsgid:%d", ch->chid, tsg->tsgid); + + /* Preempt tsg. Channel preempt is NOOP */ + return g->ops.fifo.preempt_tsg(g, tsg); +} + +int gv11b_fifo_preempt_tsg(struct gk20a *g, struct tsg_gk20a *tsg) +{ + struct fifo_gk20a *f = &g->fifo; + int ret = 0; + u32 token = PMU_INVALID_MUTEX_OWNER_ID; + int mutex_ret = 0; + u32 runlist_id; + + nvgpu_log_fn(g, "tsgid: %d", tsg->tsgid); + + runlist_id = tsg->runlist_id; + nvgpu_log_fn(g, "runlist_id: %d", runlist_id); + if (runlist_id == FIFO_INVAL_RUNLIST_ID) { + return 0; + } + + nvgpu_mutex_acquire(&f->runlist_info[runlist_id]->runlist_lock); + + /* WAR for Bug 2065990 */ + gk20a_tsg_disable_sched(g, tsg); + + mutex_ret = nvgpu_pmu_lock_acquire(g, &g->pmu, + PMU_MUTEX_ID_FIFO, &token); + + ret = gv11b_fifo_preempt_locked(g, tsg->tsgid, ID_TYPE_TSG); + + if (mutex_ret == 0) { + int err = nvgpu_pmu_lock_release(g, &g->pmu, PMU_MUTEX_ID_FIFO, + &token); + if (err != 0) { + nvgpu_err(g, "PMU_MUTEX_ID_FIFO not released err=%d", + err); + } + } + + /* WAR for Bug 2065990 */ + gk20a_tsg_enable_sched(g, tsg); + + nvgpu_mutex_release(&f->runlist_info[runlist_id]->runlist_lock); + + if (ret != 0) { + if (nvgpu_platform_is_silicon(g)) { + nvgpu_err(g, "preempt timed out for tsgid: %u, " + "ctxsw timeout will trigger recovery if needed", + tsg->tsgid); + } else { + nvgpu_rc_preempt_timeout(g, tsg); + } + } + + return ret; +} diff --git a/drivers/gpu/nvgpu/hal/fifo/preempt_gv11b.h b/drivers/gpu/nvgpu/hal/fifo/preempt_gv11b.h new file mode 100644 index 000000000..395bdafa6 --- /dev/null +++ b/drivers/gpu/nvgpu/hal/fifo/preempt_gv11b.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2016-2019, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ +#ifndef FIFO_PREEMPT_GV11B_H +#define FIFO_PREEMPT_GV11B_H + +#include + +#define PREEMPT_PENDING_POLL_PRE_SI_RETRIES 200000U /* 1G/500KHz * 100 */ + +struct gk20a; +struct channel_gk20a; +struct tsg_gk20a; + +void gv11b_fifo_preempt_trigger(struct gk20a *g, u32 id, unsigned int id_type); +int gv11b_fifo_preempt_channel(struct gk20a *g, struct channel_gk20a *ch); +int gv11b_fifo_preempt_tsg(struct gk20a *g, struct tsg_gk20a *tsg); +int gv11b_fifo_is_preempt_pending(struct gk20a *g, u32 id, + unsigned int id_type); +void gv11b_fifo_preempt_runlists_for_rc(struct gk20a *g, u32 runlists_mask); + +int gv11b_fifo_preempt_poll_pbdma(struct gk20a *g, u32 tsgid, u32 pbdma_id); + +#endif /* FIFO_PREEMPT_GV11B_H */ diff --git a/drivers/gpu/nvgpu/hal/fifo/runlist_gk20a.c b/drivers/gpu/nvgpu/hal/fifo/runlist_gk20a.c index ea1f6126a..f25383333 100644 --- a/drivers/gpu/nvgpu/hal/fifo/runlist_gk20a.c +++ b/drivers/gpu/nvgpu/hal/fifo/runlist_gk20a.c @@ -87,7 +87,7 @@ int gk20a_fifo_reschedule_preempt_next(struct channel_gk20a *ch, return ret; } - gk20a_fifo_issue_preempt(g, preempt_id, preempt_type != 0U); + g->ops.fifo.preempt_trigger(g, preempt_id, preempt_type != 0U); #ifdef TRACEPOINTS_ENABLED trace_gk20a_reschedule_preempt_next(ch->chid, fecsstat0, engine_status.reg_data, diff --git a/drivers/gpu/nvgpu/hal/init/hal_gm20b.c b/drivers/gpu/nvgpu/hal/init/hal_gm20b.c index 0f419efcc..a178a1d06 100644 --- a/drivers/gpu/nvgpu/hal/init/hal_gm20b.c +++ b/drivers/gpu/nvgpu/hal/init/hal_gm20b.c @@ -57,6 +57,7 @@ #include "hal/fuse/fuse_gm20b.h" #include "hal/ptimer/ptimer_gk20a.h" #include "hal/regops/regops_gm20b.h" +#include "hal/fifo/preempt_gk20a.h" #include "hal/fifo/pbdma_gm20b.h" #include "hal/fifo/engines_gm20b.h" #include "hal/fifo/engine_status_gm20b.h" @@ -663,6 +664,7 @@ static const struct gpu_ops gm20b_ops = { .default_timeslice_us = gk20a_fifo_default_timeslice_us, .preempt_channel = gk20a_fifo_preempt_channel, .preempt_tsg = gk20a_fifo_preempt_tsg, + .preempt_trigger = gk20a_fifo_preempt_trigger, .tsg_set_timeslice = gk20a_fifo_tsg_set_timeslice, .init_pbdma_map = gk20a_fifo_init_pbdma_map, .is_preempt_pending = gk20a_fifo_is_preempt_pending, diff --git a/drivers/gpu/nvgpu/hal/init/hal_gp10b.c b/drivers/gpu/nvgpu/hal/init/hal_gp10b.c index e6ae7da1a..42c358ceb 100644 --- a/drivers/gpu/nvgpu/hal/init/hal_gp10b.c +++ b/drivers/gpu/nvgpu/hal/init/hal_gp10b.c @@ -65,6 +65,7 @@ #include "hal/fuse/fuse_gp10b.h" #include "hal/ptimer/ptimer_gk20a.h" #include "hal/regops/regops_gp10b.h" +#include "hal/fifo/preempt_gk20a.h" #include "hal/fifo/pbdma_gm20b.h" #include "hal/fifo/pbdma_gp10b.h" #include "hal/fifo/engines_gm20b.h" @@ -730,6 +731,7 @@ static const struct gpu_ops gp10b_ops = { .default_timeslice_us = gk20a_fifo_default_timeslice_us, .preempt_channel = gk20a_fifo_preempt_channel, .preempt_tsg = gk20a_fifo_preempt_tsg, + .preempt_trigger = gk20a_fifo_preempt_trigger, .tsg_set_timeslice = gk20a_fifo_tsg_set_timeslice, .init_pbdma_map = gk20a_fifo_init_pbdma_map, .is_preempt_pending = gk20a_fifo_is_preempt_pending, diff --git a/drivers/gpu/nvgpu/hal/init/hal_gv100.c b/drivers/gpu/nvgpu/hal/init/hal_gv100.c index 4151f4e59..ddbee2844 100644 --- a/drivers/gpu/nvgpu/hal/init/hal_gv100.c +++ b/drivers/gpu/nvgpu/hal/init/hal_gv100.c @@ -58,6 +58,7 @@ #include "hal/fuse/fuse_gp106.h" #include "hal/ptimer/ptimer_gk20a.h" #include "hal/regops/regops_gv100.h" +#include "hal/fifo/preempt_gv11b.h" #include "hal/fifo/pbdma_gm20b.h" #include "hal/fifo/pbdma_gp10b.h" #include "hal/fifo/pbdma_gv11b.h" @@ -902,11 +903,13 @@ static const struct gpu_ops gv100_ops = { gr_gv100_pg_gr_load_gating_prod, }, .fifo = { - .get_preempt_timeout = gv11b_fifo_get_preempt_timeout, .init_fifo_setup_hw = gv11b_init_fifo_setup_hw, .default_timeslice_us = gk20a_fifo_default_timeslice_us, .preempt_channel = gv11b_fifo_preempt_channel, .preempt_tsg = gv11b_fifo_preempt_tsg, + .preempt_trigger = gv11b_fifo_preempt_trigger, + .preempt_runlists_for_rc = gv11b_fifo_preempt_runlists_for_rc, + .preempt_poll_pbdma = gv11b_fifo_preempt_poll_pbdma, .tsg_set_timeslice = gk20a_fifo_tsg_set_timeslice, .init_pbdma_map = gk20a_fifo_init_pbdma_map, .is_preempt_pending = gv11b_fifo_is_preempt_pending, diff --git a/drivers/gpu/nvgpu/hal/init/hal_gv11b.c b/drivers/gpu/nvgpu/hal/init/hal_gv11b.c index e7aba11e8..bca9eb488 100644 --- a/drivers/gpu/nvgpu/hal/init/hal_gv11b.c +++ b/drivers/gpu/nvgpu/hal/init/hal_gv11b.c @@ -62,6 +62,7 @@ #include "hal/ptimer/ptimer_gk20a.h" #include "hal/regops/regops_gv11b.h" #include "hal/fifo/pbdma_gm20b.h" +#include "hal/fifo/preempt_gv11b.h" #include "hal/fifo/pbdma_gp10b.h" #include "hal/fifo/pbdma_gv11b.h" #include "hal/fifo/engine_status_gv100.h" @@ -875,11 +876,13 @@ static const struct gpu_ops gv11b_ops = { gr_gv11b_pg_gr_load_gating_prod, }, .fifo = { - .get_preempt_timeout = gv11b_fifo_get_preempt_timeout, .init_fifo_setup_hw = gv11b_init_fifo_setup_hw, .default_timeslice_us = gk20a_fifo_default_timeslice_us, .preempt_channel = gv11b_fifo_preempt_channel, .preempt_tsg = gv11b_fifo_preempt_tsg, + .preempt_trigger = gv11b_fifo_preempt_trigger, + .preempt_runlists_for_rc = gv11b_fifo_preempt_runlists_for_rc, + .preempt_poll_pbdma = gv11b_fifo_preempt_poll_pbdma, .tsg_set_timeslice = gk20a_fifo_tsg_set_timeslice, .init_pbdma_map = gk20a_fifo_init_pbdma_map, .is_preempt_pending = gv11b_fifo_is_preempt_pending, diff --git a/drivers/gpu/nvgpu/hal/init/hal_tu104.c b/drivers/gpu/nvgpu/hal/init/hal_tu104.c index d8e2aafd6..55ea166a7 100644 --- a/drivers/gpu/nvgpu/hal/init/hal_tu104.c +++ b/drivers/gpu/nvgpu/hal/init/hal_tu104.c @@ -60,6 +60,7 @@ #include "hal/fuse/fuse_gm20b.h" #include "hal/fuse/fuse_gp10b.h" #include "hal/fuse/fuse_gp106.h" +#include "hal/fifo/preempt_gv11b.h" #include "hal/fifo/usermode_gv11b.h" #include "hal/fifo/usermode_tu104.h" #include "hal/fifo/pbdma_gm20b.h" @@ -935,11 +936,13 @@ static const struct gpu_ops tu104_ops = { tu104_blcg_xbar_load_gating_prod, }, .fifo = { - .get_preempt_timeout = gv100_fifo_get_preempt_timeout, .init_fifo_setup_hw = tu104_init_fifo_setup_hw, .default_timeslice_us = gk20a_fifo_default_timeslice_us, .preempt_channel = gv11b_fifo_preempt_channel, .preempt_tsg = gv11b_fifo_preempt_tsg, + .preempt_trigger = gv11b_fifo_preempt_trigger, + .preempt_runlists_for_rc = gv11b_fifo_preempt_runlists_for_rc, + .preempt_poll_pbdma = gv11b_fifo_preempt_poll_pbdma, .tsg_set_timeslice = gk20a_fifo_tsg_set_timeslice, .init_pbdma_map = gk20a_fifo_init_pbdma_map, .is_preempt_pending = gv11b_fifo_is_preempt_pending, diff --git a/drivers/gpu/nvgpu/include/nvgpu/gk20a.h b/drivers/gpu/nvgpu/include/nvgpu/gk20a.h index 99f2be9a6..e87499724 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/gk20a.h +++ b/drivers/gpu/nvgpu/include/nvgpu/gk20a.h @@ -972,6 +972,12 @@ struct gpu_ops { int (*init_fifo_setup_hw)(struct gk20a *g); int (*preempt_channel)(struct gk20a *g, struct channel_gk20a *ch); int (*preempt_tsg)(struct gk20a *g, struct tsg_gk20a *tsg); + void (*preempt_runlists_for_rc)(struct gk20a *g, + u32 runlists_bitmask); + void (*preempt_trigger)(struct gk20a *g, + u32 id, unsigned int id_type); + int (*preempt_poll_pbdma)(struct gk20a *g, u32 tsgid, + u32 pbdma_id); int (*tsg_set_timeslice)(struct tsg_gk20a *tsg, u32 timeslice); u32 (*default_timeslice_us)(struct gk20a *g); int (*init_pbdma_map)(struct gk20a *g, @@ -984,7 +990,6 @@ struct gpu_ops { struct mmu_fault_info *mmfault); void (*intr_set_recover_mask)(struct gk20a *g); void (*intr_unset_recover_mask)(struct gk20a *g); - u32 (*get_preempt_timeout)(struct gk20a *g); int (*init_pdb_cache_war)(struct gk20a *g); void (*deinit_pdb_cache_war)(struct gk20a *g); int (*set_sm_exception_type_mask)(struct channel_gk20a *ch, diff --git a/drivers/gpu/nvgpu/include/nvgpu/preempt.h b/drivers/gpu/nvgpu/include/nvgpu/preempt.h new file mode 100644 index 000000000..54d5ebf77 --- /dev/null +++ b/drivers/gpu/nvgpu/include/nvgpu/preempt.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2011-2019, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ +#ifndef NVGPU_PREEMPT_H +#define NVGPU_PREEMPT_H + +#include + +struct gk20a; +struct channel_gk20a; +struct tsg_gk20a; + +u32 nvgpu_preempt_get_timeout(struct gk20a *g); +int nvgpu_preempt_channel(struct gk20a *g, struct channel_gk20a *ch); + +void nvgpu_preempt_poll_tsg_on_pbdma(struct gk20a *g, + struct tsg_gk20a *tsg); +#endif /* NVGPU_PREEMPT_H */ diff --git a/drivers/gpu/nvgpu/os/linux/ioctl_channel.c b/drivers/gpu/nvgpu/os/linux/ioctl_channel.c index 61347031c..ece80ed11 100644 --- a/drivers/gpu/nvgpu/os/linux/ioctl_channel.c +++ b/drivers/gpu/nvgpu/os/linux/ioctl_channel.c @@ -42,6 +42,7 @@ #include #include #include +#include #include "platform_gk20a.h" #include "ioctl_channel.h" @@ -1309,7 +1310,7 @@ long gk20a_channel_ioctl(struct file *filp, __func__, cmd); break; } - err = gk20a_fifo_preempt(ch->g, ch); + err = nvgpu_preempt_channel(ch->g, ch); gk20a_idle(ch->g); break; case NVGPU_IOCTL_CHANNEL_RESCHEDULE_RUNLIST: