diff --git a/drivers/gpu/nvgpu/common/fifo/preempt.c b/drivers/gpu/nvgpu/common/fifo/preempt.c index 75fbddad6..7c4890d08 100644 --- a/drivers/gpu/nvgpu/common/fifo/preempt.c +++ b/drivers/gpu/nvgpu/common/fifo/preempt.c @@ -72,7 +72,7 @@ int nvgpu_fifo_preempt_tsg(struct gk20a *g, u32 runlist_id, u32 tsgid) mutex_ret = nvgpu_pmu_lock_acquire(g, g->pmu, PMU_MUTEX_ID_FIFO, &token); #endif - g->ops.fifo.preempt_trigger(g, tsgid, ID_TYPE_TSG); + g->ops.fifo.preempt_trigger(g, runlist_id, tsgid, ID_TYPE_TSG); /* * Poll for preempt done. if stalling interrupts are pending @@ -83,7 +83,7 @@ int nvgpu_fifo_preempt_tsg(struct gk20a *g, u32 runlist_id, u32 tsgid) * the engines hung and set the runlist reset_eng_bitmask * and mark preemption completion. */ - ret = g->ops.fifo.is_preempt_pending(g, tsgid, + ret = g->ops.fifo.is_preempt_pending(g, runlist_id, tsgid, ID_TYPE_TSG, preempt_retry_count > 1U); #ifdef CONFIG_NVGPU_LS_PMU @@ -198,7 +198,7 @@ void nvgpu_fifo_preempt_runlists_for_rc(struct gk20a *g, u32 runlists_bitmask) continue; } /* issue runlist preempt */ - g->ops.fifo.preempt_trigger(g, runlist->id, + g->ops.fifo.preempt_trigger(g, runlist->id, INVAL_ID, ID_TYPE_RUNLIST); #ifdef CONFIG_NVGPU_RECOVERY /* diff --git a/drivers/gpu/nvgpu/hal/fifo/preempt_ga10b.h b/drivers/gpu/nvgpu/hal/fifo/preempt_ga10b.h index 0db9492a5..907d9d2da 100644 --- a/drivers/gpu/nvgpu/hal/fifo/preempt_ga10b.h +++ b/drivers/gpu/nvgpu/hal/fifo/preempt_ga10b.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -26,6 +26,7 @@ struct gk20a; -void ga10b_fifo_preempt_trigger(struct gk20a *g, u32 id, unsigned int id_type); +void ga10b_fifo_preempt_trigger(struct gk20a *g, + u32 runlist_id, u32 id, unsigned int id_type); #endif /* FIFO_PREEMPT_GA10B_H */ diff --git a/drivers/gpu/nvgpu/hal/fifo/preempt_ga10b_fusa.c b/drivers/gpu/nvgpu/hal/fifo/preempt_ga10b_fusa.c index 4704365f1..22a8714f6 100644 --- a/drivers/gpu/nvgpu/hal/fifo/preempt_ga10b_fusa.c +++ b/drivers/gpu/nvgpu/hal/fifo/preempt_ga10b_fusa.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2021, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -31,22 +31,24 @@ #include -void ga10b_fifo_preempt_trigger(struct gk20a *g, u32 id, unsigned int id_type) +void ga10b_fifo_preempt_trigger(struct gk20a *g, + u32 runlist_id, u32 tsgid, unsigned int id_type) { - struct nvgpu_runlist *runlist = NULL; + struct nvgpu_runlist *runlist; - if (id == INVAL_ID) { + if (runlist_id == INVAL_ID || + (tsgid == INVAL_ID && id_type == ID_TYPE_TSG)) { nvgpu_log(g, gpu_dbg_info, "Invalid id, cannot preempt"); return; } + runlist = g->fifo.runlists[runlist_id]; + if (id_type == ID_TYPE_TSG) { - struct nvgpu_tsg *tsg = &g->fifo.tsg[id]; - nvgpu_runlist_writel(g, tsg->runlist, runlist_preempt_r(), - runlist_preempt_id_f(id) | + nvgpu_runlist_writel(g, runlist, runlist_preempt_r(), + runlist_preempt_id_f(tsgid) | runlist_preempt_type_tsg_f()); } else if (id_type == ID_TYPE_RUNLIST) { - runlist = g->fifo.runlists[id]; nvgpu_runlist_writel(g, runlist, runlist_preempt_r(), runlist_preempt_type_runlist_f()); } else { diff --git a/drivers/gpu/nvgpu/hal/fifo/preempt_gk20a.c b/drivers/gpu/nvgpu/hal/fifo/preempt_gk20a.c index 74b62a2e4..527ef16a6 100644 --- a/drivers/gpu/nvgpu/hal/fifo/preempt_gk20a.c +++ b/drivers/gpu/nvgpu/hal/fifo/preempt_gk20a.c @@ -40,15 +40,16 @@ #include -void gk20a_fifo_preempt_trigger(struct gk20a *g, u32 id, unsigned int id_type) +void gk20a_fifo_preempt_trigger(struct gk20a *g, + u32 runlist_id, u32 tsgid, unsigned int id_type) { if (id_type == ID_TYPE_TSG) { nvgpu_writel(g, fifo_preempt_r(), - fifo_preempt_id_f(id) | + fifo_preempt_id_f(tsgid) | fifo_preempt_type_tsg_f()); } else { nvgpu_writel(g, fifo_preempt_r(), - fifo_preempt_chid_f(id) | + fifo_preempt_chid_f(runlist_id) | fifo_preempt_type_channel_f()); } } @@ -58,14 +59,14 @@ static int gk20a_fifo_preempt_locked(struct gk20a *g, u32 id, { nvgpu_log_fn(g, "id: %d id_type: %d", id, id_type); - /* issue preempt */ - g->ops.fifo.preempt_trigger(g, id, id_type); + /* issue preempt, runlist_id not used for gm20b and prior */ + g->ops.fifo.preempt_trigger(g, INVAL_ID, id, id_type); - /* wait for preempt */ - return g->ops.fifo.is_preempt_pending(g, id, id_type, false); + /* wait for preempt, runlist_id not used for gm20b and prior */ + return g->ops.fifo.is_preempt_pending(g, INVAL_ID, id, id_type, false); } -int gk20a_fifo_is_preempt_pending(struct gk20a *g, u32 id, +int gk20a_fifo_is_preempt_pending(struct gk20a *g, u32 runlist_id, u32 id, unsigned int id_type, bool preempt_retries_left) { struct nvgpu_timeout timeout; @@ -73,6 +74,7 @@ int gk20a_fifo_is_preempt_pending(struct gk20a *g, u32 id, int ret; (void)preempt_retries_left; + (void)runlist_id; nvgpu_timeout_init_cpu_timer(g, &timeout, nvgpu_preempt_get_timeout(g)); diff --git a/drivers/gpu/nvgpu/hal/fifo/preempt_gk20a.h b/drivers/gpu/nvgpu/hal/fifo/preempt_gk20a.h index d5dc77391..a708c310d 100644 --- a/drivers/gpu/nvgpu/hal/fifo/preempt_gk20a.h +++ b/drivers/gpu/nvgpu/hal/fifo/preempt_gk20a.h @@ -28,10 +28,11 @@ struct gk20a; struct nvgpu_channel; struct nvgpu_tsg; -void gk20a_fifo_preempt_trigger(struct gk20a *g, u32 id, unsigned int id_type); +void gk20a_fifo_preempt_trigger(struct gk20a *g, + u32 runlist_id, u32 tsgid, unsigned int id_type); int gk20a_fifo_preempt_channel(struct gk20a *g, struct nvgpu_channel *ch); int gk20a_fifo_preempt_tsg(struct gk20a *g, u32 runlist_id, u32 tsgid); -int gk20a_fifo_is_preempt_pending(struct gk20a *g, u32 id, +int gk20a_fifo_is_preempt_pending(struct gk20a *g, u32 runlist_id, u32 id, unsigned int id_type, bool preempt_retries_left); #endif /* FIFO_PREEMPT_GK20A_H */ diff --git a/drivers/gpu/nvgpu/hal/fifo/preempt_gv11b.h b/drivers/gpu/nvgpu/hal/fifo/preempt_gv11b.h index 3ce660151..db4f960ab 100644 --- a/drivers/gpu/nvgpu/hal/fifo/preempt_gv11b.h +++ b/drivers/gpu/nvgpu/hal/fifo/preempt_gv11b.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2021, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2016-2023, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -30,9 +30,10 @@ struct gk20a; struct nvgpu_channel; struct nvgpu_tsg; -void gv11b_fifo_preempt_trigger(struct gk20a *g, u32 id, unsigned int id_type); +void gv11b_fifo_preempt_trigger(struct gk20a *g, + u32 runlist_id, u32 id, unsigned int id_type); int gv11b_fifo_preempt_channel(struct gk20a *g, struct nvgpu_channel *ch); -int gv11b_fifo_is_preempt_pending(struct gk20a *g, u32 id, +int gv11b_fifo_is_preempt_pending(struct gk20a *g, u32 runlist_id, u32 id, unsigned int id_type, bool preempt_retries_left); int gv11b_fifo_preempt_poll_pbdma(struct gk20a *g, u32 tsgid, u32 pbdma_id); diff --git a/drivers/gpu/nvgpu/hal/fifo/preempt_gv11b_fusa.c b/drivers/gpu/nvgpu/hal/fifo/preempt_gv11b_fusa.c index c2eb9f33d..866ed9df7 100644 --- a/drivers/gpu/nvgpu/hal/fifo/preempt_gv11b_fusa.c +++ b/drivers/gpu/nvgpu/hal/fifo/preempt_gv11b_fusa.c @@ -44,7 +44,7 @@ #include -void gv11b_fifo_preempt_trigger(struct gk20a *g, u32 id, unsigned int id_type) +void gv11b_fifo_preempt_trigger(struct gk20a *g, u32 runlist_id, u32 id, unsigned int id_type) { if (id_type == ID_TYPE_TSG) { nvgpu_writel(g, fifo_preempt_r(), @@ -54,7 +54,7 @@ void gv11b_fifo_preempt_trigger(struct gk20a *g, u32 id, unsigned int id_type) u32 reg_val; reg_val = nvgpu_readl(g, fifo_runlist_preempt_r()); - reg_val |= BIT32(id); + reg_val |= BIT32(runlist_id); nvgpu_writel(g, fifo_runlist_preempt_r(), reg_val); } else { nvgpu_log_info(g, "channel preempt is noop"); @@ -312,7 +312,7 @@ static int gv11b_fifo_preempt_poll_eng(struct gk20a *g, u32 id, return ret; } -int gv11b_fifo_is_preempt_pending(struct gk20a *g, u32 id, +int gv11b_fifo_is_preempt_pending(struct gk20a *g, u32 runlist_id, u32 id, unsigned int id_type, bool preempt_retries_left) { struct nvgpu_fifo *f = &g->fifo; @@ -325,13 +325,11 @@ int gv11b_fifo_is_preempt_pending(struct gk20a *g, u32 id, int err, ret = 0; u32 tsgid; - if (id_type == ID_TYPE_TSG) { - rl = f->tsg[id].runlist; - tsgid = id; - } else { - rl = f->channel[id].runlist; - tsgid = f->channel[id].tsgid; - } + /* GV11B onward, the function only supports tsg preemption */ + nvgpu_assert(id_type == ID_TYPE_TSG); + + rl = f->runlists[runlist_id]; + tsgid = id; nvgpu_log_info(g, "Check preempt pending for tsgid = %u", tsgid); diff --git a/drivers/gpu/nvgpu/hal/fifo/runlist_fifo_ga10b.c b/drivers/gpu/nvgpu/hal/fifo/runlist_fifo_ga10b.c index 13744b68d..2ad1d1c9f 100644 --- a/drivers/gpu/nvgpu/hal/fifo/runlist_fifo_ga10b.c +++ b/drivers/gpu/nvgpu/hal/fifo/runlist_fifo_ga10b.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -100,7 +100,8 @@ int ga10b_fifo_reschedule_preempt_next(struct nvgpu_channel *ch, return 0; } - g->ops.fifo.preempt_trigger(g, preempt_id, preempt_type != 0U); + g->ops.fifo.preempt_trigger(g, + runlist->id, preempt_id, preempt_type != 0U); #ifdef TRACEPOINTS_ENABLED trace_gk20a_reschedule_preempt_next(ch->chid, fecsstat0, engine_status.reg_data, fecsstat1, @@ -109,7 +110,7 @@ int ga10b_fifo_reschedule_preempt_next(struct nvgpu_channel *ch, nvgpu_runlist_readl(g, runlist, runlist_preempt_r()); #endif if (wait_preempt) { - if (g->ops.fifo.is_preempt_pending(g, preempt_id, + if (g->ops.fifo.is_preempt_pending(g, runlist->id, preempt_id, preempt_type, false) != 0) { nvgpu_err(g, "fifo preempt timed out"); /* diff --git a/drivers/gpu/nvgpu/hal/fifo/runlist_fifo_gk20a.c b/drivers/gpu/nvgpu/hal/fifo/runlist_fifo_gk20a.c index e6d5da201..92abd0506 100644 --- a/drivers/gpu/nvgpu/hal/fifo/runlist_fifo_gk20a.c +++ b/drivers/gpu/nvgpu/hal/fifo/runlist_fifo_gk20a.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2022, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -100,7 +100,7 @@ int gk20a_fifo_reschedule_preempt_next(struct nvgpu_channel *ch, return ret; } - g->ops.fifo.preempt_trigger(g, preempt_id, preempt_type != 0U); + g->ops.fifo.preempt_trigger(g, runlist->id, preempt_id, preempt_type != 0U); #ifdef TRACEPOINTS_ENABLED trace_gk20a_reschedule_preempt_next(ch->chid, fecsstat0, engine_status.reg_data, fecsstat1, @@ -109,7 +109,7 @@ int gk20a_fifo_reschedule_preempt_next(struct nvgpu_channel *ch, nvgpu_readl(g, fifo_preempt_r())); #endif if (wait_preempt) { - if (g->ops.fifo.is_preempt_pending(g, preempt_id, + if (g->ops.fifo.is_preempt_pending(g, runlist->id, preempt_id, preempt_type, false) != 0) { nvgpu_err(g, "fifo preempt timed out"); /* diff --git a/drivers/gpu/nvgpu/hal/gr/gr/gr_gp10b.c b/drivers/gpu/nvgpu/hal/gr/gr/gr_gp10b.c index 091e2aeee..3fbcc94f8 100644 --- a/drivers/gpu/nvgpu/hal/gr/gr/gr_gp10b.c +++ b/drivers/gpu/nvgpu/hal/gr/gr/gr_gp10b.c @@ -1,7 +1,7 @@ /* * GP10B GPU GR * - * Copyright (c) 2015-2022, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2015-2023, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -417,7 +417,8 @@ static int gr_gp10b_disable_channel_or_tsg(struct gk20a *g, struct nvgpu_channel nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg | gpu_dbg_intr, "CILP: tsgid: 0x%x", tsg->tsgid); - g->ops.fifo.preempt_trigger(g, tsg->tsgid, ID_TYPE_TSG); + g->ops.fifo.preempt_trigger(g, + tsg->runlist->id, tsg->tsgid, ID_TYPE_TSG); nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg | gpu_dbg_intr, "CILP: preempted tsg"); return ret; diff --git a/drivers/gpu/nvgpu/include/nvgpu/gops/fifo.h b/drivers/gpu/nvgpu/include/nvgpu/gops/fifo.h index 1d3bc4ef2..36af36eda 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/gops/fifo.h +++ b/drivers/gpu/nvgpu/include/nvgpu/gops/fifo.h @@ -209,10 +209,11 @@ struct gops_fifo { * * @return: None */ - void (*preempt_trigger)(struct gk20a *g, u32 id, unsigned int id_type); + void (*preempt_trigger)(struct gk20a *g, + u32 runlist_id, u32 tsgid, unsigned int id_type); int (*preempt_poll_pbdma)(struct gk20a *g, u32 tsgid, u32 pbdma_id); - int (*is_preempt_pending)(struct gk20a *g, u32 id, + int (*is_preempt_pending)(struct gk20a *g, u32 runlist_id, u32 id, unsigned int id_type, bool preempt_retries_left); void (*intr_set_recover_mask)(struct gk20a *g); void (*intr_unset_recover_mask)(struct gk20a *g); diff --git a/userspace/units/fifo/preempt/gv11b/nvgpu-preempt-gv11b.c b/userspace/units/fifo/preempt/gv11b/nvgpu-preempt-gv11b.c index a6cfe4827..9c29d8c5a 100644 --- a/userspace/units/fifo/preempt/gv11b/nvgpu-preempt-gv11b.c +++ b/userspace/units/fifo/preempt/gv11b/nvgpu-preempt-gv11b.c @@ -95,14 +95,14 @@ int test_gv11b_fifo_preempt_trigger(struct unit_module *m, struct gk20a *g, __func__, branches_str(branches, f_preempt_trigger)); if (branches & F_PREEMPT_TRIGGER_TSG) { - gv11b_fifo_preempt_trigger(g, 5U, ID_TYPE_TSG); + gv11b_fifo_preempt_trigger(g, 0U, 5U, ID_TYPE_TSG); expected_reg_val = fifo_preempt_id_f(5U) | fifo_preempt_type_tsg_f(); unit_assert(expected_reg_val == nvgpu_readl(g, fifo_preempt_r()), goto done); nvgpu_writel(g, fifo_preempt_r(), orig_reg_val); } else { - gv11b_fifo_preempt_trigger(g, 5U, ID_TYPE_CHANNEL); + gv11b_fifo_preempt_trigger(g, 0U, 5U, ID_TYPE_CHANNEL); unit_assert(orig_reg_val == nvgpu_readl(g, fifo_preempt_r()), goto done); } @@ -199,20 +199,22 @@ done: return ret; } -static void stub_fifo_preempt_trigger(struct gk20a *g, u32 id, +static void stub_fifo_preempt_trigger(struct gk20a *g, u32 runlist_id, u32 id, unsigned int id_type) { } -static int stub_fifo_is_preempt_pending_ebusy(struct gk20a *g, u32 id, +static int stub_fifo_is_preempt_pending_ebusy(struct gk20a *g, + u32 runlist_id, u32 id, unsigned int id_type, bool preempt_retries_left) { return -EBUSY; } -static int stub_fifo_is_preempt_pending_pass(struct gk20a *g, u32 id, +static int stub_fifo_is_preempt_pending_pass(struct gk20a *g, + u32 runlist_id, u32 id, unsigned int id_type, bool preempt_retries_left) { @@ -457,7 +459,7 @@ int test_gv11b_fifo_is_preempt_pending(struct unit_module *m, struct gk20a *g, /* Modify eng_stat for engine 0 */ nvgpu_writel(g, fifo_engine_status_r(0U), stub.eng_stat); - err = gv11b_fifo_is_preempt_pending(g, 0U, id_type, false); + err = gv11b_fifo_is_preempt_pending(g, 0U, 0U, id_type, false); if (branches & F_PREEMPT_PENDING_POLL_PBDMA_FAIL) { unit_assert(err == -ETIMEDOUT, goto done);