From 75963b47f1cb69fa3d85124fcc5a9d03c50688bf Mon Sep 17 00:00:00 2001 From: Thomas Fleury Date: Wed, 27 Mar 2019 16:54:04 -0700 Subject: [PATCH] gpu: nvgpu: move bind/unbind from fifo to tsg Moved the following HALs from fifo to tsg: - tsg.bind_channel - tsg.unbind_channel - tsg.unbind_channel_check_hw_state - tsg.unbind_channel_check_ctx_reload - tsg.unbind_channel_check_eng_faulted bind_channel and unbind_channel HALs are optional, and only implemented for vgpu: - vgpu_tsg_bind_channel - vgpu_tsg_unbind_channel Moved the following code from fifo to tsg: - nvgpu_tsg_bind_channel - nvgpu_tsg_unbind_channel - nvgpu_tsg_unbind_channel_check_hw_state - nvgpu_tsg_unbind_channel_check_ctx_reload - gv11b_tsg_unbind_channel_check_eng_faulted tsg is now explictly passed to bind/unbind operations, along with ch Jira NVGPU-2979 Change-Id: I337a3d73ceef5ff320b036b14739ef0e831a28ee Signed-off-by: Thomas Fleury Reviewed-on: https://git-master.nvidia.com/r/2084029 Reviewed-by: mobile promotions Tested-by: mobile promotions --- drivers/gpu/nvgpu/common/fifo/channel.c | 6 +- drivers/gpu/nvgpu/common/fifo/tsg.c | 151 ++++++++++++++++-- .../gpu/nvgpu/common/vgpu/fifo/fifo_vgpu.h | 5 +- .../nvgpu/common/vgpu/gp10b/vgpu_hal_gp10b.c | 9 +- .../nvgpu/common/vgpu/gv11b/vgpu_hal_gv11b.c | 11 +- .../nvgpu/common/vgpu/gv11b/vgpu_tsg_gv11b.c | 9 +- drivers/gpu/nvgpu/common/vgpu/tsg_vgpu.c | 25 +-- drivers/gpu/nvgpu/gk20a/ce2_gk20a.c | 2 +- drivers/gpu/nvgpu/gk20a/fifo_gk20a.c | 96 ----------- drivers/gpu/nvgpu/gk20a/fifo_gk20a.h | 3 - drivers/gpu/nvgpu/gm20b/fifo_gm20b.c | 20 --- drivers/gpu/nvgpu/gm20b/fifo_gm20b.h | 1 - drivers/gpu/nvgpu/gm20b/hal_gm20b.c | 11 +- drivers/gpu/nvgpu/gp10b/hal_gp10b.c | 11 +- drivers/gpu/nvgpu/gv100/hal_gv100.c | 13 +- drivers/gpu/nvgpu/gv11b/fifo_gv11b.c | 32 ---- drivers/gpu/nvgpu/gv11b/fifo_gv11b.h | 1 - drivers/gpu/nvgpu/gv11b/hal_gv11b.c | 13 +- drivers/gpu/nvgpu/hal/fifo/tsg_gv11b.c | 31 ++++ drivers/gpu/nvgpu/hal/fifo/tsg_gv11b.h | 5 + drivers/gpu/nvgpu/include/nvgpu/gk20a.h | 18 ++- drivers/gpu/nvgpu/include/nvgpu/tsg.h | 12 +- drivers/gpu/nvgpu/os/linux/cde.c | 2 +- drivers/gpu/nvgpu/os/linux/ioctl_tsg.c | 19 +-- drivers/gpu/nvgpu/tu104/hal_tu104.c | 13 +- 25 files changed, 265 insertions(+), 254 deletions(-) diff --git a/drivers/gpu/nvgpu/common/fifo/channel.c b/drivers/gpu/nvgpu/common/fifo/channel.c index 76bf8af89..574a52348 100644 --- a/drivers/gpu/nvgpu/common/fifo/channel.c +++ b/drivers/gpu/nvgpu/common/fifo/channel.c @@ -305,6 +305,7 @@ void gk20a_wait_until_counter_is_N( static void gk20a_free_channel(struct channel_gk20a *ch, bool force) { struct gk20a *g = ch->g; + struct tsg_gk20a *tsg; struct fifo_gk20a *f = &g->fifo; struct vm_gk20a *ch_vm = ch->vm; unsigned long timeout = nvgpu_get_poll_timeout(g); @@ -331,13 +332,14 @@ static void gk20a_free_channel(struct channel_gk20a *ch, bool force) */ if (!nvgpu_is_enabled(g, NVGPU_DRIVER_IS_DYING)) { /* abort channel and remove from runlist */ - if (tsg_gk20a_from_ch(ch) != NULL) { + tsg = tsg_gk20a_from_ch(ch); + if (tsg != NULL) { /* Between tsg is not null and unbind_channel call, * ioctl cannot be called anymore because user doesn't * have an open channel fd anymore to use for the unbind * ioctl. */ - err = gk20a_tsg_unbind_channel(ch); + err = nvgpu_tsg_unbind_channel(tsg, ch); if (err != 0) { nvgpu_err(g, "failed to unbind channel %d from TSG", diff --git a/drivers/gpu/nvgpu/common/fifo/tsg.c b/drivers/gpu/nvgpu/common/fifo/tsg.c index 563ce4426..22751116b 100644 --- a/drivers/gpu/nvgpu/common/fifo/tsg.c +++ b/drivers/gpu/nvgpu/common/fifo/tsg.c @@ -68,12 +68,12 @@ static bool gk20a_is_channel_active(struct gk20a *g, struct channel_gk20a *ch) * * Note that channel is not runnable when we bind it to TSG */ -int gk20a_tsg_bind_channel(struct tsg_gk20a *tsg, - struct channel_gk20a *ch) +int nvgpu_tsg_bind_channel(struct tsg_gk20a *tsg, struct channel_gk20a *ch) { struct gk20a *g = ch->g; + int err = 0; - nvgpu_log_fn(g, " "); + nvgpu_log_fn(g, "bind tsg:%u ch:%u\n", tsg->tsgid, ch->chid); /* check if channel is already bound to some TSG */ if (tsg_gk20a_from_ch(ch) != NULL) { @@ -85,17 +85,20 @@ int gk20a_tsg_bind_channel(struct tsg_gk20a *tsg, return -EINVAL; } - /* all the channel part of TSG should need to be same runlist_id */ if (tsg->runlist_id == FIFO_INVAL_TSG_ID) { tsg->runlist_id = ch->runlist_id; } else if (tsg->runlist_id != ch->runlist_id) { nvgpu_err(tsg->g, - "Error: TSG channel should be share same runlist ch[%d] tsg[%d]", + "runlist_id mismatch ch[%d] tsg[%d]", ch->runlist_id, tsg->runlist_id); return -EINVAL; } + if (g->ops.tsg.bind_channel != NULL) { + err = g->ops.tsg.bind_channel(tsg, ch); + } + nvgpu_rwsem_down_write(&tsg->ch_list_lock); nvgpu_list_add_tail(&ch->ch_entry, &tsg->ch_list); ch->tsgid = tsg->tsgid; @@ -103,28 +106,23 @@ int gk20a_tsg_bind_channel(struct tsg_gk20a *tsg, nvgpu_ref_get(&tsg->refcount); - nvgpu_log(g, gpu_dbg_fn, "BIND tsg:%d channel:%d\n", - tsg->tsgid, ch->chid); - - nvgpu_log_fn(g, "done"); - return 0; + return err; } /* The caller must ensure that channel belongs to a tsg */ -int gk20a_tsg_unbind_channel(struct channel_gk20a *ch) +int nvgpu_tsg_unbind_channel(struct tsg_gk20a *tsg, struct channel_gk20a *ch) { struct gk20a *g = ch->g; - struct tsg_gk20a *tsg = tsg_gk20a_from_ch(ch); int err; - nvgpu_assert(tsg != NULL); + nvgpu_log_fn(g, "unbind tsg:%u ch:%u\n", tsg->tsgid, ch->chid); - err = g->ops.fifo.tsg_unbind_channel(ch); + err = nvgpu_tsg_unbind_channel_common(tsg, ch); if (err != 0) { nvgpu_err(g, "Channel %d unbind failed, tearing down TSG %d", ch->chid, tsg->tsgid); - gk20a_fifo_abort_tsg(ch->g, tsg, true); + gk20a_fifo_abort_tsg(g, tsg, true); /* If channel unbind fails, channel is still part of runlist */ channel_gk20a_update_runlist(ch, false); @@ -133,14 +131,133 @@ int gk20a_tsg_unbind_channel(struct channel_gk20a *ch) ch->tsgid = NVGPU_INVALID_TSG_ID; nvgpu_rwsem_up_write(&tsg->ch_list_lock); } - nvgpu_log(g, gpu_dbg_fn, "UNBIND tsg:%d channel:%d", - tsg->tsgid, ch->chid); + + if (g->ops.tsg.unbind_channel != NULL) { + err = g->ops.tsg.unbind_channel(tsg, ch); + } nvgpu_ref_put(&tsg->refcount, gk20a_tsg_release); return 0; } +int nvgpu_tsg_unbind_channel_common(struct tsg_gk20a *tsg, + struct channel_gk20a *ch) +{ + struct gk20a *g = ch->g; + int err; + bool tsg_timedout; + + /* If one channel in TSG times out, we disable all channels */ + nvgpu_rwsem_down_write(&tsg->ch_list_lock); + tsg_timedout = gk20a_channel_check_unserviceable(ch); + nvgpu_rwsem_up_write(&tsg->ch_list_lock); + + /* Disable TSG and examine status before unbinding channel */ + g->ops.tsg.disable(tsg); + + err = g->ops.fifo.preempt_tsg(g, tsg); + if (err != 0) { + goto fail_enable_tsg; + } + + if (!tsg_timedout && + (g->ops.tsg.unbind_channel_check_hw_state != NULL)) { + err = g->ops.tsg.unbind_channel_check_hw_state(tsg, ch); + if (err != 0) { + nvgpu_err(g, "invalid hw_state for ch %u", ch->chid); + goto fail_enable_tsg; + } + } + + /* Channel should be seen as TSG channel while updating runlist */ + err = channel_gk20a_update_runlist(ch, false); + if (err != 0) { + nvgpu_err(g, "update runlist failed ch:%u tsg:%u", + ch->chid, tsg->tsgid); + goto fail_enable_tsg; + } + + /* Remove channel from TSG and re-enable rest of the channels */ + nvgpu_rwsem_down_write(&tsg->ch_list_lock); + nvgpu_list_del(&ch->ch_entry); + ch->tsgid = NVGPU_INVALID_TSG_ID; + + /* another thread could have re-enabled the channel because it was + * still on the list at that time, so make sure it's truly disabled + */ + g->ops.channel.disable(ch); + nvgpu_rwsem_up_write(&tsg->ch_list_lock); + + /* + * Don't re-enable all channels if TSG has timed out already + * + * Note that we can skip disabling and preempting TSG too in case of + * time out, but we keep that to ensure TSG is kicked out + */ + if (!tsg_timedout) { + g->ops.tsg.enable(tsg); + } + + if (g->ops.channel.abort_clean_up != NULL) { + g->ops.channel.abort_clean_up(ch); + } + + return 0; + +fail_enable_tsg: + if (!tsg_timedout) { + g->ops.tsg.enable(tsg); + } + return err; +} + +int nvgpu_tsg_unbind_channel_check_hw_state(struct tsg_gk20a *tsg, + struct channel_gk20a *ch) +{ + struct gk20a *g = ch->g; + struct nvgpu_channel_hw_state hw_state; + + g->ops.channel.read_state(g, ch, &hw_state); + + if (hw_state.next) { + nvgpu_err(g, "Channel %d to be removed from TSG %d has NEXT set!", + ch->chid, ch->tsgid); + return -EINVAL; + } + + if (g->ops.tsg.unbind_channel_check_ctx_reload != NULL) { + g->ops.tsg.unbind_channel_check_ctx_reload(tsg, ch, &hw_state); + } + + if (g->ops.tsg.unbind_channel_check_eng_faulted != NULL) { + g->ops.tsg.unbind_channel_check_eng_faulted(tsg, ch, + &hw_state); + } + + return 0; +} + +void nvgpu_tsg_unbind_channel_check_ctx_reload(struct tsg_gk20a *tsg, + struct channel_gk20a *ch, + struct nvgpu_channel_hw_state *hw_state) +{ + struct gk20a *g = ch->g; + struct channel_gk20a *temp_ch; + + /* If CTX_RELOAD is set on a channel, move it to some other channel */ + if (hw_state->ctx_reload) { + nvgpu_rwsem_down_read(&tsg->ch_list_lock); + nvgpu_list_for_each_entry(temp_ch, &tsg->ch_list, + channel_gk20a, ch_entry) { + if (temp_ch->chid != ch->chid) { + g->ops.channel.force_ctx_reload(temp_ch); + break; + } + } + nvgpu_rwsem_up_read(&tsg->ch_list_lock); + } +} void nvgpu_tsg_recover(struct gk20a *g, struct tsg_gk20a *tsg, bool verbose, u32 rc_type) diff --git a/drivers/gpu/nvgpu/common/vgpu/fifo/fifo_vgpu.h b/drivers/gpu/nvgpu/common/vgpu/fifo/fifo_vgpu.h index 0c5f2b1ae..c19f55f20 100644 --- a/drivers/gpu/nvgpu/common/vgpu/fifo/fifo_vgpu.h +++ b/drivers/gpu/nvgpu/common/vgpu/fifo/fifo_vgpu.h @@ -52,9 +52,8 @@ int vgpu_tsg_force_reset_ch(struct channel_gk20a *ch, u32 vgpu_fifo_default_timeslice_us(struct gk20a *g); int vgpu_tsg_open(struct tsg_gk20a *tsg); void vgpu_tsg_release(struct tsg_gk20a *tsg); -int vgpu_tsg_bind_channel(struct tsg_gk20a *tsg, - struct channel_gk20a *ch); -int vgpu_tsg_unbind_channel(struct channel_gk20a *ch); +int vgpu_tsg_bind_channel(struct tsg_gk20a *tsg, struct channel_gk20a *ch); +int vgpu_tsg_unbind_channel(struct tsg_gk20a *tsg, struct channel_gk20a *ch); int vgpu_tsg_set_timeslice(struct tsg_gk20a *tsg, u32 timeslice); void vgpu_tsg_enable(struct tsg_gk20a *tsg); int vgpu_set_sm_exception_type_mask(struct channel_gk20a *ch, u32 mask); diff --git a/drivers/gpu/nvgpu/common/vgpu/gp10b/vgpu_hal_gp10b.c b/drivers/gpu/nvgpu/common/vgpu/gp10b/vgpu_hal_gp10b.c index 310d0c7f8..c38cd9860 100644 --- a/drivers/gpu/nvgpu/common/vgpu/gp10b/vgpu_hal_gp10b.c +++ b/drivers/gpu/nvgpu/common/vgpu/gp10b/vgpu_hal_gp10b.c @@ -416,8 +416,6 @@ static const struct gpu_ops vgpu_gp10b_ops = { .default_timeslice_us = vgpu_fifo_default_timeslice_us, .preempt_channel = vgpu_fifo_preempt_channel, .preempt_tsg = vgpu_fifo_preempt_tsg, - .tsg_verify_channel_status = NULL, - .tsg_verify_status_ctx_reload = NULL, .tsg_set_timeslice = vgpu_tsg_set_timeslice, .tsg_open = vgpu_tsg_open, .tsg_release = vgpu_tsg_release, @@ -425,8 +423,6 @@ static const struct gpu_ops vgpu_gp10b_ops = { .is_preempt_pending = NULL, .reset_enable_hw = NULL, .teardown_ch_tsg = NULL, - .tsg_bind_channel = vgpu_tsg_bind_channel, - .tsg_unbind_channel = vgpu_tsg_unbind_channel, .post_event_id = gk20a_tsg_event_id_post_event, .setup_sw = vgpu_fifo_setup_sw, .cleanup_sw = vgpu_fifo_cleanup_sw, @@ -546,6 +542,11 @@ static const struct gpu_ops vgpu_gp10b_ops = { .tsg = { .enable = vgpu_tsg_enable, .disable = nvgpu_tsg_disable, + .bind_channel = vgpu_tsg_bind_channel, + .unbind_channel = vgpu_tsg_unbind_channel, + .unbind_channel_check_hw_state = NULL, + .unbind_channel_check_ctx_reload = NULL, + .unbind_channel_check_eng_faulted = NULL, .check_ctxsw_timeout = nvgpu_tsg_check_ctxsw_timeout, .force_reset = vgpu_tsg_force_reset_ch, }, diff --git a/drivers/gpu/nvgpu/common/vgpu/gv11b/vgpu_hal_gv11b.c b/drivers/gpu/nvgpu/common/vgpu/gv11b/vgpu_hal_gv11b.c index c17f8e94d..a3bc56efb 100644 --- a/drivers/gpu/nvgpu/common/vgpu/gv11b/vgpu_hal_gv11b.c +++ b/drivers/gpu/nvgpu/common/vgpu/gv11b/vgpu_hal_gv11b.c @@ -499,10 +499,6 @@ static const struct gpu_ops vgpu_gv11b_ops = { .default_timeslice_us = vgpu_fifo_default_timeslice_us, .preempt_channel = vgpu_fifo_preempt_channel, .preempt_tsg = vgpu_fifo_preempt_tsg, - .tsg_verify_channel_status = NULL, - .tsg_verify_status_ctx_reload = NULL, - /* TODO: implement it for CE fault */ - .tsg_verify_status_faulted = NULL, .tsg_set_timeslice = vgpu_tsg_set_timeslice, .tsg_open = vgpu_tsg_open, .tsg_release = vgpu_tsg_release, @@ -513,8 +509,6 @@ static const struct gpu_ops vgpu_gv11b_ops = { .init_eng_method_buffers = gv11b_fifo_init_eng_method_buffers, .deinit_eng_method_buffers = gv11b_fifo_deinit_eng_method_buffers, - .tsg_bind_channel = vgpu_gv11b_tsg_bind_channel, - .tsg_unbind_channel = vgpu_tsg_unbind_channel, .post_event_id = gk20a_tsg_event_id_post_event, .setup_sw = vgpu_fifo_setup_sw, .cleanup_sw = vgpu_fifo_cleanup_sw, @@ -636,6 +630,11 @@ static const struct gpu_ops vgpu_gv11b_ops = { .tsg = { .enable = gv11b_tsg_enable, .disable = nvgpu_tsg_disable, + .bind_channel = vgpu_gv11b_tsg_bind_channel, + .unbind_channel = vgpu_tsg_unbind_channel, + .unbind_channel_check_hw_state = NULL, + .unbind_channel_check_ctx_reload = NULL, + .unbind_channel_check_eng_faulted = NULL, .check_ctxsw_timeout = nvgpu_tsg_check_ctxsw_timeout, .force_reset = vgpu_tsg_force_reset_ch, }, diff --git a/drivers/gpu/nvgpu/common/vgpu/gv11b/vgpu_tsg_gv11b.c b/drivers/gpu/nvgpu/common/vgpu/gv11b/vgpu_tsg_gv11b.c index 907f35243..1afce23a1 100644 --- a/drivers/gpu/nvgpu/common/vgpu/gv11b/vgpu_tsg_gv11b.c +++ b/drivers/gpu/nvgpu/common/vgpu/gv11b/vgpu_tsg_gv11b.c @@ -39,11 +39,6 @@ int vgpu_gv11b_tsg_bind_channel(struct tsg_gk20a *tsg, nvgpu_log_fn(g, " "); - err = gk20a_tsg_bind_channel(tsg, ch); - if (err) { - return err; - } - msg.cmd = TEGRA_VGPU_CMD_TSG_BIND_CHANNEL_EX; msg.handle = vgpu_get_handle(tsg->g); p->tsg_id = tsg->tsgid; @@ -53,10 +48,8 @@ int vgpu_gv11b_tsg_bind_channel(struct tsg_gk20a *tsg, err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg)); err = err ? err : msg.ret; if (err) { - nvgpu_err(tsg->g, - "vgpu_gv11b_tsg_bind_channel failed, ch %d tsgid %d", + nvgpu_err(g, "vgpu bind channel failed, ch %d tsgid %d", ch->chid, tsg->tsgid); - gk20a_tsg_unbind_channel(ch); } return err; diff --git a/drivers/gpu/nvgpu/common/vgpu/tsg_vgpu.c b/drivers/gpu/nvgpu/common/vgpu/tsg_vgpu.c index d8f4efac6..621ed911b 100644 --- a/drivers/gpu/nvgpu/common/vgpu/tsg_vgpu.c +++ b/drivers/gpu/nvgpu/common/vgpu/tsg_vgpu.c @@ -88,8 +88,7 @@ void vgpu_tsg_enable(struct tsg_gk20a *tsg) nvgpu_rwsem_up_read(&tsg->ch_list_lock); } -int vgpu_tsg_bind_channel(struct tsg_gk20a *tsg, - struct channel_gk20a *ch) +int vgpu_tsg_bind_channel(struct tsg_gk20a *tsg, struct channel_gk20a *ch) { struct tegra_vgpu_cmd_msg msg = {}; struct tegra_vgpu_tsg_bind_unbind_channel_params *p = @@ -99,28 +98,21 @@ int vgpu_tsg_bind_channel(struct tsg_gk20a *tsg, nvgpu_log_fn(g, " "); - err = gk20a_tsg_bind_channel(tsg, ch); - if (err) { - return err; - } - msg.cmd = TEGRA_VGPU_CMD_TSG_BIND_CHANNEL; - msg.handle = vgpu_get_handle(tsg->g); + msg.handle = vgpu_get_handle(g); p->tsg_id = tsg->tsgid; p->ch_handle = ch->virt_ctx; err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg)); err = err ? err : msg.ret; if (err) { - nvgpu_err(tsg->g, - "vgpu_tsg_bind_channel failed, ch %d tsgid %d", + nvgpu_err(g, "vgpu_tsg_bind_channel failed, ch %d tsgid %d", ch->chid, tsg->tsgid); - gk20a_tsg_unbind_channel(ch); } return err; } -int vgpu_tsg_unbind_channel(struct channel_gk20a *ch) +int vgpu_tsg_unbind_channel(struct tsg_gk20a *tsg, struct channel_gk20a *ch) { struct tegra_vgpu_cmd_msg msg = {}; struct tegra_vgpu_tsg_bind_unbind_channel_params *p = @@ -130,13 +122,8 @@ int vgpu_tsg_unbind_channel(struct channel_gk20a *ch) nvgpu_log_fn(g, " "); - err = gk20a_fifo_tsg_unbind_channel(ch); - if (err) { - return err; - } - msg.cmd = TEGRA_VGPU_CMD_TSG_UNBIND_CHANNEL; - msg.handle = vgpu_get_handle(ch->g); + msg.handle = vgpu_get_handle(g); p->ch_handle = ch->virt_ctx; err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg)); err = err ? err : msg.ret; @@ -156,7 +143,7 @@ int vgpu_tsg_set_timeslice(struct tsg_gk20a *tsg, u32 timeslice) nvgpu_log_fn(g, " "); msg.cmd = TEGRA_VGPU_CMD_TSG_SET_TIMESLICE; - msg.handle = vgpu_get_handle(tsg->g); + msg.handle = vgpu_get_handle(g); p->tsg_id = tsg->tsgid; p->timeslice_us = timeslice; err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg)); diff --git a/drivers/gpu/nvgpu/gk20a/ce2_gk20a.c b/drivers/gpu/nvgpu/gk20a/ce2_gk20a.c index 520aafa87..57e920a8f 100644 --- a/drivers/gpu/nvgpu/gk20a/ce2_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/ce2_gk20a.c @@ -494,7 +494,7 @@ u32 gk20a_ce_create_context(struct gk20a *g, goto end; } - err = gk20a_tsg_bind_channel(ce_ctx->tsg, ce_ctx->ch); + err = nvgpu_tsg_bind_channel(ce_ctx->tsg, ce_ctx->ch); if (err != 0) { nvgpu_err(g, "ce: unable to bind to tsg"); goto end; diff --git a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c index c807a38f2..fe3ea87fa 100644 --- a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c @@ -634,102 +634,6 @@ void gk20a_fifo_recover(struct gk20a *g, u32 engine_ids, rc_type, NULL); } -int gk20a_fifo_tsg_unbind_channel_verify_status(struct channel_gk20a *ch) -{ - struct gk20a *g = ch->g; - struct nvgpu_channel_hw_state hw_state; - - g->ops.channel.read_state(g, ch, &hw_state); - - if (hw_state.next) { - nvgpu_err(g, "Channel %d to be removed from TSG %d has NEXT set!", - ch->chid, ch->tsgid); - return -EINVAL; - } - - if (g->ops.fifo.tsg_verify_status_ctx_reload != NULL) { - g->ops.fifo.tsg_verify_status_ctx_reload(ch); - } - - if (g->ops.fifo.tsg_verify_status_faulted != NULL) { - g->ops.fifo.tsg_verify_status_faulted(ch); - } - - return 0; -} - -int gk20a_fifo_tsg_unbind_channel(struct channel_gk20a *ch) -{ - struct gk20a *g = ch->g; - struct tsg_gk20a *tsg = tsg_gk20a_from_ch(ch); - int err; - bool tsg_timedout = false; - - if (tsg == NULL) { - nvgpu_err(g, "chid: %d is not bound to tsg", ch->chid); - return 0; - } - - /* If one channel in TSG times out, we disable all channels */ - nvgpu_rwsem_down_write(&tsg->ch_list_lock); - tsg_timedout = gk20a_channel_check_unserviceable(ch); - nvgpu_rwsem_up_write(&tsg->ch_list_lock); - - /* Disable TSG and examine status before unbinding channel */ - g->ops.tsg.disable(tsg); - - err = g->ops.fifo.preempt_tsg(g, tsg); - if (err != 0) { - goto fail_enable_tsg; - } - - if ((g->ops.fifo.tsg_verify_channel_status != NULL) && !tsg_timedout) { - err = g->ops.fifo.tsg_verify_channel_status(ch); - if (err != 0) { - goto fail_enable_tsg; - } - } - - /* Channel should be seen as TSG channel while updating runlist */ - err = channel_gk20a_update_runlist(ch, false); - if (err != 0) { - goto fail_enable_tsg; - } - - /* Remove channel from TSG and re-enable rest of the channels */ - nvgpu_rwsem_down_write(&tsg->ch_list_lock); - nvgpu_list_del(&ch->ch_entry); - ch->tsgid = NVGPU_INVALID_TSG_ID; - - /* another thread could have re-enabled the channel because it was - * still on the list at that time, so make sure it's truly disabled - */ - g->ops.channel.disable(ch); - nvgpu_rwsem_up_write(&tsg->ch_list_lock); - - /* - * Don't re-enable all channels if TSG has timed out already - * - * Note that we can skip disabling and preempting TSG too in case of - * time out, but we keep that to ensure TSG is kicked out - */ - if (!tsg_timedout) { - g->ops.tsg.enable(tsg); - } - - if (ch->g->ops.channel.abort_clean_up != NULL) { - ch->g->ops.channel.abort_clean_up(ch); - } - - return 0; - -fail_enable_tsg: - if (!tsg_timedout) { - g->ops.tsg.enable(tsg); - } - return err; -} - u32 gk20a_fifo_get_failing_engine_data(struct gk20a *g, u32 *__id, bool *__is_tsg) { diff --git a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.h b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.h index ae3850da9..7173bbbda 100644 --- a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.h +++ b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.h @@ -265,7 +265,6 @@ void gk20a_fifo_recover(struct gk20a *g, bool id_is_tsg, /* ignored if hw_id == ~0 */ bool id_is_known, bool verbose, u32 rc_type); int gk20a_init_fifo_reset_enable_hw(struct gk20a *g); -int gk20a_fifo_tsg_unbind_channel(struct channel_gk20a *ch); void fifo_gk20a_finish_mmu_fault_handling(struct gk20a *g, unsigned long fault_id); @@ -305,8 +304,6 @@ void gk20a_debug_dump_all_channel_status_ramfc(struct gk20a *g, struct gk20a_debug_output *o); const char *gk20a_decode_pbdma_chan_eng_ctx_status(u32 index); -int gk20a_fifo_tsg_unbind_channel_verify_status(struct channel_gk20a *ch); - int gk20a_fifo_is_preempt_pending(struct gk20a *g, u32 id, unsigned int id_type); int __locked_fifo_preempt(struct gk20a *g, u32 id, bool is_tsg); diff --git a/drivers/gpu/nvgpu/gm20b/fifo_gm20b.c b/drivers/gpu/nvgpu/gm20b/fifo_gm20b.c index 89c0c3aea..a02658105 100644 --- a/drivers/gpu/nvgpu/gm20b/fifo_gm20b.c +++ b/drivers/gpu/nvgpu/gm20b/fifo_gm20b.c @@ -38,23 +38,3 @@ #include "gk20a/fifo_gk20a.h" #include "fifo_gm20b.h" -void gm20b_fifo_tsg_verify_status_ctx_reload(struct channel_gk20a *ch) -{ - struct gk20a *g = ch->g; - struct tsg_gk20a *tsg = &g->fifo.tsg[ch->tsgid]; - struct channel_gk20a *temp_ch; - struct nvgpu_channel_hw_state hw_state; - - /* If CTX_RELOAD is set on a channel, move it to some other channel */ - g->ops.channel.read_state(g, ch, &hw_state); - if (hw_state.ctx_reload) { - nvgpu_rwsem_down_read(&tsg->ch_list_lock); - nvgpu_list_for_each_entry(temp_ch, &tsg->ch_list, channel_gk20a, ch_entry) { - if (temp_ch->chid != ch->chid) { - g->ops.channel.force_ctx_reload(temp_ch); - break; - } - } - nvgpu_rwsem_up_read(&tsg->ch_list_lock); - } -} diff --git a/drivers/gpu/nvgpu/gm20b/fifo_gm20b.h b/drivers/gpu/nvgpu/gm20b/fifo_gm20b.h index 756b9dc07..602dc118d 100644 --- a/drivers/gpu/nvgpu/gm20b/fifo_gm20b.h +++ b/drivers/gpu/nvgpu/gm20b/fifo_gm20b.h @@ -30,6 +30,5 @@ struct fifo_gk20a; struct mmu_fault_info; void gm20b_fifo_init_pbdma_intr_descs(struct fifo_gk20a *f); -void gm20b_fifo_tsg_verify_status_ctx_reload(struct channel_gk20a *ch); #endif /* NVGPU_GM20B_FIFO_GM20B_H */ diff --git a/drivers/gpu/nvgpu/gm20b/hal_gm20b.c b/drivers/gpu/nvgpu/gm20b/hal_gm20b.c index 7d2b1472f..95095ca42 100644 --- a/drivers/gpu/nvgpu/gm20b/hal_gm20b.c +++ b/drivers/gpu/nvgpu/gm20b/hal_gm20b.c @@ -628,8 +628,6 @@ static const struct gpu_ops gm20b_ops = { .default_timeslice_us = gk20a_fifo_default_timeslice_us, .preempt_channel = gk20a_fifo_preempt_channel, .preempt_tsg = gk20a_fifo_preempt_tsg, - .tsg_verify_channel_status = gk20a_fifo_tsg_unbind_channel_verify_status, - .tsg_verify_status_ctx_reload = gm20b_fifo_tsg_verify_status_ctx_reload, .tsg_set_timeslice = gk20a_fifo_tsg_set_timeslice, .init_pbdma_info = gk20a_fifo_init_pbdma_info, .dump_channel_status_ramfc = gk20a_dump_channel_status_ramfc, @@ -638,8 +636,6 @@ static const struct gpu_ops gm20b_ops = { .teardown_ch_tsg = gk20a_fifo_teardown_ch_tsg, .teardown_mask_intr = gk20a_fifo_teardown_mask_intr, .teardown_unmask_intr = gk20a_fifo_teardown_unmask_intr, - .tsg_bind_channel = gk20a_tsg_bind_channel, - .tsg_unbind_channel = gk20a_fifo_tsg_unbind_channel, .post_event_id = gk20a_tsg_event_id_post_event, .setup_sw = nvgpu_fifo_setup_sw, .cleanup_sw = nvgpu_fifo_cleanup_sw, @@ -768,6 +764,13 @@ static const struct gpu_ops gm20b_ops = { .tsg = { .enable = gk20a_tsg_enable, .disable = nvgpu_tsg_disable, + .bind_channel = NULL, + .unbind_channel = NULL, + .unbind_channel_check_hw_state = + nvgpu_tsg_unbind_channel_check_hw_state, + .unbind_channel_check_ctx_reload = + nvgpu_tsg_unbind_channel_check_ctx_reload, + .unbind_channel_check_eng_faulted = NULL, .check_ctxsw_timeout = nvgpu_tsg_check_ctxsw_timeout, .force_reset = nvgpu_tsg_force_reset_ch, }, diff --git a/drivers/gpu/nvgpu/gp10b/hal_gp10b.c b/drivers/gpu/nvgpu/gp10b/hal_gp10b.c index 7aa5abee5..5b61d0863 100644 --- a/drivers/gpu/nvgpu/gp10b/hal_gp10b.c +++ b/drivers/gpu/nvgpu/gp10b/hal_gp10b.c @@ -717,8 +717,6 @@ static const struct gpu_ops gp10b_ops = { .default_timeslice_us = gk20a_fifo_default_timeslice_us, .preempt_channel = gk20a_fifo_preempt_channel, .preempt_tsg = gk20a_fifo_preempt_tsg, - .tsg_verify_channel_status = gk20a_fifo_tsg_unbind_channel_verify_status, - .tsg_verify_status_ctx_reload = gm20b_fifo_tsg_verify_status_ctx_reload, .tsg_set_timeslice = gk20a_fifo_tsg_set_timeslice, .init_pbdma_info = gk20a_fifo_init_pbdma_info, .dump_channel_status_ramfc = gk20a_dump_channel_status_ramfc, @@ -727,8 +725,6 @@ static const struct gpu_ops gp10b_ops = { .teardown_ch_tsg = gk20a_fifo_teardown_ch_tsg, .teardown_mask_intr = gk20a_fifo_teardown_mask_intr, .teardown_unmask_intr = gk20a_fifo_teardown_unmask_intr, - .tsg_bind_channel = gk20a_tsg_bind_channel, - .tsg_unbind_channel = gk20a_fifo_tsg_unbind_channel, .post_event_id = gk20a_tsg_event_id_post_event, .setup_sw = nvgpu_fifo_setup_sw, .cleanup_sw = nvgpu_fifo_cleanup_sw, @@ -860,6 +856,13 @@ static const struct gpu_ops gp10b_ops = { .tsg = { .enable = gk20a_tsg_enable, .disable = nvgpu_tsg_disable, + .bind_channel = NULL, + .unbind_channel = NULL, + .unbind_channel_check_hw_state = + nvgpu_tsg_unbind_channel_check_hw_state, + .unbind_channel_check_ctx_reload = + nvgpu_tsg_unbind_channel_check_ctx_reload, + .unbind_channel_check_eng_faulted = NULL, .check_ctxsw_timeout = nvgpu_tsg_check_ctxsw_timeout, .force_reset = nvgpu_tsg_force_reset_ch, }, diff --git a/drivers/gpu/nvgpu/gv100/hal_gv100.c b/drivers/gpu/nvgpu/gv100/hal_gv100.c index e4c1a693e..849825566 100644 --- a/drivers/gpu/nvgpu/gv100/hal_gv100.c +++ b/drivers/gpu/nvgpu/gv100/hal_gv100.c @@ -898,9 +898,6 @@ static const struct gpu_ops gv100_ops = { .default_timeslice_us = gk20a_fifo_default_timeslice_us, .preempt_channel = gv11b_fifo_preempt_channel, .preempt_tsg = gv11b_fifo_preempt_tsg, - .tsg_verify_channel_status = gk20a_fifo_tsg_unbind_channel_verify_status, - .tsg_verify_status_ctx_reload = gm20b_fifo_tsg_verify_status_ctx_reload, - .tsg_verify_status_faulted = gv11b_fifo_tsg_verify_status_faulted, .tsg_set_timeslice = gk20a_fifo_tsg_set_timeslice, .init_pbdma_info = gk20a_fifo_init_pbdma_info, .dump_channel_status_ramfc = gv11b_dump_channel_status_ramfc, @@ -912,8 +909,6 @@ static const struct gpu_ops gv100_ops = { .init_eng_method_buffers = gv11b_fifo_init_eng_method_buffers, .deinit_eng_method_buffers = gv11b_fifo_deinit_eng_method_buffers, - .tsg_bind_channel = gk20a_tsg_bind_channel, - .tsg_unbind_channel = gk20a_fifo_tsg_unbind_channel, .post_event_id = gk20a_tsg_event_id_post_event, .setup_sw = nvgpu_fifo_setup_sw, .cleanup_sw = nvgpu_fifo_cleanup_sw, @@ -1046,6 +1041,14 @@ static const struct gpu_ops gv100_ops = { .tsg = { .enable = gv11b_tsg_enable, .disable = nvgpu_tsg_disable, + .bind_channel = NULL, + .unbind_channel = NULL, + .unbind_channel_check_hw_state = + nvgpu_tsg_unbind_channel_check_hw_state, + .unbind_channel_check_ctx_reload = + nvgpu_tsg_unbind_channel_check_ctx_reload, + .unbind_channel_check_eng_faulted = + gv11b_tsg_unbind_channel_check_eng_faulted, .check_ctxsw_timeout = nvgpu_tsg_check_ctxsw_timeout, .force_reset = nvgpu_tsg_force_reset_ch, }, diff --git a/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c b/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c index a5cb02c9e..ba3f70c45 100644 --- a/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c +++ b/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c @@ -1192,35 +1192,3 @@ void gv11b_mmu_fault_id_to_eng_pbdma_id_and_veid(struct gk20a *g, *pbdma_id = FIFO_INVAL_PBDMA_ID; } } - -void gv11b_fifo_tsg_verify_status_faulted(struct channel_gk20a *ch) -{ - struct gk20a *g = ch->g; - struct tsg_gk20a *tsg = &g->fifo.tsg[ch->tsgid]; - struct nvgpu_channel_hw_state hw_state; - - g->ops.channel.read_state(g, ch, &hw_state); - /* - * If channel has FAULTED set, clear the CE method buffer - * if saved out channel is same as faulted channel - */ - if (!hw_state.eng_faulted) { - return; - } - - if (tsg->eng_method_buffers == NULL) { - return; - } - - /* - * CE method buffer format : - * DWord0 = method count - * DWord1 = channel id - * - * It is sufficient to write 0 to method count to invalidate - */ - if ((u32)ch->chid == - nvgpu_mem_rd32(g, &tsg->eng_method_buffers[ASYNC_CE_RUNQUE], 1)) { - nvgpu_mem_wr32(g, &tsg->eng_method_buffers[ASYNC_CE_RUNQUE], 0, 0); - } -} diff --git a/drivers/gpu/nvgpu/gv11b/fifo_gv11b.h b/drivers/gpu/nvgpu/gv11b/fifo_gv11b.h index 69f7a68b1..44edbd48e 100644 --- a/drivers/gpu/nvgpu/gv11b/fifo_gv11b.h +++ b/drivers/gpu/nvgpu/gv11b/fifo_gv11b.h @@ -67,7 +67,6 @@ void gv11b_fifo_deinit_eng_method_buffers(struct gk20a *g, struct tsg_gk20a *tsg); int gv11b_init_fifo_setup_hw(struct gk20a *g); -void gv11b_fifo_tsg_verify_status_faulted(struct channel_gk20a *ch); u32 gv11b_fifo_get_preempt_timeout(struct gk20a *g); void gv11b_fifo_init_ramfc_eng_method_buffer(struct gk20a *g, diff --git a/drivers/gpu/nvgpu/gv11b/hal_gv11b.c b/drivers/gpu/nvgpu/gv11b/hal_gv11b.c index fecb945eb..e76abc86f 100644 --- a/drivers/gpu/nvgpu/gv11b/hal_gv11b.c +++ b/drivers/gpu/nvgpu/gv11b/hal_gv11b.c @@ -854,9 +854,6 @@ static const struct gpu_ops gv11b_ops = { .default_timeslice_us = gk20a_fifo_default_timeslice_us, .preempt_channel = gv11b_fifo_preempt_channel, .preempt_tsg = gv11b_fifo_preempt_tsg, - .tsg_verify_channel_status = gk20a_fifo_tsg_unbind_channel_verify_status, - .tsg_verify_status_ctx_reload = gm20b_fifo_tsg_verify_status_ctx_reload, - .tsg_verify_status_faulted = gv11b_fifo_tsg_verify_status_faulted, .tsg_set_timeslice = gk20a_fifo_tsg_set_timeslice, .init_pbdma_info = gk20a_fifo_init_pbdma_info, .dump_channel_status_ramfc = gv11b_dump_channel_status_ramfc, @@ -868,8 +865,6 @@ static const struct gpu_ops gv11b_ops = { .init_eng_method_buffers = gv11b_fifo_init_eng_method_buffers, .deinit_eng_method_buffers = gv11b_fifo_deinit_eng_method_buffers, - .tsg_bind_channel = gk20a_tsg_bind_channel, - .tsg_unbind_channel = gk20a_fifo_tsg_unbind_channel, .post_event_id = gk20a_tsg_event_id_post_event, .setup_sw = nvgpu_fifo_setup_sw, .cleanup_sw = nvgpu_fifo_cleanup_sw, @@ -1004,6 +999,14 @@ static const struct gpu_ops gv11b_ops = { .tsg = { .enable = gv11b_tsg_enable, .disable = nvgpu_tsg_disable, + .bind_channel = NULL, + .unbind_channel = NULL, + .unbind_channel_check_hw_state = + nvgpu_tsg_unbind_channel_check_hw_state, + .unbind_channel_check_ctx_reload = + nvgpu_tsg_unbind_channel_check_ctx_reload, + .unbind_channel_check_eng_faulted = + gv11b_tsg_unbind_channel_check_eng_faulted, .check_ctxsw_timeout = nvgpu_tsg_check_ctxsw_timeout, .force_reset = nvgpu_tsg_force_reset_ch, }, diff --git a/drivers/gpu/nvgpu/hal/fifo/tsg_gv11b.c b/drivers/gpu/nvgpu/hal/fifo/tsg_gv11b.c index d02d9a835..a0b24bb36 100644 --- a/drivers/gpu/nvgpu/hal/fifo/tsg_gv11b.c +++ b/drivers/gpu/nvgpu/hal/fifo/tsg_gv11b.c @@ -21,11 +21,14 @@ */ #include +#include #include #include #include "hal/fifo/tsg_gv11b.h" +#include "gv11b/fifo_gv11b.h" + /* TSG enable sequence applicable for Volta and onwards */ void gv11b_tsg_enable(struct tsg_gk20a *tsg) { @@ -44,3 +47,31 @@ void gv11b_tsg_enable(struct tsg_gk20a *tsg) g->ops.fifo.ring_channel_doorbell(last_ch); } } + +void gv11b_tsg_unbind_channel_check_eng_faulted(struct tsg_gk20a *tsg, + struct channel_gk20a *ch, + struct nvgpu_channel_hw_state *hw_state) +{ + struct gk20a *g = tsg->g; + struct nvgpu_mem *mem; + + /* + * If channel has FAULTED set, clear the CE method buffer + * if saved out channel is same as faulted channel + */ + if (!hw_state->eng_faulted || (tsg->eng_method_buffers == NULL)) { + return; + } + + /* + * CE method buffer format : + * DWord0 = method count + * DWord1 = channel id + * + * It is sufficient to write 0 to method count to invalidate + */ + mem = &tsg->eng_method_buffers[ASYNC_CE_RUNQUE]; + if (ch->chid == nvgpu_mem_rd32(g, mem, 1)) { + nvgpu_mem_wr32(g, mem, 0, 0); + } +} diff --git a/drivers/gpu/nvgpu/hal/fifo/tsg_gv11b.h b/drivers/gpu/nvgpu/hal/fifo/tsg_gv11b.h index 1bc7e42f2..6453064ce 100644 --- a/drivers/gpu/nvgpu/hal/fifo/tsg_gv11b.h +++ b/drivers/gpu/nvgpu/hal/fifo/tsg_gv11b.h @@ -24,7 +24,12 @@ #define NVGPU_TSG_GV11B_H struct tsg_gk20a; +struct channel_gk20a; +struct nvgpu_channel_hw_state; void gv11b_tsg_enable(struct tsg_gk20a *tsg); +void gv11b_tsg_unbind_channel_check_eng_faulted(struct tsg_gk20a *tsg, + struct channel_gk20a *ch, + struct nvgpu_channel_hw_state *hw_state); #endif /* NVGPU_TSG_GV11B_H */ diff --git a/drivers/gpu/nvgpu/include/nvgpu/gk20a.h b/drivers/gpu/nvgpu/include/nvgpu/gk20a.h index b1ba328e6..7770732c0 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/gk20a.h +++ b/drivers/gpu/nvgpu/include/nvgpu/gk20a.h @@ -952,15 +952,9 @@ struct gpu_ops { int (*init_fifo_setup_hw)(struct gk20a *g); int (*preempt_channel)(struct gk20a *g, struct channel_gk20a *ch); int (*preempt_tsg)(struct gk20a *g, struct tsg_gk20a *tsg); - int (*tsg_verify_channel_status)(struct channel_gk20a *ch); - void (*tsg_verify_status_ctx_reload)(struct channel_gk20a *ch); - void (*tsg_verify_status_faulted)(struct channel_gk20a *ch); void (*apply_pb_timeout)(struct gk20a *g); int (*tsg_set_timeslice)(struct tsg_gk20a *tsg, u32 timeslice); u32 (*default_timeslice_us)(struct gk20a *g); - int (*tsg_bind_channel)(struct tsg_gk20a *tsg, - struct channel_gk20a *ch); - int (*tsg_unbind_channel)(struct channel_gk20a *ch); int (*tsg_open)(struct tsg_gk20a *tsg); void (*tsg_release)(struct tsg_gk20a *tsg); int (*init_pbdma_info)(struct fifo_gk20a *f); @@ -1156,6 +1150,18 @@ struct gpu_ops { struct { void (*enable)(struct tsg_gk20a *tsg); void (*disable)(struct tsg_gk20a *tsg); + int (*bind_channel)(struct tsg_gk20a *tsg, + struct channel_gk20a *ch); + int (*unbind_channel)(struct tsg_gk20a *tsg, + struct channel_gk20a *ch); + int (*unbind_channel_check_hw_state)(struct tsg_gk20a *tsg, + struct channel_gk20a *ch); + void (*unbind_channel_check_ctx_reload)(struct tsg_gk20a *tsg, + struct channel_gk20a *ch, + struct nvgpu_channel_hw_state *state); + void (*unbind_channel_check_eng_faulted)(struct tsg_gk20a *tsg, + struct channel_gk20a *ch, + struct nvgpu_channel_hw_state *state); bool (*check_ctxsw_timeout)(struct tsg_gk20a *tsg, bool *verbose, u32 *ms); int (*force_reset)(struct channel_gk20a *ch, diff --git a/drivers/gpu/nvgpu/include/nvgpu/tsg.h b/drivers/gpu/nvgpu/include/nvgpu/tsg.h index 33c4fc8a3..da5d81d15 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/tsg.h +++ b/drivers/gpu/nvgpu/include/nvgpu/tsg.h @@ -33,6 +33,7 @@ struct gk20a; struct channel_gk20a; struct nvgpu_gr_ctx; +struct nvgpu_channel_hw_state; struct nvgpu_tsg_sm_error_state { u32 hww_global_esr; @@ -93,9 +94,16 @@ void nvgpu_tsg_cleanup_sw(struct gk20a *g); struct tsg_gk20a *tsg_gk20a_from_ch(struct channel_gk20a *ch); void nvgpu_tsg_disable(struct tsg_gk20a *tsg); -int gk20a_tsg_bind_channel(struct tsg_gk20a *tsg, +int nvgpu_tsg_bind_channel(struct tsg_gk20a *tsg, struct channel_gk20a *ch); -int gk20a_tsg_unbind_channel(struct channel_gk20a *ch); +int nvgpu_tsg_unbind_channel(struct tsg_gk20a *tsg, struct channel_gk20a *ch); +int nvgpu_tsg_unbind_channel_common(struct tsg_gk20a *tsg, + struct channel_gk20a *ch); +int nvgpu_tsg_unbind_channel_check_hw_state(struct tsg_gk20a *tsg, + struct channel_gk20a *ch); +void nvgpu_tsg_unbind_channel_check_ctx_reload(struct tsg_gk20a *tsg, + struct channel_gk20a *ch, + struct nvgpu_channel_hw_state *hw_state); int nvgpu_tsg_force_reset_ch(struct channel_gk20a *ch, u32 err_code, bool verbose); void nvgpu_tsg_recover(struct gk20a *g, struct tsg_gk20a *tsg, diff --git a/drivers/gpu/nvgpu/os/linux/cde.c b/drivers/gpu/nvgpu/os/linux/cde.c index fa85e4d4a..1bb0f509e 100644 --- a/drivers/gpu/nvgpu/os/linux/cde.c +++ b/drivers/gpu/nvgpu/os/linux/cde.c @@ -1357,7 +1357,7 @@ static int gk20a_cde_load(struct gk20a_cde_ctx *cde_ctx) goto err_commit_va; } - err = gk20a_tsg_bind_channel(tsg, ch); + err = nvgpu_tsg_bind_channel(tsg, ch); if (err) { nvgpu_err(g, "cde: unable to bind to tsg"); goto err_setup_bind; diff --git a/drivers/gpu/nvgpu/os/linux/ioctl_tsg.c b/drivers/gpu/nvgpu/os/linux/ioctl_tsg.c index 95f82d236..cc62119be 100644 --- a/drivers/gpu/nvgpu/os/linux/ioctl_tsg.c +++ b/drivers/gpu/nvgpu/os/linux/ioctl_tsg.c @@ -41,7 +41,7 @@ struct tsg_private { struct tsg_gk20a *tsg; }; -static int gk20a_tsg_bind_channel_fd(struct tsg_gk20a *tsg, int ch_fd) +static int nvgpu_tsg_bind_channel_fd(struct tsg_gk20a *tsg, int ch_fd) { struct channel_gk20a *ch; int err; @@ -50,7 +50,7 @@ static int gk20a_tsg_bind_channel_fd(struct tsg_gk20a *tsg, int ch_fd) if (!ch) return -EINVAL; - err = ch->g->ops.fifo.tsg_bind_channel(tsg, ch); + err = nvgpu_tsg_bind_channel(tsg, ch); gk20a_channel_put(ch); return err; @@ -113,7 +113,7 @@ static int gk20a_tsg_ioctl_bind_channel_ex(struct gk20a *g, if (ch->subctx_id > CHANNEL_INFO_VEID0) ch->runqueue_sel = 1; - err = ch->g->ops.fifo.tsg_bind_channel(tsg, ch); + err = nvgpu_tsg_bind_channel(tsg, ch); ch_put: gk20a_channel_put(ch); idle: @@ -123,21 +123,22 @@ mutex_release: return err; } -static int gk20a_tsg_unbind_channel_fd(struct tsg_gk20a *tsg, int ch_fd) +static int nvgpu_tsg_unbind_channel_fd(struct tsg_gk20a *tsg, int ch_fd) { struct channel_gk20a *ch; int err = 0; ch = gk20a_get_channel_from_file(ch_fd); - if (!ch) + if (!ch) { return -EINVAL; + } - if (ch->tsgid != tsg->tsgid) { + if (tsg != tsg_gk20a_from_ch(ch)) { err = -EINVAL; goto out; } - err = gk20a_tsg_unbind_channel(ch); + err = nvgpu_tsg_unbind_channel(tsg, ch); /* * Mark the channel unserviceable since channel unbound from TSG @@ -644,7 +645,7 @@ long nvgpu_ioctl_tsg_dev_ioctl(struct file *filp, unsigned int cmd, err = -EINVAL; break; } - err = gk20a_tsg_bind_channel_fd(tsg, ch_fd); + err = nvgpu_tsg_bind_channel_fd(tsg, ch_fd); break; } @@ -669,7 +670,7 @@ long nvgpu_ioctl_tsg_dev_ioctl(struct file *filp, unsigned int cmd, "failed to host gk20a for ioctl cmd: 0x%x", cmd); break; } - err = gk20a_tsg_unbind_channel_fd(tsg, ch_fd); + err = nvgpu_tsg_unbind_channel_fd(tsg, ch_fd); gk20a_idle(g); break; } diff --git a/drivers/gpu/nvgpu/tu104/hal_tu104.c b/drivers/gpu/nvgpu/tu104/hal_tu104.c index d88b1f34c..5e432294e 100644 --- a/drivers/gpu/nvgpu/tu104/hal_tu104.c +++ b/drivers/gpu/nvgpu/tu104/hal_tu104.c @@ -931,9 +931,6 @@ static const struct gpu_ops tu104_ops = { .default_timeslice_us = gk20a_fifo_default_timeslice_us, .preempt_channel = gv11b_fifo_preempt_channel, .preempt_tsg = gv11b_fifo_preempt_tsg, - .tsg_verify_channel_status = gk20a_fifo_tsg_unbind_channel_verify_status, - .tsg_verify_status_ctx_reload = gm20b_fifo_tsg_verify_status_ctx_reload, - .tsg_verify_status_faulted = gv11b_fifo_tsg_verify_status_faulted, .tsg_set_timeslice = gk20a_fifo_tsg_set_timeslice, .init_pbdma_info = gk20a_fifo_init_pbdma_info, .dump_channel_status_ramfc = gv11b_dump_channel_status_ramfc, @@ -945,8 +942,6 @@ static const struct gpu_ops tu104_ops = { .init_eng_method_buffers = gv11b_fifo_init_eng_method_buffers, .deinit_eng_method_buffers = gv11b_fifo_deinit_eng_method_buffers, - .tsg_bind_channel = gk20a_tsg_bind_channel, - .tsg_unbind_channel = gk20a_fifo_tsg_unbind_channel, .post_event_id = gk20a_tsg_event_id_post_event, .setup_sw = nvgpu_fifo_setup_sw, .cleanup_sw = nvgpu_fifo_cleanup_sw, @@ -1081,6 +1076,14 @@ static const struct gpu_ops tu104_ops = { .tsg = { .enable = gv11b_tsg_enable, .disable = nvgpu_tsg_disable, + .bind_channel = NULL, + .unbind_channel = NULL, + .unbind_channel_check_hw_state = + nvgpu_tsg_unbind_channel_check_hw_state, + .unbind_channel_check_ctx_reload = + nvgpu_tsg_unbind_channel_check_ctx_reload, + .unbind_channel_check_eng_faulted = + gv11b_tsg_unbind_channel_check_eng_faulted, .check_ctxsw_timeout = nvgpu_tsg_check_ctxsw_timeout, .force_reset = nvgpu_tsg_force_reset_ch, },