diff --git a/drivers/gpu/nvgpu/Makefile b/drivers/gpu/nvgpu/Makefile index 0378ec263..ee3db2abc 100644 --- a/drivers/gpu/nvgpu/Makefile +++ b/drivers/gpu/nvgpu/Makefile @@ -234,6 +234,8 @@ nvgpu-y += \ hal/fifo/ramfc_gp10b.o \ hal/fifo/ramfc_gv11b.o \ hal/fifo/ramfc_tu104.o \ + hal/fifo/tsg_gk20a.o \ + hal/fifo/tsg_gv11b.o \ hal/fifo/userd_gk20a.o \ hal/fifo/userd_gv11b.o \ hal/fifo/fifo_intr_gk20a.o \ diff --git a/drivers/gpu/nvgpu/Makefile.sources b/drivers/gpu/nvgpu/Makefile.sources index 142db38a5..eb98d1f25 100644 --- a/drivers/gpu/nvgpu/Makefile.sources +++ b/drivers/gpu/nvgpu/Makefile.sources @@ -365,6 +365,8 @@ srcs += common/sim.c \ hal/fifo/ramfc_gp10b.c \ hal/fifo/ramfc_gv11b.c \ hal/fifo/ramfc_tu104.c \ + hal/fifo/tsg_gk20a.c \ + hal/fifo/tsg_gv11b.c \ hal/fifo/userd_gk20a.c \ hal/fifo/userd_gv11b.c \ hal/fifo/fifo_intr_gk20a.c \ diff --git a/drivers/gpu/nvgpu/common/fifo/channel.c b/drivers/gpu/nvgpu/common/fifo/channel.c index 2f7d4f171..23b5d1e6c 100644 --- a/drivers/gpu/nvgpu/common/fifo/channel.c +++ b/drivers/gpu/nvgpu/common/fifo/channel.c @@ -173,7 +173,7 @@ int gk20a_enable_channel_tsg(struct gk20a *g, struct channel_gk20a *ch) tsg = tsg_gk20a_from_ch(ch); if (tsg != NULL) { - g->ops.fifo.enable_tsg(tsg); + g->ops.tsg.enable(tsg); return 0; } else { return -EINVAL; @@ -186,7 +186,7 @@ int gk20a_disable_channel_tsg(struct gk20a *g, struct channel_gk20a *ch) tsg = tsg_gk20a_from_ch(ch); if (tsg != NULL) { - g->ops.fifo.disable_tsg(tsg); + g->ops.tsg.disable(tsg); return 0; } else { return -EINVAL; diff --git a/drivers/gpu/nvgpu/common/fifo/tsg.c b/drivers/gpu/nvgpu/common/fifo/tsg.c index b92331fce..b9e3d0f29 100644 --- a/drivers/gpu/nvgpu/common/fifo/tsg.c +++ b/drivers/gpu/nvgpu/common/fifo/tsg.c @@ -35,48 +35,7 @@ #include "gk20a/gr_gk20a.h" -int gk20a_enable_tsg(struct tsg_gk20a *tsg) -{ - struct gk20a *g = tsg->g; - struct channel_gk20a *ch; - - gk20a_tsg_disable_sched(g, tsg); - - /* - * Due to h/w bug that exists in Maxwell and Pascal, - * we first need to enable all channels with NEXT and CTX_RELOAD set, - * and then rest of the channels should be enabled - */ - nvgpu_rwsem_down_read(&tsg->ch_list_lock); - nvgpu_list_for_each_entry(ch, &tsg->ch_list, channel_gk20a, ch_entry) { - struct nvgpu_channel_hw_state hw_state; - - g->ops.channel.read_state(g, ch, &hw_state); - - if (hw_state.next || hw_state.ctx_reload) { - g->ops.channel.enable(ch); - } - } - - nvgpu_list_for_each_entry(ch, &tsg->ch_list, channel_gk20a, ch_entry) { - struct nvgpu_channel_hw_state hw_state; - - g->ops.channel.read_state(g, ch, &hw_state); - - if (hw_state.next || hw_state.ctx_reload) { - continue; - } - - g->ops.channel.enable(ch); - } - nvgpu_rwsem_up_read(&tsg->ch_list_lock); - - gk20a_tsg_enable_sched(g, tsg); - - return 0; -} - -void gk20a_disable_tsg(struct tsg_gk20a *tsg) +void nvgpu_tsg_disable(struct tsg_gk20a *tsg) { struct gk20a *g = tsg->g; struct channel_gk20a *ch; @@ -192,7 +151,7 @@ void nvgpu_tsg_recover(struct gk20a *g, struct tsg_gk20a *tsg, nvgpu_mutex_acquire(&g->dbg_sessions_lock); /* disable tsg so that it does not get scheduled again */ - g->ops.fifo.disable_tsg(tsg); + g->ops.tsg.disable(tsg); /* * On hitting engine reset, h/w drops the ctxsw_status to INVALID in diff --git a/drivers/gpu/nvgpu/common/vgpu/fifo/fifo_vgpu.h b/drivers/gpu/nvgpu/common/vgpu/fifo/fifo_vgpu.h index c7c3d08c5..efaa51184 100644 --- a/drivers/gpu/nvgpu/common/vgpu/fifo/fifo_vgpu.h +++ b/drivers/gpu/nvgpu/common/vgpu/fifo/fifo_vgpu.h @@ -53,7 +53,7 @@ int vgpu_tsg_bind_channel(struct tsg_gk20a *tsg, struct channel_gk20a *ch); int vgpu_tsg_unbind_channel(struct channel_gk20a *ch); int vgpu_tsg_set_timeslice(struct tsg_gk20a *tsg, u32 timeslice); -int vgpu_enable_tsg(struct tsg_gk20a *tsg); +void vgpu_tsg_enable(struct tsg_gk20a *tsg); int vgpu_set_sm_exception_type_mask(struct channel_gk20a *ch, u32 mask); void vgpu_channel_free_ctx_header(struct channel_gk20a *c); #endif /* NVGPU_FIFO_VGPU_H */ diff --git a/drivers/gpu/nvgpu/common/vgpu/gp10b/vgpu_hal_gp10b.c b/drivers/gpu/nvgpu/common/vgpu/gp10b/vgpu_hal_gp10b.c index 39229eafc..0d2a43185 100644 --- a/drivers/gpu/nvgpu/common/vgpu/gp10b/vgpu_hal_gp10b.c +++ b/drivers/gpu/nvgpu/common/vgpu/gp10b/vgpu_hal_gp10b.c @@ -407,8 +407,6 @@ static const struct gpu_ops vgpu_gp10b_ops = { .default_timeslice_us = vgpu_fifo_default_timeslice_us, .preempt_channel = vgpu_fifo_preempt_channel, .preempt_tsg = vgpu_fifo_preempt_tsg, - .enable_tsg = vgpu_enable_tsg, - .disable_tsg = gk20a_disable_tsg, .tsg_verify_channel_status = NULL, .tsg_verify_status_ctx_reload = NULL, .trigger_mmu_fault = NULL, @@ -523,6 +521,8 @@ static const struct gpu_ops vgpu_gp10b_ops = { .count = vgpu_channel_count, }, .tsg = { + .enable = vgpu_tsg_enable, + .disable = nvgpu_tsg_disable, .check_ctxsw_timeout = nvgpu_tsg_check_ctxsw_timeout, }, .netlist = { diff --git a/drivers/gpu/nvgpu/common/vgpu/gv11b/vgpu_hal_gv11b.c b/drivers/gpu/nvgpu/common/vgpu/gv11b/vgpu_hal_gv11b.c index 5e6a941e5..3dd9ca4a6 100644 --- a/drivers/gpu/nvgpu/common/vgpu/gv11b/vgpu_hal_gv11b.c +++ b/drivers/gpu/nvgpu/common/vgpu/gv11b/vgpu_hal_gv11b.c @@ -26,6 +26,7 @@ #include "hal/fifo/pbdma_gm20b.h" #include "hal/fifo/pbdma_gp10b.h" #include "hal/fifo/pbdma_gv11b.h" +#include "hal/fifo/tsg_gv11b.h" #include "hal/fifo/userd_gk20a.h" #include "hal/fifo/userd_gv11b.h" #include "hal/fifo/fifo_intr_gv11b.h" @@ -477,8 +478,6 @@ static const struct gpu_ops vgpu_gv11b_ops = { .default_timeslice_us = vgpu_fifo_default_timeslice_us, .preempt_channel = vgpu_fifo_preempt_channel, .preempt_tsg = vgpu_fifo_preempt_tsg, - .enable_tsg = vgpu_gv11b_enable_tsg, - .disable_tsg = gk20a_disable_tsg, .tsg_verify_channel_status = NULL, .tsg_verify_status_ctx_reload = NULL, /* TODO: implement it for CE fault */ @@ -602,6 +601,8 @@ static const struct gpu_ops vgpu_gv11b_ops = { .count = vgpu_channel_count, }, .tsg = { + .enable = gv11b_tsg_enable, + .disable = nvgpu_tsg_disable, .check_ctxsw_timeout = nvgpu_tsg_check_ctxsw_timeout, }, .netlist = { diff --git a/drivers/gpu/nvgpu/common/vgpu/gv11b/vgpu_tsg_gv11b.c b/drivers/gpu/nvgpu/common/vgpu/gv11b/vgpu_tsg_gv11b.c index c75a0ab5a..9c8ea4ade 100644 --- a/drivers/gpu/nvgpu/common/vgpu/gv11b/vgpu_tsg_gv11b.c +++ b/drivers/gpu/nvgpu/common/vgpu/gv11b/vgpu_tsg_gv11b.c @@ -60,24 +60,3 @@ int vgpu_gv11b_tsg_bind_channel(struct tsg_gk20a *tsg, return err; } - -int vgpu_gv11b_enable_tsg(struct tsg_gk20a *tsg) -{ - struct gk20a *g = tsg->g; - struct channel_gk20a *ch; - struct channel_gk20a *last_ch = NULL; - - nvgpu_rwsem_down_read(&tsg->ch_list_lock); - nvgpu_list_for_each_entry(ch, &tsg->ch_list, channel_gk20a, ch_entry) { - g->ops.channel.enable(ch); - last_ch = ch; - } - nvgpu_rwsem_up_read(&tsg->ch_list_lock); - - if (last_ch) { - g->ops.fifo.ring_channel_doorbell(last_ch); - } - - return 0; -} - diff --git a/drivers/gpu/nvgpu/common/vgpu/gv11b/vgpu_tsg_gv11b.h b/drivers/gpu/nvgpu/common/vgpu/gv11b/vgpu_tsg_gv11b.h index 288e49be7..7e1b24e29 100644 --- a/drivers/gpu/nvgpu/common/vgpu/gv11b/vgpu_tsg_gv11b.h +++ b/drivers/gpu/nvgpu/common/vgpu/gv11b/vgpu_tsg_gv11b.h @@ -25,5 +25,5 @@ int vgpu_gv11b_tsg_bind_channel(struct tsg_gk20a *tsg, struct channel_gk20a *ch); -int vgpu_gv11b_enable_tsg(struct tsg_gk20a *tsg); + #endif /* NVGPU_VGPU_TSG_GV11B_H */ diff --git a/drivers/gpu/nvgpu/common/vgpu/tsg_vgpu.c b/drivers/gpu/nvgpu/common/vgpu/tsg_vgpu.c index b3df95169..c2e8ddc8e 100644 --- a/drivers/gpu/nvgpu/common/vgpu/tsg_vgpu.c +++ b/drivers/gpu/nvgpu/common/vgpu/tsg_vgpu.c @@ -75,7 +75,7 @@ void vgpu_tsg_release(struct tsg_gk20a *tsg) } } -int vgpu_enable_tsg(struct tsg_gk20a *tsg) +void vgpu_tsg_enable(struct tsg_gk20a *tsg) { struct gk20a *g = tsg->g; struct channel_gk20a *ch; @@ -85,8 +85,6 @@ int vgpu_enable_tsg(struct tsg_gk20a *tsg) g->ops.channel.enable(ch); } nvgpu_rwsem_up_read(&tsg->ch_list_lock); - - return 0; } int vgpu_tsg_bind_channel(struct tsg_gk20a *tsg, diff --git a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c index 14a35b48b..4e9e2ff64 100644 --- a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c @@ -503,7 +503,7 @@ void gk20a_fifo_abort_tsg(struct gk20a *g, struct tsg_gk20a *tsg, bool preempt) WARN_ON(tsg->abortable == false); - g->ops.fifo.disable_tsg(tsg); + g->ops.tsg.disable(tsg); if (preempt) { g->ops.fifo.preempt_tsg(g, tsg); @@ -732,7 +732,7 @@ static bool gk20a_fifo_handle_mmu_fault_locked( */ if (tsg != NULL) { if (deferred_reset_pending) { - gk20a_disable_tsg(tsg); + g->ops.tsg.disable(tsg); } else { if (!fake_fault) { nvgpu_tsg_set_ctx_mmu_error(g, tsg); @@ -1061,7 +1061,7 @@ int gk20a_fifo_tsg_unbind_channel(struct channel_gk20a *ch) nvgpu_rwsem_up_write(&tsg->ch_list_lock); /* Disable TSG and examine status before unbinding channel */ - g->ops.fifo.disable_tsg(tsg); + g->ops.tsg.disable(tsg); err = g->ops.fifo.preempt_tsg(g, tsg); if (err != 0) { @@ -1094,7 +1094,7 @@ int gk20a_fifo_tsg_unbind_channel(struct channel_gk20a *ch) * time out, but we keep that to ensure TSG is kicked out */ if (!tsg_timedout) { - g->ops.fifo.enable_tsg(tsg); + g->ops.tsg.enable(tsg); } if (ch->g->ops.fifo.ch_abort_clean_up != NULL) { @@ -1105,7 +1105,7 @@ int gk20a_fifo_tsg_unbind_channel(struct channel_gk20a *ch) fail_enable_tsg: if (!tsg_timedout) { - g->ops.fifo.enable_tsg(tsg); + g->ops.tsg.enable(tsg); } return err; } diff --git a/drivers/gpu/nvgpu/gm20b/hal_gm20b.c b/drivers/gpu/nvgpu/gm20b/hal_gm20b.c index 667faa894..cd415849f 100644 --- a/drivers/gpu/nvgpu/gm20b/hal_gm20b.c +++ b/drivers/gpu/nvgpu/gm20b/hal_gm20b.c @@ -55,6 +55,7 @@ #include "hal/fifo/engine_status_gm20b.h" #include "hal/fifo/pbdma_status_gm20b.h" #include "hal/fifo/ramfc_gk20a.h" +#include "hal/fifo/tsg_gk20a.h" #include "hal/fifo/userd_gk20a.h" #include "hal/fifo/fifo_intr_gk20a.h" #include "hal/fifo/ctxsw_timeout_gk20a.h" @@ -584,8 +585,6 @@ static const struct gpu_ops gm20b_ops = { .default_timeslice_us = gk20a_fifo_default_timeslice_us, .preempt_channel = gk20a_fifo_preempt_channel, .preempt_tsg = gk20a_fifo_preempt_tsg, - .enable_tsg = gk20a_enable_tsg, - .disable_tsg = gk20a_disable_tsg, .tsg_verify_channel_status = gk20a_fifo_tsg_unbind_channel_verify_status, .tsg_verify_status_ctx_reload = gm20b_fifo_tsg_verify_status_ctx_reload, .trigger_mmu_fault = gm20b_fifo_trigger_mmu_fault, @@ -711,6 +710,8 @@ static const struct gpu_ops gm20b_ops = { .force_ctx_reload = gm20b_channel_force_ctx_reload, }, .tsg = { + .enable = gk20a_tsg_enable, + .disable = nvgpu_tsg_disable, .check_ctxsw_timeout = nvgpu_tsg_check_ctxsw_timeout, }, .netlist = { diff --git a/drivers/gpu/nvgpu/gp10b/hal_gp10b.c b/drivers/gpu/nvgpu/gp10b/hal_gp10b.c index a50ef8528..fd20c977d 100644 --- a/drivers/gpu/nvgpu/gp10b/hal_gp10b.c +++ b/drivers/gpu/nvgpu/gp10b/hal_gp10b.c @@ -65,6 +65,7 @@ #include "hal/fifo/pbdma_status_gm20b.h" #include "hal/fifo/ramfc_gk20a.h" #include "hal/fifo/ramfc_gp10b.h" +#include "hal/fifo/tsg_gk20a.h" #include "hal/fifo/userd_gk20a.h" #include "hal/fifo/fifo_intr_gk20a.h" #include "hal/fifo/ctxsw_timeout_gk20a.h" @@ -665,8 +666,6 @@ static const struct gpu_ops gp10b_ops = { .default_timeslice_us = gk20a_fifo_default_timeslice_us, .preempt_channel = gk20a_fifo_preempt_channel, .preempt_tsg = gk20a_fifo_preempt_tsg, - .enable_tsg = gk20a_enable_tsg, - .disable_tsg = gk20a_disable_tsg, .tsg_verify_channel_status = gk20a_fifo_tsg_unbind_channel_verify_status, .tsg_verify_status_ctx_reload = gm20b_fifo_tsg_verify_status_ctx_reload, .trigger_mmu_fault = gm20b_fifo_trigger_mmu_fault, @@ -795,6 +794,8 @@ static const struct gpu_ops gp10b_ops = { .set_syncpt = nvgpu_channel_set_syncpt, }, .tsg = { + .enable = gk20a_tsg_enable, + .disable = nvgpu_tsg_disable, .check_ctxsw_timeout = nvgpu_tsg_check_ctxsw_timeout, }, .netlist = { diff --git a/drivers/gpu/nvgpu/gv100/hal_gv100.c b/drivers/gpu/nvgpu/gv100/hal_gv100.c index e1907d320..08e2cc5e6 100644 --- a/drivers/gpu/nvgpu/gv100/hal_gv100.c +++ b/drivers/gpu/nvgpu/gv100/hal_gv100.c @@ -54,6 +54,7 @@ #include "hal/fifo/pbdma_status_gm20b.h" #include "hal/fifo/ramfc_gp10b.h" #include "hal/fifo/ramfc_gv11b.h" +#include "hal/fifo/tsg_gv11b.h" #include "hal/fifo/userd_gk20a.h" #include "hal/fifo/userd_gv11b.h" #include "hal/fifo/fifo_intr_gk20a.h" @@ -839,8 +840,6 @@ static const struct gpu_ops gv100_ops = { .default_timeslice_us = gk20a_fifo_default_timeslice_us, .preempt_channel = gv11b_fifo_preempt_channel, .preempt_tsg = gv11b_fifo_preempt_tsg, - .enable_tsg = gv11b_fifo_enable_tsg, - .disable_tsg = gk20a_disable_tsg, .tsg_verify_channel_status = gk20a_fifo_tsg_unbind_channel_verify_status, .tsg_verify_status_ctx_reload = gm20b_fifo_tsg_verify_status_ctx_reload, .tsg_verify_status_faulted = gv11b_fifo_tsg_verify_status_faulted, @@ -975,6 +974,8 @@ static const struct gpu_ops gv100_ops = { .reset_faulted = gv11b_channel_reset_faulted, }, .tsg = { + .enable = gv11b_tsg_enable, + .disable = nvgpu_tsg_disable, .check_ctxsw_timeout = nvgpu_tsg_check_ctxsw_timeout, }, .netlist = { diff --git a/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c b/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c index 4e5cef1cb..3cf580239 100644 --- a/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c +++ b/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c @@ -524,27 +524,6 @@ int gv11b_fifo_preempt_channel(struct gk20a *g, struct channel_gk20a *ch) return g->ops.fifo.preempt_tsg(g, tsg); } -/* TSG enable sequence applicable for Volta and onwards */ -int gv11b_fifo_enable_tsg(struct tsg_gk20a *tsg) -{ - struct gk20a *g = tsg->g; - struct channel_gk20a *ch; - struct channel_gk20a *last_ch = NULL; - - nvgpu_rwsem_down_read(&tsg->ch_list_lock); - nvgpu_list_for_each_entry(ch, &tsg->ch_list, channel_gk20a, ch_entry) { - g->ops.channel.enable(ch); - last_ch = ch; - } - nvgpu_rwsem_up_read(&tsg->ch_list_lock); - - if (last_ch != NULL) { - g->ops.fifo.ring_channel_doorbell(last_ch); - } - - return 0; -} - int gv11b_fifo_preempt_tsg(struct gk20a *g, struct tsg_gk20a *tsg) { struct fifo_gk20a *f = &g->fifo; @@ -679,7 +658,7 @@ static void gv11b_fifo_locked_abort_runlist_active_tsgs(struct gk20a *g, } nvgpu_log(g, gpu_dbg_info, "abort tsg id %lu", tsgid); - gk20a_disable_tsg(tsg); + g->ops.tsg.disable(tsg); /* assume all pbdma and eng faulted are set */ nvgpu_log(g, gpu_dbg_info, "reset pbdma and eng faulted"); @@ -848,7 +827,7 @@ void gv11b_fifo_teardown_ch_tsg(struct gk20a *g, u32 act_eng_bitmask, } if (tsg != NULL) { - gk20a_disable_tsg(tsg); + g->ops.tsg.disable(tsg); } /* @@ -931,7 +910,7 @@ void gv11b_fifo_teardown_ch_tsg(struct gk20a *g, u32 act_eng_bitmask, #endif if (tsg != NULL) { if (deferred_reset_pending) { - gk20a_disable_tsg(tsg); + g->ops.tsg.disable(tsg); } else { if (rc_type == RC_TYPE_MMU_FAULT) { nvgpu_tsg_set_ctx_mmu_error(g, tsg); diff --git a/drivers/gpu/nvgpu/gv11b/fifo_gv11b.h b/drivers/gpu/nvgpu/gv11b/fifo_gv11b.h index 323e9651c..69f7a68b1 100644 --- a/drivers/gpu/nvgpu/gv11b/fifo_gv11b.h +++ b/drivers/gpu/nvgpu/gv11b/fifo_gv11b.h @@ -54,7 +54,6 @@ int gv11b_fifo_is_preempt_pending(struct gk20a *g, u32 id, unsigned int id_type); int gv11b_fifo_preempt_channel(struct gk20a *g, struct channel_gk20a *ch); int gv11b_fifo_preempt_tsg(struct gk20a *g, struct tsg_gk20a *tsg); -int gv11b_fifo_enable_tsg(struct tsg_gk20a *tsg); void gv11b_fifo_teardown_ch_tsg(struct gk20a *g, u32 act_eng_bitmask, u32 id, unsigned int id_type, unsigned int rc_type, struct mmu_fault_info *mmfault); diff --git a/drivers/gpu/nvgpu/gv11b/hal_gv11b.c b/drivers/gpu/nvgpu/gv11b/hal_gv11b.c index 6a95a8048..276b8593a 100644 --- a/drivers/gpu/nvgpu/gv11b/hal_gv11b.c +++ b/drivers/gpu/nvgpu/gv11b/hal_gv11b.c @@ -56,6 +56,7 @@ #include "hal/fifo/engines_gv11b.h" #include "hal/fifo/ramfc_gp10b.h" #include "hal/fifo/ramfc_gv11b.h" +#include "hal/fifo/tsg_gv11b.h" #include "hal/fifo/userd_gk20a.h" #include "hal/fifo/userd_gv11b.h" #include "hal/fifo/fifo_intr_gk20a.h" @@ -794,8 +795,6 @@ static const struct gpu_ops gv11b_ops = { .default_timeslice_us = gk20a_fifo_default_timeslice_us, .preempt_channel = gv11b_fifo_preempt_channel, .preempt_tsg = gv11b_fifo_preempt_tsg, - .enable_tsg = gv11b_fifo_enable_tsg, - .disable_tsg = gk20a_disable_tsg, .tsg_verify_channel_status = gk20a_fifo_tsg_unbind_channel_verify_status, .tsg_verify_status_ctx_reload = gm20b_fifo_tsg_verify_status_ctx_reload, .tsg_verify_status_faulted = gv11b_fifo_tsg_verify_status_faulted, @@ -932,6 +931,8 @@ static const struct gpu_ops gv11b_ops = { .reset_faulted = gv11b_channel_reset_faulted, }, .tsg = { + .enable = gv11b_tsg_enable, + .disable = nvgpu_tsg_disable, .check_ctxsw_timeout = nvgpu_tsg_check_ctxsw_timeout, }, .netlist = { diff --git a/drivers/gpu/nvgpu/hal/fifo/tsg_gk20a.c b/drivers/gpu/nvgpu/hal/fifo/tsg_gk20a.c new file mode 100644 index 000000000..b39cc0ec4 --- /dev/null +++ b/drivers/gpu/nvgpu/hal/fifo/tsg_gk20a.c @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2011-2019, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include +#include + +#include "hal/fifo/tsg_gk20a.h" + +void gk20a_tsg_enable(struct tsg_gk20a *tsg) +{ + struct gk20a *g = tsg->g; + struct channel_gk20a *ch; + + gk20a_tsg_disable_sched(g, tsg); + + /* + * Due to h/w bug that exists in Maxwell and Pascal, + * we first need to enable all channels with NEXT and CTX_RELOAD set, + * and then rest of the channels should be enabled + */ + nvgpu_rwsem_down_read(&tsg->ch_list_lock); + nvgpu_list_for_each_entry(ch, &tsg->ch_list, channel_gk20a, ch_entry) { + struct nvgpu_channel_hw_state hw_state; + + g->ops.channel.read_state(g, ch, &hw_state); + + if (hw_state.next || hw_state.ctx_reload) { + g->ops.channel.enable(ch); + } + } + + nvgpu_list_for_each_entry(ch, &tsg->ch_list, channel_gk20a, ch_entry) { + struct nvgpu_channel_hw_state hw_state; + + g->ops.channel.read_state(g, ch, &hw_state); + + if (hw_state.next || hw_state.ctx_reload) { + continue; + } + + g->ops.channel.enable(ch); + } + nvgpu_rwsem_up_read(&tsg->ch_list_lock); + + gk20a_tsg_enable_sched(g, tsg); +} diff --git a/drivers/gpu/nvgpu/hal/fifo/tsg_gk20a.h b/drivers/gpu/nvgpu/hal/fifo/tsg_gk20a.h new file mode 100644 index 000000000..db1d92312 --- /dev/null +++ b/drivers/gpu/nvgpu/hal/fifo/tsg_gk20a.h @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2011-2019, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef NVGPU_TSG_GK20A_H +#define NVGPU_TSG_GK20A_H + +struct tsg_gk20a; + +void gk20a_tsg_enable(struct tsg_gk20a *tsg); + +#endif /* NVGPU_TSG_GK20A_H */ diff --git a/drivers/gpu/nvgpu/hal/fifo/tsg_gv11b.c b/drivers/gpu/nvgpu/hal/fifo/tsg_gv11b.c new file mode 100644 index 000000000..d02d9a835 --- /dev/null +++ b/drivers/gpu/nvgpu/hal/fifo/tsg_gv11b.c @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2016-2019, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include + +#include "hal/fifo/tsg_gv11b.h" + +/* TSG enable sequence applicable for Volta and onwards */ +void gv11b_tsg_enable(struct tsg_gk20a *tsg) +{ + struct gk20a *g = tsg->g; + struct channel_gk20a *ch; + struct channel_gk20a *last_ch = NULL; + + nvgpu_rwsem_down_read(&tsg->ch_list_lock); + nvgpu_list_for_each_entry(ch, &tsg->ch_list, channel_gk20a, ch_entry) { + g->ops.channel.enable(ch); + last_ch = ch; + } + nvgpu_rwsem_up_read(&tsg->ch_list_lock); + + if (last_ch != NULL) { + g->ops.fifo.ring_channel_doorbell(last_ch); + } +} diff --git a/drivers/gpu/nvgpu/hal/fifo/tsg_gv11b.h b/drivers/gpu/nvgpu/hal/fifo/tsg_gv11b.h new file mode 100644 index 000000000..1bc7e42f2 --- /dev/null +++ b/drivers/gpu/nvgpu/hal/fifo/tsg_gv11b.h @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2016-2019, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef NVGPU_TSG_GV11B_H +#define NVGPU_TSG_GV11B_H + +struct tsg_gk20a; + +void gv11b_tsg_enable(struct tsg_gk20a *tsg); + +#endif /* NVGPU_TSG_GV11B_H */ diff --git a/drivers/gpu/nvgpu/include/nvgpu/gk20a.h b/drivers/gpu/nvgpu/include/nvgpu/gk20a.h index dae30729a..46d1070eb 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/gk20a.h +++ b/drivers/gpu/nvgpu/include/nvgpu/gk20a.h @@ -902,8 +902,6 @@ struct gpu_ops { void (*free_inst)(struct gk20a *g, struct channel_gk20a *ch); int (*preempt_channel)(struct gk20a *g, struct channel_gk20a *ch); int (*preempt_tsg)(struct gk20a *g, struct tsg_gk20a *tsg); - int (*enable_tsg)(struct tsg_gk20a *tsg); - void (*disable_tsg)(struct tsg_gk20a *tsg); int (*tsg_verify_channel_status)(struct channel_gk20a *ch); void (*tsg_verify_status_ctx_reload)(struct channel_gk20a *ch); void (*tsg_verify_status_faulted)(struct channel_gk20a *ch); @@ -1086,8 +1084,10 @@ struct gpu_ops { int (*set_syncpt)(struct channel_gk20a *ch); } channel; struct { + void (*enable)(struct tsg_gk20a *tsg); + void (*disable)(struct tsg_gk20a *tsg); bool (*check_ctxsw_timeout)(struct tsg_gk20a *tsg, - bool *verbose, u32 *ms); + bool *verbose, u32 *ms); } tsg; struct { void (*read_engine_status_info) (struct gk20a *g, diff --git a/drivers/gpu/nvgpu/include/nvgpu/tsg.h b/drivers/gpu/nvgpu/include/nvgpu/tsg.h index 191839607..90aca9887 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/tsg.h +++ b/drivers/gpu/nvgpu/include/nvgpu/tsg.h @@ -19,8 +19,8 @@ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER * DEALINGS IN THE SOFTWARE. */ -#ifndef TSG_GK20A_H -#define TSG_GK20A_H +#ifndef NVGPU_TSG_H +#define NVGPU_TSG_H #include #include @@ -92,8 +92,7 @@ void nvgpu_tsg_cleanup_sw(struct gk20a *g); struct tsg_gk20a *tsg_gk20a_from_ch(struct channel_gk20a *ch); -int gk20a_enable_tsg(struct tsg_gk20a *tsg); -void gk20a_disable_tsg(struct tsg_gk20a *tsg); +void nvgpu_tsg_disable(struct tsg_gk20a *tsg); int gk20a_tsg_bind_channel(struct tsg_gk20a *tsg, struct channel_gk20a *ch); int gk20a_tsg_unbind_channel(struct channel_gk20a *ch); @@ -150,4 +149,4 @@ void nvgpu_tsg_set_error_notifier(struct gk20a *g, struct tsg_gk20a *tsg, bool nvgpu_tsg_ctxsw_timeout_debug_dump_state(struct tsg_gk20a *tsg); void nvgpu_tsg_set_ctxsw_timeout_accumulated_ms(struct tsg_gk20a *tsg, u32 ms); -#endif /* TSG_GK20A_H */ +#endif /* NVGPU_TSG_H */ diff --git a/drivers/gpu/nvgpu/os/linux/ioctl_tsg.c b/drivers/gpu/nvgpu/os/linux/ioctl_tsg.c index 7b65b6d89..95f82d236 100644 --- a/drivers/gpu/nvgpu/os/linux/ioctl_tsg.c +++ b/drivers/gpu/nvgpu/os/linux/ioctl_tsg.c @@ -682,7 +682,7 @@ long nvgpu_ioctl_tsg_dev_ioctl(struct file *filp, unsigned int cmd, "failed to host gk20a for ioctl cmd: 0x%x", cmd); return err; } - g->ops.fifo.enable_tsg(tsg); + g->ops.tsg.enable(tsg); gk20a_idle(g); break; } @@ -695,7 +695,7 @@ long nvgpu_ioctl_tsg_dev_ioctl(struct file *filp, unsigned int cmd, "failed to host gk20a for ioctl cmd: 0x%x", cmd); return err; } - g->ops.fifo.disable_tsg(tsg); + g->ops.tsg.disable(tsg); gk20a_idle(g); break; } diff --git a/drivers/gpu/nvgpu/tu104/hal_tu104.c b/drivers/gpu/nvgpu/tu104/hal_tu104.c index 4d8f09ffe..0ffff13bd 100644 --- a/drivers/gpu/nvgpu/tu104/hal_tu104.c +++ b/drivers/gpu/nvgpu/tu104/hal_tu104.c @@ -58,6 +58,7 @@ #include "hal/fifo/ramfc_gp10b.h" #include "hal/fifo/ramfc_gv11b.h" #include "hal/fifo/ramfc_tu104.h" +#include "hal/fifo/tsg_gv11b.h" #include "hal/fifo/userd_gk20a.h" #include "hal/fifo/userd_gv11b.h" #include "hal/fifo/fifo_intr_gk20a.h" @@ -875,8 +876,6 @@ static const struct gpu_ops tu104_ops = { .default_timeslice_us = gk20a_fifo_default_timeslice_us, .preempt_channel = gv11b_fifo_preempt_channel, .preempt_tsg = gv11b_fifo_preempt_tsg, - .enable_tsg = gv11b_fifo_enable_tsg, - .disable_tsg = gk20a_disable_tsg, .tsg_verify_channel_status = gk20a_fifo_tsg_unbind_channel_verify_status, .tsg_verify_status_ctx_reload = gm20b_fifo_tsg_verify_status_ctx_reload, .tsg_verify_status_faulted = gv11b_fifo_tsg_verify_status_faulted, @@ -1013,6 +1012,8 @@ static const struct gpu_ops tu104_ops = { .reset_faulted = gv11b_channel_reset_faulted, }, .tsg = { + .enable = gv11b_tsg_enable, + .disable = nvgpu_tsg_disable, .check_ctxsw_timeout = nvgpu_tsg_check_ctxsw_timeout, }, .netlist = {