diff --git a/drivers/gpu/nvgpu/Makefile b/drivers/gpu/nvgpu/Makefile index 834584be5..6e975d501 100644 --- a/drivers/gpu/nvgpu/Makefile +++ b/drivers/gpu/nvgpu/Makefile @@ -353,6 +353,9 @@ nvgpu-y += \ common/sec2/sec2.o \ common/sec2/sec2_ipc.o \ common/io/io.o \ + common/power_features/power_features.o \ + common/power_features/cg/cg.o \ + common/power_features/pg/pg.o \ common/sim.o \ common/sim_pci.o \ common/fifo/channel.o \ diff --git a/drivers/gpu/nvgpu/Makefile.sources b/drivers/gpu/nvgpu/Makefile.sources index 4053b6d5b..72ff51373 100644 --- a/drivers/gpu/nvgpu/Makefile.sources +++ b/drivers/gpu/nvgpu/Makefile.sources @@ -177,6 +177,9 @@ srcs += common/sim.c \ common/semaphore/semaphore_pool.c \ common/semaphore/semaphore_hw.c \ common/semaphore/semaphore.c \ + common/power_features/power_features.c \ + common/power_features/cg/cg.c \ + common/power_features/pg/pg.c \ common/fifo/channel.c \ common/fifo/submit.c \ common/fifo/tsg.c \ diff --git a/drivers/gpu/nvgpu/common/debugger.c b/drivers/gpu/nvgpu/common/debugger.c index 867fcc697..e4dac4dae 100644 --- a/drivers/gpu/nvgpu/common/debugger.c +++ b/drivers/gpu/nvgpu/common/debugger.c @@ -1,7 +1,7 @@ /* * Tegra GK20A GPU Debugger/Profiler Driver * - * Copyright (c) 2013-2018, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2013-2019, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -34,8 +34,7 @@ #include #include #include - -#include "gk20a/gr_gk20a.h" +#include /* * API to get first channel from the list of all channels @@ -208,60 +207,29 @@ int nvgpu_dbg_set_powergate(struct dbg_session_gk20a *dbg_s, bool disable_powerg return err; } - /*do elpg disable before clock gating */ - nvgpu_pmu_pg_global_enable(g, false); + err = nvgpu_cg_pg_disable(g); - if (g->ops.clock_gating.slcg_gr_load_gating_prod != NULL) { - g->ops.clock_gating.slcg_gr_load_gating_prod(g, - false); + if (err == 0) { + dbg_s->is_pg_disabled = true; + nvgpu_log(g, gpu_dbg_gpu_dbg | gpu_dbg_fn, + "pg disabled"); } - if (g->ops.clock_gating.slcg_perf_load_gating_prod != NULL) { - g->ops.clock_gating.slcg_perf_load_gating_prod(g, - false); - } - if (g->ops.clock_gating.slcg_ltc_load_gating_prod != NULL) { - g->ops.clock_gating.slcg_ltc_load_gating_prod(g, - false); - } - - gr_gk20a_init_cg_mode(g, BLCG_MODE, BLCG_RUN); - gr_gk20a_init_cg_mode(g, ELCG_MODE, ELCG_RUN); - - dbg_s->is_pg_disabled = true; } else { /* restore (can) powergate, clk state */ /* release pending exceptions to fault/be handled as usual */ /*TBD: ordering of these? */ - if (g->elcg_enabled) { - gr_gk20a_init_cg_mode(g, ELCG_MODE, ELCG_AUTO); - } + err = nvgpu_cg_pg_enable(g); - if (g->blcg_enabled) { - gr_gk20a_init_cg_mode(g, BLCG_MODE, BLCG_AUTO); - } + nvgpu_log(g, gpu_dbg_gpu_dbg | gpu_dbg_fn, "module idle"); - if (g->slcg_enabled) { - if (g->ops.clock_gating.slcg_ltc_load_gating_prod != NULL) { - g->ops.clock_gating.slcg_ltc_load_gating_prod(g, - g->slcg_enabled); - } - if (g->ops.clock_gating.slcg_perf_load_gating_prod != NULL) { - g->ops.clock_gating.slcg_perf_load_gating_prod(g, - g->slcg_enabled); - } - if (g->ops.clock_gating.slcg_gr_load_gating_prod != NULL) { - g->ops.clock_gating.slcg_gr_load_gating_prod(g, - g->slcg_enabled); - } - } - nvgpu_pmu_pg_global_enable(g, true); - - nvgpu_log(g, gpu_dbg_gpu_dbg | gpu_dbg_fn, - "module idle"); gk20a_idle(g); - dbg_s->is_pg_disabled = false; + if (err == 0) { + dbg_s->is_pg_disabled = false; + nvgpu_log(g, gpu_dbg_gpu_dbg | gpu_dbg_fn, + "pg enabled"); + } } nvgpu_log(g, gpu_dbg_fn|gpu_dbg_gpu_dbg, "%s powergate mode = %s done", diff --git a/drivers/gpu/nvgpu/common/mm/mm.c b/drivers/gpu/nvgpu/common/mm/mm.c index feba585e5..5ea3964e3 100644 --- a/drivers/gpu/nvgpu/common/mm/mm.c +++ b/drivers/gpu/nvgpu/common/mm/mm.c @@ -30,6 +30,7 @@ #include #include #include +#include #include "gk20a/ce2_gk20a.h" @@ -360,22 +361,9 @@ static int nvgpu_init_mm_reset_enable_hw(struct gk20a *g) g->ops.mc.fb_reset(g); } - if (g->ops.clock_gating.slcg_fb_load_gating_prod != NULL) { - g->ops.clock_gating.slcg_fb_load_gating_prod(g, - g->slcg_enabled); - } - if (g->ops.clock_gating.slcg_ltc_load_gating_prod != NULL) { - g->ops.clock_gating.slcg_ltc_load_gating_prod(g, - g->slcg_enabled); - } - if (g->ops.clock_gating.blcg_fb_load_gating_prod != NULL) { - g->ops.clock_gating.blcg_fb_load_gating_prod(g, - g->blcg_enabled); - } - if (g->ops.clock_gating.blcg_ltc_load_gating_prod != NULL) { - g->ops.clock_gating.blcg_ltc_load_gating_prod(g, - g->blcg_enabled); - } + nvgpu_cg_slcg_fb_ltc_load_enable(g); + + nvgpu_cg_blcg_fb_ltc_load_enable(g); if (g->ops.fb.init_fs_state != NULL) { g->ops.fb.init_fs_state(g); diff --git a/drivers/gpu/nvgpu/common/pmu/pmu.c b/drivers/gpu/nvgpu/common/pmu/pmu.c index bdfd47213..a9785057e 100644 --- a/drivers/gpu/nvgpu/common/pmu/pmu.c +++ b/drivers/gpu/nvgpu/common/pmu/pmu.c @@ -33,6 +33,7 @@ #include #include #include +#include static int nvgpu_pg_init_task(void *arg); @@ -47,15 +48,9 @@ static int pmu_enable_hw(struct nvgpu_pmu *pmu, bool enable) /* bring PMU falcon/engine out of reset */ g->ops.pmu.reset_engine(g, true); - if (g->ops.clock_gating.slcg_pmu_load_gating_prod != NULL) { - g->ops.clock_gating.slcg_pmu_load_gating_prod(g, - g->slcg_enabled); - } + nvgpu_cg_slcg_pmu_load_enable(g); - if (g->ops.clock_gating.blcg_pmu_load_gating_prod != NULL) { - g->ops.clock_gating.blcg_pmu_load_gating_prod(g, - g->blcg_enabled); - } + nvgpu_cg_blcg_pmu_load_enable(g); if (nvgpu_falcon_mem_scrub_wait(pmu->flcn) != 0) { /* keep PMU falcon/engine in reset @@ -580,7 +575,7 @@ static void pmu_setup_hw_enable_elpg(struct gk20a *g) g->ops.pmu.save_zbc(g, 0xf); } - if (g->elpg_enabled) { + if (g->can_elpg && g->elpg_enabled) { /* Init reg with prod values*/ if (g->ops.pmu.pmu_setup_elpg != NULL) { g->ops.pmu.pmu_setup_elpg(g); diff --git a/drivers/gpu/nvgpu/common/pmu/pmu_gm20b.c b/drivers/gpu/nvgpu/common/pmu/pmu_gm20b.c index 8ee355fb1..ad0b15222 100644 --- a/drivers/gpu/nvgpu/common/pmu/pmu_gm20b.c +++ b/drivers/gpu/nvgpu/common/pmu/pmu_gm20b.c @@ -105,7 +105,7 @@ int gm20b_pmu_setup_elpg(struct gk20a *g) nvgpu_log_fn(g, " "); - if (g->elpg_enabled) { + if (g->can_elpg && g->elpg_enabled) { reg_writes = ARRAY_SIZE(_pginitseq_gm20b); /* Initialize registers with production values*/ for (index = 0; index < reg_writes; index++) { diff --git a/drivers/gpu/nvgpu/common/pmu/pmu_gp10b.c b/drivers/gpu/nvgpu/common/pmu/pmu_gp10b.c index 58a4b0024..ad8b0dc2d 100644 --- a/drivers/gpu/nvgpu/common/pmu/pmu_gp10b.c +++ b/drivers/gpu/nvgpu/common/pmu/pmu_gp10b.c @@ -296,7 +296,7 @@ int gp10b_pmu_setup_elpg(struct gk20a *g) nvgpu_log_fn(g, " "); - if (g->elpg_enabled) { + if (g->can_elpg && g->elpg_enabled) { reg_writes = ARRAY_SIZE(_pginitseq_gp10b); /* Initialize registers with production values*/ for (index = 0; index < reg_writes; index++) { diff --git a/drivers/gpu/nvgpu/common/pmu/pmu_gv11b.c b/drivers/gpu/nvgpu/common/pmu/pmu_gv11b.c index 4fd1ffca8..d17b1d182 100644 --- a/drivers/gpu/nvgpu/common/pmu/pmu_gv11b.c +++ b/drivers/gpu/nvgpu/common/pmu/pmu_gv11b.c @@ -124,7 +124,7 @@ int gv11b_pmu_setup_elpg(struct gk20a *g) nvgpu_log_fn(g, " "); - if (g->elpg_enabled) { + if (g->can_elpg && g->elpg_enabled) { reg_writes = ARRAY_SIZE(_pginitseq_gv11b); /* Initialize registers with production values*/ for (index = 0; index < reg_writes; index++) { diff --git a/drivers/gpu/nvgpu/common/power_features/cg/cg.c b/drivers/gpu/nvgpu/common/power_features/cg/cg.c new file mode 100644 index 000000000..1b587cee5 --- /dev/null +++ b/drivers/gpu/nvgpu/common/power_features/cg/cg.c @@ -0,0 +1,566 @@ +/* + * Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include + +static void nvgpu_cg_set_mode(struct gk20a *g, int cgmode, int mode_config) +{ + u32 engine_idx; + u32 active_engine_id = 0; + struct fifo_engine_info_gk20a *engine_info = NULL; + struct fifo_gk20a *f = &g->fifo; + + nvgpu_log_fn(g, " "); + + for (engine_idx = 0; engine_idx < f->num_engines; ++engine_idx) { + active_engine_id = f->active_engines_list[engine_idx]; + engine_info = &f->engine_info[active_engine_id]; + + /* gr_engine supports both BLCG and ELCG */ + if ((cgmode == BLCG_MODE) && (engine_info->engine_enum == + NVGPU_ENGINE_GR_GK20A)) { + g->ops.therm.init_blcg_mode(g, (u32)mode_config, + active_engine_id); + break; + } else if (cgmode == ELCG_MODE) { + g->ops.therm.init_elcg_mode(g, (u32)mode_config, + active_engine_id); + } else { + nvgpu_err(g, "invalid cg mode %d, config %d for " + "act_eng_id %d", + cgmode, mode_config, active_engine_id); + } + } +} + +void nvgpu_cg_elcg_enable(struct gk20a *g) +{ + nvgpu_log_fn(g, " "); + + if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_ELCG)) { + return; + } + + nvgpu_mutex_acquire(&g->cg_pg_lock); + if (g->elcg_enabled) { + nvgpu_cg_set_mode(g, ELCG_MODE, ELCG_AUTO); + } + nvgpu_mutex_release(&g->cg_pg_lock); +} + +void nvgpu_cg_elcg_disable(struct gk20a *g) +{ + nvgpu_log_fn(g, " "); + + if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_ELCG)) { + return; + } + + nvgpu_mutex_acquire(&g->cg_pg_lock); + if (g->elcg_enabled) { + nvgpu_cg_set_mode(g, ELCG_MODE, ELCG_RUN); + } + nvgpu_mutex_release(&g->cg_pg_lock); + +} + +void nvgpu_cg_blcg_mode_enable(struct gk20a *g) +{ + nvgpu_log_fn(g, " "); + + if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_BLCG)) { + return; + } + + nvgpu_mutex_acquire(&g->cg_pg_lock); + if (g->blcg_enabled) { + nvgpu_cg_set_mode(g, BLCG_MODE, BLCG_AUTO); + } + nvgpu_mutex_release(&g->cg_pg_lock); + +} + +void nvgpu_cg_blcg_mode_disable(struct gk20a *g) +{ + nvgpu_log_fn(g, " "); + + if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_BLCG)) { + return; + } + + nvgpu_mutex_acquire(&g->cg_pg_lock); + if (g->blcg_enabled) { + nvgpu_cg_set_mode(g, BLCG_MODE, BLCG_RUN); + } + nvgpu_mutex_release(&g->cg_pg_lock); + + +} + +void nvgpu_cg_blcg_fb_ltc_load_enable(struct gk20a *g) +{ + nvgpu_log_fn(g, " "); + + if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_BLCG)) { + return; + } + nvgpu_mutex_acquire(&g->cg_pg_lock); + if (!g->blcg_enabled) { + goto done; + } + if (g->ops.clock_gating.blcg_fb_load_gating_prod != NULL) { + g->ops.clock_gating.blcg_fb_load_gating_prod(g, true); + } + if (g->ops.clock_gating.blcg_ltc_load_gating_prod != NULL) { + g->ops.clock_gating.blcg_ltc_load_gating_prod(g, true); + } +done: + nvgpu_mutex_release(&g->cg_pg_lock); +} + +void nvgpu_cg_blcg_fifo_load_enable(struct gk20a *g) +{ + nvgpu_log_fn(g, " "); + + if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_BLCG)) { + return; + } + nvgpu_mutex_acquire(&g->cg_pg_lock); + if (!g->blcg_enabled) { + goto done; + } + if (g->ops.clock_gating.blcg_fifo_load_gating_prod != NULL) { + g->ops.clock_gating.blcg_fifo_load_gating_prod(g, true); + } +done: + nvgpu_mutex_release(&g->cg_pg_lock); +} + +void nvgpu_cg_blcg_pmu_load_enable(struct gk20a *g) +{ + nvgpu_log_fn(g, " "); + + if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_BLCG)) { + return; + } + nvgpu_mutex_acquire(&g->cg_pg_lock); + if (!g->blcg_enabled) { + goto done; + } + if (g->ops.clock_gating.blcg_pmu_load_gating_prod != NULL) { + g->ops.clock_gating.blcg_pmu_load_gating_prod(g, true); + } +done: + nvgpu_mutex_release(&g->cg_pg_lock); +} + +void nvgpu_cg_blcg_ce_load_enable(struct gk20a *g) +{ + nvgpu_log_fn(g, " "); + + if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_BLCG)) { + return; + } + nvgpu_mutex_acquire(&g->cg_pg_lock); + if (!g->blcg_enabled) { + goto done; + } + if (g->ops.clock_gating.blcg_ce_load_gating_prod != NULL) { + g->ops.clock_gating.blcg_ce_load_gating_prod(g, true); + } +done: + nvgpu_mutex_release(&g->cg_pg_lock); +} + +void nvgpu_cg_blcg_gr_load_enable(struct gk20a *g) +{ + nvgpu_log_fn(g, " "); + + if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_BLCG)) { + return; + } + nvgpu_mutex_acquire(&g->cg_pg_lock); + if (!g->blcg_enabled) { + goto done; + } + if (g->ops.clock_gating.blcg_gr_load_gating_prod != NULL) { + g->ops.clock_gating.blcg_gr_load_gating_prod(g, true); + } +done: + nvgpu_mutex_release(&g->cg_pg_lock); +} + +void nvgpu_cg_slcg_fb_ltc_load_enable(struct gk20a *g) +{ + nvgpu_log_fn(g, " "); + + if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_SLCG)) { + return; + } + nvgpu_mutex_acquire(&g->cg_pg_lock); + if (!g->slcg_enabled) { + goto done; + } + if (g->ops.clock_gating.slcg_fb_load_gating_prod != NULL) { + g->ops.clock_gating.slcg_fb_load_gating_prod(g, true); + } + if (g->ops.clock_gating.slcg_ltc_load_gating_prod != NULL) { + g->ops.clock_gating.slcg_ltc_load_gating_prod(g, true); + } +done: + nvgpu_mutex_release(&g->cg_pg_lock); +} + +void nvgpu_cg_slcg_priring_load_enable(struct gk20a *g) +{ + nvgpu_log_fn(g, " "); + + if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_SLCG)) { + return; + } + nvgpu_mutex_acquire(&g->cg_pg_lock); + if (!g->slcg_enabled) { + goto done; + } + if (g->ops.clock_gating.slcg_priring_load_gating_prod != NULL) { + g->ops.clock_gating.slcg_priring_load_gating_prod(g, true); + } +done: + nvgpu_mutex_release(&g->cg_pg_lock); +} + +void nvgpu_cg_slcg_gr_perf_ltc_load_enable(struct gk20a *g) +{ + nvgpu_log_fn(g, " "); + + if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_SLCG)) { + return; + } + nvgpu_mutex_acquire(&g->cg_pg_lock); + if (!g->slcg_enabled) { + goto done; + } + if (g->ops.clock_gating.slcg_ltc_load_gating_prod != NULL) { + g->ops.clock_gating.slcg_ltc_load_gating_prod(g, true); + } + if (g->ops.clock_gating.slcg_perf_load_gating_prod != NULL) { + g->ops.clock_gating.slcg_perf_load_gating_prod(g, true); + } + if (g->ops.clock_gating.slcg_gr_load_gating_prod != NULL) { + g->ops.clock_gating.slcg_gr_load_gating_prod(g, true); + } +done: + nvgpu_mutex_release(&g->cg_pg_lock); +} + +void nvgpu_cg_slcg_gr_perf_ltc_load_disable(struct gk20a *g) +{ + nvgpu_log_fn(g, " "); + + if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_SLCG)) { + return; + } + nvgpu_mutex_acquire(&g->cg_pg_lock); + if (!g->slcg_enabled) { + goto done; + } + if (g->ops.clock_gating.slcg_gr_load_gating_prod != NULL) { + g->ops.clock_gating.slcg_gr_load_gating_prod(g, false); + } + if (g->ops.clock_gating.slcg_perf_load_gating_prod != NULL) { + g->ops.clock_gating.slcg_perf_load_gating_prod(g, false); + } + if (g->ops.clock_gating.slcg_ltc_load_gating_prod != NULL) { + g->ops.clock_gating.slcg_ltc_load_gating_prod(g, false); + } +done: + nvgpu_mutex_release(&g->cg_pg_lock); +} + +void nvgpu_cg_slcg_fifo_load_enable(struct gk20a *g) +{ + nvgpu_log_fn(g, " "); + + if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_SLCG)) { + return; + } + nvgpu_mutex_acquire(&g->cg_pg_lock); + if (!g->slcg_enabled) { + goto done; + } + if (g->ops.clock_gating.slcg_fifo_load_gating_prod != NULL) { + g->ops.clock_gating.slcg_fifo_load_gating_prod(g, true); + } +done: + nvgpu_mutex_release(&g->cg_pg_lock); +} + +void nvgpu_cg_slcg_pmu_load_enable(struct gk20a *g) +{ + nvgpu_log_fn(g, " "); + + if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_SLCG)) { + return; + } + nvgpu_mutex_acquire(&g->cg_pg_lock); + if (!g->slcg_enabled) { + goto done; + } + if (g->ops.clock_gating.slcg_pmu_load_gating_prod != NULL) { + g->ops.clock_gating.slcg_pmu_load_gating_prod(g, true); + } +done: + nvgpu_mutex_release(&g->cg_pg_lock); +} + +void nvgpu_cg_slcg_ce2_load_enable(struct gk20a *g) +{ + nvgpu_log_fn(g, " "); + + if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_SLCG)) { + return; + } + nvgpu_mutex_acquire(&g->cg_pg_lock); + if (!g->slcg_enabled) { + goto done; + } + if (g->ops.clock_gating.slcg_ce2_load_gating_prod != NULL) { + g->ops.clock_gating.slcg_ce2_load_gating_prod(g, true); + } +done: + nvgpu_mutex_release(&g->cg_pg_lock); +} + +void nvgpu_cg_init_gr_load_gating_prod(struct gk20a *g) +{ + nvgpu_log_fn(g, " "); + + nvgpu_mutex_acquire(&g->cg_pg_lock); + + if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_SLCG)) { + goto check_can_blcg; + } + if (!g->slcg_enabled) { + goto check_can_blcg; + } + + if (g->ops.clock_gating.slcg_bus_load_gating_prod != NULL) { + g->ops.clock_gating.slcg_bus_load_gating_prod(g, true); + } + if (g->ops.clock_gating.slcg_chiplet_load_gating_prod != NULL) { + g->ops.clock_gating.slcg_chiplet_load_gating_prod(g, true); + } + if (g->ops.clock_gating.slcg_gr_load_gating_prod != NULL) { + g->ops.clock_gating.slcg_gr_load_gating_prod(g, true); + } + if (g->ops.clock_gating.slcg_ctxsw_firmware_load_gating_prod != NULL) { + g->ops.clock_gating.slcg_ctxsw_firmware_load_gating_prod(g, + true); + } + if (g->ops.clock_gating.slcg_perf_load_gating_prod != NULL) { + g->ops.clock_gating.slcg_perf_load_gating_prod(g, true); + } + if (g->ops.clock_gating.slcg_xbar_load_gating_prod != NULL) { + g->ops.clock_gating.slcg_xbar_load_gating_prod(g, true); + } + +check_can_blcg: + if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_BLCG)) { + goto pg_gr_load; + } + if (!g->blcg_enabled) { + goto pg_gr_load; + } + if (g->ops.clock_gating.blcg_bus_load_gating_prod != NULL) { + g->ops.clock_gating.blcg_bus_load_gating_prod(g, true); + } + if (g->ops.clock_gating.blcg_gr_load_gating_prod != NULL) { + g->ops.clock_gating.blcg_gr_load_gating_prod(g, true); + } + if (g->ops.clock_gating.blcg_ctxsw_firmware_load_gating_prod != NULL) { + g->ops.clock_gating.blcg_ctxsw_firmware_load_gating_prod(g, + true); + } + if (g->ops.clock_gating.blcg_xbar_load_gating_prod != NULL) { + g->ops.clock_gating.blcg_xbar_load_gating_prod(g, true); + } +pg_gr_load: + if (g->ops.clock_gating.pg_gr_load_gating_prod != NULL) { + g->ops.clock_gating.pg_gr_load_gating_prod(g, true); + } + + nvgpu_mutex_release(&g->cg_pg_lock); +} + +void nvgpu_cg_elcg_set_elcg_enabled(struct gk20a *g, bool enable) +{ + nvgpu_log_fn(g, " "); + + if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_ELCG)) { + return; + } + + nvgpu_mutex_release(&g->cg_pg_lock); + if (enable) { + if (!g->elcg_enabled) { + g->elcg_enabled = true; + nvgpu_cg_set_mode(g, ELCG_MODE, ELCG_AUTO); + } + } else { + if (g->elcg_enabled) { + g->elcg_enabled = false; + nvgpu_cg_set_mode(g, ELCG_MODE, ELCG_RUN); + } + } + nvgpu_mutex_release(&g->cg_pg_lock); +} + +void nvgpu_cg_blcg_set_blcg_enabled(struct gk20a *g, bool enable) +{ + bool load = false; + + nvgpu_log_fn(g, " "); + + if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_BLCG)) { + return; + } + + nvgpu_mutex_acquire(&g->cg_pg_lock); + if (enable) { + if (!g->blcg_enabled) { + load = true; + g->blcg_enabled = true; + } + } else { + if (g->blcg_enabled) { + load = true; + g->blcg_enabled = false; + } + } + if (!load ) { + goto done; + } + + if (g->ops.clock_gating.blcg_bus_load_gating_prod != NULL) { + g->ops.clock_gating.blcg_bus_load_gating_prod(g, enable); + } + if (g->ops.clock_gating.blcg_ce_load_gating_prod != NULL) { + g->ops.clock_gating.blcg_ce_load_gating_prod(g, enable); + } + if (g->ops.clock_gating.blcg_ctxsw_firmware_load_gating_prod != NULL) { + g->ops.clock_gating.blcg_ctxsw_firmware_load_gating_prod(g, + enable); + } + if (g->ops.clock_gating.blcg_fb_load_gating_prod != NULL) { + g->ops.clock_gating.blcg_fb_load_gating_prod(g, enable); + } + if (g->ops.clock_gating.blcg_fifo_load_gating_prod != NULL) { + g->ops.clock_gating.blcg_fifo_load_gating_prod(g, enable); + } + if (g->ops.clock_gating.blcg_gr_load_gating_prod != NULL) { + g->ops.clock_gating.blcg_gr_load_gating_prod(g, enable); + } + if (g->ops.clock_gating.blcg_ltc_load_gating_prod != NULL) { + g->ops.clock_gating.blcg_ltc_load_gating_prod(g, enable); + } + if (g->ops.clock_gating.blcg_pmu_load_gating_prod != NULL) { + g->ops.clock_gating.blcg_pmu_load_gating_prod(g, enable); + } + if (g->ops.clock_gating.blcg_xbar_load_gating_prod != NULL) { + g->ops.clock_gating.blcg_xbar_load_gating_prod(g, enable); + } + +done: + nvgpu_mutex_release(&g->cg_pg_lock); +} + +void nvgpu_cg_slcg_set_slcg_enabled(struct gk20a *g, bool enable) +{ + bool load = false; + + nvgpu_log_fn(g, " "); + + if (!nvgpu_is_enabled(g, NVGPU_GPU_CAN_SLCG)) { + return; + } + + nvgpu_mutex_acquire(&g->cg_pg_lock); + if (enable) { + if (!g->slcg_enabled) { + load = true; + g->slcg_enabled = true; + } + } else { + if (g->slcg_enabled) { + load = true; + g->slcg_enabled = false; + } + } + if (!load ) { + goto done; + } + + if (g->ops.clock_gating.slcg_bus_load_gating_prod != NULL) { + g->ops.clock_gating.slcg_bus_load_gating_prod(g, enable); + } + if (g->ops.clock_gating.slcg_ce2_load_gating_prod != NULL) { + g->ops.clock_gating.slcg_ce2_load_gating_prod(g, enable); + } + if (g->ops.clock_gating.slcg_chiplet_load_gating_prod != NULL) { + g->ops.clock_gating.slcg_chiplet_load_gating_prod(g, enable); + } + if (g->ops.clock_gating.slcg_ctxsw_firmware_load_gating_prod != + NULL) { + g->ops.clock_gating.slcg_ctxsw_firmware_load_gating_prod(g, + enable); + } + if (g->ops.clock_gating.slcg_fb_load_gating_prod != NULL) { + g->ops.clock_gating.slcg_fb_load_gating_prod(g, enable); + } + if (g->ops.clock_gating.slcg_fifo_load_gating_prod != NULL) { + g->ops.clock_gating.slcg_fifo_load_gating_prod(g, enable); + } + if (g->ops.clock_gating.slcg_gr_load_gating_prod != NULL) { + g->ops.clock_gating.slcg_gr_load_gating_prod(g, enable); + } + if (g->ops.clock_gating.slcg_ltc_load_gating_prod != NULL) { + g->ops.clock_gating.slcg_ltc_load_gating_prod(g, enable); + } + if (g->ops.clock_gating.slcg_perf_load_gating_prod != NULL) { + g->ops.clock_gating.slcg_perf_load_gating_prod(g, enable); + } + if (g->ops.clock_gating.slcg_priring_load_gating_prod != NULL) { + g->ops.clock_gating.slcg_priring_load_gating_prod(g, + enable); + } + if (g->ops.clock_gating.slcg_pmu_load_gating_prod != NULL) { + g->ops.clock_gating.slcg_pmu_load_gating_prod(g, enable); + } + if (g->ops.clock_gating.slcg_xbar_load_gating_prod != NULL) { + g->ops.clock_gating.slcg_xbar_load_gating_prod(g, enable); + } + +done: + nvgpu_mutex_release(&g->cg_pg_lock); +} diff --git a/drivers/gpu/nvgpu/common/power_features/pg/pg.c b/drivers/gpu/nvgpu/common/power_features/pg/pg.c new file mode 100644 index 000000000..fa31f4e3a --- /dev/null +++ b/drivers/gpu/nvgpu/common/power_features/pg/pg.c @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include + +bool nvgpu_pg_elpg_is_enabled(struct gk20a *g) +{ + bool elpg_enabled; + + nvgpu_log_fn(g, " "); + + nvgpu_mutex_acquire(&g->cg_pg_lock); + elpg_enabled = g->elpg_enabled; + nvgpu_mutex_release(&g->cg_pg_lock); + return elpg_enabled; +} + +int nvgpu_pg_elpg_enable(struct gk20a *g) +{ + int err = 0; + + nvgpu_log_fn(g, " "); + + if (!g->can_elpg) { + return 0; + } + + nvgpu_mutex_acquire(&g->cg_pg_lock); + if (g->elpg_enabled) { + err = nvgpu_pmu_pg_global_enable(g, true); + } + nvgpu_mutex_release(&g->cg_pg_lock); + return err; +} + +int nvgpu_pg_elpg_disable(struct gk20a *g) +{ + int err = 0; + + nvgpu_log_fn(g, " "); + + if (!g->can_elpg) { + return 0; + } + + nvgpu_mutex_acquire(&g->cg_pg_lock); + if (g->elpg_enabled) { + err = nvgpu_pmu_pg_global_enable(g, false); + } + nvgpu_mutex_release(&g->cg_pg_lock); + return err; +} + +int nvgpu_pg_elpg_set_elpg_enabled(struct gk20a *g, bool enable) +{ + int err = 0; + bool change_mode = false; + + nvgpu_log_fn(g, " "); + + if (!g->can_elpg) { + return 0; + } + + nvgpu_mutex_acquire(&g->cg_pg_lock); + if (enable) { + if (!g->elpg_enabled) { + change_mode = true; + g->elpg_enabled = true; + } + } else { + if (g->elpg_enabled) { + change_mode = true; + g->elpg_enabled = false; + } + } + if (!change_mode) { + goto done; + } + + err = nvgpu_pmu_pg_global_enable(g, enable); +done: + nvgpu_mutex_release(&g->cg_pg_lock); + return err; +} diff --git a/drivers/gpu/nvgpu/common/power_features/power_features.c b/drivers/gpu/nvgpu/common/power_features/power_features.c new file mode 100644 index 000000000..792fdc015 --- /dev/null +++ b/drivers/gpu/nvgpu/common/power_features/power_features.c @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include +#include + +int nvgpu_cg_pg_disable(struct gk20a *g) +{ + int err = 0; + + nvgpu_log_fn(g, " "); + + /* disable elpg before clock gating */ + err = nvgpu_pg_elpg_disable(g); + if (err != 0) { + nvgpu_err(g, "failed to set disable elpg"); + } + nvgpu_cg_slcg_gr_perf_ltc_load_disable(g); + + nvgpu_cg_blcg_mode_disable(g); + + nvgpu_cg_elcg_disable(g); + + return err; +} + +int nvgpu_cg_pg_enable(struct gk20a *g) +{ + int err = 0; + + nvgpu_log_fn(g, " "); + + nvgpu_cg_elcg_enable(g); + + nvgpu_cg_blcg_mode_enable(g); + + nvgpu_cg_slcg_gr_perf_ltc_load_enable(g); + + err = nvgpu_pg_elpg_enable(g); + if (err != 0) { + nvgpu_err(g, "failed to set enable elpg"); + } + + return err; +} diff --git a/drivers/gpu/nvgpu/gk20a/ce2_gk20a.c b/drivers/gpu/nvgpu/gk20a/ce2_gk20a.c index 27874546e..58be800f8 100644 --- a/drivers/gpu/nvgpu/gk20a/ce2_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/ce2_gk20a.c @@ -32,6 +32,7 @@ #include #include #include +#include #include "gk20a/fence_gk20a.h" #include "gk20a/ce2_gk20a.h" @@ -354,14 +355,9 @@ int gk20a_init_ce_support(struct gk20a *g) g->ops.mc.reset(g, ce_reset_mask); - if (g->ops.clock_gating.slcg_ce2_load_gating_prod != NULL) { - g->ops.clock_gating.slcg_ce2_load_gating_prod(g, - g->slcg_enabled); - } - if (g->ops.clock_gating.blcg_ce_load_gating_prod != NULL) { - g->ops.clock_gating.blcg_ce_load_gating_prod(g, - g->blcg_enabled); - } + nvgpu_cg_slcg_ce2_load_enable(g); + + nvgpu_cg_blcg_ce_load_enable(g); if (ce_app->initialised) { /* assume this happen during poweron/poweroff GPU sequence */ diff --git a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c index e808d6580..f8568fbea 100644 --- a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c @@ -55,6 +55,8 @@ #include #include #include +#include +#include #include "mm_gk20a.h" @@ -311,14 +313,9 @@ int gk20a_init_fifo_reset_enable_hw(struct gk20a *g) /* enable pmc pfifo */ g->ops.mc.reset(g, g->ops.mc.reset_mask(g, NVGPU_UNIT_FIFO)); - if (g->ops.clock_gating.slcg_fifo_load_gating_prod != NULL) { - g->ops.clock_gating.slcg_fifo_load_gating_prod(g, - g->slcg_enabled); - } - if (g->ops.clock_gating.blcg_fifo_load_gating_prod != NULL) { - g->ops.clock_gating.blcg_fifo_load_gating_prod(g, - g->blcg_enabled); - } + nvgpu_cg_slcg_fifo_load_enable(g); + + nvgpu_cg_blcg_fifo_load_enable(g); timeout = gk20a_readl(g, fifo_fb_timeout_r()); timeout = set_field(timeout, fifo_fb_timeout_period_m(), @@ -931,10 +928,8 @@ void gk20a_fifo_reset_engine(struct gk20a *g, u32 engine_id) } if (engine_enum == NVGPU_ENGINE_GR_GK20A) { - if (g->can_elpg) { - if (nvgpu_pmu_disable_elpg(g) != 0) { - nvgpu_err(g, "failed to set disable elpg"); - } + if (nvgpu_pg_elpg_disable(g) != 0 ) { + nvgpu_err(g, "failed to set disable elpg"); } #ifdef CONFIG_GK20A_CTXSW_TRACE @@ -961,8 +956,8 @@ void gk20a_fifo_reset_engine(struct gk20a *g, u32 engine_id) "HALT gr pipe not supported and " "gr cannot be reset without halting gr pipe"); } - if (g->can_elpg) { - nvgpu_pmu_enable_elpg(g); + if (nvgpu_pg_elpg_enable(g) != 0 ) { + nvgpu_err(g, "failed to set enable elpg"); } } if ((engine_enum == NVGPU_ENGINE_GRCE_GK20A) || @@ -1122,26 +1117,9 @@ static bool gk20a_fifo_handle_mmu_fault_locked( g->fifo.deferred_reset_pending = false; - /* Disable power management */ - if (g->can_elpg) { - if (nvgpu_pmu_disable_elpg(g) != 0) { - nvgpu_err(g, "failed to set disable elpg"); - } + if (nvgpu_cg_pg_disable(g) != 0) { + nvgpu_warn(g, "fail to disable power mgmt"); } - if (g->ops.clock_gating.slcg_gr_load_gating_prod != NULL) { - g->ops.clock_gating.slcg_gr_load_gating_prod(g, - false); - } - if (g->ops.clock_gating.slcg_perf_load_gating_prod != NULL) { - g->ops.clock_gating.slcg_perf_load_gating_prod(g, - false); - } - if (g->ops.clock_gating.slcg_ltc_load_gating_prod != NULL) { - g->ops.clock_gating.slcg_ltc_load_gating_prod(g, - false); - } - - gr_gk20a_init_cg_mode(g, ELCG_MODE, ELCG_RUN); /* Disable fifo access */ grfifo_ctl = gk20a_readl(g, gr_gpfifo_ctl_r()); @@ -1326,11 +1304,9 @@ static bool gk20a_fifo_handle_mmu_fault_locked( gr_gpfifo_ctl_access_enabled_f() | gr_gpfifo_ctl_semaphore_access_enabled_f()); - /* It is safe to enable ELPG again. */ - if (g->can_elpg) { - nvgpu_pmu_enable_elpg(g); + if (nvgpu_cg_pg_enable(g) != 0) { + nvgpu_warn(g, "fail to enable power mgmt"); } - return verbose; } diff --git a/drivers/gpu/nvgpu/gk20a/gr_gk20a.c b/drivers/gpu/nvgpu/gk20a/gr_gk20a.c index 08ad2456c..589d36469 100644 --- a/drivers/gpu/nvgpu/gk20a/gr_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/gr_gk20a.c @@ -57,6 +57,7 @@ #include #include #include +#include #include "gr_gk20a.h" #include "gr_pri_gk20a.h" @@ -92,8 +93,6 @@ void nvgpu_report_gr_exception(struct gk20a *g, u32 inst, static int gk20a_init_gr_bind_fecs_elpg(struct gk20a *g); -/*elcg init */ -static void gr_gk20a_enable_elcg(struct gk20a *g); void gk20a_fecs_dump_falcon_stats(struct gk20a *g) { @@ -1277,9 +1276,7 @@ int gr_gk20a_init_golden_ctx_image(struct gk20a *g, g->ops.gr.init_preemption_state(g); } - if (g->ops.clock_gating.blcg_gr_load_gating_prod != NULL) { - g->ops.clock_gating.blcg_gr_load_gating_prod(g, g->blcg_enabled); - } + nvgpu_cg_blcg_gr_load_enable(g); err = gr_gk20a_wait_idle(g); if (err != 0) { @@ -2486,33 +2483,6 @@ int gr_gk20a_get_zcull_info(struct gk20a *g, struct gr_gk20a *gr, return 0; } -void gr_gk20a_init_cg_mode(struct gk20a *g, u32 cgmode, u32 mode_config) -{ - u32 engine_idx; - u32 active_engine_id = 0; - struct fifo_engine_info_gk20a *engine_info = NULL; - struct fifo_gk20a *f = &g->fifo; - - for (engine_idx = 0; engine_idx < f->num_engines; ++engine_idx) { - active_engine_id = f->active_engines_list[engine_idx]; - engine_info = &f->engine_info[active_engine_id]; - - /* gr_engine supports both BLCG and ELCG */ - if ((cgmode == BLCG_MODE) && - (engine_info->engine_enum == NVGPU_ENGINE_GR_GK20A)) { - g->ops.therm.init_blcg_mode(g, mode_config, active_engine_id); - break; - } else if (cgmode == ELCG_MODE) { - g->ops.therm.init_elcg_mode(g, mode_config, - active_engine_id); - } else { - nvgpu_err(g, "invalid cg mode %d, config %d for " - "act_eng_id %d", - cgmode, mode_config, active_engine_id); - } - } -} - void gr_gk20a_program_zcull_mapping(struct gk20a *g, u32 zcull_num_entries, u32 *zcull_map_tiles) { @@ -2927,60 +2897,6 @@ out: return err; } -static void gr_gk20a_load_gating_prod(struct gk20a *g) -{ - nvgpu_log_fn(g, " "); - - /* slcg prod values */ - if (g->ops.clock_gating.slcg_bus_load_gating_prod != NULL) { - g->ops.clock_gating.slcg_bus_load_gating_prod(g, - g->slcg_enabled); - } - if (g->ops.clock_gating.slcg_chiplet_load_gating_prod != NULL) { - g->ops.clock_gating.slcg_chiplet_load_gating_prod(g, - g->slcg_enabled); - } - if (g->ops.clock_gating.slcg_gr_load_gating_prod != NULL) { - g->ops.clock_gating.slcg_gr_load_gating_prod(g, - g->slcg_enabled); - } - if (g->ops.clock_gating.slcg_ctxsw_firmware_load_gating_prod != NULL) { - g->ops.clock_gating.slcg_ctxsw_firmware_load_gating_prod(g, - g->slcg_enabled); - } - if (g->ops.clock_gating.slcg_perf_load_gating_prod != NULL) { - g->ops.clock_gating.slcg_perf_load_gating_prod(g, - g->slcg_enabled); - } - if (g->ops.clock_gating.slcg_xbar_load_gating_prod != NULL) { - g->ops.clock_gating.slcg_xbar_load_gating_prod(g, - g->slcg_enabled); - } - - /* blcg prod values */ - if (g->ops.clock_gating.blcg_bus_load_gating_prod != NULL) { - g->ops.clock_gating.blcg_bus_load_gating_prod(g, - g->blcg_enabled); - } - if (g->ops.clock_gating.blcg_gr_load_gating_prod != NULL) { - g->ops.clock_gating.blcg_gr_load_gating_prod(g, - g->blcg_enabled); - } - if (g->ops.clock_gating.blcg_ctxsw_firmware_load_gating_prod != NULL) { - g->ops.clock_gating.blcg_ctxsw_firmware_load_gating_prod(g, - g->blcg_enabled); - } - if (g->ops.clock_gating.blcg_xbar_load_gating_prod != NULL) { - g->ops.clock_gating.blcg_xbar_load_gating_prod(g, - g->blcg_enabled); - } - if (g->ops.clock_gating.pg_gr_load_gating_prod != NULL) { - g->ops.clock_gating.pg_gr_load_gating_prod(g, true); - } - - nvgpu_log_fn(g, "done"); -} - static void gk20a_init_gr_prepare(struct gk20a *g) { /* reset gr engine */ @@ -2988,10 +2904,10 @@ static void gk20a_init_gr_prepare(struct gk20a *g) g->ops.mc.reset_mask(g, NVGPU_UNIT_BLG) | g->ops.mc.reset_mask(g, NVGPU_UNIT_PERFMON)); - gr_gk20a_load_gating_prod(g); + nvgpu_cg_init_gr_load_gating_prod(g); /* Disable elcg until it gets enabled later in the init*/ - gr_gk20a_init_cg_mode(g, ELCG_MODE, ELCG_RUN); + nvgpu_cg_elcg_disable(g); /* enable fifo access */ gk20a_writel(g, gr_gpfifo_ctl_r(), @@ -3321,7 +3237,8 @@ int gk20a_init_gr_support(struct gk20a *g) } } - gr_gk20a_enable_elcg(g); + nvgpu_cg_elcg_enable(g); + /* GR is inialized, signal possible waiters */ g->gr.initialized = true; nvgpu_cond_signal(&g->gr.init_wq); @@ -3411,15 +3328,6 @@ int gk20a_enable_gr_hw(struct gk20a *g) return 0; } -static void gr_gk20a_enable_elcg(struct gk20a *g) -{ - if (g->elcg_enabled) { - gr_gk20a_init_cg_mode(g, ELCG_MODE, ELCG_AUTO); - } else { - gr_gk20a_init_cg_mode(g, ELCG_MODE, ELCG_RUN); - } -} - int gk20a_gr_reset(struct gk20a *g) { int err; @@ -3476,8 +3384,9 @@ int gk20a_gr_reset(struct gk20a *g) return err; } - gr_gk20a_load_gating_prod(g); - gr_gk20a_enable_elcg(g); + nvgpu_cg_init_gr_load_gating_prod(g); + + nvgpu_cg_elcg_enable(g); return err; } diff --git a/drivers/gpu/nvgpu/gk20a/gr_gk20a.h b/drivers/gpu/nvgpu/gk20a/gr_gk20a.h index d35764d46..a2382cb6e 100644 --- a/drivers/gpu/nvgpu/gk20a/gr_gk20a.h +++ b/drivers/gpu/nvgpu/gk20a/gr_gk20a.h @@ -26,6 +26,7 @@ #include #include +#include #include "mm_gk20a.h" @@ -402,17 +403,13 @@ u32 gk20a_gr_get_sm_no_lock_down_hww_global_esr_mask(struct gk20a *g); #define gr_gk20a_elpg_protected_call(g, func) \ ({ \ int err = 0; \ - if ((g)->elpg_enabled) {\ - err = nvgpu_pmu_disable_elpg(g); \ - if (err != 0) {\ - nvgpu_pmu_enable_elpg(g); \ - } \ - } \ + err = nvgpu_pg_elpg_disable(g);\ + if (err != 0) {\ + err = nvgpu_pg_elpg_enable(g);\ + }\ if (err == 0) { \ err = (func); \ - if ((g)->elpg_enabled) {\ - nvgpu_pmu_enable_elpg(g); \ - } \ + (void)nvgpu_pg_elpg_enable(g);\ } \ err; \ }) diff --git a/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c b/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c index a0d07a4b0..cad6fef2c 100644 --- a/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c +++ b/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c @@ -48,6 +48,8 @@ #include #include #include +#include +#include #include "gk20a/fifo_gk20a.h" @@ -1044,26 +1046,9 @@ void gv11b_fifo_teardown_ch_tsg(struct gk20a *g, u32 act_eng_bitmask, g->fifo.deferred_reset_pending = false; - /* Disable power management */ - if (g->elpg_enabled) { - if (nvgpu_pmu_disable_elpg(g) != 0) { - nvgpu_err(g, "failed to set disable elpg"); - } + if (nvgpu_cg_pg_disable(g) != 0) { + nvgpu_warn(g, "fail to disable power mgmt"); } - if (g->ops.clock_gating.slcg_gr_load_gating_prod != NULL) { - g->ops.clock_gating.slcg_gr_load_gating_prod(g, - false); - } - if (g->ops.clock_gating.slcg_perf_load_gating_prod != NULL) { - g->ops.clock_gating.slcg_perf_load_gating_prod(g, - false); - } - if (g->ops.clock_gating.slcg_ltc_load_gating_prod != NULL) { - g->ops.clock_gating.slcg_ltc_load_gating_prod(g, - false); - } - - gr_gk20a_init_cg_mode(g, ELCG_MODE, ELCG_RUN); if (rc_type == RC_TYPE_MMU_FAULT) { gk20a_debug_dump(g); @@ -1175,11 +1160,8 @@ void gv11b_fifo_teardown_ch_tsg(struct gk20a *g, u32 act_eng_bitmask, gk20a_fifo_set_runlist_state(g, runlists_mask, RUNLIST_ENABLED); - /* It is safe to enable ELPG again. */ - if (g->elpg_enabled) { - if (nvgpu_pmu_enable_elpg(g) != 0) { - nvgpu_err(g, "ELPG enable failed"); - } + if (nvgpu_cg_pg_enable(g) != 0) { + nvgpu_warn(g, "fail to enable power mgmt"); } g->ops.fifo.teardown_unmask_intr(g); @@ -1267,18 +1249,11 @@ int gv11b_init_fifo_reset_enable_hw(struct gk20a *g) /* enable pmc pfifo */ g->ops.mc.reset(g, g->ops.mc.reset_mask(g, NVGPU_UNIT_FIFO)); - if (g->ops.clock_gating.slcg_ce2_load_gating_prod != NULL) { - g->ops.clock_gating.slcg_ce2_load_gating_prod(g, - g->slcg_enabled); - } - if (g->ops.clock_gating.slcg_fifo_load_gating_prod != NULL) { - g->ops.clock_gating.slcg_fifo_load_gating_prod(g, - g->slcg_enabled); - } - if (g->ops.clock_gating.blcg_fifo_load_gating_prod != NULL) { - g->ops.clock_gating.blcg_fifo_load_gating_prod(g, - g->blcg_enabled); - } + nvgpu_cg_slcg_ce2_load_enable(g); + + nvgpu_cg_slcg_fifo_load_enable(g); + + nvgpu_cg_blcg_fifo_load_enable(g); timeout = gk20a_readl(g, fifo_fb_timeout_r()); nvgpu_log_info(g, "fifo_fb_timeout reg val = 0x%08x", timeout); diff --git a/drivers/gpu/nvgpu/hal/priv_ring/priv_ring_gm20b.c b/drivers/gpu/nvgpu/hal/priv_ring/priv_ring_gm20b.c index 318800c18..269254a07 100644 --- a/drivers/gpu/nvgpu/hal/priv_ring/priv_ring_gm20b.c +++ b/drivers/gpu/nvgpu/hal/priv_ring/priv_ring_gm20b.c @@ -28,6 +28,7 @@ #include #include #include +#include #include "priv_ring_gm20b.h" @@ -44,10 +45,7 @@ void gm20b_priv_ring_enable(struct gk20a *g) nvgpu_log_info(g, "enabling priv ring"); - if (g->ops.clock_gating.slcg_priring_load_gating_prod != NULL) { - g->ops.clock_gating.slcg_priring_load_gating_prod(g, - g->slcg_enabled); - } + nvgpu_cg_slcg_priring_load_enable(g); nvgpu_writel(g,pri_ringmaster_command_r(), 0x4); diff --git a/drivers/gpu/nvgpu/include/nvgpu/gk20a.h b/drivers/gpu/nvgpu/include/nvgpu/gk20a.h index ac1242eb2..7e9884cbb 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/gk20a.h +++ b/drivers/gpu/nvgpu/include/nvgpu/gk20a.h @@ -1768,6 +1768,7 @@ struct gk20a { u32 max_timeslice_us; bool runlist_interleave; + struct nvgpu_mutex cg_pg_lock; bool slcg_enabled; bool blcg_enabled; bool elcg_enabled; diff --git a/drivers/gpu/nvgpu/include/nvgpu/power_features/cg.h b/drivers/gpu/nvgpu/include/nvgpu/power_features/cg.h new file mode 100644 index 000000000..3bb86267b --- /dev/null +++ b/drivers/gpu/nvgpu/include/nvgpu/power_features/cg.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + + +#ifndef NVGPU_POWER_FEATURES_CG_H +#define NVGPU_POWER_FEATURES_CG_H + +#include + +struct gk20a; +struct fifo_gk20a; + +void nvgpu_cg_init_gr_load_gating_prod(struct gk20a *g); +void nvgpu_cg_elcg_enable(struct gk20a *g); +void nvgpu_cg_elcg_disable(struct gk20a *g); +void nvgpu_cg_elcg_set_elcg_enabled(struct gk20a *g, bool enable); + +void nvgpu_cg_blcg_mode_enable(struct gk20a *g); +void nvgpu_cg_blcg_mode_disable(struct gk20a *g); +void nvgpu_cg_blcg_fb_ltc_load_enable(struct gk20a *g); +void nvgpu_cg_blcg_fifo_load_enable(struct gk20a *g); +void nvgpu_cg_blcg_pmu_load_enable(struct gk20a *g); +void nvgpu_cg_blcg_ce_load_enable(struct gk20a *g); +void nvgpu_cg_blcg_gr_load_enable(struct gk20a *g); +void nvgpu_cg_blcg_set_blcg_enabled(struct gk20a *g, bool enable); + +void nvgpu_cg_slcg_gr_perf_ltc_load_enable(struct gk20a *g); +void nvgpu_cg_slcg_gr_perf_ltc_load_disable(struct gk20a *g); +void nvgpu_cg_slcg_fb_ltc_load_enable(struct gk20a *g); +void nvgpu_cg_slcg_priring_load_enable(struct gk20a *g); +void nvgpu_cg_slcg_fifo_load_enable(struct gk20a *g); +void nvgpu_cg_slcg_pmu_load_enable(struct gk20a *g); +void nvgpu_cg_slcg_ce2_load_enable(struct gk20a *g); +void nvgpu_cg_slcg_set_slcg_enabled(struct gk20a *g, bool enable); + +#endif /*NVGPU_POWER_FEATURES_CG_H*/ diff --git a/drivers/gpu/nvgpu/include/nvgpu/power_features/pg.h b/drivers/gpu/nvgpu/include/nvgpu/power_features/pg.h new file mode 100644 index 000000000..d73578071 --- /dev/null +++ b/drivers/gpu/nvgpu/include/nvgpu/power_features/pg.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + + +#ifndef NVGPU_POWER_FEATURES_PG_H +#define NVGPU_POWER_FEATURES_PG_H + +#include + +struct gk20a; + +int nvgpu_pg_elpg_disable(struct gk20a *g); +int nvgpu_pg_elpg_enable(struct gk20a *g); +bool nvgpu_pg_elpg_is_enabled(struct gk20a *g); +int nvgpu_pg_elpg_set_elpg_enabled(struct gk20a *g, bool enable); + +#endif /*NVGPU_POWER_FEATURES_PG_H*/ diff --git a/drivers/gpu/nvgpu/include/nvgpu/power_features/power_features.h b/drivers/gpu/nvgpu/include/nvgpu/power_features/power_features.h new file mode 100644 index 000000000..f6ffccf18 --- /dev/null +++ b/drivers/gpu/nvgpu/include/nvgpu/power_features/power_features.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + + +#ifndef NVGPU_POWER_FEATURES_H +#define NVGPU_POWER_FEATURES_H + +#include + +struct gk20a; + +int nvgpu_cg_pg_disable(struct gk20a *g); +int nvgpu_cg_pg_enable(struct gk20a *g); + +#endif /*NVGPU_POWER_FEATURES_H*/ diff --git a/drivers/gpu/nvgpu/os/linux/driver_common.c b/drivers/gpu/nvgpu/os/linux/driver_common.c index 852ab329b..6f9ffd75e 100644 --- a/drivers/gpu/nvgpu/os/linux/driver_common.c +++ b/drivers/gpu/nvgpu/os/linux/driver_common.c @@ -65,6 +65,7 @@ static void nvgpu_init_vars(struct gk20a *g) nvgpu_mutex_init(&g->ctxsw_disable_lock); nvgpu_mutex_init(&g->tpc_pg_lock); nvgpu_mutex_init(&g->clk_arb_enable_lock); + nvgpu_mutex_init(&g->cg_pg_lock); /* Init the clock req count to 0 */ nvgpu_atomic_set(&g->clk_arb_global_nr, 0); diff --git a/drivers/gpu/nvgpu/os/linux/sysfs.c b/drivers/gpu/nvgpu/os/linux/sysfs.c index 3187b65a5..e8b0bd531 100644 --- a/drivers/gpu/nvgpu/os/linux/sysfs.c +++ b/drivers/gpu/nvgpu/os/linux/sysfs.c @@ -24,6 +24,8 @@ #include #include #include +#include +#include #include "os_linux.h" #include "sysfs.h" @@ -48,20 +50,19 @@ static ssize_t elcg_enable_store(struct device *dev, return -EINVAL; err = gk20a_busy(g); - if (err) + if (err) { return err; + } if (val) { - g->elcg_enabled = true; - gr_gk20a_init_cg_mode(g, ELCG_MODE, ELCG_AUTO); + nvgpu_cg_elcg_set_elcg_enabled(g, true); } else { - g->elcg_enabled = false; - gr_gk20a_init_cg_mode(g, ELCG_MODE, ELCG_RUN); + nvgpu_cg_elcg_set_elcg_enabled(g, false); } gk20a_idle(g); - nvgpu_info(g, "ELCG is %s.", g->elcg_enabled ? "enabled" : + nvgpu_info(g, "ELCG is %s.", val ? "enabled" : "disabled"); return count; @@ -87,45 +88,21 @@ static ssize_t blcg_enable_store(struct device *dev, if (kstrtoul(buf, 10, &val) < 0) return -EINVAL; - if (val) - g->blcg_enabled = true; - else - g->blcg_enabled = false; - err = gk20a_busy(g); - if (err) + if (err) { return err; + } + + if (val) { + nvgpu_cg_blcg_set_blcg_enabled(g, true); + } else { + nvgpu_cg_blcg_set_blcg_enabled(g, false); + } + - if (g->ops.clock_gating.blcg_bus_load_gating_prod) - g->ops.clock_gating.blcg_bus_load_gating_prod(g, - g->blcg_enabled); - if (g->ops.clock_gating.blcg_ce_load_gating_prod) - g->ops.clock_gating.blcg_ce_load_gating_prod(g, - g->blcg_enabled); - if (g->ops.clock_gating.blcg_ctxsw_firmware_load_gating_prod) - g->ops.clock_gating.blcg_ctxsw_firmware_load_gating_prod(g, - g->blcg_enabled); - if (g->ops.clock_gating.blcg_fb_load_gating_prod) - g->ops.clock_gating.blcg_fb_load_gating_prod(g, - g->blcg_enabled); - if (g->ops.clock_gating.blcg_fifo_load_gating_prod) - g->ops.clock_gating.blcg_fifo_load_gating_prod(g, - g->blcg_enabled); - if (g->ops.clock_gating.blcg_gr_load_gating_prod) - g->ops.clock_gating.blcg_gr_load_gating_prod(g, - g->blcg_enabled); - if (g->ops.clock_gating.blcg_ltc_load_gating_prod) - g->ops.clock_gating.blcg_ltc_load_gating_prod(g, - g->blcg_enabled); - if (g->ops.clock_gating.blcg_pmu_load_gating_prod) - g->ops.clock_gating.blcg_pmu_load_gating_prod(g, - g->blcg_enabled); - if (g->ops.clock_gating.blcg_xbar_load_gating_prod) - g->ops.clock_gating.blcg_xbar_load_gating_prod(g, - g->blcg_enabled); gk20a_idle(g); - nvgpu_info(g, "BLCG is %s.", g->blcg_enabled ? "enabled" : + nvgpu_info(g, "BLCG is %s.", val ? "enabled" : "disabled"); return count; @@ -152,59 +129,26 @@ static ssize_t slcg_enable_store(struct device *dev, if (kstrtoul(buf, 10, &val) < 0) return -EINVAL; - if (val) - g->slcg_enabled = true; - else - g->slcg_enabled = false; + err = gk20a_busy(g); + if (err) { + return err; + } + + if (val) { + nvgpu_cg_slcg_set_slcg_enabled(g, true); + } else { + nvgpu_cg_slcg_set_slcg_enabled(g, false); + } /* * TODO: slcg_therm_load_gating is not enabled anywhere during * init. Therefore, it would be incongruous to add it here. Once * it is added to init, we should add it here too. */ - err = gk20a_busy(g); - if (err) - return err; - if (g->ops.clock_gating.slcg_bus_load_gating_prod) - g->ops.clock_gating.slcg_bus_load_gating_prod(g, - g->slcg_enabled); - if (g->ops.clock_gating.slcg_ce2_load_gating_prod) - g->ops.clock_gating.slcg_ce2_load_gating_prod(g, - g->slcg_enabled); - if (g->ops.clock_gating.slcg_chiplet_load_gating_prod) - g->ops.clock_gating.slcg_chiplet_load_gating_prod(g, - g->slcg_enabled); - if (g->ops.clock_gating.slcg_ctxsw_firmware_load_gating_prod) - g->ops.clock_gating.slcg_ctxsw_firmware_load_gating_prod(g, - g->slcg_enabled); - if (g->ops.clock_gating.slcg_fb_load_gating_prod) - g->ops.clock_gating.slcg_fb_load_gating_prod(g, - g->slcg_enabled); - if (g->ops.clock_gating.slcg_fifo_load_gating_prod) - g->ops.clock_gating.slcg_fifo_load_gating_prod(g, - g->slcg_enabled); - if (g->ops.clock_gating.slcg_gr_load_gating_prod) - g->ops.clock_gating.slcg_gr_load_gating_prod(g, - g->slcg_enabled); - if (g->ops.clock_gating.slcg_ltc_load_gating_prod) - g->ops.clock_gating.slcg_ltc_load_gating_prod(g, - g->slcg_enabled); - if (g->ops.clock_gating.slcg_perf_load_gating_prod) - g->ops.clock_gating.slcg_perf_load_gating_prod(g, - g->slcg_enabled); - if (g->ops.clock_gating.slcg_priring_load_gating_prod) - g->ops.clock_gating.slcg_priring_load_gating_prod(g, - g->slcg_enabled); - if (g->ops.clock_gating.slcg_pmu_load_gating_prod) - g->ops.clock_gating.slcg_pmu_load_gating_prod(g, - g->slcg_enabled); - if (g->ops.clock_gating.slcg_xbar_load_gating_prod) - g->ops.clock_gating.slcg_xbar_load_gating_prod(g, - g->slcg_enabled); gk20a_idle(g); - nvgpu_info(g, "SLCG is %s.", g->slcg_enabled ? "enabled" : + nvgpu_info(g, "SLCG is %s.", val ? "enabled" : "disabled"); return count; @@ -337,8 +281,9 @@ static ssize_t railgate_enable_store(struct device *dev, } /* wake-up system to make rail-gating setting effective */ err = gk20a_busy(g); - if (err) + if (err) { return err; + } gk20a_idle(g); nvgpu_info(g, "railgate is %s.", @@ -383,8 +328,9 @@ static ssize_t railgate_delay_store(struct device *dev, /* wake-up system to make rail-gating delay effective immediately */ err = gk20a_busy(g); - if (err) + if (err) { return err; + } gk20a_idle(g); return count; @@ -452,8 +398,9 @@ static ssize_t gk20a_load_show(struct device *dev, busy_time = 0; } else { err = gk20a_busy(g); - if (err) + if (err) { return err; + } nvgpu_pmu_load_update(g); nvgpu_pmu_load_norm(g, &busy_time); @@ -477,36 +424,22 @@ static ssize_t elpg_enable_store(struct device *dev, return -EINVAL; if (!g->power_on) { - g->elpg_enabled = val ? true : false; + return -EAGAIN; } else { err = gk20a_busy(g); - if (err) + if (err != 0) { return -EAGAIN; - /* - * Since elpg is refcounted, we should not unnecessarily call - * enable/disable if it is already so. - */ - if (val && !g->elpg_enabled) { - g->elpg_enabled = true; - nvgpu_pmu_pg_global_enable(g, true); - - } else if (!val && g->elpg_enabled) { - if (g->ops.pmu.pmu_pg_engines_feature_list && - g->ops.pmu.pmu_pg_engines_feature_list(g, - PMU_PG_ELPG_ENGINE_ID_GRAPHICS) != - NVGPU_PMU_GR_FEATURE_MASK_POWER_GATING) { - nvgpu_pmu_pg_global_enable(g, false); - g->elpg_enabled = false; - } else { - g->elpg_enabled = false; - nvgpu_pmu_pg_global_enable(g, false); - } + } + if (val != 0) { + nvgpu_pg_elpg_set_elpg_enabled(g, true); + } else { + nvgpu_pg_elpg_set_elpg_enabled(g, false); } gk20a_idle(g); - } - nvgpu_info(g, "ELPG is %s.", g->elpg_enabled ? "enabled" : - "disabled"); + nvgpu_info(g, "ELPG is %s.", val ? "enabled" : + "disabled"); + } return count; } @@ -515,7 +448,8 @@ static ssize_t elpg_enable_read(struct device *dev, { struct gk20a *g = get_gk20a(dev); - return snprintf(buf, PAGE_SIZE, "%d\n", g->elpg_enabled ? 1 : 0); + return snprintf(buf, PAGE_SIZE, "%d\n", + nvgpu_pg_elpg_is_enabled(g) ? 1 : 0); } static DEVICE_ATTR(elpg_enable, ROOTRW, elpg_enable_read, elpg_enable_store); @@ -544,8 +478,9 @@ static ssize_t ldiv_slowdown_factor_store(struct device *dev, g->ldiv_slowdown_factor = val; } else { err = gk20a_busy(g); - if (err) + if (err) { return -EAGAIN; + } g->ldiv_slowdown_factor = val; @@ -587,8 +522,9 @@ static ssize_t mscg_enable_store(struct device *dev, g->mscg_enabled = val ? true : false; } else { err = gk20a_busy(g); - if (err) + if (err) { return -EAGAIN; + } /* * Since elpg is refcounted, we should not unnecessarily call * enable/disable if it is already so. @@ -613,8 +549,9 @@ static ssize_t mscg_enable_store(struct device *dev, /* make status visible */ smp_mb(); g->mscg_enabled = false; - if (g->elpg_enabled) - nvgpu_pmu_pg_global_enable(g, true); + if (nvgpu_pg_elpg_is_enabled(g)) { + nvgpu_pg_elpg_enable(g); + } } g->mscg_enabled = false; } @@ -706,8 +643,9 @@ static ssize_t aelpg_enable_store(struct device *dev, return -EINVAL; err = gk20a_busy(g); - if (err) + if (err) { return err; + } if (g->pmu.pmu_ready) { if (val && !g->aelpg_enabled) { diff --git a/drivers/gpu/nvgpu/os/linux/vgpu/vgpu_linux.c b/drivers/gpu/nvgpu/os/linux/vgpu/vgpu_linux.c index e1a8a65cb..4ca043a52 100644 --- a/drivers/gpu/nvgpu/os/linux/vgpu/vgpu_linux.c +++ b/drivers/gpu/nvgpu/os/linux/vgpu/vgpu_linux.c @@ -78,6 +78,7 @@ static void vgpu_init_vars(struct gk20a *g, struct gk20a_platform *platform) nvgpu_mutex_init(&g->power_lock); nvgpu_mutex_init(&g->ctxsw_disable_lock); nvgpu_mutex_init(&g->clk_arb_enable_lock); + nvgpu_mutex_init(&g->cg_pg_lock); nvgpu_mutex_init(&priv->vgpu_clk_get_freq_lock);