diff --git a/drivers/gpu/nvgpu/common/gr/obj_ctx.c b/drivers/gpu/nvgpu/common/gr/obj_ctx.c index 76dda1636..69c1d206a 100644 --- a/drivers/gpu/nvgpu/common/gr/obj_ctx.c +++ b/drivers/gpu/nvgpu/common/gr/obj_ctx.c @@ -750,7 +750,7 @@ int nvgpu_gr_obj_ctx_alloc_golden_ctx_image(struct gk20a *g, golden_image->ready = true; #ifdef CONFIG_NVGPU_POWER_PG - nvgpu_pmu_set_golden_image_initialized(g, true); + nvgpu_pmu_set_golden_image_initialized(g, GOLDEN_IMG_READY); #endif g->ops.gr.falcon.set_current_ctx_invalid(g); @@ -984,7 +984,7 @@ void nvgpu_gr_obj_ctx_deinit(struct gk20a *g, golden_image->local_golden_image = NULL; } #ifdef CONFIG_NVGPU_POWER_PG - nvgpu_pmu_set_golden_image_initialized(g, false); + nvgpu_pmu_set_golden_image_initialized(g, GOLDEN_IMG_NOT_READY); #endif golden_image->ready = false; nvgpu_kfree(g, golden_image); diff --git a/drivers/gpu/nvgpu/common/init/nvgpu_init.c b/drivers/gpu/nvgpu/common/init/nvgpu_init.c index a7d831190..f254a535f 100644 --- a/drivers/gpu/nvgpu/common/init/nvgpu_init.c +++ b/drivers/gpu/nvgpu/common/init/nvgpu_init.c @@ -58,6 +58,10 @@ #include #endif +#ifdef CONFIG_NVGPU_POWER_PG +#include +#endif + bool is_nvgpu_gpu_state_valid(struct gk20a *g) { u32 boot_0 = g->ops.mc.get_chip_details(g, NULL, NULL, NULL); @@ -916,6 +920,10 @@ int nvgpu_finalize_poweron(struct gk20a *g) NVGPU_INIT_TABLE_ENTRY(&nvgpu_init_syncpt_mem, NO_FLAG), #ifdef CONFIG_NVGPU_PROFILER NVGPU_INIT_TABLE_ENTRY(&nvgpu_pm_reservation_init, NO_FLAG), +#endif +#ifdef CONFIG_NVGPU_POWER_PG + NVGPU_INIT_TABLE_ENTRY(g->ops.pmu.pmu_restore_golden_img_state, + NO_FLAG), #endif NVGPU_INIT_TABLE_ENTRY(g->ops.channel.resume_all_serviceable_ch, NO_FLAG), diff --git a/drivers/gpu/nvgpu/common/pmu/pg/pmu_pg.c b/drivers/gpu/nvgpu/common/pmu/pg/pmu_pg.c index e1cf29acb..5883e84b2 100644 --- a/drivers/gpu/nvgpu/common/pmu/pg/pmu_pg.c +++ b/drivers/gpu/nvgpu/common/pmu/pg/pmu_pg.c @@ -34,6 +34,7 @@ #include #include #include +#include #include "pg_sw_gm20b.h" #include "pg_sw_gv11b.h" @@ -273,7 +274,8 @@ int nvgpu_pmu_enable_elpg(struct gk20a *g) /* do NOT enable elpg until golden ctx is created, * which is related with the ctx that ELPG save and restore. */ - if (unlikely(!pmu->pg->golden_image_initialized)) { + if (unlikely((nvgpu_atomic_read(&pmu->pg->golden_image_initialized)) != + GOLDEN_IMG_READY)) { goto exit_unlock; } @@ -612,7 +614,8 @@ int nvgpu_pmu_enable_elpg_ms(struct gk20a *g) } /* do NOT enable elpg_ms until golden ctx is created */ - if (unlikely(!pmu->pg->golden_image_initialized)) { + if (unlikely((nvgpu_atomic_read(&pmu->pg->golden_image_initialized)) != + GOLDEN_IMG_READY)) { goto exit_unlock; } @@ -962,13 +965,25 @@ int nvgpu_pmu_pg_sw_setup(struct gk20a *g, struct nvgpu_pmu *pmu, pg->elpg_refcnt = 0; pg->elpg_ms_refcnt = 0; - /* skip seq_buf alloc during unrailgate path */ + /* During un-railgate path, skip seq_buf alloc + * and do not update golden_image_initialized flag + * in un-railgate path. + */ if (!nvgpu_mem_is_valid(&pg->seq_buf)) { err = pmu_pg_init_seq_buf(g, pmu, pg); if (err != 0) { nvgpu_err(g, "failed to allocate memory"); return err; } + + /* + * During first boot set golden_image_intialized + * to not_ready. + * This will set to ready state after golden + * ctx is created. + */ + nvgpu_atomic_set(&pg->golden_image_initialized, + GOLDEN_IMG_NOT_READY); } if (nvgpu_thread_is_running(&pg->pg_init.state_task)) { @@ -1001,6 +1016,17 @@ void nvgpu_pmu_pg_destroy(struct gk20a *g, struct nvgpu_pmu *pmu, g->pg_ingating_time_us += (u64)pg_stat_data.ingating_time; g->pg_ungating_time_us += (u64)pg_stat_data.ungating_time; g->pg_gating_cnt += pg_stat_data.gating_cnt; + /* + * if golden image is ready then set the + * golden_image_initialized to suspended state as + * part of railgate sequence. This will be set to + * ready in un-railgate sequence. + */ + if (nvgpu_atomic_read(&pg->golden_image_initialized) == + GOLDEN_IMG_READY) { + nvgpu_atomic_set(&pg->golden_image_initialized, + GOLDEN_IMG_SUSPEND); + } pg->zbc_ready = false; } @@ -1097,7 +1123,7 @@ void nvgpu_pmu_pg_deinit(struct gk20a *g, struct nvgpu_pmu *pmu, nvgpu_kfree(g, pg); } -void nvgpu_pmu_set_golden_image_initialized(struct gk20a *g, bool initialized) +void nvgpu_pmu_set_golden_image_initialized(struct gk20a *g, u8 state) { struct nvgpu_pmu *pmu = g->pmu; @@ -1105,7 +1131,7 @@ void nvgpu_pmu_set_golden_image_initialized(struct gk20a *g, bool initialized) return; } - pmu->pg->golden_image_initialized = initialized; + nvgpu_atomic_set(&pmu->pg->golden_image_initialized, state); } int nvgpu_pmu_elpg_statistics(struct gk20a *g, u32 pg_engine_id, @@ -1168,3 +1194,31 @@ void *nvgpu_pmu_pg_buf_get_cpu_va(struct gk20a *g, struct nvgpu_pmu *pmu) return pmu->pg->pg_buf.cpu_va; } + +int nvgpu_pmu_restore_golden_img_state(struct gk20a *g) +{ + struct nvgpu_pmu *pmu = g->pmu; + int err = 0; + + if (!is_pg_supported(g, pmu->pg)) { + goto out; + } + + if (nvgpu_atomic_read(&pmu->pg->golden_image_initialized) == + GOLDEN_IMG_SUSPEND) { + /* + * this becomes part of un-railgate sequence. + * set the golden_image_initialized to ready state + * and re-enable elpg. + */ + nvgpu_atomic_set(&pmu->pg->golden_image_initialized, + GOLDEN_IMG_READY); + err = nvgpu_pmu_reenable_elpg(g); + if (err != 0) { + nvgpu_err(g, "fail to re-enable elpg"); + goto out; + } + } +out: + return err; +} diff --git a/drivers/gpu/nvgpu/hal/init/hal_ga100.c b/drivers/gpu/nvgpu/hal/init/hal_ga100.c index 9ceef924e..b1450b755 100644 --- a/drivers/gpu/nvgpu/hal/init/hal_ga100.c +++ b/drivers/gpu/nvgpu/hal/init/hal_ga100.c @@ -69,7 +69,9 @@ #include #include #include - +#ifdef CONFIG_NVGPU_POWER_PG +#include +#endif #include "hal/mm/mm_gm20b.h" #include "hal/mm/mm_gp10b.h" #include "hal/mm/mm_gv11b.h" @@ -1278,6 +1280,9 @@ static const struct gops_pmu ga100_ops_pmu = { .ecc_free = gv11b_pmu_ecc_free, /* Init */ .pmu_early_init = nvgpu_pmu_early_init, +#ifdef CONFIG_NVGPU_POWER_PG + .pmu_restore_golden_img_state = nvgpu_pmu_restore_golden_img_state, +#endif .pmu_rtos_init = nvgpu_pmu_rtos_init, .pmu_pstate_sw_setup = nvgpu_pmu_pstate_sw_setup, .pmu_pstate_pmu_setup = nvgpu_pmu_pstate_pmu_setup, diff --git a/drivers/gpu/nvgpu/hal/init/hal_ga10b.c b/drivers/gpu/nvgpu/hal/init/hal_ga10b.c index cb3c5bff0..045492491 100644 --- a/drivers/gpu/nvgpu/hal/init/hal_ga10b.c +++ b/drivers/gpu/nvgpu/hal/init/hal_ga10b.c @@ -47,6 +47,9 @@ #include #endif #include +#ifdef CONFIG_NVGPU_POWER_PG +#include +#endif #include "hal/mm/mm_gp10b.h" #include "hal/mm/mm_gv11b.h" @@ -1303,6 +1306,9 @@ static const struct gops_pmu ga10b_ops_pmu = { */ /* Basic init ops */ .pmu_early_init = nvgpu_pmu_early_init, +#ifdef CONFIG_NVGPU_POWER_PG + .pmu_restore_golden_img_state = nvgpu_pmu_restore_golden_img_state, +#endif .is_pmu_supported = ga10b_is_pmu_supported, .falcon_base_addr = gv11b_pmu_falcon_base_addr, .falcon2_base_addr = ga10b_pmu_falcon2_base_addr, diff --git a/drivers/gpu/nvgpu/hal/init/hal_gm20b.c b/drivers/gpu/nvgpu/hal/init/hal_gm20b.c index bbbfe057a..848a645f4 100644 --- a/drivers/gpu/nvgpu/hal/init/hal_gm20b.c +++ b/drivers/gpu/nvgpu/hal/init/hal_gm20b.c @@ -56,6 +56,9 @@ #include #include #include +#ifdef CONFIG_NVGPU_POWER_PG +#include +#endif #include "hal/mm/mm_gk20a.h" #include "hal/mm/mm_gm20b.h" @@ -816,6 +819,9 @@ static const struct gops_pmu gm20b_ops_pmu = { .pmu_pstate_pmu_setup = nvgpu_pmu_pstate_pmu_setup, .pmu_destroy = nvgpu_pmu_destroy, .pmu_early_init = nvgpu_pmu_early_init, +#ifdef CONFIG_NVGPU_POWER_PG + .pmu_restore_golden_img_state = nvgpu_pmu_restore_golden_img_state, +#endif .pmu_rtos_init = nvgpu_pmu_rtos_init, .is_pmu_supported = gm20b_is_pmu_supported, .falcon_base_addr = gk20a_pmu_falcon_base_addr, diff --git a/drivers/gpu/nvgpu/hal/init/hal_gv11b.c b/drivers/gpu/nvgpu/hal/init/hal_gv11b.c index f731fde76..1a837727c 100644 --- a/drivers/gpu/nvgpu/hal/init/hal_gv11b.c +++ b/drivers/gpu/nvgpu/hal/init/hal_gv11b.c @@ -43,6 +43,9 @@ #include #endif #include +#ifdef CONFIG_NVGPU_POWER_PG +#include +#endif #include "hal/mm/mm_gp10b.h" #include "hal/mm/mm_gv11b.h" @@ -1115,6 +1118,9 @@ static const struct gops_pmu gv11b_ops_pmu = { */ /* Basic init ops */ .pmu_early_init = nvgpu_pmu_early_init, +#ifdef CONFIG_NVGPU_POWER_PG + .pmu_restore_golden_img_state = nvgpu_pmu_restore_golden_img_state, +#endif .is_pmu_supported = gv11b_is_pmu_supported, .falcon_base_addr = gv11b_pmu_falcon_base_addr, .pmu_reset = nvgpu_pmu_reset, diff --git a/drivers/gpu/nvgpu/include/nvgpu/gops/pmu.h b/drivers/gpu/nvgpu/include/nvgpu/gops/pmu.h index 29e82cc0c..0a574cffd 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/gops/pmu.h +++ b/drivers/gpu/nvgpu/include/nvgpu/gops/pmu.h @@ -104,6 +104,9 @@ struct gops_pmu { */ int (*pmu_early_init)(struct gk20a *g); +#ifdef CONFIG_NVGPU_POWER_PG + int (*pmu_restore_golden_img_state)(struct gk20a *g); +#endif /** @cond DOXYGEN_SHOULD_SKIP_THIS */ #ifdef CONFIG_NVGPU_LS_PMU diff --git a/drivers/gpu/nvgpu/include/nvgpu/pmu/pmu_pg.h b/drivers/gpu/nvgpu/include/nvgpu/pmu/pmu_pg.h index f8bbd547a..980658acd 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/pmu/pmu_pg.h +++ b/drivers/gpu/nvgpu/include/nvgpu/pmu/pmu_pg.h @@ -63,6 +63,13 @@ struct rpc_handler_payload; #define APCTRL_POWER_BREAKEVEN_DEFAULT_US (2000U) #define APCTRL_CYCLES_PER_SAMPLE_MAX_DEFAULT (200U) +/* State of golden image */ +enum { + GOLDEN_IMG_NOT_READY = 0, + GOLDEN_IMG_SUSPEND, + GOLDEN_IMG_READY, +}; + struct nvgpu_pg_init { bool state_change; bool state_destroy; @@ -90,7 +97,7 @@ struct nvgpu_pmu_pg { bool initialized; u32 stat_dmem_offset[PMU_PG_ELPG_ENGINE_ID_INVALID_ENGINE]; struct nvgpu_mem seq_buf; - bool golden_image_initialized; + nvgpu_atomic_t golden_image_initialized; u32 mscg_stat; u32 mscg_transition_state; int (*elpg_statistics)(struct gk20a *g, u32 pg_engine_id, @@ -142,6 +149,7 @@ int nvgpu_pmu_pg_sw_setup(struct gk20a *g, struct nvgpu_pmu *pmu, struct nvgpu_pmu_pg *pg); void nvgpu_pmu_pg_destroy(struct gk20a *g, struct nvgpu_pmu *pmu, struct nvgpu_pmu_pg *pg); +int nvgpu_pmu_restore_golden_img_state(struct gk20a *g); /* PG enable/disable */ int nvgpu_pmu_reenable_elpg(struct gk20a *g); @@ -161,7 +169,7 @@ int nvgpu_aelpg_init_and_enable(struct gk20a *g, u8 ctrl_id); int nvgpu_pmu_ap_send_command(struct gk20a *g, union pmu_ap_cmd *p_ap_cmd, bool b_block); -void nvgpu_pmu_set_golden_image_initialized(struct gk20a *g, bool initialized); +void nvgpu_pmu_set_golden_image_initialized(struct gk20a *g, u8 state); /* PG ops*/ int nvgpu_pmu_elpg_statistics(struct gk20a *g, u32 pg_engine_id,