diff --git a/drivers/gpu/nvgpu/common/pmu/ipc/pmu_msg.c b/drivers/gpu/nvgpu/common/pmu/ipc/pmu_msg.c index 7bed652f8..0396c7650 100644 --- a/drivers/gpu/nvgpu/common/pmu/ipc/pmu_msg.c +++ b/drivers/gpu/nvgpu/common/pmu/ipc/pmu_msg.c @@ -585,7 +585,7 @@ static void pmu_rpc_handler(struct gk20a *g, struct pmu_msg *msg, case PMU_UNIT_PG_LOADING: case PMU_UNIT_PG: if (pmu->pg->rpc_handler != NULL) { - pmu->pg->rpc_handler(g, pmu, &rpc); + pmu->pg->rpc_handler(g, pmu, &rpc, rpc_payload); } break; default: diff --git a/drivers/gpu/nvgpu/common/pmu/pg/pg_sw_ga10b.c b/drivers/gpu/nvgpu/common/pmu/pg/pg_sw_ga10b.c index ed7a558d1..e396c11d2 100644 --- a/drivers/gpu/nvgpu/common/pmu/pg/pg_sw_ga10b.c +++ b/drivers/gpu/nvgpu/common/pmu/pg/pg_sw_ga10b.c @@ -292,10 +292,12 @@ static int ga10b_pmu_pg_load_buff(struct gk20a *g, struct nvgpu_pmu *pmu) } static void ga10b_pg_rpc_handler(struct gk20a *g, struct nvgpu_pmu *pmu, - struct nv_pmu_rpc_header *rpc) + struct nv_pmu_rpc_header *rpc, struct rpc_handler_payload *rpc_payload) { - nvgpu_log_fn(g, " "); + struct pmu_rpc_struct_lpwr_pg_ctrl_allow *rpc_allow; + struct pmu_rpc_struct_lpwr_pg_ctrl_disallow *rpc_disallow; + nvgpu_log_fn(g, " "); switch (rpc->function) { case NV_PMU_RPC_ID_PG_LOADING_PRE_INIT: nvgpu_pmu_dbg(g, "Reply to PG_PRE_INIT"); @@ -320,11 +322,25 @@ static void ga10b_pg_rpc_handler(struct gk20a *g, struct nvgpu_pmu *pmu, break; case NV_PMU_RPC_ID_PG_ALLOW: nvgpu_pmu_dbg(g, "Reply to PG_ALLOW"); + rpc_allow = (struct pmu_rpc_struct_lpwr_pg_ctrl_allow *)rpc_payload->rpc_buff; + if (rpc_allow->ctrl_id == PMU_PG_ELPG_ENGINE_ID_GRAPHICS) { pmu->pg->elpg_stat = PMU_ELPG_STAT_ON; + } else if (rpc_allow->ctrl_id == PMU_PG_ELPG_ENGINE_ID_MS_LTC) { + pmu->pg->elpg_ms_stat = PMU_ELPG_MS_STAT_ON; + } else { + nvgpu_err(g, "Invalid pg_engine_id"); + } break; case NV_PMU_RPC_ID_PG_DISALLOW: nvgpu_pmu_dbg(g, "Reply to PG_DISALLOW"); + rpc_disallow = (struct pmu_rpc_struct_lpwr_pg_ctrl_disallow *)(void *)rpc_payload->rpc_buff; + if (rpc_disallow->ctrl_id == PMU_PG_ELPG_ENGINE_ID_GRAPHICS) { pmu->pg->elpg_stat = PMU_ELPG_STAT_OFF; + } else if (rpc_disallow->ctrl_id == PMU_PG_ELPG_ENGINE_ID_MS_LTC) { + pmu->pg->elpg_ms_stat = PMU_ELPG_MS_STAT_OFF; + } else { + nvgpu_err(g, "Invalid pg_engine_id"); + } break; default: nvgpu_err(g, diff --git a/drivers/gpu/nvgpu/common/pmu/pg/pmu_pg.c b/drivers/gpu/nvgpu/common/pmu/pg/pmu_pg.c index b779c65fb..ca5f9dc79 100644 --- a/drivers/gpu/nvgpu/common/pmu/pg/pmu_pg.c +++ b/drivers/gpu/nvgpu/common/pmu/pg/pmu_pg.c @@ -294,12 +294,17 @@ int nvgpu_pmu_enable_elpg(struct gk20a *g) pmu->pg->mscg_stat == PMU_MSCG_DISABLED) { continue; } - if ((BIT32(pg_engine_id) & pg_engine_id_list) != 0U) { - ret = pmu_enable_elpg_locked(g, pg_engine_id); + if (pg_engine_id == PMU_PG_ELPG_ENGINE_ID_GRAPHICS) { + ret = pmu_enable_elpg_locked(g, pg_engine_id); + } else if (pg_engine_id == PMU_PG_ELPG_ENGINE_ID_MS_LTC) { + ret = nvgpu_pmu_enable_elpg_ms(g); + } else { + ret = -EINVAL; + nvgpu_err(g, "Inavlid pg_engine_id"); + } } } - exit_unlock: nvgpu_mutex_release(&pmu->pg->elpg_mutex); nvgpu_log_fn(g, "done"); @@ -399,6 +404,12 @@ int nvgpu_pmu_disable_elpg(struct gk20a *g) pmu->pg->mscg_transition_state = PMU_ELPG_STAT_OFF_PENDING; } + + if (pg_engine_id == PMU_PG_ELPG_ENGINE_ID_MS_LTC) { + ret = nvgpu_pmu_disable_elpg_ms(g); + continue; + } + if (pg_engine_id == PMU_PG_ELPG_ENGINE_ID_GRAPHICS) { ptr = &pmu->pg->elpg_stat; } else if (pg_engine_id == PMU_PG_ELPG_ENGINE_ID_MS) { @@ -468,6 +479,148 @@ exit: return ret; } +int nvgpu_pmu_disable_elpg_ms(struct gk20a *g) +{ + struct nvgpu_pmu *pmu = g->pmu; + int ret = 0; + u32 *ptr = NULL; + + nvgpu_log_fn(g, " "); + + if (!is_pg_supported(g, pmu->pg)) { + return ret; + } + + nvgpu_mutex_acquire(&pmu->pg->elpg_ms_mutex); + + pmu->pg->elpg_ms_refcnt = nvgpu_safe_sub_s32( + pmu->pg->elpg_ms_refcnt, 1); + if (pmu->pg->elpg_ms_refcnt > 0) { + nvgpu_warn(g, + "%s(): possible elpg_ms refcnt mismatch. elpg_ms refcnt=%d", + __func__, pmu->pg->elpg_ms_refcnt); + WARN_ON(true); + ret = 0; + goto exit_unlock; + } + + /* cancel off_on_pending and return */ + if (pmu->pg->elpg_ms_stat == PMU_ELPG_MS_STAT_OFF_ON_PENDING) { + pmu->pg->elpg_ms_stat = PMU_ELPG_MS_STAT_OFF; + ret = 0; + goto exit_unlock; + } else if (pmu->pg->elpg_ms_stat == PMU_ELPG_MS_STAT_ON_PENDING) { + /* wait if on_pending */ + pmu_wait_message_cond(pmu, nvgpu_get_poll_timeout(g), + &pmu->pg->elpg_ms_stat, PMU_ELPG_MS_STAT_ON); + + if (pmu->pg->elpg_ms_stat != PMU_ELPG_MS_STAT_ON) { + nvgpu_err(g, "ELPG_MS_ALLOW_ACK failed, elpg_ms_stat=%d", + pmu->pg->elpg_ms_stat); + pmu_dump_elpg_stats(pmu); + nvgpu_pmu_dump_falcon_stats(pmu); + ret = -EBUSY; + goto exit_unlock; + } + } else if (pmu->pg->elpg_ms_stat != PMU_ELPG_MS_STAT_ON) { + /* return if ELPG_MS is already off */ + ret = 0; + nvgpu_err(g, "ELPG_MS already disabled"); + goto exit_unlock; + } else { + pmu->pg->elpg_ms_stat = PMU_ELPG_MS_STAT_OFF_PENDING; + ptr = &pmu->pg->elpg_ms_stat; + + nvgpu_pmu_dbg(g, "cmd post PMU_PG_ELPG_CMD_DISALLOW for MS_LTC"); + if (pmu->pg->disallow == NULL) { + nvgpu_err(g, + "PG disallow function not assigned"); + ret = -EINVAL; + goto exit_unlock; + } + ret = pmu->pg->disallow(g, pmu, PMU_PG_ELPG_ENGINE_ID_MS_LTC); + if (ret != 0) { + nvgpu_err(g, "PMU_PG_ELPG_CMD_DISALLOW " + "cmd post for MS_LTC failed"); + goto exit_unlock; + } + + pmu_wait_message_cond(pmu, + nvgpu_get_poll_timeout(g), + ptr, PMU_ELPG_MS_STAT_OFF); + if (*ptr != PMU_ELPG_MS_STAT_OFF) { + nvgpu_err(g, "ELPG_MS_DISALLOW_ACK failed"); + pmu_dump_elpg_stats(pmu); + nvgpu_pmu_dump_falcon_stats(pmu); + ret = -EBUSY; + goto exit_unlock; + } + } +exit_unlock: + nvgpu_mutex_release(&pmu->pg->elpg_ms_mutex); + nvgpu_log_fn(g, "done"); + return ret; +} + +int nvgpu_pmu_enable_elpg_ms(struct gk20a *g) +{ + struct nvgpu_pmu *pmu = g->pmu; + int status = 0; + + nvgpu_log_fn(g, " "); + + if (!is_pg_supported(g, g->pmu->pg)) { + return status; + } + + nvgpu_mutex_acquire(&pmu->pg->elpg_ms_mutex); + + pmu->pg->elpg_ms_refcnt = nvgpu_safe_add_s32( + pmu->pg->elpg_ms_refcnt, 1); + if (pmu->pg->elpg_ms_refcnt <= 0) { + goto exit_unlock; + } + + /* something is not right if we end up in following code path */ + if (unlikely(pmu->pg->elpg_ms_refcnt > 1)) { + nvgpu_warn(g, + "%s(): possible elpg_ms_refcnt mismatch.elpg_ms refcnt=%d", + __func__, pmu->pg->elpg_ms_refcnt); + WARN_ON(true); + } + + /* do NOT enable elpg_ms until golden ctx is created */ + if (unlikely(!pmu->pg->golden_image_initialized)) { + goto exit_unlock; + } + + if (pmu->pg->elpg_ms_stat != PMU_ELPG_MS_STAT_OFF) { + nvgpu_err(g, "ELPG_MS already enabled"); + } + + pmu->pg->elpg_ms_stat = PMU_ELPG_MS_STAT_ON_PENDING; + + nvgpu_pmu_dbg(g, "cmd post PMU_PG_ELPG_CMD_ALLOW for MS_LTC"); + if (pmu->pg->allow == NULL) { + nvgpu_err(g, "PG allow function not assigned"); + status = -EINVAL; + goto exit_unlock; + } + status = pmu->pg->allow(g, pmu, PMU_PG_ELPG_ENGINE_ID_MS_LTC); + + if (status != 0) { + nvgpu_log_fn(g, "PG allow for MS_LTC FAILED err=%d", + status); + } else { + nvgpu_log_fn(g, "done"); + } + +exit_unlock: + nvgpu_mutex_release(&pmu->pg->elpg_ms_mutex); + nvgpu_log_fn(g, "done"); + return status; +} + /* PG init */ void pmu_handle_pg_stat_msg(struct gk20a *g, struct pmu_msg *msg, void *param, u32 status) @@ -789,8 +942,9 @@ int nvgpu_pmu_pg_sw_setup(struct gk20a *g, struct nvgpu_pmu *pmu, return 0; } - /* start with elpg disabled until first enable call */ + /* start with elpg and elpg_ms disabled until first enable call */ pg->elpg_refcnt = 0; + pg->elpg_ms_refcnt = 0; /* skip seq_buf alloc during unrailgate path */ if (!nvgpu_mem_is_valid(&pg->seq_buf)) { diff --git a/drivers/gpu/nvgpu/common/pmu/pg/pmu_pg.h b/drivers/gpu/nvgpu/common/pmu/pg/pmu_pg.h index 8b5a7dcea..f669fa335 100644 --- a/drivers/gpu/nvgpu/common/pmu/pg/pmu_pg.h +++ b/drivers/gpu/nvgpu/common/pmu/pg/pmu_pg.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2020-2021, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -40,6 +40,19 @@ */ #define PMU_ELPG_STAT_OFF_ON_PENDING 4U +/* elpg_ms is off */ +#define PMU_ELPG_MS_STAT_OFF 0U +/* elpg_ms is on */ +#define PMU_ELPG_MS_STAT_ON 1U +/* elpg_ms is off, ALLOW cmd has been sent, wait for ack */ +#define PMU_ELPG_MS_STAT_ON_PENDING 2U +/* elpg_ms is on, DISALLOW cmd has been sent, wait for ack */ +#define PMU_ELPG_MS_STAT_OFF_PENDING 3U +/* elpg_ms is off, caller has requested on, but ALLOW + * cmd hasn't been sent due to ENABLE_ALLOW delay + */ +#define PMU_ELPG_MS_STAT_OFF_ON_PENDING 4U + #define PMU_PGENG_GR_BUFFER_IDX_INIT 0U #define PMU_PGENG_GR_BUFFER_IDX_ZBC 1U #define PMU_PGENG_GR_BUFFER_IDX_FECS 2U diff --git a/drivers/gpu/nvgpu/common/pmu/pmu.c b/drivers/gpu/nvgpu/common/pmu/pmu.c index 0ea5d2563..2718aa77c 100644 --- a/drivers/gpu/nvgpu/common/pmu/pmu.c +++ b/drivers/gpu/nvgpu/common/pmu/pmu.c @@ -210,6 +210,7 @@ int nvgpu_pmu_early_init(struct gk20a *g) g->can_elpg = false; g->elpg_enabled = false; g->aelpg_enabled = false; + g->elpg_ms_enabled = false; nvgpu_set_enabled(g, NVGPU_PMU_PERFMON, false); nvgpu_set_enabled(g, NVGPU_ELPG_MS_ENABLED, false); #ifdef CONFIG_NVGPU_DGPU diff --git a/drivers/gpu/nvgpu/common/power_features/pg/pg.c b/drivers/gpu/nvgpu/common/power_features/pg/pg.c index dd1f68fda..a57a55735 100644 --- a/drivers/gpu/nvgpu/common/power_features/pg/pg.c +++ b/drivers/gpu/nvgpu/common/power_features/pg/pg.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2019-2021, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -116,3 +116,50 @@ done: nvgpu_mutex_release(&g->cg_pg_lock); return err; } + +int nvgpu_pg_elpg_ms_enable(struct gk20a *g) +{ + int err = 0; +#ifdef CONFIG_NVGPU_LS_PMU + nvgpu_log_fn(g, " "); + + if (!g->can_elpg) { + return 0; + } + + if (g->pmu->pg->initialized) { + g->ops.gr.init.wait_initialized(g); + + nvgpu_mutex_acquire(&g->cg_pg_lock); + if ((g->elpg_enabled) && (g->elpg_ms_enabled)) { + err = nvgpu_pmu_enable_elpg_ms(g); + } + nvgpu_mutex_release(&g->cg_pg_lock); + } +#endif + return err; +} + +int nvgpu_pg_elpg_ms_disable(struct gk20a *g) +{ + int err = 0; +#ifdef CONFIG_NVGPU_LS_PMU + nvgpu_log_fn(g, " "); + + if (!g->can_elpg) { + return 0; + } + + if (g->pmu->pg->initialized) { + g->ops.gr.init.wait_initialized(g); + + nvgpu_mutex_acquire(&g->cg_pg_lock); + if ((g->elpg_enabled) && (g->elpg_ms_enabled)) { + err = nvgpu_pmu_disable_elpg_ms(g); + } + nvgpu_mutex_release(&g->cg_pg_lock); + } +#endif + return err; +} + diff --git a/drivers/gpu/nvgpu/include/nvgpu/gk20a.h b/drivers/gpu/nvgpu/include/nvgpu/gk20a.h index 0d7207416..100c9e357 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/gk20a.h +++ b/drivers/gpu/nvgpu/include/nvgpu/gk20a.h @@ -556,6 +556,7 @@ struct gk20a { bool elcg_enabled; /** @cond DOXYGEN_SHOULD_SKIP_THIS */ bool elpg_enabled; + bool elpg_ms_enabled; bool aelpg_enabled; bool can_elpg; bool mscg_enabled; diff --git a/drivers/gpu/nvgpu/include/nvgpu/pmu/pmu_pg.h b/drivers/gpu/nvgpu/include/nvgpu/pmu/pmu_pg.h index b42e3a2f2..f67539dd0 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/pmu/pmu_pg.h +++ b/drivers/gpu/nvgpu/include/nvgpu/pmu/pmu_pg.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2020, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2019-2021, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -32,10 +32,12 @@ #include #include #include +#include struct nvgpu_pmu; struct vm_gk20a; struct pmu_pg_stats_data; +struct rpc_handler_payload; /*PG defines used by nvpgu-pmu*/ #define PMU_PG_SEQ_BUF_SIZE 4096U @@ -70,12 +72,15 @@ struct nvgpu_pg_init { struct nvgpu_pmu_pg { u32 elpg_stat; + u32 elpg_ms_stat; #define PMU_ELPG_ENABLE_ALLOW_DELAY_MSEC 1U /* msec */ struct nvgpu_pg_init pg_init; struct nvgpu_mutex pg_mutex; /* protect pg-RPPG/MSCG enable/disable */ struct nvgpu_mutex elpg_mutex; /* protect elpg enable/disable */ + struct nvgpu_mutex elpg_ms_mutex; /* protect elpg_ms enable/disable */ /* disable -1, enable +1, <=0 elpg disabled, > 0 elpg enabled */ int elpg_refcnt; + int elpg_ms_refcnt; u32 aelpg_param[5]; bool zbc_ready; bool zbc_save_done; @@ -113,7 +118,7 @@ struct nvgpu_pmu_pg { int (*load_buff)(struct gk20a *g, struct nvgpu_pmu *pmu); int (*hw_load_zbc)(struct gk20a *g, struct nvgpu_pmu *pmu); void (*rpc_handler)(struct gk20a *g, struct nvgpu_pmu *pmu, - struct nv_pmu_rpc_header *rpc); + struct nv_pmu_rpc_header *rpc, struct rpc_handler_payload *rpc_payload); int (*init_send)(struct gk20a *g, struct nvgpu_pmu *pmu, u8 pg_engine_id); }; @@ -140,6 +145,9 @@ void nvgpu_pmu_pg_destroy(struct gk20a *g, struct nvgpu_pmu *pmu, int nvgpu_pmu_reenable_elpg(struct gk20a *g); int nvgpu_pmu_enable_elpg(struct gk20a *g); int nvgpu_pmu_disable_elpg(struct gk20a *g); +int nvgpu_pmu_enable_elpg_ms(struct gk20a *g); +int nvgpu_pmu_disable_elpg_ms(struct gk20a *g); + int nvgpu_pmu_pg_global_enable(struct gk20a *g, bool enable_pg); int nvgpu_pmu_get_pg_stats(struct gk20a *g, u32 pg_engine_id, diff --git a/drivers/gpu/nvgpu/include/nvgpu/power_features/pg.h b/drivers/gpu/nvgpu/include/nvgpu/power_features/pg.h index ad7ec0491..23fac0f46 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/power_features/pg.h +++ b/drivers/gpu/nvgpu/include/nvgpu/power_features/pg.h @@ -42,12 +42,29 @@ struct gk20a; } \ err; \ }) + +#define nvgpu_pg_elpg_ms_protected_call(g, func) \ + ({ \ + int status = 0; \ + status = nvgpu_pg_elpg_ms_disable(g);\ + if (status != 0) {\ + (void)nvgpu_pg_elpg_ms_enable(g);\ + } \ + if (status == 0) { \ + status = (func); \ + (void)nvgpu_pg_elpg_ms_enable(g);\ + } \ + status; \ + }) #else #define nvgpu_pg_elpg_protected_call(g, func) func +#define nvgpu_pg_elpg_ms_protected_call(g, func) func #endif int nvgpu_pg_elpg_disable(struct gk20a *g); int nvgpu_pg_elpg_enable(struct gk20a *g); +int nvgpu_pg_elpg_ms_disable(struct gk20a *g); +int nvgpu_pg_elpg_ms_enable(struct gk20a *g); bool nvgpu_pg_elpg_is_enabled(struct gk20a *g); int nvgpu_pg_elpg_set_elpg_enabled(struct gk20a *g, bool enable); diff --git a/drivers/gpu/nvgpu/os/linux/driver_common.c b/drivers/gpu/nvgpu/os/linux/driver_common.c index f66f5935e..a65c19ab1 100644 --- a/drivers/gpu/nvgpu/os/linux/driver_common.c +++ b/drivers/gpu/nvgpu/os/linux/driver_common.c @@ -223,6 +223,7 @@ static void nvgpu_init_pm_vars(struct gk20a *g) if (g->elpg_enabled) { nvgpu_set_enabled(g, NVGPU_ELPG_MS_ENABLED, platform->enable_elpg_ms); + g->elpg_ms_enabled = platform->enable_elpg_ms; } }