gpu: nvgpu: MSCG support

- Added enable_mscg, mscg_enabled & mscg_stat flags,
  mscg_enabled flag can be used to controll
  mscg enable/disable at runtime along with mscg_stat flag.
- Added defines & interface to support ms/mclk-change/post-init-param
- Added defines for lpwr tables read  from vbios.
- HAL to support post init param which is require
  to setup clockgating interface in PMU & interfaces used during
  mscg state machine.
- gk20a_pmu_pg_global_enable() can be called when pg support
  required to enable/disable, this also checks & wait
  if pstate switch is in progress till it complets
- pg_mutex to protect PG-RPPG/MSCG enable/disable

JIRA DNVGPU-71

Change-Id: If312cefc888a4de0a5c96898baeaac1a76e53e46
Signed-off-by: Mahantesh Kumbar <mkumbar@nvidia.com>
Reviewed-on: http://git-master/r/1247554
(cherry picked from commit e6c94948b8058ba642ea56677ad798fc56b8a28a)
Reviewed-on: http://git-master/r/1270971
GVS: Gerrit_Virtual_Submit
Reviewed-by: Terje Bergstrom <tbergstrom@nvidia.com>

drivers/gpu/nvgpu/gk20a/dbg_gpu_gk20a.c

@@ -1169,8 +1169,8 @@ static int dbg_set_powergate(struct dbg_session_gk20a *dbg_s, u32  powermode)
 				return -EPERM;
 
 			/*do elpg disable before clock gating */
-			if (support_gk20a_pmu(g->dev))
+			gk20a_pmu_pg_global_enable(g, false);
-				gk20a_pmu_disable_elpg(g);
+
 			if (g->ops.clock_gating.slcg_gr_load_gating_prod)
 				g->ops.clock_gating.slcg_gr_load_gating_prod(g,
 						false);
@@ -1216,8 +1216,7 @@ static int dbg_set_powergate(struct dbg_session_gk20a *dbg_s, u32  powermode)
 				g->ops.clock_gating.slcg_gr_load_gating_prod(g,
 						g->slcg_enabled);
 
-			if (support_gk20a_pmu(g->dev))
+			gk20a_pmu_pg_global_enable(g, true);
-				gk20a_pmu_enable_elpg(g);
 
 			gk20a_dbg(gpu_dbg_gpu_dbg | gpu_dbg_fn, "module idle");
 			gk20a_idle(dbg_s->dev);

drivers/gpu/nvgpu/gk20a/gk20a.h

@@ -610,6 +610,9 @@ struct gpu_ops {
 		u32 (*pmu_pg_supported_engines_list)(struct gk20a *g);
 		u32 (*pmu_pg_engines_feature_list)(struct gk20a *g,
 			u32 pg_engine_id);
+		int (*pmu_lpwr_enable_pg)(struct gk20a *g, bool pstate_lock);
+		int (*pmu_lpwr_disable_pg)(struct gk20a *g, bool pstate_lock);
+		u32 (*pmu_pg_param_post_init)(struct gk20a *g);
 		int (*send_lrf_tex_ltc_dram_overide_en_dis_cmd)
 			(struct gk20a *g, u32 mask);
 		void (*dump_secure_fuses)(struct gk20a *g);
@@ -847,6 +850,7 @@ struct gk20a {
 	bool elcg_enabled;
 	bool elpg_enabled;
 	bool aelpg_enabled;
+	bool mscg_enabled;
 	bool forced_idle;
 	bool forced_reset;
 	bool allow_all;

drivers/gpu/nvgpu/gk20a/platform_gk20a.h

@@ -93,6 +93,9 @@ struct gk20a_platform {
 	/* Adaptative ELPG: true = enable flase = disable */
 	bool enable_aelpg;
 
+	/* Memory System Clock Gating: true = enable flase = disable*/
+	bool enable_mscg;
+
 	/* Timeout for per-channel watchdog (in mS) */
 	u32 ch_wdt_timeout_ms;
 

drivers/gpu/nvgpu/gk20a/pmu_api.h

@@ -526,17 +526,77 @@ enum {
 };
 
 #define PMU_PG_PARAM_CMD_GR_INIT_PARAM  0x0
+#define PMU_PG_PARAM_CMD_MS_INIT_PARAM  0x01
+#define PMU_PG_PARAM_CMD_MCLK_CHANGE  0x04
+#define PMU_PG_PARAM_CMD_POST_INIT  0x06
 
 #define PMU_PG_FEATURE_GR_SDIV_SLOWDOWN_ENABLED	(1 << 0)
 #define PMU_PG_FEATURE_GR_POWER_GATING_ENABLED	(1 << 2)
 #define PMU_PG_FEATURE_GR_RPPG_ENABLED		(1 << 3)
 
+#define NVGPU_PMU_GR_FEATURE_MASK_RPPG	(1 << 3)
+#define NVGPU_PMU_GR_FEATURE_MASK_ALL	\
+	(	\
+		NVGPU_PMU_GR_FEATURE_MASK_RPPG   \
+	)
+
+#define NVGPU_PMU_MS_FEATURE_MASK_CLOCK_GATING  (1 << 0)
+#define NVGPU_PMU_MS_FEATURE_MASK_SW_ASR        (1 << 1)
+#define NVGPU_PMU_MS_FEATURE_MASK_RPPG          (1 << 8)
+#define NVGPU_PMU_MS_FEATURE_MASK_FB_TRAINING   (1 << 5)
+
+#define NVGPU_PMU_MS_FEATURE_MASK_ALL	\
+	(	\
+		NVGPU_PMU_MS_FEATURE_MASK_CLOCK_GATING  |\
+		NVGPU_PMU_MS_FEATURE_MASK_SW_ASR        |\
+		NVGPU_PMU_MS_FEATURE_MASK_RPPG          |\
+		NVGPU_PMU_MS_FEATURE_MASK_FB_TRAINING   \
+	)
+
+#define PG_REQUEST_TYPE_GLOBAL 0x0
+#define PG_REQUEST_TYPE_PSTATE 0x1
+
 struct pmu_pg_cmd_gr_init_param {
 	u8 cmd_type;
 	u16 sub_cmd_id;
 	u8 featuremask;
 };
 
+struct pmu_pg_cmd_ms_init_param {
+	u8 cmd_type;
+	u16 cmd_id;
+	u8 psi;
+	u8 idle_flipped_test_enabled;
+	u16 psiSettleTimeUs;
+	u8 rsvd[2];
+	u32 support_mask;
+	u32 abort_timeout_us;
+};
+
+struct pmu_pg_cmd_mclk_change {
+	u8 cmd_type;
+	u16 cmd_id;
+	u8 rsvd;
+	u32 data;
+};
+
+#define PG_VOLT_RAIL_IDX_MAX 2
+
+struct pmu_pg_volt_rail {
+	u8    volt_rail_idx;
+	u8    sleep_volt_dev_idx;
+	u8    sleep_vfe_idx;
+	u32   sleep_voltage_uv;
+	u32   therm_vid0_cache;
+	u32   therm_vid1_cache;
+};
+
+struct pmu_pg_cmd_post_init_param {
+	u8 cmd_type;
+	u16 cmd_id;
+	struct pmu_pg_volt_rail pg_volt_rail[PG_VOLT_RAIL_IDX_MAX];
+};
+
 struct pmu_pg_cmd_stat {
 	u8 cmd_type;
 	u8 engine_id;
@@ -553,6 +613,9 @@ struct pmu_pg_cmd {
 		struct pmu_pg_cmd_eng_buf_load_v2 eng_buf_load_v2;
 		struct pmu_pg_cmd_stat stat;
 		struct pmu_pg_cmd_gr_init_param gr_init_param;
+		struct pmu_pg_cmd_ms_init_param ms_init_param;
+		struct pmu_pg_cmd_mclk_change mclk_change;
+		struct pmu_pg_cmd_post_init_param post_init;
 		/* TBD: other pg commands */
 		union pmu_ap_cmd ap_cmd;
 		struct nv_pmu_rppg_cmd rppg_cmd;

drivers/gpu/nvgpu/gk20a/pmu_gk20a.c

@@ -1364,6 +1364,7 @@ int gk20a_init_pmu(struct pmu_gk20a *pmu)
 	struct pmu_v *pv = &g->ops.pmu_ver;
 
 	mutex_init(&pmu->elpg_mutex);
+	mutex_init(&pmu->pg_mutex);
 	mutex_init(&pmu->isr_mutex);
 	mutex_init(&pmu->pmu_copy_lock);
 	mutex_init(&pmu->pmu_seq_lock);
@@ -3298,6 +3299,9 @@ void gk20a_init_pmu_ops(struct gpu_ops *gops)
 	gops->pmu.pmu_pg_init_param = NULL;
 	gops->pmu.pmu_pg_supported_engines_list = gk20a_pmu_pg_engines_list;
 	gops->pmu.pmu_pg_engines_feature_list = gk20a_pmu_pg_feature_list;
+	gops->pmu.pmu_lpwr_enable_pg = NULL;
+	gops->pmu.pmu_lpwr_disable_pg = NULL;
+	gops->pmu.pmu_pg_param_post_init = NULL;
 	gops->pmu.send_lrf_tex_ltc_dram_overide_en_dis_cmd = NULL;
 	gops->pmu.dump_secure_fuses = NULL;
 	gops->pmu.is_lazy_bootstrap = NULL;
@@ -3378,6 +3382,7 @@ static void pmu_handle_pg_elpg_msg(struct gk20a *g, struct pmu_msg *msg,
 				PMU_PG_FEATURE_GR_POWER_GATING_ENABLED) {
 				pmu->initialized = true;
 				pmu->pmu_state = PMU_STATE_STARTED;
+				pmu->mscg_stat = PMU_MSCG_DISABLED;
 			} else
 				schedule_work(&pmu->pg_init);
 		}
@@ -3506,6 +3511,9 @@ static int pmu_init_powergating(struct gk20a *g)
 		}
 	}
 
+	if (g->ops.pmu.pmu_pg_param_post_init)
+		g->ops.pmu.pmu_pg_param_post_init(g);
+
 	return 0;
 }
 
@@ -4693,44 +4701,62 @@ clean_up:
 	return err;
 }
 
-static int gk20a_pmu_enable_elpg_locked(struct gk20a *g)
+int gk20a_pmu_pg_global_enable(struct gk20a *g, u32 enable_pg)
+{
+	u32 status = 0;
+
+	if (enable_pg == true) {
+		if (g->ops.pmu.pmu_pg_engines_feature_list &&
+			g->ops.pmu.pmu_pg_engines_feature_list(g,
+			PMU_PG_ELPG_ENGINE_ID_GRAPHICS) !=
+			PMU_PG_FEATURE_GR_POWER_GATING_ENABLED) {
+			if (g->ops.pmu.pmu_lpwr_enable_pg)
+				status = g->ops.pmu.pmu_lpwr_enable_pg(g,
+						true);
+		} else if (support_gk20a_pmu(g->dev))
+			status = gk20a_pmu_enable_elpg(g);
+	} else if (enable_pg == false) {
+		if (g->ops.pmu.pmu_pg_engines_feature_list &&
+			g->ops.pmu.pmu_pg_engines_feature_list(g,
+			PMU_PG_ELPG_ENGINE_ID_GRAPHICS) !=
+			PMU_PG_FEATURE_GR_POWER_GATING_ENABLED) {
+			if (g->ops.pmu.pmu_lpwr_disable_pg)
+				status = g->ops.pmu.pmu_lpwr_disable_pg(g,
+						true);
+		} else if (support_gk20a_pmu(g->dev))
+			status = gk20a_pmu_disable_elpg(g);
+	}
+
+	return status;
+}
+
+static int gk20a_pmu_enable_elpg_locked(struct gk20a *g, u32 pg_engine_id)
 {
 	struct pmu_gk20a *pmu = &g->pmu;
 	struct pmu_cmd cmd;
 	u32 seq, status;
-	u32 pg_engine_id;
-	u32 pg_engine_id_list = 0;
 
 	gk20a_dbg_fn("");
 
-	if (g->ops.pmu.pmu_pg_supported_engines_list)
+	memset(&cmd, 0, sizeof(struct pmu_cmd));
-		pg_engine_id_list = g->ops.pmu.pmu_pg_supported_engines_list(g);
+	cmd.hdr.unit_id = PMU_UNIT_PG;
-	for (pg_engine_id = PMU_PG_ELPG_ENGINE_ID_GRAPHICS;
+	cmd.hdr.size = PMU_CMD_HDR_SIZE +
-		pg_engine_id < PMU_PG_ELPG_ENGINE_ID_INVALID_ENGINE;
+		sizeof(struct pmu_pg_cmd_elpg_cmd);
-		pg_engine_id++) {
+	cmd.cmd.pg.elpg_cmd.cmd_type = PMU_PG_CMD_ID_ELPG_CMD;
+	cmd.cmd.pg.elpg_cmd.engine_id = pg_engine_id;
+	cmd.cmd.pg.elpg_cmd.cmd = PMU_PG_ELPG_CMD_ALLOW;
 
-		if (BIT(pg_engine_id) & pg_engine_id_list) {
+   /* no need to wait ack for ELPG enable but set
-			memset(&cmd, 0, sizeof(struct pmu_cmd));
+	* pending to sync with follow up ELPG disable
-			cmd.hdr.unit_id = PMU_UNIT_PG;
+	*/
-			cmd.hdr.size = PMU_CMD_HDR_SIZE +
+	if (pg_engine_id == PMU_PG_ELPG_ENGINE_ID_GRAPHICS)
-				sizeof(struct pmu_pg_cmd_elpg_cmd);
+		pmu->elpg_stat = PMU_ELPG_STAT_ON_PENDING;
-			cmd.cmd.pg.elpg_cmd.cmd_type = PMU_PG_CMD_ID_ELPG_CMD;
-			cmd.cmd.pg.elpg_cmd.engine_id = pg_engine_id;
-			cmd.cmd.pg.elpg_cmd.cmd = PMU_PG_ELPG_CMD_ALLOW;
 
-		   /* no need to wait ack for ELPG enable but set
+	gk20a_dbg_pmu("cmd post PMU_PG_ELPG_CMD_ALLOW");
-		    * pending to sync with follow up ELPG disable
+	status = gk20a_pmu_cmd_post(g, &cmd, NULL, NULL,
-		    */
+		PMU_COMMAND_QUEUE_HPQ, pmu_handle_pg_elpg_msg,
-			if (pg_engine_id == PMU_PG_ELPG_ENGINE_ID_GRAPHICS)
+		pmu, &seq, ~0);
-				pmu->elpg_stat = PMU_ELPG_STAT_ON_PENDING;
+	WARN_ON(status != 0);
-
-			gk20a_dbg_pmu("cmd post PMU_PG_ELPG_CMD_ALLOW");
-			status = gk20a_pmu_cmd_post(g, &cmd, NULL, NULL,
-				PMU_COMMAND_QUEUE_HPQ, pmu_handle_pg_elpg_msg,
-				pmu, &seq, ~0);
-			WARN_ON(status != 0);
-		}
-	}
 
 	gk20a_dbg_fn("done");
 	return 0;
@@ -4740,12 +4766,13 @@ int gk20a_pmu_enable_elpg(struct gk20a *g)
 {
 	struct pmu_gk20a *pmu = &g->pmu;
 	struct gr_gk20a *gr = &g->gr;
+	u32 pg_engine_id;
+	u32 pg_engine_id_list = 0;
 
 	int ret = 0;
 
 	gk20a_dbg_fn("");
 
-
 	if (!support_gk20a_pmu(g->dev))
 		return ret;
 
@@ -4772,7 +4799,20 @@ int gk20a_pmu_enable_elpg(struct gk20a *g)
 	if (pmu->elpg_stat != PMU_ELPG_STAT_OFF)
 		goto exit_unlock;
 
-	ret = gk20a_pmu_enable_elpg_locked(g);
+	if (g->ops.pmu.pmu_pg_supported_engines_list)
+		pg_engine_id_list = g->ops.pmu.pmu_pg_supported_engines_list(g);
+
+	for (pg_engine_id = PMU_PG_ELPG_ENGINE_ID_GRAPHICS;
+		pg_engine_id < PMU_PG_ELPG_ENGINE_ID_INVALID_ENGINE;
+		pg_engine_id++) {
+
+		if (pg_engine_id == PMU_PG_ELPG_ENGINE_ID_MS &&
+			pmu->mscg_stat == PMU_MSCG_DISABLED)
+			continue;
+
+		if (BIT(pg_engine_id) & pg_engine_id_list)
+			ret = gk20a_pmu_enable_elpg_locked(g, pg_engine_id);
+	}
 
 exit_unlock:
 	mutex_unlock(&pmu->elpg_mutex);
@@ -4845,6 +4885,10 @@ int gk20a_pmu_disable_elpg(struct gk20a *g)
 		pg_engine_id < PMU_PG_ELPG_ENGINE_ID_INVALID_ENGINE;
 		pg_engine_id++) {
 
+		if (pg_engine_id == PMU_PG_ELPG_ENGINE_ID_MS &&
+			pmu->mscg_stat == PMU_MSCG_DISABLED)
+			continue;
+
 		if (BIT(pg_engine_id) & pg_engine_id_list) {
 			memset(&cmd, 0, sizeof(struct pmu_cmd));
 			cmd.hdr.unit_id = PMU_UNIT_PG;

drivers/gpu/nvgpu/gk20a/pmu_gk20a.h

@@ -629,6 +629,9 @@ struct pmu_pg_stats {
 #define PMU_ELPG_STAT_OFF_ON_PENDING	4   /* elpg is off, caller has requested on, but ALLOW
 					       cmd hasn't been sent due to ENABLE_ALLOW delay */
 
+#define PMU_MSCG_DISABLED 0
+#define PMU_MSCG_ENABLED 1
+
 /* Falcon Register index */
 #define PMU_FALCON_REG_R0		(0)
 #define PMU_FALCON_REG_R1		(1)
@@ -716,10 +719,13 @@ struct pmu_gk20a {
 
 	u32 elpg_stat;
 
+	u32 mscg_stat;
+
 	int pmu_state;
 
 #define PMU_ELPG_ENABLE_ALLOW_DELAY_MSEC	1 /* msec */
 	struct work_struct pg_init;
+	struct mutex pg_mutex; /* protect pg-RPPG/MSCG enable/disable */
 	struct mutex elpg_mutex; /* protect elpg enable/disable */
 	int elpg_refcnt; /* disable -1, enable +1, <=0 elpg disabled, > 0 elpg enabled */
 
@@ -774,6 +780,7 @@ int gk20a_pmu_cmd_post(struct gk20a *g, struct pmu_cmd *cmd, struct pmu_msg *msg
 
 int gk20a_pmu_enable_elpg(struct gk20a *g);
 int gk20a_pmu_disable_elpg(struct gk20a *g);
+int gk20a_pmu_pg_global_enable(struct gk20a *g, u32 enable_pg);
 
 u32 gk20a_pmu_pg_engines_list(struct gk20a *g);
 u32 gk20a_pmu_pg_feature_list(struct gk20a *g, u32 pg_engine_id);

drivers/gpu/nvgpu/gm206/bios_gm206.h

@@ -44,6 +44,9 @@ enum {
 	VOLTAGE_RAIL_TABLE = 26,
 	VOLTAGE_DEVICE_TABLE,
 	VOLTAGE_POLICY_TABLE,
+	LOWPOWER_TABLE,
+	LOWPOWER_GR_TABLE = 32,
+	LOWPOWER_MS_TABLE = 33,
 };
 
 enum {

drivers/gpu/nvgpu/gm206/pmu_gm206.c

@@ -159,6 +159,9 @@ void gm206_init_pmu_ops(struct gpu_ops *gops)
 	gops->pmu.pmu_pg_init_param = NULL;
 	gops->pmu.pmu_pg_supported_engines_list = NULL;
 	gops->pmu.pmu_pg_engines_feature_list = NULL;
+	gops->pmu.pmu_lpwr_enable_pg = NULL;
+	gops->pmu.pmu_lpwr_disable_pg = NULL;
+	gops->pmu.pmu_pg_param_post_init = NULL;
 	gops->pmu.send_lrf_tex_ltc_dram_overide_en_dis_cmd = NULL;
 	gops->pmu.dump_secure_fuses = NULL;
 	gops->pmu.reset = gk20a_pmu_reset;

drivers/gpu/nvgpu/gm20b/pmu_gm20b.c

@@ -288,6 +288,9 @@ void gm20b_init_pmu_ops(struct gpu_ops *gops)
 	gops->pmu.pmu_pg_init_param = NULL;
 	gops->pmu.pmu_pg_supported_engines_list = gk20a_pmu_pg_engines_list;
 	gops->pmu.pmu_pg_engines_feature_list = gk20a_pmu_pg_feature_list;
+	gops->pmu.pmu_lpwr_enable_pg = NULL;
+	gops->pmu.pmu_lpwr_disable_pg = NULL;
+	gops->pmu.pmu_pg_param_post_init = NULL;
 	gops->pmu.send_lrf_tex_ltc_dram_overide_en_dis_cmd = NULL;
 	gops->pmu.dump_secure_fuses = pmu_dump_security_fuses_gm20b;
 	gops->pmu.reset = gk20a_pmu_reset;

drivers/gpu/nvgpu/nvgpu_common.c

@@ -89,6 +89,8 @@ static void nvgpu_init_pm_vars(struct gk20a *g)
 		tegra_platform_is_silicon() ? platform->enable_elpg : false;
 	g->aelpg_enabled =
 		tegra_platform_is_silicon() ? platform->enable_aelpg : false;
+	g->mscg_enabled =
+		tegra_platform_is_silicon() ? platform->enable_mscg : false;
 
 	/* set default values to aelpg parameters */
 	g->pmu.aelpg_param[0] = APCTRL_SAMPLING_PERIOD_PG_DEFAULT_US;