gpu: nvgpu: fix possible PMU isr race

Possible race description :
- while PMU is booting, it sends messages to kernel which we process
  in gk20a_pmu_isr()
- but when messages are processed it is possible that we are on the way
  to rail gate the GPU and we have already called pmu_destroy()
- this could lead to hangs if while processing messages, GR is
  already off

To fix this, introduce another mutex isr_enable_lock and a flag to
turn on/off ISRs
- when we enable PMU, get the lock and set the flag
- in pmu_destroy(), get the lock and remove the flag
- in pmu_isr(), take the lock, check if flag is set or not. If flag
  is not set return, otherwise proceed with the messages

Bug 200014542
Bug 200014887

Change-Id: I0204d8a00e4563859eebc807d4ac7d26161316ea
Signed-off-by: Deepak Nibade <dnibade@nvidia.com>
Reviewed-on: http://git-master/r/428371
(cherry picked from commit 9a37528314f2a2504e4530719f817a93db9a5bf0)
Reviewed-on: http://git-master/r/428352
Reviewed-by: Automatic_Commit_Validation_User
GVS: Gerrit_Virtual_Submit
Reviewed-by: Sachin Nikam <snikam@nvidia.com>

drivers/gpu/nvgpu/gk20a/pmu_gk20a.c

@@ -490,6 +490,7 @@ int gk20a_init_pmu(struct pmu_gk20a *pmu)
 
 	mutex_init(&pmu->elpg_mutex);
 	mutex_init(&pmu->isr_mutex);
+	mutex_init(&pmu->isr_enable_lock);
 	mutex_init(&pmu->pmu_copy_lock);
 	mutex_init(&pmu->pmu_seq_lock);
 
@@ -1771,7 +1772,11 @@ int gk20a_init_pmu_setup_hw1(struct gk20a *g)
 	int err;
 
 	gk20a_dbg_fn("");
+
+	mutex_lock(&pmu->isr_enable_lock);
 	pmu_reset(pmu);
+	pmu->isr_enabled = true;
+	mutex_unlock(&pmu->isr_enable_lock);
 
 	/* setup apertures - virtual */
 	gk20a_writel(g, pwr_fbif_transcfg_r(GK20A_PMU_DMAIDX_UCODE),
@@ -2881,6 +2886,12 @@ void gk20a_pmu_isr(struct gk20a *g)
 
 	gk20a_dbg_fn("");
 
+	mutex_lock(&pmu->isr_enable_lock);
+	if (!pmu->isr_enabled) {
+		mutex_unlock(&pmu->isr_enable_lock);
+		return;
+	}
+
 	mutex_lock(&pmu->isr_mutex);
 
 	mask = gk20a_readl(g, pwr_falcon_irqmask_r()) &
@@ -2893,6 +2904,7 @@ void gk20a_pmu_isr(struct gk20a *g)
 	if (!intr || pmu->pmu_state == PMU_STATE_OFF) {
 		gk20a_writel(g, pwr_falcon_irqsclr_r(), intr);
 		mutex_unlock(&pmu->isr_mutex);
+		mutex_unlock(&pmu->isr_enable_lock);
 		return;
 	}
 
@@ -2925,6 +2937,7 @@ void gk20a_pmu_isr(struct gk20a *g)
 	}
 
 	mutex_unlock(&pmu->isr_mutex);
+	mutex_unlock(&pmu->isr_enable_lock);
 }
 
 static bool pmu_validate_cmd(struct pmu_gk20a *pmu, struct pmu_cmd *cmd,
@@ -3342,7 +3355,11 @@ int gk20a_pmu_destroy(struct gk20a *g)
 	g->pg_ungating_time_us += (u64)elpg_ungating_time;
 	g->pg_gating_cnt += gating_cnt;
 
+	mutex_lock(&pmu->isr_enable_lock);
 	pmu_enable(pmu, false);
+	pmu->isr_enabled = false;
+	mutex_unlock(&pmu->isr_enable_lock);
+
 	pmu->pmu_state = PMU_STATE_OFF;
 	pmu->pmu_ready = false;
 	pmu->perfmon_ready = false;

drivers/gpu/nvgpu/gk20a/pmu_gk20a.h

@@ -1054,6 +1054,9 @@ struct pmu_gk20a {
 	u32 sample_buffer;
 
 	struct mutex isr_mutex;
+	struct mutex isr_enable_lock;
+	bool isr_enabled;
+
 	bool zbc_ready;
 	union {
 		struct pmu_cmdline_args_v0 args_v0;