gpu: nvgpu: protect nvgpu power state access using spinlock

IRQs can get triggered during nvgpu power-on due to MMU fault, invalid
PRIV ring or bus access etc. Handlers for those IRQs can't access the
full state related to the IRQ unless nvgpu is fully powered on.

In order to let the IRQ handlers know about the nvgpu power-on state
gk20a.power_on_state variable has to be protected through spinlock
to avoid the deadlock due to usage of earlier power_lock mutex.

Further the IRQs need to be disabled on local CPU while updating the
power state variable hence use spin_lock_irqsave and spin_unlock_-
irqrestore APIs for protecting the access.

JIRA NVGPU-1592

Change-Id: If5d1b5e2617ad90a68faa56ff47f62bb3f0b232b
Signed-off-by: Sagar Kamble <skamble@nvidia.com>
Reviewed-on: https://git-master.nvidia.com/r/2203860
Reviewed-by: mobile promotions <svcmobile_promotions@nvidia.com>
Tested-by: mobile promotions <svcmobile_promotions@nvidia.com>

drivers/gpu/nvgpu/common/fifo/channel.c

@@ -1305,7 +1305,7 @@ void nvgpu_channel_clean_up_jobs(struct nvgpu_channel *c,
 		return;
 	}
 
-	if (!c->g->power_on) { /* shutdown case */
+	if (nvgpu_is_powered_off(c->g)) { /* shutdown case */
 		nvgpu_channel_put(c);
 		return;
 	}
@@ -1462,7 +1462,7 @@ void nvgpu_channel_clean_up_jobs(struct nvgpu_channel *c,
  */
 void nvgpu_channel_update(struct nvgpu_channel *c)
 {
-	if (!c->g->power_on) { /* shutdown case */
+	if (nvgpu_is_powered_off(c->g)) { /* shutdown case */
 		return;
 	}
 

drivers/gpu/nvgpu/common/init/nvgpu_init.c

@@ -92,7 +92,7 @@ static int nvgpu_sw_quiesce_thread(void *data)
 
 	nvgpu_mutex_acquire(&g->power_lock);
 
-	if (!g->power_on || g->is_virtual) {
+	if (nvgpu_is_powered_off(g) || g->is_virtual) {
 		err = -EINVAL;
 		goto idle;
 	}
@@ -289,8 +289,6 @@ int nvgpu_prepare_poweroff(struct gk20a *g)
 #endif
 	gk20a_mask_interrupts(g);
 
-	g->power_on = false;
-
 	return ret;
 }
 
@@ -561,12 +559,6 @@ int nvgpu_finalize_poweron(struct gk20a *g)
 
 	nvgpu_log_fn(g, " ");
 
-	if (g->power_on) {
-		return 0;
-	}
-
-	g->power_on = true;
-
 #ifdef CONFIG_NVGPU_RECOVERY
 	nvgpu_set_enabled(g, NVGPU_SUPPORT_FAULT_RECOVERY, true);
 #else
@@ -622,10 +614,6 @@ done:
 	nvgpu_falcons_sw_free(g);
 
 exit:
-	if (err != 0) {
-		g->power_on = false;
-	}
-
 	return err;
 }
 

drivers/gpu/nvgpu/common/pmu/perfmon/pmu_perfmon.c

@@ -384,7 +384,7 @@ int nvgpu_pmu_busy_cycles_norm(struct gk20a *g, u32 *norm)
 	u32 intr_status;
 
 	gk20a_busy_noresume(g);
-	if (!g->power_on) {
+	if (nvgpu_is_powered_off(g)) {
 		*norm = 0;
 		goto exit;
 	}
@@ -423,7 +423,7 @@ exit:
 void nvgpu_pmu_get_load_counters(struct gk20a *g, u32 *busy_cycles,
 				 u32 *total_cycles)
 {
-	if (!g->power_on || gk20a_busy(g) != 0) {
+	if (nvgpu_is_powered_off(g) || gk20a_busy(g) != 0) {
 		*busy_cycles = 0;
 		*total_cycles = 0;
 		return;
@@ -437,7 +437,7 @@ void nvgpu_pmu_get_load_counters(struct gk20a *g, u32 *busy_cycles,
 
 void nvgpu_pmu_reset_load_counters(struct gk20a *g)
 {
-	if (!g->power_on || gk20a_busy(g) != 0) {
+	if (nvgpu_is_powered_off(g) || gk20a_busy(g) != 0) {
 		return;
 	}
 

drivers/gpu/nvgpu/hal/fb/fb_gm20b_fusa.c

@@ -31,6 +31,7 @@
 #include <nvgpu/gk20a.h>
 #include <nvgpu/bug.h>
 #include <nvgpu/ltc.h>
+#include <nvgpu/nvgpu_init.h>
 
 #include "fb_gm20b.h"
 
@@ -90,7 +91,7 @@ int gm20b_fb_tlb_invalidate(struct gk20a *g, struct nvgpu_mem *pdb)
 	   hw. Use the power_on flag to skip tlb invalidation when gpu
 	   power is turned off */
 
-	if (!g->power_on) {
+	if (nvgpu_is_powered_off(g)) {
 		return err;
 	}
 

drivers/gpu/nvgpu/hal/fb/fb_tu104.c

@@ -32,6 +32,7 @@
 #include <nvgpu/utils.h>
 #include <nvgpu/gk20a.h>
 #include <nvgpu/bug.h>
+#include <nvgpu/nvgpu_init.h>
 
 #include "hal/fb/fb_gv11b.h"
 #include "hal/fb/fb_gv100.h"
@@ -60,7 +61,7 @@ int fb_tu104_tlb_invalidate(struct gk20a *g, struct nvgpu_mem *pdb)
 	 * hw. Use the power_on flag to skip tlb invalidation when gpu
 	 * power is turned off
 	 */
-	if (!g->power_on) {
+	if (nvgpu_is_powered_off(g)) {
 		return 0;
 	}
 

drivers/gpu/nvgpu/hal/mm/cache/flush_gk20a.c

@@ -45,7 +45,7 @@ void gk20a_mm_cbc_clean(struct gk20a *g)
 	nvgpu_log_fn(g, " ");
 
 	gk20a_busy_noresume(g);
-	if (!g->power_on) {
+	if (nvgpu_is_powered_off(g)) {
 		goto hw_was_off;
 	}
 

drivers/gpu/nvgpu/hal/mm/cache/flush_gk20a_fusa.c

@@ -45,7 +45,7 @@ int gk20a_mm_fb_flush(struct gk20a *g)
 	nvgpu_log_fn(g, " ");
 
 	gk20a_busy_noresume(g);
-	if (!g->power_on) {
+	if (nvgpu_is_powered_off(g)) {
 		gk20a_idle_nosuspend(g);
 		return 0;
 	}
@@ -157,7 +157,7 @@ void gk20a_mm_l2_invalidate(struct gk20a *g)
 {
 	struct mm_gk20a *mm = &g->mm;
 	gk20a_busy_noresume(g);
-	if (g->power_on) {
+	if (!nvgpu_is_powered_off(g)) {
 		nvgpu_mutex_acquire(&mm->l2_op_lock);
 		gk20a_mm_l2_invalidate_locked(g);
 		nvgpu_mutex_release(&mm->l2_op_lock);
@@ -176,7 +176,7 @@ int gk20a_mm_l2_flush(struct gk20a *g, bool invalidate)
 	nvgpu_log_fn(g, " ");
 
 	gk20a_busy_noresume(g);
-	if (!g->power_on) {
+	if (nvgpu_is_powered_off(g)) {
 		goto hw_was_off;
 	}
 

drivers/gpu/nvgpu/include/nvgpu/gk20a.h

@@ -1680,8 +1680,9 @@ struct gk20a {
 
 	const char *name;
 
+	u32 power_on_state;
+
 	bool gpu_reset_done;
-	bool power_on;
 	bool suspended;
 	bool sw_ready;
 
@@ -1742,6 +1743,8 @@ struct gk20a {
 
 	struct nvgpu_mutex power_lock;
 
+	struct nvgpu_spinlock power_spinlock;
+
 	/* Channel priorities */
 	u32 tsg_timeslice_low_priority_us;
 	u32 tsg_timeslice_medium_priority_us;

drivers/gpu/nvgpu/include/nvgpu/linux/lock.h

@@ -64,6 +64,12 @@ static inline void nvgpu_spinlock_release(struct nvgpu_spinlock *spinlock)
 	spin_unlock(&spinlock->spinlock);
 };
 
+#define nvgpu_spinlock_irqsave(lock, flags) \
+		spin_lock_irqsave(lock.spinlock, flags)
+
+#define nvgpu_spinunlock_irqrestore(lock, flags) \
+		spin_unlock_irqrestore(lock.spinlock, flags)
+
 static inline void nvgpu_raw_spinlock_init(struct nvgpu_raw_spinlock *spinlock)
 {
 	raw_spin_lock_init(&spinlock->spinlock);

drivers/gpu/nvgpu/include/nvgpu/nvgpu_init.h

@@ -23,6 +23,8 @@
 #ifndef NVGPU_INIT_H
 #define NVGPU_INIT_H
 
+#include <nvgpu/types.h>
+
 struct gk20a;
 struct nvgpu_ref;
 
@@ -154,6 +156,47 @@ int nvgpu_enable_irqs(struct gk20a *g);
  */
 void nvgpu_disable_irqs(struct gk20a *g);
 
+#define NVGPU_STATE_POWERED_OFF		0U
+#define NVGPU_STATE_POWERING_ON		1U
+#define NVGPU_STATE_POWERED_ON		2U
+
+/**
+ * @brief Set the nvgpu power state
+ *
+ * @param g [in] The GPU
+ * @param state Power state
+ *
+ * Set the nvgpu power state.
+ */
+void nvgpu_set_power_state(struct gk20a *g, u32 state);
+
+/**
+ * @brief Get the nvgpu power state
+ *
+ * @param g [in] The GPU
+ *
+ * Returns the nvgpu power state.
+ */
+const char *nvgpu_get_power_state(struct gk20a *g);
+
+/**
+ * @brief Return the nvgpu power-on state
+ *
+ * @param g [in] The GPU
+ *
+ * Return true if nvgpu is in powered on state, else false.
+ */
+bool nvgpu_is_powered_on(struct gk20a *g);
+
+/**
+ * @brief Return the nvgpu power-off state
+ *
+ * @param g [in] The GPU
+ *
+ * Return true if nvgpu is in powered off state, else false.
+ */
+bool nvgpu_is_powered_off(struct gk20a *g);
+
 /**
  * @brief Check if the device can go busy
  *

drivers/gpu/nvgpu/libnvgpu-drv_safe.export

@@ -343,6 +343,7 @@ nvgpu_tsg_unbind_channel_check_hw_state
 nvgpu_tsg_unbind_channel_check_ctx_reload
 nvgpu_set_bit
 nvgpu_set_enabled
+nvgpu_set_power_state
 nvgpu_set_pte
 nvgpu_sgt_alignment
 nvgpu_sgt_create_from_mem

drivers/gpu/nvgpu/os/linux/debug_pmu.c

@@ -83,7 +83,7 @@ static int mscg_stat_show(struct seq_file *s, void *data)
 	int err;
 
 	/* Don't unnecessarily power on the device */
-	if (g->power_on) {
+	if (nvgpu_is_powered_on(g)) {
 		err = gk20a_busy(g);
 		if (err)
 			return err;
@@ -141,7 +141,7 @@ static int mscg_transitions_show(struct seq_file *s, void *data)
 	u32 total_gating_cnt;
 	int err;
 
-	if (g->power_on) {
+	if (nvgpu_is_powered_on(g)) {
 		err = gk20a_busy(g);
 		if (err)
 			return err;
@@ -177,7 +177,7 @@ static int elpg_stat_show(struct seq_file *s, void *data)
 	int err;
 
 	/* Don't unnecessarily power on the device */
-	if (g->power_on) {
+	if (nvgpu_is_powered_on(g)) {
 		err = gk20a_busy(g);
 		if (err)
 			return err;
@@ -234,7 +234,7 @@ static int elpg_transitions_show(struct seq_file *s, void *data)
 	u32 total_gating_cnt;
 	int err;
 
-	if (g->power_on) {
+	if (nvgpu_is_powered_on(g)) {
 		err = gk20a_busy(g);
 		if (err)
 			return err;
@@ -358,7 +358,7 @@ static ssize_t perfmon_events_enable_write(struct file *file,
 		return -EINVAL;
 
 	/* Don't turn on gk20a unnecessarily */
-	if (g->power_on) {
+	if (nvgpu_is_powered_on(g)) {
 		err = gk20a_busy(g);
 		if (err)
 			return err;

drivers/gpu/nvgpu/os/linux/driver_common.c

@@ -60,6 +60,8 @@ static void nvgpu_init_vars(struct gk20a *g)
 
 	nvgpu_spinlock_init(&g->mc_enable_lock);
 
+	nvgpu_spinlock_init(&g->power_spinlock);
+
 	nvgpu_mutex_init(&platform->railgate_lock);
 	nvgpu_mutex_init(&g->dbg_sessions_lock);
 	nvgpu_mutex_init(&g->client_lock);

drivers/gpu/nvgpu/os/linux/intr.c

@@ -17,12 +17,10 @@
 #include <linux/irqreturn.h>
 
 #include <nvgpu/gk20a.h>
+#include <nvgpu/nvgpu_init.h>
 
 #include <nvgpu/atomic.h>
 #include <nvgpu/unit.h>
-#ifndef CONFIG_NVGPU_RECOVERY
-#include <nvgpu/nvgpu_init.h>
-#endif
 #include "os_linux.h"
 
 irqreturn_t nvgpu_intr_stall(struct gk20a *g)
@@ -33,7 +31,7 @@ irqreturn_t nvgpu_intr_stall(struct gk20a *g)
 	trace_mc_gk20a_intr_stall(g->name);
 #endif
 
-	if (!g->power_on)
+	if (nvgpu_is_powered_off(g))
 		return IRQ_NONE;
 
 	/* not from gpu when sharing irq with others */
@@ -89,7 +87,7 @@ irqreturn_t nvgpu_intr_nonstall(struct gk20a *g)
 	int ops_old, ops_new, ops = 0;
 	struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
 
-	if (!g->power_on)
+	if (nvgpu_is_powered_off(g))
 		return IRQ_NONE;
 
 	/* not from gpu when sharing irq with others */

drivers/gpu/nvgpu/os/linux/module.c

@@ -400,9 +400,11 @@ int gk20a_pm_finalize_poweron(struct device *dev)
 
 	nvgpu_mutex_acquire(&g->power_lock);
 
-	if (g->power_on)
+	if (nvgpu_is_powered_on(g))
 		goto done;
 
+	nvgpu_set_power_state(g, NVGPU_STATE_POWERING_ON);
+
 #ifdef CONFIG_NVGPU_TRACE
 	trace_gk20a_finalize_poweron(dev_name(dev));
 #endif
@@ -508,10 +510,9 @@ int gk20a_pm_finalize_poweron(struct device *dev)
 
 	g->sw_ready = true;
 
-done:
-	if (err)
-		g->power_on = false;
+	nvgpu_set_power_state(g, NVGPU_STATE_POWERED_ON);
 
+done:
 	nvgpu_mutex_release(&g->power_lock);
 	return err;
 }
@@ -557,6 +558,67 @@ void nvgpu_disable_irqs(struct gk20a *g)
 	}
 }
 
+void nvgpu_set_power_state(struct gk20a *g, u32 state)
+{
+	unsigned long flags;
+
+	nvgpu_spinlock_irqsave(&g->power_spinlock, flags);
+	g->power_on_state = state;
+	nvgpu_spinunlock_irqrestore(&g->power_spinlock, flags);
+}
+
+const char *nvgpu_get_power_state(struct gk20a *g)
+{
+	unsigned long flags;
+	const char *str = NULL;
+	u32 state;
+
+	nvgpu_spinlock_irqsave(&g->power_spinlock, flags);
+	state = g->power_on_state;
+	nvgpu_spinunlock_irqrestore(&g->power_spinlock, flags);
+
+	switch (state) {
+	case NVGPU_STATE_POWERED_OFF:
+		str = "off";
+		break;
+	case NVGPU_STATE_POWERING_ON:
+		str = "powering on";
+		break;
+	case NVGPU_STATE_POWERED_ON:
+		str = "on";
+		break;
+	default:
+		nvgpu_err(g, "Invalid nvgpu power state.");
+		break;
+	}
+
+	return str;
+}
+
+bool nvgpu_is_powered_on(struct gk20a *g)
+{
+	unsigned long flags;
+	u32 power_on;
+
+	nvgpu_spinlock_irqsave(&g->power_spinlock, flags);
+	power_on = g->power_on_state;
+	nvgpu_spinunlock_irqrestore(&g->power_spinlock, flags);
+
+	return power_on == NVGPU_STATE_POWERED_ON;
+}
+
+bool nvgpu_is_powered_off(struct gk20a *g)
+{
+	unsigned long flags;
+	u32 power_on;
+
+	nvgpu_spinlock_irqsave(&g->power_spinlock, flags);
+	power_on = g->power_on_state;
+	nvgpu_spinunlock_irqrestore(&g->power_spinlock, flags);
+
+	return power_on == NVGPU_STATE_POWERED_OFF;
+}
+
 static int gk20a_pm_prepare_poweroff(struct device *dev)
 {
 	struct gk20a *g = get_gk20a(dev);
@@ -571,7 +633,7 @@ static int gk20a_pm_prepare_poweroff(struct device *dev)
 
 	nvgpu_mutex_acquire(&g->power_lock);
 
-	if (!g->power_on)
+	if (nvgpu_is_powered_off(g))
 		goto done;
 
 	/* disable IRQs and wait for completion */
@@ -596,6 +658,9 @@ static int gk20a_pm_prepare_poweroff(struct device *dev)
 	gk20a_lockout_registers(g);
 
 	nvgpu_hide_usermode_for_poweroff(g);
+
+	nvgpu_set_power_state(g, NVGPU_STATE_POWERED_OFF);
+
 	nvgpu_mutex_release(&g->power_lock);
 	return 0;
 
@@ -1102,7 +1167,7 @@ int nvgpu_quiesce(struct gk20a *g)
 	int err;
 	struct device *dev = dev_from_gk20a(g);
 
-	if (g->power_on) {
+	if (nvgpu_is_powered_on(g)) {
 		err = nvgpu_wait_for_idle(g);
 		if (err) {
 			nvgpu_err(g, "failed to idle GPU, err=%d", err);
@@ -1149,7 +1214,7 @@ static void gk20a_pm_shutdown(struct platform_device *pdev)
 	if (gk20a_gpu_is_virtual(&pdev->dev))
 		return;
 
-	if (!g->power_on)
+	if (nvgpu_is_powered_off(g))
 		goto finish;
 
 	gk20a_driver_start_unload(g);
@@ -1237,7 +1302,7 @@ static int gk20a_pm_suspend(struct device *dev)
 	int ret = 0;
 	int idle_usage_count = 0;
 
-	if (!g->power_on) {
+	if (nvgpu_is_powered_off(g)) {
 		if (platform->suspend)
 			ret = platform->suspend(dev);
 		if (ret)

drivers/gpu/nvgpu/os/linux/sysfs.c

@@ -260,7 +260,7 @@ static ssize_t gpu_powered_on_show(struct device *dev,
 {
 	struct gk20a *g = get_gk20a(dev);
 
-	return snprintf(buf, PAGE_SIZE, "%u\n", g->power_on);
+	return snprintf(buf, PAGE_SIZE, "%s\n", nvgpu_get_power_state(g));
 }
 
 static DEVICE_ATTR(gpu_powered_on, S_IRUGO, gpu_powered_on_show, NULL);
@@ -406,7 +406,7 @@ static ssize_t gk20a_load_show(struct device *dev,
 	ssize_t res;
 	int err;
 
-	if (!g->power_on) {
+	if (nvgpu_is_powered_off(g)) {
 		busy_time = 0;
 	} else {
 		err = gk20a_busy(g);
@@ -435,7 +435,7 @@ static ssize_t elpg_enable_store(struct device *dev,
 	if (kstrtoul(buf, 10, &val) < 0)
 		return -EINVAL;
 
-	if (!g->power_on) {
+	if (nvgpu_is_powered_off(g)) {
 		return -EAGAIN;
 	} else {
 		err = gk20a_busy(g);
@@ -487,7 +487,7 @@ static ssize_t ldiv_slowdown_factor_store(struct device *dev,
 	if (val == g->ldiv_slowdown_factor)
 		return count;
 
-	if (!g->power_on) {
+	if (nvgpu_is_powered_off(g)) {
 		g->ldiv_slowdown_factor = val;
 	} else {
 		err = gk20a_busy(g);
@@ -530,7 +530,7 @@ static ssize_t mscg_enable_store(struct device *dev,
 	if (kstrtoul(buf, 10, &val) < 0)
 		return -EINVAL;
 
-	if (!g->power_on) {
+	if (nvgpu_is_powered_off(g)) {
 		g->mscg_enabled = val ? true : false;
 	} else {
 		err = gk20a_busy(g);

drivers/gpu/nvgpu/os/linux/vgpu/vgpu_linux.c

@@ -80,6 +80,8 @@ static void vgpu_init_vars(struct gk20a *g, struct gk20a_platform *platform)
 	struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
 	struct vgpu_priv_data *priv = vgpu_get_priv_data(g);
 
+	nvgpu_spinlock_init(&g->power_spinlock);
+
 	nvgpu_mutex_init(&g->power_lock);
 	nvgpu_mutex_init(&g->clk_arb_enable_lock);
 	nvgpu_mutex_init(&g->cg_pg_lock);
@@ -174,7 +176,7 @@ int vgpu_pm_prepare_poweroff(struct device *dev)
 
 	nvgpu_mutex_acquire(&g->power_lock);
 
-	if (!g->power_on)
+	if (nvgpu_is_powered_off(g))
 		goto done;
 
 	if (g->ops.channel.suspend_all_serviceable_ch != NULL) {
@@ -185,7 +187,8 @@ int vgpu_pm_prepare_poweroff(struct device *dev)
 		goto done;
 	}
 
-	g->power_on = false;
+	nvgpu_set_power_state(g, NVGPU_STATE_POWERED_OFF);
+
  done:
 	nvgpu_mutex_release(&g->power_lock);
 
@@ -202,10 +205,10 @@ int vgpu_pm_finalize_poweron(struct device *dev)
 
 	nvgpu_mutex_acquire(&g->power_lock);
 
-	if (g->power_on)
+	if (nvgpu_is_powered_on(g))
 		goto done;
 
-	g->power_on = true;
+	nvgpu_set_power_state(g, NVGPU_STATE_POWERING_ON);
 
 	err = vgpu_finalize_poweron_common(g);
 	if (err)
@@ -222,10 +225,9 @@ int vgpu_pm_finalize_poweron(struct device *dev)
 
 	g->sw_ready = true;
 
-done:
-	if (err)
-		g->power_on = false;
+	nvgpu_set_power_state(g, NVGPU_STATE_POWERED_ON);
 
+done:
 	nvgpu_mutex_release(&g->power_lock);
 	return err;
 }

drivers/gpu/nvgpu/os/posix/nvgpu.c

@@ -72,6 +72,62 @@ void nvgpu_disable_irqs(struct gk20a *g)
 {
 }
 
+void nvgpu_set_power_state(struct gk20a *g, u32 state)
+{
+	nvgpu_spinlock_acquire(&g->power_spinlock);
+	g->power_on_state = state;
+	nvgpu_spinlock_release(&g->power_spinlock);
+}
+
+const char *nvgpu_get_power_state(struct gk20a *g)
+{
+	const char *str = NULL;
+	u32 state;
+
+	nvgpu_spinlock_acquire(&g->power_spinlock);
+	state = g->power_on_state;
+	nvgpu_spinlock_release(&g->power_spinlock);
+
+	switch (state) {
+	case NVGPU_STATE_POWERED_OFF:
+		str = "off";
+		break;
+	case NVGPU_STATE_POWERING_ON:
+		str = "powering on";
+		break;
+	case NVGPU_STATE_POWERED_ON:
+		str = "on";
+		break;
+	default:
+		nvgpu_err(g, "Invalid nvgpu power state.");
+		break;
+	}
+
+	return str;
+}
+
+bool nvgpu_is_powered_on(struct gk20a *g)
+{
+	u32 power_on;
+
+	nvgpu_spinlock_acquire(&g->power_spinlock);
+	power_on = g->power_on_state;
+	nvgpu_spinlock_release(&g->power_spinlock);
+
+	return power_on == NVGPU_STATE_POWERED_ON;
+}
+
+bool nvgpu_is_powered_off(struct gk20a *g)
+{
+	u32 power_on;
+
+	nvgpu_spinlock_acquire(&g->power_spinlock);
+	power_on = g->power_on_state;
+	nvgpu_spinlock_release(&g->power_spinlock);
+
+	return power_on == NVGPU_STATE_POWERED_OFF;
+}
+
 /*
  * We have no runtime PM stuff in userspace so these are really just noops.
  */

userspace/units/init/nvgpu-init.c

@@ -390,7 +390,7 @@ int test_poweron(struct unit_module *m, struct gk20a *g, void *args)
 	/* loop over the simple cases */
 	for (i = 0; i < simple_init_func_ptrs_count; i++) {
 		*simple_init_func_ptrs[i] = return_fail;
-		g->power_on = false;
+		nvgpu_set_power_state(g, NVGPU_STATE_POWERED_OFF);
 		err = nvgpu_finalize_poweron(g);
 		if (err == 0) {
 			unit_return_fail(m,
@@ -403,7 +403,7 @@ int test_poweron(struct unit_module *m, struct gk20a *g, void *args)
 	/* handle the exceptions */
 
 	falcon_fail_on_id = FALCON_ID_PMU;
-	g->power_on = false;
+	nvgpu_set_power_state(g, NVGPU_STATE_POWERED_OFF);
 	err = nvgpu_finalize_poweron(g);
 	if (err == 0) {
 		unit_return_fail(m,
@@ -411,7 +411,7 @@ int test_poweron(struct unit_module *m, struct gk20a *g, void *args)
 	}
 
 	falcon_fail_on_id = FALCON_ID_FECS;
-	g->power_on = false;
+	nvgpu_set_power_state(g, NVGPU_STATE_POWERED_OFF);
 	err = nvgpu_finalize_poweron(g);
 	if (err == 0) {
 		unit_return_fail(m,
@@ -420,7 +420,7 @@ int test_poweron(struct unit_module *m, struct gk20a *g, void *args)
 	falcon_fail_on_id = U32_MAX; /* stop failing */
 
 	g->ops.tpc.tpc_powergate = return_failure_u32_param;
-	g->power_on = false;
+	nvgpu_set_power_state(g, NVGPU_STATE_POWERED_OFF);
 	err = nvgpu_finalize_poweron(g);
 	if (err == 0) {
 		unit_return_fail(m,
@@ -429,7 +429,7 @@ int test_poweron(struct unit_module *m, struct gk20a *g, void *args)
 	g->ops.tpc.tpc_powergate = return_success_u32_param;
 
 	/* test the case of already being powered on */
-	g->power_on = true;
+	nvgpu_set_power_state(g, NVGPU_STATE_POWERED_ON);
 	err = nvgpu_finalize_poweron(g);
 	if (err != 0) {
 		unit_return_fail(m,
@@ -458,7 +458,7 @@ int test_poweron_branches(struct unit_module *m, struct gk20a *g, void *args)
 	g->ops.therm.elcg_init_idle_filters = NULL;
 	g->ops.gr.ecc.ecc_init_support = NULL;
 	g->ops.channel.resume_all_serviceable_ch = NULL;
-	g->power_on = false;
+	nvgpu_set_power_state(g, NVGPU_STATE_POWERED_OFF);
 	err = nvgpu_finalize_poweron(g);
 	if (err != 0) {
 		unit_return_fail(m,
@@ -468,14 +468,14 @@ int test_poweron_branches(struct unit_module *m, struct gk20a *g, void *args)
 	/* test the syncpoint paths here */
 	nvgpu_set_enabled(g, NVGPU_HAS_SYNCPOINTS, true);
 	g->syncpt_unit_size = 0UL;
-	g->power_on = false;
+	nvgpu_set_power_state(g, NVGPU_STATE_POWERED_OFF);
 	err = nvgpu_finalize_poweron(g);
 	if (err != 0) {
 		unit_return_fail(m,
 			"nvgpu_finalize_poweron returned fail\n");
 	}
 	g->syncpt_unit_size = 2UL;
-	g->power_on = false;
+	nvgpu_set_power_state(g, NVGPU_STATE_POWERED_OFF);
 	err = nvgpu_finalize_poweron(g);
 	if (err != 0) {
 		unit_return_fail(m,
@@ -485,7 +485,7 @@ int test_poweron_branches(struct unit_module *m, struct gk20a *g, void *args)
 	 * This redundant call will hit the case where memory is already
 	 * valid
 	 */
-	g->power_on = false;
+	nvgpu_set_power_state(g, NVGPU_STATE_POWERED_OFF);
 	err = nvgpu_finalize_poweron(g);
 	if (err != 0) {
 		unit_return_fail(m,
@@ -493,7 +493,7 @@ int test_poweron_branches(struct unit_module *m, struct gk20a *g, void *args)
 	}
 	nvgpu_dma_free(g, &g->syncpt_mem);
 	nvgpu_posix_enable_fault_injection(kmem_fi, true, 0);
-	g->power_on = false;
+	nvgpu_set_power_state(g, NVGPU_STATE_POWERED_OFF);
 	err = nvgpu_finalize_poweron(g);
 	if (err == 0) {
 		unit_return_fail(m,

userspace/units/mm/mm/mm.c

@@ -25,6 +25,7 @@
 #include <unit/unit.h>
 #include <unit/core.h>
 
+#include <nvgpu/nvgpu_init.h>
 #include <nvgpu/posix/io.h>
 #include "os/posix/os_posix.h"
 
@@ -525,14 +526,14 @@ int test_mm_suspend(struct unit_module *m, struct gk20a *g, void *args)
 {
 	int err;
 
-	g->power_on = false;
+	nvgpu_set_power_state(g, NVGPU_STATE_POWERED_OFF);
 	err = nvgpu_mm_suspend(g);
 	if (err != -ETIMEDOUT) {
 		unit_return_fail(m, "suspend did not fail as expected err=%d\n",
 				err);
 	}
 
-	g->power_on = true;
+	nvgpu_set_power_state(g, NVGPU_STATE_POWERED_ON);
 	err = nvgpu_mm_suspend(g);
 	if (err != 0) {
 		unit_return_fail(m, "suspend fail err=%d\n", err);
@@ -545,7 +546,7 @@ int test_mm_suspend(struct unit_module *m, struct gk20a *g, void *args)
 	 */
 	g->ops.fb.intr.disable = gv11b_fb_intr_disable;
 	g->ops.mm.mmu_fault.disable_hw = gv11b_mm_mmu_fault_disable_hw;
-	g->power_on = true;
+	nvgpu_set_power_state(g, NVGPU_STATE_POWERED_ON);
 	err = nvgpu_mm_suspend(g);
 	if (err != 0) {
 		unit_return_fail(m, "suspend fail err=%d\n", err);