gpu: nvgpu: support suspend/resume with user disabled railgating

We take an extra power refcount when we disable railgating
through railgate_enable sysfs
And that breaks suspend/resume since we check for power
refcount first in gk20a_pm_suspend()

Fix this with following :
- set a flag user_railgate_disabled when User
  disables railgating through sysfs railgate_enable
- in gk20a_pm_suspend(), drop one power refcount
  if flag is set
- in gk20a_pm_resume(), take one refcount again
  if flag is set

Fix __gk20a_do_idle() to consider this extra refcount as well.
Add new variable target_ref_cnt and use it instead of
assuming target refcount of 1

In case User has disabled rail gating, set this target
refcount as 2

Also, export gk20a_idle_nosuspend() which drop power refcount
without triggering suspend

Bug 200233570

Change-Id: Ic0e35c73eb74ffefea1cd90d1b152650d9d2043d
Signed-off-by: Deepak Nibade <dnibade@nvidia.com>
Reviewed-on: http://git-master/r/1236047
(cherry picked from commit 6e002d57da4b5c58ed79889728bb678d3aa1f1b1)
Reviewed-on: http://git-master/r/1235219
Reviewed-by: mobile promotions <svcmobile_promotions@nvidia.com>
Tested-by: mobile promotions <svcmobile_promotions@nvidia.com>

drivers/gpu/nvgpu/gk20a/gk20a.c

@@ -1395,9 +1395,14 @@ static int gk20a_pm_suspend(struct device *dev)
 	struct gk20a *g = get_gk20a(dev);
 	int ret = 0;
 
+	if (platform->user_railgate_disabled)
+		gk20a_idle_nosuspend(dev);
+
 #ifdef CONFIG_PM
-	if (atomic_read(&dev->power.usage_count) > 1)
-		return -EBUSY;
+	if (atomic_read(&dev->power.usage_count) > 1) {
+		ret = -EBUSY;
+		goto fail;
+	}
 #endif
 
 	if (!g->power_on)
@@ -1405,7 +1410,7 @@ static int gk20a_pm_suspend(struct device *dev)
 
 	ret = gk20a_pm_runtime_suspend(dev);
 	if (ret)
-		return ret;
+		goto fail;
 
 	if (platform->suspend)
 		platform->suspend(dev);
@@ -1413,13 +1418,23 @@ static int gk20a_pm_suspend(struct device *dev)
 	g->suspended = true;
 
 	return 0;
+
+fail:
+	if (platform->user_railgate_disabled)
+		gk20a_busy_noresume(dev);
+
+	return ret;
 }
 
 static int gk20a_pm_resume(struct device *dev)
 {
 	struct gk20a *g = get_gk20a(dev);
+	struct gk20a_platform *platform = dev_get_drvdata(dev);
 	int ret = 0;
 
+	if (platform->user_railgate_disabled)
+		gk20a_busy_noresume(dev);
+
 	if (!g->suspended)
 		return 0;
 
@@ -1711,6 +1726,11 @@ fail:
 	return ret < 0 ? ret : 0;
 }
 
+void gk20a_idle_nosuspend(struct device *dev)
+{
+	pm_runtime_put_noidle(dev);
+}
+
 void gk20a_idle(struct device *dev)
 {
 	if (pm_runtime_enabled(dev)) {
@@ -1783,6 +1803,7 @@ int __gk20a_do_idle(struct device *dev, bool force_reset)
 	unsigned long timeout = jiffies +
 		msecs_to_jiffies(GK20A_WAIT_FOR_IDLE_MS);
 	int ref_cnt;
+	int target_ref_cnt = 0;
 	bool is_railgated;
 	int err = 0;
 
@@ -1802,15 +1823,25 @@ int __gk20a_do_idle(struct device *dev, bool force_reset)
 	 */
 	mutex_unlock(&platform->railgate_lock);
 	pm_runtime_get_sync(dev);
+
+	/*
+	 * One refcount taken in this API
+	 * If User disables rail gating, we take one more
+	 * extra refcount
+	 */
+	if (platform->user_railgate_disabled)
+		target_ref_cnt = 2;
+	else
+		target_ref_cnt = 1;
 	mutex_lock(&platform->railgate_lock);
 
 	/* check and wait until GPU is idle (with a timeout) */
 	do {
 		msleep(1);
 		ref_cnt = atomic_read(&dev->power.usage_count);
-	} while (ref_cnt != 1 && time_before(jiffies, timeout));
+	} while (ref_cnt != target_ref_cnt && time_before(jiffies, timeout));
 
-	if (ref_cnt != 1) {
+	if (ref_cnt != target_ref_cnt) {
 		gk20a_err(dev, "failed to idle - refcount %d != 1\n",
 			ref_cnt);
 		goto fail_drop_usage_count;

drivers/gpu/nvgpu/gk20a/gk20a.h

@@ -1174,6 +1174,7 @@ void gk20a_remove_sysfs(struct device *dev);
 #define GK20A_SIM_IORESOURCE_MEM 2
 
 void gk20a_busy_noresume(struct device *dev);
+void gk20a_idle_nosuspend(struct device *dev);
 int __must_check gk20a_busy(struct device *dev);
 void gk20a_idle(struct device *dev);
 void gk20a_disable(struct gk20a *g, u32 units);

drivers/gpu/nvgpu/gk20a/gk20a_sysfs.c

@@ -294,12 +294,14 @@ static ssize_t railgate_enable_store(struct device *dev,
 		/* release extra ref count */
 		gk20a_idle(dev);
 		platform->can_railgate = true;
+		platform->user_railgate_disabled = false;
 	} else if (railgate_enable == 0 && platform->can_railgate) {
 		/* take extra ref count */
 		err = gk20a_busy(dev);
 		if (err)
 			return err;
 		platform->can_railgate = false;
+		platform->user_railgate_disabled = true;
 	}
 	if (err)
 		return err;

drivers/gpu/nvgpu/gk20a/platform_gk20a.h

@@ -44,6 +44,9 @@ struct gk20a_platform {
 	/* Should be populated at probe. */
 	bool can_railgate;
 
+	/* Set by User while disabling railgating */
+	bool user_railgate_disabled;
+
 	/* Should be populated at probe. */
 	bool can_elpg;