gpu: nvgpu: split away power node removal

Presently, gk20a_user_deinit is used to remove all device nodes
including "power" node as well.

Split removal of power node into a separate function
gk20a_power_node_deinit to enable other device removal during the
normal runtime_suspend path to facilitate the fast path for MIG
reconfiguration. Powernode can be removed only during a call to
Rmmod. This also enables separately powering off the device nodes
in the unlikely case of a poweron failure.

Bug 3308828
Jira NVGPU-6920

Change-Id: Ib045a09a992a63c468492a837b273cca41e20f15
Signed-off-by: Debarshi Dutta <ddutta@nvidia.com>
Reviewed-on: https://git-master.nvidia.com/r/c/linux-nvgpu/+/2543014
Tested-by: mobile promotions <svcmobile_promotions@nvidia.com>
Reviewed-by: svcacv <svcacv@nvidia.com>
Reviewed-by: svc_kernel_abi <svc_kernel_abi@nvidia.com>
Reviewed-by: Dinesh T <dt@nvidia.com>
Reviewed-by: Lakshmanan M <lm@nvidia.com>
Reviewed-by: Vijayakumar Subbu <vsubbu@nvidia.com>
Reviewed-by: mobile promotions <svcmobile_promotions@nvidia.com>

drivers/gpu/nvgpu/os/linux/driver_common.c

@@ -307,7 +307,7 @@ int nvgpu_probe(struct gk20a *g,
 	* need to be removed.
 	*/
 	if (platform->platform_chip_id == TEGRA_210) {
-		err = gk20a_user_init(dev);
+		err = gk20a_user_nodes_init(dev);
 		if (err)
 			return err;
 		l->dev_nodes_created = true;

drivers/gpu/nvgpu/os/linux/ioctl.c

@@ -298,38 +298,58 @@ static int gk20a_create_device(
 	return 0;
 }
 
-void gk20a_user_deinit(struct device *dev)
+void gk20a_remove_devices_and_classes(struct gk20a *g, bool power_node)
 {
-	struct gk20a *g = gk20a_from_dev(dev);
 	struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
 	struct nvgpu_cdev *cdev, *n;
 	struct nvgpu_class *class, *p;
 
 	nvgpu_list_for_each_entry_safe(cdev, n, &l->cdev_list_head, nvgpu_cdev, list_entry) {
-		nvgpu_list_del(&cdev->list_entry);
+		class = cdev->class;
+		if (class->power_node != power_node)
+			continue;
 
-		device_destroy(cdev->class, cdev->cdev.dev);
+		nvgpu_list_del(&cdev->list_entry);
+		device_destroy(nvgpu_class_get_class(cdev->class), cdev->cdev.dev);
 		cdev_del(&cdev->cdev);
 
 		nvgpu_kfree(g, cdev);
 	}
 
+	nvgpu_list_for_each_entry_safe(class, p, &l->class_list_head, nvgpu_class, list_entry) {
+		if (class->power_node != power_node)
+			continue;
+
+		nvgpu_list_del(&class->list_entry);
+		class_destroy(class->class);
+		nvgpu_kfree(g, class);
+	}
+}
+
+void gk20a_power_node_deinit(struct device *dev)
+{
+	struct gk20a *g = gk20a_from_dev(dev);
+	struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
+
+	gk20a_remove_devices_and_classes(g, true);
+
 	if (l->power_cdev_region) {
 		unregister_chrdev_region(l->power_cdev_region, l->power_cdevs);
 	}
+}
+
+void gk20a_user_nodes_deinit(struct device *dev)
+{
+	struct gk20a *g = gk20a_from_dev(dev);
+	struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
+
+	gk20a_remove_devices_and_classes(g, false);
 
 	if (l->cdev_region) {
 		unregister_chrdev_region(l->cdev_region, l->num_cdevs);
 		l->num_cdevs = 0;
 	}
 
-	nvgpu_list_for_each_entry_safe(class, p, &l->class_list_head, nvgpu_class, list_entry) {
-		nvgpu_list_del(&class->list_entry);
-
-		class_destroy(class->class);
-		nvgpu_kfree(g, class);
-	}
-
 	l->dev_nodes_created = false;
 }
 
@@ -585,7 +605,7 @@ int gk20a_power_node_init(struct device *dev)
 			goto fail;
 		}
 
-		cdev->class = class->class;
+		cdev->class = class;
 		nvgpu_init_list_node(&cdev->list_entry);
 		nvgpu_list_add(&cdev->list_entry, &l->cdev_list_head);
 	}
@@ -593,12 +613,12 @@ int gk20a_power_node_init(struct device *dev)
 	l->power_cdevs = total_cdevs;
 	return 0;
 fail:
-	gk20a_user_deinit(dev);
+	gk20a_power_node_deinit(dev);
 	return err;
 
 }
 
-int gk20a_user_init(struct device *dev)
+int gk20a_user_nodes_init(struct device *dev)
 {
 	int err;
 	dev_t devno;
@@ -672,7 +692,7 @@ int gk20a_user_init(struct device *dev)
 				goto fail;
 			}
 
-			cdev->class = class->class;
+			cdev->class = class;
 			nvgpu_init_list_node(&cdev->list_entry);
 			nvgpu_list_add(&cdev->list_entry, &l->cdev_list_head);
 		}
@@ -682,7 +702,7 @@ int gk20a_user_init(struct device *dev)
 
 	return 0;
 fail:
-	gk20a_user_deinit(dev);
+	gk20a_user_nodes_deinit(dev);
 	return err;
 }
 

drivers/gpu/nvgpu/os/linux/ioctl.h

@@ -21,10 +21,25 @@
 struct device;
 struct class;
 
+struct nvgpu_class {
+	struct class *class;
+	struct nvgpu_list_node list_entry;
+
+	struct nvgpu_cdev_class_priv_data *priv_data;
+
+	enum nvgpu_mig_gpu_instance_type instance_type;
+	bool power_node;
+};
+
+static inline struct class *nvgpu_class_get_class(struct nvgpu_class *class)
+{
+	return class->class;
+}
+
 struct nvgpu_cdev {
 	struct cdev cdev;
 	struct device *node;
-	struct class *class;
+	struct nvgpu_class *class;
 	struct nvgpu_list_node list_entry;
 };
 
@@ -43,16 +58,6 @@ struct nvgpu_cdev_class_priv_data {
 	bool pci;
 };
 
-struct nvgpu_class {
-	struct class *class;
-	struct nvgpu_list_node list_entry;
-
-	struct nvgpu_cdev_class_priv_data *priv_data;
-
-	enum nvgpu_mig_gpu_instance_type instance_type;
-	bool power_node;
-};
-
 static inline struct nvgpu_class *
 nvgpu_class_from_list_entry(struct nvgpu_list_node *node)
 {
@@ -60,9 +65,10 @@ nvgpu_class_from_list_entry(struct nvgpu_list_node *node)
 		((uintptr_t)node - offsetof(struct nvgpu_class, list_entry));
 };
 
-int gk20a_user_init(struct device *dev);
+int gk20a_user_nodes_init(struct device *dev);
 int gk20a_power_node_init(struct device *dev);
-void gk20a_user_deinit(struct device *dev);
+void gk20a_user_nodes_deinit(struct device *dev);
+void gk20a_power_node_deinit(struct device *dev);
 
 struct gk20a *nvgpu_get_gk20a_from_cdev(struct nvgpu_cdev *cdev);
 u32 nvgpu_get_gpu_instance_id_from_cdev(struct gk20a *g, struct nvgpu_cdev *cdev);

drivers/gpu/nvgpu/os/linux/module.c

@@ -442,7 +442,7 @@ int gk20a_pm_finalize_poweron(struct device *dev)
 	}
 
 	if (!l->dev_nodes_created) {
-		err = gk20a_user_init(dev);
+		err = gk20a_user_nodes_init(dev);
 		if (err) {
 			goto done;
 		}
@@ -542,7 +542,7 @@ done:
 		nvgpu_disable_irqs(g);
 		nvgpu_remove_sim_support_linux(g);
 		if (l->dev_nodes_created) {
-			gk20a_user_deinit(dev);
+			gk20a_user_nodes_deinit(dev);
 		}
 	}
 
@@ -1783,7 +1783,9 @@ int nvgpu_remove(struct device *dev)
 
 	nvgpu_clk_arb_cleanup_arbiter(g);
 
-	gk20a_user_deinit(dev);
+	gk20a_user_nodes_deinit(dev_from_gk20a(g));
+
+	gk20a_power_node_deinit(dev_from_gk20a(g));
 
 	gk20a_debug_deinit(g);
 

drivers/gpu/nvgpu/os/linux/vgpu/vgpu_linux.c

@@ -217,7 +217,7 @@ int vgpu_pm_finalize_poweron(struct device *dev)
 		goto done;
 
 	if (!l->dev_nodes_created) {
-		err = gk20a_user_init(dev);
+		err = gk20a_user_nodes_init(dev);
 		if (err) {
 			goto done;
 		}
@@ -509,7 +509,7 @@ int vgpu_remove(struct platform_device *pdev)
 
 	vgpu_comm_deinit();
 	gk20a_sched_ctrl_cleanup(g);
-	gk20a_user_deinit(dev);
+	gk20a_user_nodes_deinit(dev);
 	vgpu_remove_sysfs(dev);
 	gk20a_get_platform(dev)->g = NULL;
 	nvgpu_put(g);