gpu: nvgpu: enumerate dev nodes per GPU instance in MIG mode

In MIG mode, each of the dev nodes should be enumerated for each fGPU. And for physical instance only the "ctrl" node should be enumerated. Support this with below set of changes : - Add struct nvgpu_mig_static_info that describes static GPU instance configuration. GPCs are enumerated only during poweron and grmgr unit will populate instance information based on number of GPCs. For linux, GPU poweron happens only with first gk20a_busy() call and instance information is not available during probe() time. Hence this static table is a temporary solution until proper solution is identified. - Add nvgpu_default_mig_static_info for iGPU and nvgpu_default_pci_mig_static_info for dGPU that describes GPU instance partition. - Add new function nvgpu_prepare_mig_dev_node_class_list() that parses the static table and creates one class per instance in MIG mode. Non-MIG mode classes are now enumerated in nvgpu_prepare_default_dev_node_class_list(). - Add new structure nvgpu_cdev_class_priv_data to store private data for each cdev. This will hold instance specific information and pointer to private data will be maintained in struct class and also passed as private data while creating device node with device_create() - Add nvgpu_mig_phys_devnode() to set dev node path/names for fGPUs and add nvgpu_mig_fgpu_devnode() to set dev node path/names for physical instance in MIG mode. - Add new field mig_physical_node to struct nvgpu_dev_node. This field is set if corresponding dev node should be created for physical instance in MIG mode. For now set it only for "ctrl" node. Jira NVGPU-5648 Change-Id: Ic97874eece1fbe0083b3ac4c48e36e06004f1bc2 Signed-off-by: Deepak Nibade <dnibade@nvidia.com> Reviewed-on: https://git-master.nvidia.com/r/c/linux-nvgpu/+/2434586 Reviewed-by: automaticguardword <automaticguardword@nvidia.com> Reviewed-by: Lakshmanan M <lm@nvidia.com> Reviewed-by: mobile promotions <svcmobile_promotions@nvidia.com> GVS: Gerrit_Virtual_Submit Tested-by: mobile promotions <svcmobile_promotions@nvidia.com>
2025-12-22 09:12:24 +03:00 · 2020-10-19 16:06:34 +05:30
parent c1173d11df
commit a1bbcff476
2 changed files with 210 additions and 16 deletions
--- a/drivers/gpu/nvgpu/os/linux/ioctl.c
+++ b/drivers/gpu/nvgpu/os/linux/ioctl.c
@@ -21,6 +21,8 @@

 #include <nvgpu/nvgpu_common.h>
 #include <nvgpu/gk20a.h>
+#include <nvgpu/mig.h>
+#include <nvgpu/nvgpu_init.h>

 #include "ioctl_channel.h"
 #include "ioctl_ctrl.h"
@@ -149,23 +151,27 @@ static const struct file_operations gk20a_sched_ops = {
 };

 struct nvgpu_dev_node {
+	/* Device node name */
 	char name[20];
+	/* file operations for device */
 	const struct file_operations *fops;
+	/* If node should be created for physical instance in MIG mode */
+	bool mig_physical_node;
 };

 static const struct nvgpu_dev_node dev_node_list[] = {
-	{"as",		&gk20a_as_ops},
-	{"channel",	&gk20a_channel_ops},
-	{"ctrl",	&gk20a_ctrl_ops},
+	{"as",		&gk20a_as_ops,		false	},
+	{"channel",	&gk20a_channel_ops,	false	},
+	{"ctrl",	&gk20a_ctrl_ops,	true	},
 #if defined(CONFIG_NVGPU_FECS_TRACE)
-	{"ctxsw",	&gk20a_ctxsw_ops},
+	{"ctxsw",	&gk20a_ctxsw_ops,	false	},
 #endif
-	{"dbg",		&gk20a_dbg_ops},
-	{"prof",	&gk20a_prof_ops},
-	{"prof-ctx",	&gk20a_prof_ctx_ops},
-	{"prof-dev",	&gk20a_prof_dev_ops},
-	{"sched",	&gk20a_sched_ops},
-	{"tsg",		&gk20a_tsg_ops},
+	{"dbg",		&gk20a_dbg_ops,		false	},
+	{"prof",	&gk20a_prof_ops,	false	},
+	{"prof-ctx",	&gk20a_prof_ctx_ops,	false	},
+	{"prof-dev",	&gk20a_prof_dev_ops,	false	},
+	{"sched",	&gk20a_sched_ops,	false	},
+	{"tsg",		&gk20a_tsg_ops,		false	},
 };

 static char *nvgpu_devnode(const char *cdev_name)
@@ -194,12 +200,51 @@ static char *nvgpu_pci_devnode(struct device *dev, umode_t *mode)
 			dev_name(dev->parent), dev_name(dev));
 }

+static char *nvgpu_mig_phys_devnode(struct device *dev, umode_t *mode)
+{
+	struct nvgpu_cdev_class_priv_data *priv_data;
+
+	if (mode) {
+		*mode = S_IRUSR | S_IWUSR;
+	}
+
+	priv_data = dev_get_drvdata(dev);
+
+	if (priv_data->pci) {
+		return kasprintf(GFP_KERNEL, "nvgpu/dgpu-%s/%s",
+				dev_name(dev->parent), dev_name(dev));
+	}
+
+	return kasprintf(GFP_KERNEL, "nvgpu/igpu0/%s", dev_name(dev));
+}
+
+static char *nvgpu_mig_fgpu_devnode(struct device *dev, umode_t *mode)
+{
+	struct nvgpu_cdev_class_priv_data *priv_data;
+
+	if (mode) {
+		*mode = S_IRUSR | S_IWUSR;
+	}
+
+	priv_data = dev_get_drvdata(dev);
+
+	if (priv_data->pci) {
+		return kasprintf(GFP_KERNEL, "nvgpu/dgpu-%s/fgpu-%u-%u/%s",
+				dev_name(dev->parent), priv_data->major_instance_id,
+				priv_data->minor_instance_id, dev_name(dev));
+	}
+
+	return kasprintf(GFP_KERNEL, "nvgpu/igpu0/fgpu-%u-%u/%s",
+				priv_data->major_instance_id,
+				priv_data->minor_instance_id, dev_name(dev));
+}
+
 static int gk20a_create_device(
 	struct device *dev, int devno,
 	const char *cdev_name,
 	struct cdev *cdev, struct device **out,
 	const struct file_operations *ops,
-	struct class *class)
+	struct nvgpu_class *class)
 {
 	struct device *subdev;
 	int err;
@@ -217,11 +262,12 @@ static int gk20a_create_device(
 		return err;
 	}

-	if (class->devnode == NULL) {
+	if (class->class->devnode == NULL) {
 		device_name = nvgpu_devnode(cdev_name);
 	}

-	subdev = device_create(class, dev, devno, NULL,
+	subdev = device_create(class->class, dev, devno,
+			class->priv_data ? class->priv_data : NULL,
 			device_name ? device_name : cdev_name);
 	if (IS_ERR(subdev)) {
 		err = PTR_ERR(dev);
@@ -268,7 +314,7 @@ void gk20a_user_deinit(struct device *dev)
 	}
 }

-struct nvgpu_class *nvgpu_create_class(struct gk20a *g, const char *class_name)
+static struct nvgpu_class *nvgpu_create_class(struct gk20a *g, const char *class_name)
 {
 	struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g);
 	struct nvgpu_class *class;
@@ -291,7 +337,113 @@ struct nvgpu_class *nvgpu_create_class(struct gk20a *g, const char *class_name)
 	return class;
 }

-int nvgpu_prepare_dev_node_class_list(struct gk20a *g, u32 *num_classes)
+/*
+ * GPU instance information in MIG mode should be fetched from
+ * common.grmgr unit. But instance information is populated during GPU
+ * poweron and device nodes are enumerated during probe.
+ *
+ * Handle this temporarily by adding static information of instances
+ * where GPU is partitioned into two instances. In long term, this will
+ * need to be handled with design changes.
+ *
+ * This static information should be removed once instance information
+ * is fetched from common.grmgr unit.
+ */
+struct nvgpu_mig_static_info {
+	enum nvgpu_mig_gpu_instance_type instance_type;
+	u32 major_instance_id;
+	u32 minor_instance_id;
+};
+
+static const struct nvgpu_mig_static_info nvgpu_default_mig_static_info[] =
+{
+	{
+		.instance_type = NVGPU_MIG_TYPE_PHYSICAL,
+	},
+	{
+		.instance_type = NVGPU_MIG_TYPE_MIG,
+		.major_instance_id = 0,
+		.minor_instance_id = 0,
+	},
+	{
+		.instance_type = NVGPU_MIG_TYPE_MIG,
+		.major_instance_id = 0,
+		.minor_instance_id = 1,
+	},
+};
+
+static const struct nvgpu_mig_static_info nvgpu_default_pci_mig_static_info[] =
+{
+	{
+		.instance_type = NVGPU_MIG_TYPE_PHYSICAL,
+	},
+	{
+		.instance_type = NVGPU_MIG_TYPE_MIG,
+		.major_instance_id = 1,
+		.minor_instance_id = 0,
+	},
+	{
+		.instance_type = NVGPU_MIG_TYPE_MIG,
+		.major_instance_id = 2,
+		.minor_instance_id = 0,
+	},
+};
+
+static int nvgpu_prepare_mig_dev_node_class_list(struct gk20a *g, u32 *num_classes)
+{
+	u32 class_count = 0U;
+	const struct nvgpu_mig_static_info *info;
+	struct nvgpu_class *class;
+	u32 i;
+	u32 num_instances;
+	struct nvgpu_cdev_class_priv_data *priv_data;
+
+	if (g->pci_class != 0U) {
+		info = &nvgpu_default_pci_mig_static_info[0];
+		num_instances = sizeof(nvgpu_default_pci_mig_static_info) /
+				sizeof(nvgpu_default_pci_mig_static_info[0]);
+	} else {
+		info = &nvgpu_default_mig_static_info[0];
+		num_instances = sizeof(nvgpu_default_mig_static_info) /
+				sizeof(nvgpu_default_mig_static_info[0]);
+	}
+
+	for (i = 0U; i < num_instances; i++) {
+		priv_data = nvgpu_kzalloc(g, sizeof(*priv_data));
+		if (priv_data == NULL) {
+			return -ENOMEM;
+		}
+
+		snprintf(priv_data->class_name, sizeof(priv_data->class_name),
+			"nvidia%s-gpu-fgpu%u",
+			(g->pci_class != 0U) ? "-pci" : "", i);
+
+		class = nvgpu_create_class(g, priv_data->class_name);
+		if (class == NULL) {
+			kfree(priv_data);
+			return -ENOMEM;
+		}
+		class_count++;
+
+		if (info[i].instance_type == NVGPU_MIG_TYPE_PHYSICAL) {
+			class->class->devnode = nvgpu_mig_phys_devnode;
+		} else {
+			class->class->devnode = nvgpu_mig_fgpu_devnode;
+		}
+
+		priv_data->local_instance_id = i;
+		priv_data->major_instance_id = info[i].major_instance_id;
+		priv_data->minor_instance_id = info[i].minor_instance_id;
+		priv_data->pci = (g->pci_class != 0U);
+		class->priv_data = priv_data;
+		class->instance_type = info[i].instance_type;
+	}
+
+	*num_classes = class_count;
+	return 0;
+}
+
+static int nvgpu_prepare_default_dev_node_class_list(struct gk20a *g, u32 *num_classes)
 {
 	struct nvgpu_class *class;
 	u32 count = 0U;
@@ -316,6 +468,32 @@ int nvgpu_prepare_dev_node_class_list(struct gk20a *g, u32 *num_classes)
 	return 0;
 }

+static int nvgpu_prepare_dev_node_class_list(struct gk20a *g, u32 *num_classes)
+{
+	int err;
+
+	if (nvgpu_is_enabled(g, NVGPU_SUPPORT_MIG)) {
+		err = nvgpu_prepare_mig_dev_node_class_list(g, num_classes);
+	} else {
+		err = nvgpu_prepare_default_dev_node_class_list(g, num_classes);
+	}
+
+	return err;
+}
+
+static bool check_valid_dev_node(struct gk20a *g, struct nvgpu_class *class,
+		const struct nvgpu_dev_node *node)
+{
+	if (nvgpu_is_enabled(g, NVGPU_SUPPORT_MIG)) {
+		if ((class->instance_type == NVGPU_MIG_TYPE_PHYSICAL) &&
+		    !node->mig_physical_node) {
+			return false;
+		}
+	}
+
+	return true;
+}
+
 int gk20a_user_init(struct device *dev)
 {
 	int err;
@@ -348,6 +526,10 @@ int gk20a_user_init(struct device *dev)

 	nvgpu_list_for_each_entry(class, &l->class_list_head, nvgpu_class, list_entry) {
 		for (cdev_index = 0; cdev_index < num_cdevs; cdev_index++) {
+			if (!check_valid_dev_node(g, class, &dev_node_list[cdev_index])) {
+				continue;
+			}
+
 			cdev = nvgpu_kzalloc(g, sizeof(*cdev));
 			if (cdev == NULL) {
 				dev_err(dev, "failed to allocate cdev\n");
@@ -358,7 +540,7 @@ int gk20a_user_init(struct device *dev)
 					dev_node_list[cdev_index].name,
 					&cdev->cdev, &cdev->node,
 					dev_node_list[cdev_index].fops,
-					class->class);
+					class);
 			if (err) {
 				goto fail;
 			}
--- a/drivers/gpu/nvgpu/os/linux/os_linux.h
+++ b/drivers/gpu/nvgpu/os/linux/os_linux.h
@@ -87,9 +87,21 @@ nvgpu_cdev_from_list_entry(struct nvgpu_list_node *node)
 		((uintptr_t)node - offsetof(struct nvgpu_cdev, list_entry));
 };

+struct nvgpu_cdev_class_priv_data {
+	char class_name[64];
+	u32 local_instance_id;
+	u32 major_instance_id;
+	u32 minor_instance_id;
+	bool pci;
+};
+
 struct nvgpu_class {
 	struct class *class;
 	struct nvgpu_list_node list_entry;
+
+	struct nvgpu_cdev_class_priv_data *priv_data;
+
+	enum nvgpu_mig_gpu_instance_type instance_type;
 };

 static inline struct nvgpu_class *