nvgpu: gpu: update default nvs domain implementation

In current form, the default domain acts like any schedulable
domain. TSGs are bound to it and it can be enumerated via the
public interfaces.

The new expectation for the default domain is meant to change
from the current form to a pseudo domain that cannot act like
an ordinary domain in other ways, i.e. it must not be reachable
by in particular the domain management API, it can't be removed,
does not show up in lists, and TSGs cannot be explicitly bound to
this domain. It won't participate in round-robin domain scheduling.
It is not really a domain, and acts like one only when activated in
the manual mode.

Following changes are made overall to support the above change in
definition.

1) Domain creation and attaching the domain to the scheduler are now
split into two separate functions. The new default domain (having ID
= UINT64_MAX) is created separately from a static function without
linking it with other domains in the scheduler.

2) struct nvgpu_nvs_scheduler explicitely stores the default domain
to support direct lookups.

3) TSGs are initially not bound to default domain/rl_domain.

Jira NVGPU-8165

Signed-off-by: Debarshi Dutta <ddutta@nvidia.com>
Change-Id: I916d11f4eea5124d8d64176dc77f3806c6139695
Reviewed-on: https://git-master.nvidia.com/r/c/linux-nvgpu/+/2697477
Tested-by: mobile promotions <svcmobile_promotions@nvidia.com>
Reviewed-by: mobile promotions <svcmobile_promotions@nvidia.com>

drivers/gpu/nvgpu/common/fifo/tsg.c

@@ -89,15 +89,6 @@ int nvgpu_tsg_bind_channel(struct nvgpu_tsg *tsg, struct nvgpu_channel *ch)
 		return -EINVAL;
 	}
 
-	/*
-	 * This runlist domain is set either by default or in an explicit
-	 * bind. If the default domain has been deleted, explicit bind is
-	 * mandatory.
-	 */
-	if (tsg->rl_domain == NULL) {
-		return -EINVAL;
-	}
-
 	/* cannot bind more channels than MAX channels supported per TSG */
 	nvgpu_rwsem_down_read(&tsg->ch_list_lock);
 	max_ch_per_tsg = g->ops.runlist.get_max_channels_per_tsg();
@@ -119,14 +110,17 @@ int nvgpu_tsg_bind_channel(struct nvgpu_tsg *tsg, struct nvgpu_channel *ch)
 	 */
 	if (tsg->runlist == NULL) {
 		tsg->runlist = ch->runlist;
-		/*
-		 * The rl domain identifier is stashed in tsg->rl_domain->name
-		 * when the tsg is bound to a domain, but at that point there
-		 * are no channels yet to describe which runlist id should be
-		 * used. Now we know.
-		 */
-		tsg->rl_domain = nvgpu_rl_domain_get(g, tsg->runlist->id, tsg->rl_domain->name);
-		WARN_ON(tsg->rl_domain == NULL);
+		if (tsg->rl_domain != NULL) {
+			/*
+			 * The rl domain identifier is stashed in tsg->rl_domain->name
+			 * when the tsg is bound to a domain, but at that point there
+			 * are no channels yet to describe which runlist id should be
+			 * used. Now we know.
+			 */
+			tsg->rl_domain = nvgpu_rl_domain_get(g, tsg->runlist->id,
+				tsg->rl_domain->name);
+			WARN_ON(tsg->rl_domain == NULL);
+		}
 	} else {
 		if (tsg->runlist != ch->runlist) {
 			nvgpu_err(tsg->g,
@@ -862,12 +856,8 @@ int nvgpu_tsg_open_common(struct gk20a *g, struct nvgpu_tsg *tsg, pid_t pid)
 	tsg->interleave_level = NVGPU_FIFO_RUNLIST_INTERLEAVE_LEVEL_LOW;
 	tsg->timeslice_us = g->ops.tsg.default_timeslice_us(g);
 	tsg->runlist = NULL;
-	/*
-	 * The domain ptr will get updated with the right id once the runlist
-	 * gets specified based on the first channel.
-	 */
-	tsg->rl_domain = nvgpu_rl_domain_get(g, 0, "(default)");
-	tsg->nvs_domain = nvgpu_nvs_domain_by_name(g, "(default)");
+	tsg->rl_domain = NULL;
+	tsg->nvs_domain = NULL;
 #ifdef CONFIG_NVGPU_DEBUGGER
 	tsg->sm_exception_mask_type = NVGPU_SM_EXCEPTION_TYPE_MASK_NONE;
 #endif

drivers/gpu/nvgpu/common/nvs/nvs_sched.c

@@ -22,6 +22,7 @@
 
 #include <nvs/log.h>
 #include <nvs/sched.h>
+#include <nvgpu/types.h>
 
 #include <nvgpu/nvs.h>
 #include <nvgpu/kmem.h>
@@ -92,7 +93,7 @@ static u64 nvgpu_nvs_tick(struct gk20a *g)
 {
 	struct nvgpu_nvs_scheduler *sched = g->scheduler;
 	struct nvgpu_nvs_domain *domain;
-	struct nvs_domain *nvs_domain;
+	struct nvs_domain *nvs_next;
 	u64 timeslice;
 
 	nvs_dbg(g, "nvs tick");
@@ -101,20 +102,16 @@ static u64 nvgpu_nvs_tick(struct gk20a *g)
 
 	domain = sched->active_domain;
 
-	if (domain == NULL) {
-		/* nothing to schedule, TODO wait for an event instead */
-		nvgpu_mutex_release(&g->sched_mutex);
-		return 100 * NSEC_PER_MSEC;
+	/* If active_domain == shadow_domain, then nvs_next is NULL */
+	nvs_next = nvs_domain_get_next_domain(sched->sched, domain->parent);
+	if (nvs_next == NULL) {
+		nvs_next = sched->shadow_domain->parent;
 	}
 
-	nvs_domain = domain->parent->next;
-	if (nvs_domain == NULL) {
-		nvs_domain = g->scheduler->sched->domain_list->domains;
-	}
-	timeslice = nvs_domain->timeslice_ns;
+	timeslice = nvs_next->timeslice_ns;
 
 	nvgpu_runlist_tick(g);
-	sched->active_domain = nvs_domain->priv;
+	sched->active_domain = nvs_next->priv;
 
 	nvgpu_mutex_release(&g->sched_mutex);
 
@@ -165,9 +162,93 @@ static void nvgpu_nvs_worker_deinit(struct gk20a *g)
 	nvs_dbg(g, "NVS worker suspended");
 }
 
+static struct nvgpu_nvs_domain *
+	nvgpu_nvs_gen_domain(struct gk20a *g, const char *name, u64 id,
+		u64 timeslice, u64 preempt_grace)
+{
+	struct nvs_domain *nvs_dom = NULL;
+	struct nvgpu_nvs_domain *nvgpu_dom = NULL;
+
+	nvs_dbg(g, "Adding new domain: %s", name);
+
+	nvgpu_dom = nvgpu_kzalloc(g, sizeof(*nvgpu_dom));
+	if (nvgpu_dom == NULL) {
+		nvs_dbg(g, "failed to allocate memory for domain %s", name);
+		return nvgpu_dom;
+	}
+
+	nvgpu_dom->id = id;
+	nvgpu_dom->ref = 1U;
+
+	nvs_dom = nvs_domain_create(g->scheduler->sched, name,
+				timeslice, preempt_grace, nvgpu_dom);
+
+	if (nvs_dom == NULL) {
+		nvs_dbg(g, "failed to create nvs domain for %s", name);
+		nvgpu_kfree(g, nvgpu_dom);
+		nvgpu_dom = NULL;
+		return nvgpu_dom;
+	}
+
+	nvgpu_dom->parent = nvs_dom;
+
+	return nvgpu_dom;
+}
+
+static int nvgpu_nvs_gen_shadow_domain(struct gk20a *g)
+{
+	int err = 0;
+	struct nvgpu_nvs_domain *nvgpu_dom;
+
+	if (g->scheduler->shadow_domain != NULL) {
+		goto error;
+	}
+
+	nvgpu_dom = nvgpu_nvs_gen_domain(g, SHADOW_DOMAIN_NAME, U64_MAX,
+		100U * NSEC_PER_MSEC, 0U);
+	if (nvgpu_dom == NULL) {
+		err = -ENOMEM;
+		goto error;
+	}
+
+	g->scheduler->shadow_domain = nvgpu_dom;
+
+	/* Set active_domain to shadow_domain during Init */
+	g->scheduler->active_domain = g->scheduler->shadow_domain;
+
+error:
+	return err;
+}
+
+static void nvgpu_nvs_remove_shadow_domain(struct gk20a *g)
+{
+	struct nvgpu_nvs_scheduler *sched = g->scheduler;
+	struct nvs_domain *nvs_dom;
+
+	if (sched == NULL) {
+		/* never powered on to init anything */
+		return;
+	}
+
+	if (sched->shadow_domain == NULL) {
+		return;
+	}
+
+	if (sched->shadow_domain->ref != 1U) {
+		nvgpu_warn(g,
+				"domain %llu is still in use during shutdown! refs: %u",
+				sched->shadow_domain->id, sched->shadow_domain->ref);
+	}
+
+	nvs_dom = sched->shadow_domain->parent;
+	nvs_domain_destroy(sched->sched, nvs_dom);
+
+	nvgpu_kfree(g, sched->shadow_domain);
+	sched->shadow_domain = NULL;
+}
+
 int nvgpu_nvs_init(struct gk20a *g)
 {
-	struct nvgpu_nvs_domain *domain;
 	int err;
 
 	nvgpu_mutex_init(&g->sched_mutex);
@@ -177,16 +258,6 @@ int nvgpu_nvs_init(struct gk20a *g)
 		return err;
 	}
 
-	if (nvgpu_rl_domain_get(g, 0, "(default)") == NULL) {
-		int err = nvgpu_nvs_add_domain(g, "(default)",
-				100U * NSEC_PER_MSEC,
-				0U,
-				&domain);
-		if (err != 0) {
-			return err;
-		}
-	}
-
 	return 0;
 }
 
@@ -214,6 +285,8 @@ void nvgpu_nvs_remove_support(struct gk20a *g)
 		nvgpu_kfree(g, nvgpu_dom);
 	}
 
+	nvgpu_nvs_remove_shadow_domain(g);
+
 	nvs_sched_close(sched->sched);
 	nvgpu_kfree(g, sched->sched);
 	nvgpu_kfree(g, sched);
@@ -247,15 +320,22 @@ int nvgpu_nvs_open(struct gk20a *g)
 		goto unlock;
 	}
 
-	err = nvgpu_nvs_worker_init(g);
+	nvs_dbg(g, "  Creating NVS scheduler.");
+	err = nvs_sched_create(g->scheduler->sched, &nvgpu_nvs_ops, g);
 	if (err != 0) {
 		goto unlock;
 	}
 
-	nvs_dbg(g, "  Creating NVS scheduler.");
-	err = nvs_sched_create(g->scheduler->sched, &nvgpu_nvs_ops, g);
+	if (nvgpu_rl_domain_get(g, 0, SHADOW_DOMAIN_NAME) == NULL) {
+		err = nvgpu_nvs_gen_shadow_domain(g);
+		if (err != 0) {
+			goto unlock;
+		}
+	}
+
+	err = nvgpu_nvs_worker_init(g);
 	if (err != 0) {
-		nvgpu_nvs_worker_deinit(g);
+		nvgpu_nvs_remove_shadow_domain(g);
 		goto unlock;
 	}
 
@@ -289,8 +369,6 @@ int nvgpu_nvs_add_domain(struct gk20a *g, const char *name, u64 timeslice,
 	struct nvs_domain *nvs_dom;
 	struct nvgpu_nvs_domain *nvgpu_dom;
 
-	nvs_dbg(g, "Adding new domain: %s", name);
-
 	nvgpu_mutex_acquire(&g->sched_mutex);
 
 	if (nvs_domain_by_name(g->scheduler->sched, name) != NULL) {
@@ -298,37 +376,29 @@ int nvgpu_nvs_add_domain(struct gk20a *g, const char *name, u64 timeslice,
 		goto unlock;
 	}
 
-	nvgpu_dom = nvgpu_kzalloc(g, sizeof(*nvgpu_dom));
+	nvgpu_dom = nvgpu_nvs_gen_domain(g, name, nvgpu_nvs_new_id(g),
+		timeslice, preempt_grace);
 	if (nvgpu_dom == NULL) {
 		err = -ENOMEM;
 		goto unlock;
 	}
 
-	nvgpu_dom->id = nvgpu_nvs_new_id(g);
-	nvgpu_dom->ref = 1U;
+	nvs_dom = nvgpu_dom->parent;
 
-	nvs_dom = nvs_domain_create(g->scheduler->sched, name,
-				    timeslice, preempt_grace, nvgpu_dom);
-
-	if (nvs_dom == NULL) {
-		nvs_dbg(g, "failed to create nvs domain for %s", name);
-		nvgpu_kfree(g, nvgpu_dom);
-		err = -ENOMEM;
-		goto unlock;
-	}
+	nvs_domain_scheduler_attach(g->scheduler->sched, nvs_dom);
 
 	err = nvgpu_rl_domain_alloc(g, name);
 	if (err != 0) {
 		nvs_dbg(g, "failed to alloc rl domain for %s", name);
-		nvs_domain_destroy(g->scheduler->sched, nvs_dom);
+		nvs_domain_unlink_and_destroy(g->scheduler->sched, nvs_dom);
 		nvgpu_kfree(g, nvgpu_dom);
 		goto unlock;
 	}
 
-
 	nvgpu_dom->parent = nvs_dom;
 
-	if (g->scheduler->active_domain == NULL) {
+	/* Set the first user created domain as active domain */
+	if (g->scheduler->active_domain == g->scheduler->shadow_domain) {
 		g->scheduler->active_domain = nvgpu_dom;
 	}
 
@@ -464,14 +534,15 @@ int nvgpu_nvs_del_domain(struct gk20a *g, u64 dom_id)
 
 	/* note: same wraparound logic as in RL domains to keep in sync */
 	if (s->active_domain == nvgpu_dom) {
-		nvs_next = nvs_dom->next;
-		if (nvs_next == NULL) {
-			nvs_next = s->sched->domain_list->domains;
+		nvs_next = nvs_domain_get_next_domain(s->sched, nvs_dom);
+		/* Its the only entry in the list. Set the default domain as the active domain */
+		if (nvs_next == nvs_dom) {
+			nvs_next = s->shadow_domain->parent;
 		}
 		s->active_domain = nvs_next->priv;
 	}
 
-	nvs_domain_destroy(s->sched, nvs_dom);
+	nvs_domain_unlink_and_destroy(s->sched, nvs_dom);
 	nvgpu_kfree(g, nvgpu_dom);
 
 unlock:

drivers/gpu/nvgpu/hal/fifo/runlist_fifo_ga10b_fusa.c

@@ -72,6 +72,9 @@ void ga10b_runlist_hw_submit(struct gk20a *g, struct nvgpu_runlist *runlist)
 			runlist_submit_base_hi_ptr_hi_f(runlist_iova_hi));
 	}
 
+	rl_dbg(g, "Submitting domain[%s], mem=0x%16llx", runlist->domain->name,
+		(u64)nvgpu_mem_get_addr(g, &runlist->domain->mem_hw->mem));
+
 	/* TODO offset in runlist support */
 	nvgpu_runlist_writel(g, runlist, runlist_submit_r(),
 			runlist_submit_offset_f(0U) |

drivers/gpu/nvgpu/include/nvgpu/nvs.h

@@ -84,6 +84,7 @@ struct nvgpu_nvs_scheduler {
 	nvgpu_atomic64_t id_counter;
 	struct nvgpu_nvs_worker worker;
 	struct nvgpu_nvs_domain *active_domain;
+	struct nvgpu_nvs_domain *shadow_domain;
 };
 
 #ifdef CONFIG_NVS_PRESENT

drivers/gpu/nvgpu/include/nvgpu/runlist.h

@@ -156,7 +156,8 @@ struct nvgpu_runlist {
 	 */
 	struct nvgpu_runlist_domain *shadow_rl_domain;
 	/*
-	 * All scheduling domains of this RL, see nvgpu_runlist_domain::domain_node.
+	 * All scheduling domains(created by users) of this RL,
+	 * see nvgpu_runlist_domain::domain_node.
 	 *
 	 * Design note: the runlist hardware unit should not own the actual
 	 * domain memory; this arrangement is temporary to aid in transition

drivers/gpu/nvgpu/include/nvgpu/tsg.h

@@ -179,7 +179,7 @@ struct nvgpu_tsg {
 	struct nvgpu_runlist *runlist;
 
 	/**
-	 * Runlist domain this TSG is bound to. Bound with an ioctl, initially the default domain.
+	 * Runlist domain this TSG is bound to. Bound with an ioctl, initially empty
 	 */
 	struct nvgpu_runlist_domain *rl_domain;
 

nvsched/include/nvs/domain.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 NVIDIA Corporation.  All rights reserved.
+ * Copyright (c) 2021-2022 NVIDIA Corporation.  All rights reserved.
  *
  * NVIDIA Corporation and its licensors retain all intellectual property
  * and proprietary rights in and to this software, related documentation
@@ -68,7 +68,11 @@ struct nvs_domain {
 struct nvs_domain *nvs_domain_create(struct nvs_sched *sched,
 		  const char *name, u64 timeslice, u64 preempt_grace,
 		  void *priv);
+struct nvs_domain *nvs_domain_get_next_domain(struct nvs_sched *sched, struct nvs_domain *dom);
+void nvs_domain_scheduler_attach(struct nvs_sched *sched, struct nvs_domain *dom);
 void nvs_domain_destroy(struct nvs_sched *sched, struct nvs_domain *dom);
+void nvs_domain_unlink_and_destroy(struct nvs_sched *sched,
+			struct nvs_domain *dom);
 void nvs_domain_clear_all(struct nvs_sched *sched);
 u32 nvs_domain_count(struct nvs_sched *sched);
 struct nvs_domain *nvs_domain_by_name(struct nvs_sched *sched, const char *name);

nvsched/src/domain.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 NVIDIA Corporation.  All rights reserved.
+ * Copyright (c) 2021-2022 NVIDIA Corporation.  All rights reserved.
  *
  * NVIDIA Corporation and its licensors retain all intellectual property
  * and proprietary rights in and to this software, related documentation
@@ -13,13 +13,12 @@
 #include <nvs/domain.h>
 
 /*
- * Create and add a new domain to the end of the domain list.
+ * Create a new domain and populate it with default values
  */
 struct nvs_domain *nvs_domain_create(struct nvs_sched *sched,
 		  const char *name, u64 timeslice, u64 preempt_grace,
 		  void *priv)
 {
-	struct nvs_domain_list *dlist = sched->domain_list;
 	struct nvs_domain *dom = nvs_malloc(sched, sizeof(*dom));
 
 	nvs_log(sched, "Creating domain - %s", name);
@@ -34,8 +33,28 @@ struct nvs_domain *nvs_domain_create(struct nvs_sched *sched,
 	dom->timeslice_ns     = timeslice;
 	dom->preempt_grace_ns = preempt_grace;
 	dom->priv             = priv;
+	dom->next             = NULL;
 
-	nvs_log_event(sched, NVS_EV_CREATE_DOMAIN, 0U);
+	return dom;
+}
+
+struct nvs_domain *nvs_domain_get_next_domain(struct nvs_sched *sched, struct nvs_domain *dom)
+{
+	struct nvs_domain *nvs_next = dom->next;
+
+	if (nvs_next == NULL) {
+		nvs_next = sched->domain_list->domains;
+	}
+
+	return nvs_next;
+}
+
+/*
+ * Attach a new domain to the end of the domain list.
+ */
+void nvs_domain_scheduler_attach(struct nvs_sched *sched, struct nvs_domain *dom)
+{
+	struct nvs_domain_list *dlist = sched->domain_list;
 
 	/*
 	 * Now add the domain to the list of domains. If this is the first
@@ -46,14 +65,13 @@ struct nvs_domain *nvs_domain_create(struct nvs_sched *sched,
 	if (dlist->domains == NULL) {
 		dlist->domains = dom;
 		dlist->last    = dom;
-		return dom;
+		return;
 	}
 
 	dlist->last->next = dom;
 	dlist->last       = dom;
 
-	nvs_log(sched, "%s: Domain added", name);
-	return dom;
+	nvs_log(sched, "%s: Domain added", dom->name);
 }
 
 /*
@@ -66,7 +84,6 @@ static void nvs_domain_unlink(struct nvs_sched *sched,
 	struct nvs_domain_list *dlist = sched->domain_list;
 	struct nvs_domain *tmp;
 
-
 	if (dlist->domains == dom) {
 		dlist->domains = dom->next;
 
@@ -76,6 +93,7 @@ static void nvs_domain_unlink(struct nvs_sched *sched,
 		 * pointer as well.
 		 */
 		if (dom == dlist->last) {
+			dlist->domains = NULL;
 			dlist->last = NULL;
 		}
 		return;
@@ -99,15 +117,20 @@ static void nvs_domain_unlink(struct nvs_sched *sched,
 
 void nvs_domain_destroy(struct nvs_sched *sched,
 			struct nvs_domain *dom)
+{
+	nvs_memset(dom, 0, sizeof(*dom));
+	nvs_free(sched, dom);
+}
+
+void nvs_domain_unlink_and_destroy(struct nvs_sched *sched,
+			struct nvs_domain *dom)
 {
 	nvs_log_event(sched, NVS_EV_REMOVE_DOMAIN, 0);
 
 	nvs_domain_unlink(sched, dom);
-
-	nvs_memset(dom, 0, sizeof(*dom));
-	nvs_free(sched, dom);
-
 	sched->domain_list->nr--;
+
+	nvs_domain_destroy(sched, dom);
 }
 
 void nvs_domain_clear_all(struct nvs_sched *sched)
@@ -115,7 +138,7 @@ void nvs_domain_clear_all(struct nvs_sched *sched)
 	struct nvs_domain_list *dlist = sched->domain_list;
 
 	while (dlist->domains != NULL) {
-		nvs_domain_destroy(sched, dlist->domains);
+		nvs_domain_unlink_and_destroy(sched, dlist->domains);
 	}
 }