nvgpu: gpu: separate runlist submit from construction

This patch primary separates runlist modification from runlist submits. Instead of submitting the runlist(domain) immediately after modification, a worker thread interface is now being used to synchronously schedule runlist submits. If the runlist being scheduled is currently active, the submit happens instantly, otherwise, it will happen in the next iteration when the nvs thread will schedule the domain. This external interface uses a condition variable to wait for the completion of the synchronous submits. A pending_update variable is used to synchronize domain memory swaps just before being submitted. To facilitate faster scheduling via the NVS thread, nvgpu_dom itself contains an array of rl_domain pointers. This can then be used to select the appropriate rl_domain directly for scheduling as against the earlier approach of maintaining nvs domains and rl domains in sync everytime. Signed-off-by: Debarshi Dutta <ddutta@nvidia.com> Change-Id: I1725c7cf56407cca2e3d2589833d1c0b66a7ad7b Reviewed-on: https://git-master.nvidia.com/r/c/linux-nvgpu/+/2739795 Reviewed-by: svc-mobile-coverity <svc-mobile-coverity@nvidia.com> Reviewed-by: Ramesh Mylavarapu <rmylavarapu@nvidia.com> Reviewed-by: Vaibhav Kachore <vkachore@nvidia.com> GVS: Gerrit_Virtual_Submit
2025-12-24 10:34:43 +03:00 · 2022-05-26 11:12:56 +05:30
parent e355ab9c21
commit d8e8eb65d3
9 changed files with 457 additions and 221 deletions
--- a/drivers/gpu/nvgpu/common/nvs/nvs_sched.c
+++ b/drivers/gpu/nvgpu/common/nvs/nvs_sched.c
@@ -43,8 +43,18 @@ static struct nvs_sched_ops nvgpu_nvs_ops = {
 *    - currently it just locks all affected runlists
 *    - consider pausing the scheduler logic and signaling users
 */
+
 struct nvgpu_nvs_worker_item {
+	struct gk20a *g;
+	struct nvgpu_runlist *rl;
+	struct nvgpu_runlist_domain *rl_domain;
+	struct nvgpu_cond cond;
+	bool swap_buffer;
+	bool wait_for_finish;
+	bool locked;
+	int status;
 	struct nvgpu_list_node list;
+	nvgpu_atomic_t state;
 };

 static inline struct nvgpu_nvs_worker_item *
@@ -70,6 +80,9 @@ static void nvgpu_nvs_worker_poll_init(struct nvgpu_worker *worker)
 	nvs_worker->current_timeout = 100;
 	nvgpu_timeout_init_cpu_timer_sw(worker->g, &nvs_worker->timeout,
 			nvs_worker->current_timeout);
+
+	nvgpu_atomic_set(&nvs_worker->nvs_sched_init, 1);
+	nvgpu_cond_signal(&nvs_worker->worker.wq);
 }

 static u32 nvgpu_nvs_worker_wakeup_timeout(struct nvgpu_worker *worker)
@@ -80,20 +93,12 @@ static u32 nvgpu_nvs_worker_wakeup_timeout(struct nvgpu_worker *worker)
 	return nvs_worker->current_timeout;
 }

-static void nvgpu_nvs_worker_wakeup_process_item(
-		struct nvgpu_list_node *work_item)
-{
-	struct nvgpu_nvs_worker_item *item =
-		nvgpu_nvs_worker_item_from_worker_item(work_item);
-	(void)item;
-	/* placeholder; never called yet */
-}
-
 static u64 nvgpu_nvs_tick(struct gk20a *g)
 {
 	struct nvgpu_nvs_scheduler *sched = g->scheduler;
 	struct nvgpu_nvs_domain *domain;
 	struct nvs_domain *nvs_next;
+	struct nvgpu_nvs_domain *nvgpu_domain_next;
 	u64 timeslice;

 	nvs_dbg(g, "nvs tick");
@@ -109,8 +114,9 @@ static u64 nvgpu_nvs_tick(struct gk20a *g)
 	}

 	timeslice = nvs_next->timeslice_ns;
+	nvgpu_domain_next = nvs_next->priv;

-	nvgpu_runlist_tick(g);
+	nvgpu_runlist_tick(g, nvgpu_domain_next->rl_domains);
 	sched->active_domain = nvs_next->priv;

 	nvgpu_mutex_release(&g->sched_mutex);
@@ -118,6 +124,113 @@ static u64 nvgpu_nvs_tick(struct gk20a *g)
 	return timeslice;
 }

+static void nvgpu_nvs_worker_wakeup_process_item(struct nvgpu_list_node *work_item)
+{
+	struct nvgpu_nvs_worker_item *work =
+			nvgpu_nvs_worker_item_from_worker_item(work_item);
+	struct gk20a *g = work->g;
+	int ret = 0;
+	struct nvgpu_nvs_scheduler *sched = g->scheduler;
+	struct nvs_domain *nvs_domain;
+	struct nvgpu_runlist *runlist = work->rl;
+	struct nvgpu_runlist_domain *rl_domain = work->rl_domain;
+
+	nvgpu_mutex_acquire(&g->sched_mutex);
+
+	if (rl_domain == NULL) {
+		nvs_domain = sched->shadow_domain->parent;
+		rl_domain = runlist->shadow_rl_domain;
+	} else if (strcmp(rl_domain->name, SHADOW_DOMAIN_NAME) == 0) {
+		nvs_domain = sched->shadow_domain->parent;
+	} else {
+		nvs_domain = nvs_domain_by_name(sched->sched, rl_domain->name);
+		if (nvs_domain == NULL) {
+			nvgpu_err(g, "Unable to find domain[%s]", rl_domain->name);
+			ret = -EINVAL;
+			goto done;
+		}
+	}
+
+	if (sched->active_domain == nvs_domain->priv) {
+		/* Instantly switch domain and force runlist updates */
+		ret = nvgpu_rl_domain_sync_submit(g, runlist, rl_domain, work->swap_buffer, work->wait_for_finish);
+	} else {
+		/* Swap buffers here even if its deferred for correctness */
+		if (work->swap_buffer) {
+			nvgpu_runlist_swap_mem(g, rl_domain);
+		}
+		ret = 1;
+	}
+
+	nvs_dbg(g, " ");
+
+done:
+	nvgpu_mutex_release(&g->sched_mutex);
+	work->status = ret;
+	nvgpu_atomic_set(&work->state, 1);
+	/* Wakeup threads waiting on runlist submit */
+	nvgpu_cond_signal(&work->cond);
+}
+
+static int nvgpu_nvs_worker_submit(struct gk20a *g, struct nvgpu_runlist *rl,
+		struct nvgpu_runlist_domain *rl_domain, bool swap_buffer,
+		bool wait_for_finish)
+{
+	struct nvgpu_nvs_scheduler *sched = g->scheduler;
+	struct nvgpu_nvs_worker *worker = &sched->worker;
+	struct nvgpu_nvs_worker_item *work;
+	int ret = 0;
+
+	if (sched == NULL) {
+		return -ENODEV;
+	}
+
+	nvs_dbg(g, " ");
+
+	work = nvgpu_kzalloc(g, sizeof(*work));
+	if (work == NULL) {
+		nvgpu_err(g, "Unable to allocate memory for runlist job");
+		ret = -ENOMEM;
+		goto free_domain;
+	}
+
+	work->g = g;
+	work->rl = rl;
+	work->rl_domain = rl_domain;
+	nvgpu_cond_init(&work->cond);
+	nvgpu_init_list_node(&work->list);
+	work->swap_buffer = swap_buffer;
+	work->wait_for_finish = wait_for_finish;
+	nvgpu_atomic_set(&work->state, 0);
+
+	nvs_dbg(g, " enqueueing runlist submit");
+
+	ret = nvgpu_worker_enqueue(&worker->worker, &work->list);
+	if (ret != 0) {
+		goto fail;
+	}
+
+	nvs_dbg(g, " ");
+
+	ret = NVGPU_COND_WAIT(&work->cond, nvgpu_atomic_read(&work->state) == 1, 0U);
+	if (ret != 0) {
+		nvgpu_err(g, "Runlist submit interrupted while waiting for submit");
+		goto fail;
+	}
+
+	nvs_dbg(g, " ");
+
+	ret = work->status;
+
+fail:
+	nvgpu_cond_destroy(&work->cond);
+	nvgpu_kfree(g, work);
+
+free_domain:
+
+	return ret;
+}
+
 static void nvgpu_nvs_worker_wakeup_post_process(struct nvgpu_worker *worker)
 {
 	struct gk20a *g = worker->g;
@@ -146,19 +259,40 @@ static const struct nvgpu_worker_ops nvs_worker_ops = {

 static int nvgpu_nvs_worker_init(struct gk20a *g)
 {
+	int err = 0;
 	struct nvgpu_worker *worker = &g->scheduler->worker.worker;
+	struct nvgpu_nvs_worker *nvs_worker = &g->scheduler->worker;
+
+	nvgpu_cond_init(&nvs_worker->wq_init);
+	nvgpu_atomic_set(&nvs_worker->nvs_sched_init, 0);

 	nvgpu_worker_init_name(worker, "nvgpu_nvs", g->name);

-	return nvgpu_worker_init(g, worker, &nvs_worker_ops);
+	err = nvgpu_worker_init(g, worker, &nvs_worker_ops);
+	if (err != 0) {
+		/* Ensure that scheduler thread is started as soon as possible to handle
+		 * minimal uptime for applications.
+		 */
+		err = NVGPU_COND_WAIT(&nvs_worker->worker.wq,
+				nvgpu_atomic_read(&nvs_worker->nvs_sched_init) == 1, 0);
+		if (err != 0) {
+			nvgpu_err(g, "Interrupted while waiting for scheduler thread");
+		}
+	}
+
+	return err;
 }

 static void nvgpu_nvs_worker_deinit(struct gk20a *g)
 {
 	struct nvgpu_worker *worker = &g->scheduler->worker.worker;
+	struct nvgpu_nvs_worker *nvs_worker = &g->scheduler->worker;

 	nvgpu_worker_deinit(worker);

+	nvgpu_atomic_set(&nvs_worker->nvs_sched_init, 0);
+	nvgpu_cond_destroy(&nvs_worker->wq_init);
+
 	nvs_dbg(g, "NVS worker suspended");
 }

@@ -166,8 +300,10 @@ static struct nvgpu_nvs_domain *
 	nvgpu_nvs_gen_domain(struct gk20a *g, const char *name, u64 id,
 		u64 timeslice, u64 preempt_grace)
 {
+	struct nvgpu_fifo *f = &g->fifo;
 	struct nvs_domain *nvs_dom = NULL;
 	struct nvgpu_nvs_domain *nvgpu_dom = NULL;
+	u32 num_runlists = f->num_runlists;

 	nvs_dbg(g, "Adding new domain: %s", name);

@@ -177,6 +313,14 @@ static struct nvgpu_nvs_domain *
 		return nvgpu_dom;
 	}

+	nvgpu_dom->rl_domains = nvgpu_kzalloc(g, sizeof(*nvgpu_dom->rl_domains) * num_runlists);
+	if (nvgpu_dom->rl_domains == NULL) {
+		nvs_dbg(g, "failed to allocate memory for domain->rl_domains");
+		nvgpu_kfree(g, nvgpu_dom);
+		nvgpu_dom = NULL;
+		return nvgpu_dom;
+	}
+
 	nvgpu_dom->id = id;
 	nvgpu_dom->ref = 1U;

@@ -185,6 +329,7 @@ static struct nvgpu_nvs_domain *

 	if (nvs_dom == NULL) {
 		nvs_dbg(g, "failed to create nvs domain for %s", name);
+		nvgpu_kfree(g, nvgpu_dom->rl_domains);
 		nvgpu_kfree(g, nvgpu_dom);
 		nvgpu_dom = NULL;
 		return nvgpu_dom;
@@ -195,6 +340,19 @@ static struct nvgpu_nvs_domain *
 	return nvgpu_dom;
 }

+static void nvgpu_nvs_link_shadow_rl_domains(struct gk20a *g,
+		struct nvgpu_nvs_domain *nvgpu_dom)
+{
+	struct nvgpu_fifo *f = &g->fifo;
+	u32 num_runlists = f->num_runlists;
+	u32 i;
+
+	for (i = 0U; i < num_runlists; i++) {
+		struct nvgpu_runlist *runlist = &f->active_runlists[i];
+		nvgpu_dom->rl_domains[i] = runlist->shadow_rl_domain;
+	}
+}
+
 static int nvgpu_nvs_gen_shadow_domain(struct gk20a *g)
 {
 	int err = 0;
@@ -211,6 +369,8 @@ static int nvgpu_nvs_gen_shadow_domain(struct gk20a *g)
 		goto error;
 	}

+	nvgpu_nvs_link_shadow_rl_domains(g, nvgpu_dom);
+
 	g->scheduler->shadow_domain = nvgpu_dom;

 	/* Set active_domain to shadow_domain during Init */
@@ -243,6 +403,8 @@ static void nvgpu_nvs_remove_shadow_domain(struct gk20a *g)
 	nvs_dom = sched->shadow_domain->parent;
 	nvs_domain_destroy(sched->sched, nvs_dom);

+	nvgpu_kfree(g, sched->shadow_domain->rl_domains);
+	sched->shadow_domain->rl_domains = NULL;
 	nvgpu_kfree(g, sched->shadow_domain);
 	sched->shadow_domain = NULL;
 }
@@ -339,6 +501,7 @@ int nvgpu_nvs_open(struct gk20a *g)
 		goto unlock;
 	}

+	g->nvs_worker_submit = nvgpu_nvs_worker_submit;
 unlock:
 	if (err) {
 		nvs_dbg(g, "  Failed! Error code: %d", err);
@@ -362,12 +525,52 @@ static u64 nvgpu_nvs_new_id(struct gk20a *g)
 	return nvgpu_atomic64_inc_return(&g->scheduler->id_counter);
 }

+static int nvgpu_nvs_create_rl_domain_mem(struct gk20a *g,
+		struct nvgpu_nvs_domain *domain, const char *name)
+{
+	struct nvgpu_fifo *f = &g->fifo;
+	u32 i, j;
+	int err = 0;
+
+	for (i = 0U; i < f->num_runlists; i++) {
+		domain->rl_domains[i] = nvgpu_runlist_domain_alloc(g, name);
+		if (domain->rl_domains[i] == NULL) {
+			err = -ENOMEM;
+			break;
+		}
+	}
+
+	if (err != 0) {
+		for (j = 0; j != i; j++) {
+			nvgpu_runlist_domain_free(g, domain->rl_domains[j]);
+			domain->rl_domains[j] = NULL;
+		}
+	}
+
+	return err;
+}
+
+static void nvgpu_nvs_link_rl_domains(struct gk20a *g,
+		struct nvgpu_nvs_domain *domain)
+{
+	struct nvgpu_fifo *f = &g->fifo;
+	u32 i;
+
+	for (i = 0U; i < f->num_runlists; i++) {
+		struct nvgpu_runlist *runlist;
+
+		runlist = &f->active_runlists[i];
+		nvgpu_runlist_link_domain(runlist, domain->rl_domains[i]);
+	}
+}
+
 int nvgpu_nvs_add_domain(struct gk20a *g, const char *name, u64 timeslice,
 			 u64 preempt_grace, struct nvgpu_nvs_domain **pdomain)
 {
 	int err = 0;
 	struct nvs_domain *nvs_dom;
 	struct nvgpu_nvs_domain *nvgpu_dom;
+	struct nvgpu_nvs_scheduler *sched = g->scheduler;

 	nvgpu_mutex_acquire(&g->sched_mutex);

@@ -383,28 +586,26 @@ int nvgpu_nvs_add_domain(struct gk20a *g, const char *name, u64 timeslice,
 		goto unlock;
 	}

-	nvs_dom = nvgpu_dom->parent;
-
-	nvs_domain_scheduler_attach(g->scheduler->sched, nvs_dom);
-
-	err = nvgpu_rl_domain_alloc(g, name);
+	err = nvgpu_nvs_create_rl_domain_mem(g, nvgpu_dom, name);
 	if (err != 0) {
-		nvs_dbg(g, "failed to alloc rl domain for %s", name);
-		nvs_domain_unlink_and_destroy(g->scheduler->sched, nvs_dom);
+		nvs_domain_destroy(sched->sched, nvgpu_dom->parent);
+		nvgpu_kfree(g, nvgpu_dom->rl_domains);
 		nvgpu_kfree(g, nvgpu_dom);
 		goto unlock;
 	}

-	nvgpu_dom->parent = nvs_dom;
+	nvgpu_nvs_link_rl_domains(g, nvgpu_dom);

-	/* Set the first user created domain as active domain */
-	if (g->scheduler->active_domain == g->scheduler->shadow_domain) {
-		g->scheduler->active_domain = nvgpu_dom;
-	}
+	nvs_dom = nvgpu_dom->parent;
+
+	nvs_domain_scheduler_attach(g->scheduler->sched, nvs_dom);
+
+	nvgpu_dom->parent = nvs_dom;

 	*pdomain = nvgpu_dom;
 unlock:
 	nvgpu_mutex_release(&g->sched_mutex);
+
 	return err;
 }

@@ -492,6 +693,30 @@ void nvgpu_nvs_domain_put(struct gk20a *g, struct nvgpu_nvs_domain *dom)
 	nvgpu_mutex_release(&g->sched_mutex);
 }

+static void nvgpu_nvs_delete_rl_domain_mem(struct gk20a *g, struct nvgpu_nvs_domain *dom)
+{
+	struct nvgpu_fifo *f = &g->fifo;
+	u32 i;
+
+	for (i = 0U; i < f->num_runlists; i++) {
+		nvgpu_runlist_domain_free(g, dom->rl_domains[i]);
+		dom->rl_domains[i] = NULL;
+	}
+}
+
+static void nvgpu_nvs_unlink_rl_domains(struct gk20a *g, struct nvgpu_nvs_domain *domain)
+{
+	struct nvgpu_fifo *f = &g->fifo;
+	u32 i;
+
+	for (i = 0; i < f->num_runlists; i++) {
+		struct nvgpu_runlist *runlist;
+		runlist = &f->active_runlists[i];
+
+		nvgpu_runlist_unlink_domain(runlist, domain->rl_domains[i]);
+	}
+}
+
 int nvgpu_nvs_del_domain(struct gk20a *g, u64 dom_id)
 {
 	struct nvgpu_nvs_scheduler *s = g->scheduler;
@@ -519,23 +744,13 @@ int nvgpu_nvs_del_domain(struct gk20a *g, u64 dom_id)

 	nvs_dom = nvgpu_dom->parent;

-	err = nvgpu_rl_domain_delete(g, nvs_dom->name);
-	if (err != 0) {
-		nvs_dbg(g, "failed to delete RL domains on %llu!", dom_id);
-		/*
-		 * The RL domains require the existence of at least one domain;
-		 * this path inherits that logic until it's been made more
-		 * flexible.
-		 */
-		goto unlock;
-	}
-
+	nvgpu_nvs_unlink_rl_domains(g, nvgpu_dom);
+	nvgpu_nvs_delete_rl_domain_mem(g, nvgpu_dom);
 	nvgpu_dom->ref = 0U;

-	/* note: same wraparound logic as in RL domains to keep in sync */
 	if (s->active_domain == nvgpu_dom) {
 		nvs_next = nvs_domain_get_next_domain(s->sched, nvs_dom);
-		/* Its the only entry in the list. Set the default domain as the active domain */
+		/* Its the only entry in the list. Set the shadow domain as the active domain */
 		if (nvs_next == nvs_dom) {
 			nvs_next = s->shadow_domain->parent;
 		}
@@ -543,6 +758,8 @@ int nvgpu_nvs_del_domain(struct gk20a *g, u64 dom_id)
 	}

 	nvs_domain_unlink_and_destroy(s->sched, nvs_dom);
+
+	nvgpu_kfree(g, nvgpu_dom->rl_domains);
 	nvgpu_kfree(g, nvgpu_dom);

 unlock: