diff --git a/drivers/gpu/nvgpu/common/fifo/tsg.c b/drivers/gpu/nvgpu/common/fifo/tsg.c index 777adb663..8e22be557 100644 --- a/drivers/gpu/nvgpu/common/fifo/tsg.c +++ b/drivers/gpu/nvgpu/common/fifo/tsg.c @@ -89,15 +89,6 @@ int nvgpu_tsg_bind_channel(struct nvgpu_tsg *tsg, struct nvgpu_channel *ch) return -EINVAL; } - /* - * This runlist domain is set either by default or in an explicit - * bind. If the default domain has been deleted, explicit bind is - * mandatory. - */ - if (tsg->rl_domain == NULL) { - return -EINVAL; - } - /* cannot bind more channels than MAX channels supported per TSG */ nvgpu_rwsem_down_read(&tsg->ch_list_lock); max_ch_per_tsg = g->ops.runlist.get_max_channels_per_tsg(); @@ -119,14 +110,17 @@ int nvgpu_tsg_bind_channel(struct nvgpu_tsg *tsg, struct nvgpu_channel *ch) */ if (tsg->runlist == NULL) { tsg->runlist = ch->runlist; - /* - * The rl domain identifier is stashed in tsg->rl_domain->name - * when the tsg is bound to a domain, but at that point there - * are no channels yet to describe which runlist id should be - * used. Now we know. - */ - tsg->rl_domain = nvgpu_rl_domain_get(g, tsg->runlist->id, tsg->rl_domain->name); - WARN_ON(tsg->rl_domain == NULL); + if (tsg->rl_domain != NULL) { + /* + * The rl domain identifier is stashed in tsg->rl_domain->name + * when the tsg is bound to a domain, but at that point there + * are no channels yet to describe which runlist id should be + * used. Now we know. + */ + tsg->rl_domain = nvgpu_rl_domain_get(g, tsg->runlist->id, + tsg->rl_domain->name); + WARN_ON(tsg->rl_domain == NULL); + } } else { if (tsg->runlist != ch->runlist) { nvgpu_err(tsg->g, @@ -862,12 +856,8 @@ int nvgpu_tsg_open_common(struct gk20a *g, struct nvgpu_tsg *tsg, pid_t pid) tsg->interleave_level = NVGPU_FIFO_RUNLIST_INTERLEAVE_LEVEL_LOW; tsg->timeslice_us = g->ops.tsg.default_timeslice_us(g); tsg->runlist = NULL; - /* - * The domain ptr will get updated with the right id once the runlist - * gets specified based on the first channel. - */ - tsg->rl_domain = nvgpu_rl_domain_get(g, 0, "(default)"); - tsg->nvs_domain = nvgpu_nvs_domain_by_name(g, "(default)"); + tsg->rl_domain = NULL; + tsg->nvs_domain = NULL; #ifdef CONFIG_NVGPU_DEBUGGER tsg->sm_exception_mask_type = NVGPU_SM_EXCEPTION_TYPE_MASK_NONE; #endif diff --git a/drivers/gpu/nvgpu/common/nvs/nvs_sched.c b/drivers/gpu/nvgpu/common/nvs/nvs_sched.c index 5066b9e80..e11ad8668 100644 --- a/drivers/gpu/nvgpu/common/nvs/nvs_sched.c +++ b/drivers/gpu/nvgpu/common/nvs/nvs_sched.c @@ -22,6 +22,7 @@ #include #include +#include #include #include @@ -92,7 +93,7 @@ static u64 nvgpu_nvs_tick(struct gk20a *g) { struct nvgpu_nvs_scheduler *sched = g->scheduler; struct nvgpu_nvs_domain *domain; - struct nvs_domain *nvs_domain; + struct nvs_domain *nvs_next; u64 timeslice; nvs_dbg(g, "nvs tick"); @@ -101,20 +102,16 @@ static u64 nvgpu_nvs_tick(struct gk20a *g) domain = sched->active_domain; - if (domain == NULL) { - /* nothing to schedule, TODO wait for an event instead */ - nvgpu_mutex_release(&g->sched_mutex); - return 100 * NSEC_PER_MSEC; + /* If active_domain == shadow_domain, then nvs_next is NULL */ + nvs_next = nvs_domain_get_next_domain(sched->sched, domain->parent); + if (nvs_next == NULL) { + nvs_next = sched->shadow_domain->parent; } - nvs_domain = domain->parent->next; - if (nvs_domain == NULL) { - nvs_domain = g->scheduler->sched->domain_list->domains; - } - timeslice = nvs_domain->timeslice_ns; + timeslice = nvs_next->timeslice_ns; nvgpu_runlist_tick(g); - sched->active_domain = nvs_domain->priv; + sched->active_domain = nvs_next->priv; nvgpu_mutex_release(&g->sched_mutex); @@ -165,9 +162,93 @@ static void nvgpu_nvs_worker_deinit(struct gk20a *g) nvs_dbg(g, "NVS worker suspended"); } +static struct nvgpu_nvs_domain * + nvgpu_nvs_gen_domain(struct gk20a *g, const char *name, u64 id, + u64 timeslice, u64 preempt_grace) +{ + struct nvs_domain *nvs_dom = NULL; + struct nvgpu_nvs_domain *nvgpu_dom = NULL; + + nvs_dbg(g, "Adding new domain: %s", name); + + nvgpu_dom = nvgpu_kzalloc(g, sizeof(*nvgpu_dom)); + if (nvgpu_dom == NULL) { + nvs_dbg(g, "failed to allocate memory for domain %s", name); + return nvgpu_dom; + } + + nvgpu_dom->id = id; + nvgpu_dom->ref = 1U; + + nvs_dom = nvs_domain_create(g->scheduler->sched, name, + timeslice, preempt_grace, nvgpu_dom); + + if (nvs_dom == NULL) { + nvs_dbg(g, "failed to create nvs domain for %s", name); + nvgpu_kfree(g, nvgpu_dom); + nvgpu_dom = NULL; + return nvgpu_dom; + } + + nvgpu_dom->parent = nvs_dom; + + return nvgpu_dom; +} + +static int nvgpu_nvs_gen_shadow_domain(struct gk20a *g) +{ + int err = 0; + struct nvgpu_nvs_domain *nvgpu_dom; + + if (g->scheduler->shadow_domain != NULL) { + goto error; + } + + nvgpu_dom = nvgpu_nvs_gen_domain(g, SHADOW_DOMAIN_NAME, U64_MAX, + 100U * NSEC_PER_MSEC, 0U); + if (nvgpu_dom == NULL) { + err = -ENOMEM; + goto error; + } + + g->scheduler->shadow_domain = nvgpu_dom; + + /* Set active_domain to shadow_domain during Init */ + g->scheduler->active_domain = g->scheduler->shadow_domain; + +error: + return err; +} + +static void nvgpu_nvs_remove_shadow_domain(struct gk20a *g) +{ + struct nvgpu_nvs_scheduler *sched = g->scheduler; + struct nvs_domain *nvs_dom; + + if (sched == NULL) { + /* never powered on to init anything */ + return; + } + + if (sched->shadow_domain == NULL) { + return; + } + + if (sched->shadow_domain->ref != 1U) { + nvgpu_warn(g, + "domain %llu is still in use during shutdown! refs: %u", + sched->shadow_domain->id, sched->shadow_domain->ref); + } + + nvs_dom = sched->shadow_domain->parent; + nvs_domain_destroy(sched->sched, nvs_dom); + + nvgpu_kfree(g, sched->shadow_domain); + sched->shadow_domain = NULL; +} + int nvgpu_nvs_init(struct gk20a *g) { - struct nvgpu_nvs_domain *domain; int err; nvgpu_mutex_init(&g->sched_mutex); @@ -177,16 +258,6 @@ int nvgpu_nvs_init(struct gk20a *g) return err; } - if (nvgpu_rl_domain_get(g, 0, "(default)") == NULL) { - int err = nvgpu_nvs_add_domain(g, "(default)", - 100U * NSEC_PER_MSEC, - 0U, - &domain); - if (err != 0) { - return err; - } - } - return 0; } @@ -214,6 +285,8 @@ void nvgpu_nvs_remove_support(struct gk20a *g) nvgpu_kfree(g, nvgpu_dom); } + nvgpu_nvs_remove_shadow_domain(g); + nvs_sched_close(sched->sched); nvgpu_kfree(g, sched->sched); nvgpu_kfree(g, sched); @@ -247,15 +320,22 @@ int nvgpu_nvs_open(struct gk20a *g) goto unlock; } - err = nvgpu_nvs_worker_init(g); + nvs_dbg(g, " Creating NVS scheduler."); + err = nvs_sched_create(g->scheduler->sched, &nvgpu_nvs_ops, g); if (err != 0) { goto unlock; } - nvs_dbg(g, " Creating NVS scheduler."); - err = nvs_sched_create(g->scheduler->sched, &nvgpu_nvs_ops, g); + if (nvgpu_rl_domain_get(g, 0, SHADOW_DOMAIN_NAME) == NULL) { + err = nvgpu_nvs_gen_shadow_domain(g); + if (err != 0) { + goto unlock; + } + } + + err = nvgpu_nvs_worker_init(g); if (err != 0) { - nvgpu_nvs_worker_deinit(g); + nvgpu_nvs_remove_shadow_domain(g); goto unlock; } @@ -289,8 +369,6 @@ int nvgpu_nvs_add_domain(struct gk20a *g, const char *name, u64 timeslice, struct nvs_domain *nvs_dom; struct nvgpu_nvs_domain *nvgpu_dom; - nvs_dbg(g, "Adding new domain: %s", name); - nvgpu_mutex_acquire(&g->sched_mutex); if (nvs_domain_by_name(g->scheduler->sched, name) != NULL) { @@ -298,37 +376,29 @@ int nvgpu_nvs_add_domain(struct gk20a *g, const char *name, u64 timeslice, goto unlock; } - nvgpu_dom = nvgpu_kzalloc(g, sizeof(*nvgpu_dom)); + nvgpu_dom = nvgpu_nvs_gen_domain(g, name, nvgpu_nvs_new_id(g), + timeslice, preempt_grace); if (nvgpu_dom == NULL) { err = -ENOMEM; goto unlock; } - nvgpu_dom->id = nvgpu_nvs_new_id(g); - nvgpu_dom->ref = 1U; + nvs_dom = nvgpu_dom->parent; - nvs_dom = nvs_domain_create(g->scheduler->sched, name, - timeslice, preempt_grace, nvgpu_dom); - - if (nvs_dom == NULL) { - nvs_dbg(g, "failed to create nvs domain for %s", name); - nvgpu_kfree(g, nvgpu_dom); - err = -ENOMEM; - goto unlock; - } + nvs_domain_scheduler_attach(g->scheduler->sched, nvs_dom); err = nvgpu_rl_domain_alloc(g, name); if (err != 0) { nvs_dbg(g, "failed to alloc rl domain for %s", name); - nvs_domain_destroy(g->scheduler->sched, nvs_dom); + nvs_domain_unlink_and_destroy(g->scheduler->sched, nvs_dom); nvgpu_kfree(g, nvgpu_dom); goto unlock; } - nvgpu_dom->parent = nvs_dom; - if (g->scheduler->active_domain == NULL) { + /* Set the first user created domain as active domain */ + if (g->scheduler->active_domain == g->scheduler->shadow_domain) { g->scheduler->active_domain = nvgpu_dom; } @@ -464,14 +534,15 @@ int nvgpu_nvs_del_domain(struct gk20a *g, u64 dom_id) /* note: same wraparound logic as in RL domains to keep in sync */ if (s->active_domain == nvgpu_dom) { - nvs_next = nvs_dom->next; - if (nvs_next == NULL) { - nvs_next = s->sched->domain_list->domains; + nvs_next = nvs_domain_get_next_domain(s->sched, nvs_dom); + /* Its the only entry in the list. Set the default domain as the active domain */ + if (nvs_next == nvs_dom) { + nvs_next = s->shadow_domain->parent; } s->active_domain = nvs_next->priv; } - nvs_domain_destroy(s->sched, nvs_dom); + nvs_domain_unlink_and_destroy(s->sched, nvs_dom); nvgpu_kfree(g, nvgpu_dom); unlock: diff --git a/drivers/gpu/nvgpu/hal/fifo/runlist_fifo_ga10b_fusa.c b/drivers/gpu/nvgpu/hal/fifo/runlist_fifo_ga10b_fusa.c index b535e5d50..852a4e2f7 100644 --- a/drivers/gpu/nvgpu/hal/fifo/runlist_fifo_ga10b_fusa.c +++ b/drivers/gpu/nvgpu/hal/fifo/runlist_fifo_ga10b_fusa.c @@ -72,6 +72,9 @@ void ga10b_runlist_hw_submit(struct gk20a *g, struct nvgpu_runlist *runlist) runlist_submit_base_hi_ptr_hi_f(runlist_iova_hi)); } + rl_dbg(g, "Submitting domain[%s], mem=0x%16llx", runlist->domain->name, + (u64)nvgpu_mem_get_addr(g, &runlist->domain->mem_hw->mem)); + /* TODO offset in runlist support */ nvgpu_runlist_writel(g, runlist, runlist_submit_r(), runlist_submit_offset_f(0U) | diff --git a/drivers/gpu/nvgpu/include/nvgpu/nvs.h b/drivers/gpu/nvgpu/include/nvgpu/nvs.h index 7210bbc18..1ba508d6e 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/nvs.h +++ b/drivers/gpu/nvgpu/include/nvgpu/nvs.h @@ -84,6 +84,7 @@ struct nvgpu_nvs_scheduler { nvgpu_atomic64_t id_counter; struct nvgpu_nvs_worker worker; struct nvgpu_nvs_domain *active_domain; + struct nvgpu_nvs_domain *shadow_domain; }; #ifdef CONFIG_NVS_PRESENT diff --git a/drivers/gpu/nvgpu/include/nvgpu/runlist.h b/drivers/gpu/nvgpu/include/nvgpu/runlist.h index efef0cd5b..6a2da5d2d 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/runlist.h +++ b/drivers/gpu/nvgpu/include/nvgpu/runlist.h @@ -156,7 +156,8 @@ struct nvgpu_runlist { */ struct nvgpu_runlist_domain *shadow_rl_domain; /* - * All scheduling domains of this RL, see nvgpu_runlist_domain::domain_node. + * All scheduling domains(created by users) of this RL, + * see nvgpu_runlist_domain::domain_node. * * Design note: the runlist hardware unit should not own the actual * domain memory; this arrangement is temporary to aid in transition diff --git a/drivers/gpu/nvgpu/include/nvgpu/tsg.h b/drivers/gpu/nvgpu/include/nvgpu/tsg.h index e31172c28..345b64f3e 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/tsg.h +++ b/drivers/gpu/nvgpu/include/nvgpu/tsg.h @@ -179,7 +179,7 @@ struct nvgpu_tsg { struct nvgpu_runlist *runlist; /** - * Runlist domain this TSG is bound to. Bound with an ioctl, initially the default domain. + * Runlist domain this TSG is bound to. Bound with an ioctl, initially empty */ struct nvgpu_runlist_domain *rl_domain; diff --git a/nvsched/include/nvs/domain.h b/nvsched/include/nvs/domain.h index 637caa573..36383eb4c 100644 --- a/nvsched/include/nvs/domain.h +++ b/nvsched/include/nvs/domain.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 NVIDIA Corporation. All rights reserved. + * Copyright (c) 2021-2022 NVIDIA Corporation. All rights reserved. * * NVIDIA Corporation and its licensors retain all intellectual property * and proprietary rights in and to this software, related documentation @@ -68,7 +68,11 @@ struct nvs_domain { struct nvs_domain *nvs_domain_create(struct nvs_sched *sched, const char *name, u64 timeslice, u64 preempt_grace, void *priv); +struct nvs_domain *nvs_domain_get_next_domain(struct nvs_sched *sched, struct nvs_domain *dom); +void nvs_domain_scheduler_attach(struct nvs_sched *sched, struct nvs_domain *dom); void nvs_domain_destroy(struct nvs_sched *sched, struct nvs_domain *dom); +void nvs_domain_unlink_and_destroy(struct nvs_sched *sched, + struct nvs_domain *dom); void nvs_domain_clear_all(struct nvs_sched *sched); u32 nvs_domain_count(struct nvs_sched *sched); struct nvs_domain *nvs_domain_by_name(struct nvs_sched *sched, const char *name); diff --git a/nvsched/src/domain.c b/nvsched/src/domain.c index 92e98cee8..bd1958435 100644 --- a/nvsched/src/domain.c +++ b/nvsched/src/domain.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 NVIDIA Corporation. All rights reserved. + * Copyright (c) 2021-2022 NVIDIA Corporation. All rights reserved. * * NVIDIA Corporation and its licensors retain all intellectual property * and proprietary rights in and to this software, related documentation @@ -13,13 +13,12 @@ #include /* - * Create and add a new domain to the end of the domain list. + * Create a new domain and populate it with default values */ struct nvs_domain *nvs_domain_create(struct nvs_sched *sched, const char *name, u64 timeslice, u64 preempt_grace, void *priv) { - struct nvs_domain_list *dlist = sched->domain_list; struct nvs_domain *dom = nvs_malloc(sched, sizeof(*dom)); nvs_log(sched, "Creating domain - %s", name); @@ -34,8 +33,28 @@ struct nvs_domain *nvs_domain_create(struct nvs_sched *sched, dom->timeslice_ns = timeslice; dom->preempt_grace_ns = preempt_grace; dom->priv = priv; + dom->next = NULL; - nvs_log_event(sched, NVS_EV_CREATE_DOMAIN, 0U); + return dom; +} + +struct nvs_domain *nvs_domain_get_next_domain(struct nvs_sched *sched, struct nvs_domain *dom) +{ + struct nvs_domain *nvs_next = dom->next; + + if (nvs_next == NULL) { + nvs_next = sched->domain_list->domains; + } + + return nvs_next; +} + +/* + * Attach a new domain to the end of the domain list. + */ +void nvs_domain_scheduler_attach(struct nvs_sched *sched, struct nvs_domain *dom) +{ + struct nvs_domain_list *dlist = sched->domain_list; /* * Now add the domain to the list of domains. If this is the first @@ -46,14 +65,13 @@ struct nvs_domain *nvs_domain_create(struct nvs_sched *sched, if (dlist->domains == NULL) { dlist->domains = dom; dlist->last = dom; - return dom; + return; } dlist->last->next = dom; dlist->last = dom; - nvs_log(sched, "%s: Domain added", name); - return dom; + nvs_log(sched, "%s: Domain added", dom->name); } /* @@ -66,7 +84,6 @@ static void nvs_domain_unlink(struct nvs_sched *sched, struct nvs_domain_list *dlist = sched->domain_list; struct nvs_domain *tmp; - if (dlist->domains == dom) { dlist->domains = dom->next; @@ -76,6 +93,7 @@ static void nvs_domain_unlink(struct nvs_sched *sched, * pointer as well. */ if (dom == dlist->last) { + dlist->domains = NULL; dlist->last = NULL; } return; @@ -99,15 +117,20 @@ static void nvs_domain_unlink(struct nvs_sched *sched, void nvs_domain_destroy(struct nvs_sched *sched, struct nvs_domain *dom) +{ + nvs_memset(dom, 0, sizeof(*dom)); + nvs_free(sched, dom); +} + +void nvs_domain_unlink_and_destroy(struct nvs_sched *sched, + struct nvs_domain *dom) { nvs_log_event(sched, NVS_EV_REMOVE_DOMAIN, 0); nvs_domain_unlink(sched, dom); - - nvs_memset(dom, 0, sizeof(*dom)); - nvs_free(sched, dom); - sched->domain_list->nr--; + + nvs_domain_destroy(sched, dom); } void nvs_domain_clear_all(struct nvs_sched *sched) @@ -115,7 +138,7 @@ void nvs_domain_clear_all(struct nvs_sched *sched) struct nvs_domain_list *dlist = sched->domain_list; while (dlist->domains != NULL) { - nvs_domain_destroy(sched, dlist->domains); + nvs_domain_unlink_and_destroy(sched, dlist->domains); } }