diff --git a/drivers/gpu/nvgpu/common/fifo/runlist.c b/drivers/gpu/nvgpu/common/fifo/runlist.c index 36522fb8b..2e9a02d01 100644 --- a/drivers/gpu/nvgpu/common/fifo/runlist.c +++ b/drivers/gpu/nvgpu/common/fifo/runlist.c @@ -663,6 +663,14 @@ int nvgpu_runlist_update(struct gk20a *g, struct nvgpu_runlist *rl, return -EINVAL; } + if (tsg->rl_domain == NULL) { + /* + * "Success" case because the TSG is not participating in + * scheduling at the moment, so there is nothing to be done. + */ + return 0; + } + return nvgpu_runlist_do_update(g, rl, tsg->rl_domain, ch, add, wait_for_finish); } @@ -1146,10 +1154,33 @@ static void nvgpu_init_active_runlist_mapping(struct gk20a *g) } } +static int nvgpu_runlist_alloc_default_domain(struct gk20a *g) +{ +#ifndef CONFIG_NVS_PRESENT + struct nvgpu_fifo *f = &g->fifo; + u32 i; + + for (i = 0; i < g->fifo.num_runlists; i++) { + struct nvgpu_runlist *runlist = &f->active_runlists[i]; + + runlist->domain = nvgpu_runlist_domain_alloc(g, runlist, "(default)"); + if (runlist->domain == NULL) { + nvgpu_err(g, "memory allocation failed"); + /* + * deletion of prior domains happens in + * nvgpu_runlist_cleanup_sw() via the caller. + */ + return -ENOMEM; + } + } +#endif + return 0; +} + int nvgpu_runlist_setup_sw(struct gk20a *g) { struct nvgpu_fifo *f = &g->fifo; - u32 num_runlists = 0U, i; + u32 num_runlists = 0U; unsigned int runlist_id; int err = 0; @@ -1190,18 +1221,11 @@ int nvgpu_runlist_setup_sw(struct gk20a *g) nvgpu_init_active_runlist_mapping(g); - for (i = 0; i < g->fifo.num_runlists; i++) { - struct nvgpu_runlist *runlist = &f->active_runlists[i]; - - runlist->domain = nvgpu_runlist_domain_alloc(g, runlist, "(default)"); - if (runlist->domain == NULL) { - nvgpu_err(g, "memory allocation failed"); - err = -ENOMEM; - goto clean_up_runlist; - } + err = nvgpu_runlist_alloc_default_domain(g); + if (err != 0) { + goto clean_up_runlist; } - g->ops.runlist.init_enginfo(g, f); return 0; diff --git a/drivers/gpu/nvgpu/common/fifo/tsg.c b/drivers/gpu/nvgpu/common/fifo/tsg.c index f923d4125..aeea43749 100644 --- a/drivers/gpu/nvgpu/common/fifo/tsg.c +++ b/drivers/gpu/nvgpu/common/fifo/tsg.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #ifdef CONFIG_NVGPU_PROFILER @@ -156,7 +157,8 @@ int nvgpu_tsg_bind_channel(struct nvgpu_tsg *tsg, struct nvgpu_channel *ch) #ifdef CONFIG_NVS_PRESENT int nvgpu_tsg_bind_domain(struct nvgpu_tsg *tsg, const char *domain_name) { - struct nvgpu_runlist_domain *domain; + struct nvgpu_runlist_domain *rl_domain; + struct nvgpu_nvs_domain *nvs_domain; struct gk20a *g = tsg->g; /* Hopping channels from one domain to another is not allowed */ @@ -164,16 +166,28 @@ int nvgpu_tsg_bind_domain(struct nvgpu_tsg *tsg, const char *domain_name) return -EINVAL; } + nvs_domain = nvgpu_nvs_domain_get(g, domain_name); + if (nvs_domain == NULL) { + return -ENOENT; + } + /* * The domain ptr will get updated with the right id once the runlist * gets specified based on the first channel. */ - domain = nvgpu_rl_domain_get(g, 0, domain_name); - if (domain == NULL) { + rl_domain = nvgpu_rl_domain_get(g, 0, domain_name); + if (rl_domain == NULL) { + /* + * This shouldn't happen because the nvs domain guarantees RL domains. + * + * TODO: query this via the nvs domain. + */ + nvgpu_nvs_domain_put(g, nvs_domain); return -ENOENT; } - tsg->rl_domain = domain; + tsg->rl_domain = rl_domain; + tsg->nvs_domain = nvs_domain; return 0; } @@ -844,6 +858,7 @@ int nvgpu_tsg_open_common(struct gk20a *g, struct nvgpu_tsg *tsg, pid_t pid) * gets specified based on the first channel. */ tsg->rl_domain = nvgpu_rl_domain_get(g, 0, "(default)"); + tsg->nvs_domain = nvgpu_nvs_domain_get(g, "(default)"); #ifdef CONFIG_NVGPU_DEBUGGER tsg->sm_exception_mask_type = NVGPU_SM_EXCEPTION_TYPE_MASK_NONE; #endif @@ -926,6 +941,12 @@ void nvgpu_tsg_release_common(struct gk20a *g, struct nvgpu_tsg *tsg) } #endif + if (tsg->nvs_domain != NULL) { + nvgpu_nvs_domain_put(g, tsg->nvs_domain); + tsg->nvs_domain = NULL; + tsg->rl_domain = NULL; + } + if (tsg->vm != NULL) { nvgpu_vm_put(tsg->vm); tsg->vm = NULL; diff --git a/drivers/gpu/nvgpu/common/init/nvgpu_init.c b/drivers/gpu/nvgpu/common/init/nvgpu_init.c index 3d9c202cd..a7d831190 100644 --- a/drivers/gpu/nvgpu/common/init/nvgpu_init.c +++ b/drivers/gpu/nvgpu/common/init/nvgpu_init.c @@ -52,9 +52,7 @@ #include #include #include -#ifdef CONFIG_NVS_PRESENT #include -#endif #ifdef CONFIG_NVGPU_LS_PMU #include @@ -858,6 +856,7 @@ int nvgpu_finalize_poweron(struct gk20a *g) NVGPU_INIT_TABLE_ENTRY(g->ops.mm.init_mm_support, NO_FLAG), NVGPU_INIT_TABLE_ENTRY(g->ops.fifo.fifo_init_support, NO_FLAG), + NVGPU_INIT_TABLE_ENTRY(nvgpu_nvs_init, NO_FLAG), NVGPU_INIT_TABLE_ENTRY(g->ops.therm.elcg_init_idle_filters, NO_FLAG), NVGPU_INIT_TABLE_ENTRY(&nvgpu_netlist_init_ctx_vars, NO_FLAG), @@ -923,9 +922,6 @@ int nvgpu_finalize_poweron(struct gk20a *g) #if defined(CONFIG_NVGPU_GSP_SCHEDULER) || defined(CONFIG_NVGPU_GSP_STRESS_TEST) /* Init gsp ops */ NVGPU_INIT_TABLE_ENTRY(&nvgpu_gsp_sw_init, NO_FLAG), -#endif -#ifdef CONFIG_NVS_PRESENT - NVGPU_INIT_TABLE_ENTRY(nvgpu_nvs_init, NO_FLAG), #endif }; size_t i; diff --git a/drivers/gpu/nvgpu/common/nvs/nvs_sched.c b/drivers/gpu/nvgpu/common/nvs/nvs_sched.c index 7989c6e38..333ca665c 100644 --- a/drivers/gpu/nvgpu/common/nvs/nvs_sched.c +++ b/drivers/gpu/nvgpu/common/nvs/nvs_sched.c @@ -26,20 +26,32 @@ #include #include #include +#include static struct nvs_sched_ops nvgpu_nvs_ops = { .preempt = NULL, .recover = NULL, }; -/** - * Init call to prepare the scheduler mutex. We won't actually allocate a - * scheduler until someone opens the scheduler node. - */ int nvgpu_nvs_init(struct gk20a *g) { + struct nvgpu_nvs_domain *domain; + int err; + nvgpu_mutex_init(&g->sched_mutex); + err = nvgpu_nvs_open(g); + if (err != 0) { + return err; + } + + if (nvgpu_rl_domain_get(g, 0, "(default)") == NULL) { + int err = nvgpu_nvs_add_domain(g, "(default)", 1000*1000, 10*1000, &domain); + if (err != 0) { + return err; + } + } + return 0; } @@ -121,6 +133,7 @@ int nvgpu_nvs_add_domain(struct gk20a *g, const char *name, u32 timeslice, } nvgpu_dom->id = nvgpu_nvs_new_id(g); + nvgpu_dom->ref = 1U; nvs_dom = nvs_domain_create(g->scheduler->sched, name, timeslice, preempt_grace, nvgpu_dom); @@ -131,6 +144,14 @@ int nvgpu_nvs_add_domain(struct gk20a *g, const char *name, u32 timeslice, goto unlock; } + err = nvgpu_rl_domain_alloc(g, name); + if (err != 0) { + nvs_domain_destroy(g->scheduler->sched, nvs_dom); + nvgpu_kfree(g, nvgpu_dom); + return err; + } + + nvgpu_dom->parent = nvs_dom; *pdomain = nvgpu_dom; @@ -155,6 +176,36 @@ nvgpu_nvs_get_dom_by_id(struct gk20a *g, struct nvs_sched *sched, u64 dom_id) return NULL; } +struct nvgpu_nvs_domain * +nvgpu_nvs_domain_get(struct gk20a *g, const char *name) +{ + struct nvs_domain *nvs_dom; + struct nvgpu_nvs_domain *dom = NULL; + struct nvgpu_nvs_scheduler *sched = g->scheduler; + + nvgpu_mutex_acquire(&g->sched_mutex); + + nvs_dom = nvs_domain_by_name(sched->sched, name); + if (nvs_dom == NULL) { + goto unlock; + } + + dom = nvs_dom->priv; + dom->ref++; + +unlock: + nvgpu_mutex_release(&g->sched_mutex); + return dom; +} + +void nvgpu_nvs_domain_put(struct gk20a *g, struct nvgpu_nvs_domain *dom) +{ + nvgpu_mutex_acquire(&g->sched_mutex); + dom->ref--; + WARN_ON(dom->ref == 0U); + nvgpu_mutex_release(&g->sched_mutex); +} + int nvgpu_nvs_del_domain(struct gk20a *g, u64 dom_id) { struct nvgpu_nvs_domain *nvgpu_dom; @@ -168,12 +219,32 @@ int nvgpu_nvs_del_domain(struct gk20a *g, u64 dom_id) nvgpu_dom = nvgpu_nvs_get_dom_by_id(g, g->scheduler->sched, dom_id); if (nvgpu_dom == NULL) { nvs_dbg(g, "domain %llu does not exist!", dom_id); - err = -EINVAL; + err = -ENOENT; + goto unlock; + } + + if (nvgpu_dom->ref != 1U) { + nvs_dbg(g, "domain %llu is still in use! refs: %u", + dom_id, nvgpu_dom->ref); + err = -EBUSY; goto unlock; } nvs_dom = nvgpu_dom->parent; + err = nvgpu_rl_domain_delete(g, nvs_dom->name); + if (err != 0) { + nvs_dbg(g, "failed to delete RL domains on %llu!", dom_id); + /* + * The RL domains require the existence of at least one domain; + * this path inherits that logic until it's been made more + * flexible. + */ + goto unlock; + } + + nvgpu_dom->ref = 0U; + nvs_domain_destroy(g->scheduler->sched, nvs_dom); nvgpu_kfree(g, nvgpu_dom); diff --git a/drivers/gpu/nvgpu/common/vgpu/init/init_vgpu.c b/drivers/gpu/nvgpu/common/vgpu/init/init_vgpu.c index 993cfdb0f..6a4b4bd78 100644 --- a/drivers/gpu/nvgpu/common/vgpu/init/init_vgpu.c +++ b/drivers/gpu/nvgpu/common/vgpu/init/init_vgpu.c @@ -38,6 +38,7 @@ #include #include #include +#include #include "init_vgpu.h" #include "hal/vgpu/init/init_hal_vgpu.h" @@ -228,6 +229,12 @@ int vgpu_finalize_poweron_common(struct gk20a *g) return err; } + err = nvgpu_nvs_init(g); + if (err != 0) { + nvgpu_err(g, "failed to init gk20a nvs"); + return err; + } + err = vgpu_fbp_init_support(g); if (err != 0) { nvgpu_err(g, "failed to init gk20a fbp"); diff --git a/drivers/gpu/nvgpu/include/nvgpu/nvs.h b/drivers/gpu/nvgpu/include/nvgpu/nvs.h index 1169d3511..3debccafc 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/nvs.h +++ b/drivers/gpu/nvgpu/include/nvgpu/nvs.h @@ -23,7 +23,9 @@ #ifndef NVGPU_NVS_H #define NVGPU_NVS_H +#ifdef CONFIG_NVS_PRESENT #include +#endif #include #include @@ -52,6 +54,15 @@ struct nvgpu_nvs_domain { * Convenience pointer for linking back to the parent object. */ struct nvs_domain *parent; + + /* + * Domains are dynamically used by their participant TSGs and the + * runlist HW. A refcount prevents them from getting prematurely freed. + * + * This is not the usual refcount. The primary owner is userspace via the + * ioctl layer and a TSG putting a ref does not result in domain deletion. + */ + u32 ref; }; struct nvgpu_nvs_scheduler { @@ -59,6 +70,7 @@ struct nvgpu_nvs_scheduler { nvgpu_atomic64_t id_counter; }; +#ifdef CONFIG_NVS_PRESENT int nvgpu_nvs_init(struct gk20a *g); int nvgpu_nvs_open(struct gk20a *g); void nvgpu_nvs_get_log(struct gk20a *g, s64 *timestamp, const char **msg); @@ -70,10 +82,28 @@ struct nvgpu_nvs_domain * nvgpu_nvs_get_dom_by_id(struct gk20a *g, struct nvs_sched *sched, u64 dom_id); void nvgpu_nvs_print_domain(struct gk20a *g, struct nvgpu_nvs_domain *domain); +struct nvgpu_nvs_domain * +nvgpu_nvs_domain_get(struct gk20a *g, const char *name); +void nvgpu_nvs_domain_put(struct gk20a *g, struct nvgpu_nvs_domain *dom); /* * Debug wrapper for NVS code. */ #define nvs_dbg(g, fmt, arg...) \ nvgpu_log(g, gpu_dbg_nvs, fmt, ##arg) +#else +static inline int nvgpu_nvs_init(struct gk20a *g) +{ + return 0; +} +static inline struct nvgpu_nvs_domain * +nvgpu_nvs_domain_get(struct gk20a *g, const char *name) +{ + return NULL; +} +static inline void nvgpu_nvs_domain_put(struct gk20a *g, struct nvgpu_nvs_domain *dom) +{ +} +#endif + #endif diff --git a/drivers/gpu/nvgpu/include/nvgpu/tsg.h b/drivers/gpu/nvgpu/include/nvgpu/tsg.h index 8a22a42c7..b17a14884 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/tsg.h +++ b/drivers/gpu/nvgpu/include/nvgpu/tsg.h @@ -55,6 +55,8 @@ struct nvgpu_gr_ctx; struct nvgpu_channel_hw_state; struct nvgpu_profiler_object; struct nvgpu_runlist; +struct nvgpu_runlist_domain; +struct nvgpu_nvs_domain; #ifdef CONFIG_NVGPU_CHANNEL_TSG_CONTROL enum nvgpu_event_id_type; @@ -177,10 +179,15 @@ struct nvgpu_tsg { struct nvgpu_runlist *runlist; /** - * Scheduling domain this TSG is bound to. Bound with an ioctl, initially the default domain. + * Runlist domain this TSG is bound to. Bound with an ioctl, initially the default domain. */ struct nvgpu_runlist_domain *rl_domain; + /* + * A TSG keeps a ref to its scheduling domain so that active domains can't be deleted. + */ + struct nvgpu_nvs_domain *nvs_domain; + /** tgid (OS specific) of the process that openend the TSG. */ /** diff --git a/libs/dgpu/libnvgpu-drv-dgpu_safe.export b/libs/dgpu/libnvgpu-drv-dgpu_safe.export index f74a9712a..8e2f4859a 100644 --- a/libs/dgpu/libnvgpu-drv-dgpu_safe.export +++ b/libs/dgpu/libnvgpu-drv-dgpu_safe.export @@ -676,6 +676,7 @@ nvgpu_runlist_unlock_active_runlists nvgpu_runlist_unlock_runlists nvgpu_runlist_update nvgpu_runlist_update_locked +nvgpu_rl_domain_alloc nvgpu_rwsem_init nvgpu_rwsem_down_read nvgpu_rwsem_down_write diff --git a/libs/igpu/libnvgpu-drv-igpu_safe.export b/libs/igpu/libnvgpu-drv-igpu_safe.export index b05f2a888..5a6118d8e 100644 --- a/libs/igpu/libnvgpu-drv-igpu_safe.export +++ b/libs/igpu/libnvgpu-drv-igpu_safe.export @@ -693,6 +693,7 @@ nvgpu_runlist_unlock_active_runlists nvgpu_runlist_unlock_runlists nvgpu_runlist_update nvgpu_runlist_update_locked +nvgpu_rl_domain_alloc nvgpu_rwsem_init nvgpu_rwsem_down_read nvgpu_rwsem_down_write diff --git a/userspace/units/fifo/channel/nvgpu-channel.c b/userspace/units/fifo/channel/nvgpu-channel.c index 34c7afb25..b11ca7e35 100644 --- a/userspace/units/fifo/channel/nvgpu-channel.c +++ b/userspace/units/fifo/channel/nvgpu-channel.c @@ -1529,6 +1529,13 @@ static int stub_fifo_preempt_channel(struct gk20a *g, struct nvgpu_channel *ch) return -1; } +static int stub_runlist_reload(struct gk20a *g, struct nvgpu_runlist *rl, + struct nvgpu_runlist_domain *domain, + bool add, bool wait_for_finish) +{ + return 0; +} + static void stub_channel_work_completion_cancel_sync(struct nvgpu_channel *ch) { @@ -1560,6 +1567,7 @@ int test_channel_suspend_resume_serviceable_chs(struct unit_module *m, g->ops.fifo.preempt_tsg = stub_fifo_preempt_tsg; g->ops.fifo.preempt_channel = stub_fifo_preempt_channel; + g->ops.runlist.reload = stub_runlist_reload; orig_ch_tsgid = ch->tsgid; for (branches = 0U; branches < F_CHANNEL_SUSPEND_RESUME_CHS_LAST; diff --git a/userspace/units/fifo/fifo/nvgpu-fifo.c b/userspace/units/fifo/fifo/nvgpu-fifo.c index 5fbe79fbe..f80d2c230 100644 --- a/userspace/units/fifo/fifo/nvgpu-fifo.c +++ b/userspace/units/fifo/fifo/nvgpu-fifo.c @@ -124,6 +124,7 @@ int test_fifo_suspend(struct unit_module *m, struct gk20a *g, void *args) gv11b_init_hal(g); gops = g->ops; + nvgpu_device_init(g); g->ops.fifo.bar1_snooping_disable = stub_fifo_bar1_snooping_disable; err = nvgpu_fifo_init_support(g); unit_assert(err == 0, goto done); @@ -175,6 +176,7 @@ int test_fifo_sw_quiesce(struct unit_module *m, struct gk20a *g, void *args) gv11b_init_hal(g); gops = g->ops; + nvgpu_device_init(g); err = nvgpu_fifo_init_support(g); unit_assert(err == 0, goto done); @@ -206,15 +208,19 @@ done: * NOTE: nvgpu_engine_setup_sw() consists of 2 memory allocations. * Selecting branch for nvgpu_runlist_setup_sw() fail case accordingly. */ +#define F_FIFO_SETUP_SW_COMMON_ENGINE_FAIL2 BIT(5) #define F_FIFO_SETUP_SW_COMMON_RUNLIST_FAIL BIT(6) -#define F_FIFO_SETUP_SW_PBDMA_NULL BIT(7) -#define F_FIFO_CLEANUP_SW_PBDMA_NULL BIT(8) -#define F_FIFO_SETUP_HW_PASS BIT(9) -#define F_FIFO_SETUP_HW_FAIL BIT(10) -#define F_FIFO_INIT_LAST BIT(11) +/* + * The fifo setup too contains another allocation. + */ +#define F_FIFO_SETUP_SW_COMMON_RUNLIST_FAIL2 BIT(7) +#define F_FIFO_SETUP_SW_PBDMA_NULL BIT(8) +#define F_FIFO_CLEANUP_SW_PBDMA_NULL BIT(9) +#define F_FIFO_SETUP_HW_PASS BIT(10) +#define F_FIFO_SETUP_HW_FAIL BIT(11) +#define F_FIFO_INIT_LAST BIT(12) static const char *f_fifo_init[] = { - "fifo init support", "fifo init sw ready", "channel setup sw fail", "tsg setup sw fail", @@ -222,6 +228,7 @@ static const char *f_fifo_init[] = { "engine setup sw fail", "", "runlist setup sw fail", + "runlist setup 2 sw fail", "pbdma setup sw NULL", "pbdma cleanup sw NULL", "fifo setup hw pass", @@ -250,8 +257,7 @@ int test_init_support(struct unit_module *m, struct gk20a *g, void *args) F_FIFO_SETUP_SW_COMMON_TSG_FAIL | F_FIFO_SETUP_SW_COMMON_PBDMA_FAIL | F_FIFO_SETUP_SW_COMMON_ENGINE_FAIL | - F_FIFO_SETUP_SW_COMMON_RUNLIST_FAIL | - F_FIFO_CLEANUP_SW_PBDMA_NULL; + F_FIFO_SETUP_SW_COMMON_RUNLIST_FAIL; u32 fail = F_FIFO_SETUP_HW_FAIL | alloc_fail; u32 prune = F_FIFO_SETUP_SW_READY | F_FIFO_SETUP_SW_PBDMA_NULL | F_FIFO_SETUP_HW_PASS | fail; diff --git a/userspace/units/fifo/runlist/gk20a/nvgpu-runlist-gk20a.c b/userspace/units/fifo/runlist/gk20a/nvgpu-runlist-gk20a.c index d015aaf27..4de5b083b 100644 --- a/userspace/units/fifo/runlist/gk20a/nvgpu-runlist-gk20a.c +++ b/userspace/units/fifo/runlist/gk20a/nvgpu-runlist-gk20a.c @@ -78,14 +78,17 @@ int test_gk20a_runlist_hw_submit(struct unit_module *m, struct nvgpu_fifo *f = &g->fifo; int ret = UNIT_FAIL; u32 runlist_id = nvgpu_engine_get_gr_runlist_id(g); + struct nvgpu_runlist *runlist = g->fifo.runlists[runlist_id]; u32 count; + nvgpu_rl_domain_alloc(g, "(default)"); + for (count = 0; count < 2; count++) { nvgpu_writel(g, fifo_runlist_r(), 0); nvgpu_writel(g, fifo_runlist_base_r(), 0); - f->runlists[runlist_id]->domain->mem_hw->count = count; + runlist->domain->mem_hw->count = count; gk20a_runlist_hw_submit(g, f->runlists[runlist_id]); if (count == 0) {