From fba96fdc094a78c573a3f1f3a61e2445debd818c Mon Sep 17 00:00:00 2001 From: Alex Waterman Date: Tue, 19 May 2020 16:41:41 -0500 Subject: [PATCH] gpu: nvgpu: Replace nvgpu_engine_info with nvgpu_device Delete the struct nvgpu_engine_info as it's essentially identical to struct nvgpu_device. Duplicating data structures is not ideal as it's terribly confusing what does what. Update all uses of nvgpu_engine_info to use struct nvgpu_device. This is often a fairly straight forward replacement. Couple of places though where things got interesting: - The enum_type that engine_info uses is defined in engines.h and has a bit of SW abstraction - in particular the GRCE type. The only place this seemed to be actually relevant (the IOCTL providing device info to userspace) the GRCE engines can be worked out by comparing runlist ID. - Addition of masks based on intr_id and reset_id; those can be computed easily enough using BIT32() but this is an area that could be improved on. This reaches into a lot of extraneous code that traverses the fifo active engines list and dramtically simplifies this. Now, instead of having to go through a table of engine IDs that point to the list of all host engines, the active engine list is just a list of pointers to valid engines. It's now trivial to do a for-all-active-engines type loop. This could even be turned into a generic macro or otherwise abstracted in the future. JIRA NVGPU-5421 Change-Id: I3a810deb55a7dd8c09836fd2dae85d3e28eb23cf Signed-off-by: Alex Waterman Reviewed-on: https://git-master.nvidia.com/r/c/linux-nvgpu/+/2319895 Tested-by: mobile promotions Reviewed-by: mobile promotions --- drivers/gpu/nvgpu/common/ce/ce.c | 8 +- drivers/gpu/nvgpu/common/device.c | 12 +- drivers/gpu/nvgpu/common/fifo/engines.c | 763 ++++++++---------- drivers/gpu/nvgpu/common/fifo/runlist.c | 12 +- drivers/gpu/nvgpu/common/gr/gr.c | 3 +- drivers/gpu/nvgpu/common/init/nvgpu_init.c | 6 + drivers/gpu/nvgpu/common/nvlink/nvlink.c | 3 +- .../gpu/nvgpu/common/power_features/cg/cg.c | 15 +- .../gpu/nvgpu/common/vgpu/fifo/fifo_vgpu.c | 13 + drivers/gpu/nvgpu/common/vgpu/top/top_vgpu.c | 24 +- .../nvgpu/hal/fifo/ctxsw_timeout_gv11b_fusa.c | 22 +- drivers/gpu/nvgpu/hal/fifo/engines_gm20b.c | 63 +- .../gpu/nvgpu/hal/fifo/engines_gp10b_fusa.c | 51 +- drivers/gpu/nvgpu/hal/fifo/mmu_fault_gm20b.c | 19 +- .../gpu/nvgpu/hal/fifo/runlist_fifo_gk20a.c | 17 +- drivers/gpu/nvgpu/hal/init/hal_gm20b.c | 1 - drivers/gpu/nvgpu/hal/init/hal_gp10b.c | 1 - drivers/gpu/nvgpu/hal/init/hal_gv11b.c | 1 - drivers/gpu/nvgpu/hal/init/hal_tu104.c | 1 - drivers/gpu/nvgpu/hal/mc/mc_gm20b.c | 24 +- drivers/gpu/nvgpu/hal/mc/mc_gm20b_fusa.c | 35 +- drivers/gpu/nvgpu/hal/mc/mc_gp10b.h | 4 +- drivers/gpu/nvgpu/hal/mc/mc_gp10b_fusa.c | 33 +- drivers/gpu/nvgpu/hal/mc/mc_tu104.c | 29 +- drivers/gpu/nvgpu/hal/rc/rc_gk20a.c | 5 +- drivers/gpu/nvgpu/hal/therm/therm_gm20b.c | 19 +- drivers/gpu/nvgpu/hal/therm/therm_gp10b.c | 13 +- .../gpu/nvgpu/hal/therm/therm_gv11b_fusa.c | 12 +- drivers/gpu/nvgpu/hal/top/top_gm20b.c | 6 - drivers/gpu/nvgpu/hal/top/top_gm20b_fusa.c | 19 +- drivers/gpu/nvgpu/include/nvgpu/engines.h | 151 +--- drivers/gpu/nvgpu/include/nvgpu/fifo.h | 51 +- drivers/gpu/nvgpu/include/nvgpu/gops_top.h | 21 - drivers/gpu/nvgpu/include/nvgpu/runlist.h | 2 +- drivers/gpu/nvgpu/os/linux/debug_fifo.c | 11 +- drivers/gpu/nvgpu/os/linux/ioctl_ctrl.c | 52 +- libs/dgpu/libnvgpu-drv-dgpu_safe.export | 2 - libs/igpu/libnvgpu-drv-igpu_safe.export | 2 - 38 files changed, 597 insertions(+), 929 deletions(-) diff --git a/drivers/gpu/nvgpu/common/ce/ce.c b/drivers/gpu/nvgpu/common/ce/ce.c index 5e97ea0d7..2ba7c169d 100644 --- a/drivers/gpu/nvgpu/common/ce/ce.c +++ b/drivers/gpu/nvgpu/common/ce/ce.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -41,16 +42,11 @@ int nvgpu_ce_init_support(struct gk20a *g) #if defined(CONFIG_NVGPU_NON_FUSA) && defined(CONFIG_NVGPU_NEXT) if (g->ops.mc.reset_engine != NULL) { - err = nvgpu_next_mc_reset_engine(g, NVGPU_ENGINE_GRCE); + err = nvgpu_next_mc_reset_engine(g, NVGPU_DEVTYPE_LCE); if (err != 0) { nvgpu_err(g, "NVGPU_ENGINE_GRCE reset failed"); return err; } - err = nvgpu_next_mc_reset_engine(g, NVGPU_ENGINE_ASYNC_CE); - if (err != 0) { - nvgpu_err(g, "NVGPU_ENGINE_ASYNC_CE reset failed"); - return err; - } } else { #endif ce_reset_mask = nvgpu_engine_get_all_ce_reset_mask(g); diff --git a/drivers/gpu/nvgpu/common/device.c b/drivers/gpu/nvgpu/common/device.c index 12e67fc59..5d3609711 100644 --- a/drivers/gpu/nvgpu/common/device.c +++ b/drivers/gpu/nvgpu/common/device.c @@ -121,16 +121,17 @@ int nvgpu_device_init(struct gk20a *g) { u32 i; + device_dbg(g, "Initializating GPU device list"); + /* * Ground work - make sure we aren't doing this again and that we have * all the necessary data structures. */ if (g->devs != NULL) { + device_dbg(g, " GPU device list already present. Done."); return 0; } - device_dbg(g, "Initializating GPU device list"); - g->devs = nvgpu_kzalloc(g, sizeof(*g->devs)); if (g->devs == NULL) { return -ENOMEM; @@ -162,10 +163,13 @@ void nvgpu_device_cleanup(struct gk20a *g) u32 i; struct nvgpu_list_node *devlist; + device_dbg(g, "Releasing GPU device list"); + /* * Make unit testing a bit easier. */ if (g->devs == NULL) { + device_dbg(g, " Already done."); return; } @@ -193,14 +197,12 @@ void nvgpu_device_cleanup(struct gk20a *g) static const struct nvgpu_device *dev_instance_from_devlist( struct nvgpu_list_node *devlist, u32 inst_id) { - u32 i = 0U; struct nvgpu_device *dev; nvgpu_list_for_each_entry(dev, devlist, nvgpu_device, dev_list_node) { - if (inst_id == i) { + if (dev->inst_id == inst_id) { return dev; } - i++; } return NULL; diff --git a/drivers/gpu/nvgpu/common/fifo/engines.c b/drivers/gpu/nvgpu/common/fifo/engines.c index fed9543b4..45deff8b1 100644 --- a/drivers/gpu/nvgpu/common/fifo/engines.c +++ b/drivers/gpu/nvgpu/common/fifo/engines.c @@ -69,240 +69,159 @@ enum nvgpu_fifo_engine nvgpu_engine_enum_from_dev(struct gk20a *g, return ret; } -struct nvgpu_engine_info *nvgpu_engine_get_active_eng_info( +const struct nvgpu_device *nvgpu_engine_get_active_eng_info( struct gk20a *g, u32 engine_id) { - struct nvgpu_fifo *f = NULL; - u32 i; - struct nvgpu_engine_info *info = NULL; + struct nvgpu_fifo *f = &g->fifo; - if (g == NULL) { - return info; + if (engine_id >= f->max_engines) { + return NULL; } - f = &g->fifo; - - if (engine_id < f->max_engines) { - for (i = 0U; i < f->num_engines; i++) { - if (engine_id == f->active_engines_list[i]) { - info = &f->engine_info[engine_id]; - break; - } - } - } - - if (info == NULL) { - nvgpu_err(g, "engine_id is not in active list/invalid %d", - engine_id); - } - - return info; -} - -u32 nvgpu_engine_get_ids(struct gk20a *g, - u32 *engine_ids, u32 engine_id_sz, - enum nvgpu_fifo_engine engine_enum) -{ - struct nvgpu_fifo *f = NULL; - u32 instance_cnt = 0; - u32 i; - u32 engine_id = 0; - struct nvgpu_engine_info *info = NULL; - - if ((g == NULL) || (engine_id_sz == 0U) || - (engine_enum == NVGPU_ENGINE_INVAL)) { - return instance_cnt; - } - - f = &g->fifo; - for (i = 0U; i < f->num_engines; i++) { - engine_id = f->active_engines_list[i]; - info = &f->engine_info[engine_id]; - - if (info->engine_enum == engine_enum) { - if (instance_cnt < engine_id_sz) { - engine_ids[instance_cnt] = engine_id; - ++instance_cnt; - } else { - nvgpu_log_info(g, "warning engine_id table sz is small %d", - engine_id_sz); - } - } - } - return instance_cnt; + return f->host_engines[engine_id]; } bool nvgpu_engine_check_valid_id(struct gk20a *g, u32 engine_id) { - struct nvgpu_fifo *f = NULL; - u32 i; - bool valid = false; + struct nvgpu_fifo *f = &g->fifo; - if (g == NULL) { - return valid; + if (engine_id >= f->max_engines) { + return false; } - f = &g->fifo; - - if (engine_id < f->max_engines) { - for (i = 0U; i < f->num_engines; i++) { - if (engine_id == f->active_engines_list[i]) { - valid = true; - break; - } - } - } - - if (!valid) { - nvgpu_err(g, "engine_id is not in active list/invalid %d", - engine_id); - } - - return valid; + return f->host_engines[engine_id] != NULL; } u32 nvgpu_engine_get_gr_id(struct gk20a *g) { - u32 gr_engine_cnt = 0; - u32 gr_engine_id = NVGPU_INVALID_ENG_ID; - + const struct nvgpu_device *dev; /* Consider 1st available GR engine */ - gr_engine_cnt = nvgpu_engine_get_ids(g, &gr_engine_id, - 1, NVGPU_ENGINE_GR); - if (gr_engine_cnt == 0U) { - nvgpu_err(g, "No GR engine available on this device!"); + dev = nvgpu_device_get(g, NVGPU_DEVTYPE_GRAPHICS, 0); + if (dev == NULL) { + nvgpu_warn(g, "No GR devices on this GPU?!"); + return NVGPU_INVALID_ENG_ID; } - return gr_engine_id; + return dev->engine_id; } u32 nvgpu_engine_act_interrupt_mask(struct gk20a *g, u32 engine_id) { - struct nvgpu_engine_info *engine_info = NULL; + const struct nvgpu_device *dev = NULL; - engine_info = nvgpu_engine_get_active_eng_info(g, engine_id); - if (engine_info != NULL) { - return engine_info->intr_mask; + dev = nvgpu_engine_get_active_eng_info(g, engine_id); + if (dev == NULL) { + return 0; } - return 0; + return BIT32(dev->intr_id); } u32 nvgpu_gr_engine_interrupt_mask(struct gk20a *g) { - u32 eng_intr_mask = 0; - unsigned int i; - u32 engine_id = 0; - enum nvgpu_fifo_engine engine_enum; + const struct nvgpu_device *dev; - for (i = 0; i < g->fifo.num_engines; i++) { - u32 intr_mask; - - engine_id = g->fifo.active_engines_list[i]; - intr_mask = g->fifo.engine_info[engine_id].intr_mask; - engine_enum = g->fifo.engine_info[engine_id].engine_enum; - - if (engine_enum != NVGPU_ENGINE_GR) { - continue; - } - - eng_intr_mask |= intr_mask; + dev = nvgpu_device_get(g, NVGPU_DEVTYPE_GRAPHICS, 0); + if (dev == NULL) { + return 0U; } - return eng_intr_mask; + return BIT32(dev->intr_id); } u32 nvgpu_ce_engine_interrupt_mask(struct gk20a *g) { - u32 eng_intr_mask = 0; - unsigned int i; - u32 engine_id = 0; - enum nvgpu_fifo_engine engine_enum; + const struct nvgpu_device *dev; + u32 i; + u32 mask = 0U; if ((g->ops.ce.isr_stall == NULL) || (g->ops.ce.isr_nonstall == NULL)) { return 0U; } - for (i = 0; i < g->fifo.num_engines; i++) { - u32 intr_mask; - - engine_id = g->fifo.active_engines_list[i]; - intr_mask = g->fifo.engine_info[engine_id].intr_mask; - engine_enum = g->fifo.engine_info[engine_id].engine_enum; - - if ((engine_enum == NVGPU_ENGINE_GRCE) || - (engine_enum == NVGPU_ENGINE_ASYNC_CE)) { - eng_intr_mask |= intr_mask; + /* + * For old chips - pre-Pascal - we have COPY[0-2], for new chips we + * have some number of LCE instances. For the purpose of this code we + * imagine a system that could have both; in reality that'll never be + * the case. + * + * This can be cleaned up in the future by defining a SW type for CE and + * hiding this ugliness in the device management code. + */ + for (i = NVGPU_DEVTYPE_COPY0; i <= NVGPU_DEVTYPE_COPY2; i++) { + dev = nvgpu_device_get(g, i, i - NVGPU_DEVTYPE_COPY0); + if (dev == NULL) { + continue; } + + mask |= BIT32(dev->intr_id); } - return eng_intr_mask; + /* + * Now take care of LCEs. + */ + for (i = 0U; i < nvgpu_device_count(g, NVGPU_DEVTYPE_LCE); i++) { + dev = nvgpu_device_get(g, NVGPU_DEVTYPE_LCE, i); + nvgpu_assert(dev != NULL); + + mask |= BIT32(dev->intr_id); + } + + return mask; } u32 nvgpu_engine_get_all_ce_reset_mask(struct gk20a *g) { - u32 reset_mask = 0; - enum nvgpu_fifo_engine engine_enum; - struct nvgpu_fifo *f = NULL; + u32 mask = 0U; + const struct nvgpu_device *dev; u32 i; - struct nvgpu_engine_info *engine_info; - u32 engine_id = 0; - if (g == NULL) { - return reset_mask; - } - - f = &g->fifo; - - for (i = 0U; i < f->num_engines; i++) { - engine_id = f->active_engines_list[i]; - engine_info = &f->engine_info[engine_id]; - engine_enum = engine_info->engine_enum; - - if ((engine_enum == NVGPU_ENGINE_GRCE) || - (engine_enum == NVGPU_ENGINE_ASYNC_CE)) { - reset_mask |= engine_info->reset_mask; + /* + * Same principle as nvgpu_ce_engine_interrupt_mask. + */ + for (i = NVGPU_DEVTYPE_COPY0; i <= NVGPU_DEVTYPE_COPY2; i++) { + dev = nvgpu_device_get(g, i, i - NVGPU_DEVTYPE_COPY0); + if (dev == NULL) { + continue; } + + mask |= BIT32(dev->reset_id); } - return reset_mask; + /* + * Now take care of LCEs. + */ + for (i = 0U; i < nvgpu_device_count(g, NVGPU_DEVTYPE_LCE); i++) { + dev = nvgpu_device_get(g, NVGPU_DEVTYPE_LCE, i); + nvgpu_assert(dev != NULL); + + mask |= BIT32(dev->reset_id); + } + + return mask; } #ifdef CONFIG_NVGPU_FIFO_ENGINE_ACTIVITY -int nvgpu_engine_enable_activity(struct gk20a *g, - struct nvgpu_engine_info *eng_info) +static void nvgpu_engine_enable_activity(struct gk20a *g, + const struct nvgpu_device *dev) { - nvgpu_log(g, gpu_dbg_info, "start"); - - nvgpu_runlist_set_state(g, BIT32(eng_info->runlist_id), - RUNLIST_ENABLED); - return 0; + nvgpu_runlist_set_state(g, BIT32(dev->runlist_id), RUNLIST_ENABLED); } -int nvgpu_engine_enable_activity_all(struct gk20a *g) +void nvgpu_engine_enable_activity_all(struct gk20a *g) { - unsigned int i; - int err = 0, ret = 0; + u32 i; for (i = 0; i < g->fifo.num_engines; i++) { - u32 engine_id = g->fifo.active_engines_list[i]; - err = nvgpu_engine_enable_activity(g, - &g->fifo.engine_info[engine_id]); - if (err != 0) { - nvgpu_err(g, - "failed to enable engine %d activity", engine_id); - ret = err; - } + nvgpu_engine_enable_activity(g, g->fifo.active_engines[i]); } - - return ret; } int nvgpu_engine_disable_activity(struct gk20a *g, - struct nvgpu_engine_info *eng_info, + const struct nvgpu_device *dev, bool wait_for_idle) { u32 pbdma_chid = NVGPU_INVALID_CHANNEL_ID; @@ -311,14 +230,14 @@ int nvgpu_engine_disable_activity(struct gk20a *g, u32 token = PMU_INVALID_MUTEX_OWNER_ID; int mutex_ret = -EINVAL; #endif - struct nvgpu_channel *ch = NULL; int err = 0; + struct nvgpu_channel *ch = NULL; struct nvgpu_engine_status_info engine_status; struct nvgpu_pbdma_status_info pbdma_status; nvgpu_log_fn(g, " "); - g->ops.engine_status.read_engine_status_info(g, eng_info->engine_id, + g->ops.engine_status.read_engine_status_info(g, dev->engine_id, &engine_status); if (engine_status.is_busy && !wait_for_idle) { return -EBUSY; @@ -331,11 +250,12 @@ int nvgpu_engine_disable_activity(struct gk20a *g, } #endif - nvgpu_runlist_set_state(g, BIT32(eng_info->runlist_id), + nvgpu_runlist_set_state(g, BIT32(dev->runlist_id), RUNLIST_DISABLED); /* chid from pbdma status */ - g->ops.pbdma_status.read_pbdma_status_info(g, eng_info->pbdma_id, + g->ops.pbdma_status.read_pbdma_status_info(g, + dev->pbdma_id, &pbdma_status); if (nvgpu_pbdma_status_is_chsw_valid(&pbdma_status) || nvgpu_pbdma_status_is_chsw_save(&pbdma_status)) { @@ -359,7 +279,7 @@ int nvgpu_engine_disable_activity(struct gk20a *g, } /* chid from engine status */ - g->ops.engine_status.read_engine_status_info(g, eng_info->engine_id, + g->ops.engine_status.read_engine_status_info(g, dev->engine_id, &engine_status); if (nvgpu_engine_status_is_ctxsw_valid(&engine_status) || nvgpu_engine_status_is_ctxsw_save(&engine_status)) { @@ -393,10 +313,7 @@ clean_up: #endif if (err != 0) { nvgpu_log_fn(g, "failed"); - if (nvgpu_engine_enable_activity(g, eng_info) != 0) { - nvgpu_err(g, - "failed to enable gr engine activity"); - } + nvgpu_engine_enable_activity(g, dev); } else { nvgpu_log_fn(g, "done"); } @@ -408,16 +325,14 @@ int nvgpu_engine_disable_activity_all(struct gk20a *g, { unsigned int i; int err = 0, ret = 0; - u32 engine_id; for (i = 0; i < g->fifo.num_engines; i++) { - engine_id = g->fifo.active_engines_list[i]; err = nvgpu_engine_disable_activity(g, - &g->fifo.engine_info[engine_id], + g->fifo.active_engines[i], wait_for_idle); if (err != 0) { nvgpu_err(g, "failed to disable engine %d activity", - engine_id); + g->fifo.active_engines[i]->engine_id); ret = err; break; } @@ -425,14 +340,8 @@ int nvgpu_engine_disable_activity_all(struct gk20a *g, if (err != 0) { while (i-- != 0U) { - engine_id = g->fifo.active_engines_list[i]; - err = nvgpu_engine_enable_activity(g, - &g->fifo.engine_info[engine_id]); - if (err != 0) { - nvgpu_err(g, - "failed to re-enable engine %d activity", - engine_id); - } + nvgpu_engine_enable_activity(g, + g->fifo.active_engines[i]); } } @@ -503,21 +412,23 @@ int nvgpu_engine_setup_sw(struct gk20a *g) size_t size; f->max_engines = nvgpu_get_litter_value(g, GPU_LIT_HOST_NUM_ENGINES); - size = nvgpu_safe_mult_u64(f->max_engines, sizeof(*f->engine_info)); - f->engine_info = nvgpu_kzalloc(g, size); - if (f->engine_info == NULL) { - nvgpu_err(g, "no mem for engine info"); + size = nvgpu_safe_mult_u64(f->max_engines, + sizeof(struct nvgpu_device *)); + + /* + * Allocate the two device lists for host devices. + */ + f->host_engines = nvgpu_kzalloc(g, size); + if (f->host_engines == NULL) { + nvgpu_err(g, "OOM allocating host engine list"); return -ENOMEM; } - - size = nvgpu_safe_mult_u64(f->max_engines, sizeof(u32)); - f->active_engines_list = nvgpu_kzalloc(g, size); - if (f->active_engines_list == NULL) { + f->active_engines = nvgpu_kzalloc(g, size); + if (f->active_engines == NULL) { nvgpu_err(g, "no mem for active engine list"); err = -ENOMEM; goto clean_up_engine_info; } - (void) memset(f->active_engines_list, 0xff, size); err = nvgpu_engine_init_info(f); if (err != 0) { @@ -528,12 +439,12 @@ int nvgpu_engine_setup_sw(struct gk20a *g) return 0; clean_up: - nvgpu_kfree(g, f->active_engines_list); - f->active_engines_list = NULL; + nvgpu_kfree(g, f->active_engines); + f->active_engines = NULL; clean_up_engine_info: - nvgpu_kfree(g, f->engine_info); - f->engine_info = NULL; + nvgpu_kfree(g, f->host_engines); + f->host_engines = NULL; return err; } @@ -543,18 +454,17 @@ void nvgpu_engine_cleanup_sw(struct gk20a *g) struct nvgpu_fifo *f = &g->fifo; f->num_engines = 0; - nvgpu_kfree(g, f->engine_info); - f->engine_info = NULL; - nvgpu_kfree(g, f->active_engines_list); - f->active_engines_list = NULL; + nvgpu_kfree(g, f->host_engines); + f->host_engines = NULL; + nvgpu_kfree(g, f->active_engines); + f->active_engines = NULL; } #ifdef CONFIG_NVGPU_ENGINE_RESET void nvgpu_engine_reset(struct gk20a *g, u32 engine_id) { - enum nvgpu_fifo_engine engine_enum = NVGPU_ENGINE_INVAL; - struct nvgpu_engine_info *engine_info; struct nvgpu_swprofiler *prof = &g->fifo.eng_reset_profiler; + const struct nvgpu_device *dev; nvgpu_log_fn(g, " "); @@ -564,170 +474,168 @@ void nvgpu_engine_reset(struct gk20a *g, u32 engine_id) nvgpu_swprofile_begin_sample(prof); - engine_info = nvgpu_engine_get_active_eng_info(g, engine_id); - - if (engine_info != NULL) { - engine_enum = engine_info->engine_enum; - } - - if (engine_enum == NVGPU_ENGINE_INVAL) { + dev = nvgpu_engine_get_active_eng_info(g, engine_id); + if (dev == NULL) { nvgpu_err(g, "unsupported engine_id %d", engine_id); + return; } - nvgpu_swprofile_snapshot(prof, PROF_ENG_RESET_PREAMBLE); - - if (engine_enum == NVGPU_ENGINE_GR) { -#ifdef CONFIG_NVGPU_POWER_PG - if (nvgpu_pg_elpg_disable(g) != 0 ) { - nvgpu_err(g, "failed to set disable elpg"); - } -#endif - nvgpu_swprofile_snapshot(prof, PROF_ENG_RESET_ELPG_DISABLE); - -#ifdef CONFIG_NVGPU_FECS_TRACE - /* - * Resetting engine will alter read/write index. Need to flush - * circular buffer before re-enabling FECS. - */ - if (g->ops.gr.fecs_trace.reset != NULL) { - if (g->ops.gr.fecs_trace.reset(g) != 0) { - nvgpu_warn(g, "failed to reset fecs traces"); - } - } -#endif - - nvgpu_swprofile_snapshot(prof, PROF_ENG_RESET_FECS_TRACE_RESET); - - if (!nvgpu_platform_is_simulation(g)) { - int err = 0; - - /*HALT_PIPELINE method, halt GR engine*/ - err = g->ops.gr.falcon.ctrl_ctxsw(g, - NVGPU_GR_FALCON_METHOD_HALT_PIPELINE, 0U, NULL); - if (err != 0) { - nvgpu_err(g, "failed to halt gr pipe"); - } - - nvgpu_swprofile_snapshot(prof, PROF_ENG_RESET_HALT_PIPELINE); - - /* - * resetting engine using mc_enable_r() is not - * enough, we do full init sequence - */ - nvgpu_log(g, gpu_dbg_info, "resetting gr engine"); - - err = nvgpu_gr_reset(g); - if (err != 0) { - nvgpu_err(g, "failed to reset gr engine"); - } - nvgpu_swprofile_snapshot(prof, PROF_ENG_RESET_GR_RESET); - } else { - nvgpu_log(g, gpu_dbg_info, - "HALT gr pipe not supported and " - "gr cannot be reset without halting gr pipe"); - } - -#ifdef CONFIG_NVGPU_POWER_PG - if (nvgpu_pg_elpg_enable(g) != 0 ) { - nvgpu_err(g, "failed to set enable elpg"); - } -#endif + if (!nvgpu_device_is_ce(g, dev) && + !nvgpu_device_is_graphics(g, dev)) { + nvgpu_warn(g, "Ignoring reset for non-host engine."); + return; } - nvgpu_swprofile_snapshot(prof, PROF_ENG_RESET_ELPG_REENABLE); - - if ((engine_enum == NVGPU_ENGINE_GRCE) || - (engine_enum == NVGPU_ENGINE_ASYNC_CE)) { -#if defined(CONFIG_NVGPU_NON_FUSA) && defined(CONFIG_NVGPU_NEXT) + /* + * Simple case first: reset a copy engine. + */ + if (nvgpu_device_is_ce(g, dev)) { +#if !defined(CONFIG_NVGPU_NON_FUSA) || !defined(CONFIG_NVGPU_NEXT) + g->ops.mc.reset(g, BIT32(dev->reset_id)); +#else int err = 0; if (g->ops.mc.reset_engine != NULL) { - err = g->ops.mc.reset_engine(g, - engine_info->nvgpu_next.reset_id); + err = g->ops.mc.reset_engine(g, dev->type); if (err != 0) { nvgpu_err(g, "failed to reset ce engine"); } } else { -#endif - g->ops.mc.reset(g, engine_info->reset_mask); -#if defined(CONFIG_NVGPU_NON_FUSA) && defined(CONFIG_NVGPU_NEXT) + g->ops.mc.reset(g, BIT32(dev->reset_id)); } #endif + return; } + + nvgpu_swprofile_snapshot(prof, PROF_ENG_RESET_PREAMBLE); + + /* + * Now reset a GR engine. + */ +#ifdef CONFIG_NVGPU_POWER_PG + if (nvgpu_pg_elpg_disable(g) != 0 ) { + nvgpu_err(g, "failed to set disable elpg"); + } +#endif + nvgpu_swprofile_snapshot(prof, PROF_ENG_RESET_ELPG_DISABLE); + +#ifdef CONFIG_NVGPU_FECS_TRACE + /* + * Resetting engine will alter read/write index. Need to flush + * circular buffer before re-enabling FECS. + */ + if (g->ops.gr.fecs_trace.reset != NULL) { + if (g->ops.gr.fecs_trace.reset(g) != 0) { + nvgpu_warn(g, "failed to reset fecs traces"); + } + } +#endif + + nvgpu_swprofile_snapshot(prof, PROF_ENG_RESET_FECS_TRACE_RESET); + + if (!nvgpu_platform_is_simulation(g)) { + int err = 0; + + /*HALT_PIPELINE method, halt GR engine*/ + err = g->ops.gr.falcon.ctrl_ctxsw(g, + NVGPU_GR_FALCON_METHOD_HALT_PIPELINE, 0U, NULL); + if (err != 0) { + nvgpu_err(g, "failed to halt gr pipe"); + } + + nvgpu_swprofile_snapshot(prof, PROF_ENG_RESET_HALT_PIPELINE); + + /* + * resetting engine using mc_enable_r() is not + * enough, we do full init sequence + */ + nvgpu_log(g, gpu_dbg_info, "resetting gr engine"); + + err = nvgpu_gr_reset(g); + if (err != 0) { + nvgpu_err(g, "failed to reset gr engine"); + } + } else { + nvgpu_log(g, gpu_dbg_info, + "HALT gr pipe not supported and " + "gr cannot be reset without halting gr pipe"); + } + +#ifdef CONFIG_NVGPU_POWER_PG + if (nvgpu_pg_elpg_enable(g) != 0) { + nvgpu_err(g, "failed to set enable elpg"); + } + nvgpu_swprofile_snapshot(prof, PROF_ENG_RESET_ELPG_REENABLE); +#endif } #endif u32 nvgpu_engine_get_fast_ce_runlist_id(struct gk20a *g) { - u32 ce_runlist_id = nvgpu_engine_get_gr_runlist_id(g); - enum nvgpu_fifo_engine engine_enum; - struct nvgpu_fifo *f = NULL; + const struct nvgpu_device *dev; + u32 nr_lces; u32 i; - struct nvgpu_engine_info *engine_info; - u32 engine_id = 0U; - if (g == NULL) { - return ce_runlist_id; + /* + * Obtain a runlist ID for the fastest available CE. The priority order + * is: + * + * 1. Last available LCE + * 2. Last available COPY[0-2] + * 3. GRAPHICS runlist as a last resort. + */ + nr_lces = nvgpu_device_count(g, NVGPU_DEVTYPE_LCE); + if (nr_lces > 0U) { + dev = nvgpu_device_get(g, + NVGPU_DEVTYPE_LCE, + nr_lces - 1U); + nvgpu_assert(dev != NULL); + + return dev->runlist_id; } - f = &g->fifo; - - for (i = 0U; i < f->num_engines; i++) { - engine_id = f->active_engines_list[i]; - engine_info = &f->engine_info[engine_id]; - engine_enum = engine_info->engine_enum; - - /* select last available ASYNC_CE if available */ - if (engine_enum == NVGPU_ENGINE_ASYNC_CE) { - ce_runlist_id = engine_info->runlist_id; + /* + * Note: this only works since NVGPU_DEVTYPE_GRAPHICS is 0 and the COPYx + * are all > 0. + */ + for (i = NVGPU_DEVTYPE_COPY2; i >= NVGPU_DEVTYPE_COPY0; i--) { + dev = nvgpu_device_get(g, i, i - NVGPU_DEVTYPE_COPY0); + if (dev != NULL) { + return dev->runlist_id; } } - return ce_runlist_id; + /* + * Fall back to GR. + */ + dev = nvgpu_device_get(g, NVGPU_DEVTYPE_GRAPHICS, 0); + nvgpu_assert(dev != NULL); + + return dev->runlist_id; } u32 nvgpu_engine_get_gr_runlist_id(struct gk20a *g) { - struct nvgpu_fifo *f = &g->fifo; - u32 gr_engine_cnt = 0; - u32 gr_engine_id = NVGPU_INVALID_ENG_ID; - struct nvgpu_engine_info *engine_info; - u32 gr_runlist_id = U32_MAX; + const struct nvgpu_device *dev; - /* Consider 1st available GR engine */ - gr_engine_cnt = nvgpu_engine_get_ids(g, &gr_engine_id, - 1, NVGPU_ENGINE_GR); - - if (gr_engine_cnt == 0U) { - nvgpu_err(g, - "No GR engine available on this device!"); - goto end; + dev = nvgpu_device_get(g, NVGPU_DEVTYPE_GRAPHICS, 0); + if (dev == NULL) { + nvgpu_warn(g, "No GR device on this GPU?!"); + return NVGPU_INVALID_RUNLIST_ID; } - engine_info = &f->engine_info[gr_engine_id]; - gr_runlist_id = engine_info->runlist_id; - -end: - return gr_runlist_id; + return dev->runlist_id; } bool nvgpu_engine_is_valid_runlist_id(struct gk20a *g, u32 runlist_id) { - struct nvgpu_fifo *f = NULL; u32 i; - u32 engine_id; - struct nvgpu_engine_info *engine_info; - - if (g == NULL) { - return false; - } - - f = &g->fifo; + struct nvgpu_fifo *f = &g->fifo; for (i = 0U; i < f->num_engines; i++) { - engine_id = f->active_engines_list[i]; - engine_info = &f->engine_info[engine_id]; - if (engine_info->runlist_id == runlist_id) { + const struct nvgpu_device *dev = f->active_engines[i]; + + if (dev->runlist_id == runlist_id) { return true; } } @@ -740,37 +648,35 @@ bool nvgpu_engine_is_valid_runlist_id(struct gk20a *g, u32 runlist_id) */ u32 nvgpu_engine_id_to_mmu_fault_id(struct gk20a *g, u32 engine_id) { - u32 fault_id = NVGPU_INVALID_ENG_ID; - struct nvgpu_engine_info *engine_info; + const struct nvgpu_device *dev; - engine_info = nvgpu_engine_get_active_eng_info(g, engine_id); + dev = nvgpu_engine_get_active_eng_info(g, engine_id); - if (engine_info != NULL) { - fault_id = engine_info->fault_id; - } else { - nvgpu_err(g, "engine_id: %d is not in active list/invalid", - engine_id); + if (dev == NULL) { + nvgpu_err(g, + "engine_id: %u is not in active list", + engine_id); + return NVGPU_INVALID_ENG_ID; } - return fault_id; + + return dev->fault_id; } u32 nvgpu_engine_mmu_fault_id_to_engine_id(struct gk20a *g, u32 fault_id) { u32 i; - u32 engine_id; - struct nvgpu_engine_info *engine_info; + const struct nvgpu_device *dev; struct nvgpu_fifo *f = &g->fifo; for (i = 0U; i < f->num_engines; i++) { - engine_id = f->active_engines_list[i]; - engine_info = &g->fifo.engine_info[engine_id]; + dev = f->active_engines[i]; - if (engine_info->fault_id == fault_id) { - break; + if (dev->fault_id == fault_id) { + return dev->engine_id; } - engine_id = NVGPU_INVALID_ENG_ID; } - return engine_id; + + return NVGPU_INVALID_ENG_ID; } u32 nvgpu_engine_get_mask_on_id(struct gk20a *g, u32 id, bool is_tsg) @@ -783,10 +689,10 @@ u32 nvgpu_engine_get_mask_on_id(struct gk20a *g, u32 id, bool is_tsg) bool busy; for (i = 0; i < g->fifo.num_engines; i++) { - u32 engine_id = g->fifo.active_engines_list[i]; + const struct nvgpu_device *dev = g->fifo.active_engines[i]; g->ops.engine_status.read_engine_status_info(g, - engine_id, &engine_status); + dev->engine_id, &engine_status); if (nvgpu_engine_status_is_ctxsw_load( &engine_status)) { @@ -799,110 +705,90 @@ u32 nvgpu_engine_get_mask_on_id(struct gk20a *g, u32 id, bool is_tsg) busy = engine_status.is_busy; - if (busy && (ctx_id == id)) { - if ((is_tsg && (type == - ENGINE_STATUS_CTX_ID_TYPE_TSGID)) || - (!is_tsg && (type == - ENGINE_STATUS_CTX_ID_TYPE_CHID))) { - engines |= BIT32(engine_id); - } + if (!busy || !(ctx_id == id)) { + continue; + } + + if ((is_tsg && (type == ENGINE_STATUS_CTX_ID_TYPE_TSGID)) || + (!is_tsg && (type == ENGINE_STATUS_CTX_ID_TYPE_CHID))) { + engines |= BIT32(dev->engine_id); } } return engines; } -static int nvgpu_engine_init_from_device_info(struct gk20a *g, - struct nvgpu_engine_info *info, - const struct nvgpu_device *dev) +static int nvgpu_engine_init_one_dev(struct nvgpu_fifo *f, + const struct nvgpu_device *dev) { bool found; - struct nvgpu_device *dev_rw = (struct nvgpu_device *)dev; + struct nvgpu_device *dev_rw; + struct gk20a *g = f->g; - info->engine_id = dev->engine_id; - info->intr_mask |= BIT32(dev->intr_id); - info->reset_mask |= BIT32(dev->reset_id); - info->runlist_id = dev->runlist_id; - info->inst_id = dev->inst_id; - info->pri_base = dev->pri_base; - info->engine_enum = nvgpu_engine_enum_from_dev(g, dev); - info->fault_id = dev->fault_id; + dev_rw = (struct nvgpu_device *)dev; /* - * Populate the PBDMA info for this device; ideally it'd be done during - * device init, but the FIFO unit is not out of reset that early in the - * nvgpu_finalize_poweron() sequence. + * Populate the PBDMA info for this device; ideally it'd be done + * during device init, but the FIFO unit is not out of reset that + * early in the nvgpu_finalize_poweron() sequence. * - * We only need to do this for native; vGPU already has pbdma_id populated - * during device initialization. + * We only need to do this for native; vGPU already has pbdma_id + * populated during device initialization. */ if (g->ops.fifo.find_pbdma_for_runlist != NULL) { found = g->ops.fifo.find_pbdma_for_runlist(g, - dev->runlist_id, - &dev_rw->pbdma_id); + dev->runlist_id, + &dev_rw->pbdma_id); if (!found) { nvgpu_err(g, "busted pbdma map"); return -EINVAL; } } - info->pbdma_id = dev->pbdma_id; #if defined(CONFIG_NVGPU_NEXT) - return nvgpu_next_engine_init_from_device_info(g, info, dev); -#else - return 0; + { + int err = nvgpu_next_engine_init_one_dev(g, dev); + if (err != 0) { + return err; + } + } #endif -} -static int nvgpu_engine_populate_gr_info(struct nvgpu_fifo *f, - u32 gr_inst) -{ - struct gk20a *g = f->g; - const struct nvgpu_device *dev; - int ret; - - dev = nvgpu_device_get(g, NVGPU_DEVTYPE_GRAPHICS, gr_inst); - if (dev == NULL) { - nvgpu_err(g, "Failed to get graphics engine inst: %d", gr_inst); - return -EINVAL; - } - - ret = nvgpu_engine_init_from_device_info(g, - &g->fifo.engine_info[dev->engine_id], - dev); - if (ret != 0) { - nvgpu_err(g, "Failed to init engine_info for engine_id: %d", - dev->engine_id); - return -EINVAL; - } - - /* engine_id starts from 0 to NV_HOST_NUM_ENGINES */ - f->active_engines_list[f->num_engines] = dev->engine_id; - f->num_engines = nvgpu_safe_add_u32(f->num_engines, 1U); + f->host_engines[dev->engine_id] = dev; + f->active_engines[f->num_engines] = dev; + ++f->num_engines; return 0; } int nvgpu_engine_init_info(struct nvgpu_fifo *f) { + u32 i; + int err; struct gk20a *g = f->g; - int ret = 0; - u32 gr_inst; f->num_engines = 0; - for (gr_inst = 0U; - gr_inst < nvgpu_device_count(g, NVGPU_DEVTYPE_GRAPHICS); - gr_inst++) { - ret = nvgpu_engine_populate_gr_info(f, gr_inst); - if (ret != 0) { - return ret; + nvgpu_log(g, gpu_dbg_device, "Loading host engines from device list"); + nvgpu_log(g, gpu_dbg_device, " GFX devices: %u", + nvgpu_device_count(g, NVGPU_DEVTYPE_GRAPHICS)); + + for (i = 0U; i < nvgpu_device_count(g, NVGPU_DEVTYPE_GRAPHICS); i++) { + const struct nvgpu_device *dev = + nvgpu_device_get(g, NVGPU_DEVTYPE_GRAPHICS, i); + + if (dev == NULL) { + nvgpu_err(g, "Failed to get graphics engine %d", i); + return -EINVAL; + } + + err = nvgpu_engine_init_one_dev(f, dev); + if (err != 0) { + return err; } } - ret = g->ops.engine.init_ce_info(f); - - return ret; + return g->ops.engine.init_ce_info(f); } void nvgpu_engine_get_id_and_type(struct gk20a *g, u32 engine_id, @@ -931,13 +817,13 @@ u32 nvgpu_engine_find_busy_doing_ctxsw(struct gk20a *g, u32 id = U32_MAX; bool is_tsg = false; u32 mailbox2; - u32 engine_id = NVGPU_INVALID_ENG_ID; struct nvgpu_engine_status_info engine_status; + const struct nvgpu_device *dev = NULL; for (i = 0U; i < g->fifo.num_engines; i++) { + dev = g->fifo.active_engines[i]; - engine_id = g->fifo.active_engines_list[i]; - g->ops.engine_status.read_engine_status_info(g, engine_id, + g->ops.engine_status.read_engine_status_info(g, dev->engine_id, &engine_status); /* @@ -946,7 +832,6 @@ u32 nvgpu_engine_find_busy_doing_ctxsw(struct gk20a *g, */ if (!engine_status.is_busy || !nvgpu_engine_status_is_ctxsw(&engine_status)) { - engine_id = NVGPU_INVALID_ENG_ID; continue; } @@ -977,7 +862,7 @@ u32 nvgpu_engine_find_busy_doing_ctxsw(struct gk20a *g, *id_ptr = id; *is_tsg_ptr = is_tsg; - return engine_id; + return dev->engine_id; } u32 nvgpu_engine_get_runlist_busy_engines(struct gk20a *g, u32 runlist_id) @@ -987,16 +872,13 @@ u32 nvgpu_engine_get_runlist_busy_engines(struct gk20a *g, u32 runlist_id) struct nvgpu_engine_status_info engine_status; for (i = 0U; i < f->num_engines; i++) { - u32 engine_id = f->active_engines_list[i]; - u32 engine_runlist = f->engine_info[engine_id].runlist_id; - bool engine_busy; + const struct nvgpu_device *dev = f->active_engines[i]; - g->ops.engine_status.read_engine_status_info(g, engine_id, + g->ops.engine_status.read_engine_status_info(g, dev->engine_id, &engine_status); - engine_busy = engine_status.is_busy; - if (engine_busy && (engine_runlist == runlist_id)) { - eng_bitmask |= BIT32(engine_id); + if (engine_status.is_busy && (dev->runlist_id == runlist_id)) { + eng_bitmask |= BIT32(dev->engine_id); } } @@ -1007,20 +889,10 @@ u32 nvgpu_engine_get_runlist_busy_engines(struct gk20a *g, u32 runlist_id) bool nvgpu_engine_should_defer_reset(struct gk20a *g, u32 engine_id, u32 engine_subid, bool fake_fault) { - enum nvgpu_fifo_engine engine_enum = NVGPU_ENGINE_INVAL; - struct nvgpu_engine_info *engine_info; + const struct nvgpu_device *dev; - if (g == NULL) { - return false; - } - - engine_info = nvgpu_engine_get_active_eng_info(g, engine_id); - - if (engine_info != NULL) { - engine_enum = engine_info->engine_enum; - } - - if (engine_enum == NVGPU_ENGINE_INVAL) { + dev = nvgpu_engine_get_active_eng_info(g, engine_id); + if (dev == NULL) { return false; } @@ -1038,7 +910,7 @@ bool nvgpu_engine_should_defer_reset(struct gk20a *g, u32 engine_id, return false; } - if (engine_enum != NVGPU_ENGINE_GR) { + if (dev->type != NVGPU_DEVTYPE_GRAPHICS) { return false; } @@ -1064,26 +936,25 @@ u32 nvgpu_engine_mmu_fault_id_to_veid(struct gk20a *g, u32 mmu_fault_id, return veid; } -u32 nvgpu_engine_mmu_fault_id_to_eng_id_and_veid(struct gk20a *g, +static u32 nvgpu_engine_mmu_fault_id_to_eng_id_and_veid(struct gk20a *g, u32 mmu_fault_id, u32 *veid) { u32 i; u32 engine_id = INVAL_ID; - struct nvgpu_engine_info *engine_info; + const struct nvgpu_device *dev; struct nvgpu_fifo *f = &g->fifo; for (i = 0U; i < f->num_engines; i++) { - engine_id = f->active_engines_list[i]; - engine_info = &g->fifo.engine_info[engine_id]; + dev = f->active_engines[i]; - if (engine_info->engine_enum == NVGPU_ENGINE_GR) { + if (dev->type == NVGPU_DEVTYPE_GRAPHICS) { *veid = nvgpu_engine_mmu_fault_id_to_veid(g, - mmu_fault_id, engine_info->fault_id); + mmu_fault_id, dev->fault_id); if (*veid != INVAL_ID) { break; } } else { - if (engine_info->fault_id == mmu_fault_id) { + if (dev->fault_id == mmu_fault_id) { *veid = INVAL_ID; break; } diff --git a/drivers/gpu/nvgpu/common/fifo/runlist.c b/drivers/gpu/nvgpu/common/fifo/runlist.c index e4dd48534..0e05e9c1e 100644 --- a/drivers/gpu/nvgpu/common/fifo/runlist.c +++ b/drivers/gpu/nvgpu/common/fifo/runlist.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -681,8 +682,8 @@ void nvgpu_runlist_cleanup_sw(struct gk20a *g) void nvgpu_runlist_init_enginfo(struct gk20a *g, struct nvgpu_fifo *f) { struct nvgpu_runlist_info *runlist; - struct nvgpu_engine_info *engine_info; - u32 i, engine_id, j; + const struct nvgpu_device *dev; + u32 i, j; nvgpu_log_fn(g, " "); @@ -700,11 +701,10 @@ void nvgpu_runlist_init_enginfo(struct gk20a *g, struct nvgpu_fifo *f) runlist->runlist_id, runlist->pbdma_bitmask); for (j = 0; j < f->num_engines; j++) { - engine_id = f->active_engines_list[j]; - engine_info = &f->engine_info[engine_id]; + dev = f->active_engines[j]; - if (engine_info->runlist_id == runlist->runlist_id) { - runlist->eng_bitmask |= BIT32(engine_id); + if (dev->runlist_id == runlist->runlist_id) { + runlist->eng_bitmask |= BIT32(dev->engine_id); } } nvgpu_log(g, gpu_dbg_info, "runlist %d: act eng bitmask 0x%x", diff --git a/drivers/gpu/nvgpu/common/gr/gr.c b/drivers/gpu/nvgpu/common/gr/gr.c index 42517c50a..ef67c54e2 100644 --- a/drivers/gpu/nvgpu/common/gr/gr.c +++ b/drivers/gpu/nvgpu/common/gr/gr.c @@ -44,6 +44,7 @@ #include #include #include +#include #include #if defined(CONFIG_NVGPU_NON_FUSA) && defined(CONFIG_NVGPU_NEXT) #include @@ -665,7 +666,7 @@ static int gr_init_prepare_hw(struct gk20a *g) if (g->ops.mc.reset_engine != NULL) { g->ops.mc.reset(g, g->ops.mc.reset_mask(g, NVGPU_UNIT_PERFMON)); - err = nvgpu_next_mc_reset_engine(g, NVGPU_ENGINE_GR); + err = nvgpu_next_mc_reset_engine(g, NVGPU_DEVTYPE_GRAPHICS); if (err != 0) { nvgpu_err(g, "NVGPU_ENGINE_GR reset failed"); return err; diff --git a/drivers/gpu/nvgpu/common/init/nvgpu_init.c b/drivers/gpu/nvgpu/common/init/nvgpu_init.c index 66fb1381b..27a074163 100644 --- a/drivers/gpu/nvgpu/common/init/nvgpu_init.c +++ b/drivers/gpu/nvgpu/common/init/nvgpu_init.c @@ -900,6 +900,12 @@ static void gk20a_free_cb(struct nvgpu_ref *refcount) g->ops.ltc.ltc_remove_support(g); } + /* + * Free the device list once the gk20a struct is removed. We don't want + * to do this during the railgate poweroff sequence since that means + * that the device list disappears every time we rail-gate. That will + * cause the fifo engine code to explode. + */ nvgpu_device_cleanup(g); #ifdef CONFIG_NVGPU_PROFILER diff --git a/drivers/gpu/nvgpu/common/nvlink/nvlink.c b/drivers/gpu/nvgpu/common/nvlink/nvlink.c index 1172bc986..07685226e 100644 --- a/drivers/gpu/nvgpu/common/nvlink/nvlink.c +++ b/drivers/gpu/nvgpu/common/nvlink/nvlink.c @@ -26,6 +26,7 @@ #include #include #include +#include #ifdef CONFIG_NVGPU_NVLINK @@ -156,8 +157,8 @@ fail: static int nvgpu_nvlink_discover_ioctrl(struct gk20a *g) { u32 i; - struct nvgpu_nvlink_ioctrl_list *ioctrl_table; u32 ioctrl_num_entries = 0U; + struct nvgpu_nvlink_ioctrl_list *ioctrl_table; ioctrl_num_entries = nvgpu_device_count(g, NVGPU_DEVTYPE_IOCTRL); nvgpu_log_info(g, "ioctrl_num_entries: %d", ioctrl_num_entries); diff --git a/drivers/gpu/nvgpu/common/power_features/cg/cg.c b/drivers/gpu/nvgpu/common/power_features/cg/cg.c index ebdf5fe48..0c598a104 100644 --- a/drivers/gpu/nvgpu/common/power_features/cg/cg.c +++ b/drivers/gpu/nvgpu/common/power_features/cg/cg.c @@ -22,6 +22,7 @@ #include #include +#include #include #include @@ -29,22 +30,18 @@ static void nvgpu_cg_set_mode(struct gk20a *g, u32 cgmode, u32 mode_config) { u32 n; u32 engine_id = 0; -#ifdef CONFIG_NVGPU_NON_FUSA - struct nvgpu_engine_info *engine_info = NULL; -#endif + const struct nvgpu_device *dev = NULL; struct nvgpu_fifo *f = &g->fifo; nvgpu_log_fn(g, " "); for (n = 0; n < f->num_engines; n++) { - engine_id = f->active_engines_list[n]; + dev = f->active_engines[n]; #ifdef CONFIG_NVGPU_NON_FUSA - engine_info = &f->engine_info[engine_id]; - /* gr_engine supports both BLCG and ELCG */ - if ((cgmode == BLCG_MODE) && (engine_info->engine_enum == - NVGPU_ENGINE_GR)) { + if ((cgmode == BLCG_MODE) && + (dev->type == NVGPU_DEVTYPE_GRAPHICS)) { g->ops.therm.init_blcg_mode(g, (u32)mode_config, engine_id); break; @@ -52,7 +49,7 @@ static void nvgpu_cg_set_mode(struct gk20a *g, u32 cgmode, u32 mode_config) #endif if (cgmode == ELCG_MODE) { g->ops.therm.init_elcg_mode(g, (u32)mode_config, - engine_id); + dev->engine_id); } else { nvgpu_err(g, "invalid cg mode %d, config %d for " "engine_id %d", diff --git a/drivers/gpu/nvgpu/common/vgpu/fifo/fifo_vgpu.c b/drivers/gpu/nvgpu/common/vgpu/fifo/fifo_vgpu.c index 1ff6af71d..a1ccf376a 100644 --- a/drivers/gpu/nvgpu/common/vgpu/fifo/fifo_vgpu.c +++ b/drivers/gpu/nvgpu/common/vgpu/fifo/fifo_vgpu.c @@ -47,6 +47,19 @@ void vgpu_fifo_cleanup_sw(struct gk20a *g) { + u32 i; + struct nvgpu_fifo *f = &g->fifo; + + for (i = 0U; i < f->max_engines; i++) { + if (f->host_engines[i] == NULL) { + continue; + } + + /* + * Cast to (void *) to get rid of the constness. + */ + nvgpu_kfree(g, (void *)f->host_engines[i]); + } nvgpu_fifo_cleanup_sw_common(g); } diff --git a/drivers/gpu/nvgpu/common/vgpu/top/top_vgpu.c b/drivers/gpu/nvgpu/common/vgpu/top/top_vgpu.c index a899a781b..82d4ad311 100644 --- a/drivers/gpu/nvgpu/common/vgpu/top/top_vgpu.c +++ b/drivers/gpu/nvgpu/common/vgpu/top/top_vgpu.c @@ -54,6 +54,7 @@ struct nvgpu_device *vgpu_top_parse_next_dev(struct gk20a *g, u32 *token) /* * Copy the engine data into the device and return it to our caller. */ + dev->type = engines->info[*token].engine_enum; dev->engine_id = engines->info[*token].engine_id; dev->intr_id = nvgpu_ffs(engines->info[*token].intr_mask) - 1; dev->reset_id = nvgpu_ffs(engines->info[*token].reset_mask) - 1; @@ -63,29 +64,6 @@ struct nvgpu_device *vgpu_top_parse_next_dev(struct gk20a *g, u32 *token) dev->pri_base = engines->info[*token].pri_base; dev->fault_id = engines->info[*token].fault_id; - /* - * vGPU sends us an engine enum; this'll be fixed once we remove - * the engine_info struct. For now just do a quick reverse map. - * - * GRCEs and ASYNC_CEs are both LCEs in terms of engine types. - */ - switch (engines->info[*token].engine_enum) { - case NVGPU_ENGINE_GR: - dev->type = NVGPU_DEVTYPE_GRAPHICS; - break; - case NVGPU_ENGINE_GRCE: - dev->type = NVGPU_DEVTYPE_LCE; - break; - case NVGPU_ENGINE_ASYNC_CE: - dev->type = NVGPU_DEVTYPE_LCE; - break; - default: - nvgpu_err(g, "Unknown engine_enum: %d", - engines->info[*token].engine_enum); - nvgpu_assert(true); - break; - } - (*token)++; return dev; diff --git a/drivers/gpu/nvgpu/hal/fifo/ctxsw_timeout_gv11b_fusa.c b/drivers/gpu/nvgpu/hal/fifo/ctxsw_timeout_gv11b_fusa.c index 797971fd1..5b9f9151f 100644 --- a/drivers/gpu/nvgpu/hal/fifo/ctxsw_timeout_gv11b_fusa.c +++ b/drivers/gpu/nvgpu/hal/fifo/ctxsw_timeout_gv11b_fusa.c @@ -28,6 +28,7 @@ #include #include #include +#include #include @@ -174,7 +175,7 @@ bool gv11b_fifo_handle_ctxsw_timeout(struct gk20a *g) { bool recover = false; u32 tsgid = NVGPU_INVALID_TSG_ID; - u32 engine_id, active_eng_id; + u32 i; u32 timeout_val, ctxsw_timeout_engines; u32 info_status; struct nvgpu_tsg *tsg = NULL; @@ -191,12 +192,12 @@ bool gv11b_fifo_handle_ctxsw_timeout(struct gk20a *g) nvgpu_log_info(g, "eng ctxsw timeout period = 0x%x", timeout_val); - for (engine_id = 0; engine_id < g->fifo.num_engines; engine_id++) { - active_eng_id = g->fifo.active_engines_list[engine_id]; + for (i = 0; i < g->fifo.num_engines; i++) { + const struct nvgpu_device *dev = g->fifo.active_engines[i]; if ((ctxsw_timeout_engines & fifo_intr_ctxsw_timeout_engine_pending_f( - active_eng_id)) != 0U) { + dev->engine_id)) != 0U) { u32 ms = 0; #ifdef CONFIG_NVGPU_KERNEL_MODE_SUBMIT bool debug_dump = false; @@ -207,7 +208,7 @@ bool gv11b_fifo_handle_ctxsw_timeout(struct gk20a *g) "dropped timeout" }; #endif - tsgid = gv11b_fifo_ctxsw_timeout_info(g, active_eng_id, + tsgid = gv11b_fifo_ctxsw_timeout_info(g, dev->engine_id, &info_status); tsg = nvgpu_tsg_check_and_get_from_id(g, tsgid); if (tsg == NULL) { @@ -229,12 +230,13 @@ bool gv11b_fifo_handle_ctxsw_timeout(struct gk20a *g) ctxsw_timeout_status_desc[info_status]; } - nvgpu_err(g, "ctxsw timeout error: " - "active engine id =%u, %s=%d, info: %s ms=%u", - active_eng_id, "tsg", tsgid, info_status_str, - ms); + nvgpu_err(g, + "ctxsw timeout error: engine_id=%u" + "%s=%d, info: %s ms=%u", + dev->engine_id, "tsg", tsgid, + info_status_str, ms); - nvgpu_rc_ctxsw_timeout(g, BIT32(active_eng_id), + nvgpu_rc_ctxsw_timeout(g, BIT32(dev->engine_id), tsg, debug_dump); continue; } diff --git a/drivers/gpu/nvgpu/hal/fifo/engines_gm20b.c b/drivers/gpu/nvgpu/hal/fifo/engines_gm20b.c index b432f64ef..f14f163f9 100644 --- a/drivers/gpu/nvgpu/hal/fifo/engines_gm20b.c +++ b/drivers/gpu/nvgpu/hal/fifo/engines_gm20b.c @@ -39,23 +39,14 @@ int gm20b_engine_init_ce_info(struct nvgpu_fifo *f) { struct gk20a *g = f->g; u32 i; - enum nvgpu_fifo_engine engine_enum; - u32 pbdma_mask = 0U; - u32 gr_runlist_id; bool found; - gr_runlist_id = nvgpu_engine_get_gr_runlist_id(g); - nvgpu_log_info(g, "gr_runlist_id: %d", gr_runlist_id); - for (i = NVGPU_DEVTYPE_COPY0; i <= NVGPU_DEVTYPE_COPY2; i++) { - struct nvgpu_device *dev; - struct nvgpu_engine_info *info; + const struct nvgpu_device *dev; + struct nvgpu_device *dev_rw; - /* - * Cast to a non-const version since we have to hack up a few fields for - * SW to work. - */ - dev = (struct nvgpu_device *)nvgpu_device_get(g, i, 0); + dev = (struct nvgpu_device *)nvgpu_device_get(g, i, + i - NVGPU_DEVTYPE_COPY0); if (dev == NULL) { /* * Not an error condition; gm20b has only 1 CE. @@ -63,51 +54,23 @@ int gm20b_engine_init_ce_info(struct nvgpu_fifo *f) continue; } + /* + * Cast to a non-const version since we have to hack up a few fields for + * SW to work. + */ + dev_rw = (struct nvgpu_device *)dev; + found = g->ops.fifo.find_pbdma_for_runlist(g, dev->runlist_id, - &pbdma_mask); + &dev_rw->pbdma_id); if (!found) { nvgpu_err(g, "busted pbdma map"); return -EINVAL; } - info = &g->fifo.engine_info[dev->engine_id]; - - engine_enum = nvgpu_engine_enum_from_dev(g, dev); - - /* GR and GR_COPY shares same runlist_id */ - if ((engine_enum == NVGPU_ENGINE_ASYNC_CE) && - (gr_runlist_id == dev->runlist_id)) { - engine_enum = NVGPU_ENGINE_GRCE; - } - info->engine_enum = engine_enum; - - if (g->ops.top.get_ce_inst_id != NULL) { - dev->inst_id = g->ops.top.get_ce_inst_id(g, dev->type); - } - - info->fault_id = dev->fault_id; - info->intr_mask |= BIT32(dev->intr_id); - info->reset_mask |= BIT32(dev->reset_id); - info->runlist_id = dev->runlist_id; - info->pbdma_id = nvgpu_safe_sub_u32( - nvgpu_safe_cast_u64_to_u32(nvgpu_ffs(pbdma_mask)), 1U); - info->inst_id = dev->inst_id; - info->pri_base = dev->pri_base; - - /* engine_id starts from 0 to NV_HOST_NUM_ENGINES */ - f->active_engines_list[f->num_engines] = dev->engine_id; + f->host_engines[dev->engine_id] = dev; + f->active_engines[f->num_engines] = dev; ++f->num_engines; - nvgpu_log_info(g, "gr info: engine_id %d runlist_id %d " - "intr_id %d reset_id %d type %d " - "engine_enum %d inst_id %d", - dev->engine_id, - dev->runlist_id, - dev->intr_id, - dev->reset_id, - dev->type, - engine_enum, - dev->inst_id); } return 0; diff --git a/drivers/gpu/nvgpu/hal/fifo/engines_gp10b_fusa.c b/drivers/gpu/nvgpu/hal/fifo/engines_gp10b_fusa.c index 60fd632f7..c38dec035 100644 --- a/drivers/gpu/nvgpu/hal/fifo/engines_gp10b_fusa.c +++ b/drivers/gpu/nvgpu/hal/fifo/engines_gp10b_fusa.c @@ -34,22 +34,12 @@ int gp10b_engine_init_ce_info(struct nvgpu_fifo *f) { struct gk20a *g = f->g; - enum nvgpu_fifo_engine engine_enum; u32 i; - u32 gr_runlist_id; - u32 lce_num_entries = 0; bool found; - gr_runlist_id = nvgpu_engine_get_gr_runlist_id(g); - nvgpu_log_info(g, "gr_runlist_id: %d", gr_runlist_id); - - lce_num_entries = nvgpu_device_count(g, NVGPU_DEVTYPE_LCE); - nvgpu_log_info(g, "lce_num_entries: %d", lce_num_entries); - - for (i = 0; i < lce_num_entries; i++) { + for (i = 0; i < nvgpu_device_count(g, NVGPU_DEVTYPE_LCE); i++) { const struct nvgpu_device *dev; struct nvgpu_device *dev_rw; - struct nvgpu_engine_info *info; dev = nvgpu_device_get(g, NVGPU_DEVTYPE_LCE, i); if (dev == NULL) { @@ -58,8 +48,6 @@ int gp10b_engine_init_ce_info(struct nvgpu_fifo *f) } dev_rw = (struct nvgpu_device *)dev; - info = &g->fifo.engine_info[dev->engine_id]; - /* * vGPU consideration. Not present in older chips. See * nvgpu_engine_init_from_device_info() for more details in the @@ -74,37 +62,20 @@ int gp10b_engine_init_ce_info(struct nvgpu_fifo *f) return -EINVAL; } } - info->pbdma_id = dev->pbdma_id; - engine_enum = nvgpu_engine_enum_from_dev(g, dev); - /* GR and GR_COPY shares same runlist_id */ - if ((engine_enum == NVGPU_ENGINE_ASYNC_CE) && - (gr_runlist_id == dev->runlist_id)) { - engine_enum = NVGPU_ENGINE_GRCE; +#if defined(CONFIG_NVGPU_NEXT) + { + int err = nvgpu_next_engine_init_one_dev(g, dev); + if (err != 0) { + return err; + } } - info->engine_enum = engine_enum; +#endif - info->fault_id = dev->fault_id; - info->intr_mask |= BIT32(dev->intr_id); - info->reset_mask |= BIT32(dev->reset_id); - info->runlist_id = dev->runlist_id; - info->inst_id = dev->inst_id; - info->pri_base = dev->pri_base; - info->engine_id = dev->engine_id; - - /* engine_id starts from 0 to NV_HOST_NUM_ENGINES */ - f->active_engines_list[f->num_engines] = dev->engine_id; + f->host_engines[dev->engine_id] = dev; + f->active_engines[f->num_engines] = dev; f->num_engines = nvgpu_safe_add_u32(f->num_engines, 1U); - nvgpu_log_info(g, "gr info: engine_id %d runlist_id %d " - "intr_id %d reset_id %d engine_type %d " - "engine_enum %d inst_id %d", - dev->engine_id, - dev->runlist_id, - dev->intr_id, - dev->reset_id, - dev->type, - engine_enum, - dev->inst_id); } + return 0; } diff --git a/drivers/gpu/nvgpu/hal/fifo/mmu_fault_gm20b.c b/drivers/gpu/nvgpu/hal/fifo/mmu_fault_gm20b.c index 96956870e..a9d97dde8 100644 --- a/drivers/gpu/nvgpu/hal/fifo/mmu_fault_gm20b.c +++ b/drivers/gpu/nvgpu/hal/fifo/mmu_fault_gm20b.c @@ -26,6 +26,7 @@ #include #include #include +#include #include @@ -66,18 +67,18 @@ void gm20b_fifo_get_mmu_fault_gpc_desc(struct mmu_fault_info *mmufault) static inline u32 gm20b_engine_id_to_fault_id(struct gk20a *g, u32 engine_id) { - u32 fault_id = INVAL_ID; - struct nvgpu_engine_info *engine_info; + const struct nvgpu_device *dev; - engine_info = nvgpu_engine_get_active_eng_info(g, engine_id); + dev = nvgpu_engine_get_active_eng_info(g, engine_id); - if (engine_info != NULL) { - fault_id = engine_info->fault_id; - } else { - nvgpu_err(g, "engine_id is not in active list/invalid %d", - engine_id); + if (dev == NULL) { + nvgpu_err(g, + "engine_id is not in active list/invalid %d", + engine_id); + return INVAL_ID; } - return fault_id; + + return dev->fault_id; } void gm20b_fifo_trigger_mmu_fault(struct gk20a *g, diff --git a/drivers/gpu/nvgpu/hal/fifo/runlist_fifo_gk20a.c b/drivers/gpu/nvgpu/hal/fifo/runlist_fifo_gk20a.c index 3e87c7a05..8a7406efb 100644 --- a/drivers/gpu/nvgpu/hal/fifo/runlist_fifo_gk20a.c +++ b/drivers/gpu/nvgpu/hal/fifo/runlist_fifo_gk20a.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include "runlist_fifo_gk20a.h" @@ -54,17 +55,19 @@ int gk20a_fifo_reschedule_preempt_next(struct nvgpu_channel *ch, struct nvgpu_runlist_info *runlist = g->fifo.runlist_info[ch->runlist_id]; int ret = 0; - u32 gr_eng_id = 0; u32 fecsstat0 = 0, fecsstat1 = 0; u32 preempt_id; u32 preempt_type = 0; + const struct nvgpu_device *dev; struct nvgpu_engine_status_info engine_status; - if (nvgpu_engine_get_ids( - g, &gr_eng_id, 1, NVGPU_ENGINE_GR) != 1U) { - return ret; + dev = nvgpu_device_get(g, NVGPU_DEVTYPE_GRAPHICS, 0); + if (dev == NULL) { + nvgpu_warn(g, "GPU has no GR engine?!"); + return -EINVAL; } - if ((runlist->eng_bitmask & BIT32(gr_eng_id)) == 0U) { + + if ((runlist->eng_bitmask & BIT32(dev->engine_id)) == 0U) { return ret; } @@ -78,8 +81,8 @@ int gk20a_fifo_reschedule_preempt_next(struct nvgpu_channel *ch, fecsstat0 = g->ops.gr.falcon.read_fecs_ctxsw_mailbox(g, NVGPU_GR_FALCON_FECS_CTXSW_MAILBOX0); - g->ops.engine_status.read_engine_status_info(g, gr_eng_id, - &engine_status); + g->ops.engine_status.read_engine_status_info(g, dev->engine_id, + &engine_status); if (nvgpu_engine_status_is_ctxsw_switch(&engine_status)) { nvgpu_engine_status_get_next_ctx_id_type(&engine_status, &preempt_id, &preempt_type); diff --git a/drivers/gpu/nvgpu/hal/init/hal_gm20b.c b/drivers/gpu/nvgpu/hal/init/hal_gm20b.c index f871e424f..00deb1db3 100644 --- a/drivers/gpu/nvgpu/hal/init/hal_gm20b.c +++ b/drivers/gpu/nvgpu/hal/init/hal_gm20b.c @@ -1114,7 +1114,6 @@ static const struct gpu_ops gm20b_ops = { .parse_next_device = gm20b_top_parse_next_dev, .device_info_parse_enum = gm20b_device_info_parse_enum, .device_info_parse_data = gm20b_device_info_parse_data, - .get_ce_inst_id = gm20b_get_ce_inst_id, .get_max_gpc_count = gm20b_top_get_max_gpc_count, .get_max_tpc_per_gpc_count = gm20b_top_get_max_tpc_per_gpc_count, diff --git a/drivers/gpu/nvgpu/hal/init/hal_gp10b.c b/drivers/gpu/nvgpu/hal/init/hal_gp10b.c index 21b63d2fb..69aacd7f1 100644 --- a/drivers/gpu/nvgpu/hal/init/hal_gp10b.c +++ b/drivers/gpu/nvgpu/hal/init/hal_gp10b.c @@ -1227,7 +1227,6 @@ static const struct gpu_ops gp10b_ops = { .parse_next_device = gm20b_top_parse_next_dev, .device_info_parse_enum = gm20b_device_info_parse_enum, .device_info_parse_data = gp10b_device_info_parse_data, - .get_ce_inst_id = NULL, .get_max_gpc_count = gm20b_top_get_max_gpc_count, .get_max_tpc_per_gpc_count = gm20b_top_get_max_tpc_per_gpc_count, diff --git a/drivers/gpu/nvgpu/hal/init/hal_gv11b.c b/drivers/gpu/nvgpu/hal/init/hal_gv11b.c index 675afffc8..6c2bfbcb8 100644 --- a/drivers/gpu/nvgpu/hal/init/hal_gv11b.c +++ b/drivers/gpu/nvgpu/hal/init/hal_gv11b.c @@ -1487,7 +1487,6 @@ NVGPU_COV_WHITELIST_BLOCK_END(NVGPU_MISRA(Rule, 8_7)) .parse_next_device = gm20b_top_parse_next_dev, .device_info_parse_enum = gm20b_device_info_parse_enum, .device_info_parse_data = gv11b_device_info_parse_data, - .get_ce_inst_id = NULL, .get_max_gpc_count = gm20b_top_get_max_gpc_count, .get_max_tpc_per_gpc_count = gm20b_top_get_max_tpc_per_gpc_count, diff --git a/drivers/gpu/nvgpu/hal/init/hal_tu104.c b/drivers/gpu/nvgpu/hal/init/hal_tu104.c index 1a7ffbbd8..a879e32e8 100644 --- a/drivers/gpu/nvgpu/hal/init/hal_tu104.c +++ b/drivers/gpu/nvgpu/hal/init/hal_tu104.c @@ -1624,7 +1624,6 @@ static const struct gpu_ops tu104_ops = { .parse_next_device = gm20b_top_parse_next_dev, .device_info_parse_enum = gm20b_device_info_parse_enum, .device_info_parse_data = gv11b_device_info_parse_data, - .get_ce_inst_id = NULL, .get_max_gpc_count = gm20b_top_get_max_gpc_count, .get_max_tpc_per_gpc_count = gm20b_top_get_max_tpc_per_gpc_count, diff --git a/drivers/gpu/nvgpu/hal/mc/mc_gm20b.c b/drivers/gpu/nvgpu/hal/mc/mc_gm20b.c index 27022cac3..3b4d097a1 100644 --- a/drivers/gpu/nvgpu/hal/mc/mc_gm20b.c +++ b/drivers/gpu/nvgpu/hal/mc/mc_gm20b.c @@ -30,6 +30,7 @@ #include #include #include +#include #include #include @@ -41,36 +42,32 @@ void gm20b_mc_isr_stall(struct gk20a *g) { u32 mc_intr_0; u32 i; - u32 engine_id = 0U; - enum nvgpu_fifo_engine engine_enum; + const struct nvgpu_device *dev; mc_intr_0 = g->ops.mc.intr_stall(g); nvgpu_log(g, gpu_dbg_intr, "stall intr %08x", mc_intr_0); for (i = 0U; i < g->fifo.num_engines; i++) { - engine_id = g->fifo.active_engines_list[i]; + dev = g->fifo.active_engines[i]; - if ((mc_intr_0 & - g->fifo.engine_info[engine_id].intr_mask) == 0U) { + if ((mc_intr_0 & BIT32(dev->intr_id)) == 0U) { continue; } - engine_enum = g->fifo.engine_info[engine_id].engine_enum; + /* GR Engine */ - if (engine_enum == NVGPU_ENGINE_GR) { + if (nvgpu_device_is_graphics(g, dev)) { nvgpu_pg_elpg_protected_call(g, g->ops.gr.intr.stall_isr(g)); } /* CE Engine */ - if (((engine_enum == NVGPU_ENGINE_GRCE) || - (engine_enum == NVGPU_ENGINE_ASYNC_CE)) && - (g->ops.ce.isr_stall != NULL)) { - g->ops.ce.isr_stall(g, - g->fifo.engine_info[engine_id].inst_id, - g->fifo.engine_info[engine_id].pri_base); + if (nvgpu_device_is_ce(g, dev) && + (g->ops.ce.isr_stall != NULL)) { + g->ops.ce.isr_stall(g, dev->inst_id, dev->pri_base); } } + if ((mc_intr_0 & mc_intr_pfifo_pending_f()) != 0U) { g->ops.fifo.intr_0_isr(g); } @@ -296,4 +293,3 @@ bool gm20b_mc_is_mmu_fault_pending(struct gk20a *g) { return g->ops.fifo.is_mmu_fault_pending(g); } - diff --git a/drivers/gpu/nvgpu/hal/mc/mc_gm20b_fusa.c b/drivers/gpu/nvgpu/hal/mc/mc_gm20b_fusa.c index 1ae292d91..a8eea958e 100644 --- a/drivers/gpu/nvgpu/hal/mc/mc_gm20b_fusa.c +++ b/drivers/gpu/nvgpu/hal/mc/mc_gm20b_fusa.c @@ -26,6 +26,7 @@ #include #include #include +#include #include "mc_gm20b.h" @@ -60,8 +61,6 @@ u32 gm20b_mc_isr_nonstall(struct gk20a *g) u32 ops = 0U; u32 mc_intr_1; u32 i; - u32 engine_id = 0U; - enum nvgpu_fifo_engine engine_enum; mc_intr_1 = g->ops.mc.intr_nonstall(g); @@ -74,25 +73,23 @@ u32 gm20b_mc_isr_nonstall(struct gk20a *g) } for (i = 0U; i < g->fifo.num_engines; i++) { - struct nvgpu_engine_info *engine_info; + const struct nvgpu_device *dev = g->fifo.active_engines[i]; - engine_id = g->fifo.active_engines_list[i]; - engine_info = &g->fifo.engine_info[engine_id]; + if ((mc_intr_1 & BIT32(dev->intr_id)) == 0U) { + continue; + } - if ((mc_intr_1 & engine_info->intr_mask) != 0U) { - engine_enum = engine_info->engine_enum; - /* GR Engine */ - if (engine_enum == NVGPU_ENGINE_GR) { - ops |= g->ops.gr.intr.nonstall_isr(g); - } - /* CE Engine */ - if (((engine_enum == NVGPU_ENGINE_GRCE) || - (engine_enum == NVGPU_ENGINE_ASYNC_CE)) && - (g->ops.ce.isr_nonstall != NULL)) { - ops |= g->ops.ce.isr_nonstall(g, - engine_info->inst_id, - engine_info->pri_base); - } + /* GR Engine */ + if (nvgpu_device_is_graphics(g, dev)) { + ops |= g->ops.gr.intr.nonstall_isr(g); + } + + /* CE Engine */ + if (nvgpu_device_is_ce(g, dev) && + (g->ops.ce.isr_nonstall != NULL)) { + ops |= g->ops.ce.isr_nonstall(g, + dev->inst_id, + dev->pri_base); } } diff --git a/drivers/gpu/nvgpu/hal/mc/mc_gp10b.h b/drivers/gpu/nvgpu/hal/mc/mc_gp10b.h index d9258c458..80da0e540 100644 --- a/drivers/gpu/nvgpu/hal/mc/mc_gp10b.h +++ b/drivers/gpu/nvgpu/hal/mc/mc_gp10b.h @@ -30,6 +30,7 @@ #endif struct gk20a; +struct nvgpu_device; enum nvgpu_unit; enum nvgpu_fifo_engine; @@ -38,8 +39,7 @@ void mc_gp10b_intr_stall_unit_config(struct gk20a *g, u32 unit, bool enable); void mc_gp10b_intr_nonstall_unit_config(struct gk20a *g, u32 unit, bool enable); void mc_gp10b_isr_stall_secondary_1(struct gk20a *g, u32 mc_intr_0); void mc_gp10b_isr_stall_secondary_0(struct gk20a *g, u32 mc_intr_0); -void mc_gp10b_isr_stall_engine(struct gk20a *g, - enum nvgpu_fifo_engine engine_enum, u32 engine_id); +void mc_gp10b_isr_stall_engine(struct gk20a *g, const struct nvgpu_device *dev); void mc_gp10b_isr_stall(struct gk20a *g); bool mc_gp10b_is_intr1_pending(struct gk20a *g, enum nvgpu_unit unit, u32 mc_intr_1); diff --git a/drivers/gpu/nvgpu/hal/mc/mc_gp10b_fusa.c b/drivers/gpu/nvgpu/hal/mc/mc_gp10b_fusa.c index ebd983129..22c63fc98 100644 --- a/drivers/gpu/nvgpu/hal/mc/mc_gp10b_fusa.c +++ b/drivers/gpu/nvgpu/hal/mc/mc_gp10b_fusa.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include "mc_gp10b.h" @@ -119,24 +120,23 @@ void mc_gp10b_isr_stall_secondary_0(struct gk20a *g, u32 mc_intr_0) } void mc_gp10b_isr_stall_engine(struct gk20a *g, - enum nvgpu_fifo_engine engine_enum, u32 engine_id) + const struct nvgpu_device *dev) { + int err; + /* GR Engine */ - if (engine_enum == NVGPU_ENGINE_GR) { - int ret_err = nvgpu_pg_elpg_protected_call(g, + if (nvgpu_device_is_graphics(g, dev)) { + err = nvgpu_pg_elpg_protected_call(g, g->ops.gr.intr.stall_isr(g)); - if (ret_err != 0) { + if (err != 0) { nvgpu_err(g, "Unable to handle gr interrupt"); } } /* CE Engine */ - if (((engine_enum == NVGPU_ENGINE_GRCE) || - (engine_enum == NVGPU_ENGINE_ASYNC_CE)) && - (g->ops.ce.isr_stall != NULL)) { - g->ops.ce.isr_stall(g, - g->fifo.engine_info[engine_id].inst_id, - g->fifo.engine_info[engine_id].pri_base); + if (nvgpu_device_is_ce(g, dev) && + (g->ops.ce.isr_stall != NULL)) { + g->ops.ce.isr_stall(g, dev->inst_id, dev->pri_base); } } @@ -180,8 +180,7 @@ void mc_gp10b_isr_stall(struct gk20a *g) { u32 mc_intr_0; u32 i; - u32 engine_id = 0U; - enum nvgpu_fifo_engine engine_enum; + const struct nvgpu_device *dev; mc_intr_0 = nvgpu_readl(g, mc_intr_r(NVGPU_MC_INTR_STALLING)); @@ -190,14 +189,14 @@ void mc_gp10b_isr_stall(struct gk20a *g) mc_gp10b_isr_stall_primary(g, mc_intr_0); for (i = 0U; i < g->fifo.num_engines; i++) { - engine_id = g->fifo.active_engines_list[i]; + dev = g->fifo.active_engines[i]; - if ((mc_intr_0 & - g->fifo.engine_info[engine_id].intr_mask) == 0U) { + if ((mc_intr_0 & BIT32(dev->intr_id)) == 0U) { continue; } - engine_enum = g->fifo.engine_info[engine_id].engine_enum; - mc_gp10b_isr_stall_engine(g, engine_enum, engine_id); + + + mc_gp10b_isr_stall_engine(g, dev); } mc_gp10b_isr_stall_secondary_0(g, mc_intr_0); diff --git a/drivers/gpu/nvgpu/hal/mc/mc_tu104.c b/drivers/gpu/nvgpu/hal/mc/mc_tu104.c index 64f1b19ce..3379b1a6d 100644 --- a/drivers/gpu/nvgpu/hal/mc/mc_tu104.c +++ b/drivers/gpu/nvgpu/hal/mc/mc_tu104.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include @@ -139,7 +140,7 @@ static void intr_tu104_nonstall_enable(struct gk20a *g) u32 i; u32 nonstall_intr_base = 0; u64 nonstall_intr_mask = 0; - u32 engine_id, intr_mask; + u32 intr_mask; /* Keep NV_PMC_INTR(1) disabled */ nvgpu_writel(g, mc_intr_en_clear_r(NVGPU_MC_INTR_NONSTALLING), U32_MAX); @@ -156,8 +157,9 @@ static void intr_tu104_nonstall_enable(struct gk20a *g) ctrl_legacy_engine_nonstall_intr_base_vectorid_r()); for (i = 0; i < g->fifo.num_engines; i++) { - engine_id = g->fifo.active_engines_list[i]; - intr_mask = g->fifo.engine_info[engine_id].intr_mask; + const struct nvgpu_device *dev = g->fifo.active_engines[i]; + + intr_mask = BIT32(dev->intr_id); nonstall_intr_mask |= U64(intr_mask) << U64(nonstall_intr_base); } @@ -309,7 +311,7 @@ u32 intr_tu104_isr_nonstall(struct gk20a *g) u64 nonstall_intr_mask = 0U; u32 nonstall_intr_mask_lo, nonstall_intr_mask_hi; u32 intr_leaf_reg0, intr_leaf_reg1; - u32 engine_id, intr_mask; + u32 intr_mask; u32 ops = 0U; intr_leaf_reg0 = nvgpu_func_readl(g, @@ -326,8 +328,9 @@ u32 intr_tu104_isr_nonstall(struct gk20a *g) ctrl_legacy_engine_nonstall_intr_base_vectorid_r()); for (i = 0U; i < g->fifo.num_engines; i++) { - engine_id = g->fifo.active_engines_list[i]; - intr_mask = g->fifo.engine_info[engine_id].intr_mask; + const struct nvgpu_device *dev = g->fifo.active_engines[i]; + + intr_mask = BIT32(dev->intr_id); nonstall_intr_mask = U64(intr_mask) << U64(nonstall_intr_base); nonstall_intr_mask_lo = u64_lo32(nonstall_intr_mask); @@ -337,7 +340,7 @@ u32 intr_tu104_isr_nonstall(struct gk20a *g) (nonstall_intr_mask_hi & intr_leaf_reg1) != 0U) { nvgpu_log(g, gpu_dbg_intr, "nonstall intr from engine %d", - engine_id); + dev->engine_id); nvgpu_func_writel(g, func_priv_cpu_intr_leaf_r( @@ -482,8 +485,7 @@ void mc_tu104_isr_stall(struct gk20a *g) { u32 mc_intr_0; u32 i; - u32 engine_id = 0U; - enum nvgpu_fifo_engine engine_enum; + const struct nvgpu_device *dev; mc_intr_0 = nvgpu_readl(g, mc_intr_r(NVGPU_MC_INTR_STALLING)); @@ -492,14 +494,13 @@ void mc_tu104_isr_stall(struct gk20a *g) mc_tu104_isr_stall_primary(g, mc_intr_0); for (i = 0U; i < g->fifo.num_engines; i++) { - engine_id = g->fifo.active_engines_list[i]; + dev = g->fifo.active_engines[i]; - if ((mc_intr_0 & - g->fifo.engine_info[engine_id].intr_mask) == 0U) { + if ((mc_intr_0 & BIT32(dev->intr_id)) == 0U) { continue; } - engine_enum = g->fifo.engine_info[engine_id].engine_enum; - mc_gp10b_isr_stall_engine(g, engine_enum, engine_id); + + mc_gp10b_isr_stall_engine(g, dev); } mc_gp10b_isr_stall_secondary_0(g, mc_intr_0); diff --git a/drivers/gpu/nvgpu/hal/rc/rc_gk20a.c b/drivers/gpu/nvgpu/hal/rc/rc_gk20a.c index cca661460..f53d6f0e5 100644 --- a/drivers/gpu/nvgpu/hal/rc/rc_gk20a.c +++ b/drivers/gpu/nvgpu/hal/rc/rc_gk20a.c @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -93,7 +94,9 @@ void gk20a_fifo_recover(struct gk20a *g, u32 eng_bitmask, * same channel as faulty engine */ for (i = 0; i < g->fifo.num_engines; i++) { - u32 active_engine_id = g->fifo.active_engines_list[i]; + const struct nvgpu_device *dev = + g->fifo.active_engines[i]; + u32 active_engine_id = dev->engine_id; u32 type; u32 id; diff --git a/drivers/gpu/nvgpu/hal/therm/therm_gm20b.c b/drivers/gpu/nvgpu/hal/therm/therm_gm20b.c index 7c5f2e303..e2da26038 100644 --- a/drivers/gpu/nvgpu/hal/therm/therm_gm20b.c +++ b/drivers/gpu/nvgpu/hal/therm/therm_gm20b.c @@ -26,8 +26,10 @@ #include #include #include -#include #include +#include + +#include #include "therm_gm20b.h" @@ -84,16 +86,15 @@ int gm20b_init_therm_setup_hw(struct gk20a *g) int gm20b_elcg_init_idle_filters(struct gk20a *g) { - u32 gate_ctrl, idle_filter; - u32 engine_id; - u32 active_engine_id = 0; + u32 gate_ctrl, idle_filter, i; + const struct nvgpu_device *dev; struct nvgpu_fifo *f = &g->fifo; nvgpu_log_fn(g, " "); - for (engine_id = 0; engine_id < f->num_engines; engine_id++) { - active_engine_id = f->active_engines_list[engine_id]; - gate_ctrl = nvgpu_readl(g, therm_gate_ctrl_r(active_engine_id)); + for (i = 0; i < f->num_engines; i++) { + dev = f->active_engines[i]; + gate_ctrl = nvgpu_readl(g, therm_gate_ctrl_r(dev->engine_id)); #ifdef CONFIG_NVGPU_SIM if (nvgpu_is_enabled(g, NVGPU_IS_FMODEL)) { @@ -110,7 +111,7 @@ int gm20b_elcg_init_idle_filters(struct gk20a *g) gate_ctrl = set_field(gate_ctrl, therm_gate_ctrl_eng_idle_filt_mant_m(), therm_gate_ctrl_eng_idle_filt_mant_f(2)); - nvgpu_writel(g, therm_gate_ctrl_r(active_engine_id), gate_ctrl); + nvgpu_writel(g, therm_gate_ctrl_r(dev->engine_id), gate_ctrl); } /* default fecs_idle_filter to 0 */ @@ -227,4 +228,4 @@ void gm20b_therm_init_blcg_mode(struct gk20a *g, u32 mode, u32 engine) } nvgpu_writel(g, therm_gate_ctrl_r(engine), gate_ctrl); -} \ No newline at end of file +} diff --git a/drivers/gpu/nvgpu/hal/therm/therm_gp10b.c b/drivers/gpu/nvgpu/hal/therm/therm_gp10b.c index f7483db9e..e622aa694 100644 --- a/drivers/gpu/nvgpu/hal/therm/therm_gp10b.c +++ b/drivers/gpu/nvgpu/hal/therm/therm_gp10b.c @@ -25,6 +25,7 @@ #include #include #include +#include #include "therm_gp10b.h" @@ -92,16 +93,14 @@ int gp10b_init_therm_setup_hw(struct gk20a *g) int gp10b_elcg_init_idle_filters(struct gk20a *g) { - u32 gate_ctrl, idle_filter; - u32 engine_id; - u32 active_engine_id = 0; + u32 gate_ctrl = 0, idle_filter; + u32 i; struct nvgpu_fifo *f = &g->fifo; nvgpu_log_fn(g, " "); - for (engine_id = 0; engine_id < f->num_engines; engine_id++) { - active_engine_id = f->active_engines_list[engine_id]; - gate_ctrl = nvgpu_readl(g, therm_gate_ctrl_r(active_engine_id)); + for (i = 0; i < f->num_engines; i++) { + const struct nvgpu_device *dev = f->active_engines[i]; if (nvgpu_platform_is_simulation(g)) { gate_ctrl = set_field(gate_ctrl, @@ -119,7 +118,7 @@ int gp10b_elcg_init_idle_filters(struct gk20a *g) gate_ctrl = set_field(gate_ctrl, therm_gate_ctrl_eng_delay_before_m(), therm_gate_ctrl_eng_delay_before_f(4)); - nvgpu_writel(g, therm_gate_ctrl_r(active_engine_id), gate_ctrl); + nvgpu_writel(g, therm_gate_ctrl_r(dev->engine_id), gate_ctrl); } /* default fecs_idle_filter to 0 */ diff --git a/drivers/gpu/nvgpu/hal/therm/therm_gv11b_fusa.c b/drivers/gpu/nvgpu/hal/therm/therm_gv11b_fusa.c index 7c3001a46..5fcd9d8e0 100644 --- a/drivers/gpu/nvgpu/hal/therm/therm_gv11b_fusa.c +++ b/drivers/gpu/nvgpu/hal/therm/therm_gv11b_fusa.c @@ -26,8 +26,10 @@ #include #include #include -#include #include +#include + +#include #include "therm_gv11b.h" @@ -142,7 +144,7 @@ int gv11b_elcg_init_idle_filters(struct gk20a *g) { u32 gate_ctrl, idle_filter; u32 i; - u32 engine_id = 0; + const struct nvgpu_device *dev; struct nvgpu_fifo *f = &g->fifo; if (nvgpu_platform_is_simulation(g)) { @@ -152,9 +154,9 @@ int gv11b_elcg_init_idle_filters(struct gk20a *g) nvgpu_log_info(g, "init clock/power gate reg"); for (i = 0; i < f->num_engines; i++) { - engine_id = f->active_engines_list[i]; + dev = f->active_engines[i]; - gate_ctrl = nvgpu_readl(g, therm_gate_ctrl_r(engine_id)); + gate_ctrl = nvgpu_readl(g, therm_gate_ctrl_r(dev->engine_id)); gate_ctrl = set_field(gate_ctrl, therm_gate_ctrl_eng_idle_filt_exp_m(), therm_gate_ctrl_eng_idle_filt_exp__prod_f()); @@ -167,7 +169,7 @@ int gv11b_elcg_init_idle_filters(struct gk20a *g) gate_ctrl = set_field(gate_ctrl, therm_gate_ctrl_eng_delay_after_m(), therm_gate_ctrl_eng_delay_after__prod_f()); - nvgpu_writel(g, therm_gate_ctrl_r(engine_id), gate_ctrl); + nvgpu_writel(g, therm_gate_ctrl_r(dev->engine_id), gate_ctrl); } idle_filter = nvgpu_readl(g, therm_fecs_idle_filter_r()); diff --git a/drivers/gpu/nvgpu/hal/top/top_gm20b.c b/drivers/gpu/nvgpu/hal/top/top_gm20b.c index 7eb08ba80..91054c88e 100644 --- a/drivers/gpu/nvgpu/hal/top/top_gm20b.c +++ b/drivers/gpu/nvgpu/hal/top/top_gm20b.c @@ -67,9 +67,3 @@ int gm20b_device_info_parse_data(struct gk20a *g, u32 table_entry, u32 *inst_id, return 0; } - -u32 gm20b_get_ce_inst_id(struct gk20a *g, u32 engine_type) -{ - /* inst_id starts from CE0 to CE2 */ - return (engine_type - NVGPU_DEVTYPE_COPY0); -} diff --git a/drivers/gpu/nvgpu/hal/top/top_gm20b_fusa.c b/drivers/gpu/nvgpu/hal/top/top_gm20b_fusa.c index 648986e28..17bacb9da 100644 --- a/drivers/gpu/nvgpu/hal/top/top_gm20b_fusa.c +++ b/drivers/gpu/nvgpu/hal/top/top_gm20b_fusa.c @@ -85,7 +85,6 @@ struct nvgpu_device *gm20b_top_parse_next_dev(struct gk20a *g, u32 *token) struct nvgpu_device *dev; while (true) { - /* * The core code relies on us to manage the index - a.k.a the * token. If token crosses the device table size, then break and @@ -162,6 +161,24 @@ struct nvgpu_device *gm20b_top_parse_next_dev(struct gk20a *g, u32 *token) return NULL; } + /* + * SW hack: override the HW inst_id field for COPY1 and 2. + * Although each CE on gm20b is considered its own device type + * that's not really very sensible. HW fixes this in future + * chips, but for now, set the inst_id field to a more intuitive + * value. + * + * It's possible this can be fixed in the future such that nvgpu + * does not rely on this; it'd make a lot of code less arbitrary. + */ + if (dev->type == NVGPU_DEVTYPE_COPY1) { + dev->inst_id = 1U; + } + if (dev->type == NVGPU_DEVTYPE_COPY2) { + dev->inst_id = 2U; + } + + break; } diff --git a/drivers/gpu/nvgpu/include/nvgpu/engines.h b/drivers/gpu/nvgpu/include/nvgpu/engines.h index 532d5ee82..4e3a40efd 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/engines.h +++ b/drivers/gpu/nvgpu/include/nvgpu/engines.h @@ -61,55 +61,6 @@ enum nvgpu_fifo_engine { NVGPU_ENGINE_INVAL = 3U, }; -struct nvgpu_engine_info { - /** - * Contains valid engine id read from device info h/w register or - * invalid engine id, U32_MAX. - */ - u32 engine_id; - /** - * Contains valid runlist id read from device info h/w register or - * invalid runlist id, U32_MAX. - */ - u32 runlist_id; - /** - * Contains bit mask for the intr id read from device info h/w register - * or invalid intr id mask, U32_MAX. This is used to check pending - * interrupt for the engine_id. - */ - u32 intr_mask; - /** - * Contains bit mask for the reset id read from device info h/w register - * or invalid reset id mask, U32_MAX. - */ - u32 reset_mask; - /** Pbdma id servicing #runlist_id. */ - u32 pbdma_id; - /** - * Specifies instance of a device, allowing s/w to distinguish between - * multiple copies of a device present on the chip. - */ - u32 inst_id; - /** - * Used to determine the start of the h/w register address space - * for #inst_id 0. - */ - u32 pri_base; - /** - * Contains valid mmu fault id read from device info h/w register or - * invalid mmu fault id, U32_MAX. - */ - u32 fault_id; - /** Engine enum type used for s/w purpose. */ - enum nvgpu_fifo_engine engine_enum; - - /** @cond DOXYGEN_SHOULD_SKIP_THIS */ -#if defined(CONFIG_NVGPU_NON_FUSA) && defined(CONFIG_NVGPU_NEXT) - /* nvgpu next engine info additions */ - struct nvgpu_next_engine_info nvgpu_next; -#endif - /** @endcond DOXYGEN_SHOULD_SKIP_THIS */ -}; /** * @brief Get s/w defined engine enum type for engine enum type defined by h/w. * See device.h for engine enum types defined by h/w. @@ -129,7 +80,7 @@ struct nvgpu_engine_info { enum nvgpu_fifo_engine nvgpu_engine_enum_from_dev(struct gk20a *g, const struct nvgpu_device *dev); /** - * @brief Get pointer to #nvgpu_engine_info for the h/w engine id. + * @brief Get pointer to #nvgpu_device for the h/w engine id. * * @param g [in] The GPU driver struct. * @param engine_id [in] Active (h/w) Engine id. @@ -138,39 +89,16 @@ enum nvgpu_fifo_engine nvgpu_engine_enum_from_dev(struct gk20a *g, * #nvgpu_engine_info from an array of structures that is indexed by h/w * engine id. * - * @return Pointer to #nvgpu_engine_info. + * @return Pointer to #nvgpu_device. * @retval NULL if #g is NULL. * @retval NULL if engine_id is not less than max supported number of engines * i.e. #nvgpu_fifo.max_engines or if #engine_id does not match with * any engine id supported by h/w or number of available engines * i.e. #nvgpu_fifo.num_engines is 0. */ -struct nvgpu_engine_info *nvgpu_engine_get_active_eng_info( +const struct nvgpu_device *nvgpu_engine_get_active_eng_info( struct gk20a *g, u32 engine_id); -/** - * @brief Get instance count and h/w engine id/s for s/w defined engine - * enum type. See #nvgpu_fifo_engine for s/w defined engine enum types. - * - * @param g [in] The GPU driver struct. - * @param engine_ids [in,out] Pointer to memory area to store h/w engine ids. - * @param engine_id_sz [in] Number of h/w engine ids to be stored in - * memory area pointed by #engine_ids. - * @param engine_enum [in] Engine enum types defined by #nvgpu_fifo_engine. - * - * - Check validity of input parameters. - * - Get #nvgpu_engine_info for each of #nvgpu_fifo.num_engines. - * - Increase instance count and store h/w engine id in #engine_ids if - * #nvgpu_engine_info.engine_enum matches with #engine_enum until - * instance count is less than #engine_id_sz. - * - * @return Instance count. - * @retval 0 if #g is NULL or #engine_id_sz is 0 or #engine_enum is - * #NVGPU_ENGINE_INVAL - * @retval 0 if #nvgpu_fifo.num_engines is 0. - */ -u32 nvgpu_engine_get_ids(struct gk20a *g, - u32 *engine_ids, u32 engine_id_sz, - enum nvgpu_fifo_engine engine_enum); + /** * @brief Check if engine id is one of the supported h/w engine ids. * @@ -204,16 +132,9 @@ u32 nvgpu_engine_get_gr_id(struct gk20a *g); * * @param g[in] The GPU driver struct. * - * For each of #nvgpu_fifo.num_engines, get pointer to - * #nvgpu_engine_info. Use this to get #nvgpu_engine_info.intr_mask. - * If #nvgpu_engine_info.engine_num type matches with - * #NVGPU_ENGINE_GR, local intr_mask variable is logically ORed with - * #nvgpu_engine_info.intr_mask. + * Return bitmask of each GR engine's interrupt bit. * * @return Interrupt mask for GR engine. - * @retval 0 if #nvgpu_fifo.num_engines is 0. - * @retval 0 if all of the supported engine enum types don't match with - * #NVGPU_ENGINE_GR. */ u32 nvgpu_gr_engine_interrupt_mask(struct gk20a *g); /** @@ -221,34 +142,23 @@ u32 nvgpu_gr_engine_interrupt_mask(struct gk20a *g); * * @param g [in] The GPU driver struct. * - * For each of #nvgpu_fifo.num_engines, get pointer to - * #nvgpu_engine_info. Use this to get #nvgpu_engine_info.intr_mask. - * If #nvgpu_engine_info.engine_num type matches with - * #NVGPU_ENGINE_GRCE or #NVGPU_ENGINE_ASYNC_CE but interrupt handlers - * are not supported or engine_enum type matches with #NVGPU_ENGINE_GR - * , local intr_mask variable is not logically ORed with - * #nvgpu_engine_info.intr_mask. + * Query all types of copy engine devices and OR their interrupt bits into + * a CE interrupt mask. * - * @return Interrupt mask for CE engines that support interrupt handlers. - * @retval 0 if #nvgpu_fifo.num_engines is 0. - * @retval 0 if all of the supported engine enum types match with - * #NVGPU_ENGINE_GRCE or #NVGPU_ENGINE_ASYNC_CE and does not - * support interrupt handler or engine enum type matches with - * #NVGPU_ENGINE_GR. + * @return 0U if there is no CE support in the system. + * @return The logical OR of all interrupt bits for all CE devices present. */ u32 nvgpu_ce_engine_interrupt_mask(struct gk20a *g); /** - * @brief Get intr mask for the h/w engine id. + * @brief Get intr mask for the device corresponding the provided engine_id. * * @param g [in] The GPU driver struct. - * @param engine_id [in] H/w Engine id. + * @param engine_id [in] HW engine_id. * - * Get pointer to #nvgpu_engine_info for the #engine_id. Use this to - * get intr mask for the #engine_id. + * Return the interrupt mask for the host device corresponding to \a engine_id. * - * @return Intr mask for the #engine_id. - * @retval 0 if pointer to #nvgpu_engine_info is NULL for the - * #engine_id. + * @return Intr mask for the #engine_id or 0 if the engine_id does not have a + * corresponding device. */ u32 nvgpu_engine_act_interrupt_mask(struct gk20a *g, u32 engine_id); /** @@ -256,16 +166,7 @@ u32 nvgpu_engine_act_interrupt_mask(struct gk20a *g, u32 engine_id); * * @param g [in] The GPU driver struct. * - * For each #nvgpu_fifo.num_engines, get pointer to #nvgpu_engine_info. - * Use this pointer to check if engine enum type matches with - * #NVGPU_ENGINE_GRCE or #NVGPU_ENGINE_ASYNC_CE. Upon match, logical OR - * the reset mask (init to 0) with reset_mask read from pointer to - * #nvgpu_engine_info. - * - * @return Reset mask for all the supported CE engine enum types. - * @retval NULL if #g is NULL. - * @retval 0 if none of the supported engine enum types match with - * #NVGPU_ENGINE_GRCE or #NVGPU_ENGINE_ASYNC_CE. + * @return The logical OR of the reset mask of each CE present on the GPU. */ u32 nvgpu_engine_get_all_ce_reset_mask(struct gk20a *g); /** @@ -307,11 +208,9 @@ int nvgpu_engine_setup_sw(struct gk20a *g); void nvgpu_engine_cleanup_sw(struct gk20a *g); #ifdef CONFIG_NVGPU_FIFO_ENGINE_ACTIVITY -int nvgpu_engine_enable_activity(struct gk20a *g, - struct nvgpu_engine_info *eng_info); -int nvgpu_engine_enable_activity_all(struct gk20a *g); +void nvgpu_engine_enable_activity_all(struct gk20a *g); int nvgpu_engine_disable_activity(struct gk20a *g, - struct nvgpu_engine_info *eng_info, + const struct nvgpu_device *dev, bool wait_for_idle); int nvgpu_engine_disable_activity_all(struct gk20a *g, bool wait_for_idle); @@ -483,22 +382,6 @@ bool nvgpu_engine_should_defer_reset(struct gk20a *g, u32 engine_id, */ u32 nvgpu_engine_mmu_fault_id_to_veid(struct gk20a *g, u32 mmu_fault_id, u32 gr_eng_fault_id); -/** - * @brief Get h/w engine id and veid from mmu fault id. - * - * @param g [in] The GPU driver struct. - * @param mmu_fault_id [in] Mmu fault id. - * @param veid [in,out] Pointer to store veid. - * - * Get valid h/w engine id for given #mmu_fault_id. Also get veid if engine - * enum for h/w engine id is of type #NVGPU_ENGINE_GR. - * - * @return Valid h/w (active) engine id. Updated #veid with valid veid - * if engine enum type for h/w engine id is of type #NVGPU_ENGINE_GR. - * @retval #INVAL_ID. - */ -u32 nvgpu_engine_mmu_fault_id_to_eng_id_and_veid(struct gk20a *g, - u32 mmu_fault_id, u32 *veid); /** * @brief Get engine id, veid and pbdma id from mmu fault id. * diff --git a/drivers/gpu/nvgpu/include/nvgpu/fifo.h b/drivers/gpu/nvgpu/include/nvgpu/fifo.h index 97c987042..10511ce16 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/fifo.h +++ b/drivers/gpu/nvgpu/include/nvgpu/fifo.h @@ -135,10 +135,6 @@ * * TODO * - * + struct nvgpu_engine_info - * - * TODO - * * + struct nvgpu_channel * * TODO @@ -229,7 +225,6 @@ #define CHANNEL_INFO_VEID0 0U struct gk20a; -struct nvgpu_engine_info; struct nvgpu_runlist_info; struct nvgpu_channel; struct nvgpu_tsg; @@ -238,41 +233,47 @@ struct nvgpu_swprofiler; struct nvgpu_fifo { /** Pointer to GPU driver struct. */ struct gk20a *g; + /** Number of channels supported by the h/w. */ unsigned int num_channels; + /** Runlist entry size in bytes as supported by h/w. */ unsigned int runlist_entry_size; + /** Number of runlist entries per runlist as supported by the h/w. */ unsigned int num_runlist_entries; /** - * This is the area of memory allocated by kernel to keep information for - * #max_engines supported by the chip. This information is filled up - * with device info h/w registers' values. Pointer is indexed by - * engine_id defined by h/w. + * Array of pointers to the engines that host controls. The size is + * based on the GPU litter value HOST_NUM_ENGINES. This is indexed by + * engine ID. That is to say, if you want to get a device that + * corresponds to engine ID, E, then host_engines[E] will give you a + * pointer to that device. + * + * If a given element is NULL, that means that there is no engine for + * the given engine ID. This is expected for chips that do not populate + * the full set of possible engines for a given family of chips. E.g + * a GV100 has a lot more engines than a gv11b. */ - struct nvgpu_engine_info *engine_info; + const struct nvgpu_device **host_engines; + /** - * Total number of engines supported on the chip. This variable is - * updated with one of the h/w register's value defined for chip - * configuration related settings. + * Total number of engines supported by the chip family. See + * #host_engines above. */ u32 max_engines; + /** - * This represents total number of active engines supported on the chip. - * This is calculated based on total number of available engines - * read from device info h/w registers. This variable can be less than - * or equal to #max_engines. + * The list of active engines; it can be (and often is) smaller than + * #host_engines. This list will have exactly #num_engines engines; + * use #num_engines to iterate over the list of devices with a for-loop. + */ + const struct nvgpu_device **active_engines; + + /** + * Length of the #active_engines array. */ u32 num_engines; - /** - * This is the area of memory allocated by kernel for #max_engines - * supported by the chip. This is needed to map engine_id defined - * by s/w to engine_id defined by device info h/w registers. - * This area of memory is indexed by s/w defined engine_id starting - * with 0. - */ - u32 *active_engines_list; /** * Pointers to runlists, indexed by real hw runlist_id. diff --git a/drivers/gpu/nvgpu/include/nvgpu/gops_top.h b/drivers/gpu/nvgpu/include/nvgpu/gops_top.h index f4b2e7059..cb5c82464 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/gops_top.h +++ b/drivers/gpu/nvgpu/include/nvgpu/gops_top.h @@ -65,27 +65,6 @@ struct gops_top { */ struct nvgpu_device *(*parse_next_device)(struct gk20a *g, u32 *token); - /** - * @brief Get the instance ID for particular copy engine - * - * @param g [in] GPU device struct pointer - * @param engine_type [in] Engine enumeration value - * - * 1. This HAL is valid for chips prior to Pascal when each instance of - * copy engine had unique engine_type. The three instances of copy - * engine are allocated engine_type in ascending starting from 1. - * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - * COPY_ENGINE_INSTANCE0 engine_type - 1 - * COPY_ENGINE_INSTANCE1 engine_type - 2 - * COPY_ENGINE_INSTANCE2 engine_type - 3 - * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - * 2. Calculate the instance id by subtracting COPY_ENGINE_INSTANCE0 - * enum value from \a engine_type. - * - * @return Calculated instance ID as explained above. - */ - u32 (*get_ce_inst_id)(struct gk20a *g, u32 engine_type); - /** * @brief Gets maximum number of GPCs in a GPU as programmed in HW * diff --git a/drivers/gpu/nvgpu/include/nvgpu/runlist.h b/drivers/gpu/nvgpu/include/nvgpu/runlist.h index 0f45a9551..658aac7a5 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/runlist.h +++ b/drivers/gpu/nvgpu/include/nvgpu/runlist.h @@ -35,7 +35,7 @@ /** @cond DOXYGEN_SHOULD_SKIP_THIS */ #if defined(CONFIG_NVGPU_NON_FUSA) && defined(CONFIG_NVGPU_NEXT) -#include "include/nvgpu/nvgpu_next_runlist.h" +#include #endif /** @endcond DOXYGEN_SHOULD_SKIP_THIS */ diff --git a/drivers/gpu/nvgpu/os/linux/debug_fifo.c b/drivers/gpu/nvgpu/os/linux/debug_fifo.c index 03f9a141d..bf628d223 100644 --- a/drivers/gpu/nvgpu/os/linux/debug_fifo.c +++ b/drivers/gpu/nvgpu/os/linux/debug_fifo.c @@ -22,6 +22,7 @@ #include #include #include +#include #include static void *gk20a_fifo_sched_debugfs_seq_start( @@ -62,15 +63,15 @@ static int gk20a_fifo_sched_debugfs_seq_show( struct nvgpu_channel *ch = v; struct nvgpu_tsg *tsg = NULL; - struct nvgpu_engine_info *engine_info; + const struct nvgpu_device *dev; struct nvgpu_runlist_info *runlist; u32 runlist_id; int ret = SEQ_SKIP; - u32 engine_id; - engine_id = nvgpu_engine_get_gr_id(g); - engine_info = (f->engine_info + engine_id); - runlist_id = engine_info->runlist_id; + dev = nvgpu_device_get(g, NVGPU_DEVTYPE_GRAPHICS, 0); + nvgpu_assert(dev != NULL); + + runlist_id = dev->runlist_id; runlist = f->runlist_info[runlist_id]; if (ch == f->channel) { diff --git a/drivers/gpu/nvgpu/os/linux/ioctl_ctrl.c b/drivers/gpu/nvgpu/os/linux/ioctl_ctrl.c index 3d9a289f0..22babdf93 100644 --- a/drivers/gpu/nvgpu/os/linux/ioctl_ctrl.c +++ b/drivers/gpu/nvgpu/os/linux/ioctl_ctrl.c @@ -36,6 +36,7 @@ #include #include #include +#include #include #ifdef CONFIG_NVGPU_GRAPHICS #include @@ -1009,46 +1010,43 @@ static int nvgpu_gpu_get_engine_info( struct nvgpu_gpu_get_engine_info_args *args) { int err = 0; - u32 engine_enum = NVGPU_ENGINE_INVAL; u32 report_index = 0; - u32 engine_id_idx; + u32 i; + const struct nvgpu_device *gr_dev; const u32 max_buffer_engines = args->engine_info_buf_size / sizeof(struct nvgpu_gpu_get_engine_info_item); struct nvgpu_gpu_get_engine_info_item __user *dst_item_list = (void __user *)(uintptr_t)args->engine_info_buf_addr; - for (engine_id_idx = 0; engine_id_idx < g->fifo.num_engines; - ++engine_id_idx) { - u32 active_engine_id = g->fifo.active_engines_list[engine_id_idx]; - const struct nvgpu_engine_info *src_info = - &g->fifo.engine_info[active_engine_id]; + gr_dev = nvgpu_device_get(g, NVGPU_DEVTYPE_GRAPHICS, 0); + nvgpu_assert(gr_dev != NULL); + + for (i = 0; i < g->fifo.num_engines; i++) { + const struct nvgpu_device *dev = g->fifo.active_engines[i]; struct nvgpu_gpu_get_engine_info_item dst_info; (void) memset(&dst_info, 0, sizeof(dst_info)); - engine_enum = src_info->engine_enum; - - switch (engine_enum) { - case NVGPU_ENGINE_GR: + if (nvgpu_device_is_graphics(g, dev)) { dst_info.engine_id = NVGPU_GPU_ENGINE_ID_GR; - break; - - case NVGPU_ENGINE_GRCE: - dst_info.engine_id = NVGPU_GPU_ENGINE_ID_GR_COPY; - break; - - case NVGPU_ENGINE_ASYNC_CE: - dst_info.engine_id = NVGPU_GPU_ENGINE_ID_ASYNC_COPY; - break; - - default: - nvgpu_err(g, "Unmapped engine enum %u", - engine_enum); - continue; + } else if (nvgpu_device_is_ce(g, dev)) { + /* + * There's two types of CE userpsace is interested in: + * ASYNC_CEs which are copy engines with their own + * runlists and GRCEs which are CEs that share a runlist + * with GR. + */ + if (dev->runlist_id == gr_dev->runlist_id) { + dst_info.engine_id = + NVGPU_GPU_ENGINE_ID_GR_COPY; + } else { + dst_info.engine_id = + NVGPU_GPU_ENGINE_ID_ASYNC_COPY; + } } - dst_info.engine_instance = src_info->inst_id; - dst_info.runlist_id = src_info->runlist_id; + dst_info.engine_instance = dev->inst_id; + dst_info.runlist_id = dev->runlist_id; if (report_index < max_buffer_engines) { err = copy_to_user(&dst_item_list[report_index], diff --git a/libs/dgpu/libnvgpu-drv-dgpu_safe.export b/libs/dgpu/libnvgpu-drv-dgpu_safe.export index 0f9f60321..80befb586 100644 --- a/libs/dgpu/libnvgpu-drv-dgpu_safe.export +++ b/libs/dgpu/libnvgpu-drv-dgpu_safe.export @@ -353,14 +353,12 @@ nvgpu_engine_get_fast_ce_runlist_id nvgpu_engine_get_gr_id nvgpu_engine_get_gr_runlist_id nvgpu_engine_get_id_and_type -nvgpu_engine_get_ids nvgpu_engine_get_mask_on_id nvgpu_engine_get_runlist_busy_engines nvgpu_engine_id_to_mmu_fault_id nvgpu_engine_is_valid_runlist_id nvgpu_engine_init_info nvgpu_engine_mmu_fault_id_to_engine_id -nvgpu_engine_mmu_fault_id_to_eng_id_and_veid nvgpu_engine_mmu_fault_id_to_eng_ve_pbdma_id nvgpu_engine_mmu_fault_id_to_veid nvgpu_engine_setup_sw diff --git a/libs/igpu/libnvgpu-drv-igpu_safe.export b/libs/igpu/libnvgpu-drv-igpu_safe.export index ff2f923a5..372c70457 100644 --- a/libs/igpu/libnvgpu-drv-igpu_safe.export +++ b/libs/igpu/libnvgpu-drv-igpu_safe.export @@ -365,14 +365,12 @@ nvgpu_engine_get_fast_ce_runlist_id nvgpu_engine_get_gr_id nvgpu_engine_get_gr_runlist_id nvgpu_engine_get_id_and_type -nvgpu_engine_get_ids nvgpu_engine_get_mask_on_id nvgpu_engine_get_runlist_busy_engines nvgpu_engine_id_to_mmu_fault_id nvgpu_engine_is_valid_runlist_id nvgpu_engine_init_info nvgpu_engine_mmu_fault_id_to_engine_id -nvgpu_engine_mmu_fault_id_to_eng_id_and_veid nvgpu_engine_mmu_fault_id_to_eng_ve_pbdma_id nvgpu_engine_mmu_fault_id_to_veid nvgpu_engine_setup_sw