From e05c0d13a019604daaf8e0fc0b3e6d15b345eff7 Mon Sep 17 00:00:00 2001 From: Konsta Holtta Date: Wed, 28 Nov 2018 13:44:50 +0200 Subject: [PATCH] gpu: nvgpu: add runlist unit to common Extract non-chip-specific code that manages the runlists (init, update, reschedule etc.) to a new file in the common directory. Move the declarations to a new matching runlist.h header. Jira NVGPU-1309 Change-Id: I3c7e0032899516487037f47ddc9a7e7aa4b0b33a Signed-off-by: Konsta Holtta Reviewed-on: https://git-master.nvidia.com/r/1978058 Reviewed-by: mobile promotions Tested-by: mobile promotions --- drivers/gpu/nvgpu/Makefile | 1 + drivers/gpu/nvgpu/Makefile.sources | 3 +- drivers/gpu/nvgpu/common/fifo/channel.c | 3 +- drivers/gpu/nvgpu/common/fifo/runlist.c | 642 +++++++++++++++++++ drivers/gpu/nvgpu/gk20a/fifo_gk20a.c | 618 +----------------- drivers/gpu/nvgpu/gk20a/fifo_gk20a.h | 21 - drivers/gpu/nvgpu/gm20b/hal_gm20b.c | 1 + drivers/gpu/nvgpu/gp10b/hal_gp10b.c | 1 + drivers/gpu/nvgpu/gv100/hal_gv100.c | 1 + drivers/gpu/nvgpu/gv11b/fifo_gv11b.c | 3 +- drivers/gpu/nvgpu/gv11b/hal_gv11b.c | 1 + drivers/gpu/nvgpu/include/nvgpu/runlist.h | 57 ++ drivers/gpu/nvgpu/os/linux/ioctl_channel.c | 3 +- drivers/gpu/nvgpu/tu104/hal_tu104.c | 1 + drivers/gpu/nvgpu/vgpu/fifo_vgpu.c | 3 +- userspace/units/fifo/runlist/nvgpu-runlist.c | 3 +- 16 files changed, 719 insertions(+), 643 deletions(-) create mode 100644 drivers/gpu/nvgpu/common/fifo/runlist.c create mode 100644 drivers/gpu/nvgpu/include/nvgpu/runlist.h diff --git a/drivers/gpu/nvgpu/Makefile b/drivers/gpu/nvgpu/Makefile index ec28f2f77..72015e129 100644 --- a/drivers/gpu/nvgpu/Makefile +++ b/drivers/gpu/nvgpu/Makefile @@ -278,6 +278,7 @@ nvgpu-y += \ common/fifo/channel.o \ common/fifo/submit.o \ common/fifo/tsg.o \ + common/fifo/runlist.o \ common/ecc.o \ common/ce2.o \ common/debugger.o \ diff --git a/drivers/gpu/nvgpu/Makefile.sources b/drivers/gpu/nvgpu/Makefile.sources index f81a755e0..0701ce395 100644 --- a/drivers/gpu/nvgpu/Makefile.sources +++ b/drivers/gpu/nvgpu/Makefile.sources @@ -1,6 +1,6 @@ # -*- mode: makefile -*- # -# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the "Software"), @@ -150,6 +150,7 @@ srcs := os/posix/nvgpu.c \ common/fifo/channel.c \ common/fifo/submit.c \ common/fifo/tsg.c \ + common/fifo/runlist.c \ common/mc/mc.c \ common/mc/mc_gm20b.c \ common/mc/mc_gp10b.c \ diff --git a/drivers/gpu/nvgpu/common/fifo/channel.c b/drivers/gpu/nvgpu/common/fifo/channel.c index 9919fdfde..ceeea1995 100644 --- a/drivers/gpu/nvgpu/common/fifo/channel.c +++ b/drivers/gpu/nvgpu/common/fifo/channel.c @@ -1,7 +1,7 @@ /* * GK20A Graphics channel * - * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2011-2019, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -47,6 +47,7 @@ #include #include #include +#include #include "gk20a/fence_gk20a.h" #include "gk20a/gr_gk20a.h" diff --git a/drivers/gpu/nvgpu/common/fifo/runlist.c b/drivers/gpu/nvgpu/common/fifo/runlist.c new file mode 100644 index 000000000..c81559e5e --- /dev/null +++ b/drivers/gpu/nvgpu/common/fifo/runlist.c @@ -0,0 +1,642 @@ +/* + * Copyright (c) 2011-2019, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include +#include +#include + +static u32 nvgpu_runlist_append_tsg(struct gk20a *g, + struct fifo_runlist_info_gk20a *runlist, + u32 **runlist_entry, + u32 *entries_left, + struct tsg_gk20a *tsg) +{ + struct fifo_gk20a *f = &g->fifo; + u32 runlist_entry_words = f->runlist_entry_size / (u32)sizeof(u32); + struct channel_gk20a *ch; + u32 count = 0; + + nvgpu_log_fn(f->g, " "); + + if (*entries_left == 0U) { + return RUNLIST_APPEND_FAILURE; + } + + /* add TSG entry */ + nvgpu_log_info(g, "add TSG %d to runlist", tsg->tsgid); + g->ops.fifo.get_tsg_runlist_entry(tsg, *runlist_entry); + nvgpu_log_info(g, "tsg rl entries left %d runlist [0] %x [1] %x", + *entries_left, + (*runlist_entry)[0], (*runlist_entry)[1]); + *runlist_entry += runlist_entry_words; + count++; + (*entries_left)--; + + nvgpu_rwsem_down_read(&tsg->ch_list_lock); + /* add runnable channels bound to this TSG */ + nvgpu_list_for_each_entry(ch, &tsg->ch_list, + channel_gk20a, ch_entry) { + if (!test_bit((int)ch->chid, + runlist->active_channels)) { + continue; + } + + if (*entries_left == 0U) { + nvgpu_rwsem_up_read(&tsg->ch_list_lock); + return RUNLIST_APPEND_FAILURE; + } + + nvgpu_log_info(g, "add channel %d to runlist", + ch->chid); + g->ops.fifo.get_ch_runlist_entry(ch, *runlist_entry); + nvgpu_log_info(g, "rl entries left %d runlist [0] %x [1] %x", + *entries_left, + (*runlist_entry)[0], (*runlist_entry)[1]); + count++; + *runlist_entry += runlist_entry_words; + (*entries_left)--; + } + nvgpu_rwsem_up_read(&tsg->ch_list_lock); + + return count; +} + + +static u32 nvgpu_runlist_append_prio(struct fifo_gk20a *f, + struct fifo_runlist_info_gk20a *runlist, + u32 **runlist_entry, + u32 *entries_left, + u32 interleave_level) +{ + u32 count = 0; + unsigned long tsgid; + + nvgpu_log_fn(f->g, " "); + + for_each_set_bit(tsgid, runlist->active_tsgs, f->num_channels) { + struct tsg_gk20a *tsg = &f->tsg[tsgid]; + u32 entries; + + if (tsg->interleave_level == interleave_level) { + entries = nvgpu_runlist_append_tsg(f->g, runlist, + runlist_entry, entries_left, tsg); + if (entries == RUNLIST_APPEND_FAILURE) { + return RUNLIST_APPEND_FAILURE; + } + count += entries; + } + } + + return count; +} + +static u32 nvgpu_runlist_append_hi(struct fifo_gk20a *f, + struct fifo_runlist_info_gk20a *runlist, + u32 **runlist_entry, + u32 *entries_left) +{ + nvgpu_log_fn(f->g, " "); + + /* + * No higher levels - this is where the "recursion" ends; just add all + * active TSGs at this level. + */ + return nvgpu_runlist_append_prio(f, runlist, runlist_entry, + entries_left, + NVGPU_FIFO_RUNLIST_INTERLEAVE_LEVEL_HIGH); +} + +static u32 nvgpu_runlist_append_med(struct fifo_gk20a *f, + struct fifo_runlist_info_gk20a *runlist, + u32 **runlist_entry, + u32 *entries_left) +{ + u32 count = 0; + unsigned long tsgid; + + nvgpu_log_fn(f->g, " "); + + for_each_set_bit(tsgid, runlist->active_tsgs, f->num_channels) { + struct tsg_gk20a *tsg = &f->tsg[tsgid]; + u32 entries; + + if (tsg->interleave_level != + NVGPU_FIFO_RUNLIST_INTERLEAVE_LEVEL_MEDIUM) { + continue; + } + + /* LEVEL_MEDIUM list starts with a LEVEL_HIGH, if any */ + + entries = nvgpu_runlist_append_hi(f, runlist, + runlist_entry, entries_left); + if (entries == RUNLIST_APPEND_FAILURE) { + return RUNLIST_APPEND_FAILURE; + } + count += entries; + + entries = nvgpu_runlist_append_tsg(f->g, runlist, + runlist_entry, entries_left, tsg); + if (entries == RUNLIST_APPEND_FAILURE) { + return RUNLIST_APPEND_FAILURE; + } + count += entries; + } + + return count; +} + +static u32 nvgpu_runlist_append_low(struct fifo_gk20a *f, + struct fifo_runlist_info_gk20a *runlist, + u32 **runlist_entry, + u32 *entries_left) +{ + u32 count = 0; + unsigned long tsgid; + + nvgpu_log_fn(f->g, " "); + + for_each_set_bit(tsgid, runlist->active_tsgs, f->num_channels) { + struct tsg_gk20a *tsg = &f->tsg[tsgid]; + u32 entries; + + if (tsg->interleave_level != + NVGPU_FIFO_RUNLIST_INTERLEAVE_LEVEL_LOW) { + continue; + } + + /* The medium level starts with the highs, if any. */ + + entries = nvgpu_runlist_append_med(f, runlist, + runlist_entry, entries_left); + if (entries == RUNLIST_APPEND_FAILURE) { + return RUNLIST_APPEND_FAILURE; + } + count += entries; + + entries = nvgpu_runlist_append_hi(f, runlist, + runlist_entry, entries_left); + if (entries == RUNLIST_APPEND_FAILURE) { + return RUNLIST_APPEND_FAILURE; + } + count += entries; + + entries = nvgpu_runlist_append_tsg(f->g, runlist, + runlist_entry, entries_left, tsg); + if (entries == RUNLIST_APPEND_FAILURE) { + return RUNLIST_APPEND_FAILURE; + } + count += entries; + } + + if (count == 0U) { + /* + * No transitions to fill with higher levels, so add + * the next level once. If that's empty too, we have only + * LEVEL_HIGH jobs. + */ + count = nvgpu_runlist_append_med(f, runlist, + runlist_entry, entries_left); + if (count == 0U) { + count = nvgpu_runlist_append_hi(f, runlist, + runlist_entry, entries_left); + } + } + + return count; +} + +static u32 nvgpu_runlist_append_flat(struct fifo_gk20a *f, + struct fifo_runlist_info_gk20a *runlist, + u32 **runlist_entry, + u32 *entries_left) +{ + u32 count = 0, entries, i; + + nvgpu_log_fn(f->g, " "); + + /* Group by priority but don't interleave. High comes first. */ + + for (i = 0; i < NVGPU_FIFO_RUNLIST_INTERLEAVE_NUM_LEVELS; i++) { + u32 level = NVGPU_FIFO_RUNLIST_INTERLEAVE_LEVEL_HIGH - i; + + entries = nvgpu_runlist_append_prio(f, runlist, runlist_entry, + entries_left, level); + if (entries == RUNLIST_APPEND_FAILURE) { + return RUNLIST_APPEND_FAILURE; + } + count += entries; + } + + return count; +} + +u32 nvgpu_runlist_construct_locked(struct fifo_gk20a *f, + struct fifo_runlist_info_gk20a *runlist, + u32 buf_id, + u32 max_entries) +{ + u32 *runlist_entry_base = runlist->mem[buf_id].cpu_va; + + nvgpu_log_fn(f->g, " "); + + /* + * The entry pointer and capacity counter that live on the stack here + * keep track of the current position and the remaining space when tsg + * and channel entries are ultimately appended. + */ + if (f->g->runlist_interleave) { + return nvgpu_runlist_append_low(f, runlist, + &runlist_entry_base, &max_entries); + } else { + return nvgpu_runlist_append_flat(f, runlist, + &runlist_entry_base, &max_entries); + } +} + +int gk20a_fifo_update_runlist_locked(struct gk20a *g, u32 runlist_id, + u32 chid, bool add, + bool wait_for_finish) +{ + int ret = 0; + struct fifo_gk20a *f = &g->fifo; + struct fifo_runlist_info_gk20a *runlist = NULL; + u64 runlist_iova; + u32 new_buf; + struct channel_gk20a *ch = NULL; + struct tsg_gk20a *tsg = NULL; + + runlist = &f->runlist_info[runlist_id]; + + /* valid channel, add/remove it from active list. + Otherwise, keep active list untouched for suspend/resume. */ + if (chid != FIFO_INVAL_CHANNEL_ID) { + ch = &f->channel[chid]; + if (gk20a_is_channel_marked_as_tsg(ch)) { + tsg = &f->tsg[ch->tsgid]; + } + + if (add) { + if (test_and_set_bit(chid, + runlist->active_channels)) { + return 0; + } + if ((tsg != NULL) && (++tsg->num_active_channels != 0U)) { + set_bit((int)f->channel[chid].tsgid, + runlist->active_tsgs); + } + } else { + if (!test_and_clear_bit(chid, + runlist->active_channels)) { + return 0; + } + if ((tsg != NULL) && + (--tsg->num_active_channels == 0U)) { + clear_bit((int)f->channel[chid].tsgid, + runlist->active_tsgs); + } + } + } + + /* There just 2 buffers */ + new_buf = runlist->cur_buffer == 0U ? 1U : 0U; + + runlist_iova = nvgpu_mem_get_addr(g, &runlist->mem[new_buf]); + + nvgpu_log_info(g, "runlist_id : %d, switch to new buffer 0x%16llx", + runlist_id, (u64)runlist_iova); + + if (runlist_iova == 0ULL) { + ret = -EINVAL; + goto clean_up; + } + + if (chid != FIFO_INVAL_CHANNEL_ID || /* add/remove a valid channel */ + add /* resume to add all channels back */) { + u32 num_entries; + + num_entries = nvgpu_runlist_construct_locked(f, + runlist, + new_buf, + f->num_runlist_entries); + if (num_entries == RUNLIST_APPEND_FAILURE) { + ret = -E2BIG; + goto clean_up; + } + runlist->count = num_entries; + WARN_ON(runlist->count > f->num_runlist_entries); + } else { + /* suspend to remove all channels */ + runlist->count = 0; + } + + g->ops.fifo.runlist_hw_submit(g, runlist_id, runlist->count, new_buf); + + if (wait_for_finish) { + ret = g->ops.fifo.runlist_wait_pending(g, runlist_id); + + if (ret == -ETIMEDOUT) { + nvgpu_err(g, "runlist %d update timeout", runlist_id); + /* trigger runlist update timeout recovery */ + return ret; + + } else if (ret == -EINTR) { + nvgpu_err(g, "runlist update interrupted"); + } + } + + runlist->cur_buffer = new_buf; + +clean_up: + return ret; +} + +/* trigger host to expire current timeslice and reschedule runlist from front */ +int nvgpu_fifo_reschedule_runlist(struct channel_gk20a *ch, bool preempt_next, + bool wait_preempt) +{ + struct gk20a *g = ch->g; + struct fifo_runlist_info_gk20a *runlist; + u32 token = PMU_INVALID_MUTEX_OWNER_ID; + int mutex_ret; + int ret = 0; + + runlist = &g->fifo.runlist_info[ch->runlist_id]; + if (nvgpu_mutex_tryacquire(&runlist->runlist_lock) == 0) { + return -EBUSY; + } + + mutex_ret = nvgpu_pmu_mutex_acquire( + &g->pmu, PMU_MUTEX_ID_FIFO, &token); + + g->ops.fifo.runlist_hw_submit( + g, ch->runlist_id, runlist->count, runlist->cur_buffer); + + if (preempt_next) { + g->ops.fifo.reschedule_preempt_next_locked(ch, wait_preempt); + } + + g->ops.fifo.runlist_wait_pending(g, ch->runlist_id); + + if (mutex_ret == 0) { + nvgpu_pmu_mutex_release( + &g->pmu, PMU_MUTEX_ID_FIFO, &token); + } + nvgpu_mutex_release(&runlist->runlist_lock); + + return ret; +} + +static void gk20a_fifo_runlist_reset_engines(struct gk20a *g, u32 runlist_id) +{ + u32 engines = g->ops.fifo.runlist_busy_engines(g, runlist_id); + + if (engines != 0U) { + gk20a_fifo_recover(g, engines, ~(u32)0, false, false, true, + RC_TYPE_RUNLIST_UPDATE_TIMEOUT); + } +} + +/* add/remove a channel from runlist + special cases below: runlist->active_channels will NOT be changed. + (chid == ~0 && !add) means remove all active channels from runlist. + (chid == ~0 && add) means restore all active channels on runlist. */ +int gk20a_fifo_update_runlist(struct gk20a *g, u32 runlist_id, u32 chid, + bool add, bool wait_for_finish) +{ + struct fifo_runlist_info_gk20a *runlist = NULL; + struct fifo_gk20a *f = &g->fifo; + u32 token = PMU_INVALID_MUTEX_OWNER_ID; + int mutex_ret; + int ret = 0; + + nvgpu_log_fn(g, " "); + + runlist = &f->runlist_info[runlist_id]; + + nvgpu_mutex_acquire(&runlist->runlist_lock); + + mutex_ret = nvgpu_pmu_mutex_acquire(&g->pmu, PMU_MUTEX_ID_FIFO, &token); + + ret = gk20a_fifo_update_runlist_locked(g, runlist_id, chid, add, + wait_for_finish); + + if (mutex_ret == 0) { + nvgpu_pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token); + } + + nvgpu_mutex_release(&runlist->runlist_lock); + + if (ret == -ETIMEDOUT) { + gk20a_fifo_runlist_reset_engines(g, runlist_id); + } + + return ret; +} + +int gk20a_fifo_update_runlist_ids(struct gk20a *g, u32 runlist_ids, u32 chid, + bool add, bool wait_for_finish) +{ + int ret = -EINVAL; + unsigned long runlist_id = 0; + int errcode; + unsigned long ulong_runlist_ids = (unsigned long)runlist_ids; + + if (g == NULL) { + goto end; + } + + ret = 0; + for_each_set_bit(runlist_id, &ulong_runlist_ids, 32U) { + /* Capture the last failure error code */ + errcode = g->ops.fifo.update_runlist(g, (u32)runlist_id, chid, + add, wait_for_finish); + if (errcode != 0) { + nvgpu_err(g, + "failed to update_runlist %lu %d", + runlist_id, errcode); + ret = errcode; + } + } +end: + return ret; +} + +const char *gk20a_fifo_interleave_level_name(u32 interleave_level) +{ + const char *ret_string = NULL; + + switch (interleave_level) { + case NVGPU_FIFO_RUNLIST_INTERLEAVE_LEVEL_LOW: + ret_string = "LOW"; + break; + + case NVGPU_FIFO_RUNLIST_INTERLEAVE_LEVEL_MEDIUM: + ret_string = "MEDIUM"; + break; + + case NVGPU_FIFO_RUNLIST_INTERLEAVE_LEVEL_HIGH: + ret_string = "HIGH"; + break; + + default: + ret_string = "?"; + break; + } + + return ret_string; +} + +void gk20a_fifo_delete_runlist(struct fifo_gk20a *f) +{ + u32 i; + u32 runlist_id; + struct fifo_runlist_info_gk20a *runlist; + struct gk20a *g = NULL; + + if ((f == NULL) || (f->runlist_info == NULL)) { + return; + } + + g = f->g; + + for (runlist_id = 0; runlist_id < f->max_runlists; runlist_id++) { + runlist = &f->runlist_info[runlist_id]; + for (i = 0; i < MAX_RUNLIST_BUFFERS; i++) { + nvgpu_dma_free(g, &runlist->mem[i]); + } + + nvgpu_kfree(g, runlist->active_channels); + runlist->active_channels = NULL; + + nvgpu_kfree(g, runlist->active_tsgs); + runlist->active_tsgs = NULL; + + nvgpu_mutex_destroy(&runlist->runlist_lock); + + } + (void) memset(f->runlist_info, 0, + (sizeof(struct fifo_runlist_info_gk20a) * f->max_runlists)); + + nvgpu_kfree(g, f->runlist_info); + f->runlist_info = NULL; + f->max_runlists = 0; +} + +int nvgpu_init_runlist(struct gk20a *g, struct fifo_gk20a *f) +{ + struct fifo_runlist_info_gk20a *runlist; + struct fifo_engine_info_gk20a *engine_info; + unsigned int runlist_id; + u32 i; + size_t runlist_size; + u32 active_engine_id, pbdma_id, engine_id; + int err = 0; + + nvgpu_log_fn(g, " "); + + f->max_runlists = g->ops.fifo.eng_runlist_base_size(); + f->runlist_info = nvgpu_kzalloc(g, + sizeof(struct fifo_runlist_info_gk20a) * + f->max_runlists); + if (f->runlist_info == NULL) { + goto clean_up_runlist; + } + + (void) memset(f->runlist_info, 0, + (sizeof(struct fifo_runlist_info_gk20a) * f->max_runlists)); + + for (runlist_id = 0; runlist_id < f->max_runlists; runlist_id++) { + runlist = &f->runlist_info[runlist_id]; + + runlist->active_channels = + nvgpu_kzalloc(g, DIV_ROUND_UP(f->num_channels, + BITS_PER_BYTE)); + if (runlist->active_channels == NULL) { + goto clean_up_runlist; + } + + runlist->active_tsgs = + nvgpu_kzalloc(g, DIV_ROUND_UP(f->num_channels, + BITS_PER_BYTE)); + if (runlist->active_tsgs == NULL) { + goto clean_up_runlist; + } + + runlist_size = (size_t)f->runlist_entry_size * + (size_t)f->num_runlist_entries; + nvgpu_log(g, gpu_dbg_info, + "runlist_entries %d runlist size %zu", + f->num_runlist_entries, runlist_size); + + for (i = 0; i < MAX_RUNLIST_BUFFERS; i++) { + err = nvgpu_dma_alloc_flags_sys(g, + NVGPU_DMA_PHYSICALLY_ADDRESSED, + runlist_size, + &runlist->mem[i]); + if (err != 0) { + nvgpu_err(g, "memory allocation failed"); + goto clean_up_runlist; + } + } + + err = nvgpu_mutex_init(&runlist->runlist_lock); + if (err != 0) { + nvgpu_err(g, + "Error in runlist_lock mutex initialization"); + goto clean_up_runlist; + } + + /* None of buffers is pinned if this value doesn't change. + Otherwise, one of them (cur_buffer) must have been pinned. */ + runlist->cur_buffer = MAX_RUNLIST_BUFFERS; + + for (pbdma_id = 0; pbdma_id < f->num_pbdma; pbdma_id++) { + if ((f->pbdma_map[pbdma_id] & BIT32(runlist_id)) != 0U) { + runlist->pbdma_bitmask |= BIT32(pbdma_id); + } + } + nvgpu_log(g, gpu_dbg_info, "runlist %d : pbdma bitmask 0x%x", + runlist_id, runlist->pbdma_bitmask); + + for (engine_id = 0; engine_id < f->num_engines; ++engine_id) { + active_engine_id = f->active_engines_list[engine_id]; + engine_info = &f->engine_info[active_engine_id]; + + if ((engine_info != NULL) && + (engine_info->runlist_id == runlist_id)) { + runlist->eng_bitmask |= BIT(active_engine_id); + } + } + nvgpu_log(g, gpu_dbg_info, "runlist %d : act eng bitmask 0x%x", + runlist_id, runlist->eng_bitmask); + } + + nvgpu_log_fn(g, "done"); + return 0; + +clean_up_runlist: + gk20a_fifo_delete_runlist(f); + nvgpu_log_fn(g, "fail"); + return err; +} diff --git a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c index 2e30490ab..e73ca3d2a 100644 --- a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c @@ -44,6 +44,7 @@ #include #include #include +#include #include #include #include @@ -393,42 +394,6 @@ u32 gk20a_fifo_engine_interrupt_mask(struct gk20a *g) return eng_intr_mask; } -void gk20a_fifo_delete_runlist(struct fifo_gk20a *f) -{ - u32 i; - u32 runlist_id; - struct fifo_runlist_info_gk20a *runlist; - struct gk20a *g = NULL; - - if ((f == NULL) || (f->runlist_info == NULL)) { - return; - } - - g = f->g; - - for (runlist_id = 0; runlist_id < f->max_runlists; runlist_id++) { - runlist = &f->runlist_info[runlist_id]; - for (i = 0; i < MAX_RUNLIST_BUFFERS; i++) { - nvgpu_dma_free(g, &runlist->mem[i]); - } - - nvgpu_kfree(g, runlist->active_channels); - runlist->active_channels = NULL; - - nvgpu_kfree(g, runlist->active_tsgs); - runlist->active_tsgs = NULL; - - nvgpu_mutex_destroy(&runlist->runlist_lock); - - } - (void) memset(f->runlist_info, 0, - (sizeof(struct fifo_runlist_info_gk20a) * f->max_runlists)); - - nvgpu_kfree(g, f->runlist_info); - f->runlist_info = NULL; - f->max_runlists = 0; -} - static void gk20a_remove_fifo_support(struct fifo_gk20a *f) { struct gk20a *g = f->g; @@ -482,104 +447,6 @@ static void gk20a_remove_fifo_support(struct fifo_gk20a *f) f->active_engines_list = NULL; } -static int init_runlist(struct gk20a *g, struct fifo_gk20a *f) -{ - struct fifo_runlist_info_gk20a *runlist; - struct fifo_engine_info_gk20a *engine_info; - unsigned int runlist_id; - u32 i; - size_t runlist_size; - u32 active_engine_id, pbdma_id, engine_id; - int err = 0; - - nvgpu_log_fn(g, " "); - - f->max_runlists = g->ops.fifo.eng_runlist_base_size(); - f->runlist_info = nvgpu_kzalloc(g, - sizeof(struct fifo_runlist_info_gk20a) * - f->max_runlists); - if (f->runlist_info == NULL) { - goto clean_up_runlist; - } - - (void) memset(f->runlist_info, 0, - (sizeof(struct fifo_runlist_info_gk20a) * f->max_runlists)); - - for (runlist_id = 0; runlist_id < f->max_runlists; runlist_id++) { - runlist = &f->runlist_info[runlist_id]; - - runlist->active_channels = - nvgpu_kzalloc(g, DIV_ROUND_UP(f->num_channels, - BITS_PER_BYTE)); - if (runlist->active_channels == NULL) { - goto clean_up_runlist; - } - - runlist->active_tsgs = - nvgpu_kzalloc(g, DIV_ROUND_UP(f->num_channels, - BITS_PER_BYTE)); - if (runlist->active_tsgs == NULL) { - goto clean_up_runlist; - } - - runlist_size = (size_t)f->runlist_entry_size * - (size_t)f->num_runlist_entries; - nvgpu_log(g, gpu_dbg_info, - "runlist_entries %d runlist size %zu", - f->num_runlist_entries, runlist_size); - - for (i = 0; i < MAX_RUNLIST_BUFFERS; i++) { - err = nvgpu_dma_alloc_flags_sys(g, - NVGPU_DMA_PHYSICALLY_ADDRESSED, - runlist_size, - &runlist->mem[i]); - if (err != 0) { - nvgpu_err(g, "memory allocation failed"); - goto clean_up_runlist; - } - } - - err = nvgpu_mutex_init(&runlist->runlist_lock); - if (err != 0) { - nvgpu_err(g, - "Error in runlist_lock mutex initialization"); - goto clean_up_runlist; - } - - /* None of buffers is pinned if this value doesn't change. - Otherwise, one of them (cur_buffer) must have been pinned. */ - runlist->cur_buffer = MAX_RUNLIST_BUFFERS; - - for (pbdma_id = 0; pbdma_id < f->num_pbdma; pbdma_id++) { - if ((f->pbdma_map[pbdma_id] & BIT32(runlist_id)) != 0U) { - runlist->pbdma_bitmask |= BIT32(pbdma_id); - } - } - nvgpu_log(g, gpu_dbg_info, "runlist %d : pbdma bitmask 0x%x", - runlist_id, runlist->pbdma_bitmask); - - for (engine_id = 0; engine_id < f->num_engines; ++engine_id) { - active_engine_id = f->active_engines_list[engine_id]; - engine_info = &f->engine_info[active_engine_id]; - - if ((engine_info != NULL) && - (engine_info->runlist_id == runlist_id)) { - runlist->eng_bitmask |= BIT(active_engine_id); - } - } - nvgpu_log(g, gpu_dbg_info, "runlist %d : act eng bitmask 0x%x", - runlist_id, runlist->eng_bitmask); - } - - nvgpu_log_fn(g, "done"); - return 0; - -clean_up_runlist: - gk20a_fifo_delete_runlist(f); - nvgpu_log_fn(g, "fail"); - return err; -} - u32 gk20a_fifo_intr_0_error_mask(struct gk20a *g) { u32 intr_0_error_mask = @@ -752,7 +619,7 @@ int gk20a_init_fifo_setup_sw_common(struct gk20a *g) g->ops.fifo.init_engine_info(f); - err = init_runlist(g, f); + err = nvgpu_init_runlist(g, f); if (err != 0) { nvgpu_err(g, "failed to init runlist"); goto clean_up; @@ -2851,16 +2718,6 @@ u32 gk20a_fifo_runlist_busy_engines(struct gk20a *g, u32 runlist_id) return engines; } -static void gk20a_fifo_runlist_reset_engines(struct gk20a *g, u32 runlist_id) -{ - u32 engines = g->ops.fifo.runlist_busy_engines(g, runlist_id); - - if (engines != 0U) { - gk20a_fifo_recover(g, engines, ~(u32)0, false, false, true, - RC_TYPE_RUNLIST_UPDATE_TIMEOUT); - } -} - int gk20a_fifo_runlist_wait_pending(struct gk20a *g, u32 runlist_id) { struct nvgpu_timeout timeout; @@ -2938,254 +2795,6 @@ void gk20a_get_ch_runlist_entry(struct channel_gk20a *ch, u32 *runlist) runlist[1] = 0; } -static u32 nvgpu_runlist_append_tsg(struct gk20a *g, - struct fifo_runlist_info_gk20a *runlist, - u32 **runlist_entry, - u32 *entries_left, - struct tsg_gk20a *tsg) -{ - struct fifo_gk20a *f = &g->fifo; - u32 runlist_entry_words = f->runlist_entry_size / (u32)sizeof(u32); - struct channel_gk20a *ch; - u32 count = 0; - - nvgpu_log_fn(f->g, " "); - - if (*entries_left == 0U) { - return RUNLIST_APPEND_FAILURE; - } - - /* add TSG entry */ - nvgpu_log_info(g, "add TSG %d to runlist", tsg->tsgid); - g->ops.fifo.get_tsg_runlist_entry(tsg, *runlist_entry); - nvgpu_log_info(g, "tsg rl entries left %d runlist [0] %x [1] %x", - *entries_left, - (*runlist_entry)[0], (*runlist_entry)[1]); - *runlist_entry += runlist_entry_words; - count++; - (*entries_left)--; - - nvgpu_rwsem_down_read(&tsg->ch_list_lock); - /* add runnable channels bound to this TSG */ - nvgpu_list_for_each_entry(ch, &tsg->ch_list, - channel_gk20a, ch_entry) { - if (!test_bit((int)ch->chid, - runlist->active_channels)) { - continue; - } - - if (*entries_left == 0U) { - nvgpu_rwsem_up_read(&tsg->ch_list_lock); - return RUNLIST_APPEND_FAILURE; - } - - nvgpu_log_info(g, "add channel %d to runlist", - ch->chid); - g->ops.fifo.get_ch_runlist_entry(ch, *runlist_entry); - nvgpu_log_info(g, "rl entries left %d runlist [0] %x [1] %x", - *entries_left, - (*runlist_entry)[0], (*runlist_entry)[1]); - count++; - *runlist_entry += runlist_entry_words; - (*entries_left)--; - } - nvgpu_rwsem_up_read(&tsg->ch_list_lock); - - return count; -} - - -static u32 nvgpu_runlist_append_prio(struct fifo_gk20a *f, - struct fifo_runlist_info_gk20a *runlist, - u32 **runlist_entry, - u32 *entries_left, - u32 interleave_level) -{ - u32 count = 0; - unsigned long tsgid; - - nvgpu_log_fn(f->g, " "); - - for_each_set_bit(tsgid, runlist->active_tsgs, f->num_channels) { - struct tsg_gk20a *tsg = &f->tsg[tsgid]; - u32 entries; - - if (tsg->interleave_level == interleave_level) { - entries = nvgpu_runlist_append_tsg(f->g, runlist, - runlist_entry, entries_left, tsg); - if (entries == RUNLIST_APPEND_FAILURE) { - return RUNLIST_APPEND_FAILURE; - } - count += entries; - } - } - - return count; -} - -static u32 nvgpu_runlist_append_hi(struct fifo_gk20a *f, - struct fifo_runlist_info_gk20a *runlist, - u32 **runlist_entry, - u32 *entries_left) -{ - nvgpu_log_fn(f->g, " "); - - /* - * No higher levels - this is where the "recursion" ends; just add all - * active TSGs at this level. - */ - return nvgpu_runlist_append_prio(f, runlist, runlist_entry, - entries_left, - NVGPU_FIFO_RUNLIST_INTERLEAVE_LEVEL_HIGH); -} - -static u32 nvgpu_runlist_append_med(struct fifo_gk20a *f, - struct fifo_runlist_info_gk20a *runlist, - u32 **runlist_entry, - u32 *entries_left) -{ - u32 count = 0; - unsigned long tsgid; - - nvgpu_log_fn(f->g, " "); - - for_each_set_bit(tsgid, runlist->active_tsgs, f->num_channels) { - struct tsg_gk20a *tsg = &f->tsg[tsgid]; - u32 entries; - - if (tsg->interleave_level != - NVGPU_FIFO_RUNLIST_INTERLEAVE_LEVEL_MEDIUM) { - continue; - } - - /* LEVEL_MEDIUM list starts with a LEVEL_HIGH, if any */ - - entries = nvgpu_runlist_append_hi(f, runlist, - runlist_entry, entries_left); - if (entries == RUNLIST_APPEND_FAILURE) { - return RUNLIST_APPEND_FAILURE; - } - count += entries; - - entries = nvgpu_runlist_append_tsg(f->g, runlist, - runlist_entry, entries_left, tsg); - if (entries == RUNLIST_APPEND_FAILURE) { - return RUNLIST_APPEND_FAILURE; - } - count += entries; - } - - return count; -} - -static u32 nvgpu_runlist_append_low(struct fifo_gk20a *f, - struct fifo_runlist_info_gk20a *runlist, - u32 **runlist_entry, - u32 *entries_left) -{ - u32 count = 0; - unsigned long tsgid; - - nvgpu_log_fn(f->g, " "); - - for_each_set_bit(tsgid, runlist->active_tsgs, f->num_channels) { - struct tsg_gk20a *tsg = &f->tsg[tsgid]; - u32 entries; - - if (tsg->interleave_level != - NVGPU_FIFO_RUNLIST_INTERLEAVE_LEVEL_LOW) { - continue; - } - - /* The medium level starts with the highs, if any. */ - - entries = nvgpu_runlist_append_med(f, runlist, - runlist_entry, entries_left); - if (entries == RUNLIST_APPEND_FAILURE) { - return RUNLIST_APPEND_FAILURE; - } - count += entries; - - entries = nvgpu_runlist_append_hi(f, runlist, - runlist_entry, entries_left); - if (entries == RUNLIST_APPEND_FAILURE) { - return RUNLIST_APPEND_FAILURE; - } - count += entries; - - entries = nvgpu_runlist_append_tsg(f->g, runlist, - runlist_entry, entries_left, tsg); - if (entries == RUNLIST_APPEND_FAILURE) { - return RUNLIST_APPEND_FAILURE; - } - count += entries; - } - - if (count == 0U) { - /* - * No transitions to fill with higher levels, so add - * the next level once. If that's empty too, we have only - * LEVEL_HIGH jobs. - */ - count = nvgpu_runlist_append_med(f, runlist, - runlist_entry, entries_left); - if (count == 0U) { - count = nvgpu_runlist_append_hi(f, runlist, - runlist_entry, entries_left); - } - } - - return count; -} - -static u32 nvgpu_runlist_append_flat(struct fifo_gk20a *f, - struct fifo_runlist_info_gk20a *runlist, - u32 **runlist_entry, - u32 *entries_left) -{ - u32 count = 0, entries, i; - - nvgpu_log_fn(f->g, " "); - - /* Group by priority but don't interleave. High comes first. */ - - for (i = 0; i < NVGPU_FIFO_RUNLIST_INTERLEAVE_NUM_LEVELS; i++) { - u32 level = NVGPU_FIFO_RUNLIST_INTERLEAVE_LEVEL_HIGH - i; - - entries = nvgpu_runlist_append_prio(f, runlist, runlist_entry, - entries_left, level); - if (entries == RUNLIST_APPEND_FAILURE) { - return RUNLIST_APPEND_FAILURE; - } - count += entries; - } - - return count; -} - -u32 nvgpu_runlist_construct_locked(struct fifo_gk20a *f, - struct fifo_runlist_info_gk20a *runlist, - u32 buf_id, - u32 max_entries) -{ - u32 *runlist_entry_base = runlist->mem[buf_id].cpu_va; - - nvgpu_log_fn(f->g, " "); - - /* - * The entry pointer and capacity counter that live on the stack here - * keep track of the current position and the remaining space when tsg - * and channel entries are ultimately appended. - */ - if (f->g->runlist_interleave) { - return nvgpu_runlist_append_low(f, runlist, - &runlist_entry_base, &max_entries); - } else { - return nvgpu_runlist_append_flat(f, runlist, - &runlist_entry_base, &max_entries); - } -} - int gk20a_fifo_set_runlist_interleave(struct gk20a *g, u32 id, u32 runlist_id, @@ -3242,131 +2851,6 @@ void gk20a_fifo_runlist_hw_submit(struct gk20a *g, u32 runlist_id, nvgpu_spinlock_release(&g->fifo.runlist_submit_lock); } -int gk20a_fifo_update_runlist_locked(struct gk20a *g, u32 runlist_id, - u32 chid, bool add, - bool wait_for_finish) -{ - int ret = 0; - struct fifo_gk20a *f = &g->fifo; - struct fifo_runlist_info_gk20a *runlist = NULL; - u64 runlist_iova; - u32 new_buf; - struct channel_gk20a *ch = NULL; - struct tsg_gk20a *tsg = NULL; - - runlist = &f->runlist_info[runlist_id]; - - /* valid channel, add/remove it from active list. - Otherwise, keep active list untouched for suspend/resume. */ - if (chid != FIFO_INVAL_CHANNEL_ID) { - ch = &f->channel[chid]; - if (gk20a_is_channel_marked_as_tsg(ch)) { - tsg = &f->tsg[ch->tsgid]; - } - - if (add) { - if (test_and_set_bit(chid, - runlist->active_channels)) { - return 0; - } - if ((tsg != NULL) && (++tsg->num_active_channels != 0U)) { - set_bit((int)f->channel[chid].tsgid, - runlist->active_tsgs); - } - } else { - if (!test_and_clear_bit(chid, - runlist->active_channels)) { - return 0; - } - if ((tsg != NULL) && - (--tsg->num_active_channels == 0U)) { - clear_bit((int)f->channel[chid].tsgid, - runlist->active_tsgs); - } - } - } - - /* There just 2 buffers */ - new_buf = runlist->cur_buffer == 0U ? 1U : 0U; - - runlist_iova = nvgpu_mem_get_addr(g, &runlist->mem[new_buf]); - - nvgpu_log_info(g, "runlist_id : %d, switch to new buffer 0x%16llx", - runlist_id, (u64)runlist_iova); - - if (runlist_iova == 0ULL) { - ret = -EINVAL; - goto clean_up; - } - - if (chid != FIFO_INVAL_CHANNEL_ID || /* add/remove a valid channel */ - add /* resume to add all channels back */) { - u32 num_entries; - - num_entries = nvgpu_runlist_construct_locked(f, - runlist, - new_buf, - f->num_runlist_entries); - if (num_entries == RUNLIST_APPEND_FAILURE) { - ret = -E2BIG; - goto clean_up; - } - runlist->count = num_entries; - WARN_ON(runlist->count > f->num_runlist_entries); - } else { - /* suspend to remove all channels */ - runlist->count = 0; - } - - g->ops.fifo.runlist_hw_submit(g, runlist_id, runlist->count, new_buf); - - if (wait_for_finish) { - ret = g->ops.fifo.runlist_wait_pending(g, runlist_id); - - if (ret == -ETIMEDOUT) { - nvgpu_err(g, "runlist %d update timeout", runlist_id); - /* trigger runlist update timeout recovery */ - return ret; - - } else if (ret == -EINTR) { - nvgpu_err(g, "runlist update interrupted"); - } - } - - runlist->cur_buffer = new_buf; - -clean_up: - return ret; -} - -int gk20a_fifo_update_runlist_ids(struct gk20a *g, u32 runlist_ids, u32 chid, - bool add, bool wait_for_finish) -{ - int ret = -EINVAL; - unsigned long runlist_id = 0; - int errcode; - unsigned long ulong_runlist_ids = (unsigned long)runlist_ids; - - if (g == NULL) { - goto end; - } - - ret = 0; - for_each_set_bit(runlist_id, &ulong_runlist_ids, 32U) { - /* Capture the last failure error code */ - errcode = g->ops.fifo.update_runlist(g, (u32)runlist_id, chid, - add, wait_for_finish); - if (errcode != 0) { - nvgpu_err(g, - "failed to update_runlist %lu %d", - runlist_id, errcode); - ret = errcode; - } - } -end: - return ret; -} - /* trigger host preempt of GR pending load ctx if that ctx is not for ch */ int gk20a_fifo_reschedule_preempt_next(struct channel_gk20a *ch, bool wait_preempt) @@ -3433,79 +2917,6 @@ int gk20a_fifo_reschedule_runlist(struct channel_gk20a *ch, bool preempt_next) return nvgpu_fifo_reschedule_runlist(ch, preempt_next, true); } -/* trigger host to expire current timeslice and reschedule runlist from front */ -int nvgpu_fifo_reschedule_runlist(struct channel_gk20a *ch, bool preempt_next, - bool wait_preempt) -{ - struct gk20a *g = ch->g; - struct fifo_runlist_info_gk20a *runlist; - u32 token = PMU_INVALID_MUTEX_OWNER_ID; - int mutex_ret; - int ret = 0; - - runlist = &g->fifo.runlist_info[ch->runlist_id]; - if (nvgpu_mutex_tryacquire(&runlist->runlist_lock) == 0) { - return -EBUSY; - } - - mutex_ret = nvgpu_pmu_mutex_acquire( - &g->pmu, PMU_MUTEX_ID_FIFO, &token); - - g->ops.fifo.runlist_hw_submit( - g, ch->runlist_id, runlist->count, runlist->cur_buffer); - - if (preempt_next) { - g->ops.fifo.reschedule_preempt_next_locked(ch, wait_preempt); - } - - gk20a_fifo_runlist_wait_pending(g, ch->runlist_id); - - if (mutex_ret == 0) { - nvgpu_pmu_mutex_release( - &g->pmu, PMU_MUTEX_ID_FIFO, &token); - } - nvgpu_mutex_release(&runlist->runlist_lock); - - return ret; -} - -/* add/remove a channel from runlist - special cases below: runlist->active_channels will NOT be changed. - (chid == ~0 && !add) means remove all active channels from runlist. - (chid == ~0 && add) means restore all active channels on runlist. */ -int gk20a_fifo_update_runlist(struct gk20a *g, u32 runlist_id, u32 chid, - bool add, bool wait_for_finish) -{ - struct fifo_runlist_info_gk20a *runlist = NULL; - struct fifo_gk20a *f = &g->fifo; - u32 token = PMU_INVALID_MUTEX_OWNER_ID; - int mutex_ret; - int ret = 0; - - nvgpu_log_fn(g, " "); - - runlist = &f->runlist_info[runlist_id]; - - nvgpu_mutex_acquire(&runlist->runlist_lock); - - mutex_ret = nvgpu_pmu_mutex_acquire(&g->pmu, PMU_MUTEX_ID_FIFO, &token); - - ret = gk20a_fifo_update_runlist_locked(g, runlist_id, chid, add, - wait_for_finish); - - if (mutex_ret == 0) { - nvgpu_pmu_mutex_release(&g->pmu, PMU_MUTEX_ID_FIFO, &token); - } - - nvgpu_mutex_release(&runlist->runlist_lock); - - if (ret == -ETIMEDOUT) { - gk20a_fifo_runlist_reset_engines(g, runlist_id); - } - - return ret; -} - int gk20a_fifo_suspend(struct gk20a *g) { nvgpu_log_fn(g, " "); @@ -4201,31 +3612,6 @@ u32 gk20a_fifo_pbdma_acquire_val(u64 timeout) return val; } -const char *gk20a_fifo_interleave_level_name(u32 interleave_level) -{ - const char *ret_string = NULL; - - switch (interleave_level) { - case NVGPU_FIFO_RUNLIST_INTERLEAVE_LEVEL_LOW: - ret_string = "LOW"; - break; - - case NVGPU_FIFO_RUNLIST_INTERLEAVE_LEVEL_MEDIUM: - ret_string = "MEDIUM"; - break; - - case NVGPU_FIFO_RUNLIST_INTERLEAVE_LEVEL_HIGH: - ret_string = "HIGH"; - break; - - default: - ret_string = "?"; - break; - } - - return ret_string; -} - u32 gk20a_fifo_get_sema_wait_cmd_size(void) { return 8; diff --git a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.h b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.h index 677efb322..a7680c5e3 100644 --- a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.h +++ b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.h @@ -266,18 +266,9 @@ void gk20a_fifo_disable_tsg_sched(struct gk20a *g, struct tsg_gk20a *tsg); u32 gk20a_fifo_engines_on_ch(struct gk20a *g, u32 chid); -int gk20a_fifo_reschedule_runlist(struct channel_gk20a *ch, bool preempt_next); -int nvgpu_fifo_reschedule_runlist(struct channel_gk20a *ch, bool preempt_next, - bool wait_preempt); int gk20a_fifo_reschedule_preempt_next(struct channel_gk20a *ch, bool wait_preempt); -int gk20a_fifo_update_runlist(struct gk20a *g, u32 runlist_id, u32 chid, - bool add, bool wait_for_finish); - -int gk20a_fifo_update_runlist_locked(struct gk20a *g, u32 runlist_id, - u32 chid, bool add, - bool wait_for_finish); int gk20a_fifo_suspend(struct gk20a *g); bool gk20a_fifo_mmu_fault_pending(struct gk20a *g); @@ -312,16 +303,12 @@ int gk20a_fifo_set_runlist_interleave(struct gk20a *g, u32 new_level); int gk20a_fifo_tsg_set_timeslice(struct tsg_gk20a *tsg, u32 timeslice); -const char *gk20a_fifo_interleave_level_name(u32 interleave_level); - enum fifo_engine gk20a_fifo_engine_enum_from_type(struct gk20a *g, u32 engine_type); u32 gk20a_fifo_get_engine_ids(struct gk20a *g, u32 engine_id[], u32 engine_id_sz, enum fifo_engine engine_enum); -void gk20a_fifo_delete_runlist(struct fifo_gk20a *f); - struct fifo_engine_info_gk20a *gk20a_fifo_get_engine_info(struct gk20a *g, u32 engine_id); @@ -339,9 +326,6 @@ u32 gk20a_fifo_get_gr_runlist_id(struct gk20a *g); bool gk20a_fifo_is_valid_runlist_id(struct gk20a *g, u32 runlist_id); -int gk20a_fifo_update_runlist_ids(struct gk20a *g, u32 runlist_ids, u32 chid, - bool add, bool wait_for_finish); - void gk20a_get_tsg_runlist_entry(struct tsg_gk20a *tsg, u32 *runlist); void gk20a_get_ch_runlist_entry(struct channel_gk20a *ch, u32 *runlist); void gk20a_fifo_set_runlist_state(struct gk20a *g, u32 runlists_mask, @@ -414,11 +398,6 @@ int gk20a_fifo_setup_userd(struct channel_gk20a *c); u32 gk20a_fifo_pbdma_acquire_val(u64 timeout); -#define RUNLIST_APPEND_FAILURE 0xffffffffU -u32 nvgpu_runlist_construct_locked(struct fifo_gk20a *f, - struct fifo_runlist_info_gk20a *runlist, - u32 buf_id, - u32 max_entries); void gk20a_fifo_runlist_hw_submit(struct gk20a *g, u32 runlist_id, u32 count, u32 buffer_index); u32 gk20a_fifo_runlist_busy_engines(struct gk20a *g, u32 runlist_id); diff --git a/drivers/gpu/nvgpu/gm20b/hal_gm20b.c b/drivers/gpu/nvgpu/gm20b/hal_gm20b.c index e8358f588..59e40de04 100644 --- a/drivers/gpu/nvgpu/gm20b/hal_gm20b.c +++ b/drivers/gpu/nvgpu/gm20b/hal_gm20b.c @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include diff --git a/drivers/gpu/nvgpu/gp10b/hal_gp10b.c b/drivers/gpu/nvgpu/gp10b/hal_gp10b.c index 94c87bb27..d91738b17 100644 --- a/drivers/gpu/nvgpu/gp10b/hal_gp10b.c +++ b/drivers/gpu/nvgpu/gp10b/hal_gp10b.c @@ -30,6 +30,7 @@ #include #include #include +#include #include #include #include diff --git a/drivers/gpu/nvgpu/gv100/hal_gv100.c b/drivers/gpu/nvgpu/gv100/hal_gv100.c index f50794478..88fe43a3f 100644 --- a/drivers/gpu/nvgpu/gv100/hal_gv100.c +++ b/drivers/gpu/nvgpu/gv100/hal_gv100.c @@ -120,6 +120,7 @@ #include #include #include +#include #include #include diff --git a/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c b/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c index ecf6ef8fe..16486bab8 100644 --- a/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c +++ b/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c @@ -1,7 +1,7 @@ /* * GV11B fifo * - * Copyright (c) 2015-2018, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2015-2019, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -41,6 +41,7 @@ #include #include #include +#include #include #include #include diff --git a/drivers/gpu/nvgpu/gv11b/hal_gv11b.c b/drivers/gpu/nvgpu/gv11b/hal_gv11b.c index 821eca2fa..19d298470 100644 --- a/drivers/gpu/nvgpu/gv11b/hal_gv11b.c +++ b/drivers/gpu/nvgpu/gv11b/hal_gv11b.c @@ -99,6 +99,7 @@ #include #include #include +#include #include #include diff --git a/drivers/gpu/nvgpu/include/nvgpu/runlist.h b/drivers/gpu/nvgpu/include/nvgpu/runlist.h new file mode 100644 index 000000000..47d5a8a97 --- /dev/null +++ b/drivers/gpu/nvgpu/include/nvgpu/runlist.h @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef NVGPU_RUNLIST_H +#define NVGPU_RUNLIST_H + +#include + +struct gk20a; +struct fifo_runlist_info_gk20a; +struct tsg_gk20a; +struct fifo_gk20a; +struct channel_gk20a; + +#define RUNLIST_APPEND_FAILURE 0xffffffffU +u32 nvgpu_runlist_construct_locked(struct fifo_gk20a *f, + struct fifo_runlist_info_gk20a *runlist, + u32 buf_id, + u32 max_entries); +int gk20a_fifo_update_runlist_locked(struct gk20a *g, u32 runlist_id, + u32 chid, bool add, + bool wait_for_finish); + +int gk20a_fifo_reschedule_runlist(struct channel_gk20a *ch, bool preempt_next); +int nvgpu_fifo_reschedule_runlist(struct channel_gk20a *ch, bool preempt_next, + bool wait_preempt); + +int gk20a_fifo_update_runlist(struct gk20a *g, u32 runlist_id, u32 chid, + bool add, bool wait_for_finish); +int gk20a_fifo_update_runlist_ids(struct gk20a *g, u32 runlist_ids, u32 chid, + bool add, bool wait_for_finish); + +const char *gk20a_fifo_interleave_level_name(u32 interleave_level); + +void gk20a_fifo_delete_runlist(struct fifo_gk20a *f); +int nvgpu_init_runlist(struct gk20a *g, struct fifo_gk20a *f); + +#endif /* NVGPU_RUNLIST_H */ diff --git a/drivers/gpu/nvgpu/os/linux/ioctl_channel.c b/drivers/gpu/nvgpu/os/linux/ioctl_channel.c index 318aeeef6..eb17238ba 100644 --- a/drivers/gpu/nvgpu/os/linux/ioctl_channel.c +++ b/drivers/gpu/nvgpu/os/linux/ioctl_channel.c @@ -1,7 +1,7 @@ /* * GK20A Graphics channel * - * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2011-2019, NVIDIA CORPORATION. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, @@ -38,6 +38,7 @@ #include #include #include +#include #include "gk20a/fence_gk20a.h" diff --git a/drivers/gpu/nvgpu/tu104/hal_tu104.c b/drivers/gpu/nvgpu/tu104/hal_tu104.c index f8815e74d..8ee425075 100644 --- a/drivers/gpu/nvgpu/tu104/hal_tu104.c +++ b/drivers/gpu/nvgpu/tu104/hal_tu104.c @@ -135,6 +135,7 @@ #include #include #include +#include #include #include diff --git a/drivers/gpu/nvgpu/vgpu/fifo_vgpu.c b/drivers/gpu/nvgpu/vgpu/fifo_vgpu.c index 209d47c1c..a3dd1aad5 100644 --- a/drivers/gpu/nvgpu/vgpu/fifo_vgpu.c +++ b/drivers/gpu/nvgpu/vgpu/fifo_vgpu.c @@ -1,7 +1,7 @@ /* * Virtualized GPU Fifo * - * Copyright (c) 2014-2018, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2014-2019, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -35,6 +35,7 @@ #include #include #include +#include #include #include diff --git a/userspace/units/fifo/runlist/nvgpu-runlist.c b/userspace/units/fifo/runlist/nvgpu-runlist.c index 2dea03fc2..a82c0ffb9 100644 --- a/userspace/units/fifo/runlist/nvgpu-runlist.c +++ b/userspace/units/fifo/runlist/nvgpu-runlist.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -26,6 +26,7 @@ #include #include +#include #include static void setup_fifo(struct gk20a *g, unsigned long *tsg_map,