diff --git a/arch/nvgpu-common.yaml b/arch/nvgpu-common.yaml index 435cdd4a0..4f7886e34 100644 --- a/arch/nvgpu-common.yaml +++ b/arch/nvgpu-common.yaml @@ -172,6 +172,11 @@ ptimer: include/nvgpu/gops_ptimer.h ] deps: +sched: + safe: no + owner: Thomas F + sources: [ include/nvgpu/sched.h ] + semaphore: safe: no owner: Alex W diff --git a/drivers/gpu/nvgpu/include/nvgpu/gk20a.h b/drivers/gpu/nvgpu/include/nvgpu/gk20a.h index 129cbc43f..b458f6b82 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/gk20a.h +++ b/drivers/gpu/nvgpu/include/nvgpu/gk20a.h @@ -137,6 +137,7 @@ enum nvgpu_unit; #include #include #include +#include #include #include @@ -736,6 +737,9 @@ struct gk20a { struct nvgpu_ecc ecc; struct pmgr_pmupstate *pmgr_pmu; struct nvgpu_sec2 sec2; +#ifdef CONFIG_NVGPU_CHANNEL_TSG_SCHEDULING + struct nvgpu_sched_ctrl sched_ctrl; +#endif #ifdef CONFIG_DEBUG_FS struct railgate_stats pstats; diff --git a/drivers/gpu/nvgpu/include/nvgpu/sched.h b/drivers/gpu/nvgpu/include/nvgpu/sched.h new file mode 100644 index 000000000..ca4cd7bec --- /dev/null +++ b/drivers/gpu/nvgpu/include/nvgpu/sched.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ +#ifndef NVGPU_SCHED_COMMON_H +#define NVGPU_SCHED_COMMON_H + +#include +#include +#include + +struct nvgpu_sched_ctrl { + struct nvgpu_mutex control_lock; + bool control_locked; + bool sw_ready; + struct nvgpu_mutex status_lock; + struct nvgpu_mutex busy_lock; + + u64 status; + + size_t bitmap_size; + u64 *active_tsg_bitmap; + u64 *recent_tsg_bitmap; + u64 *ref_tsg_bitmap; + + struct nvgpu_cond readout_wq; +}; + +#endif /* NVGPU_SCHED_COMMON_H */ diff --git a/drivers/gpu/nvgpu/os/linux/debug_sched.c b/drivers/gpu/nvgpu/os/linux/debug_sched.c index 2b6669cb3..1d2720906 100644 --- a/drivers/gpu/nvgpu/os/linux/debug_sched.c +++ b/drivers/gpu/nvgpu/os/linux/debug_sched.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2017 NVIDIA Corporation. All rights reserved. + * Copyright (C) 2017-2020 NVIDIA Corporation. All rights reserved. * * This software is licensed under the terms of the GNU General Public * License version 2, as published by the Free Software Foundation, and @@ -22,8 +22,7 @@ static int gk20a_sched_debugfs_show(struct seq_file *s, void *unused) { struct gk20a *g = s->private; - struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); - struct gk20a_sched_ctrl *sched = &l->sched_ctrl; + struct nvgpu_sched_ctrl *sched = &g->sched_ctrl; bool sched_busy = true; int n = sched->bitmap_size / sizeof(u64); diff --git a/drivers/gpu/nvgpu/os/linux/ioctl_tsg.c b/drivers/gpu/nvgpu/os/linux/ioctl_tsg.c index 8a3d40172..e455f38d6 100644 --- a/drivers/gpu/nvgpu/os/linux/ioctl_tsg.c +++ b/drivers/gpu/nvgpu/os/linux/ioctl_tsg.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2019, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2014-2020, NVIDIA CORPORATION. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, @@ -62,8 +62,7 @@ static int nvgpu_tsg_bind_channel_fd(struct nvgpu_tsg *tsg, int ch_fd) static int gk20a_tsg_ioctl_bind_channel_ex(struct gk20a *g, struct nvgpu_tsg *tsg, struct nvgpu_tsg_bind_channel_ex_args *arg) { - struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); - struct gk20a_sched_ctrl *sched = &l->sched_ctrl; + struct nvgpu_sched_ctrl *sched = &g->sched_ctrl; struct nvgpu_channel *ch; struct nvgpu_gr_config *gr_config = nvgpu_gr_get_config_ptr(g); int err = 0; @@ -498,8 +497,7 @@ int nvgpu_ioctl_tsg_dev_release(struct inode *inode, struct file *filp) static int gk20a_tsg_ioctl_set_runlist_interleave(struct gk20a *g, struct nvgpu_tsg *tsg, struct nvgpu_runlist_interleave_args *arg) { - struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); - struct gk20a_sched_ctrl *sched = &l->sched_ctrl; + struct nvgpu_sched_ctrl *sched = &g->sched_ctrl; u32 level = arg->level; int err; @@ -528,8 +526,7 @@ done: static int gk20a_tsg_ioctl_set_timeslice(struct gk20a *g, struct nvgpu_tsg *tsg, struct nvgpu_timeslice_args *arg) { - struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); - struct gk20a_sched_ctrl *sched = &l->sched_ctrl; + struct nvgpu_sched_ctrl *sched = &g->sched_ctrl; int err; nvgpu_log(g, gpu_dbg_fn | gpu_dbg_sched, "tsgid=%u", tsg->tsgid); diff --git a/drivers/gpu/nvgpu/os/linux/os_linux.h b/drivers/gpu/nvgpu/os/linux/os_linux.h index bb6a926c4..6f975632e 100644 --- a/drivers/gpu/nvgpu/os/linux/os_linux.h +++ b/drivers/gpu/nvgpu/os/linux/os_linux.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, @@ -173,8 +173,6 @@ struct nvgpu_os_linux { struct rw_semaphore busy_lock; - struct gk20a_sched_ctrl sched_ctrl; - bool init_done; }; diff --git a/drivers/gpu/nvgpu/os/linux/sched.c b/drivers/gpu/nvgpu/os/linux/sched.c index d8224859a..225730df6 100644 --- a/drivers/gpu/nvgpu/os/linux/sched.c +++ b/drivers/gpu/nvgpu/os/linux/sched.c @@ -35,8 +35,8 @@ ssize_t gk20a_sched_dev_read(struct file *filp, char __user *buf, size_t size, loff_t *off) { - struct gk20a_sched_ctrl *sched = filp->private_data; - struct gk20a *g = sched->g; + struct gk20a *g = filp->private_data; + struct nvgpu_sched_ctrl *sched = &g->sched_ctrl; struct nvgpu_sched_event_arg event = { 0 }; int err; @@ -76,8 +76,8 @@ ssize_t gk20a_sched_dev_read(struct file *filp, char __user *buf, unsigned int gk20a_sched_dev_poll(struct file *filp, poll_table *wait) { - struct gk20a_sched_ctrl *sched = filp->private_data; - struct gk20a *g = sched->g; + struct gk20a *g = filp->private_data; + struct nvgpu_sched_ctrl *sched = &g->sched_ctrl; unsigned int mask = 0; nvgpu_log(g, gpu_dbg_fn | gpu_dbg_sched, " "); @@ -91,10 +91,10 @@ unsigned int gk20a_sched_dev_poll(struct file *filp, poll_table *wait) return mask; } -static int gk20a_sched_dev_ioctl_get_tsgs(struct gk20a_sched_ctrl *sched, +static int gk20a_sched_dev_ioctl_get_tsgs(struct gk20a *g, struct nvgpu_sched_get_tsgs_args *arg) { - struct gk20a *g = sched->g; + struct nvgpu_sched_ctrl *sched = &g->sched_ctrl; nvgpu_log(g, gpu_dbg_fn | gpu_dbg_sched, "size=%u buffer=%llx", arg->size, arg->buffer); @@ -115,10 +115,10 @@ static int gk20a_sched_dev_ioctl_get_tsgs(struct gk20a_sched_ctrl *sched, return 0; } -static int gk20a_sched_dev_ioctl_get_recent_tsgs(struct gk20a_sched_ctrl *sched, +static int gk20a_sched_dev_ioctl_get_recent_tsgs(struct gk20a *g, struct nvgpu_sched_get_tsgs_args *arg) { - struct gk20a *g = sched->g; + struct nvgpu_sched_ctrl *sched = &g->sched_ctrl; nvgpu_log(g, gpu_dbg_fn | gpu_dbg_sched, "size=%u buffer=%llx", arg->size, arg->buffer); @@ -141,10 +141,10 @@ static int gk20a_sched_dev_ioctl_get_recent_tsgs(struct gk20a_sched_ctrl *sched, return 0; } -static int gk20a_sched_dev_ioctl_get_tsgs_by_pid(struct gk20a_sched_ctrl *sched, +static int gk20a_sched_dev_ioctl_get_tsgs_by_pid(struct gk20a *g, struct nvgpu_sched_get_tsgs_by_pid_args *arg) { - struct gk20a *g = sched->g; + struct nvgpu_sched_ctrl *sched = &g->sched_ctrl; struct nvgpu_fifo *f = &g->fifo; struct nvgpu_tsg *tsg; u64 *bitmap; @@ -161,7 +161,7 @@ static int gk20a_sched_dev_ioctl_get_tsgs_by_pid(struct gk20a_sched_ctrl *sched, return -ENOSPC; } - bitmap = nvgpu_kzalloc(sched->g, sched->bitmap_size); + bitmap = nvgpu_kzalloc(g, sched->bitmap_size); if (!bitmap) return -ENOMEM; @@ -180,15 +180,14 @@ static int gk20a_sched_dev_ioctl_get_tsgs_by_pid(struct gk20a_sched_ctrl *sched, bitmap, sched->bitmap_size)) err = -EFAULT; - nvgpu_kfree(sched->g, bitmap); + nvgpu_kfree(g, bitmap); return err; } -static int gk20a_sched_dev_ioctl_get_params(struct gk20a_sched_ctrl *sched, +static int gk20a_sched_dev_ioctl_get_params(struct gk20a *g, struct nvgpu_sched_tsg_get_params_args *arg) { - struct gk20a *g = sched->g; struct nvgpu_fifo *f = &g->fifo; struct nvgpu_tsg *tsg; u32 tsgid = arg->tsgid; @@ -219,10 +218,9 @@ static int gk20a_sched_dev_ioctl_get_params(struct gk20a_sched_ctrl *sched, } static int gk20a_sched_dev_ioctl_tsg_set_timeslice( - struct gk20a_sched_ctrl *sched, + struct gk20a *g, struct nvgpu_sched_tsg_timeslice_args *arg) { - struct gk20a *g = sched->g; struct nvgpu_fifo *f = &g->fifo; struct nvgpu_tsg *tsg; u32 tsgid = arg->tsgid; @@ -254,10 +252,9 @@ done: } static int gk20a_sched_dev_ioctl_tsg_set_runlist_interleave( - struct gk20a_sched_ctrl *sched, + struct gk20a *g, struct nvgpu_sched_tsg_runlist_interleave_args *arg) { - struct gk20a *g = sched->g; struct nvgpu_fifo *f = &g->fifo; struct nvgpu_tsg *tsg; u32 tsgid = arg->tsgid; @@ -288,9 +285,9 @@ done: return err; } -static int gk20a_sched_dev_ioctl_lock_control(struct gk20a_sched_ctrl *sched) +static int gk20a_sched_dev_ioctl_lock_control(struct gk20a *g) { - struct gk20a *g = sched->g; + struct nvgpu_sched_ctrl *sched = &g->sched_ctrl; nvgpu_log(g, gpu_dbg_fn | gpu_dbg_sched, " "); @@ -300,9 +297,9 @@ static int gk20a_sched_dev_ioctl_lock_control(struct gk20a_sched_ctrl *sched) return 0; } -static int gk20a_sched_dev_ioctl_unlock_control(struct gk20a_sched_ctrl *sched) +static int gk20a_sched_dev_ioctl_unlock_control(struct gk20a *g) { - struct gk20a *g = sched->g; + struct nvgpu_sched_ctrl *sched = &g->sched_ctrl; nvgpu_log(g, gpu_dbg_fn | gpu_dbg_sched, " "); @@ -312,21 +309,19 @@ static int gk20a_sched_dev_ioctl_unlock_control(struct gk20a_sched_ctrl *sched) return 0; } -static int gk20a_sched_dev_ioctl_get_api_version(struct gk20a_sched_ctrl *sched, +static int gk20a_sched_dev_ioctl_get_api_version(struct gk20a *g, struct nvgpu_sched_api_version_args *args) { - struct gk20a *g = sched->g; - nvgpu_log(g, gpu_dbg_fn | gpu_dbg_sched, " "); args->version = NVGPU_SCHED_API_VERSION; return 0; } -static int gk20a_sched_dev_ioctl_get_tsg(struct gk20a_sched_ctrl *sched, +static int gk20a_sched_dev_ioctl_get_tsg(struct gk20a *g, struct nvgpu_sched_tsg_refcount_args *arg) { - struct gk20a *g = sched->g; + struct nvgpu_sched_ctrl *sched = &g->sched_ctrl; struct nvgpu_fifo *f = &g->fifo; struct nvgpu_tsg *tsg; u32 tsgid = arg->tsgid; @@ -360,10 +355,10 @@ static int gk20a_sched_dev_ioctl_get_tsg(struct gk20a_sched_ctrl *sched, return 0; } -static int gk20a_sched_dev_ioctl_put_tsg(struct gk20a_sched_ctrl *sched, +static int gk20a_sched_dev_ioctl_put_tsg(struct gk20a *g, struct nvgpu_sched_tsg_refcount_args *arg) { - struct gk20a *g = sched->g; + struct nvgpu_sched_ctrl *sched = &g->sched_ctrl; struct nvgpu_fifo *f = &g->fifo; struct nvgpu_tsg *tsg; u32 tsgid = arg->tsgid; @@ -395,13 +390,13 @@ int gk20a_sched_dev_open(struct inode *inode, struct file *filp) struct nvgpu_os_linux *l = container_of(inode->i_cdev, struct nvgpu_os_linux, sched.cdev); struct gk20a *g; - struct gk20a_sched_ctrl *sched; + struct nvgpu_sched_ctrl *sched; int err = 0; g = nvgpu_get(&l->g); if (!g) return -ENODEV; - sched = &l->sched_ctrl; + sched = &g->sched_ctrl; nvgpu_log(g, gpu_dbg_fn | gpu_dbg_sched, "g=%p", g); @@ -423,7 +418,7 @@ int gk20a_sched_dev_open(struct inode *inode, struct file *filp) sched->bitmap_size); (void) memset(sched->ref_tsg_bitmap, 0, sched->bitmap_size); - filp->private_data = sched; + filp->private_data = g; nvgpu_log(g, gpu_dbg_sched, "filp=%p sched=%p", filp, sched); free_ref: @@ -435,8 +430,7 @@ free_ref: long gk20a_sched_dev_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) { - struct gk20a_sched_ctrl *sched = filp->private_data; - struct gk20a *g = sched->g; + struct gk20a *g = filp->private_data; u8 buf[NVGPU_CTXSW_IOCTL_MAX_ARG_SIZE]; int err = 0; @@ -457,45 +451,45 @@ long gk20a_sched_dev_ioctl(struct file *filp, unsigned int cmd, nvgpu_speculation_barrier(); switch (cmd) { case NVGPU_SCHED_IOCTL_GET_TSGS: - err = gk20a_sched_dev_ioctl_get_tsgs(sched, + err = gk20a_sched_dev_ioctl_get_tsgs(g, (struct nvgpu_sched_get_tsgs_args *)buf); break; case NVGPU_SCHED_IOCTL_GET_RECENT_TSGS: - err = gk20a_sched_dev_ioctl_get_recent_tsgs(sched, + err = gk20a_sched_dev_ioctl_get_recent_tsgs(g, (struct nvgpu_sched_get_tsgs_args *)buf); break; case NVGPU_SCHED_IOCTL_GET_TSGS_BY_PID: - err = gk20a_sched_dev_ioctl_get_tsgs_by_pid(sched, + err = gk20a_sched_dev_ioctl_get_tsgs_by_pid(g, (struct nvgpu_sched_get_tsgs_by_pid_args *)buf); break; case NVGPU_SCHED_IOCTL_TSG_GET_PARAMS: - err = gk20a_sched_dev_ioctl_get_params(sched, + err = gk20a_sched_dev_ioctl_get_params(g, (struct nvgpu_sched_tsg_get_params_args *)buf); break; case NVGPU_SCHED_IOCTL_TSG_SET_TIMESLICE: - err = gk20a_sched_dev_ioctl_tsg_set_timeslice(sched, + err = gk20a_sched_dev_ioctl_tsg_set_timeslice(g, (struct nvgpu_sched_tsg_timeslice_args *)buf); break; case NVGPU_SCHED_IOCTL_TSG_SET_RUNLIST_INTERLEAVE: - err = gk20a_sched_dev_ioctl_tsg_set_runlist_interleave(sched, + err = gk20a_sched_dev_ioctl_tsg_set_runlist_interleave(g, (struct nvgpu_sched_tsg_runlist_interleave_args *)buf); break; case NVGPU_SCHED_IOCTL_LOCK_CONTROL: - err = gk20a_sched_dev_ioctl_lock_control(sched); + err = gk20a_sched_dev_ioctl_lock_control(g); break; case NVGPU_SCHED_IOCTL_UNLOCK_CONTROL: - err = gk20a_sched_dev_ioctl_unlock_control(sched); + err = gk20a_sched_dev_ioctl_unlock_control(g); break; case NVGPU_SCHED_IOCTL_GET_API_VERSION: - err = gk20a_sched_dev_ioctl_get_api_version(sched, + err = gk20a_sched_dev_ioctl_get_api_version(g, (struct nvgpu_sched_api_version_args *)buf); break; case NVGPU_SCHED_IOCTL_GET_TSG: - err = gk20a_sched_dev_ioctl_get_tsg(sched, + err = gk20a_sched_dev_ioctl_get_tsg(g, (struct nvgpu_sched_tsg_refcount_args *)buf); break; case NVGPU_SCHED_IOCTL_PUT_TSG: - err = gk20a_sched_dev_ioctl_put_tsg(sched, + err = gk20a_sched_dev_ioctl_put_tsg(g, (struct nvgpu_sched_tsg_refcount_args *)buf); break; default: @@ -518,8 +512,8 @@ long gk20a_sched_dev_ioctl(struct file *filp, unsigned int cmd, int gk20a_sched_dev_release(struct inode *inode, struct file *filp) { - struct gk20a_sched_ctrl *sched = filp->private_data; - struct gk20a *g = sched->g; + struct gk20a *g = filp->private_data; + struct nvgpu_sched_ctrl *sched = &g->sched_ctrl; struct nvgpu_fifo *f = &g->fifo; struct nvgpu_tsg *tsg; unsigned int tsgid; @@ -546,8 +540,7 @@ int gk20a_sched_dev_release(struct inode *inode, struct file *filp) void gk20a_sched_ctrl_tsg_added(struct gk20a *g, struct nvgpu_tsg *tsg) { - struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); - struct gk20a_sched_ctrl *sched = &l->sched_ctrl; + struct nvgpu_sched_ctrl *sched = &g->sched_ctrl; int err; nvgpu_log(g, gpu_dbg_fn | gpu_dbg_sched, "tsgid=%u", tsg->tsgid); @@ -572,8 +565,7 @@ void gk20a_sched_ctrl_tsg_added(struct gk20a *g, struct nvgpu_tsg *tsg) void gk20a_sched_ctrl_tsg_removed(struct gk20a *g, struct nvgpu_tsg *tsg) { - struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); - struct gk20a_sched_ctrl *sched = &l->sched_ctrl; + struct nvgpu_sched_ctrl *sched = &g->sched_ctrl; nvgpu_log(g, gpu_dbg_fn | gpu_dbg_sched, "tsgid=%u", tsg->tsgid); @@ -595,15 +587,13 @@ void gk20a_sched_ctrl_tsg_removed(struct gk20a *g, struct nvgpu_tsg *tsg) int gk20a_sched_ctrl_init(struct gk20a *g) { - struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); - struct gk20a_sched_ctrl *sched = &l->sched_ctrl; + struct nvgpu_sched_ctrl *sched = &g->sched_ctrl; struct nvgpu_fifo *f = &g->fifo; int err; if (sched->sw_ready) return 0; - sched->g = g; sched->bitmap_size = round_up(f->num_channels, 64) / 8; sched->status = 0; @@ -646,8 +636,7 @@ free_active: void gk20a_sched_ctrl_cleanup(struct gk20a *g) { - struct nvgpu_os_linux *l = nvgpu_os_linux_from_gk20a(g); - struct gk20a_sched_ctrl *sched = &l->sched_ctrl; + struct nvgpu_sched_ctrl *sched = &g->sched_ctrl; nvgpu_kfree(g, sched->active_tsg_bitmap); nvgpu_kfree(g, sched->recent_tsg_bitmap); diff --git a/drivers/gpu/nvgpu/os/linux/sched.h b/drivers/gpu/nvgpu/os/linux/sched.h index 4c58a0d3d..9479dfbe3 100644 --- a/drivers/gpu/nvgpu/os/linux/sched.h +++ b/drivers/gpu/nvgpu/os/linux/sched.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2017, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2016-2020, NVIDIA CORPORATION. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, @@ -21,25 +21,6 @@ struct gpu_ops; struct nvgpu_tsg; struct poll_table_struct; -struct gk20a_sched_ctrl { - struct gk20a *g; - - struct nvgpu_mutex control_lock; - bool control_locked; - bool sw_ready; - struct nvgpu_mutex status_lock; - struct nvgpu_mutex busy_lock; - - u64 status; - - size_t bitmap_size; - u64 *active_tsg_bitmap; - u64 *recent_tsg_bitmap; - u64 *ref_tsg_bitmap; - - struct nvgpu_cond readout_wq; -}; - int gk20a_sched_dev_release(struct inode *inode, struct file *filp); int gk20a_sched_dev_open(struct inode *inode, struct file *filp); long gk20a_sched_dev_ioctl(struct file *, unsigned int, unsigned long);