Open source GPL/LGPL release

.gitignore

@@ -0,0 +1,101 @@
+#
+# NOTE! Don't add files that are generated in specific
+# subdirectories here. Add them in the ".gitignore" file
+# in that subdirectory instead.
+#
+# NOTE! Please use 'git ls-files -i --exclude-standard'
+# command after changing this file, to see if there are
+# any tracked files which get ignored after the change.
+#
+# Normal rules
+#
+.*
+*.o
+*.o.*
+*.a
+*.s
+*.ko
+*.so
+*.so.dbg
+*.mod.c
+*.i
+*.lst
+*.symtypes
+*.order
+*.elf
+*.bin
+*.gz
+*.bz2
+*.lzma
+*.xz
+*.lz4
+*.lzo
+*.patch
+*.gcno
+modules.builtin
+Module.symvers
+*.dwo
+
+#
+# Top-level generic files
+#
+/tags
+/TAGS
+/linux
+/vmlinux
+/vmlinuz
+/System.map
+/Module.markers
+
+#
+# Debian directory (make deb-pkg)
+#
+/debian/
+
+#
+# git files that we don't want to ignore even it they are dot-files
+#
+!.gitignore
+!.mailmap
+
+#
+# Generated include files
+#
+include/config
+include/generated
+arch/*/include/generated
+
+# stgit generated dirs
+patches-*
+
+# quilt's files
+patches
+series
+
+# cscope files
+cscope.*
+ncscope.*
+
+# gnu global files
+GPATH
+GRTAGS
+GSYMS
+GTAGS
+
+*.orig
+*~
+\#*#
+
+#
+# Leavings from module signing
+#
+extra_certificates
+signing_key.priv
+signing_key.x509
+x509.genkey
+
+# Kconfig presets
+all.config
+
+# Source files we generate from arch for doxygen.
+Doxyfile.sources.safety

Makefile.umbrella.tmk

@@ -0,0 +1,172 @@
+################################### tell Emacs this is a -*- makefile-gmake -*-
+#
+# Copyright (c) 2018-2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+#
+# tmake for SW Mobile
+#
+# Repository umbrella makefile fragment for "nvgpu"
+#
+###############################################################################
+
+#
+# Components common to all builds
+#
+NV_REPOSITORY_COMPONENTS :=
+
+ifneq ($(NV_BUILD_CONFIGURATION_OS_IS_INTEGRITY),1)
+
+ifeq ($(NV_BUILD_CONFIGURATION_OS_IS_QNX),1)
+ifeq ($(NV_BUILD_CONFIGURATION_IS_SAFETY),1)
+# On QNX, the unit tests are built only for safety profile.
+build_nvgpu_ut := 1
+endif
+else
+# On L4T, the unit tests are built with safety profile forced.
+build_nvgpu_ut := 1
+endif
+
+ifeq ($(build_nvgpu_ut),1)
+NV_REPOSITORY_COMPONENTS := libs/igpu
+NV_REPOSITORY_COMPONENTS += libs/dgpu
+NV_REPOSITORY_COMPONENTS += userspace
+NV_REPOSITORY_COMPONENTS += userspace/units/posix/bitops
+NV_REPOSITORY_COMPONENTS += userspace/units/posix/env
+NV_REPOSITORY_COMPONENTS += userspace/units/posix/fault-injection
+NV_REPOSITORY_COMPONENTS += userspace/units/posix/bug
+NV_REPOSITORY_COMPONENTS += userspace/units/posix/os_sched
+NV_REPOSITORY_COMPONENTS += userspace/units/posix/sizes
+NV_REPOSITORY_COMPONENTS += userspace/units/init
+NV_REPOSITORY_COMPONENTS += userspace/units/posix/log2
+NV_REPOSITORY_COMPONENTS += userspace/units/posix/thread
+NV_REPOSITORY_COMPONENTS += userspace/units/posix/cond
+NV_REPOSITORY_COMPONENTS += userspace/units/posix/timers
+NV_REPOSITORY_COMPONENTS += userspace/units/posix/kmem
+NV_REPOSITORY_COMPONENTS += userspace/units/posix/rwsem
+NV_REPOSITORY_COMPONENTS += userspace/units/posix/queue
+NV_REPOSITORY_COMPONENTS += userspace/units/posix/utils
+NV_REPOSITORY_COMPONENTS += userspace/units/posix/circ_buf
+NV_REPOSITORY_COMPONENTS += userspace/units/interface/bit-utils
+NV_REPOSITORY_COMPONENTS += userspace/units/interface/lock
+NV_REPOSITORY_COMPONENTS += userspace/units/interface/nvgpu_gk20a
+NV_REPOSITORY_COMPONENTS += userspace/units/interface/atomic
+NV_REPOSITORY_COMPONENTS += userspace/units/interface/rbtree
+NV_REPOSITORY_COMPONENTS += userspace/units/interface/static_analysis
+NV_REPOSITORY_COMPONENTS += userspace/units/interface/string
+NV_REPOSITORY_COMPONENTS += userspace/units/interface/worker
+NV_REPOSITORY_COMPONENTS += userspace/units/interface/kref
+NV_REPOSITORY_COMPONENTS += userspace/units/interface/list
+NV_REPOSITORY_COMPONENTS += userspace/units/bus
+NV_REPOSITORY_COMPONENTS += userspace/units/pramin
+NV_REPOSITORY_COMPONENTS += userspace/units/priv_ring
+NV_REPOSITORY_COMPONENTS += userspace/units/ptimer
+NV_REPOSITORY_COMPONENTS += userspace/units/mc
+NV_REPOSITORY_COMPONENTS += userspace/units/mm/nvgpu_sgt
+NV_REPOSITORY_COMPONENTS += userspace/units/mm/nvgpu_mem
+NV_REPOSITORY_COMPONENTS += userspace/units/mm/nvgpu_mem/dgpu
+NV_REPOSITORY_COMPONENTS += userspace/units/mm/allocators/buddy_allocator
+NV_REPOSITORY_COMPONENTS += userspace/units/mm/allocators/nvgpu_allocator
+NV_REPOSITORY_COMPONENTS += userspace/units/mm/allocators/bitmap_allocator
+NV_REPOSITORY_COMPONENTS += userspace/units/mm/allocators/page_allocator
+NV_REPOSITORY_COMPONENTS += userspace/units/mm/as
+NV_REPOSITORY_COMPONENTS += userspace/units/mm/dma
+NV_REPOSITORY_COMPONENTS += userspace/units/mm/gmmu/pd_cache
+NV_REPOSITORY_COMPONENTS += userspace/units/mm/gmmu/page_table
+NV_REPOSITORY_COMPONENTS += userspace/units/mm/hal/cache/flush_gk20a_fusa
+NV_REPOSITORY_COMPONENTS += userspace/units/mm/hal/cache/flush_gv11b_fusa
+NV_REPOSITORY_COMPONENTS += userspace/units/mm/hal/gmmu/gmmu_gk20a_fusa
+NV_REPOSITORY_COMPONENTS += userspace/units/mm/hal/gmmu/gmmu_gm20b_fusa
+NV_REPOSITORY_COMPONENTS += userspace/units/mm/hal/gmmu/gmmu_gp10b_fusa
+NV_REPOSITORY_COMPONENTS += userspace/units/mm/hal/gmmu/gmmu_gv11b_fusa
+NV_REPOSITORY_COMPONENTS += userspace/units/mm/hal/gp10b_fusa
+NV_REPOSITORY_COMPONENTS += userspace/units/mm/hal/gv11b_fusa
+NV_REPOSITORY_COMPONENTS += userspace/units/mm/hal/mmu_fault/gv11b_fusa
+NV_REPOSITORY_COMPONENTS += userspace/units/mm/mm
+NV_REPOSITORY_COMPONENTS += userspace/units/mm/page_table_faults
+NV_REPOSITORY_COMPONENTS += userspace/units/mm/vm
+NV_REPOSITORY_COMPONENTS += userspace/units/netlist
+NV_REPOSITORY_COMPONENTS += userspace/units/fb
+NV_REPOSITORY_COMPONENTS += userspace/units/fbp
+NV_REPOSITORY_COMPONENTS += userspace/units/fifo
+NV_REPOSITORY_COMPONENTS += userspace/units/fifo/channel
+NV_REPOSITORY_COMPONENTS += userspace/units/fifo/channel/gk20a
+NV_REPOSITORY_COMPONENTS += userspace/units/fifo/channel/gm20b
+NV_REPOSITORY_COMPONENTS += userspace/units/fifo/channel/gv11b
+NV_REPOSITORY_COMPONENTS += userspace/units/fifo/ctxsw_timeout/gv11b
+NV_REPOSITORY_COMPONENTS += userspace/units/fifo/engine
+NV_REPOSITORY_COMPONENTS += userspace/units/fifo/engine/gm20b
+NV_REPOSITORY_COMPONENTS += userspace/units/fifo/engine/gp10b
+NV_REPOSITORY_COMPONENTS += userspace/units/fifo/engine/gv100
+NV_REPOSITORY_COMPONENTS += userspace/units/fifo/engine/gv11b
+NV_REPOSITORY_COMPONENTS += userspace/units/fifo/fifo
+NV_REPOSITORY_COMPONENTS += userspace/units/fifo/fifo/gk20a
+NV_REPOSITORY_COMPONENTS += userspace/units/fifo/fifo/gv11b
+NV_REPOSITORY_COMPONENTS += userspace/units/fifo/pbdma
+NV_REPOSITORY_COMPONENTS += userspace/units/fifo/pbdma/gv11b
+NV_REPOSITORY_COMPONENTS += userspace/units/fifo/pbdma/gm20b
+NV_REPOSITORY_COMPONENTS += userspace/units/fifo/pbdma/gp10b
+NV_REPOSITORY_COMPONENTS += userspace/units/fifo/preempt
+NV_REPOSITORY_COMPONENTS += userspace/units/fifo/preempt/gv11b
+NV_REPOSITORY_COMPONENTS += userspace/units/fifo/ramfc/gp10b
+NV_REPOSITORY_COMPONENTS += userspace/units/fifo/ramfc/gv11b
+NV_REPOSITORY_COMPONENTS += userspace/units/fifo/ramin/gk20a
+NV_REPOSITORY_COMPONENTS += userspace/units/fifo/ramin/gm20b
+NV_REPOSITORY_COMPONENTS += userspace/units/fifo/ramin/gv11b
+NV_REPOSITORY_COMPONENTS += userspace/units/fifo/runlist
+NV_REPOSITORY_COMPONENTS += userspace/units/fifo/runlist/gk20a
+NV_REPOSITORY_COMPONENTS += userspace/units/fifo/runlist/gv11b
+NV_REPOSITORY_COMPONENTS += userspace/units/fifo/tsg
+NV_REPOSITORY_COMPONENTS += userspace/units/fifo/tsg/gv11b
+NV_REPOSITORY_COMPONENTS += userspace/units/fifo/userd/gk20a
+NV_REPOSITORY_COMPONENTS += userspace/units/fifo/usermode/gv11b
+NV_REPOSITORY_COMPONENTS += userspace/units/fuse
+NV_REPOSITORY_COMPONENTS += userspace/units/ltc
+NV_REPOSITORY_COMPONENTS += userspace/units/enabled
+NV_REPOSITORY_COMPONENTS += userspace/units/falcon
+NV_REPOSITORY_COMPONENTS += userspace/units/falcon/falcon_tests
+NV_REPOSITORY_COMPONENTS += userspace/units/pmu
+NV_REPOSITORY_COMPONENTS += userspace/units/therm
+NV_REPOSITORY_COMPONENTS += userspace/units/top
+NV_REPOSITORY_COMPONENTS += userspace/units/class
+NV_REPOSITORY_COMPONENTS += userspace/units/gr
+NV_REPOSITORY_COMPONENTS += userspace/units/gr/falcon
+NV_REPOSITORY_COMPONENTS += userspace/units/gr/config
+NV_REPOSITORY_COMPONENTS += userspace/units/gr/init
+NV_REPOSITORY_COMPONENTS += userspace/units/gr/setup
+NV_REPOSITORY_COMPONENTS += userspace/units/gr/fs_state
+NV_REPOSITORY_COMPONENTS += userspace/units/gr/global_ctx
+NV_REPOSITORY_COMPONENTS += userspace/units/gr/ctx
+NV_REPOSITORY_COMPONENTS += userspace/units/gr/obj_ctx
+NV_REPOSITORY_COMPONENTS += userspace/units/gr/intr
+NV_REPOSITORY_COMPONENTS += userspace/units/acr
+NV_REPOSITORY_COMPONENTS += userspace/units/ce
+NV_REPOSITORY_COMPONENTS += userspace/units/cg
+NV_REPOSITORY_COMPONENTS += userspace/units/rc
+NV_REPOSITORY_COMPONENTS += userspace/units/sync
+NV_REPOSITORY_COMPONENTS += userspace/units/ecc
+NV_REPOSITORY_COMPONENTS += userspace/units/io
+endif
+endif
+
+# Local Variables:
+# indent-tabs-mode: t
+# tab-width: 8
+# End:
+# vi: set tabstop=8 noexpandtab:

NVIDIA-REVIEWERS

@@ -0,0 +1,65 @@
+This is a reviewers file that can be parsed by get_nv_reviewers.py that
+internally uses scripts/get_maintainer.pl from kernel.
+
+See the MAINTAINERS file in the Linux kernel source tree for details of the
+file format. The file format is defined by the upstream Linux kernel community,
+so don't modify it without upstreaming any changes to get_maintainer.pl.
+
+Descriptions of section entries (copied from MAINTAINERS):
+
+	P: Person (obsolete)
+	M: Mail patches to: FullName <address@domain>
+	L: Mailing list that is relevant to this area
+	B: NvBugs Module Name
+	W: Web-page with status/info
+	Q: Patchwork web based patch tracking system site
+	T: SCM tree type and location.  Type is one of: git, hg, quilt, stgit, topgit.
+	S: Status, one of the following:
+	   Supported:	Someone is actually paid to look after this.
+	   Maintained:	Someone actually looks after it.
+	   Odd Fixes:	It has a maintainer but they don't have time to do
+			much other than throw the odd patch in. See below..
+	   Orphan:	No current maintainer [but maybe you could take the
+			role as you write your new code].
+	   Obsolete:	Old code. Something tagged obsolete generally means
+			it has been replaced by a better system and you
+			should be using that.
+	F: Files and directories with wildcard patterns.
+	   A trailing slash includes all files and subdirectory files.
+	   F:	drivers/net/	all files in and below drivers/net
+	   F:	drivers/net/*	all files in drivers/net, but not below
+	   F:	*/net/*		all files in "any top level directory"/net
+	   One pattern per line.  Multiple F: lines acceptable.
+	X: Files and directories that are NOT maintained, same rules as F:
+	   Files exclusions are tested before file matches.
+	   Can be useful for excluding a specific subdirectory, for instance:
+	   F:	net/
+	   X:	net/ipv6/
+	   matches all files in and below net excluding net/ipv6/
+	K: Keyword perl extended regex pattern to match content in a
+	   patch or file.  For instance:
+	   K: of_get_profile
+	      matches patches or files that contain "of_get_profile"
+	   K: \b(printk|pr_(info|err))\b
+	      matches patches or files that contain one or more of the words
+	      printk, pr_info or pr_err
+	   One regex pattern per line.  Multiple K: lines acceptable.
+
+Note: For the hard of thinking, this list is meant to remain in alphabetical
+order. If you could add yourselves to it in alphabetical order that would be
+so much easier [Ed]
+
+Maintainers List (try to look for most precise areas first)
+
+----------------------------------------------------------------------
+GPU
+M:	Vijayakumar Subbu <vsubbu@nvidia.com>
+M:	Terje Bergstrom <tbergstrom@nvidia.com>
+L:	sw-mobile-nvgpu-core@exchange.nvidia.com
+B:	Mobile_Android_Kernel <keyword = nvgpu-tegra>
+B:	Mobile_Linux_Kernel <keyword = nvgpu-tegra>
+B:	Embedded Resource Manager - Linux <keyword = nvgpu-tegra>
+S:	Supported
+F:	drivers/gpu/nvgpu/*
+F:	include/*
+F:	../../gpu-firmware-private/

arch/nvgpu-gpu_hw.yaml

@@ -0,0 +1,224 @@
+# Copyright (c) 2019, NVIDIA CORPORATION.  All Rights Reserved.
+#
+# Define meta elements and units for describing GPU HW interactions in
+# nvgpu.
+#
+
+headers:
+  safe: yes
+  owner: Terje B
+  sources: [ include/nvgpu/hw/gk20a/hw_bus_gk20a.h,
+             include/nvgpu/hw/gk20a/hw_ccsr_gk20a.h,
+             include/nvgpu/hw/gk20a/hw_ce2_gk20a.h,
+             include/nvgpu/hw/gk20a/hw_ctxsw_prog_gk20a.h,
+             include/nvgpu/hw/gk20a/hw_falcon_gk20a.h,
+             include/nvgpu/hw/gk20a/hw_fb_gk20a.h,
+             include/nvgpu/hw/gk20a/hw_fifo_gk20a.h,
+             include/nvgpu/hw/gk20a/hw_flush_gk20a.h,
+             include/nvgpu/hw/gk20a/hw_gmmu_gk20a.h,
+             include/nvgpu/hw/gk20a/hw_gr_gk20a.h,
+             include/nvgpu/hw/gk20a/hw_ltc_gk20a.h,
+             include/nvgpu/hw/gk20a/hw_mc_gk20a.h,
+             include/nvgpu/hw/gk20a/hw_pbdma_gk20a.h,
+             include/nvgpu/hw/gk20a/hw_perf_gk20a.h,
+             include/nvgpu/hw/gk20a/hw_pram_gk20a.h,
+             include/nvgpu/hw/gk20a/hw_pri_ringmaster_gk20a.h,
+             include/nvgpu/hw/gk20a/hw_pri_ringstation_gpc_gk20a.h,
+             include/nvgpu/hw/gk20a/hw_pri_ringstation_sys_gk20a.h,
+             include/nvgpu/hw/gk20a/hw_proj_gk20a.h,
+             include/nvgpu/hw/gk20a/hw_pwr_gk20a.h,
+             include/nvgpu/hw/gk20a/hw_ram_gk20a.h,
+             include/nvgpu/hw/gk20a/hw_therm_gk20a.h,
+             include/nvgpu/hw/gk20a/hw_timer_gk20a.h,
+             include/nvgpu/hw/gk20a/hw_top_gk20a.h,
+             include/nvgpu/hw/gk20a/hw_trim_gk20a.h,
+             include/nvgpu/hw/gm20b/hw_bus_gm20b.h,
+             include/nvgpu/hw/gm20b/hw_ccsr_gm20b.h,
+             include/nvgpu/hw/gm20b/hw_ce2_gm20b.h,
+             include/nvgpu/hw/gm20b/hw_ctxsw_prog_gm20b.h,
+             include/nvgpu/hw/gm20b/hw_falcon_gm20b.h,
+             include/nvgpu/hw/gm20b/hw_fb_gm20b.h,
+             include/nvgpu/hw/gm20b/hw_fifo_gm20b.h,
+             include/nvgpu/hw/gm20b/hw_flush_gm20b.h,
+             include/nvgpu/hw/gm20b/hw_fuse_gm20b.h,
+             include/nvgpu/hw/gm20b/hw_gmmu_gm20b.h,
+             include/nvgpu/hw/gm20b/hw_gr_gm20b.h,
+             include/nvgpu/hw/gm20b/hw_ltc_gm20b.h,
+             include/nvgpu/hw/gm20b/hw_mc_gm20b.h,
+             include/nvgpu/hw/gm20b/hw_pbdma_gm20b.h,
+             include/nvgpu/hw/gm20b/hw_perf_gm20b.h,
+             include/nvgpu/hw/gm20b/hw_pram_gm20b.h,
+             include/nvgpu/hw/gm20b/hw_pri_ringmaster_gm20b.h,
+             include/nvgpu/hw/gm20b/hw_pri_ringstation_gpc_gm20b.h,
+             include/nvgpu/hw/gm20b/hw_pri_ringstation_sys_gm20b.h,
+             include/nvgpu/hw/gm20b/hw_proj_gm20b.h,
+             include/nvgpu/hw/gm20b/hw_pwr_gm20b.h,
+             include/nvgpu/hw/gm20b/hw_ram_gm20b.h,
+             include/nvgpu/hw/gm20b/hw_therm_gm20b.h,
+             include/nvgpu/hw/gm20b/hw_timer_gm20b.h,
+             include/nvgpu/hw/gm20b/hw_top_gm20b.h,
+             include/nvgpu/hw/gm20b/hw_trim_gm20b.h,
+             include/nvgpu/hw/gp106/hw_bus_gp106.h,
+             include/nvgpu/hw/gp106/hw_ccsr_gp106.h,
+             include/nvgpu/hw/gp106/hw_ce_gp106.h,
+             include/nvgpu/hw/gp106/hw_ctxsw_prog_gp106.h,
+             include/nvgpu/hw/gp106/hw_falcon_gp106.h,
+             include/nvgpu/hw/gp106/hw_fb_gp106.h,
+             include/nvgpu/hw/gp106/hw_fbpa_gp106.h,
+             include/nvgpu/hw/gp106/hw_fifo_gp106.h,
+             include/nvgpu/hw/gp106/hw_flush_gp106.h,
+             include/nvgpu/hw/gp106/hw_fuse_gp106.h,
+             include/nvgpu/hw/gp106/hw_gmmu_gp106.h,
+             include/nvgpu/hw/gp106/hw_gr_gp106.h,
+             include/nvgpu/hw/gp106/hw_ltc_gp106.h,
+             include/nvgpu/hw/gp106/hw_mc_gp106.h,
+             include/nvgpu/hw/gp106/hw_pbdma_gp106.h,
+             include/nvgpu/hw/gp106/hw_perf_gp106.h,
+             include/nvgpu/hw/gp106/hw_pnvdec_gp106.h,
+             include/nvgpu/hw/gp106/hw_pram_gp106.h,
+             include/nvgpu/hw/gp106/hw_pri_ringmaster_gp106.h,
+             include/nvgpu/hw/gp106/hw_pri_ringstation_gpc_gp106.h,
+             include/nvgpu/hw/gp106/hw_pri_ringstation_sys_gp106.h,
+             include/nvgpu/hw/gp106/hw_proj_gp106.h,
+             include/nvgpu/hw/gp106/hw_psec_gp106.h,
+             include/nvgpu/hw/gp106/hw_pwr_gp106.h,
+             include/nvgpu/hw/gp106/hw_ram_gp106.h,
+             include/nvgpu/hw/gp106/hw_therm_gp106.h,
+             include/nvgpu/hw/gp106/hw_timer_gp106.h,
+             include/nvgpu/hw/gp106/hw_top_gp106.h,
+             include/nvgpu/hw/gp106/hw_trim_gp106.h,
+             include/nvgpu/hw/gp106/hw_xp_gp106.h,
+             include/nvgpu/hw/gp106/hw_xve_gp106.h,
+             include/nvgpu/hw/gp10b/hw_bus_gp10b.h,
+             include/nvgpu/hw/gp10b/hw_ccsr_gp10b.h,
+             include/nvgpu/hw/gp10b/hw_ce_gp10b.h,
+             include/nvgpu/hw/gp10b/hw_ctxsw_prog_gp10b.h,
+             include/nvgpu/hw/gp10b/hw_falcon_gp10b.h,
+             include/nvgpu/hw/gp10b/hw_fb_gp10b.h,
+             include/nvgpu/hw/gp10b/hw_fifo_gp10b.h,
+             include/nvgpu/hw/gp10b/hw_flush_gp10b.h,
+             include/nvgpu/hw/gp10b/hw_fuse_gp10b.h,
+             include/nvgpu/hw/gp10b/hw_gmmu_gp10b.h,
+             include/nvgpu/hw/gp10b/hw_gr_gp10b.h,
+             include/nvgpu/hw/gp10b/hw_ltc_gp10b.h,
+             include/nvgpu/hw/gp10b/hw_mc_gp10b.h,
+             include/nvgpu/hw/gp10b/hw_pbdma_gp10b.h,
+             include/nvgpu/hw/gp10b/hw_perf_gp10b.h,
+             include/nvgpu/hw/gp10b/hw_pram_gp10b.h,
+             include/nvgpu/hw/gp10b/hw_pri_ringmaster_gp10b.h,
+             include/nvgpu/hw/gp10b/hw_pri_ringstation_gpc_gp10b.h,
+             include/nvgpu/hw/gp10b/hw_pri_ringstation_sys_gp10b.h,
+             include/nvgpu/hw/gp10b/hw_proj_gp10b.h,
+             include/nvgpu/hw/gp10b/hw_pwr_gp10b.h,
+             include/nvgpu/hw/gp10b/hw_ram_gp10b.h,
+             include/nvgpu/hw/gp10b/hw_therm_gp10b.h,
+             include/nvgpu/hw/gp10b/hw_timer_gp10b.h,
+             include/nvgpu/hw/gp10b/hw_top_gp10b.h,
+             include/nvgpu/hw/gv100/hw_bus_gv100.h,
+             include/nvgpu/hw/gv100/hw_ccsr_gv100.h,
+             include/nvgpu/hw/gv100/hw_ce_gv100.h,
+             include/nvgpu/hw/gv100/hw_ctxsw_prog_gv100.h,
+             include/nvgpu/hw/gv100/hw_falcon_gv100.h,
+             include/nvgpu/hw/gv100/hw_fb_gv100.h,
+             include/nvgpu/hw/gv100/hw_fifo_gv100.h,
+             include/nvgpu/hw/gv100/hw_flush_gv100.h,
+             include/nvgpu/hw/gv100/hw_fuse_gv100.h,
+             include/nvgpu/hw/gv100/hw_gmmu_gv100.h,
+             include/nvgpu/hw/gv100/hw_gr_gv100.h,
+             include/nvgpu/hw/gv100/hw_ioctrl_gv100.h,
+             include/nvgpu/hw/gv100/hw_ioctrlmif_gv100.h,
+             include/nvgpu/hw/gv100/hw_ltc_gv100.h,
+             include/nvgpu/hw/gv100/hw_mc_gv100.h,
+             include/nvgpu/hw/gv100/hw_minion_gv100.h,
+             include/nvgpu/hw/gv100/hw_nvl_gv100.h,
+             include/nvgpu/hw/gv100/hw_nvlinkip_discovery_gv100.h,
+             include/nvgpu/hw/gv100/hw_nvlipt_gv100.h,
+             include/nvgpu/hw/gv100/hw_nvtlc_gv100.h,
+             include/nvgpu/hw/gv100/hw_pbdma_gv100.h,
+             include/nvgpu/hw/gv100/hw_perf_gv100.h,
+             include/nvgpu/hw/gv100/hw_pgsp_gv100.h,
+             include/nvgpu/hw/gv100/hw_pram_gv100.h,
+             include/nvgpu/hw/gv100/hw_pri_ringmaster_gv100.h,
+             include/nvgpu/hw/gv100/hw_pri_ringstation_gpc_gv100.h,
+             include/nvgpu/hw/gv100/hw_pri_ringstation_sys_gv100.h,
+             include/nvgpu/hw/gv100/hw_proj_gv100.h,
+             include/nvgpu/hw/gv100/hw_pwr_gv100.h,
+             include/nvgpu/hw/gv100/hw_ram_gv100.h,
+             include/nvgpu/hw/gv100/hw_therm_gv100.h,
+             include/nvgpu/hw/gv100/hw_timer_gv100.h,
+             include/nvgpu/hw/gv100/hw_top_gv100.h,
+             include/nvgpu/hw/gv100/hw_trim_gv100.h,
+             include/nvgpu/hw/gv100/hw_usermode_gv100.h,
+             include/nvgpu/hw/gv100/hw_xp_gv100.h,
+             include/nvgpu/hw/gv100/hw_xve_gv100.h,
+             include/nvgpu/hw/gv11b/hw_bus_gv11b.h,
+             include/nvgpu/hw/gv11b/hw_ccsr_gv11b.h,
+             include/nvgpu/hw/gv11b/hw_ce_gv11b.h,
+             include/nvgpu/hw/gv11b/hw_ctxsw_prog_gv11b.h,
+             include/nvgpu/hw/gv11b/hw_falcon_gv11b.h,
+             include/nvgpu/hw/gv11b/hw_fb_gv11b.h,
+             include/nvgpu/hw/gv11b/hw_fifo_gv11b.h,
+             include/nvgpu/hw/gv11b/hw_flush_gv11b.h,
+             include/nvgpu/hw/gv11b/hw_fuse_gv11b.h,
+             include/nvgpu/hw/gv11b/hw_gmmu_gv11b.h,
+             include/nvgpu/hw/gv11b/hw_gr_gv11b.h,
+             include/nvgpu/hw/gv11b/hw_ltc_gv11b.h,
+             include/nvgpu/hw/gv11b/hw_mc_gv11b.h,
+             include/nvgpu/hw/gv11b/hw_pbdma_gv11b.h,
+             include/nvgpu/hw/gv11b/hw_perf_gv11b.h,
+             include/nvgpu/hw/gv11b/hw_pram_gv11b.h,
+             include/nvgpu/hw/gv11b/hw_pri_ringmaster_gv11b.h,
+             include/nvgpu/hw/gv11b/hw_pri_ringstation_gpc_gv11b.h,
+             include/nvgpu/hw/gv11b/hw_pri_ringstation_sys_gv11b.h,
+             include/nvgpu/hw/gv11b/hw_pri_ringstation_fbp_gv11b.h,
+             include/nvgpu/hw/gv11b/hw_proj_gv11b.h,
+             include/nvgpu/hw/gv11b/hw_pwr_gv11b.h,
+             include/nvgpu/hw/gv11b/hw_ram_gv11b.h,
+             include/nvgpu/hw/gv11b/hw_therm_gv11b.h,
+             include/nvgpu/hw/gv11b/hw_timer_gv11b.h,
+             include/nvgpu/hw/gv11b/hw_top_gv11b.h,
+             include/nvgpu/hw/gv11b/hw_usermode_gv11b.h,
+             include/nvgpu/hw/tu104/hw_bus_tu104.h,
+             include/nvgpu/hw/tu104/hw_ccsr_tu104.h,
+             include/nvgpu/hw/tu104/hw_ce_tu104.h,
+             include/nvgpu/hw/tu104/hw_ctrl_tu104.h,
+             include/nvgpu/hw/tu104/hw_ctxsw_prog_tu104.h,
+             include/nvgpu/hw/tu104/hw_falcon_tu104.h,
+             include/nvgpu/hw/tu104/hw_fb_tu104.h,
+             include/nvgpu/hw/tu104/hw_fbpa_tu104.h,
+             include/nvgpu/hw/tu104/hw_fifo_tu104.h,
+             include/nvgpu/hw/tu104/hw_flush_tu104.h,
+             include/nvgpu/hw/tu104/hw_func_tu104.h,
+             include/nvgpu/hw/tu104/hw_fuse_tu104.h,
+             include/nvgpu/hw/tu104/hw_gc6_tu104.h,
+             include/nvgpu/hw/tu104/hw_gmmu_tu104.h,
+             include/nvgpu/hw/tu104/hw_gr_tu104.h,
+             include/nvgpu/hw/tu104/hw_ioctrl_tu104.h,
+             include/nvgpu/hw/tu104/hw_ioctrlmif_tu104.h,
+             include/nvgpu/hw/tu104/hw_ltc_tu104.h,
+             include/nvgpu/hw/tu104/hw_mc_tu104.h,
+             include/nvgpu/hw/tu104/hw_minion_tu104.h,
+             include/nvgpu/hw/tu104/hw_nvl_tu104.h,
+             include/nvgpu/hw/tu104/hw_nvlinkip_discovery_tu104.h,
+             include/nvgpu/hw/tu104/hw_nvlipt_tu104.h,
+             include/nvgpu/hw/tu104/hw_nvtlc_tu104.h,
+             include/nvgpu/hw/tu104/hw_pbdma_tu104.h,
+             include/nvgpu/hw/tu104/hw_perf_tu104.h,
+             include/nvgpu/hw/tu104/hw_pgsp_tu104.h,
+             include/nvgpu/hw/tu104/hw_pnvdec_tu104.h,
+             include/nvgpu/hw/tu104/hw_pram_tu104.h,
+             include/nvgpu/hw/tu104/hw_pri_ringmaster_tu104.h,
+             include/nvgpu/hw/tu104/hw_pri_ringstation_gpc_tu104.h,
+             include/nvgpu/hw/tu104/hw_pri_ringstation_sys_tu104.h,
+             include/nvgpu/hw/tu104/hw_pri_ringstation_fbp_tu104.h,
+             include/nvgpu/hw/tu104/hw_proj_tu104.h,
+             include/nvgpu/hw/tu104/hw_psec_tu104.h,
+             include/nvgpu/hw/tu104/hw_pwr_tu104.h,
+             include/nvgpu/hw/tu104/hw_ram_tu104.h,
+             include/nvgpu/hw/tu104/hw_therm_tu104.h,
+             include/nvgpu/hw/tu104/hw_timer_tu104.h,
+             include/nvgpu/hw/tu104/hw_top_tu104.h,
+             include/nvgpu/hw/tu104/hw_trim_tu104.h,
+             include/nvgpu/hw/tu104/hw_usermode_tu104.h,
+             include/nvgpu/hw/tu104/hw_xp_tu104.h,
+             include/nvgpu/hw/tu104/hw_xve_tu104.h ]

arch/nvgpu-hal-new.yaml

@@ -0,0 +1,836 @@
+# Copyright (c) 2019-2021, NVIDIA CORPORATION.  All Rights Reserved.
+#
+# HAL units. These are the units that have access to HW.
+#
+
+bus_fusa:
+  safe: yes
+  owner: Terje B
+  sources: [ include/nvgpu/gops/bus.h,
+             hal/bus/bus_gk20a_fusa.c,
+             hal/bus/bus_gk20a.h,
+             hal/bus/bus_gm20b_fusa.c, hal/bus/bus_gm20b.h,
+             hal/bus/bus_gp10b_fusa.c, hal/bus/bus_gp10b.h,
+             hal/bus/bus_gv11b_fusa.c, hal/bus/bus_gv11b.h ]
+
+bus:
+  safe: no
+  owner: Terje B
+  sources: [ hal/bus/bus_gk20a.c,
+             hal/bus/bus_gv100.c, hal/bus/bus_gv100.h,
+             hal/bus/bus_tu104.c, hal/bus/bus_tu104.h ]
+
+ltc:
+  owner: Seshendra G
+  children:
+    ltc_fusa:
+      safe: yes
+      sources: [ hal/ltc/ltc_gm20b_fusa.c,
+                 hal/ltc/ltc_gm20b.h,
+                 hal/ltc/ltc_gp10b_fusa.c,
+                 hal/ltc/ltc_gp10b.h,
+                 hal/ltc/ltc_gv11b_fusa.c,
+                 hal/ltc/ltc_gv11b.h ]
+    ltc:
+      safe: no
+      sources: [ hal/ltc/ltc_gm20b.c,
+                 hal/ltc/ltc_gm20b_dbg.c,
+                 hal/ltc/ltc_gp10b.c,
+                 hal/ltc/ltc_tu104.c,
+                 hal/ltc/ltc_tu104.h ]
+    intr_fusa:
+      safe: yes
+      sources: [ hal/ltc/intr/ltc_intr_gp10b_fusa.c,
+                 hal/ltc/intr/ltc_intr_gp10b.h,
+                 hal/ltc/intr/ltc_intr_gv11b_fusa.c,
+                 hal/ltc/intr/ltc_intr_gv11b.h ]
+    intr:
+      safe: no
+      sources: [ hal/ltc/intr/ltc_intr_gm20b.c,
+                 hal/ltc/intr/ltc_intr_gm20b.h,
+                 hal/ltc/intr/ltc_intr_gp10b.c ]
+
+init_fusa:
+  safe: yes
+  owner: Philip E
+  sources: [ hal/init/hal_gv11b.c,
+             hal/init/hal_gv11b.h,
+             hal/init/hal_init.c,
+             hal/init/hal_gv11b_litter.c,
+             hal/init/hal_gv11b_litter.h ]
+
+init:
+  safe: no
+  owner: Philip E
+  sources: [ hal/init/hal_gm20b.c,
+             hal/init/hal_gm20b.h,
+             hal/init/hal_gp10b.c,
+             hal/init/hal_gp10b.h,
+             hal/init/hal_tu104.c,
+             hal/init/hal_tu104.h,
+             hal/init/hal_gm20b_litter.c,
+             hal/init/hal_gm20b_litter.h,
+             hal/init/hal_gp10b_litter.c,
+             hal/init/hal_gp10b_litter.h,
+             hal/init/hal_tu104_litter.c,
+             hal/init/hal_tu104_litter.h ]
+
+priv_ring_fusa:
+  safe: yes
+  owner: Seema K
+  sources: [ include/nvgpu/gops/priv_ring.h,
+             hal/priv_ring/priv_ring_gm20b_fusa.c,
+             hal/priv_ring/priv_ring_gm20b.h,
+             hal/priv_ring/priv_ring_gp10b_fusa.c,
+             hal/priv_ring/priv_ring_gp10b.h ]
+
+priv_ring:
+  safe: no
+  owner: Seema K
+  sources: [ hal/priv_ring/priv_ring_gm20b.c,
+             hal/priv_ring/priv_ring_gv11b.c,
+             hal/priv_ring/priv_ring_gv11b.h ]
+
+ptimer_fusa:
+  safe: yes
+  owner: Terje B
+  sources: [ hal/ptimer/ptimer_gk20a_fusa.c,
+             hal/ptimer/ptimer_gk20a.h ]
+
+ptimer:
+  safe: no
+  owner: Deepak N
+  sources: [ hal/ptimer/ptimer_gp10b.c,
+             hal/ptimer/ptimer_gp10b.h,
+             hal/ptimer/ptimer_gv11b.c,
+             hal/ptimer/ptimer_gv11b.h ]
+
+cg_fusa:
+  safe: yes
+  owner: Seema K
+  sources: [ hal/power_features/cg/gating_reglist.h,
+             hal/power_features/cg/gv11b_gating_reglist.c,
+             hal/power_features/cg/gv11b_gating_reglist.h ]
+
+cg:
+  safe: no
+  owner: Seema K
+  sources: [ hal/power_features/cg/gm20b_gating_reglist.c,
+             hal/power_features/cg/gm20b_gating_reglist.h,
+             hal/power_features/cg/gp10b_gating_reglist.c,
+             hal/power_features/cg/gp10b_gating_reglist.h,
+             hal/power_features/cg/tu104_gating_reglist.c,
+             hal/power_features/cg/tu104_gating_reglist.h ]
+
+rc:
+  safe: no
+  owner: Seema K
+  sources: [ hal/rc/rc_gk20a.c,
+             hal/rc/rc_gk20a.h,
+             hal/rc/rc_gv11b.c,
+             hal/rc/rc_gv11b.h ]
+
+fbpa:
+  safe: yes
+  owner: Seshendra G
+  gpu: dgpu
+  sources: [ hal/fbpa/fbpa_tu104.c,   hal/fbpa/fbpa_tu104.h ]
+
+clk:
+  safe: yes
+  owner: Ramesh M
+  gpu: dgpu
+  sources: [ hal/clk/clk_tu104.c,
+             hal/clk/clk_tu104.h ]
+
+clk_mon:
+  safe: yes
+  owner: Ramesh M
+  gpu: dgpu
+  sources: [ hal/clk/clk_mon_tu104.c,
+             hal/clk/clk_mon_tu104.h,
+             include/nvgpu/clk_mon.h]
+
+clk_igpu:
+  safe: no
+  owner: Ramesh M
+  gpu: igpu
+  sources: [ hal/clk/clk_gk20a.h,
+             hal/clk/clk_gm20b.c,
+             hal/clk/clk_gm20b.h ]
+
+fifo:
+  safe: yes
+  owner: Seema K
+  children:
+    userd:
+      safe: no
+      sources: [ hal/fifo/userd_gk20a.c,
+                 hal/fifo/userd_gk20a.h,
+                 hal/fifo/userd_gv11b.c,
+                 hal/fifo/userd_gv11b.h ]
+    ramfc_fusa:
+      safe: yes
+      sources: [ hal/fifo/ramin_gk20a_fusa.c,
+                 hal/fifo/ramfc_gp10b_fusa.c,
+                 hal/fifo/ramfc_gp10b.h,
+                 hal/fifo/ramfc_gv11b_fusa.c,
+                 hal/fifo/ramfc_gv11b.h ]
+
+    ramfc:
+      safe: no
+      sources: [ hal/fifo/ramfc_gk20a.c,
+                 hal/fifo/ramfc_gk20a.h,
+                 hal/fifo/ramfc_gp10b.c,
+                 hal/fifo/ramfc_tu104.c,
+                 hal/fifo/ramfc_tu104.h ]
+
+    ramin_fusa:
+      safe: yes
+      sources: [ hal/fifo/ramin_gv11b.h,
+                 hal/fifo/ramin_gm20b.h,
+                 hal/fifo/ramin_gv11b_fusa.c,
+                 hal/fifo/ramin_gm20b_fusa.c ]
+
+    ramin:
+      safe: no
+      sources: [ hal/fifo/ramin_gk20a.h,
+                 hal/fifo/ramin_gp10b.h,
+                 hal/fifo/ramin_gk20a.c,
+                 hal/fifo/ramin_gp10b.c,
+                 hal/fifo/ramin_tu104.c,
+                 hal/fifo/ramin_tu104.h ]
+
+    runlist_fusa:
+      safe: yes
+      sources: [ hal/fifo/runlist_fifo_gk20a.c,
+                 hal/fifo/runlist_fifo_gk20a_fusa.c,
+                 hal/fifo/runlist_fifo_gk20a.h,
+                 hal/fifo/runlist_fifo_gv11b.c,
+                 hal/fifo/runlist_fifo_gv11b_fusa.c,
+                 hal/fifo/runlist_fifo_gv11b.h,
+                 hal/fifo/runlist_ram_gv11b_fusa.c,
+                 hal/fifo/runlist_ram_gv11b.h ]
+
+    runlist:
+      safe: no
+      sources: [ hal/fifo/runlist_fifo_gv100.c,
+                 hal/fifo/runlist_fifo_gv100.h,
+                 hal/fifo/runlist_fifo_tu104.c,
+                 hal/fifo/runlist_fifo_tu104.h,
+                 hal/fifo/runlist_ram_gk20a.c,
+                 hal/fifo/runlist_ram_gk20a.h,
+                 hal/fifo/runlist_ram_tu104.c,
+                 hal/fifo/runlist_ram_tu104.h ]
+
+    channel_fusa:
+      safe: yes
+      sources: [ hal/fifo/channel_gk20a_fusa.c,
+                 hal/fifo/channel_gk20a.h,
+                 hal/fifo/channel_gm20b_fusa.c,
+                 hal/fifo/channel_gm20b.h,
+                 hal/fifo/channel_gv11b_fusa.c,
+                 hal/fifo/channel_gv11b.h ]
+    channel:
+      safe: no
+      sources: [ hal/fifo/channel_gk20a.c,
+                 hal/fifo/channel_gm20b.c,
+                 hal/fifo/channel_gv100.c,
+                 hal/fifo/channel_gv100.h ]
+    tsg_fusa:
+      safe: yes
+      sources: [ hal/fifo/tsg_gk20a.h,
+                 hal/fifo/tsg_gk20a_fusa.c,
+                 hal/fifo/tsg_gv11b.h,
+                 hal/fifo/tsg_gv11b_fusa.c ]
+
+    tsg:
+      safe: no
+      sources: [ hal/fifo/tsg_gk20a.c ]
+
+    fifo_fusa:
+     safe: yes
+     sources: [ hal/fifo/fifo_intr_gk20a_fusa.c,
+                hal/fifo/fifo_intr_gk20a.h,
+                hal/fifo/fifo_intr_gv11b_fusa.c,
+                hal/fifo/fifo_intr_gv11b.h,
+                hal/fifo/ctxsw_timeout_gv11b_fusa.c,
+                hal/fifo/ctxsw_timeout_gv11b.h,
+                hal/fifo/fifo_gk20a_fusa.c,
+                hal/fifo/fifo_gk20a.h,
+                hal/fifo/fifo_gv11b_fusa.c,
+                hal/fifo/fifo_gv11b.h ]
+    fifo:
+     safe: no
+     sources: [ hal/fifo/fifo_intr_gk20a.c,
+                hal/fifo/fifo_intr_gv100.c,
+                hal/fifo/fifo_intr_gv100.h,
+                hal/fifo/ctxsw_timeout_gk20a.c,
+                hal/fifo/ctxsw_timeout_gk20a.h,
+                hal/fifo/mmu_fault_gk20a.c,
+                hal/fifo/mmu_fault_gk20a.h,
+                hal/fifo/mmu_fault_gm20b.c,
+                hal/fifo/mmu_fault_gm20b.h,
+                hal/fifo/mmu_fault_gp10b.c,
+                hal/fifo/mmu_fault_gp10b.h,
+                hal/fifo/fifo_gk20a.c,
+                hal/fifo/fifo_tu104.c,
+                hal/fifo/fifo_tu104.h ]
+
+    engine_status_fusa:
+     safe: yes
+     sources: [ hal/fifo/engine_status_gm20b_fusa.c,
+                hal/fifo/engine_status_gm20b.h,
+                hal/fifo/engine_status_gv100_fusa.c,
+                hal/fifo/engine_status_gv100.h ]
+    engine_status:
+     safe: no
+     sources: [ hal/fifo/engine_status_gm20b.c ]
+
+    engines_fusa:
+     safe: yes
+     sources: [ hal/fifo/engines_gp10b_fusa.c,
+                hal/fifo/engines_gp10b.h,
+                hal/fifo/engines_gv11b_fusa.c,
+                hal/fifo/engines_gv11b.h ]
+
+    engines:
+     safe: no
+     sources: [ hal/fifo/engines_gm20b.c,
+                hal/fifo/engines_gm20b.h ]
+
+    pbdma_status:
+      safe: yes
+      sources: [ hal/fifo/pbdma_status_gm20b_fusa.c,
+                 hal/fifo/pbdma_status_gm20b.h ]
+
+    pbdma_fusa:
+      safe: yes
+      sources: [ hal/fifo/pbdma_gm20b_fusa.c,
+                 hal/fifo/pbdma_gm20b.h,
+                 hal/fifo/pbdma_gp10b_fusa.c,
+                 hal/fifo/pbdma_gp10b.h,
+                 hal/fifo/pbdma_gv11b_fusa.c,
+                 hal/fifo/pbdma_gv11b.h ]
+
+    pbdma:
+      safe: no
+      sources: [ hal/fifo/pbdma_gm20b.c,
+                 hal/fifo/pbdma_gp10b.c,
+                 hal/fifo/pbdma_tu104.c,
+                 hal/fifo/pbdma_tu104.h ]
+
+    preempt_fusa:
+     safe: yes
+     sources: [ hal/fifo/preempt_gv11b_fusa.c,
+                hal/fifo/preempt_gv11b.h ]
+
+    preempt:
+     safe: no
+     sources: [ hal/fifo/preempt_gk20a.c,
+                hal/fifo/preempt_gk20a.h ]
+
+    usermode_fusa:
+      safe: yes
+      sources: [ hal/fifo/usermode_gv11b_fusa.c,
+                 hal/fifo/usermode_gv11b.h ]
+
+    usermode:
+      safe: no
+      sources: [ hal/fifo/usermode_tu104.c,
+                 hal/fifo/usermode_tu104.h ]
+
+fuse_fusa:
+  safe: yes
+  owner: Seema K
+  sources: [ hal/fuse/fuse_gm20b_fusa.c,
+             hal/fuse/fuse_gm20b.h,
+             hal/fuse/fuse_gp10b_fusa.c,
+             hal/fuse/fuse_gp10b.h ]
+
+fuse:
+  safe: no
+  owner: Seema K
+  sources: [ hal/fuse/fuse_gm20b.c,
+             hal/fuse/fuse_gp106.c,
+             hal/fuse/fuse_gp106.h,
+             hal/fuse/fuse_tu104.c,
+             hal/fuse/fuse_tu104.h ]
+
+gsp:
+  safe: no
+  sources: [ hal/gsp/gsp_tu104.c,
+             hal/gsp/gsp_tu104.h ]
+
+mm:
+  safe: yes
+  owner: Alex W
+  children:
+    gmmu_fusa:
+      safe: yes
+      sources: [ hal/mm/gmmu/gmmu_gk20a_fusa.c,
+                 hal/mm/gmmu/gmmu_gk20a.h,
+                 hal/mm/gmmu/gmmu_gm20b_fusa.c,
+                 hal/mm/gmmu/gmmu_gm20b.h,
+                 hal/mm/gmmu/gmmu_gp10b_fusa.c,
+                 hal/mm/gmmu/gmmu_gp10b.h,
+                 hal/mm/gmmu/gmmu_gv11b_fusa.c,
+                 hal/mm/gmmu/gmmu_gv11b.h ]
+    gmmu:
+      safe: no
+      sources: [ hal/mm/gmmu/gmmu_gk20a.c,
+                 hal/mm/gmmu/gmmu_gm20b.c]
+    cache_fusa:
+      safe: yes
+      sources: [ hal/mm/cache/flush_gk20a_fusa.c,
+                 hal/mm/cache/flush_gk20a.h,
+                 hal/mm/cache/flush_gv11b_fusa.c,
+                 hal/mm/cache/flush_gv11b.h ]
+    cache:
+      safe: no
+      sources: [ hal/mm/cache/flush_gk20a.c ]
+    mmu_fault:
+      safe: yes
+      sources: [ hal/mm/mmu_fault/mmu_fault_gv11b_fusa.c,
+                 hal/mm/mmu_fault/mmu_fault_gv11b.h ]
+    mm_fusa:
+      safe: yes
+      sources: [ hal/mm/mm_gp10b_fusa.c,
+                 hal/mm/mm_gp10b.h,
+                 hal/mm/mm_gv11b_fusa.c,
+                 hal/mm/mm_gv11b.h]
+    mm:
+      safe: no
+      sources: [ hal/mm/mm_gk20a.c,
+                 hal/mm/mm_gk20a.h,
+                 hal/mm/mm_gm20b.c,
+                 hal/mm/mm_gm20b.h,
+                 hal/mm/mm_gv100.c,
+                 hal/mm/mm_gv100.h,
+                 hal/mm/mm_tu104.c,
+                 hal/mm/mm_tu104.h]
+
+sync:
+  safe: yes
+  owner: Thomas F
+  children:
+    sema:
+      safe: no
+      sources: [ hal/sync/sema_cmdbuf_gk20a.c,
+                 hal/sync/sema_cmdbuf_gk20a.h,
+                 hal/sync/sema_cmdbuf_gv11b.c,
+                 hal/sync/sema_cmdbuf_gv11b.h ]
+    syncpt:
+      safe: no
+      sources: [ hal/sync/syncpt_cmdbuf_gk20a.c,
+                 hal/sync/syncpt_cmdbuf_gk20a.h,
+                 hal/sync/syncpt_cmdbuf_gv11b.c]
+    syncpt_fusa:
+      safe: yes
+      sources: [ hal/sync/syncpt_cmdbuf_gv11b_fusa.c,
+                 hal/sync/syncpt_cmdbuf_gv11b.h ]
+
+
+therm_fusa:
+  safe: yes
+  owner: Seshendra G
+  sources: [ hal/therm/therm_gv11b_fusa.c,
+             hal/therm/therm_gv11b.h ]
+
+therm:
+  safe: no
+  owner: Seshendra G
+  sources: [ hal/therm/therm_gm20b.c,
+             hal/therm/therm_gm20b.h,
+             hal/therm/therm_tu104.c,
+             hal/therm/therm_tu104.h,
+             hal/therm/therm_gp10b.c,
+             hal/therm/therm_gp10b.h ]
+
+cbc:
+  safe: no
+  owner: Seshendra G
+  sources: [ hal/cbc/cbc_gm20b.c,
+             hal/cbc/cbc_gm20b.h,
+             hal/cbc/cbc_gp10b.c,
+             hal/cbc/cbc_gp10b.h,
+             hal/cbc/cbc_gv11b.c,
+             hal/cbc/cbc_gv11b.h,
+             hal/cbc/cbc_tu104.c,
+             hal/cbc/cbc_tu104.h ]
+
+ce_fusa:
+  safe: yes
+  owner: Thomas F
+  sources: [ hal/ce/ce_gp10b_fusa.c,
+             hal/ce/ce_gp10b.h,
+             hal/ce/ce_gv11b_fusa.c,
+             hal/ce/ce_gv11b.h ]
+
+ce:
+  safe: no
+  owner: Thomas F
+  sources: [ hal/ce/ce2_gk20a.c,
+             hal/ce/ce2_gk20a.h,
+             hal/ce/ce_tu104.c,
+             hal/ce/ce_tu104.h ]
+
+gr:
+  safe: yes
+  owner: Deepak N
+  children:
+    ecc_fusa:
+      safe: yes
+      sources: [hal/gr/ecc/ecc_gv11b_fusa.c,
+                hal/gr/ecc/ecc_gv11b.h ]
+    ecc:
+      safe: no
+      sources: [hal/gr/ecc/ecc_gv11b.c,
+                hal/gr/ecc/ecc_gp10b.c,
+                hal/gr/ecc/ecc_gp10b.h ]
+    ctxsw_prog_fusa:
+      safe: yes
+      sources: [ hal/gr/ctxsw_prog/ctxsw_prog_gm20b_fusa.c,
+                 hal/gr/ctxsw_prog/ctxsw_prog_gm20b.h,
+                 hal/gr/ctxsw_prog/ctxsw_prog_gp10b_fusa.c,
+                 hal/gr/ctxsw_prog/ctxsw_prog_gp10b.h,
+                 hal/gr/ctxsw_prog/ctxsw_prog_gv11b_fusa.c,
+                 hal/gr/ctxsw_prog/ctxsw_prog_gv11b.h ]
+    ctxsw_prog:
+      safe: no
+      sources: [ hal/gr/ctxsw_prog/ctxsw_prog_gm20b.c,
+                 hal/gr/ctxsw_prog/ctxsw_prog_gm20b_dbg.c,
+                 hal/gr/ctxsw_prog/ctxsw_prog_gp10b.c,
+                 hal/gr/ctxsw_prog/ctxsw_prog_gv11b.c]
+    config_fusa:
+      safe: yes
+      sources: [ hal/gr/config/gr_config_gm20b_fusa.c,
+                 hal/gr/config/gr_config_gm20b.h,
+                 hal/gr/config/gr_config_gv100_fusa.c,
+                 hal/gr/config/gr_config_gv100.h ]
+
+    config:
+      safe: no
+      sources: [ hal/gr/config/gr_config_gm20b.c ]
+
+    init_fusa:
+      safe: yes
+      sources: [ hal/gr/init/gr_init_gm20b_fusa.c,
+                 hal/gr/init/gr_init_gm20b.h,
+                 hal/gr/init/gr_init_gp10b_fusa.c,
+                 hal/gr/init/gr_init_gp10b.h,
+                 hal/gr/init/gr_init_gv11b_fusa.c,
+                 hal/gr/init/gr_init_gv11b.h ]
+    init:
+      safe: no
+      sources: [ hal/gr/init/gr_init_gm20b.c,
+                 hal/gr/init/gr_init_gp10b.c,
+                 hal/gr/init/gr_init_gv100.c,
+                 hal/gr/init/gr_init_gv100.h,
+                 hal/gr/init/gr_init_gv11b.c,
+                 hal/gr/init/gr_init_tu104.c,
+                 hal/gr/init/gr_init_tu104.h ]
+    intr_fusa:
+      safe: yes
+      sources: [ hal/gr/intr/gr_intr_gm20b_fusa.c,
+                 hal/gr/intr/gr_intr_gm20b.h,
+                 hal/gr/intr/gr_intr_gp10b_fusa.c,
+                 hal/gr/intr/gr_intr_gp10b.h,
+                 hal/gr/intr/gr_intr_gv11b_fusa.c,
+                 hal/gr/intr/gr_intr_gv11b.h ]
+    intr:
+      safe: no
+      sources: [ hal/gr/intr/gr_intr_gm20b.c,
+                 hal/gr/intr/gr_intr_gp10b.c,
+                 hal/gr/intr/gr_intr_tu104.c,
+                 hal/gr/intr/gr_intr_tu104.h ]
+    falcon_fusa:
+      safe: yes
+      sources: [ hal/gr/falcon/gr_falcon_gm20b_fusa.c,
+                 hal/gr/falcon/gr_falcon_gm20b.h,
+                 hal/gr/falcon/gr_falcon_gp10b_fusa.c,
+                 hal/gr/falcon/gr_falcon_gp10b.h,
+                 hal/gr/falcon/gr_falcon_gv11b_fusa.c,
+                 hal/gr/falcon/gr_falcon_gv11b.h ]
+    falcon:
+      safe: no
+      sources: [ hal/gr/falcon/gr_falcon_gm20b.c,
+                 hal/gr/falcon/gr_falcon_tu104.c,
+                 hal/gr/falcon/gr_falcon_tu104.h ]
+    fecs_trace:
+      safe: no
+      sources: [ hal/gr/fecs_trace/fecs_trace_gm20b.c,
+                 hal/gr/fecs_trace/fecs_trace_gm20b.h,
+                 hal/gr/fecs_trace/fecs_trace_gm20b.h,
+                 hal/gr/fecs_trace/fecs_trace_gv11b.c,
+                 hal/gr/fecs_trace/fecs_trace_gv11b.h ]
+    hwpm_map:
+      safe: no
+      sources: [ hal/gr/hwpm_map/hwpm_map_gv100.c,
+                 hal/gr/hwpm_map/hwpm_map_gv100.h ]
+    zbc:
+      safe: no
+      sources: [ hal/gr/zbc/zbc_gm20b.c,
+                 hal/gr/zbc/zbc_gm20b.h,
+                 hal/gr/zbc/zbc_gp10b.c,
+                 hal/gr/zbc/zbc_gp10b.h,
+                 hal/gr/zbc/zbc_gv11b.c,
+                 hal/gr/zbc/zbc_gv11b.h ]
+    zcull:
+      safe: no
+      sources: [ hal/gr/zcull/zcull_gm20b.c,
+                 hal/gr/zcull/zcull_gm20b.h,
+                 hal/gr/zcull/zcull_gv11b.c,
+                 hal/gr/zcull/zcull_gv11b.h ]
+    gr:
+      safe: no
+      sources: [ hal/gr/gr/gr_gk20a.c, hal/gr/gr/gr_gk20a.h,
+                 hal/gr/gr/gr_gm20b.c, hal/gr/gr/gr_gm20b.h,
+                 hal/gr/gr/gr_gp10b.c, hal/gr/gr/gr_gp10b.h,
+                 hal/gr/gr/gr_gv100.c, hal/gr/gr/gr_gv100.h,
+                 hal/gr/gr/gr_gv11b.c, hal/gr/gr/gr_gv11b.h,
+                 hal/gr/gr/gr_tu104.c, hal/gr/gr/gr_tu104.h,
+                 include/nvgpu/gr/warpstate.h,
+                 hal/gr/gr/gr_pri_gk20a.h,
+                 hal/gr/gr/gr_pri_gv11b.h ]
+
+regops:
+  safe: no
+  owner: Deepak N
+  sources: [ hal/regops/regops_gm20b.c,
+             hal/regops/regops_gm20b.h,
+             hal/regops/regops_gp10b.c,
+             hal/regops/regops_gp10b.h,
+             hal/regops/regops_gv11b.c,
+             hal/regops/regops_gv11b.h,
+             hal/regops/regops_tu104.c,
+             hal/regops/regops_tu104.h,
+             hal/regops/allowlist_gv11b.c,
+             hal/regops/allowlist_gv11b.h,
+             hal/regops/allowlist_tu104.c,
+             hal/regops/allowlist_tu104.h ]
+
+falcon_fusa:
+  safe: yes
+  owner: Sagar K
+  sources: [ hal/falcon/falcon_gk20a_fusa.c,
+             hal/falcon/falcon_gk20a.h ]
+
+falcon:
+  safe: no
+  owner: Sagar K
+  sources: [ hal/falcon/falcon_gk20a.c ]
+
+mc_fusa:
+  safe: yes
+  owner: Seema K
+  sources: [ hal/mc/mc_gm20b_fusa.c,
+             hal/mc/mc_gm20b.h,
+             hal/mc/mc_gp10b_fusa.c,
+             hal/mc/mc_gp10b.h,
+             hal/mc/mc_gv11b_fusa.c,
+             hal/mc/mc_gv11b.h ]
+
+mc:
+  safe: no
+  owner: Seema K
+  sources: [ hal/mc/mc_gm20b.c,
+             hal/mc/mc_gv100.c,
+             hal/mc/mc_gv100.h,
+             hal/mc/mc_tu104.c,
+             hal/mc/mc_tu104.h ]
+
+fb_fusa:
+  safe: yes
+  owner: Seshendra G
+  sources: [ include/nvgpu/gops/fb.h,
+             hal/fb/fb_gm20b_fusa.c,
+             hal/fb/fb_gm20b.h,
+             hal/fb/fb_gv11b_fusa.c,
+             hal/fb/fb_gv11b.h,
+             hal/fb/ecc/fb_ecc_gv11b.h, hal/fb/ecc/fb_ecc_gv11b_fusa.c,
+             hal/fb/intr/fb_intr_gv11b.h, hal/fb/intr/fb_intr_gv11b_fusa.c,
+             hal/fb/fb_mmu_fault_gv11b.h, hal/fb/fb_mmu_fault_gv11b_fusa.c,
+             hal/fb/intr/fb_intr_ecc_gv11b.h, hal/fb/intr/fb_intr_ecc_gv11b_fusa.c ]
+
+fb:
+  safe: no
+  owner: Seshendra G
+  sources: [ hal/fb/fb_gm20b.c,
+             hal/fb/fb_gp106.c, hal/fb/fb_gp106.h,
+             hal/fb/fb_gp10b.c, hal/fb/fb_gp10b.h,
+             hal/fb/fb_gv100.c, hal/fb/fb_gv100.h,
+             hal/fb/fb_gv11b.c,
+             hal/fb/fb_tu104.c, hal/fb/fb_tu104.h,
+             hal/fb/intr/fb_intr_gv100.h, hal/fb/intr/fb_intr_gv100.c,
+             hal/fb/fb_mmu_fault_tu104.h, hal/fb/fb_mmu_fault_tu104.c,
+             hal/fb/intr/fb_intr_tu104.c, hal/fb/intr/fb_intr_tu104.h,
+             hal/fb/intr/fb_intr_ecc_gv11b.c ]
+
+pmu_fusa:
+  safe: yes
+  owner: Mahantesh K
+  sources: [ hal/pmu/pmu_gk20a_fusa.c,
+             hal/pmu/pmu_gk20a.h,
+             hal/pmu/pmu_gv11b_fusa.c,
+             hal/pmu/pmu_gv11b.h ]
+
+pmu:
+  safe: no
+  owner: Mahantesh K
+  sources: [ hal/pmu/pmu_gk20a.c,
+             hal/pmu/pmu_gm20b.c,
+             hal/pmu/pmu_gm20b.h,
+             hal/pmu/pmu_gp10b.c,
+             hal/pmu/pmu_gp10b.h,
+             hal/pmu/pmu_gv11b.c,
+             hal/pmu/pmu_tu104.c,
+             hal/pmu/pmu_tu104.h ]
+
+nvlink:
+  safe: yes
+  gpu: dgpu
+  sources: [ hal/nvlink/link_mode_transitions_gv100.c,
+             hal/nvlink/link_mode_transitions_gv100.h,
+             hal/nvlink/link_mode_transitions_tu104.c,
+             hal/nvlink/link_mode_transitions_tu104.h,
+             hal/nvlink/intr_and_err_handling_tu104.c,
+             hal/nvlink/intr_and_err_handling_tu104.h,
+             hal/nvlink/nvlink_gv100.c,
+             hal/nvlink/nvlink_gv100.h,
+             hal/nvlink/nvlink_tu104.c,
+             hal/nvlink/nvlink_tu104.h,
+             hal/nvlink/minion_gv100.c,
+             hal/nvlink/minion_gv100.h,
+             hal/nvlink/minion_tu104.c,
+             hal/nvlink/minion_tu104.h ]
+
+sec2:
+  safe: yes
+  owner: Sagar K
+  gpu: dgpu
+  sources: [ hal/sec2/sec2_tu104.c,
+             hal/sec2/sec2_tu104.h ]
+
+netlist_fusa:
+  safe: yes
+  owner: Seshendra G
+  gpu: both
+  sources: [ include/nvgpu/gops/netlist.h,
+             hal/netlist/netlist_gv11b_fusa.c,
+             hal/netlist/netlist_gv11b.h ]
+
+netlist:
+  safe: no
+  owner: Seshendra G
+  gpu: both
+  sources: [ hal/netlist/netlist_gm20b.c,
+             hal/netlist/netlist_gm20b.h,
+             hal/netlist/netlist_gp10b.c,
+             hal/netlist/netlist_gp10b.h,
+             hal/netlist/netlist_gv100.c,
+             hal/netlist/netlist_gv100.h,
+             hal/netlist/netlist_tu104.c,
+             hal/netlist/netlist_tu104.h ]
+
+nvdec:
+  safe: no
+  sources: [ hal/nvdec/nvdec_gp106.c,
+             hal/nvdec/nvdec_gp106.h,
+             hal/nvdec/nvdec_tu104.c,
+             hal/nvdec/nvdec_tu104.h ]
+
+perf:
+  safe: no
+  owner: Deepak N
+  sources: [ hal/perf/perf_gm20b.c,
+             hal/perf/perf_gm20b.h,
+             hal/perf/perf_gv11b.c,
+             hal/perf/perf_gv11b.h,
+             hal/perf/perf_tu104.c,
+             hal/perf/perf_tu104.h ]
+
+pramin:
+  safe: yes
+  gpu: dgpu
+  sources: [ hal/pramin/pramin_gp10b.c,
+             hal/pramin/pramin_gp10b.h,
+             hal/pramin/pramin_gv100.c,
+             hal/pramin/pramin_gv100.h,
+             hal/pramin/pramin_init.c,
+             hal/pramin/pramin_init.h,
+             hal/pramin/pramin_tu104.c,
+             hal/pramin/pramin_tu104.h ]
+
+class_fusa:
+  safe: yes
+  owner: Seshendra G
+  sources: [ hal/class/class_gm20b.h,
+             hal/class/class_gp10b.h,
+             hal/class/class_gv11b_fusa.c,
+             hal/class/class_gv11b.h ]
+
+class:
+  safe: no
+  owner: Seshendra G
+  sources: [ hal/class/class_gm20b.c,
+             hal/class/class_gp10b.c,
+             hal/class/class_tu104.c,
+             hal/class/class_tu104.h ]
+
+func:
+  safe: yes
+  owner: Terje B
+  gpu: dgpu
+  sources: [ hal/func/func_tu104.c,
+             hal/func/func_tu104.h ]
+
+top_fusa:
+  safe: yes
+  owner: Tejal K
+  sources: [ include/nvgpu/gops/top.h,
+             hal/top/top_gm20b_fusa.c,
+             hal/top/top_gm20b.h,
+             hal/top/top_gp10b.h,
+             hal/top/top_gv11b_fusa.c,
+             hal/top/top_gv11b.h ]
+
+top:
+  safe: no
+  owner: Tejal K
+  sources: [ hal/top/top_gm20b.c,
+             hal/top/top_gp106.c,
+             hal/top/top_gp106.h,
+             hal/top/top_gp10b.c,
+             hal/top/top_gv100.c,
+             hal/top/top_gv100.h ]
+
+bios:
+  safe: yes
+  owner: Tejal
+  gpu: dgpu
+  sources: [ hal/bios/bios_tu104.c,
+             hal/bios/bios_tu104.h ]
+
+xve:
+  safe: yes
+  owner: Alex W
+  gpu: dgpu
+  sources: [ include/nvgpu/xve.h,
+             hal/xve/xve_gp106.c,
+             hal/xve/xve_gp106.h,
+             hal/xve/xve_tu104.c,
+             hal/xve/xve_tu104.h ]
+
+tpc:
+  safe: no
+  owner: Divya S
+  sources: [ hal/tpc/tpc_gv11b.c,
+             hal/tpc/tpc_gv11b.h ]
+
+cic:
+  safe: yes
+  owner: Tejal K
+  sources: [ hal/cic/cic_gv11b_fusa.c,
+             hal/cic/cic_lut_gv11b_fusa.c,
+             hal/cic/cic_gv11b.h ]

arch/nvgpu-hal-vgpu.yaml

@@ -0,0 +1,29 @@
+# Copyright (c) 2019-2021, NVIDIA CORPORATION.  All Rights Reserved.
+#
+# VGPU HAL units.
+#
+
+init:
+  safe: no
+  owner: Aparna D
+  sources: [ hal/vgpu/init/init_hal_vgpu.c,
+             hal/vgpu/init/init_hal_vgpu.h,
+             hal/vgpu/init/vgpu_hal_gv11b.c,
+             hal/vgpu/init/vgpu_hal_gv11b.h ]
+
+fifo:
+  safe : no
+  owner: Aparna D
+  children:
+    fifo:
+     safe: no
+     sources: [ hal/vgpu/fifo/fifo_gv11b_vgpu.c,
+                hal/vgpu/fifo/fifo_gv11b_vgpu.h ]
+
+sync:
+  safe: no
+  owner: Aparna D
+  children:
+    syncpt:
+      sources: [ hal/vgpu/sync/syncpt_cmdbuf_gv11b_vgpu.c,
+                 hal/vgpu/sync/syncpt_cmdbuf_gv11b_vgpu.h ]

arch/nvgpu-hal.yaml

@@ -0,0 +1,10 @@
+# Copyright (c) 2019-2020, NVIDIA CORPORATION.  All Rights Reserved.
+#
+# HAL units. These are the units that have access to HW.
+#
+
+init:
+  safe: yes
+  owner: Terje B
+  sources: [ include/nvgpu/hal_init.h ]
+

arch/nvgpu-interface.yaml

@@ -0,0 +1,132 @@
+# Copyright (c) 2019, NVIDIA CORPORATION.  All Rights Reserved.
+#
+# OS interface units and utilities. Often represented by simply a header file.
+#
+
+atomic:
+  safe: yes
+  sources: [ include/nvgpu/atomic.h ]
+
+barrier:
+  safe: yes
+  sources: [ include/nvgpu/barrier.h ]
+
+bitops:
+  safe: yes
+  sources: [ include/nvgpu/bitops.h ]
+
+bsearch:
+  safe: yes
+  sources: [ include/nvgpu/bsearch.h ]
+
+bug:
+  safe: yes
+  sources: [ include/nvgpu/bug.h ]
+
+circ_buf:
+  safe: yes
+  sources: [ include/nvgpu/circ_buf.h ]
+
+cond:
+  safe: yes
+  sources: [ include/nvgpu/cond.h ]
+
+dma:
+  safe: yes
+  sources: [ include/nvgpu/dma.h ]
+
+# This one is likely not structured correctly.
+dt:
+  safe: yes
+  sources: [ include/nvgpu/dt.h ]
+
+# Also a problem.
+errno:
+  safe: yes
+  sources: [ include/nvgpu/errno.h ]
+
+firmware:
+  safe: yes
+  sources: [ include/nvgpu/firmware.h ]
+
+fuse:
+  safe: yes
+  sources: [ include/nvgpu/fuse.h ]
+
+io:
+  safe: yes
+  sources: [ include/nvgpu/io.h, include/nvgpu/io_usermode.h ]
+
+kmem:
+  safe: yes
+  sources: [ include/nvgpu/kmem.h ]
+
+kref:
+  safe: yes
+  sources: [ include/nvgpu/kref.h ]
+
+list:
+  safe: yes
+  sources: [ include/nvgpu/list.h ]
+  tags: unit-testable
+
+lock:
+  safe: yes
+  sources: [ include/nvgpu/lock.h ]
+
+log:
+  safe: yes
+  sources: [ include/nvgpu/log.h, include/nvgpu/log_common.h ]
+
+log2:
+  safe: yes
+  sources: [ include/nvgpu/log2.h ]
+
+rwsem:
+  safe: yes
+  sources: [ include/nvgpu/rwsem.h ]
+
+sizes:
+  safe: yes
+  sources: [ include/nvgpu/sizes.h ]
+
+soc:
+  safe: yes
+  sources: [ include/nvgpu/soc.h ]
+
+sort:
+  safe: yes
+  sources: [ include/nvgpu/sort.h ]
+
+string:
+  safe: yes
+  sources: [ include/nvgpu/string.h ]
+  tags: unit-testable
+
+thread:
+  safe: yes
+  sources: [ include/nvgpu/thread.h ]
+
+timers:
+  safe: yes
+  sources: [ include/nvgpu/timers.h ]
+
+types:
+  safe: yes
+  sources: [ include/nvgpu/types.h ]
+
+nvgpu_sgt:
+  safe: yes
+  sources: [ include/nvgpu/nvgpu_sgt_os.h ]
+
+os_sched:
+  safe: yes
+  sources: [ include/nvgpu/os_sched.h ]
+
+nvhost:
+  safe: yes
+  sources: [ include/nvgpu/nvhost.h ]
+
+vpr:
+  safe: no
+  sources: [ include/nvgpu/vpr.h ]

arch/nvgpu-linux.yaml

@@ -0,0 +1,256 @@
+#
+# Copyright (c) 2019-2021, NVIDIA CORPORATION.  All Rights Reserved.
+#
+# Linux elements and units in nvgpu.
+#
+# The safe: tag is ommited through out since all Linux units are by definition
+# not safe.
+#
+# I also have not put a huge amount of thought into this since none of this
+# code is "safe" code. Nor are we planning on spending a lot of effort to
+# clean this up. At least not yet.
+
+bsearch:
+  sources: [ os/linux/bsearch.c ]
+
+channel:
+  sources: [ os/linux/linux-channel.c, os/linux/channel.h ]
+
+vpr:
+  sources: [ os/linux/vpr.c ]
+
+clk:
+  sources: [ os/linux/clk.c, os/linux/clk.h ]
+
+cde:
+  sources: [ os/linux/cde.c,       os/linux/cde.h,
+             os/linux/cde_gm20b.c, os/linux/cde_gm20b.h,
+             os/linux/cde_gp10b.c, os/linux/cde_gp10b.h  ]
+
+comptags:
+  sources: [ os/linux/comptags.c ]
+
+cond:
+  sources: [ os/linux/cond.c ]
+
+dma:
+  sources: [ os/linux/linux-dma.c ]
+
+dmabuf:
+  sources: [ os/linux/dmabuf_vidmem.c, os/linux/dmabuf_vidmem.h,
+             os/linux/dmabuf_priv.c, os/linux/dmabuf_priv.h ]
+
+driver_common:
+  sources: [ os/linux/driver_common.c, os/linux/driver_common.h ]
+
+dt:
+  sources: [ os/linux/dt.c ]
+
+debug:
+  sources: [ os/linux/debug.c,
+             os/linux/debug_allocator.c,
+             os/linux/debug_allocator.h,
+             os/linux/debug_bios.c,
+             os/linux/debug_bios.h,
+             os/linux/debug_cde.c,
+             os/linux/debug_cde.h,
+             os/linux/debug_ce.c,
+             os/linux/debug_ce.h,
+             os/linux/debug_clk_gm20b.c,
+             os/linux/debug_clk_gm20b.h,
+             os/linux/debug_clk_tu104.c,
+             os/linux/debug_clk_tu104.h,
+             os/linux/debug_fecs_trace.c,
+             os/linux/debug_fecs_trace.h,
+             os/linux/debug_fifo.c,
+             os/linux/debug_fifo.h,
+             os/linux/debug_gr.c,
+             os/linux/debug_gr.h,
+             os/linux/debug_hal.c,
+             os/linux/debug_hal.h,
+             os/linux/debug_kmem.c,
+             os/linux/debug_kmem.h,
+             os/linux/debug_ltc.c,
+             os/linux/debug_ltc.h,
+             os/linux/debug_pmgr.c,
+             os/linux/debug_pmgr.h,
+             os/linux/debug_pmu.c,
+             os/linux/debug_pmu.h,
+             os/linux/debug_sched.c,
+             os/linux/debug_sched.h,
+             os/linux/debug_therm_tu104.c,
+             os/linux/debug_therm_tu104.h,
+             os/linux/debug_xve.c,
+             os/linux/debug_xve.h,
+             os/linux/debug_s_param.c,
+             os/linux/debug_s_param.h,
+             os/linux/debug_volt.c,
+             os/linux/debug_volt.h,
+             os/linux/swprofile_debugfs.c,
+             os/linux/swprofile_debugfs.h,
+             os/linux/fecs_trace_linux.c,
+             os/linux/fecs_trace_linux.h,
+             os/linux/nvlink_probe.c ]
+
+firmware:
+  sources: [ os/linux/firmware.c ]
+
+fuse:
+  sources: [ os/linux/fuse.c ]
+
+intr:
+  sources: [ os/linux/intr.c ]
+
+io:
+  sources: [ os/linux/io_usermode.c,
+             os/linux/linux-io.c ]
+
+ioctl:
+  sources: [ os/linux/ioctl.c,
+             os/linux/ioctl.h,
+             os/linux/ioctl_as.c,
+             os/linux/ioctl_as.h,
+             os/linux/ioctl_channel.c,
+             os/linux/ioctl_channel.h,
+             os/linux/ioctl_clk_arb.c,
+             os/linux/ioctl_ctrl.c,
+             os/linux/ioctl_ctrl.h,
+             os/linux/ioctl_dbg.c,
+             os/linux/ioctl_dbg.h,
+             os/linux/ioctl_prof.c,
+             os/linux/ioctl_prof.h,
+             os/linux/ioctl_tsg.c,
+             os/linux/ioctl_tsg.h,
+             os/linux/power_ops.c,
+             os/linux/power_ops.h ]
+
+kmem:
+  sources: [ os/linux/kmem.c, os/linux/kmem_priv.h ]
+
+log:
+  sources: [ os/linux/log.c ]
+
+module:
+  sources: [ os/linux/module.c,          os/linux/module.h,
+             os/linux/module_usermode.c, os/linux/module_usermode.h ]
+
+nvgpu_mem:
+  sources: [ os/linux/nvgpu_mem.c ]
+
+nvhost:
+  sources: [ os/linux/nvhost.c,
+             os/linux/nvhost_common.c,
+             os/linux/nvhost_host1x.c,
+             os/linux/nvhost_priv.h ]
+
+nvlink:
+  sources: [ os/linux/nvlink.c,
+             os/linux/nvlink.h ]
+
+fence:
+  sources: [ os/linux/os_fence_android.c,
+             os/linux/os_fence_android_sema.c,
+             os/linux/os_fence_dma.c,
+             os/linux/os_fence_dma_sema.c,
+             os/linux/os_fence_syncpt.c,
+             os/linux/os_fence_priv.h,
+             os/linux/sync_sema_android.c,
+             os/linux/sync_sema_android.h,
+             os/linux/sync_sema_dma.c,
+             os/linux/sync_sema_dma.h,
+             include/nvgpu/os_fence.h,
+             include/nvgpu/os_fence_semas.h,
+             include/nvgpu/os_fence_syncpts.h ]
+
+ops:
+  sources: [ os/linux/os_ops.c,       os/linux/os_ops.h,
+             os/linux/os_ops_gm20b.c, os/linux/os_ops_gm20b.h,
+             os/linux/os_ops_gp10b.c, os/linux/os_ops_gp10b.h,
+             os/linux/os_ops_gv100.c, os/linux/os_ops_gv100.h,
+             os/linux/os_ops_gv11b.c, os/linux/os_ops_gv11b.h,
+             os/linux/os_ops_tu104.c, os/linux/os_ops_tu104.h ]
+
+pci:
+  sources: [ os/linux/pci.c,
+             os/linux/pci.h,
+             os/linux/pci_power.c,
+             os/linux/pci_power.h,
+             include/nvgpu/pci.h ]
+
+platform:
+  sources: [ os/linux/platform_gk20a.h,
+             os/linux/platform_gk20a_tegra.c,
+             os/linux/platform_gk20a_tegra.h,
+             os/linux/platform_gp10b.h,
+             os/linux/platform_gp10b_tegra.c,
+             os/linux/platform_gv11b_tegra.c ]
+
+rwsem:
+  sources: [ os/linux/rwsem.c ]
+
+scale:
+  sources: [os/linux/scale.c, os/linux/scale.h ]
+
+sched:
+  sources: [ os/linux/os_sched.c,
+             os/linux/sched.c,
+             os/linux/sched.h ]
+
+sim:
+  sources: [ os/linux/sim.c, os/linux/sim_pci.c ]
+
+soc:
+  sources: [ os/linux/soc.c ]
+
+sysfs:
+  sources: [ os/linux/sysfs.c,     os/linux/sysfs.h,
+             os/linux/ecc_sysfs.c ]
+
+thread:
+  sources: [ os/linux/thread.c ]
+
+timers:
+  sources: [ os/linux/timers.c ]
+
+vgpu:
+  sources: [ os/linux/vgpu/fecs_trace_vgpu_linux.c,
+             os/linux/vgpu/gv11b/platform_gv11b_vgpu_tegra.c,
+             os/linux/vgpu/gv11b/platform_gv11b_vgpu_tegra.h,
+             os/linux/vgpu/platform_vgpu_tegra.c,
+             os/linux/vgpu/platform_vgpu_tegra.h,
+             os/linux/vgpu/sysfs_vgpu.c,
+             os/linux/vgpu/vgpu_ivc.c,
+             os/linux/vgpu/vgpu_ivm.c,
+             os/linux/vgpu/vgpu_linux.c,
+             os/linux/vgpu/vgpu_linux.h ]
+
+vm:
+  sources: [ os/linux/vm.c ]
+
+cic:
+  sources: [ os/linux/cic/cic_stub.c ]
+
+# Group all the Linux headers for now.
+headers:
+  sources: [ include/nvgpu/linux/atomic.h,
+             include/nvgpu/linux/barrier.h,
+             include/nvgpu/linux/cond.h,
+             include/nvgpu/linux/dma.h,
+             include/nvgpu/linux/kmem.h,
+             include/nvgpu/linux/lock.h,
+             include/nvgpu/linux/bitops.h,
+             include/nvgpu/linux/nvgpu_mem.h,
+             include/nvgpu/linux/os_fence_android.h,
+             include/nvgpu/linux/os_fence_dma.h,
+             include/nvgpu/linux/rwsem.h,
+             include/nvgpu/linux/sim.h,
+             include/nvgpu/linux/sim_pci.h,
+             include/nvgpu/linux/soc_fuse.h,
+             include/nvgpu/linux/thread.h,
+             include/nvgpu/linux/log.h,
+             include/nvgpu/linux/utils.h,
+             include/nvgpu/linux/vm.h ]
+
+# An extra unit to lump all the unclassified Linux files.
+extra:
+  sources: [ os/linux/os_linux.h ]

arch/nvgpu-posix.yaml

@@ -0,0 +1,171 @@
+# Copyright (c) 2019-2021, NVIDIA CORPORATION.  All Rights Reserved.
+#
+# POSIX elements and units in nvgpu.
+#
+
+# TODO: break this up into individual units. Will be critical for re-use in
+# QNX.
+all:
+  safe: no
+  owner: Alex W
+  sources: [ os/posix/error_notifier.c,
+             os/posix/firmware.c,
+             os/posix/fuse.c,
+             os/posix/gk20a.c,
+             os/posix/log.c,
+             os/posix/nvgpu.c,
+             os/posix/os_posix.h,
+             os/posix/posix-channel.c,
+             os/posix/posix-comptags.c,
+             os/posix/posix-dma.c,
+             os/posix/posix-fault-injection.c,
+             os/posix/posix-nvgpu_mem.c,
+             os/posix/posix-tsg.c,
+             os/posix/posix-vm.c,
+             os/posix/soc.c,
+             os/posix/bsearch.c,
+             os/posix/posix-clk_arb.c,
+             os/posix/posix-dt.c,
+             os/posix/posix-io.c,
+             os/posix/posix-nvhost.c,
+             os/posix/posix-nvlink.c,
+             os/posix/posix-vgpu.c,
+             os/posix/posix-vidmem.c,
+             os/posix/fecs_trace_posix.c,
+             os/posix/stubs.c,
+             os/posix/posix-vpr.c,
+             os/posix/mock-registers.c ]
+
+headers:
+  safe: no
+  owner: Alex W
+  sources: [ include/nvgpu/posix/barrier.h,
+             include/nvgpu/posix/dma.h,
+             include/nvgpu/posix/io.h,
+             include/nvgpu/posix/mock-regs.h,
+             include/nvgpu/posix/log.h,
+             include/nvgpu/posix/nvgpu_mem.h,
+             include/nvgpu/posix/pci.h,
+             include/nvgpu/posix/posix-fault-injection.h,
+             include/nvgpu/posix/posix-channel.h,
+             include/nvgpu/posix/probe.h,
+             include/nvgpu/posix/soc_fuse.h,
+             include/nvgpu/posix/vm.h,
+             include/nvgpu/posix/posix_vidmem.h,
+             include/nvgpu/posix/posix-nvhost.h,
+             include/nvgpu/posix/trace_gk20a.h ]
+
+queue:
+  safe: yes
+  owner: Rajesh D
+  sources: [ os/posix/queue.c,
+             include/nvgpu/posix/queue.h ]
+
+sort:
+  safe: no
+  owner: Ajesh K
+  sources: [ include/nvgpu/posix/sort.h ]
+  deps:
+
+bug:
+  safe: yes
+  owner: Ajesh K
+  sources: [ os/posix/bug.c,
+             include/nvgpu/posix/bug.h ]
+  deps:
+
+lock:
+  safe: yes
+  owner: Ajesh K
+  sources: [ os/posix/lock.c,
+             include/nvgpu/posix/lock.h ]
+
+rwsem:
+  safe: yes
+  owner: Ajesh K
+  sources: [ os/posix/rwsem.c,
+             include/nvgpu/posix/rwsem.h ]
+  deps:
+
+size:
+  safe: yes
+  owner: Ajesh K
+  sources: [ include/nvgpu/posix/sizes.h ]
+  deps:
+
+cond:
+  safe: yes
+  owner: Ajesh K
+  sources: [ os/posix/cond.c,
+             include/nvgpu/posix/cond.h ]
+  deps:
+
+threads:
+  safe: yes
+  owner: Ajesh K
+  sources: [ os/posix/thread.c,
+             include/nvgpu/posix/thread.h ]
+  deps:
+
+timers:
+  safe: yes
+  owner: Ajesh K
+  sources: [ os/posix/timers.c,
+             include/nvgpu/posix/timers.h ]
+  deps:
+
+atomic:
+  safe: yes
+  owner: Ajesh K
+  sources: [ include/nvgpu/posix/atomic.h ]
+  deps:
+
+os_sched:
+  safe: yes
+  owner: Ajesh K
+  sources: [ os/posix/os_sched.c ]
+
+kmem:
+  safe: yes
+  owner: Ajesh K
+  sources: [ os/posix/kmem.c,
+             include/nvgpu/posix/kmem.h ]
+  deps:
+
+types:
+  safe: yes
+  owner: Ajesh K
+  sources: [ include/nvgpu/posix/types.h ]
+  deps:
+
+utils:
+  safe: yes
+  owner: Ajesh K
+  sources: [ include/nvgpu/posix/utils.h ]
+  deps:
+
+bitops:
+  safe: yes
+  owner: Ajesh K
+  sources: [ os/posix/bitmap.c,
+             include/nvgpu/posix/bitops.h ]
+  deps:
+
+file_ops:
+  safe: yes
+  owner: Prateek S
+  sources: [ os/posix/file_ops.c,
+             include/nvgpu/posix/file_ops.h ]
+  deps:
+
+log2:
+  safe: yes
+  owner: Ajesh K
+  sources: [ include/nvgpu/posix/log2.h ]
+  deps:
+
+circ_buf:
+  safe: yes
+  owner: Ajesh K
+  sources: [ include/nvgpu/posix/circ_buf.h ]
+  deps:

arch/nvgpu-vgpu.yaml

@@ -0,0 +1,78 @@
+# Copyright (c) 2019-2021, NVIDIA CORPORATION.  All Rights Reserved.
+#
+# vGPU architecture: currently there hasn't been much work done on
+# decomposing the vGPU architecture so all of the vGPU files are simply
+# grouped into one super unit.
+#
+# TODO: Fix this !?!? Seems rather important given that this is the
+#       target for safety!
+#
+
+all:
+  safe: no
+  owner: Aparna D
+  sources: [ common/vgpu/cbc/cbc_vgpu.c,
+             common/vgpu/cbc/cbc_vgpu.h,
+             common/vgpu/ce_vgpu.c,
+             common/vgpu/clk_vgpu.c,
+             common/vgpu/clk_vgpu.h,
+             common/vgpu/debugger_vgpu.c,
+             common/vgpu/debugger_vgpu.h,
+             common/vgpu/pm_reservation_vgpu.c,
+             common/vgpu/pm_reservation_vgpu.h,
+             common/vgpu/ecc_vgpu.c,
+             common/vgpu/ecc_vgpu.h,
+             common/vgpu/top/top_vgpu.c,
+             common/vgpu/top/top_vgpu.h,
+             common/vgpu/fifo/fifo_vgpu.c,
+             common/vgpu/fifo/fifo_vgpu.h,
+             common/vgpu/fifo/channel_vgpu.c,
+             common/vgpu/fifo/channel_vgpu.h,
+             common/vgpu/fifo/tsg_vgpu.c,
+             common/vgpu/fifo/tsg_vgpu.h,
+             common/vgpu/fifo/preempt_vgpu.c,
+             common/vgpu/fifo/preempt_vgpu.h,
+             common/vgpu/fifo/ramfc_vgpu.c,
+             common/vgpu/fifo/ramfc_vgpu.h,
+             common/vgpu/fifo/runlist_vgpu.c,
+             common/vgpu/fifo/runlist_vgpu.h,
+             common/vgpu/fifo/userd_vgpu.c,
+             common/vgpu/fifo/userd_vgpu.h,
+             common/vgpu/gr/ctx_vgpu.c,
+             common/vgpu/gr/ctx_vgpu.h,
+             common/vgpu/gr/fecs_trace_vgpu.c,
+             common/vgpu/gr/fecs_trace_vgpu.h,
+             common/vgpu/gr/gr_vgpu.c,
+             common/vgpu/gr/gr_vgpu.h,
+             common/vgpu/gr/subctx_vgpu.c,
+             common/vgpu/gr/subctx_vgpu.h,
+             common/vgpu/ltc/ltc_vgpu.c,
+             common/vgpu/ltc/ltc_vgpu.h,
+             common/vgpu/mm/mm_vgpu.c,
+             common/vgpu/mm/mm_vgpu.h,
+             common/vgpu/mm/vm_vgpu.c,
+             common/vgpu/perf/cyclestats_snapshot_vgpu.c,
+             common/vgpu/perf/cyclestats_snapshot_vgpu.h,
+             common/vgpu/perf/perf_vgpu.c,
+             common/vgpu/perf/perf_vgpu.h,
+             common/vgpu/fbp/fbp_vgpu.c,
+             common/vgpu/fbp/fbp_vgpu.h,
+             common/vgpu/fb/fb_vgpu.c,
+             common/vgpu/fb/fb_vgpu.h,
+             common/vgpu/intr/intr_vgpu.c,
+             common/vgpu/intr/intr_vgpu.h,
+             common/vgpu/ivc/comm_vgpu.c,
+             common/vgpu/ivc/comm_vgpu.h,
+             common/vgpu/ptimer/ptimer_vgpu.c,
+             common/vgpu/ptimer/ptimer_vgpu.h,
+             common/vgpu/init/init_vgpu.c,
+             common/vgpu/init/init_vgpu.h,
+             common/vgpu/profiler/profiler_vgpu.c,
+             common/vgpu/profiler/profiler_vgpu.h,
+             include/nvgpu/vgpu/tegra_vgpu.h,
+             include/nvgpu/vgpu/vgpu.h,
+             include/nvgpu/vgpu/vgpu_ivc.h,
+             include/nvgpu/vgpu/vgpu_ivm.h,
+             include/nvgpu/vgpu/ce_vgpu.h,
+             include/nvgpu/vgpu/vm_vgpu.h,
+             include/nvgpu/vgpu/os_init_hal_vgpu.h ]

arch/nvgpu.yaml

@@ -0,0 +1,69 @@
+# Copyright (c) 2019-2020, NVIDIA CORPORATION.  All Rights Reserved.
+#
+# Top level NVGPU architecure description in YAML.
+#
+# The format of this document is structured by element and unit. Though only
+# units may have source files associated with them. All units must exist in an
+# element.
+#
+
+nvgpu:
+  safe: no
+  children:
+    # The common code element. This has gr, mm, etc.
+    common:
+      safe: no
+      children:
+        !include nvgpu-common.yaml nvgpu-next-common.yaml
+
+    # HAL units - Hardware Abstraction Layer.
+    hal:
+      safe: no
+      children:
+        !include nvgpu-hal-new.yaml nvgpu-next-hal.yaml
+
+    # The QNX OS layer implementation units.
+    qnx:
+      safe: yes
+      children:
+        !include nvgpu-qnx.yaml
+
+    # And the Linux version of the OS implementation units.
+    linux:
+      safe: no
+      children:
+        !include nvgpu-linux.yaml nvgpu-next-linux.yaml
+
+    # POSIX units for implementing the OS layer for unit testing.
+    posix:
+      children:
+        !include nvgpu-posix.yaml
+
+
+    # Inteface units - these provide interfaces for NVGPU to the underlying
+    # OS or CPU.
+    interface:
+      safe: yes
+      owner: Alex W
+      children:
+        !include nvgpu-interface.yaml
+
+    # Virtualization code.
+    vgpu:
+      safe: yes
+      children:
+        !include nvgpu-vgpu.yaml
+
+    # Virtualization HAL code.
+    hal-vgpu:
+      safe: yes
+      children:
+        !include nvgpu-hal-vgpu.yaml nvgpu-next-hal-vgpu.yaml
+
+    # A meta-element for the GPU HW. A good example of this is the HW headers.
+    # This is not code we write in nvgpu, but we import it from the GPU HW
+    # tree (with a little tranformation - the gen reg scrip).
+    gpu_hw:
+      safe: no
+      children:
+        !include nvgpu-gpu_hw.yaml nvgpu-next-gpu_hw.yaml

drivers/gpu/Makefile

@@ -0,0 +1,2 @@
+# Choose this option if you have an SoC with integrated Nvidia GPU IP.
+obj-m += nvgpu/

drivers/gpu/nvgpu/Makefile

@@ -0,0 +1,788 @@
+GCOV_PROFILE := y
+
+# When building NVGPU as an external module, srctree.nvgpu is not
+# defined. Given that NVGPU knows where its header files are located
+# relatively, we can use the path information of this Makefile to
+# set srctree.nvgpu if it is not already defined. Please note that
+# 'lastword $(MAKEFILE_LIST)' refers to this Makefile.
+srctree.nvgpu ?= $(abspath $(shell dirname $(lastword $(MAKEFILE_LIST)))/../../..)
+
+ccflags-y += -I$(srctree.nvgpu)/drivers/gpu/nvgpu/include
+ccflags-y += -I$(srctree.nvgpu)/drivers/gpu/nvgpu
+ccflags-y += -I$(srctree.nvgpu)/include
+ccflags-y += -I$(srctree.nvgpu)/include/uapi
+
+ccflags-y += -I$(srctree)/drivers/devfreq
+
+ccflags-y += -Wframe-larger-than=2048
+ccflags-y += -Wno-multichar
+ccflags-y += -Werror
+ccflags-y += -Wno-error=cpp
+ifeq ($(VERSION),4)
+ccflags-y += -Wextra -Wno-unused-parameter -Wno-missing-field-initializers
+endif
+
+# Turn off when this is fixed upstream, if ever.
+ccflags-y += -D__NVGPU_PREVENT_UNTRUSTED_SPECULATION
+
+ifneq ($(srctree.nvgpu),)
+include $(srctree.nvgpu)/drivers/gpu/nvgpu/Makefile.linux.configs
+else
+include Makefile.linux.configs
+endif
+
+# When using the upstream host1x driver, the Makefile must define the
+# srctree.host1x path in order to find the necessary header files for
+# the upstream host1x driver.
+ifeq ($(CONFIG_TEGRA_GK20A_NVHOST_HOST1X),y)
+ccflags-y += -I$(srctree.host1x)/include
+endif
+
+ifeq ($(CONFIG_NVGPU_DEBUGGER),y)
+ccflags-y += -DCONFIG_NVGPU_DEBUGGER
+ccflags-y += -DCONFIG_NVGPU_ENGINE_RESET
+endif
+
+ccflags-y += -DCONFIG_NVGPU_DETERMINISTIC_CHANNELS
+ccflags-y += -DCONFIG_NVGPU_TPC_POWERGATE
+ccflags-y += -DCONFIG_NVGPU_ACR_LEGACY
+ccflags-y += -DCONFIG_NVGPU_ENGINE_QUEUE
+ccflags-y += -DCONFIG_NVGPU_FIFO_ENGINE_ACTIVITY
+ccflags-y += -DCONFIG_NVGPU_USERD
+ccflags-y += -DCONFIG_NVGPU_CHANNEL_WDT
+ccflags-y += -DCONFIG_NVGPU_LS_PMU
+ccflags-y += -DCONFIG_NVGPU_CILP
+ccflags-y += -DCONFIG_NVGPU_GFXP
+ccflags-y += -DCONFIG_NVGPU_GRAPHICS
+ccflags-y += -DCONFIG_NVGPU_REPLAYABLE_FAULT
+ccflags-y += -DCONFIG_NVGPU_CHANNEL_TSG_SCHEDULING
+ccflags-y += -DCONFIG_NVGPU_CHANNEL_TSG_CONTROL
+ccflags-y += -DCONFIG_NVGPU_POWER_PG
+ccflags-y += -DCONFIG_NVGPU_KERNEL_MODE_SUBMIT
+ccflags-y += -DCONFIG_NVGPU_SIM
+ccflags-y += -DCONFIG_NVGPU_TRACE
+ccflags-y += -DCONFIG_NVGPU_SYSFS
+ccflags-y += -DCONFIG_NVGPU_CLK_ARB
+ccflags-y += -DCONFIG_NVGPU_FALCON_DEBUG
+ccflags-y += -DCONFIG_NVGPU_FALCON_NON_FUSA
+ccflags-y += -DCONFIG_NVGPU_IOCTL_NON_FUSA
+ccflags-y += -DCONFIG_NVGPU_NON_FUSA
+ccflags-y += -DCONFIG_NVGPU_INJECT_HWERR
+ccflags-y += -DCONFIG_NVGPU_GR_FALCON_NON_SECURE_BOOT
+ccflags-y += -DCONFIG_NVGPU_SET_FALCON_ACCESS_MAP
+ccflags-y += -DCONFIG_NVGPU_SW_SEMAPHORE
+ccflags-y += -DCONFIG_NVGPU_FENCE
+ccflags-y += -DCONFIG_NVGPU_PROFILER
+
+ifeq ($(CONFIG_NVGPU_LOGGING),y)
+ccflags-y += -DCONFIG_NVGPU_LOGGING=1
+endif
+
+ifeq ($(CONFIG_NVGPU_HAL_NON_FUSA),y)
+ccflags-y += -DCONFIG_NVGPU_HAL_NON_FUSA
+endif
+
+ifeq ($(CONFIG_NVGPU_RECOVERY),y)
+ccflags-y += -DCONFIG_NVGPU_RECOVERY
+ccflags-y += -DCONFIG_NVGPU_ENGINE_RESET
+nvgpu-y += \
+	hal/rc/rc_gk20a.o \
+	hal/rc/rc_gv11b.o
+endif
+
+obj-$(CONFIG_GK20A) := nvgpu.o
+
+# OS independent parts of nvgpu. The work to collect files here
+# is in progress.
+
+ifeq ($(CONFIG_NVGPU_DGPU),y)
+nvgpu-$(CONFIG_NVGPU_DGPU) += \
+	os/linux/pci.o \
+	os/linux/pci_power.o \
+	os/linux/dmabuf_vidmem.o \
+	os/linux/os_ops_gv100.o \
+	os/linux/os_ops_tu104.o \
+	common/sec2/sec2.o \
+	common/sec2/sec2_allocator.o \
+	common/sec2/sec2_lsfm.o \
+	common/sec2/ipc/sec2_cmd.o \
+	common/sec2/ipc/sec2_msg.o \
+	common/sec2/ipc/sec2_queue.o \
+	common/sec2/ipc/sec2_seq.o \
+	common/vbios/bios_sw_gv100.o \
+	common/vbios/bios_sw_tu104.o \
+	common/falcon/falcon_sw_tu104.o \
+	common/acr/acr_sw_tu104.o \
+	common/mm/allocators/page_allocator.o \
+	common/mm/vidmem.o \
+	common/pramin.o \
+	common/ce/ce_app.o \
+	common/clk_arb/clk_arb_gv100.o \
+	common/engine_queues/engine_emem_queue.o \
+	hal/mm/mm_gv100.o \
+	hal/mm/mm_tu104.o \
+	hal/mc/mc_gv100.o  \
+	hal/mc/mc_tu104.o  \
+	hal/bus/bus_gv100.o \
+	hal/bus/bus_tu104.o \
+	hal/class/class_tu104.o \
+	hal/clk/clk_tu104.o \
+	hal/clk/clk_mon_tu104.o \
+	hal/gr/init/gr_init_gv100.o \
+	hal/gr/init/gr_init_tu104.o \
+	hal/gr/intr/gr_intr_tu104.o \
+	hal/gr/falcon/gr_falcon_tu104.o \
+	hal/fbpa/fbpa_tu104.o \
+	hal/init/hal_tu104.o \
+	hal/init/hal_tu104_litter.o \
+	hal/power_features/cg/tu104_gating_reglist.o \
+	hal/ltc/ltc_tu104.o \
+	hal/fb/fb_gv100.o \
+	hal/fb/fb_tu104.o \
+	hal/fb/fb_mmu_fault_tu104.o \
+	hal/fb/intr/fb_intr_gv100.o \
+	hal/fb/intr/fb_intr_tu104.o \
+	hal/func/func_tu104.o \
+	hal/fifo/fifo_tu104.o \
+	hal/fifo/usermode_tu104.o \
+	hal/fifo/pbdma_tu104.o \
+	hal/fifo/ramfc_tu104.o \
+	hal/fifo/ramin_tu104.o \
+	hal/fifo/channel_gv100.o \
+	hal/fifo/runlist_ram_tu104.o \
+	hal/fifo/runlist_fifo_gv100.o \
+	hal/fifo/runlist_fifo_tu104.o \
+	hal/fifo/fifo_intr_gv100.o \
+	hal/fuse/fuse_gp106.o \
+	hal/fuse/fuse_tu104.o \
+	hal/netlist/netlist_gv100.o \
+	hal/netlist/netlist_tu104.o \
+	hal/nvdec/nvdec_gp106.o \
+	hal/nvdec/nvdec_tu104.o \
+	hal/gsp/gsp_tu104.o \
+	hal/sec2/sec2_tu104.o \
+	hal/pramin/pramin_gp10b.o \
+	hal/pramin/pramin_gv100.o \
+	hal/pramin/pramin_init.o \
+	hal/pramin/pramin_tu104.o \
+	hal/bios/bios_tu104.o \
+	hal/top/top_gv100.o \
+	hal/xve/xve_gp106.o \
+	hal/xve/xve_tu104.o
+
+nvgpu-$(CONFIG_DEBUG_FS) += \
+	os/linux/debug_therm_tu104.o \
+	os/linux/debug_bios.o \
+	os/linux/debug_xve.o \
+	os/linux/debug_clk_tu104.o
+endif
+
+# nvlink sources are not conditionally compiled. nvlink probe and
+# public functions return -ENODEV when not supported.
+
+nvgpu-y += \
+	common/vbios/nvlink_bios.o \
+	common/nvlink/probe.o \
+	common/nvlink/init/device_reginit.o \
+	common/nvlink/init/device_reginit_gv100.o \
+	common/nvlink/minion.o \
+	common/nvlink/link_mode_transitions.o \
+	common/nvlink/nvlink.o \
+	os/linux/nvlink_probe.o \
+	os/linux/nvlink.o \
+	hal/nvlink/minion_gv100.o \
+	hal/nvlink/minion_tu104.o \
+	hal/nvlink/nvlink_gv100.o \
+	hal/nvlink/nvlink_tu104.o \
+	hal/nvlink/intr_and_err_handling_tu104.o \
+	hal/nvlink/link_mode_transitions_gv100.o \
+	hal/nvlink/link_mode_transitions_tu104.o
+
+nvgpu-y += \
+	common/device.o \
+	common/utils/enabled.o \
+	common/utils/errata.o \
+	common/utils/rbtree.o \
+	common/utils/string.o \
+	common/utils/worker.o \
+	common/swdebug/profile.o \
+	common/ptimer/ptimer.o \
+	common/perf/perfbuf.o \
+	common/therm/therm.o \
+	common/mc/mc.o \
+	common/sync/channel_sync.o \
+	common/sync/channel_sync_semaphore.o \
+	common/semaphore/semaphore_sea.o \
+	common/semaphore/semaphore_pool.o \
+	common/semaphore/semaphore_hw.o \
+	common/semaphore/semaphore.o \
+	common/regops/regops.o \
+	common/ltc/ltc.o \
+	common/fb/fb.o \
+	common/fbp/fbp.o \
+	common/gr/gr_utils.o \
+	common/gr/gr.o \
+	common/gr/gr_intr.o \
+	common/gr/global_ctx.o \
+	common/gr/ctx.o \
+	common/gr/gr_falcon.o \
+	common/gr/subctx.o \
+	common/gr/zcull.o \
+	common/gr/gr_config.o \
+	common/gr/zbc.o \
+	common/gr/gr_setup.o \
+	common/gr/hwpm_map.o \
+	common/gr/obj_ctx.o \
+	common/gr/fs_state.o \
+	common/gr/gr_ecc.o \
+	common/netlist/netlist.o \
+	common/init/nvgpu_init.o \
+	common/pmu/pmu.o \
+	common/pmu/allocator.o \
+	common/pmu/pmu_mutex.o \
+	common/pmu/fw/fw.o \
+	common/pmu/fw/fw_ver_ops.o \
+	common/pmu/fw/fw_ns_bootstrap.o \
+	common/pmu/pg/pmu_pg.o \
+	common/pmu/pg/pmu_aelpg.o \
+	common/pmu/perfmon/pmu_perfmon.o \
+	common/pmu/perfmon/pmu_perfmon_sw_gm20b.o \
+	common/pmu/perfmon/pmu_perfmon_sw_gv11b.o \
+	common/pmu/pmu_debug.o \
+	common/pmu/pg/pg_sw_gm20b.o \
+	common/pmu/pg/pg_sw_gp10b.o \
+	common/pmu/pg/pg_sw_gp106.o \
+	common/pmu/pg/pg_sw_gv11b.o \
+	common/pmu/ipc/pmu_cmd.o \
+	common/pmu/ipc/pmu_msg.o \
+	common/pmu/ipc/pmu_queue.o \
+	common/pmu/ipc/pmu_seq.o \
+	common/acr/acr.o \
+	common/acr/acr_wpr.o \
+	common/acr/acr_blob_alloc.o \
+	common/acr/acr_blob_construct_v0.o \
+	common/acr/acr_blob_construct.o \
+	common/acr/acr_bootstrap.o \
+	common/acr/acr_sw_gm20b.o \
+	common/acr/acr_sw_gp10b.o \
+	common/acr/acr_sw_gv11b.o \
+	common/sbr/sbr.o \
+	common/pmu/super_surface/super_surface.o \
+	common/pmu/lsfm/lsfm.o \
+	common/pmu/lsfm/lsfm_sw_gm20b.o \
+	common/pmu/lsfm/lsfm_sw_gp10b.o \
+	common/pmu/lsfm/lsfm_sw_gv100.o \
+	common/pmu/lsfm/lsfm_sw_tu104.o \
+	common/pmu/perf/vfe_var.o \
+	common/pmu/perf/vfe_equ.o \
+	common/pmu/perf/perf.o \
+	common/pmu/perf/change_seq.o \
+	common/pmu/perf/pstate.o \
+	common/pmu/pmgr/pwrdev.o \
+	common/pmu/pmgr/pmgr.o \
+	common/pmu/pmgr/pmgrpmu.o \
+	common/pmu/pmgr/pwrmonitor.o \
+	common/pmu/pmgr/pwrpolicy.o \
+	common/pmu/volt/volt.o \
+	common/pmu/volt/volt_rail.o \
+	common/pmu/volt/volt_dev.o \
+	common/pmu/volt/volt_policy.o \
+	common/pmu/therm/thrm.o \
+	common/pmu/therm/therm_dev.o \
+	common/pmu/therm/therm_channel.o \
+	common/pmu/lpwr/rppg.o \
+	common/pmu/pmu_pstate.o \
+	common/pmu/pmu_rtos_init.o \
+	common/pmu/clk/clk_vin.o \
+	common/pmu/clk/clk_fll.o \
+	common/pmu/clk/clk_domain.o \
+	common/pmu/clk/clk_prog.o \
+	common/pmu/clk/clk_vf_point.o \
+	common/pmu/clk/clk.o \
+	common/pmu/boardobj/boardobj.o \
+	common/pmu/boardobj/boardobjgrp.o \
+	common/pmu/boardobj/boardobjgrpmask.o \
+	common/pmu/boardobj/boardobjgrp_e255.o \
+	common/pmu/boardobj/boardobjgrp_e32.o \
+	common/clk_arb/clk_arb.o \
+	common/clk_arb/clk_arb_gp10b.o \
+	common/rc/rc.o \
+	common/grmgr/grmgr.o \
+	common/cic/cic.o \
+	common/cic/cic_intr.o \
+	common/cic/ce_cic.o \
+	common/cic/ctxsw_cic.o \
+	common/cic/ecc_cic.o \
+	common/cic/host_cic.o \
+	common/cic/gr_cic.o \
+	common/cic/pri_cic.o \
+	common/cic/pmu_cic.o \
+	common/cic/mmu_cic.o \
+	common/cic/msg_cic.o \
+	hal/bus/bus_gk20a.o \
+	hal/class/class_gm20b.o \
+	hal/class/class_gp10b.o \
+	hal/clk/clk_gm20b.o \
+	hal/gr/ecc/ecc_gp10b.o \
+	hal/gr/ecc/ecc_gv11b.o \
+	hal/gr/zcull/zcull_gm20b.o \
+	hal/gr/zcull/zcull_gv11b.o \
+	hal/gr/ctxsw_prog/ctxsw_prog_gp10b.o \
+	hal/gr/ctxsw_prog/ctxsw_prog_gv11b.o \
+	hal/gr/init/gr_init_gm20b.o \
+	hal/gr/init/gr_init_gp10b.o \
+	hal/gr/init/gr_init_gv11b.o \
+	hal/gr/intr/gr_intr_gm20b.o \
+	hal/gr/intr/gr_intr_gp10b.o \
+	hal/gr/hwpm_map/hwpm_map_gv100.o \
+	hal/gr/zbc/zbc_gm20b.o \
+	hal/gr/zbc/zbc_gp10b.o \
+	hal/gr/zbc/zbc_gv11b.o \
+	hal/gr/gr/gr_gk20a.o \
+	hal/gr/gr/gr_gm20b.o \
+	hal/gr/gr/gr_gp10b.o \
+	hal/gr/gr/gr_gv100.o \
+	hal/gr/gr/gr_gv11b.o \
+	hal/gr/gr/gr_tu104.o \
+	hal/init/hal_gv11b.o \
+	hal/init/hal_gv11b_litter.o \
+	hal/init/hal_init.o \
+	hal/perf/perf_gv11b.o \
+	hal/perf/perf_tu104.o \
+	hal/power_features/cg/gp10b_gating_reglist.o \
+	hal/power_features/cg/gv11b_gating_reglist.o \
+	hal/regops/regops_gv11b.o \
+	hal/regops/allowlist_gv11b.o \
+	hal/ce/ce2_gk20a.o \
+	hal/therm/therm_gp10b.o \
+	hal/therm/therm_tu104.o \
+	hal/gr/falcon/gr_falcon_gm20b.o \
+	hal/ltc/ltc_gp10b.o \
+	hal/ltc/intr/ltc_intr_gm20b.o \
+	hal/ltc/intr/ltc_intr_gp10b.o \
+	hal/fb/fb_gm20b.o \
+	hal/fb/fb_gp10b.o \
+	hal/fb/fb_gp106.o \
+	hal/fb/fb_gv11b.o \
+	hal/fb/intr/fb_intr_ecc_gv11b.o \
+	hal/fuse/fuse_gm20b.o \
+	hal/fifo/fifo_gk20a.o \
+	hal/fifo/preempt_gk20a.o \
+	hal/fifo/ramfc_gk20a.o \
+	hal/fifo/ramfc_gp10b.o \
+	hal/fifo/ramin_gk20a.o \
+	hal/fifo/ramin_gp10b.o \
+	hal/fifo/runlist_fifo_gv11b.o \
+	hal/fifo/channel_gk20a.o \
+	hal/fifo/channel_gm20b.o \
+	hal/fifo/tsg_gk20a.o \
+	hal/fifo/userd_gk20a.o \
+	hal/fifo/userd_gv11b.o \
+	hal/fifo/fifo_intr_gk20a.o \
+	hal/fifo/ctxsw_timeout_gk20a.o \
+	hal/netlist/netlist_gp10b.o \
+	hal/sync/sema_cmdbuf_gk20a.o \
+	hal/sync/sema_cmdbuf_gv11b.o \
+	hal/pmu/pmu_gk20a.o \
+	hal/pmu/pmu_gm20b.o \
+	hal/pmu/pmu_gp10b.o \
+	hal/pmu/pmu_gv11b.o \
+	hal/pmu/pmu_tu104.o \
+	hal/top/top_gp106.o \
+	hal/top/top_gp10b.o \
+	hal/tpc/tpc_gv11b.o \
+	hal/priv_ring/priv_ring_gv11b.o \
+	hal/cic/cic_gv11b_fusa.o \
+	hal/cic/cic_lut_gv11b_fusa.o
+
+# Linux specific parts of nvgpu.
+nvgpu-y += \
+	os/linux/os_ops.o \
+	os/linux/os_ops_gm20b.o \
+	os/linux/os_ops_gp10b.o \
+	os/linux/os_ops_gv11b.o \
+	os/linux/kmem.o \
+	os/linux/timers.o \
+	os/linux/ioctl.o \
+	os/linux/ioctl_ctrl.o \
+	os/linux/ioctl_as.o \
+	os/linux/ioctl_channel.o \
+	os/linux/ioctl_tsg.o \
+	os/linux/ioctl_dbg.o \
+	os/linux/ioctl_prof.o \
+	os/linux/ioctl_clk_arb.o \
+	os/linux/cond.o \
+	os/linux/nvgpu_mem.o \
+	os/linux/linux-dma.o \
+	os/linux/driver_common.o \
+	os/linux/firmware.o \
+	os/linux/thread.o \
+	os/linux/vm.o \
+	os/linux/intr.o \
+	os/linux/sysfs.o \
+	os/linux/linux-io.o \
+	os/linux/io_usermode.o \
+	os/linux/rwsem.o \
+	os/linux/sched.o \
+	os/linux/linux-channel.o \
+	os/linux/sim.o \
+	os/linux/sim_pci.o \
+	os/linux/os_sched.o \
+	os/linux/dt.o \
+	os/linux/ecc_sysfs.o \
+	os/linux/bsearch.o \
+	os/linux/cic/cic_stub.o \
+	os/linux/dmabuf_priv.o \
+	os/linux/power_ops.o
+
+nvgpu-$(CONFIG_NVGPU_VPR) += os/linux/vpr.o
+
+nvgpu-$(CONFIG_DEBUG_FS) += \
+	os/linux/debug.o \
+	os/linux/debug_gr.o \
+	os/linux/debug_fifo.o \
+	os/linux/debug_ce.o \
+	os/linux/debug_pmu.o \
+	os/linux/debug_pmgr.o \
+	os/linux/debug_sched.o \
+	os/linux/debug_allocator.o \
+	os/linux/debug_hal.o \
+	os/linux/debug_clk_gm20b.o \
+	os/linux/debug_ltc.o \
+	os/linux/debug_volt.o \
+	os/linux/debug_s_param.o \
+	os/linux/swprofile_debugfs.o
+
+nvgpu-$(CONFIG_NVGPU_LOGGING) += os/linux/log.o
+
+ifeq ($(CONFIG_NVGPU_TRACK_MEM_USAGE),y)
+nvgpu-$(CONFIG_DEBUG_FS) += \
+	os/linux/debug_kmem.o
+endif
+
+nvgpu-$(CONFIG_NVGPU_FECS_TRACE) += \
+	common/gr/fecs_trace.o \
+	hal/gr/fecs_trace/fecs_trace_gm20b.o \
+	hal/gr/fecs_trace/fecs_trace_gv11b.o \
+	os/linux/fecs_trace_linux.o
+
+ifeq ($(CONFIG_NVGPU_FECS_TRACE),y)
+nvgpu-$(CONFIG_DEBUG_FS) += \
+	os/linux/debug_fecs_trace.o
+endif
+
+nvgpu-$(CONFIG_TEGRA_GK20A) += \
+	os/linux/fuse.o \
+	os/linux/module.o \
+	os/linux/module_usermode.o \
+	os/linux/platform_gk20a_tegra.o \
+	os/linux/platform_gp10b_tegra.o \
+	os/linux/platform_gv11b_tegra.o
+
+ifeq ($(CONFIG_TEGRA_GK20A),y)
+nvgpu-$(CONFIG_NVGPU_TEGRA_FUSE) += os/linux/soc.o
+endif
+
+nvgpu-$(CONFIG_NVGPU_SYNCFD_ANDROID) += \
+	os/linux/sync_sema_android.o \
+	os/linux/os_fence_android.o \
+	os/linux/os_fence_android_sema.o
+
+nvgpu-$(CONFIG_NVGPU_SYNCFD_STABLE) += \
+	os/linux/sync_sema_dma.o \
+	os/linux/os_fence_dma.o \
+	os/linux/os_fence_dma_sema.o
+
+nvgpu-$(CONFIG_TEGRA_GK20A_NVHOST) += \
+	common/sync/channel_sync_syncpt.o \
+	common/fence/fence_syncpt.o
+ifneq ($(CONFIG_NVGPU_SYNCFD_NONE),y)
+nvgpu-$(CONFIG_TEGRA_GK20A_NVHOST) += \
+	os/linux/os_fence_syncpt.o
+endif
+
+ifeq ($(CONFIG_TEGRA_GK20A_NVHOST_HOST1X),y)
+nvgpu-$(CONFIG_TEGRA_GK20A_NVHOST) += \
+	os/linux/nvhost_host1x.o
+else
+nvgpu-$(CONFIG_TEGRA_GK20A_NVHOST) += \
+	os/linux/nvhost.o
+endif
+
+nvgpu-$(CONFIG_TEGRA_GK20A_NVHOST) += \
+	os/linux/nvhost_common.o \
+	hal/sync/syncpt_cmdbuf_gk20a.o \
+	hal/sync/syncpt_cmdbuf_gv11b.o \
+	hal/sync/syncpt_cmdbuf_gv11b_fusa.o \
+	common/sync/channel_user_syncpt.o
+
+nvgpu-$(CONFIG_NVGPU_GR_VIRTUALIZATION) += \
+	os/linux/vgpu/platform_vgpu_tegra.o \
+	os/linux/vgpu/sysfs_vgpu.o \
+	os/linux/vgpu/vgpu_ivc.o \
+	os/linux/vgpu/vgpu_ivm.o \
+	os/linux/vgpu/vgpu_linux.o \
+	os/linux/vgpu/gv11b/platform_gv11b_vgpu_tegra.o
+
+ifeq ($(CONFIG_NVGPU_FECS_TRACE),y)
+nvgpu-$(CONFIG_NVGPU_GR_VIRTUALIZATION) += \
+	os/linux/vgpu/fecs_trace_vgpu_linux.o
+endif
+
+nvgpu-$(CONFIG_COMMON_CLK) += \
+	os/linux/clk.o
+
+nvgpu-$(CONFIG_GK20A_DEVFREQ) += \
+	os/linux/scale.o
+
+nvgpu-$(CONFIG_NVGPU_SUPPORT_CDE) += \
+	os/linux/cde.o \
+	os/linux/cde_gm20b.o \
+	os/linux/cde_gp10b.o
+
+ifeq ($(CONFIG_DEBUG_FS),y)
+nvgpu-$(CONFIG_NVGPU_SUPPORT_CDE) += \
+	os/linux/debug_cde.o
+endif
+
+nvgpu-y += \
+	common/mm/allocators/nvgpu_allocator.o \
+	common/mm/allocators/bitmap_allocator.o \
+	common/mm/allocators/buddy_allocator.o \
+	common/mm/gmmu/page_table.o \
+	common/mm/gmmu/pd_cache.o \
+	common/mm/gmmu/pte.o \
+	common/mm/as.o \
+	common/mm/vm.o \
+	common/mm/vm_area.o \
+	common/mm/nvgpu_mem.o \
+	common/mm/nvgpu_sgt.o \
+	common/mm/mm.o \
+	common/mm/dma.o \
+	common/vbios/bios.o \
+	common/falcon/falcon.o \
+	common/falcon/falcon_sw_gk20a.o \
+	common/engine_queues/engine_mem_queue.o \
+	common/engine_queues/engine_dmem_queue.o \
+	common/engine_queues/engine_fb_queue.o \
+	common/io/io.o \
+	common/power_features/power_features.o \
+	common/power_features/cg/cg.o \
+	common/power_features/pg/pg.o \
+	common/sim/sim.o \
+	common/sim/sim_pci.o \
+	common/sim/sim_netlist.o \
+	common/fifo/fifo.o \
+	common/fifo/preempt.o \
+	common/fifo/channel.o \
+	common/fifo/channel_wdt.o \
+	common/fifo/channel_worker.o \
+	common/fifo/pbdma.o \
+	common/fifo/submit.o \
+	common/fifo/job.o \
+	common/fifo/priv_cmdbuf.o \
+	common/fifo/tsg.o \
+	common/fifo/runlist.o \
+	common/fifo/engine_status.o \
+	common/fifo/engines.o \
+	common/fifo/pbdma_status.o \
+	common/fifo/userd.o \
+	common/fifo/watchdog.o \
+	common/fence/fence.o \
+	common/fence/fence_sema.o \
+	common/ecc.o \
+	common/log_common.o \
+	common/ce/ce.o \
+	common/debugger.o \
+	common/profiler/profiler.o \
+	common/profiler/pm_reservation.o
+
+nvgpu-$(CONFIG_NVGPU_GR_VIRTUALIZATION) += \
+	common/vgpu/ltc/ltc_vgpu.o \
+	common/vgpu/fbp/fbp_vgpu.o \
+	common/vgpu/fb/fb_vgpu.o \
+	common/vgpu/gr/gr_vgpu.o \
+	common/vgpu/gr/ctx_vgpu.o \
+	common/vgpu/gr/subctx_vgpu.o \
+	common/vgpu/top/top_vgpu.o \
+	common/vgpu/fifo/fifo_vgpu.o \
+	common/vgpu/fifo/channel_vgpu.o \
+	common/vgpu/fifo/tsg_vgpu.o \
+	common/vgpu/fifo/preempt_vgpu.o \
+	common/vgpu/fifo/runlist_vgpu.o \
+	common/vgpu/fifo/ramfc_vgpu.o \
+	common/vgpu/fifo/userd_vgpu.o \
+	common/vgpu/ce_vgpu.o \
+	common/vgpu/mm/mm_vgpu.o \
+	common/vgpu/mm/vm_vgpu.o \
+	common/vgpu/init/init_vgpu.o \
+	common/vgpu/ivc/comm_vgpu.o \
+	common/vgpu/intr/intr_vgpu.o \
+	common/vgpu/ptimer/ptimer_vgpu.o \
+	common/vgpu/debugger_vgpu.o \
+	common/vgpu/pm_reservation_vgpu.o \
+	common/vgpu/perf/perf_vgpu.o \
+	common/vgpu/profiler/profiler_vgpu.o \
+	common/vgpu/ecc_vgpu.o \
+	common/vgpu/clk_vgpu.o \
+	common/vgpu/gr/fecs_trace_vgpu.o \
+	hal/vgpu/init/init_hal_vgpu.o \
+	hal/vgpu/fifo/fifo_gv11b_vgpu.o \
+	hal/vgpu/sync/syncpt_cmdbuf_gv11b_vgpu.o
+
+nvgpu-$(CONFIG_NVGPU_CYCLESTATS) += \
+	common/perf/cyclestats_snapshot.o \
+	common/cyclestats/cyclestats.o
+
+ifeq ($(CONFIG_NVGPU_GR_VIRTUALIZATION),y)
+nvgpu-$(CONFIG_NVGPU_CYCLESTATS) += \
+	common/vgpu/perf/cyclestats_snapshot_vgpu.o
+nvgpu-$(CONFIG_NVGPU_COMPRESSION) += \
+	common/vgpu/cbc/cbc_vgpu.o
+endif
+
+nvgpu-$(CONFIG_NVGPU_COMPRESSION) += \
+	os/linux/comptags.o \
+	common/mm/comptags.o \
+	common/cbc/cbc.o \
+	hal/cbc/cbc_gm20b.o \
+	hal/cbc/cbc_gp10b.o \
+	hal/cbc/cbc_gv11b.o \
+	hal/cbc/cbc_tu104.o \
+
+# FUSA (Functionally Safe) HAL source files
+nvgpu-y += \
+	hal/mm/mm_gv11b_fusa.o \
+	hal/mm/mm_gp10b_fusa.o \
+	hal/mm/gmmu/gmmu_gv11b_fusa.o \
+	hal/mm/gmmu/gmmu_gp10b_fusa.o \
+	hal/mm/gmmu/gmmu_gk20a_fusa.o \
+	hal/mm/gmmu/gmmu_gm20b_fusa.o \
+	hal/mm/cache/flush_gk20a_fusa.o \
+	hal/mm/cache/flush_gv11b_fusa.o \
+	hal/mm/mmu_fault/mmu_fault_gv11b_fusa.o \
+	hal/ltc/intr/ltc_intr_gp10b_fusa.o \
+	hal/ltc/intr/ltc_intr_gv11b_fusa.o \
+	hal/bus/bus_gk20a_fusa.o \
+	hal/bus/bus_gm20b_fusa.o \
+	hal/bus/bus_gp10b_fusa.o \
+	hal/bus/bus_gv11b_fusa.o \
+	hal/ce/ce_gp10b_fusa.o \
+	hal/ce/ce_gv11b_fusa.o \
+	hal/class/class_gv11b_fusa.o \
+	hal/falcon/falcon_gk20a_fusa.o \
+	hal/fb/fb_gm20b_fusa.o \
+	hal/fb/fb_gv11b_fusa.o \
+	hal/fb/fb_mmu_fault_gv11b_fusa.o \
+	hal/fb/ecc/fb_ecc_gv11b_fusa.o \
+	hal/fb/intr/fb_intr_ecc_gv11b_fusa.o \
+	hal/fb/intr/fb_intr_gv11b_fusa.o \
+	hal/fifo/channel_gk20a_fusa.o \
+	hal/fifo/channel_gm20b_fusa.o \
+	hal/fifo/channel_gv11b_fusa.o \
+	hal/fifo/ctxsw_timeout_gv11b_fusa.o \
+	hal/fifo/engine_status_gm20b_fusa.o \
+	hal/fifo/engine_status_gv100_fusa.o \
+	hal/fifo/engines_gp10b_fusa.o \
+	hal/fifo/engines_gv11b_fusa.o \
+	hal/fifo/fifo_gk20a_fusa.o \
+	hal/fifo/fifo_gv11b_fusa.o \
+	hal/fifo/fifo_intr_gk20a_fusa.o \
+	hal/fifo/fifo_intr_gv11b_fusa.o \
+	hal/fifo/pbdma_gm20b_fusa.o \
+	hal/fifo/pbdma_gp10b_fusa.o \
+	hal/fifo/pbdma_gv11b_fusa.o \
+	hal/fifo/pbdma_status_gm20b_fusa.o \
+	hal/fifo/preempt_gv11b_fusa.o \
+	hal/fifo/ramfc_gp10b_fusa.o \
+	hal/fifo/ramfc_gv11b_fusa.o \
+	hal/fifo/ramin_gk20a_fusa.o \
+	hal/fifo/ramin_gm20b_fusa.o \
+	hal/fifo/ramin_gv11b_fusa.o \
+	hal/fifo/runlist_fifo_gk20a_fusa.o \
+	hal/fifo/runlist_fifo_gv11b_fusa.o \
+	hal/fifo/runlist_ram_gv11b_fusa.o \
+	hal/fifo/tsg_gk20a_fusa.o \
+	hal/fifo/tsg_gv11b_fusa.o \
+	hal/fifo/usermode_gv11b_fusa.o \
+	hal/fuse/fuse_gm20b_fusa.o \
+	hal/fuse/fuse_gp10b_fusa.o \
+	hal/gr/config/gr_config_gm20b_fusa.o \
+	hal/gr/config/gr_config_gv100_fusa.o \
+	hal/gr/ctxsw_prog/ctxsw_prog_gm20b_fusa.o \
+	hal/gr/ctxsw_prog/ctxsw_prog_gp10b_fusa.o \
+	hal/gr/ctxsw_prog/ctxsw_prog_gv11b_fusa.o \
+	hal/gr/ecc/ecc_gv11b_fusa.o \
+	hal/gr/falcon/gr_falcon_gm20b_fusa.o \
+	hal/gr/falcon/gr_falcon_gp10b_fusa.o \
+	hal/gr/falcon/gr_falcon_gv11b_fusa.o \
+	hal/gr/init/gr_init_gm20b_fusa.o \
+	hal/gr/init/gr_init_gp10b_fusa.o \
+	hal/gr/init/gr_init_gv11b_fusa.o \
+	hal/gr/intr/gr_intr_gm20b_fusa.o \
+	hal/gr/intr/gr_intr_gp10b_fusa.o \
+	hal/gr/intr/gr_intr_gv11b_fusa.o \
+	hal/ltc/ltc_gm20b_fusa.o \
+	hal/ltc/ltc_gp10b_fusa.o \
+	hal/ltc/ltc_gv11b_fusa.o \
+	hal/mc/mc_gm20b_fusa.o \
+	hal/mc/mc_gp10b_fusa.o \
+	hal/mc/mc_gv11b_fusa.o \
+	hal/netlist/netlist_gv11b_fusa.o \
+	hal/pmu/pmu_gk20a_fusa.o \
+	hal/pmu/pmu_gv11b_fusa.o \
+	hal/priv_ring/priv_ring_gm20b_fusa.o \
+	hal/priv_ring/priv_ring_gp10b_fusa.o \
+	hal/ptimer/ptimer_gk20a_fusa.o \
+	hal/ptimer/ptimer_gp10b.o \
+	hal/ptimer/ptimer_gv11b.o \
+	hal/therm/therm_gv11b_fusa.o \
+	hal/top/top_gm20b_fusa.o \
+	hal/top/top_gv11b_fusa.o
+
+nvgpu-$(CONFIG_NVGPU_HAL_NON_FUSA) += \
+	hal/ce/ce_tu104.o \
+	hal/clk/clk_gm20b.o \
+	hal/init/hal_gp10b.o \
+	hal/init/hal_gp10b_litter.o \
+	hal/init/hal_gm20b.o \
+	hal/init/hal_gm20b_litter.o \
+	hal/fifo/engine_status_gm20b.o \
+	hal/fifo/engines_gm20b.o \
+	hal/fifo/pbdma_gm20b.o \
+	hal/fifo/pbdma_gp10b.o \
+	hal/fifo/mmu_fault_gk20a.o \
+	hal/fifo/mmu_fault_gm20b.o \
+	hal/fifo/mmu_fault_gp10b.o \
+	hal/fifo/runlist_fifo_gk20a.o \
+	hal/fifo/runlist_ram_gk20a.o \
+	hal/gr/config/gr_config_gm20b.o \
+	hal/gr/ctxsw_prog/ctxsw_prog_gm20b.o \
+	hal/gr/ctxsw_prog/ctxsw_prog_gm20b_dbg.o \
+	hal/gr/gr/gr_gk20a.o \
+	hal/gr/gr/gr_gm20b.o \
+	hal/gr/gr/gr_gp10b.o \
+	hal/ltc/ltc_gm20b.o \
+	hal/ltc/ltc_gm20b_dbg.o \
+	hal/mc/mc_gm20b.o \
+	hal/mm/cache/flush_gk20a.o \
+	hal/mm/mm_gm20b.o \
+	hal/mm/mm_gk20a.o \
+	hal/mm/gmmu/gmmu_gk20a.o \
+	hal/mm/gmmu/gmmu_gm20b.o \
+	hal/falcon/falcon_gk20a.o \
+	hal/netlist/netlist_gm20b.o \
+	hal/perf/perf_gm20b.o \
+	hal/power_features/cg/gm20b_gating_reglist.o \
+	hal/priv_ring/priv_ring_gm20b.o \
+	hal/regops/regops_gm20b.o \
+	hal/regops/regops_gp10b.o \
+	hal/regops/regops_tu104.o \
+	hal/regops/allowlist_tu104.o \
+	hal/therm/therm_gm20b.o \
+	hal/top/top_gm20b.o
+
+ifeq ($(CONFIG_NVGPU_GR_VIRTUALIZATION),y)
+nvgpu-$(CONFIG_NVGPU_HAL_NON_FUSA) += \
+	hal/vgpu/init/vgpu_hal_gv11b.o
+endif

drivers/gpu/nvgpu/Makefile.doxygen

@@ -0,0 +1,53 @@
+##################### tell Emacs this is a -*- makefile-gmake -*-
+#
+# Copyright (c) 2019-2020 NVIDIA CORPORATION.  All Rights Reserved.
+#
+# NVIDIA CORPORATION and its licensors retain all intellectual property
+# and proprietary rights in and to this software, related documentation
+# and any modifications thereto.  Any use, reproduction, disclosure or
+# distribution of this software and related documentation without an express
+# license agreement from NVIDIA CORPORATION is strictly prohibited.
+#
+# Build the doxygen output.
+#
+# Some assumptions: this is a local build only. You need installed:
+#
+#   doxygen
+#   python2
+#
+# You also must have $TOP/$TEGRA_TOP set; For L4T:
+#
+#   $ export TOP=$TEGRA_TOP
+#
+# Should work.
+#
+
+PYTHON2    = python2.7
+ARCH       = $(TEGRA_TOP)/core-private/tools/arch/arch.py
+NVGPU_ARCH = $(TOP)/kernel/nvgpu/arch
+ARCH_YAML  = $(NVGPU_ARCH)/*.yaml
+
+# Build the doxygen output. But make sure the sources file is generated
+# first.
+all: doxygen/sources
+	doxygen doxygen/Doxyfile.safety
+
+# Generate the sources file. This has a list of files that we shall include
+# in the doxygen output.
+doxygen/sources: $(ARCH) $(ARCH_YAML)
+	$(ARCH) --arch-file $(NVGPU_ARCH)/nvgpu.yaml files	\
+		--safe --gpu igpu --gpu both			\
+		--prefix 'INPUT += ' > doxygen/Doxyfile.sources.safety
+	@if [ ! -z "$(EXTRA_PATH)" ] ; then \
+		$(ARCH) --arch-file $(NVGPU_ARCH)/nvgpu.yaml \
+			--include-path $(EXTRA_PATH) files	\
+		--safe --gpu igpu --gpu both			\
+		--prefix 'INPUT += $(EXTRA_PATH)/' \
+		>> doxygen/Doxyfile.sources.safety ; \
+		cat $(EXTRA_PATH)/unit-tests/SWUTS.sources \
+			 >> doxygen/Doxyfile.sources.safety ; \
+	fi
+
+clean:
+	rm -rf doxygen/Doxyfile.sources.safety
+	rm -rf html

drivers/gpu/nvgpu/Makefile.linux.configs

@@ -0,0 +1,225 @@
+# Turn off all other configs, if CONFIG_GK20A is not set
+CONFIG_GK20A := m
+
+# Enable GK20A PMU features.
+CONFIG_GK20A_PMU := y
+
+# Enable support for the GK20A graphics engine on Tegra
+# by adding a Tegra platfrom interface to the GK20A driver.
+CONFIG_TEGRA_GK20A := y
+
+# Enable Support for Loading High Secure binary, and using
+# Write Protected Regions (WPR) for storing ucodes, and bootstrap
+# PMU, FECS and GPCCS in Low Secure mode.
+CONFIG_TEGRA_ACR := y
+
+# Support for debugger APIs
+CONFIG_NVGPU_DEBUGGER := y
+
+# Support for iGPU LS PMU enable/disable
+CONFIG_NVGPU_LS_PMU := y
+
+# Enable/Disable NVGPU logging
+CONFIG_NVGPU_LOGGING := y
+
+# Enable/Disable the support of HALs from chips that do not have functional
+# safety certification
+CONFIG_NVGPU_HAL_NON_FUSA := y
+
+# Support recovery on failure (which may involve engine reset)
+CONFIG_NVGPU_RECOVERY := y
+
+# Support for compression
+CONFIG_NVGPU_COMPRESSION := y
+
+# Enable support for extraction of comptags for CDE.
+ifeq ($(CONFIG_NVGPU_COMPRESSION),y)
+CONFIG_NVGPU_SUPPORT_CDE := y
+endif
+
+ifeq ($(CONFIG_COMMON_CLK),y)
+ifeq ($(CONFIG_PM_DEVFREQ),y)
+# Select this entry to enable gk20a scaling
+CONFIG_GK20A_DEVFREQ := y
+
+# Disable support to pass PM_QOS constraints to devfreq based scaling.
+CONFIG_GK20A_PM_QOS := n
+
+endif
+endif
+
+# Say Y here to allow nvgpu to track and keep statistics on
+# the system memory used by the driver. This does recreate
+# some of the kmem_leak tracking but this is also applicable
+# to other OSes which do not have Linux' kmem_leak.
+#CONFIG_NVGPU_TRACK_MEM_USAGE    := n
+
+# Enable the cycle stats debugging features.
+CONFIG_NVGPU_CYCLESTATS := y
+
+# Enable support for the NVGPU Context Switch Tracing. In this mode,
+# FECS collects timestamps for contexts loaded on GR engine. This
+# allows tracking context switches on GR engine, as well as
+# identifying processes that submitted work.
+CONFIG_NVGPU_FECS_TRACE := y
+
+# Enable support in GK20A for the nvhost (host1x) dma engine hardware
+# that includes things like hardware syncpts. This requires
+# TEGRA_GRHOST
+ifdef CONFIG_TEGRA_GRHOST
+CONFIG_TEGRA_GK20A_NVHOST := y
+endif
+ifdef CONFIG_TEGRA_HOST1X_NEXT
+CONFIG_TEGRA_GK20A_NVHOST := y
+CONFIG_TEGRA_GK20A_NVHOST_HOST1X := y
+endif
+
+# Enable support for GPUs on PCIe bus.
+ifeq ($(CONFIG_PCI),y)
+# Support for NVGPU DGPU
+CONFIG_NVGPU_DGPU := y
+endif
+
+# Enable nvgpu debug facility to redirect debug spew to ftrace. This
+# affects kernel memory use, so should not be enabled by default.
+ifeq ($(CONFIG_TRACING),y)
+#CONFIG_GK20A_TRACE_PRINTK := y
+endif
+
+# Use tegra_alloc_fd() for allocating dma_buf fds. This allocates
+# the fds above 1024 which exempts them from counting against process
+# fd limit.
+ifeq ($(CONFIG_NV_TEGRA_MC),y)
+CONFIG_NVGPU_USE_TEGRA_ALLOC_FD := y
+endif
+
+# Support Nvlink
+ifeq ($(CONFIG_TEGRA_NVLINK),y)
+CONFIG_NVGPU_NVLINK := y
+endif
+
+# Support NVGPU Virtualization
+ifeq ($(CONFIG_TEGRA_GR_VIRTUALIZATION),y)
+CONFIG_NVGPU_GR_VIRTUALIZATION := y
+endif
+
+# Support for NVGPU VPR
+ifeq ($(CONFIG_TEGRA_VPR),y)
+CONFIG_NVGPU_VPR := y
+endif
+
+# Support Tegra fuse
+ifeq ($(CONFIG_TEGRA_KFUSE),y)
+CONFIG_NVGPU_TEGRA_FUSE := y
+endif
+
+# GPU job synchronization (fences before and after submits) can use raw
+# syncpoints if available and sync fds if chosen. Without syncpoints,
+# nvgpu also provides semaphore-backed sync fds to userspace.
+#
+# Select which kernel-provided API is used for sync fds. Matching
+# support is required for the userspace drivers too.
+ifeq ($(CONFIG_SYNC),y)
+CONFIG_NVGPU_SYNCFD_ANDROID := y
+else ifeq ($(CONFIG_SYNC_FILE), y)
+CONFIG_NVGPU_SYNCFD_STABLE := y
+else
+CONFIG_NVGPU_SYNCFD_NONE := y
+endif
+
+# Below check indicates the build is invoked from Nvidia's
+# internal build system.
+ifneq ($(NV_BUILD_KERNEL_OPTIONS),)
+
+# Disable the below configs for kstable
+ifneq ($(findstring stable,$(NV_BUILD_KERNEL_OPTIONS)),)
+CONFIG_GK20A_DEVFREQ := n
+CONFIG_GK20A_PM_QOS := n
+else ifneq ($(filter 4.9 4.14,$(patsubst -,$(space),$(NV_BUILD_KERNEL_OPTIONS))),)
+# Enable support to pass PM_QOS constraints to devfreq based scaling.
+CONFIG_GK20A_PM_QOS := y
+endif
+
+endif
+
+ifeq ($(CONFIG_GK20A_PMU),y)
+ccflags-y += -DCONFIG_GK20A_PMU
+endif
+ifeq ($(CONFIG_TEGRA_GK20A),y)
+ccflags-y += -DCONFIG_TEGRA_GK20A
+endif
+ifeq ($(CONFIG_TEGRA_ACR),y)
+ccflags-y += -DCONFIG_TEGRA_ACR
+endif
+ifeq ($(CONFIG_NVGPU_DEBUGGER),y)
+ccflags-y += -DCONFIG_NVGPU_DEBUGGER
+endif
+ifeq ($(CONFIG_NVGPU_LS_PMU),y)
+ccflags-y += -DCONFIG_NVGPU_LS_PMU
+endif
+ifeq ($(CONFIG_NVGPU_LOGGING),y)
+ccflags-y += -DCONFIG_NVGPU_LOGGING
+endif
+ifeq ($(CONFIG_NVGPU_HAL_NON_FUSA),y)
+ccflags-y += -DCONFIG_NVGPU_HAL_NON_FUSA
+endif
+ifeq ($(CONFIG_NVGPU_RECOVERY),y)
+ccflags-y += -DCONFIG_NVGPU_RECOVERY
+endif
+ifeq ($(CONFIG_NVGPU_COMPRESSION),y)
+ccflags-y += -DCONFIG_NVGPU_COMPRESSION
+endif
+ifeq ($(CONFIG_NVGPU_SUPPORT_CDE),y)
+ccflags-y += -DCONFIG_NVGPU_SUPPORT_CDE
+endif
+ifeq ($(CONFIG_GK20A_DEVFREQ),y)
+ccflags-y += -DCONFIG_GK20A_DEVFREQ
+endif
+ifeq ($(CONFIG_GK20A_PM_QOS),y)
+ccflags-y += -DCONFIG_GK20A_PM_QOS
+endif
+ifeq ($(CONFIG_NVGPU_TRACK_MEM_USAGE),y)
+ccflags-y += -DCONFIG_NVGPU_TRACK_MEM_USAGE
+endif
+ifeq ($(CONFIG_NVGPU_CYCLESTATS),y)
+ccflags-y += -DCONFIG_NVGPU_CYCLESTATS
+endif
+ifeq ($(CONFIG_NVGPU_FECS_TRACE),y)
+ccflags-y += -DCONFIG_NVGPU_FECS_TRACE
+endif
+ifeq ($(CONFIG_TEGRA_GK20A_NVHOST),y)
+ccflags-y += -DCONFIG_TEGRA_GK20A_NVHOST
+endif
+ifeq ($(CONFIG_TEGRA_GK20A_NVHOST_HOST1X),y)
+ccflags-y += -DCONFIG_TEGRA_GK20A_NVHOST_HOST1X
+endif
+ifeq ($(CONFIG_NVGPU_DGPU),y)
+ccflags-y += -DCONFIG_NVGPU_DGPU
+endif
+ifeq ($(CONFIG_GK20A_TRACE_PRINTK),y)
+ccflags-y += -DCONFIG_GK20A_TRACE_PRINTK
+endif
+ifeq ($(CONFIG_NVGPU_USE_TEGRA_ALLOC_FD),y)
+ccflags-y += -DCONFIG_NVGPU_USE_TEGRA_ALLOC_FD
+endif
+ifeq ($(CONFIG_NVGPU_NVLINK),y)
+ccflags-y += -DCONFIG_NVGPU_NVLINK
+endif
+ifeq ($(CONFIG_NVGPU_GR_VIRTUALIZATION),y)
+ccflags-y += -DCONFIG_NVGPU_GR_VIRTUALIZATION
+endif
+ifeq ($(CONFIG_NVGPU_VPR),y)
+ccflags-y += -DCONFIG_NVGPU_VPR
+endif
+ifeq ($(CONFIG_NVGPU_TEGRA_FUSE),y)
+ccflags-y += -DCONFIG_NVGPU_TEGRA_FUSE
+endif
+ifeq ($(CONFIG_NVGPU_SYNCFD_ANDROID),y)
+ccflags-y += -DCONFIG_NVGPU_SYNCFD_ANDROID
+endif
+ifeq ($(CONFIG_NVGPU_SYNCFD_STABLE),y)
+ccflags-y += -DCONFIG_NVGPU_SYNCFD_STABLE
+endif
+ifeq ($(CONFIG_NVGPU_SYNCFD_NONE),y)
+ccflags-y += -DCONFIG_NVGPU_SYNCFD_NONE
+endif

drivers/gpu/nvgpu/Makefile.shared.configs

@@ -0,0 +1,319 @@
+#
+# Copyright (c) 2019-2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+#
+
+# This file defines the make variables and cflags applicable to nvgpu common
+# shared by qnx, nvgpu userspace, test builds. Note that cflags are added to
+# variable NVGPU_COMMON_CFLAGS that needs to be used by the parent Makefile
+# to update corresponding cflags variable.
+
+# Default is the regular profile. That can be overridden if necessary. by
+# setting the NVGPU_FORCE_SAFETY_PROFILE. This is a useful hack while we
+# wait for the userspace tmake build to make its way into a proper safety
+# profile build.
+profile := default
+
+# Decide whether to use the safety release, safety debug or the regular profile.
+ifeq ($(NV_BUILD_CONFIGURATION_IS_SAFETY),1)
+profile := safety_release
+ifeq ($(NV_BUILD_CONFIGURATION_IS_DEBUG),1)
+profile := safety_debug
+endif
+endif
+
+ifeq ($(NVGPU_FORCE_SAFETY_PROFILE),1)
+profile := safety_release
+ifeq ($(NVGPU_FORCE_DEBUG_PROFILE),1)
+profile := safety_debug
+endif
+endif
+
+NVGPU_COMMON_CFLAGS		:=
+
+#
+# Flags always enabled regardless of build profile.
+#
+
+NVGPU_COMMON_CFLAGS		+=		\
+	-DCONFIG_TEGRA_GK20A_PMU=1		\
+	-DCONFIG_TEGRA_ACR=1			\
+	-DCONFIG_NVGPU_GR_VIRTUALIZATION	\
+	-DCONFIG_PCI_MSI
+
+CONFIG_NVGPU_LOGGING                 := 1
+NVGPU_COMMON_CFLAGS                  += -DCONFIG_NVGPU_LOGGING
+
+# Syncpoint support provided by nvhost is expected to exist.
+CONFIG_TEGRA_GK20A_NVHOST            := 1
+NVGPU_COMMON_CFLAGS                  += -DCONFIG_TEGRA_GK20A_NVHOST
+
+# Syncfds are a Linux feature.
+CONFIG_NVGPU_SYNCFD_NONE             := 1
+NVGPU_COMMON_CFLAGS                  += -DCONFIG_NVGPU_SYNCFD_NONE
+
+CONFIG_NVGPU_GRAPHICS                := 1
+NVGPU_COMMON_CFLAGS                  += -DCONFIG_NVGPU_GRAPHICS
+
+ifeq ($(profile),$(filter $(profile),safety_debug safety_release))
+
+# Enable golden context verification only for safety debug/release build
+NVGPU_COMMON_CFLAGS             +=      \
+	-DCONFIG_NVGPU_GR_GOLDEN_CTX_VERIFICATION     \
+	-DCONFIG_NVGPU_BUILD_CONFIGURATION_IS_SAFETY
+
+## For tesing of CTXSW FW error codes manually, enable below configs in safety build.
+## Enable only one config at a time, because only one error can be tested at a time.
+# NVGPU_COMMON_CFLAGS             +=      \
+# 	-DCONFIG_NVGPU_CTXSW_FW_ERROR_WDT_TESTING
+#
+# NVGPU_COMMON_CFLAGS             +=      \
+# 	-DCONFIG_NVGPU_CTXSW_FW_ERROR_CODE_TESTING
+#
+# NVGPU_COMMON_CFLAGS             +=      \
+# 	-DCONFIG_NVGPU_CTXSW_FW_ERROR_HEADER_TESTING
+
+ifeq ($(CONFIG_NVGPU_DGPU),1)
+
+NVGPU_COMMON_CFLAGS             += -DCONFIG_NVGPU_DGPU
+
+CONFIG_NVGPU_NVLINK             := 1
+NVGPU_COMMON_CFLAGS             += -DCONFIG_NVGPU_NVLINK
+
+# used by sec2 code
+CONFIG_NVGPU_ENGINE_QUEUE       := 1
+NVGPU_COMMON_CFLAGS             += -DCONFIG_NVGPU_ENGINE_QUEUE
+
+# used in ce_app
+CONFIG_NVGPU_FENCE              := 1
+NVGPU_COMMON_CFLAGS             += -DCONFIG_NVGPU_FENCE
+
+# ce_app uses syncpt (nvgpu_nvhost_syncpt_wait_timeout_ext)
+CONFIG_NVGPU_KERNEL_MODE_SUBMIT := 1
+NVGPU_COMMON_CFLAGS             += -DCONFIG_NVGPU_KERNEL_MODE_SUBMIT
+
+CONFIG_NVGPU_FALCON_NON_FUSA	:= 1
+NVGPU_COMMON_CFLAGS             += -DCONFIG_NVGPU_FALCON_NON_FUSA
+
+CONFIG_NVGPU_GR_FALCON_NON_SECURE_BOOT := 1
+NVGPU_COMMON_CFLAGS             += -DCONFIG_NVGPU_GR_FALCON_NON_SECURE_BOOT
+
+CONFIG_NVGPU_SM_DIVERSITY := 1
+NVGPU_COMMON_CFLAGS             += -DCONFIG_NVGPU_SM_DIVERSITY
+
+CONFIG_NVGPU_USE_3LSS_ERR_INJECTION := 1
+NVGPU_COMMON_CFLAGS             += -DCONFIG_NVGPU_USE_3LSS_ERR_INJECTION
+
+CONFIG_NVGPU_LS_PMU             := 1
+NVGPU_COMMON_CFLAGS             += -DCONFIG_NVGPU_LS_PMU
+
+CONFIG_NVGPU_CLK_ARB            := 1
+NVGPU_COMMON_CFLAGS             += -DCONFIG_NVGPU_CLK_ARB
+endif
+
+endif
+
+CONFIG_NVGPU_TEGRA_FUSE		:= 1
+NVGPU_COMMON_CFLAGS		+= -DCONFIG_NVGPU_TEGRA_FUSE
+
+#
+# Flags enabled only for safety debug and regular build profile.
+#
+ifneq ($(profile),safety_release)
+
+CONFIG_NVGPU_TRACE              := 1
+NVGPU_COMMON_CFLAGS             += -DCONFIG_NVGPU_TRACE
+
+NVGPU_COMMON_CFLAGS             += -DCONFIG_NVGPU_FALCON_DEBUG
+
+#
+# Flags enabled only for regular build profile.
+#
+ifneq ($(profile),safety_debug)
+
+CONFIG_NVGPU_SYSFS              := 1
+NVGPU_COMMON_CFLAGS             += -DCONFIG_NVGPU_SYSFS
+
+# ACR feature to enable old tegra ACR profile support
+CONFIG_NVGPU_ACR_LEGACY         := 1
+NVGPU_COMMON_CFLAGS             += -DCONFIG_NVGPU_ACR_LEGACY
+
+CONFIG_NVGPU_ENGINE_QUEUE       := 1
+NVGPU_COMMON_CFLAGS             += -DCONFIG_NVGPU_ENGINE_QUEUE
+
+CONFIG_NVGPU_DEBUGGER           := 1
+NVGPU_COMMON_CFLAGS             += -DCONFIG_NVGPU_DEBUGGER
+
+CONFIG_NVGPU_PROFILER           := 1
+NVGPU_COMMON_CFLAGS             += -DCONFIG_NVGPU_PROFILER
+
+CONFIG_NVGPU_RECOVERY           := 1
+NVGPU_COMMON_CFLAGS             += -DCONFIG_NVGPU_RECOVERY
+
+CONFIG_NVGPU_CILP               := 1
+NVGPU_COMMON_CFLAGS             += -DCONFIG_NVGPU_CILP
+
+CONFIG_NVGPU_GFXP               := 1
+NVGPU_COMMON_CFLAGS             += -DCONFIG_NVGPU_GFXP
+
+CONFIG_NVGPU_CYCLESTATS         := 1
+NVGPU_COMMON_CFLAGS             += -DCONFIG_NVGPU_CYCLESTATS
+
+CONFIG_NVGPU_FECS_TRACE         := 1
+NVGPU_COMMON_CFLAGS             += -DCONFIG_NVGPU_FECS_TRACE
+
+ifneq ($(CONFIG_NVGPU_DGPU),1)
+CONFIG_NVGPU_IGPU_VIRT          := 1
+NVGPU_COMMON_CFLAGS             += -DCONFIG_NVGPU_IGPU_VIRT
+endif
+
+# Enable the usage of 3LSS error injection features.
+CONFIG_NVGPU_USE_3LSS_ERR_INJECTION := 1
+NVGPU_COMMON_CFLAGS                 += -DCONFIG_NVGPU_USE_3LSS_ERR_INJECTION
+
+# Enable nvlink support for normal build.
+CONFIG_NVGPU_NVLINK             := 1
+NVGPU_COMMON_CFLAGS             += -DCONFIG_NVGPU_NVLINK
+
+# Enable tpc_powergate support for normal build.
+CONFIG_NVGPU_TPC_POWERGATE      := 1
+NVGPU_COMMON_CFLAGS             += -DCONFIG_NVGPU_TPC_POWERGATE
+
+# Enable mssnvlink0 reset control for normal build
+CONFIG_MSSNVLINK0_RST_CONTROL   := 1
+NVGPU_COMMON_CFLAGS             += -DCONFIG_MSSNVLINK0_RST_CONTROL
+
+# Enable dgpu support for normal build.
+CONFIG_NVGPU_DGPU               := 1
+NVGPU_COMMON_CFLAGS             += -DCONFIG_NVGPU_DGPU
+
+# Enable nvgpu_next for normal build
+ifneq ($(NV_BUILD_CONFIGURATION_IS_EXTERNAL), 1)
+CONFIG_NVGPU_NEXT               := 1
+NVGPU_COMMON_CFLAGS             += -DCONFIG_NVGPU_NEXT
+endif
+
+CONFIG_NVGPU_VPR                := 1
+NVGPU_COMMON_CFLAGS             += -DCONFIG_NVGPU_VPR
+
+CONFIG_NVGPU_REPLAYABLE_FAULT   := 1
+NVGPU_COMMON_CFLAGS             += -DCONFIG_NVGPU_REPLAYABLE_FAULT
+
+# Enable LS PMU support for normal build
+CONFIG_NVGPU_LS_PMU             := 1
+NVGPU_COMMON_CFLAGS             += -DCONFIG_NVGPU_LS_PMU
+
+# Enable elpg support for normal build
+CONFIG_NVGPU_POWER_PG           := 1
+NVGPU_COMMON_CFLAGS             += -DCONFIG_NVGPU_POWER_PG
+
+# Enable sim support for normal build
+CONFIG_NVGPU_SIM                := 1
+NVGPU_COMMON_CFLAGS             += -DCONFIG_NVGPU_SIM
+
+CONFIG_NVGPU_COMPRESSION        := 1
+NVGPU_COMMON_CFLAGS             += -DCONFIG_NVGPU_COMPRESSION
+
+# Enable non FUSA HALs for normal build
+CONFIG_NVGPU_HAL_NON_FUSA       := 1
+NVGPU_COMMON_CFLAGS             += -DCONFIG_NVGPU_HAL_NON_FUSA
+
+# Enable non FUSA common code for normal build
+CONFIG_NVGPU_NON_FUSA           := 1
+NVGPU_COMMON_CFLAGS             += -DCONFIG_NVGPU_NON_FUSA
+
+CONFIG_NVGPU_CLK_ARB            := 1
+NVGPU_COMMON_CFLAGS             += -DCONFIG_NVGPU_CLK_ARB
+
+NVGPU_COMMON_CFLAGS             += -DCONFIG_NVGPU_FALCON_NON_FUSA
+
+NVGPU_COMMON_CFLAGS             += -DCONFIG_NVGPU_IOCTL_NON_FUSA
+NVGPU_COMMON_CFLAGS             += -DCONFIG_NVGPU_DETERMINISTIC_CHANNELS
+
+CONFIG_NVGPU_GR_FALCON_NON_SECURE_BOOT := 1
+NVGPU_COMMON_CFLAGS                    += -DCONFIG_NVGPU_GR_FALCON_NON_SECURE_BOOT
+
+CONFIG_NVGPU_SET_FALCON_ACCESS_MAP := 1
+NVGPU_COMMON_CFLAGS                += -DCONFIG_NVGPU_SET_FALCON_ACCESS_MAP
+
+# Enable SW Semaphore for normal build
+CONFIG_NVGPU_SW_SEMAPHORE       := 1
+NVGPU_COMMON_CFLAGS             += -DCONFIG_NVGPU_SW_SEMAPHORE
+
+# Enable Channel WDT for safety build until we switch to user mode submits only
+CONFIG_NVGPU_CHANNEL_WDT := 1
+NVGPU_COMMON_CFLAGS		+= -DCONFIG_NVGPU_CHANNEL_WDT
+
+# Enable Kernel Mode submit for safety build until we switch to user mode
+# submits only
+CONFIG_NVGPU_KERNEL_MODE_SUBMIT := 1
+NVGPU_COMMON_CFLAGS             += -DCONFIG_NVGPU_KERNEL_MODE_SUBMIT
+
+# Enable fences for safety build till until we switch to user mode submits only
+CONFIG_NVGPU_FENCE              := 1
+NVGPU_COMMON_CFLAGS             += -DCONFIG_NVGPU_FENCE
+
+# Enable powergate lib for normal build
+CONFIG_NVGPU_USE_POWERGATE_LIB  := 1
+NVGPU_COMMON_CFLAGS             += -DCONFIG_NVGPU_USE_POWERGATE_LIB
+
+NVGPU_COMMON_CFLAGS             += -DCONFIG_NVGPU_FIFO_ENGINE_ACTIVITY
+# Enable dynamic busy/idle support
+CONFIG_NVGPU_DYNAMIC_BUSY_IDLE_SUPPORT := 1
+NVGPU_COMMON_CFLAGS             += -DCONFIG_NVGPU_DYNAMIC_BUSY_IDLE_SUPPORT
+
+# Enable HW based error injection support
+NVGPU_COMMON_CFLAGS             += -DCONFIG_NVGPU_INJECT_HWERR
+
+# Enable Channel/TSG Scheduling
+CONFIG_NVGPU_CHANNEL_TSG_SCHEDULING  := 1
+NVGPU_COMMON_CFLAGS                  += -DCONFIG_NVGPU_CHANNEL_TSG_SCHEDULING
+
+# Enable Channel/TSG Control
+CONFIG_NVGPU_CHANNEL_TSG_CONTROL     := 1
+NVGPU_COMMON_CFLAGS                  += -DCONFIG_NVGPU_CHANNEL_TSG_CONTROL
+
+# Enable Virtualization server for normal build
+NVGPU_COMMON_CFLAGS             += -DCONFIG_TEGRA_GR_VIRTUALIZATION_SERVER
+
+# Enable SM diversity support for normal build
+CONFIG_NVGPU_SM_DIVERSITY            := 1
+NVGPU_COMMON_CFLAGS                  += -DCONFIG_NVGPU_SM_DIVERSITY
+
+# Enable Multi Instance GPU support for normal build
+CONFIG_NVGPU_MIG                     := 1
+NVGPU_COMMON_CFLAGS                  += -DCONFIG_NVGPU_MIG
+
+endif
+endif
+
+# Enable USERD only if kernel mode submit is supported
+ifeq ($(CONFIG_NVGPU_KERNEL_MODE_SUBMIT),1)
+CONFIG_NVGPU_USERD		:= 1
+NVGPU_COMMON_CFLAGS             += -DCONFIG_NVGPU_USERD
+endif
+
+ifeq ($(CONFIG_NVGPU_DEBUGGER),1)
+NVGPU_COMMON_CFLAGS             += -DCONFIG_NVGPU_ENGINE_RESET
+endif
+
+ifeq ($(CONFIG_NVGPU_RECOVERY),1)
+NVGPU_COMMON_CFLAGS             += -DCONFIG_NVGPU_ENGINE_RESET
+endif

drivers/gpu/nvgpu/Makefile.sources

@@ -0,0 +1,711 @@
+# -*- mode: makefile -*-
+#
+# Copyright (c) 2018-2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+srcs :=
+
+ifdef NVGPU_POSIX
+srcs +=	os/posix/nvgpu.c \
+	os/posix/posix-io.c \
+	os/posix/mock-registers.c \
+	os/posix/posix-nvgpu_mem.c \
+	os/posix/posix-dma.c \
+	os/posix/posix-vm.c \
+	os/posix/firmware.c \
+	os/posix/soc.c \
+	os/posix/error_notifier.c \
+	os/posix/posix-channel.c \
+	os/posix/posix-tsg.c \
+	os/posix/stubs.c \
+	os/posix/posix-nvhost.c \
+	os/posix/posix-vgpu.c \
+	os/posix/posix-dt.c \
+	os/posix/fuse.c
+
+ifdef CONFIG_NVGPU_VPR
+srcs += os/posix/posix-vpr.c
+endif
+
+ifdef CONFIG_NVGPU_FECS_TRACE
+srcs += os/posix/fecs_trace_posix.c
+endif
+
+ifeq ($(CONFIG_NVGPU_CLK_ARB),1)
+srcs += os/posix/posix-clk_arb.c
+endif
+
+ifdef CONFIG_NVGPU_NVLINK
+srcs +=	os/posix/posix-nvlink.c
+endif
+
+ifeq ($(CONFIG_NVGPU_COMPRESSION),1)
+srcs += os/posix/posix-comptags.c
+endif
+
+ifeq ($(CONFIG_NVGPU_LOGGING),1)
+srcs += os/posix/log.c
+endif
+
+ifeq ($(CONFIG_NVGPU_DGPU),1)
+srcs += os/posix/posix-vidmem.c
+endif
+endif
+
+# POSIX sources shared between the POSIX and QNX builds.
+srcs += os/posix/bug.c \
+	os/posix/rwsem.c \
+	os/posix/timers.c \
+	os/posix/cond.c \
+	os/posix/lock.c \
+	os/posix/thread.c \
+	os/posix/os_sched.c \
+	os/posix/bitmap.c \
+	os/posix/kmem.c \
+	os/posix/file_ops.c \
+	os/posix/queue.c
+
+ifeq ($(NV_BUILD_CONFIGURATION_IS_SAFETY),0)
+srcs += os/posix/bsearch.c
+endif
+
+srcs +=	common/device.c \
+	common/utils/enabled.c \
+	common/utils/errata.c \
+	common/utils/rbtree.c \
+	common/utils/string.c \
+	common/utils/worker.c \
+	common/swdebug/profile.c \
+	common/init/nvgpu_init.c \
+	common/mm/allocators/nvgpu_allocator.c \
+	common/mm/allocators/bitmap_allocator.c \
+	common/mm/allocators/buddy_allocator.c \
+	common/mm/gmmu/page_table.c \
+	common/mm/gmmu/pd_cache.c \
+	common/mm/gmmu/pte.c \
+	common/mm/as.c \
+	common/mm/vm.c \
+	common/mm/vm_area.c \
+	common/mm/nvgpu_mem.c \
+	common/mm/nvgpu_sgt.c \
+	common/mm/mm.c \
+	common/mm/dma.c \
+	common/therm/therm.c \
+	common/ltc/ltc.c \
+	common/fb/fb.c \
+	common/fbp/fbp.c \
+	common/io/io.c \
+	common/ecc.c \
+	common/falcon/falcon.c \
+	common/falcon/falcon_sw_gk20a.c \
+	common/gr/gr.c \
+	common/gr/gr_utils.c \
+	common/gr/gr_intr.c \
+	common/gr/global_ctx.c \
+	common/gr/subctx.c \
+	common/gr/ctx.c \
+	common/gr/gr_falcon.c \
+	common/gr/gr_config.c \
+	common/gr/gr_setup.c \
+	common/gr/obj_ctx.c \
+	common/gr/fs_state.c \
+	common/gr/gr_ecc.c \
+	common/netlist/netlist.c \
+	common/pmu/pmu.c \
+	common/acr/acr.c \
+	common/acr/acr_wpr.c \
+	common/acr/acr_blob_alloc.c \
+	common/acr/acr_blob_construct.c \
+	common/acr/acr_bootstrap.c \
+	common/acr/acr_sw_gv11b.c \
+	common/ptimer/ptimer.c \
+	common/power_features/cg/cg.c \
+	common/sync/channel_user_syncpt.c \
+	common/fifo/preempt.c \
+	common/fifo/channel.c \
+	common/fifo/fifo.c \
+	common/fifo/pbdma.c \
+	common/fifo/tsg.c \
+	common/fifo/runlist.c \
+	common/fifo/engine_status.c \
+	common/fifo/engines.c \
+	common/fifo/pbdma_status.c \
+	common/mc/mc.c \
+	common/rc/rc.c \
+	common/ce/ce.c \
+	common/grmgr/grmgr.c \
+	common/cic/cic.c \
+	common/cic/cic_intr.c \
+	common/cic/ce_cic.c \
+	common/cic/ctxsw_cic.c \
+	common/cic/ecc_cic.c \
+	common/cic/host_cic.c \
+	common/cic/gr_cic.c \
+	common/cic/pri_cic.c \
+	common/cic/pmu_cic.c \
+	common/cic/mmu_cic.c \
+	common/cic/msg_cic.c \
+	hal/init/hal_gv11b.c \
+	hal/init/hal_gv11b_litter.c \
+	hal/init/hal_init.c \
+	hal/power_features/cg/gv11b_gating_reglist.c \
+	hal/fifo/runlist_fifo_gv11b.c \
+	hal/fifo/userd_gk20a.c \
+	hal/sync/syncpt_cmdbuf_gv11b.c
+
+# Source files below are functionaly safe (FuSa) and must always be included.
+srcs += hal/mm/mm_gv11b_fusa.c \
+	hal/mm/mm_gp10b_fusa.c \
+	hal/mm/gmmu/gmmu_gv11b_fusa.c \
+	hal/mm/gmmu/gmmu_gp10b_fusa.c \
+	hal/mm/gmmu/gmmu_gk20a_fusa.c \
+	hal/mm/gmmu/gmmu_gm20b_fusa.c \
+	hal/mm/cache/flush_gk20a_fusa.c \
+	hal/mm/cache/flush_gv11b_fusa.c \
+	hal/mm/mmu_fault/mmu_fault_gv11b_fusa.c \
+	hal/ltc/intr/ltc_intr_gp10b_fusa.c \
+	hal/ltc/intr/ltc_intr_gv11b_fusa.c \
+	hal/bus/bus_gk20a_fusa.c \
+	hal/bus/bus_gm20b_fusa.c \
+	hal/bus/bus_gp10b_fusa.c \
+	hal/bus/bus_gv11b_fusa.c \
+	hal/ce/ce_gp10b_fusa.c \
+	hal/ce/ce_gv11b_fusa.c \
+	hal/class/class_gv11b_fusa.c \
+	hal/falcon/falcon_gk20a_fusa.c \
+	hal/fb/fb_gm20b_fusa.c \
+	hal/fb/fb_gv11b_fusa.c \
+	hal/fb/fb_mmu_fault_gv11b_fusa.c \
+	hal/fb/ecc/fb_ecc_gv11b_fusa.c \
+	hal/fb/intr/fb_intr_ecc_gv11b_fusa.c \
+	hal/fb/intr/fb_intr_gv11b_fusa.c \
+	hal/fifo/channel_gk20a_fusa.c \
+	hal/fifo/channel_gm20b_fusa.c \
+	hal/fifo/channel_gv11b_fusa.c \
+	hal/fifo/ctxsw_timeout_gv11b_fusa.c \
+	hal/fifo/engine_status_gm20b_fusa.c \
+	hal/fifo/engine_status_gv100_fusa.c \
+	hal/fifo/engines_gp10b_fusa.c \
+	hal/fifo/engines_gv11b_fusa.c \
+	hal/fifo/fifo_gk20a_fusa.c \
+	hal/fifo/fifo_gv11b_fusa.c \
+	hal/fifo/fifo_intr_gk20a_fusa.c \
+	hal/fifo/fifo_intr_gv11b_fusa.c \
+	hal/fifo/pbdma_gm20b_fusa.c \
+	hal/fifo/pbdma_gp10b_fusa.c \
+	hal/fifo/pbdma_gv11b_fusa.c \
+	hal/fifo/pbdma_status_gm20b_fusa.c \
+	hal/fifo/preempt_gv11b_fusa.c \
+	hal/fifo/ramfc_gp10b_fusa.c \
+	hal/fifo/ramfc_gv11b_fusa.c \
+	hal/fifo/ramin_gk20a_fusa.c \
+	hal/fifo/ramin_gm20b_fusa.c \
+	hal/fifo/ramin_gv11b_fusa.c \
+	hal/fifo/runlist_fifo_gk20a_fusa.c \
+	hal/fifo/runlist_fifo_gv11b_fusa.c \
+	hal/fifo/runlist_ram_gv11b_fusa.c \
+	hal/fifo/tsg_gk20a_fusa.c \
+	hal/fifo/tsg_gv11b_fusa.c \
+	hal/fifo/usermode_gv11b_fusa.c \
+	hal/fuse/fuse_gm20b_fusa.c \
+	hal/fuse/fuse_gp10b_fusa.c \
+	hal/gr/config/gr_config_gm20b_fusa.c \
+	hal/gr/config/gr_config_gv100_fusa.c \
+	hal/gr/ctxsw_prog/ctxsw_prog_gm20b_fusa.c \
+	hal/gr/ctxsw_prog/ctxsw_prog_gp10b_fusa.c \
+	hal/gr/ctxsw_prog/ctxsw_prog_gv11b_fusa.c \
+	hal/gr/ecc/ecc_gv11b_fusa.c \
+	hal/gr/falcon/gr_falcon_gm20b_fusa.c \
+	hal/gr/falcon/gr_falcon_gp10b_fusa.c \
+	hal/gr/falcon/gr_falcon_gv11b_fusa.c \
+	hal/gr/init/gr_init_gm20b_fusa.c \
+	hal/gr/init/gr_init_gp10b_fusa.c \
+	hal/gr/init/gr_init_gv11b_fusa.c \
+	hal/gr/intr/gr_intr_gm20b_fusa.c \
+	hal/gr/intr/gr_intr_gp10b_fusa.c \
+	hal/gr/intr/gr_intr_gv11b_fusa.c \
+	hal/ltc/ltc_gm20b_fusa.c \
+	hal/ltc/ltc_gp10b_fusa.c \
+	hal/ltc/ltc_gv11b_fusa.c \
+	hal/mc/mc_gm20b_fusa.c \
+	hal/mc/mc_gp10b_fusa.c \
+	hal/mc/mc_gv11b_fusa.c \
+	hal/netlist/netlist_gv11b_fusa.c \
+	hal/pmu/pmu_gk20a_fusa.c \
+	hal/pmu/pmu_gv11b_fusa.c \
+	hal/priv_ring/priv_ring_gm20b_fusa.c \
+	hal/priv_ring/priv_ring_gp10b_fusa.c \
+	hal/ptimer/ptimer_gk20a_fusa.c \
+	hal/sync/syncpt_cmdbuf_gv11b_fusa.c \
+	hal/therm/therm_gv11b_fusa.c \
+	hal/top/top_gm20b_fusa.c \
+	hal/top/top_gv11b_fusa.c \
+	hal/cic/cic_gv11b_fusa.c \
+	hal/cic/cic_lut_gv11b_fusa.c
+
+# Source files below are not guaranteed to be functionaly safe (FuSa) and are
+# only included in the normal build.
+ifeq ($(CONFIG_NVGPU_HAL_NON_FUSA),1)
+srcs += hal/init/hal_gp10b.c \
+	hal/init/hal_gp10b_litter.c \
+	hal/init/hal_gm20b.c \
+	hal/init/hal_gm20b_litter.c \
+	hal/mm/cache/flush_gk20a.c \
+	hal/mm/mm_gm20b.c \
+	hal/mm/mm_gk20a.c \
+	hal/mm/gmmu/gmmu_gk20a.c \
+	hal/mm/gmmu/gmmu_gm20b.c \
+	hal/mc/mc_gm20b.c  \
+	hal/bus/bus_gk20a.c \
+	hal/class/class_gm20b.c \
+	hal/class/class_gp10b.c \
+	hal/clk/clk_gm20b.c \
+	hal/falcon/falcon_gk20a.c \
+	hal/gr/config/gr_config_gm20b.c \
+	hal/gr/ecc/ecc_gp10b.c \
+	hal/gr/ecc/ecc_gv11b.c \
+	hal/gr/init/gr_init_gm20b.c \
+	hal/gr/init/gr_init_gp10b.c \
+	hal/gr/init/gr_init_gv11b.c \
+	hal/gr/intr/gr_intr_gm20b.c \
+	hal/gr/intr/gr_intr_gp10b.c \
+	hal/gr/falcon/gr_falcon_gm20b.c \
+	hal/priv_ring/priv_ring_gm20b.c \
+	hal/power_features/cg/gm20b_gating_reglist.c \
+	hal/power_features/cg/gp10b_gating_reglist.c \
+	hal/ce/ce2_gk20a.c \
+	hal/therm/therm_gm20b.c \
+	hal/therm/therm_gp10b.c \
+	hal/ltc/ltc_gm20b.c \
+	hal/ltc/ltc_gp10b.c \
+	hal/ltc/intr/ltc_intr_gm20b.c \
+	hal/ltc/intr/ltc_intr_gp10b.c \
+	hal/fb/fb_gp10b.c \
+	hal/fb/fb_gp106.c \
+	hal/fb/fb_gm20b.c \
+	hal/fb/fb_gv11b.c \
+	hal/fb/intr/fb_intr_ecc_gv11b.c \
+	hal/fuse/fuse_gm20b.c \
+	hal/fifo/fifo_gk20a.c \
+	hal/fifo/preempt_gk20a.c \
+	hal/fifo/engines_gm20b.c \
+	hal/fifo/pbdma_gm20b.c \
+	hal/fifo/pbdma_gp10b.c \
+	hal/fifo/engine_status_gm20b.c \
+	hal/fifo/ramfc_gk20a.c  \
+	hal/fifo/ramfc_gp10b.c  \
+	hal/fifo/ramin_gk20a.c \
+	hal/fifo/ramin_gp10b.c \
+	hal/fifo/channel_gk20a.c \
+	hal/fifo/channel_gm20b.c \
+	hal/fifo/tsg_gk20a.c \
+	hal/fifo/fifo_intr_gk20a.c \
+	hal/fifo/mmu_fault_gk20a.c \
+	hal/fifo/mmu_fault_gm20b.c \
+	hal/fifo/mmu_fault_gp10b.c \
+	hal/fifo/ctxsw_timeout_gk20a.c \
+	hal/fifo/runlist_fifo_gk20a.c \
+	hal/fifo/runlist_ram_gk20a.c \
+	hal/netlist/netlist_gm20b.c \
+	hal/netlist/netlist_gp10b.c \
+	hal/sync/syncpt_cmdbuf_gk20a.c \
+	hal/pmu/pmu_gv11b.c \
+	hal/top/top_gm20b.c \
+	hal/top/top_gp106.c \
+	hal/top/top_gp10b.c \
+	hal/gr/ctxsw_prog/ctxsw_prog_gm20b.c \
+	hal/gr/ctxsw_prog/ctxsw_prog_gp10b.c \
+	hal/gr/ctxsw_prog/ctxsw_prog_gv11b.c
+else
+ifeq ($(CONFIG_NVGPU_DGPU),1)
+# non-FUSA files needed to build dGPU in safety
+srcs += hal/gr/falcon/gr_falcon_gm20b.c \
+	hal/fuse/fuse_gm20b.c \
+	hal/fb/fb_gp106.c \
+	hal/falcon/falcon_gk20a.c \
+	hal/bus/bus_gk20a.c \
+	hal/pmu/pmu_gv11b.c
+endif
+endif
+
+ifeq ($(CONFIG_NVGPU_CLK_ARB),1)
+srcs += \
+	common/clk_arb/clk_arb.c \
+	common/clk_arb/clk_arb_gp10b.c
+endif
+
+ifeq ($(CONFIG_NVGPU_ACR_LEGACY),1)
+srcs += \
+	common/acr/acr_blob_construct_v0.c \
+	common/acr/acr_sw_gm20b.c \
+	common/acr/acr_sw_gp10b.c
+endif
+
+ifeq ($(CONFIG_NVGPU_ENGINE_QUEUE),1)
+srcs += common/engine_queues/engine_mem_queue.c \
+	common/engine_queues/engine_dmem_queue.c \
+	common/engine_queues/engine_emem_queue.c \
+	common/engine_queues/engine_fb_queue.c
+endif
+
+ifeq ($(CONFIG_NVGPU_GRAPHICS),1)
+srcs += common/gr/zbc.c \
+	common/gr/zcull.c \
+	hal/gr/zbc/zbc_gm20b.c \
+	hal/gr/zbc/zbc_gp10b.c \
+	hal/gr/zbc/zbc_gv11b.c \
+	hal/gr/zcull/zcull_gm20b.c \
+	hal/gr/zcull/zcull_gv11b.c
+endif
+
+ifeq ($(CONFIG_NVGPU_DEBUGGER),1)
+srcs += common/debugger.c \
+	common/regops/regops.c \
+	common/gr/hwpm_map.c \
+	common/perf/perfbuf.c \
+	hal/regops/regops_gv11b.c \
+	hal/regops/allowlist_gv11b.c \
+	hal/gr/ctxsw_prog/ctxsw_prog_gm20b_dbg.c \
+	hal/gr/hwpm_map/hwpm_map_gv100.c \
+	hal/ltc/ltc_gm20b_dbg.c \
+	hal/ptimer/ptimer_gp10b.c \
+	hal/perf/perf_gv11b.c \
+	hal/perf/perf_tu104.c \
+	hal/gr/gr/gr_gk20a.c \
+	hal/gr/gr/gr_gm20b.c \
+	hal/gr/gr/gr_gp10b.c \
+	hal/gr/gr/gr_gv11b.c \
+	hal/gr/gr/gr_gv100.c \
+	hal/gr/gr/gr_tu104.c
+ifeq ($(CONFIG_NVGPU_HAL_NON_FUSA),1)
+srcs += hal/regops/regops_gm20b.c \
+	hal/regops/regops_gp10b.c \
+	hal/regops/regops_tu104.c \
+	hal/regops/allowlist_tu104.c \
+	hal/perf/perf_gm20b.c
+endif
+endif
+
+ifeq ($(CONFIG_NVGPU_PROFILER),1)
+srcs += common/profiler/profiler.c \
+	common/profiler/pm_reservation.c \
+	hal/priv_ring/priv_ring_gv11b.c \
+	hal/ptimer/ptimer_gv11b.c
+endif
+
+ifeq ($(CONFIG_NVGPU_KERNEL_MODE_SUBMIT),1)
+srcs += common/fifo/submit.c \
+        common/fifo/priv_cmdbuf.c \
+        common/fifo/job.c \
+        common/fifo/channel_worker.c \
+	common/sync/channel_sync.c \
+	common/sync/channel_sync_syncpt.c
+endif
+
+ifeq ($(CONFIG_NVGPU_CHANNEL_WDT),1)
+srcs += common/fifo/watchdog.c \
+	common/fifo/channel_wdt.c
+endif
+
+ifeq ($(CONFIG_NVGPU_SW_SEMAPHORE),1)
+srcs += common/semaphore/semaphore_sea.c \
+	common/semaphore/semaphore_pool.c \
+	common/semaphore/semaphore_hw.c \
+	common/semaphore/semaphore.c \
+	common/sync/channel_sync_semaphore.c \
+	hal/sync/sema_cmdbuf_gk20a.c \
+	hal/sync/sema_cmdbuf_gv11b.c
+endif
+
+ifeq ($(CONFIG_NVGPU_USERD),1)
+srcs += common/fifo/userd.c \
+	hal/fifo/userd_gv11b.c
+endif
+
+ifeq ($(CONFIG_NVGPU_RECOVERY),1)
+srcs += hal/rc/rc_gv11b.c
+ifeq ($(CONFIG_NVGPU_HAL_NON_FUSA),1)
+srcs += hal/rc/rc_gk20a.c
+endif
+endif
+
+ifeq ($(CONFIG_NVGPU_FENCE),1)
+srcs += common/fence/fence.c
+ifeq ($(CONFIG_TEGRA_GK20A_NVHOST),1)
+srcs += common/fence/fence_syncpt.c
+endif
+ifeq ($(CONFIG_NVGPU_SW_SEMAPHORE),1)
+srcs += common/fence/fence_sema.c
+endif
+endif
+
+ifeq ($(CONFIG_NVGPU_FECS_TRACE),1)
+srcs += common/gr/fecs_trace.c \
+	hal/gr/fecs_trace/fecs_trace_gm20b.c \
+	hal/gr/fecs_trace/fecs_trace_gv11b.c
+ifeq ($(CONFIG_NVGPU_IGPU_VIRT),1)
+srcs += common/vgpu/gr/fecs_trace_vgpu.c
+endif
+endif
+
+ifeq ($(CONFIG_NVGPU_CYCLESTATS),1)
+srcs += common/perf/cyclestats_snapshot.c \
+	common/cyclestats/cyclestats.c
+ifeq ($(CONFIG_NVGPU_IGPU_VIRT),1)
+srcs += common/vgpu/perf/cyclestats_snapshot_vgpu.c
+endif
+endif
+
+# POSIX file used for unit testing for both qnx and linux
+ifdef NVGPU_FAULT_INJECTION_ENABLEMENT
+srcs += os/posix/posix-fault-injection.c
+endif
+
+ifeq ($(CONFIG_NVGPU_LS_PMU),1)
+# Add LS PMU files which are required for normal build
+srcs += \
+	common/pmu/boardobj/boardobj.c \
+	common/pmu/boardobj/boardobjgrp.c \
+	common/pmu/boardobj/boardobjgrpmask.c \
+	common/pmu/boardobj/boardobjgrp_e255.c \
+	common/pmu/boardobj/boardobjgrp_e32.c \
+	common/pmu/clk/clk.c \
+	common/pmu/volt/volt.c \
+	common/pmu/clk/clk_domain.c \
+	common/pmu/clk/clk_fll.c \
+	common/pmu/clk/clk_prog.c \
+	common/pmu/clk/clk_vf_point.c \
+	common/pmu/clk/clk_vin.c \
+	common/pmu/fw/fw.c \
+	common/pmu/fw/fw_ver_ops.c \
+	common/pmu/fw/fw_ns_bootstrap.c \
+	common/pmu/ipc/pmu_cmd.c \
+	common/pmu/ipc/pmu_msg.c \
+	common/pmu/ipc/pmu_queue.c \
+	common/pmu/ipc/pmu_seq.c \
+	common/pmu/lpwr/rppg.c \
+	common/pmu/lsfm/lsfm.c \
+	common/pmu/lsfm/lsfm_sw_gm20b.c \
+	common/pmu/lsfm/lsfm_sw_gp10b.c \
+	common/pmu/lsfm/lsfm_sw_gv100.c \
+	common/pmu/lsfm/lsfm_sw_tu104.c \
+	common/pmu/perf/vfe_equ.c \
+	common/pmu/perf/vfe_var.c \
+	common/pmu/perf/perf.c \
+	common/pmu/perf/pstate.c \
+	common/pmu/perf/change_seq.c \
+	common/pmu/perfmon/pmu_perfmon.c \
+	common/pmu/perfmon/pmu_perfmon_sw_gm20b.c \
+	common/pmu/perfmon/pmu_perfmon_sw_gv11b.c \
+	common/pmu/pmgr/pmgr.c \
+	common/pmu/pmgr/pmgrpmu.c \
+	common/pmu/pmgr/pwrdev.c \
+	common/pmu/pmgr/pwrmonitor.c \
+	common/pmu/pmgr/pwrpolicy.c \
+	common/pmu/super_surface/super_surface.c \
+	common/pmu/therm/thrm.c \
+	common/pmu/therm/therm_channel.c \
+	common/pmu/therm/therm_dev.c \
+	common/pmu/volt/volt_dev.c \
+	common/pmu/volt/volt_policy.c \
+	common/pmu/volt/volt_rail.c \
+	common/pmu/allocator.c \
+	common/pmu/pmu_debug.c \
+	common/pmu/pmu_mutex.c \
+	common/pmu/pmu_pstate.c \
+	common/pmu/pmu_rtos_init.c \
+	hal/therm/therm_tu104.c \
+	hal/pmu/pmu_gk20a.c \
+	hal/pmu/pmu_gm20b.c \
+	hal/pmu/pmu_gp10b.c \
+	hal/pmu/pmu_tu104.c
+
+ifeq ($(CONFIG_NVGPU_POWER_PG),1)
+srcs += common/pmu/pg/pg_sw_gm20b.c \
+	common/pmu/pg/pg_sw_gp10b.c \
+	common/pmu/pg/pg_sw_gp106.c \
+	common/pmu/pg/pg_sw_gv11b.c \
+	common/pmu/pg/pmu_pg.c \
+	common/pmu/pg/pmu_aelpg.c
+endif
+
+ifeq ($(CONFIG_NVGPU_CLK_ARB),1)
+srcs += common/clk_arb/clk_arb_gv100.c
+endif
+
+endif
+
+ifeq ($(CONFIG_NVGPU_POWER_PG),1)
+srcs += common/power_features/pg/pg.c
+endif
+
+ifeq ($(CONFIG_NVGPU_IGPU_VIRT),1)
+srcs += common/vgpu/init/init_vgpu.c \
+	common/vgpu/ivc/comm_vgpu.c \
+	common/vgpu/intr/intr_vgpu.c \
+	common/vgpu/ptimer/ptimer_vgpu.c \
+	common/vgpu/top/top_vgpu.c \
+	common/vgpu/fifo/fifo_vgpu.c \
+	common/vgpu/fifo/channel_vgpu.c \
+	common/vgpu/fifo/tsg_vgpu.c \
+	common/vgpu/fifo/preempt_vgpu.c \
+	common/vgpu/fifo/runlist_vgpu.c \
+	common/vgpu/fifo/ramfc_vgpu.c \
+	common/vgpu/perf/perf_vgpu.c \
+	common/vgpu/profiler/profiler_vgpu.c \
+	common/vgpu/mm/mm_vgpu.c \
+	common/vgpu/mm/vm_vgpu.c \
+	common/vgpu/gr/gr_vgpu.c \
+	common/vgpu/fb/fb_vgpu.c \
+	common/vgpu/gr/ctx_vgpu.c \
+	common/vgpu/gr/subctx_vgpu.c \
+	common/vgpu/clk_vgpu.c \
+	common/vgpu/debugger_vgpu.c \
+	common/vgpu/pm_reservation_vgpu.c \
+	common/vgpu/ltc/ltc_vgpu.c \
+	common/vgpu/fbp/fbp_vgpu.c \
+	common/vgpu/ce_vgpu.c \
+	hal/vgpu/init/init_hal_vgpu.c \
+	hal/vgpu/init/vgpu_hal_gv11b.c \
+	hal/vgpu/fifo/fifo_gv11b_vgpu.c \
+	hal/vgpu/sync/syncpt_cmdbuf_gv11b_vgpu.c
+
+ifeq ($(CONFIG_NVGPU_USERD),1)
+srcs += common/vgpu/fifo/userd_vgpu.c
+endif
+
+ifeq ($(CONFIG_NVGPU_COMPRESSION),1)
+srcs += common/vgpu/cbc/cbc_vgpu.c
+endif
+endif
+
+ifeq ($(CONFIG_NVGPU_COMPRESSION),1)
+srcs += common/mm/comptags.c \
+	common/cbc/cbc.c \
+	hal/cbc/cbc_gm20b.c \
+	hal/cbc/cbc_gp10b.c \
+	hal/cbc/cbc_gv11b.c
+endif
+
+ifeq ($(CONFIG_NVGPU_NVLINK),1)
+srcs += common/vbios/nvlink_bios.c \
+	common/nvlink/probe.c \
+	common/nvlink/init/device_reginit.c \
+	common/nvlink/init/device_reginit_gv100.c \
+	common/nvlink/minion.c \
+	common/nvlink/link_mode_transitions.c \
+	common/nvlink/nvlink.c \
+	hal/nvlink/minion_gv100.c \
+	hal/nvlink/minion_tu104.c \
+	hal/nvlink/nvlink_gv100.c \
+	hal/nvlink/nvlink_tu104.c \
+	hal/nvlink/intr_and_err_handling_tu104.c \
+	hal/nvlink/link_mode_transitions_gv100.c \
+	hal/nvlink/link_mode_transitions_tu104.c
+endif
+
+ifeq ($(CONFIG_NVGPU_DGPU),1)
+srcs += common/sec2/sec2.c \
+	common/sec2/sec2_allocator.c \
+	common/sec2/sec2_lsfm.c \
+	common/sec2/ipc/sec2_cmd.c \
+	common/sec2/ipc/sec2_msg.c \
+	common/sec2/ipc/sec2_queue.c \
+	common/sec2/ipc/sec2_seq.c \
+	common/vbios/bios.c \
+	common/vbios/bios_sw_gv100.c \
+	common/vbios/bios_sw_tu104.c \
+	common/falcon/falcon_sw_tu104.c \
+	common/acr/acr_sw_tu104.c \
+	common/mm/allocators/page_allocator.c \
+	common/mm/vidmem.c \
+	common/pramin.c \
+	common/ce/ce_app.c \
+	common/sbr/sbr.c \
+	hal/mm/mm_gv100.c \
+	hal/mm/mm_tu104.c \
+	hal/mc/mc_gv100.c  \
+	hal/mc/mc_tu104.c  \
+	hal/bus/bus_gv100.c \
+	hal/bus/bus_tu104.c \
+	hal/ce/ce_tu104.c \
+	hal/class/class_tu104.c \
+	hal/clk/clk_tu104.c \
+	hal/clk/clk_mon_tu104.c \
+	hal/gr/init/gr_init_gv100.c \
+	hal/gr/init/gr_init_tu104.c \
+	hal/gr/intr/gr_intr_tu104.c \
+	hal/gr/falcon/gr_falcon_tu104.c \
+	hal/fbpa/fbpa_tu104.c \
+	hal/init/hal_tu104.c \
+	hal/init/hal_tu104_litter.c \
+	hal/power_features/cg/tu104_gating_reglist.c \
+	hal/ltc/ltc_tu104.c \
+	hal/fb/fb_gv100.c \
+	hal/fb/fb_tu104.c \
+	hal/fb/fb_mmu_fault_tu104.c \
+	hal/fb/intr/fb_intr_gv100.c \
+	hal/fb/intr/fb_intr_tu104.c \
+	hal/func/func_tu104.c \
+	hal/fifo/fifo_tu104.c \
+	hal/fifo/usermode_tu104.c \
+	hal/fifo/pbdma_tu104.c \
+	hal/fifo/ramfc_tu104.c  \
+	hal/fifo/ramin_tu104.c \
+	hal/fifo/channel_gv100.c \
+	hal/fifo/runlist_ram_tu104.c \
+	hal/fifo/runlist_fifo_gv100.c \
+	hal/fifo/runlist_fifo_tu104.c \
+	hal/fifo/fifo_intr_gv100.c \
+	hal/fuse/fuse_gp106.c \
+	hal/fuse/fuse_tu104.c \
+	hal/netlist/netlist_gv100.c \
+	hal/netlist/netlist_tu104.c \
+	hal/nvdec/nvdec_gp106.c \
+	hal/nvdec/nvdec_tu104.c \
+	hal/gsp/gsp_tu104.c \
+	hal/sec2/sec2_tu104.c \
+	hal/pramin/pramin_gp10b.c \
+	hal/pramin/pramin_gv100.c \
+	hal/pramin/pramin_init.c \
+	hal/pramin/pramin_tu104.c \
+	hal/bios/bios_tu104.c \
+	hal/top/top_gv100.c \
+	hal/xve/xve_gp106.c \
+	hal/xve/xve_tu104.c
+
+ifeq ($(CONFIG_NVGPU_COMPRESSION),1)
+srcs +=	hal/cbc/cbc_tu104.c
+endif
+endif
+
+ifeq ($(CONFIG_NVGPU_SIM),1)
+srcs += common/sim/sim.c \
+	common/sim/sim_pci.c \
+	common/sim/sim_netlist.c
+endif
+
+ifeq ($(CONFIG_NVGPU_NON_FUSA),1)
+srcs +=	common/power_features/power_features.c
+endif
+
+ifeq ($(CONFIG_NVGPU_TPC_POWERGATE),1)
+srcs +=	hal/tpc/tpc_gv11b.c
+endif

drivers/gpu/nvgpu/common/acr/acr.c

@@ -0,0 +1,172 @@
+/*
+ * Copyright (c) 2019-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/types.h>
+#include <nvgpu/dma.h>
+#include <nvgpu/firmware.h>
+#include <nvgpu/gk20a.h>
+#include <nvgpu/acr.h>
+
+#include "acr_priv.h"
+#ifdef CONFIG_NVGPU_ACR_LEGACY
+#include "acr_sw_gm20b.h"
+#include "acr_sw_gp10b.h"
+#endif
+#include "acr_sw_gv11b.h"
+#ifdef CONFIG_NVGPU_DGPU
+#include "acr_sw_tu104.h"
+#endif
+
+#if defined(CONFIG_NVGPU_NEXT) && defined(CONFIG_NVGPU_NON_FUSA)
+#include "nvgpu_next_gpuid.h"
+#endif
+
+/* ACR public API's */
+bool nvgpu_acr_is_lsf_lazy_bootstrap(struct gk20a *g, struct nvgpu_acr *acr,
+	u32 falcon_id)
+{
+	if (acr == NULL) {
+		return false;
+	}
+
+	if ((falcon_id == FALCON_ID_FECS) || (falcon_id == FALCON_ID_PMU) ||
+		(falcon_id == FALCON_ID_GPCCS)) {
+		return acr->lsf[falcon_id].is_lazy_bootstrap;
+	} else {
+		nvgpu_err(g, "Invalid falcon id\n");
+		return false;
+	}
+}
+
+#ifdef CONFIG_NVGPU_DGPU
+int nvgpu_acr_alloc_blob_prerequisite(struct gk20a *g, struct nvgpu_acr *acr,
+	size_t size)
+{
+	if (acr == NULL) {
+		return -EINVAL;
+	}
+
+	return acr->alloc_blob_space(g, size, &acr->ucode_blob);
+}
+#endif
+
+/* ACR blob construct & bootstrap */
+int nvgpu_acr_bootstrap_hs_acr(struct gk20a *g, struct nvgpu_acr *acr)
+{
+	int err = 0;
+
+	if (acr == NULL) {
+		return -EINVAL;
+	}
+
+	err = acr->bootstrap_hs_acr(g, acr);
+	if (err != 0) {
+		nvgpu_err(g, "ACR bootstrap failed");
+	}
+
+	nvgpu_log(g, gpu_dbg_gr, "ACR bootstrap Done");
+	return err;
+}
+
+int nvgpu_acr_construct_execute(struct gk20a *g)
+{
+	int err = 0;
+
+	if (g->acr == NULL) {
+		return -EINVAL;
+	}
+
+	err = g->acr->prepare_ucode_blob(g);
+	if (err != 0) {
+		nvgpu_err(g, "ACR ucode blob prepare failed");
+		goto done;
+	}
+
+	err = nvgpu_acr_bootstrap_hs_acr(g, g->acr);
+	if (err != 0) {
+		nvgpu_err(g, "Bootstrap HS ACR failed");
+	}
+
+done:
+	return err;
+}
+
+/* ACR init */
+int nvgpu_acr_init(struct gk20a *g)
+{
+	u32 ver = nvgpu_safe_add_u32(g->params.gpu_arch,
+					g->params.gpu_impl);
+	int err = 0;
+
+	if (g->acr != NULL) {
+		/*
+		 * Recovery/unrailgate case, we do not need to do ACR init as ACR is
+		 * set during cold boot & doesn't execute ACR clean up as part off
+		 * sequence, so reuse to perform faster boot.
+		 */
+		return err;
+	}
+
+	g->acr = (struct nvgpu_acr *)nvgpu_kzalloc(g, sizeof(struct nvgpu_acr));
+	if (g->acr == NULL) {
+		err = -ENOMEM;
+		goto done;
+	}
+
+	switch (ver) {
+#ifdef CONFIG_NVGPU_ACR_LEGACY
+	case GK20A_GPUID_GM20B:
+	case GK20A_GPUID_GM20B_B:
+		nvgpu_gm20b_acr_sw_init(g, g->acr);
+		break;
+	case NVGPU_GPUID_GP10B:
+		nvgpu_gp10b_acr_sw_init(g, g->acr);
+		break;
+#endif
+	case NVGPU_GPUID_GV11B:
+		nvgpu_gv11b_acr_sw_init(g, g->acr);
+		break;
+#if defined(CONFIG_NVGPU_NEXT)
+	case NVGPU_NEXT_GPUID:
+		nvgpu_next_acr_sw_init(g, g->acr);
+		break;
+#endif
+#ifdef CONFIG_NVGPU_DGPU
+	case NVGPU_GPUID_TU104:
+		nvgpu_tu104_acr_sw_init(g, g->acr);
+		break;
+#if defined(CONFIG_NVGPU_NEXT)
+	case NVGPU_NEXT_DGPU_GPUID:
+		nvgpu_next_dgpu_acr_sw_init(g, g->acr);
+		break;
+#endif
+#endif
+	default:
+		nvgpu_kfree(g, g->acr);
+		err = -EINVAL;
+		nvgpu_err(g, "no support for GPUID %x", ver);
+		break;
+	}
+
+done:
+	return err;
+}

drivers/gpu/nvgpu/common/acr/acr_blob_alloc.c

@@ -0,0 +1,63 @@
+/*
+ * Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/types.h>
+#include <nvgpu/dma.h>
+#include <nvgpu/gk20a.h>
+
+#include "acr_wpr.h"
+#include "acr_priv.h"
+#include "acr_blob_alloc.h"
+
+int nvgpu_acr_alloc_blob_space_sys(struct gk20a *g, size_t size,
+	struct nvgpu_mem *mem)
+{
+	return nvgpu_dma_alloc_flags_sys(g, NVGPU_DMA_PHYSICALLY_ADDRESSED,
+		size, mem);
+}
+#ifdef CONFIG_NVGPU_DGPU
+int nvgpu_acr_alloc_blob_space_vid(struct gk20a *g, size_t size,
+	struct nvgpu_mem *mem)
+{
+	struct wpr_carveout_info wpr_inf;
+	int err;
+
+	if (mem->size != 0ULL) {
+		return 0;
+	}
+
+	g->acr->get_wpr_info(g, &wpr_inf);
+
+	/*
+	 * Even though this mem_desc wouldn't be used, the wpr region needs to
+	 * be reserved in the allocator.
+	 */
+	err = nvgpu_dma_alloc_vid_at(g, wpr_inf.size,
+		&g->acr->wpr_dummy, wpr_inf.wpr_base);
+	if (err != 0) {
+		return err;
+	}
+
+	return nvgpu_dma_alloc_vid_at(g, wpr_inf.size, mem,
+		wpr_inf.nonwpr_base);
+}
+#endif

drivers/gpu/nvgpu/common/acr/acr_blob_alloc.h

@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef ACR_BLOB_ALLOC_H
+#define ACR_BLOB_ALLOC_H
+
+struct gk20a;
+struct nvgpu_mem;
+
+int nvgpu_acr_alloc_blob_space_sys(struct gk20a *g, size_t size,
+	struct nvgpu_mem *mem);
+#ifdef CONFIG_NVGPU_DGPU
+int nvgpu_acr_alloc_blob_space_vid(struct gk20a *g, size_t size,
+	struct nvgpu_mem *mem);
+#endif
+
+#endif /* ACR_BLOB_ALLOC_H */

drivers/gpu/nvgpu/common/acr/acr_blob_construct.h

@@ -0,0 +1,153 @@
+/*
+ * Copyright (c) 2019-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef ACR_BLOB_CONSTRUCT_H
+#define ACR_BLOB_CONSTRUCT_H
+
+#include <nvgpu/falcon.h>
+#include <nvgpu/flcnif_cmn.h>
+#include <nvgpu/pmu.h>
+
+#include "nvgpu_acr_interface.h"
+
+#define UCODE_NB_MAX_DATE_LENGTH  64U
+struct ls_falcon_ucode_desc {
+	u32 descriptor_size;
+	u32 image_size;
+	u32 tools_version;
+	u32 app_version;
+	char date[UCODE_NB_MAX_DATE_LENGTH];
+	u32 bootloader_start_offset;
+	u32 bootloader_size;
+	u32 bootloader_imem_offset;
+	u32 bootloader_entry_point;
+	u32 app_start_offset;
+	u32 app_size;
+	u32 app_imem_offset;
+	u32 app_imem_entry;
+	u32 app_dmem_offset;
+	u32 app_resident_code_offset;
+	u32 app_resident_code_size;
+	u32 app_resident_data_offset;
+	u32 app_resident_data_size;
+	u32 nb_imem_overlays;
+	u32 nb_dmem_overlays;
+	struct {u32 start; u32 size; } load_ovl[UCODE_NB_MAX_DATE_LENGTH];
+	u32 compressed;
+};
+
+struct ls_falcon_ucode_desc_v1 {
+	u32 descriptor_size;
+	u32 image_size;
+	u32 tools_version;
+	u32 app_version;
+	char date[UCODE_NB_MAX_DATE_LENGTH];
+	u32 secure_bootloader;
+	u32 bootloader_start_offset;
+	u32 bootloader_size;
+	u32 bootloader_imem_offset;
+	u32 bootloader_entry_point;
+	u32 app_start_offset;
+	u32 app_size;
+	u32 app_imem_offset;
+	u32 app_imem_entry;
+	u32 app_dmem_offset;
+	u32 app_resident_code_offset;
+	u32 app_resident_code_size;
+	u32 app_resident_data_offset;
+	u32 app_resident_data_size;
+	u32 nb_imem_overlays;
+	u32 nb_dmem_overlays;
+	struct {u32 start; u32 size; } load_ovl[64];
+	u32 compressed;
+};
+
+struct flcn_ucode_img {
+	u32 *data;
+	struct ls_falcon_ucode_desc *desc;
+	u32 data_size;
+	struct lsf_ucode_desc *lsf_desc;
+	bool is_next_core_img;
+#if defined(CONFIG_NVGPU_NEXT)
+	struct falcon_next_core_ucode_desc *ndesc;
+#endif
+};
+
+struct lsfm_managed_ucode_img {
+	struct lsfm_managed_ucode_img *next;
+	struct lsf_wpr_header wpr_header;
+	struct lsf_lsb_header lsb_header;
+	struct flcn_bl_dmem_desc bl_gen_desc;
+	u32 bl_gen_desc_size;
+	u32 full_ucode_size;
+	struct flcn_ucode_img ucode_img;
+};
+
+#ifdef CONFIG_NVGPU_DGPU
+/*
+ * LSF shared SubWpr Header
+ *
+ * use_case_id - Shared SubWpr use case ID (updated by nvgpu)
+ * start_addr  - start address of subWpr (updated by nvgpu)
+ * size_4K     - size of subWpr in 4K (updated by nvgpu)
+ */
+struct lsf_shared_sub_wpr_header {
+	u32 use_case_id;
+	u32 start_addr;
+	u32 size_4K;
+};
+
+/*
+ * LSFM SUB WPRs struct
+ * pnext          : Next entry in the list, NULL if last
+ * sub_wpr_header : SubWpr Header struct
+ */
+struct lsfm_sub_wpr {
+	struct lsfm_sub_wpr *pnext;
+	struct lsf_shared_sub_wpr_header sub_wpr_header;
+};
+#endif
+
+struct ls_flcn_mgr {
+	u16 managed_flcn_cnt;
+	u32 wpr_size;
+	struct lsfm_managed_ucode_img *ucode_img_list;
+#ifdef CONFIG_NVGPU_DGPU
+	u16 managed_sub_wpr_count;
+	struct lsfm_sub_wpr *psub_wpr_list;
+#endif
+};
+
+int nvgpu_acr_prepare_ucode_blob(struct gk20a *g);
+#ifdef CONFIG_NVGPU_LS_PMU
+int nvgpu_acr_lsf_pmu_ucode_details(struct gk20a *g, void *lsf_ucode_img);
+#if defined(CONFIG_NVGPU_NEXT)
+s32 nvgpu_acr_lsf_pmu_ncore_ucode_details(struct gk20a *g, void *lsf_ucode_img);
+#endif
+#endif
+int nvgpu_acr_lsf_fecs_ucode_details(struct gk20a *g, void *lsf_ucode_img);
+int nvgpu_acr_lsf_gpccs_ucode_details(struct gk20a *g, void *lsf_ucode_img);
+#ifdef CONFIG_NVGPU_DGPU
+int nvgpu_acr_lsf_sec2_ucode_details(struct gk20a *g, void *lsf_ucode_img);
+#endif
+
+#endif /* ACR_BLOB_CONSTRUCT_H */

drivers/gpu/nvgpu/common/acr/acr_blob_construct_v0.c

@@ -0,0 +1,801 @@
+/*
+ * Copyright (c) 2019-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/firmware.h>
+#include <nvgpu/pmu.h>
+#include <nvgpu/falcon.h>
+#include <nvgpu/gk20a.h>
+#include <nvgpu/string.h>
+#include <nvgpu/bug.h>
+#include <nvgpu/gr/gr_falcon.h>
+#include <nvgpu/pmu/fw.h>
+#include <nvgpu/gr/gr_utils.h>
+
+#include "acr_blob_construct_v0.h"
+#include "acr_wpr.h"
+#include "acr_priv.h"
+
+#ifdef CONFIG_NVGPU_LS_PMU
+int nvgpu_acr_lsf_pmu_ucode_details_v0(struct gk20a *g, void *lsf_ucode_img)
+{
+	struct lsf_ucode_desc_v0 *lsf_desc;
+	struct nvgpu_firmware *fw_sig;
+	struct nvgpu_firmware *fw_desc;
+	struct nvgpu_firmware *fw_image;
+	struct flcn_ucode_img_v0 *p_img = (struct flcn_ucode_img_v0 *)lsf_ucode_img;
+	int err = 0;
+
+	lsf_desc = nvgpu_kzalloc(g, sizeof(struct lsf_ucode_desc_v0));
+	if (lsf_desc == NULL) {
+		err = -ENOMEM;
+		goto exit;
+	}
+
+	fw_sig = nvgpu_pmu_fw_sig_desc(g, g->pmu);
+	fw_desc = nvgpu_pmu_fw_desc_desc(g, g->pmu);
+	fw_image = nvgpu_pmu_fw_image_desc(g, g->pmu);
+
+	nvgpu_memcpy((u8 *)lsf_desc, (u8 *)fw_sig->data,
+		min_t(size_t, sizeof(*lsf_desc), fw_sig->size));
+
+	lsf_desc->falcon_id = FALCON_ID_PMU;
+
+	p_img->desc = (struct pmu_ucode_desc *)(void *)fw_desc->data;
+	p_img->data = (u32 *)(void *)fw_image->data;
+	p_img->data_size = p_img->desc->image_size;
+	p_img->lsf_desc = (struct lsf_ucode_desc_v0 *)lsf_desc;
+
+exit:
+	return err;
+}
+#endif
+
+int nvgpu_acr_lsf_fecs_ucode_details_v0(struct gk20a *g, void *lsf_ucode_img)
+{
+	struct lsf_ucode_desc_v0 *lsf_desc;
+	struct nvgpu_firmware *fecs_sig;
+	struct flcn_ucode_img_v0 *p_img = (struct flcn_ucode_img_v0 *)lsf_ucode_img;
+	struct nvgpu_gr_falcon *gr_falcon = nvgpu_gr_get_falcon_ptr(g);
+	struct nvgpu_ctxsw_ucode_segments *fecs =
+			nvgpu_gr_falcon_get_fecs_ucode_segments(gr_falcon);
+	int err;
+
+	fecs_sig = nvgpu_request_firmware(g, GM20B_FECS_UCODE_SIG, 0);
+	if (fecs_sig == NULL) {
+		nvgpu_err(g, "failed to load fecs sig");
+		return -ENOENT;
+	}
+	lsf_desc = nvgpu_kzalloc(g, sizeof(struct lsf_ucode_desc_v0));
+	if (lsf_desc == NULL) {
+		err = -ENOMEM;
+		goto rel_sig;
+	}
+	nvgpu_memcpy((u8 *)lsf_desc, (u8 *)fecs_sig->data,
+			min_t(size_t, sizeof(*lsf_desc), fecs_sig->size));
+
+	lsf_desc->falcon_id = FALCON_ID_FECS;
+
+	p_img->desc = nvgpu_kzalloc(g, sizeof(struct pmu_ucode_desc));
+	if (p_img->desc == NULL) {
+		err = -ENOMEM;
+		goto free_lsf_desc;
+	}
+
+	p_img->desc->bootloader_start_offset = fecs->boot.offset;
+	p_img->desc->bootloader_size = NVGPU_ALIGN(fecs->boot.size, 256U);
+	p_img->desc->bootloader_imem_offset = fecs->boot_imem_offset;
+	p_img->desc->bootloader_entry_point = fecs->boot_entry;
+
+	p_img->desc->image_size = NVGPU_ALIGN(fecs->boot.size, 256U) +
+		NVGPU_ALIGN(fecs->code.size, 256U) + NVGPU_ALIGN(fecs->data.size, 256U);
+	p_img->desc->app_size = NVGPU_ALIGN(fecs->code.size, 256U) +
+					NVGPU_ALIGN(fecs->data.size, 256U);
+	p_img->desc->app_start_offset = fecs->code.offset;
+	p_img->desc->app_imem_offset = 0;
+	p_img->desc->app_imem_entry = 0;
+	p_img->desc->app_dmem_offset = 0;
+	p_img->desc->app_resident_code_offset = 0;
+	p_img->desc->app_resident_code_size = fecs->code.size;
+	p_img->desc->app_resident_data_offset =
+				fecs->data.offset - fecs->code.offset;
+	p_img->desc->app_resident_data_size = fecs->data.size;
+	p_img->data = nvgpu_gr_falcon_get_surface_desc_cpu_va(gr_falcon);
+	p_img->data_size = p_img->desc->image_size;
+
+	p_img->lsf_desc = (struct lsf_ucode_desc_v0 *)lsf_desc;
+	nvgpu_acr_dbg(g, "fecs fw loaded\n");
+	nvgpu_release_firmware(g, fecs_sig);
+	return 0;
+free_lsf_desc:
+	nvgpu_kfree(g, lsf_desc);
+rel_sig:
+	nvgpu_release_firmware(g, fecs_sig);
+	return err;
+}
+
+int nvgpu_acr_lsf_gpccs_ucode_details_v0(struct gk20a *g, void *lsf_ucode_img)
+{
+	struct lsf_ucode_desc_v0 *lsf_desc;
+	struct nvgpu_firmware *gpccs_sig;
+	struct flcn_ucode_img_v0 *p_img = (struct flcn_ucode_img_v0 *)lsf_ucode_img;
+	struct nvgpu_gr_falcon *gr_falcon = nvgpu_gr_get_falcon_ptr(g);
+	struct nvgpu_ctxsw_ucode_segments *gpccs =
+			nvgpu_gr_falcon_get_gpccs_ucode_segments(gr_falcon);
+	int err;
+
+	if (!nvgpu_is_enabled(g, NVGPU_SEC_SECUREGPCCS)) {
+		return -ENOENT;
+	}
+
+	gpccs_sig = nvgpu_request_firmware(g, T18x_GPCCS_UCODE_SIG, 0);
+	if (gpccs_sig == NULL) {
+		nvgpu_err(g, "failed to load gpccs sig");
+		return -ENOENT;
+	}
+	lsf_desc = nvgpu_kzalloc(g, sizeof(struct lsf_ucode_desc_v0));
+	if (lsf_desc == NULL) {
+		err = -ENOMEM;
+		goto rel_sig;
+	}
+	nvgpu_memcpy((u8 *)lsf_desc, (u8 *)gpccs_sig->data,
+			min_t(size_t, sizeof(*lsf_desc), gpccs_sig->size));
+	lsf_desc->falcon_id = FALCON_ID_GPCCS;
+
+	p_img->desc = nvgpu_kzalloc(g, sizeof(struct pmu_ucode_desc));
+	if (p_img->desc == NULL) {
+		err = -ENOMEM;
+		goto free_lsf_desc;
+	}
+
+	p_img->desc->bootloader_start_offset =
+		0;
+	p_img->desc->bootloader_size = NVGPU_ALIGN(gpccs->boot.size, 256U);
+	p_img->desc->bootloader_imem_offset = gpccs->boot_imem_offset;
+	p_img->desc->bootloader_entry_point = gpccs->boot_entry;
+
+	p_img->desc->image_size = NVGPU_ALIGN(gpccs->boot.size, 256U) +
+		NVGPU_ALIGN(gpccs->code.size, 256U) +
+		NVGPU_ALIGN(gpccs->data.size, 256U);
+	p_img->desc->app_size = NVGPU_ALIGN(gpccs->code.size, 256U) +
+		NVGPU_ALIGN(gpccs->data.size, 256U);
+	p_img->desc->app_start_offset = p_img->desc->bootloader_size;
+	p_img->desc->app_imem_offset = 0;
+	p_img->desc->app_imem_entry = 0;
+	p_img->desc->app_dmem_offset = 0;
+	p_img->desc->app_resident_code_offset = 0;
+	p_img->desc->app_resident_code_size = NVGPU_ALIGN(gpccs->code.size, 256U);
+	p_img->desc->app_resident_data_offset =
+		NVGPU_ALIGN(gpccs->data.offset, 256U) -
+		NVGPU_ALIGN(gpccs->code.offset, 256U);
+	p_img->desc->app_resident_data_size = NVGPU_ALIGN(gpccs->data.size, 256U);
+	p_img->data = (u32 *)
+		((u8 *)nvgpu_gr_falcon_get_surface_desc_cpu_va(gr_falcon) +
+							gpccs->boot.offset);
+	p_img->data_size = NVGPU_ALIGN(p_img->desc->image_size, 256U);
+	p_img->lsf_desc = (struct lsf_ucode_desc_v0 *)lsf_desc;
+	nvgpu_acr_dbg(g, "gpccs fw loaded\n");
+	nvgpu_release_firmware(g, gpccs_sig);
+	return 0;
+free_lsf_desc:
+	nvgpu_kfree(g, lsf_desc);
+rel_sig:
+	nvgpu_release_firmware(g, gpccs_sig);
+	return err;
+}
+
+/*
+ * @brief lsfm_fill_static_lsb_hdr_info
+ * Populate static LSB header information using the provided ucode image
+ */
+static void lsfm_fill_static_lsb_hdr_info(struct gk20a *g,
+	u32 falcon_id, struct lsfm_managed_ucode_img_v0 *pnode)
+{
+	u32 full_app_size = 0;
+	u32 data = 0;
+
+	if (pnode->ucode_img.lsf_desc != NULL) {
+		nvgpu_memcpy((u8 *)&pnode->lsb_header.signature,
+			(u8 *)pnode->ucode_img.lsf_desc,
+			sizeof(struct lsf_ucode_desc_v0));
+	}
+	pnode->lsb_header.ucode_size = pnode->ucode_img.data_size;
+
+	/* Uses a loader. that is has a desc */
+	pnode->lsb_header.data_size = 0;
+
+	/*
+	 * The loader code size is already aligned (padded) such that
+	 * the code following it is aligned, but the size in the image
+	 * desc is not, bloat it up to be on a 256 byte alignment.
+	 */
+	pnode->lsb_header.bl_code_size = NVGPU_ALIGN(
+		pnode->ucode_img.desc->bootloader_size,
+		LSF_BL_CODE_SIZE_ALIGNMENT);
+	full_app_size = NVGPU_ALIGN(pnode->ucode_img.desc->app_size,
+		LSF_BL_CODE_SIZE_ALIGNMENT) +
+		pnode->lsb_header.bl_code_size;
+	pnode->lsb_header.ucode_size = NVGPU_ALIGN(
+		pnode->ucode_img.desc->app_resident_data_offset,
+		LSF_BL_CODE_SIZE_ALIGNMENT) +
+		pnode->lsb_header.bl_code_size;
+	pnode->lsb_header.data_size = full_app_size -
+		pnode->lsb_header.ucode_size;
+	/*
+	 * Though the BL is located at 0th offset of the image, the VA
+	 * is different to make sure that it doesn't collide the actual
+	 * OS VA range
+	 */
+	pnode->lsb_header.bl_imem_off =
+		pnode->ucode_img.desc->bootloader_imem_offset;
+
+	pnode->lsb_header.flags = 0;
+
+	if (falcon_id == FALCON_ID_PMU) {
+		data = NV_FLCN_ACR_LSF_FLAG_DMACTL_REQ_CTX_TRUE;
+		pnode->lsb_header.flags = data;
+	}
+
+	if (g->acr->lsf[falcon_id].is_priv_load) {
+		pnode->lsb_header.flags |=
+			NV_FLCN_ACR_LSF_FLAG_FORCE_PRIV_LOAD_TRUE;
+	}
+}
+
+/* Adds a ucode image to the list of managed ucode images managed. */
+static int lsfm_add_ucode_img(struct gk20a *g, struct ls_flcn_mgr_v0 *plsfm,
+	struct flcn_ucode_img_v0 *ucode_image, u32 falcon_id)
+{
+
+	struct lsfm_managed_ucode_img_v0 *pnode;
+
+	pnode = nvgpu_kzalloc(g, sizeof(struct lsfm_managed_ucode_img_v0));
+	if (pnode == NULL) {
+		return -ENOMEM;
+	}
+
+	/* Keep a copy of the ucode image info locally */
+	nvgpu_memcpy((u8 *)&pnode->ucode_img, (u8 *)ucode_image,
+		sizeof(struct flcn_ucode_img_v0));
+
+	/* Fill in static WPR header info*/
+	pnode->wpr_header.falcon_id = falcon_id;
+	pnode->wpr_header.bootstrap_owner = g->acr->bootstrap_owner;
+	pnode->wpr_header.status = LSF_IMAGE_STATUS_COPY;
+
+	pnode->wpr_header.lazy_bootstrap =
+		(u32)g->acr->lsf[falcon_id].is_lazy_bootstrap;
+
+	/* Fill in static LSB header info elsewhere */
+	lsfm_fill_static_lsb_hdr_info(g, falcon_id, pnode);
+	pnode->next = plsfm->ucode_img_list;
+	plsfm->ucode_img_list = pnode;
+	return 0;
+}
+
+/* Discover all managed falcon ucode images */
+static int lsfm_discover_ucode_images(struct gk20a *g,
+	struct ls_flcn_mgr_v0 *plsfm)
+{
+	struct flcn_ucode_img_v0 ucode_img;
+	struct nvgpu_acr *acr = g->acr;
+	u32 falcon_id;
+	u32 i;
+	int err = 0;
+
+	/*
+	 * Enumerate all constructed falcon objects, as we need the ucode
+	 * image info and total falcon count
+	 */
+	for (i = 0U; i < FALCON_ID_END; i++) {
+		if (nvgpu_test_bit(i, (void *)&acr->lsf_enable_mask) &&
+			acr->lsf[i].get_lsf_ucode_details != NULL) {
+
+			(void) memset(&ucode_img, 0, sizeof(ucode_img));
+
+			if (acr->lsf[i].get_lsf_ucode_details(g,
+				(void *)&ucode_img) != 0) {
+				nvgpu_err(g, "LS falcon-%d ucode get failed", i);
+				goto exit;
+			}
+
+			if (ucode_img.lsf_desc != NULL) {
+				/*
+				 * falon_id is formed by grabbing the static
+				 * base falonId from the image and adding the
+				 * engine-designated falcon instance.
+				 */
+				falcon_id = ucode_img.lsf_desc->falcon_id;
+
+				err = lsfm_add_ucode_img(g, plsfm, &ucode_img,
+					falcon_id);
+				if (err != 0) {
+					nvgpu_err(g, " Failed to add falcon-%d to LSFM ",
+						falcon_id);
+					goto exit;
+				}
+
+				plsfm->managed_flcn_cnt++;
+			}
+		}
+	}
+
+exit:
+	return err;
+}
+
+/* Generate WPR requirements for ACR allocation request */
+static int lsf_gen_wpr_requirements(struct gk20a *g, struct ls_flcn_mgr_v0 *plsfm)
+{
+	struct lsfm_managed_ucode_img_v0 *pnode = plsfm->ucode_img_list;
+	u32 wpr_offset;
+
+	/*
+	 * Start with an array of WPR headers at the base of the WPR.
+	 * The expectation here is that the secure falcon will do a single DMA
+	 * read of this array and cache it internally so it's OK to pack these.
+	 * Also, we add 1 to the falcon count to indicate the end of the array.
+	 */
+	wpr_offset = U32(sizeof(struct lsf_wpr_header_v0)) *
+		(U32(plsfm->managed_flcn_cnt) + U32(1));
+
+	/*
+	 * Walk the managed falcons, accounting for the LSB structs
+	 * as well as the ucode images.
+	 */
+	while (pnode != NULL) {
+		/* Align, save off, and include an LSB header size */
+		wpr_offset = NVGPU_ALIGN(wpr_offset, LSF_LSB_HEADER_ALIGNMENT);
+		pnode->wpr_header.lsb_offset = wpr_offset;
+		wpr_offset += (u32)sizeof(struct lsf_lsb_header_v0);
+
+		/*
+		 * Align, save off, and include the original (static)
+		 * ucode image size
+		 */
+		wpr_offset = NVGPU_ALIGN(wpr_offset,
+			LSF_UCODE_DATA_ALIGNMENT);
+		pnode->lsb_header.ucode_off = wpr_offset;
+		wpr_offset += pnode->ucode_img.data_size;
+
+		/*
+		 * For falcons that use a boot loader (BL), we append a loader
+		 * desc structure on the end of the ucode image and consider this
+		 * the boot loader data. The host will then copy the loader desc
+		 * args to this space within the WPR region (before locking down)
+		 * and the HS bin will then copy them to DMEM 0 for the loader.
+		 */
+		/*
+		 * Track the size for LSB details filled in later
+		 * Note that at this point we don't know what kind of
+		 * boot loader desc, so we just take the size of the
+		 * generic one, which is the largest it will will ever be.
+		 */
+		/* Align (size bloat) and save off generic descriptor size */
+		pnode->lsb_header.bl_data_size = NVGPU_ALIGN(
+			(u32)sizeof(pnode->bl_gen_desc),
+			LSF_BL_DATA_SIZE_ALIGNMENT);
+
+		/* Align, save off, and include the additional BL data */
+		wpr_offset = NVGPU_ALIGN(wpr_offset,
+			LSF_BL_DATA_ALIGNMENT);
+		pnode->lsb_header.bl_data_off = wpr_offset;
+		wpr_offset += pnode->lsb_header.bl_data_size;
+
+		/* Finally, update ucode surface size to include updates */
+		pnode->full_ucode_size = wpr_offset -
+			pnode->lsb_header.ucode_off;
+		if (pnode->wpr_header.falcon_id != FALCON_ID_PMU) {
+			pnode->lsb_header.app_code_off =
+				pnode->lsb_header.bl_code_size;
+			pnode->lsb_header.app_code_size =
+				pnode->lsb_header.ucode_size -
+				pnode->lsb_header.bl_code_size;
+			pnode->lsb_header.app_data_off =
+				pnode->lsb_header.ucode_size;
+			pnode->lsb_header.app_data_size =
+				pnode->lsb_header.data_size;
+		}
+		pnode = pnode->next;
+	}
+	plsfm->wpr_size = wpr_offset;
+	return 0;
+}
+
+/* Initialize WPR contents */
+static int gm20b_pmu_populate_loader_cfg(struct gk20a *g,
+	void *lsfm, u32 *p_bl_gen_desc_size)
+{
+	struct wpr_carveout_info wpr_inf;
+	struct lsfm_managed_ucode_img_v0 *p_lsfm =
+			(struct lsfm_managed_ucode_img_v0 *)lsfm;
+	struct flcn_ucode_img_v0 *p_img = &(p_lsfm->ucode_img);
+	struct loader_config *ldr_cfg = &(p_lsfm->bl_gen_desc.loader_cfg);
+	u64 addr_base;
+	struct pmu_ucode_desc *desc;
+	u64 tmp;
+	u32 addr_code, addr_data;
+
+	if (p_img->desc == NULL) {
+		/*
+		 * This means its a header based ucode,
+		 * and so we do not fill BL gen desc structure
+		 */
+		return -EINVAL;
+	}
+	desc = p_img->desc;
+	/*
+	 * Calculate physical and virtual addresses for various portions of
+	 * the PMU ucode image
+	 * Calculate the 32-bit addresses for the application code, application
+	 * data, and bootloader code. These values are all based on IM_BASE.
+	 * The 32-bit addresses will be the upper 32-bits of the virtual or
+	 * physical addresses of each respective segment.
+	 */
+	addr_base = p_lsfm->lsb_header.ucode_off;
+	g->acr->get_wpr_info(g, &wpr_inf);
+	addr_base += wpr_inf.wpr_base;
+	nvgpu_acr_dbg(g, "pmu loader cfg u32 addrbase %x\n", (u32)addr_base);
+	/*From linux*/
+	tmp = (addr_base +
+			desc->app_start_offset +
+			desc->app_resident_code_offset) >> 8;
+	nvgpu_assert(tmp <= U32_MAX);
+	addr_code = u64_lo32(tmp);
+	nvgpu_acr_dbg(g, "app start %d app res code off %d\n",
+		desc->app_start_offset, desc->app_resident_code_offset);
+	tmp = (addr_base +
+			desc->app_start_offset +
+			desc->app_resident_data_offset) >> 8;
+	nvgpu_assert(tmp <= U32_MAX);
+	addr_data = u64_lo32(tmp);
+	nvgpu_acr_dbg(g, "app res data offset%d\n",
+		desc->app_resident_data_offset);
+	nvgpu_acr_dbg(g, "bl start off %d\n", desc->bootloader_start_offset);
+
+	/* Populate the loader_config state*/
+	ldr_cfg->dma_idx = g->acr->lsf[FALCON_ID_PMU].falcon_dma_idx;
+	ldr_cfg->code_dma_base = addr_code;
+	ldr_cfg->code_dma_base1 = 0x0;
+	ldr_cfg->code_size_total = desc->app_size;
+	ldr_cfg->code_size_to_load = desc->app_resident_code_size;
+	ldr_cfg->code_entry_point = desc->app_imem_entry;
+	ldr_cfg->data_dma_base = addr_data;
+	ldr_cfg->data_dma_base1 = 0;
+	ldr_cfg->data_size = desc->app_resident_data_size;
+	ldr_cfg->overlay_dma_base = addr_code;
+	ldr_cfg->overlay_dma_base1 = 0x0;
+
+	/* Update the argc/argv members*/
+	ldr_cfg->argc = 1;
+#ifdef CONFIG_NVGPU_LS_PMU
+	nvgpu_pmu_fw_get_cmd_line_args_offset(g, &ldr_cfg->argv);
+#endif
+	*p_bl_gen_desc_size = (u32)sizeof(struct loader_config);
+	return 0;
+}
+
+static int gm20b_flcn_populate_bl_dmem_desc(struct gk20a *g,
+	void *lsfm, u32 *p_bl_gen_desc_size, u32 falconid)
+{
+	struct wpr_carveout_info wpr_inf;
+	struct lsfm_managed_ucode_img_v0 *p_lsfm =
+			(struct lsfm_managed_ucode_img_v0 *)lsfm;
+	struct flcn_ucode_img_v0 *p_img = &(p_lsfm->ucode_img);
+	struct flcn_bl_dmem_desc_v0 *ldr_cfg =
+			&(p_lsfm->bl_gen_desc.bl_dmem_desc);
+	u64 addr_base;
+	struct pmu_ucode_desc *desc;
+	u32 addr_code, addr_data;
+	u64 tmp;
+
+	if (p_img->desc == NULL) {
+		/*
+		 * This means its a header based ucode,
+		 * and so we do not fill BL gen desc structure
+		 */
+		return -EINVAL;
+	}
+	desc = p_img->desc;
+
+	/*
+	 * Calculate physical and virtual addresses for various portions of
+	 * the PMU ucode image
+	 * Calculate the 32-bit addresses for the application code, application
+	 * data, and bootloader code. These values are all based on IM_BASE.
+	 * The 32-bit addresses will be the upper 32-bits of the virtual or
+	 * physical addresses of each respective segment.
+	 */
+	addr_base = p_lsfm->lsb_header.ucode_off;
+	g->acr->get_wpr_info(g, &wpr_inf);
+	addr_base += wpr_inf.wpr_base;
+
+	nvgpu_acr_dbg(g, "gen loader cfg %x u32 addrbase %x ID\n", (u32)addr_base,
+			p_lsfm->wpr_header.falcon_id);
+	tmp = (addr_base +
+			desc->app_start_offset +
+			desc->app_resident_code_offset) >> 8;
+	nvgpu_assert(tmp <= U32_MAX);
+	addr_code = u64_lo32(tmp);
+	tmp = (addr_base +
+			desc->app_start_offset +
+			desc->app_resident_data_offset) >> 8;
+	nvgpu_assert(tmp <= U32_MAX);
+	addr_data = u64_lo32(tmp);
+
+	nvgpu_acr_dbg(g, "gen cfg %x u32 addrcode %x & data %x load offset %xID\n",
+		(u32)addr_code, (u32)addr_data, desc->bootloader_start_offset,
+		p_lsfm->wpr_header.falcon_id);
+
+	/* Populate the LOADER_CONFIG state */
+	(void) memset((void *) ldr_cfg, 0, sizeof(struct flcn_bl_dmem_desc_v0));
+	ldr_cfg->ctx_dma = g->acr->lsf[falconid].falcon_dma_idx;
+	ldr_cfg->code_dma_base = addr_code;
+	ldr_cfg->non_sec_code_size = desc->app_resident_code_size;
+	ldr_cfg->data_dma_base = addr_data;
+	ldr_cfg->data_size = desc->app_resident_data_size;
+	ldr_cfg->code_entry_point = desc->app_imem_entry;
+	*p_bl_gen_desc_size = (u32)sizeof(struct flcn_bl_dmem_desc_v0);
+	return 0;
+}
+
+/* Populate falcon boot loader generic desc.*/
+static int lsfm_fill_flcn_bl_gen_desc(struct gk20a *g,
+		struct lsfm_managed_ucode_img_v0 *pnode)
+{
+	int err = -ENOENT;
+
+	if (pnode->wpr_header.falcon_id != FALCON_ID_PMU) {
+		nvgpu_acr_dbg(g, "non pmu. write flcn bl gen desc\n");
+		err = gm20b_flcn_populate_bl_dmem_desc(g,
+				pnode, &pnode->bl_gen_desc_size,
+				pnode->wpr_header.falcon_id);
+		if (err != 0) {
+			nvgpu_err(g, "flcn_populate_bl_dmem_desc failed=%d",
+				err);
+		}
+		return err;
+	}
+
+	if (pnode->wpr_header.falcon_id == FALCON_ID_PMU) {
+		nvgpu_acr_dbg(g, "pmu write flcn bl gen desc\n");
+		err = gm20b_pmu_populate_loader_cfg(g, pnode,
+				&pnode->bl_gen_desc_size);
+		if (err != 0) {
+			nvgpu_err(g, "pmu_populate_loader_cfg failed=%d",
+				err);
+		}
+		return err;
+	}
+
+	/* Failed to find the falcon requested. */
+	return err;
+}
+
+static int lsfm_init_wpr_contents(struct gk20a *g, struct ls_flcn_mgr_v0 *plsfm,
+	struct nvgpu_mem *ucode)
+{
+	struct lsfm_managed_ucode_img_v0 *pnode = plsfm->ucode_img_list;
+	struct lsf_wpr_header_v0 last_wpr_hdr;
+	u32 i;
+	int err = 0;
+
+	/* The WPR array is at the base of the WPR */
+	pnode = plsfm->ucode_img_list;
+	(void) memset(&last_wpr_hdr, 0, sizeof(struct lsf_wpr_header_v0));
+	i = 0;
+
+	/*
+	 * Walk the managed falcons, flush WPR and LSB headers to FB.
+	 * flush any bl args to the storage area relative to the
+	 * ucode image (appended on the end as a DMEM area).
+	 */
+	while (pnode != NULL) {
+		/* Flush WPR header to memory*/
+		nvgpu_mem_wr_n(g, ucode, i * (u32)sizeof(pnode->wpr_header),
+				&pnode->wpr_header,
+				(u32)sizeof(pnode->wpr_header));
+
+		nvgpu_acr_dbg(g, "wpr header");
+		nvgpu_acr_dbg(g, "falconid :%d",
+				pnode->wpr_header.falcon_id);
+		nvgpu_acr_dbg(g, "lsb_offset :%x",
+				pnode->wpr_header.lsb_offset);
+		nvgpu_acr_dbg(g, "bootstrap_owner :%d",
+			pnode->wpr_header.bootstrap_owner);
+		nvgpu_acr_dbg(g, "lazy_bootstrap :%d",
+				pnode->wpr_header.lazy_bootstrap);
+		nvgpu_acr_dbg(g, "status :%d",
+				pnode->wpr_header.status);
+
+		/*Flush LSB header to memory*/
+		nvgpu_mem_wr_n(g, ucode, pnode->wpr_header.lsb_offset,
+				&pnode->lsb_header,
+				(u32)sizeof(pnode->lsb_header));
+
+		nvgpu_acr_dbg(g, "lsb header");
+		nvgpu_acr_dbg(g, "ucode_off :%x",
+				pnode->lsb_header.ucode_off);
+		nvgpu_acr_dbg(g, "ucode_size :%x",
+				pnode->lsb_header.ucode_size);
+		nvgpu_acr_dbg(g, "data_size :%x",
+				pnode->lsb_header.data_size);
+		nvgpu_acr_dbg(g, "bl_code_size :%x",
+				pnode->lsb_header.bl_code_size);
+		nvgpu_acr_dbg(g, "bl_imem_off :%x",
+				pnode->lsb_header.bl_imem_off);
+		nvgpu_acr_dbg(g, "bl_data_off :%x",
+				pnode->lsb_header.bl_data_off);
+		nvgpu_acr_dbg(g, "bl_data_size :%x",
+				pnode->lsb_header.bl_data_size);
+		nvgpu_acr_dbg(g, "app_code_off :%x",
+				pnode->lsb_header.app_code_off);
+		nvgpu_acr_dbg(g, "app_code_size :%x",
+				pnode->lsb_header.app_code_size);
+		nvgpu_acr_dbg(g, "app_data_off :%x",
+				pnode->lsb_header.app_data_off);
+		nvgpu_acr_dbg(g, "app_data_size :%x",
+				pnode->lsb_header.app_data_size);
+		nvgpu_acr_dbg(g, "flags :%x",
+				pnode->lsb_header.flags);
+
+		/* this falcon has a boot loader and related args, flush them */
+		/* Populate gen bl and flush to memory */
+		err = lsfm_fill_flcn_bl_gen_desc(g, pnode);
+		if (err != 0) {
+			nvgpu_err(g, "bl_gen_desc failed err=%d", err);
+			return err;
+		}
+		nvgpu_mem_wr_n(g, ucode,
+				pnode->lsb_header.bl_data_off,
+				&pnode->bl_gen_desc,
+				pnode->bl_gen_desc_size);
+
+		/* Copying of ucode */
+		nvgpu_mem_wr_n(g, ucode, pnode->lsb_header.ucode_off,
+				pnode->ucode_img.data,
+				pnode->ucode_img.data_size);
+		pnode = pnode->next;
+		i++;
+	}
+
+	/* Tag the terminator WPR header with an invalid falcon ID. */
+	last_wpr_hdr.falcon_id = FALCON_ID_INVALID;
+	nvgpu_mem_wr_n(g, ucode,
+			(u32)plsfm->managed_flcn_cnt *
+				(u32)sizeof(struct lsf_wpr_header_v0),
+			&last_wpr_hdr,
+			(u32)sizeof(struct lsf_wpr_header_v0));
+	return err;
+}
+
+/* Free any ucode image structure resources. */
+static void lsfm_free_ucode_img_res(struct gk20a *g,
+	struct flcn_ucode_img_v0 *p_img)
+{
+	if (p_img->lsf_desc != NULL) {
+		nvgpu_kfree(g, p_img->lsf_desc);
+		p_img->lsf_desc = NULL;
+	}
+}
+
+/* Free any ucode image structure resources. */
+static void lsfm_free_nonpmu_ucode_img_res(struct gk20a *g,
+	struct flcn_ucode_img_v0 *p_img)
+{
+	if (p_img->lsf_desc != NULL) {
+		nvgpu_kfree(g, p_img->lsf_desc);
+		p_img->lsf_desc = NULL;
+	}
+	if (p_img->desc != NULL) {
+		nvgpu_kfree(g, p_img->desc);
+		p_img->desc = NULL;
+	}
+}
+
+static void free_acr_resources(struct gk20a *g, struct ls_flcn_mgr_v0 *plsfm)
+{
+	u32 cnt = plsfm->managed_flcn_cnt;
+	struct lsfm_managed_ucode_img_v0 *mg_ucode_img;
+	while (cnt != 0U) {
+		mg_ucode_img = plsfm->ucode_img_list;
+		if (mg_ucode_img->ucode_img.lsf_desc->falcon_id ==
+				FALCON_ID_PMU) {
+			lsfm_free_ucode_img_res(g, &mg_ucode_img->ucode_img);
+		} else {
+			lsfm_free_nonpmu_ucode_img_res(g,
+				&mg_ucode_img->ucode_img);
+		}
+		plsfm->ucode_img_list = mg_ucode_img->next;
+		nvgpu_kfree(g, mg_ucode_img);
+		cnt--;
+	}
+}
+
+int nvgpu_acr_prepare_ucode_blob_v0(struct gk20a *g)
+{
+	int err = 0;
+	struct ls_flcn_mgr_v0 lsfm_l, *plsfm;
+	struct wpr_carveout_info wpr_inf;
+	struct nvgpu_gr_falcon *gr_falcon = nvgpu_gr_get_falcon_ptr(g);
+
+	if (g->acr->ucode_blob.cpu_va != NULL) {
+		/* Recovery case, we do not need to form non WPR blob */
+		return err;
+	}
+	plsfm = &lsfm_l;
+	(void) memset((void *)plsfm, 0, sizeof(struct ls_flcn_mgr_v0));
+	nvgpu_acr_dbg(g, "fetching GMMU regs\n");
+	err = g->ops.fb.vpr_info_fetch(g);
+	if (err != 0) {
+		nvgpu_err(g, "fb.vpr_info_fetch failed err=%d", err);
+		return err;
+	}
+
+	err = nvgpu_gr_falcon_init_ctxsw_ucode(g, gr_falcon);
+	if (err != 0) {
+		nvgpu_err(g, "gr_falcon_init_ctxsw_ucode failed err=%d", err);
+		return err;
+	}
+
+	g->acr->get_wpr_info(g, &wpr_inf);
+	nvgpu_acr_dbg(g, "wpr carveout base:%llx\n", wpr_inf.wpr_base);
+	nvgpu_acr_dbg(g, "wpr carveout size :%llx\n", wpr_inf.size);
+
+	/* Discover all managed falcons*/
+	err = lsfm_discover_ucode_images(g, plsfm);
+	nvgpu_acr_dbg(g, " Managed Falcon cnt %d\n", plsfm->managed_flcn_cnt);
+	if (err != 0) {
+		goto exit_err;
+	}
+
+	if ((plsfm->managed_flcn_cnt != 0U) &&
+		(g->acr->ucode_blob.cpu_va == NULL)) {
+		/* Generate WPR requirements */
+		err = lsf_gen_wpr_requirements(g, plsfm);
+		if (err != 0) {
+			goto exit_err;
+		}
+
+		/* Alloc memory to hold ucode blob contents */
+		err = g->acr->alloc_blob_space(g, plsfm->wpr_size
+				, &g->acr->ucode_blob);
+		if (err != 0) {
+			goto exit_err;
+		}
+
+		nvgpu_acr_dbg(g, "managed LS falcon %d, WPR size %d bytes.\n",
+			plsfm->managed_flcn_cnt, plsfm->wpr_size);
+		err = lsfm_init_wpr_contents(g, plsfm, &g->acr->ucode_blob);
+		if (err != 0) {
+			nvgpu_kfree(g, &g->acr->ucode_blob);
+			goto free_acr;
+		}
+	} else {
+		nvgpu_acr_dbg(g, "LSFM is managing no falcons.\n");
+	}
+	nvgpu_acr_dbg(g, "prepare ucode blob return 0\n");
+
+free_acr:
+	free_acr_resources(g, plsfm);
+exit_err:
+	return err;
+}

drivers/gpu/nvgpu/common/acr/acr_blob_construct_v0.h

@@ -0,0 +1,207 @@
+/*
+ * Copyright (c) 2019-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef ACR_BLOB_CONSTRUCT_V0_H
+#define ACR_BLOB_CONSTRUCT_V0_H
+
+#include <nvgpu/falcon.h>
+#include <nvgpu/flcnif_cmn.h>
+
+/*
+ * Light Secure WPR Content Alignments
+ */
+#define LSF_WPR_HEADER_ALIGNMENT        (256U)
+#define LSF_SUB_WPR_HEADER_ALIGNMENT    (256U)
+#define LSF_LSB_HEADER_ALIGNMENT        (256U)
+#define LSF_BL_DATA_ALIGNMENT           (256U)
+#define LSF_BL_DATA_SIZE_ALIGNMENT      (256U)
+#define LSF_BL_CODE_SIZE_ALIGNMENT      (256U)
+#define LSF_DATA_SIZE_ALIGNMENT         (256U)
+#define LSF_CODE_SIZE_ALIGNMENT         (256U)
+
+#define LSF_UCODE_DATA_ALIGNMENT 4096U
+
+
+/* Defined for 1MB alignment */
+#define SHIFT_1MB	(20U)
+#define SHIFT_4KB	(12U)
+
+/*Light Secure Bootstrap header related defines*/
+#define NV_FLCN_ACR_LSF_FLAG_LOAD_CODE_AT_0_FALSE       0U
+#define NV_FLCN_ACR_LSF_FLAG_LOAD_CODE_AT_0_TRUE        BIT32(0)
+#define NV_FLCN_ACR_LSF_FLAG_DMACTL_REQ_CTX_FALSE       0U
+#define NV_FLCN_ACR_LSF_FLAG_DMACTL_REQ_CTX_TRUE        BIT32(2)
+#define NV_FLCN_ACR_LSF_FLAG_FORCE_PRIV_LOAD_TRUE       BIT32(3)
+#define NV_FLCN_ACR_LSF_FLAG_FORCE_PRIV_LOAD_FALSE      (0U)
+
+/*
+ * Image Status Defines
+ */
+#define LSF_IMAGE_STATUS_NONE                           (0U)
+#define LSF_IMAGE_STATUS_COPY                           (1U)
+#define LSF_IMAGE_STATUS_VALIDATION_CODE_FAILED         (2U)
+#define LSF_IMAGE_STATUS_VALIDATION_DATA_FAILED         (3U)
+#define LSF_IMAGE_STATUS_VALIDATION_DONE                (4U)
+#define LSF_IMAGE_STATUS_VALIDATION_SKIPPED             (5U)
+#define LSF_IMAGE_STATUS_BOOTSTRAP_READY                (6U)
+
+/*
+ * Light Secure WPR Header
+ * Defines state allowing Light Secure Falcon bootstrapping.
+ */
+struct lsf_wpr_header_v0 {
+	u32 falcon_id;
+	u32 lsb_offset;
+	u32 bootstrap_owner;
+	u32 lazy_bootstrap;
+	u32 status;
+};
+
+/*
+ * Light Secure Falcon Ucode Description Defines
+ * This structure is prelim and may change as the ucode signing flow evolves.
+ */
+struct lsf_ucode_desc_v0 {
+	u8  prd_keys[2][16];
+	u8  dbg_keys[2][16];
+	u32 b_prd_present;
+	u32 b_dbg_present;
+	u32 falcon_id;
+};
+
+/*
+ * Light Secure Bootstrap Header
+ * Defines state allowing Light Secure Falcon bootstrapping.
+ */
+struct lsf_lsb_header_v0 {
+	struct lsf_ucode_desc_v0 signature;
+	u32 ucode_off;
+	u32 ucode_size;
+	u32 data_size;
+	u32 bl_code_size;
+	u32 bl_imem_off;
+	u32 bl_data_off;
+	u32 bl_data_size;
+	u32 app_code_off;
+	u32 app_code_size;
+	u32 app_data_off;
+	u32 app_data_size;
+	u32 flags;
+};
+
+/*
+ * Union of all supported structures used by bootloaders.
+ */
+/* Falcon BL interfaces */
+/*
+ * Structure used by the boot-loader to load the rest of the code. This has
+ * to be filled by NVGPU and copied into DMEM at offset provided in the
+ * hsflcn_bl_desc.bl_desc_dmem_load_off.
+ */
+struct flcn_bl_dmem_desc_v0 {
+	u32    reserved[4];        /*Should be the first element..*/
+	u32    signature[4];        /*Should be the first element..*/
+	u32    ctx_dma;
+	u32    code_dma_base;
+	u32    non_sec_code_off;
+	u32    non_sec_code_size;
+	u32    sec_code_off;
+	u32    sec_code_size;
+	u32    code_entry_point;
+	u32    data_dma_base;
+	u32    data_size;
+	u32    code_dma_base1;
+	u32    data_dma_base1;
+};
+
+/*
+ * Legacy structure used by the current PMU bootloader.
+ */
+struct loader_config {
+	u32 dma_idx;
+	u32 code_dma_base;     /* upper 32-bits of 40-bit dma address */
+	u32 code_size_total;
+	u32 code_size_to_load;
+	u32 code_entry_point;
+	u32 data_dma_base;     /* upper 32-bits of 40-bit dma address */
+	u32 data_size;         /* initialized data of the application  */
+	u32 overlay_dma_base;  /* upper 32-bits of the 40-bit dma address */
+	u32 argc;
+	u32 argv;
+	u16 code_dma_base1;    /* upper 7 bits of 47-bit dma address */
+	u16 data_dma_base1;    /* upper 7 bits of 47-bit dma address */
+	u16 overlay_dma_base1; /* upper 7 bits of the 47-bit dma address */
+};
+
+union flcn_bl_generic_desc {
+	struct flcn_bl_dmem_desc_v0 bl_dmem_desc;
+	struct loader_config loader_cfg;
+};
+
+struct flcn_ucode_img_v0 {
+	u32  *data;
+	struct pmu_ucode_desc *desc; /* only some falcons have descriptor */
+	u32  data_size;
+	/* NULL if not a light secure falcon. */
+	struct lsf_ucode_desc_v0 *lsf_desc;
+	/* True if there a resources to freed by the client. */
+};
+
+/*
+ * LSFM Managed Ucode Image
+ * next             : Next image the list, NULL if last.
+ * wpr_header       : WPR header for this ucode image
+ * lsb_header       : LSB header for this ucode image
+ * bl_gen_desc      : Bootloader generic desc structure for this ucode image
+ * bl_gen_desc_size : Sizeof bootloader desc structure for this ucode image
+ * full_ucode_size  : Surface size required for final ucode image
+ * ucode_img        : Ucode image info
+ */
+struct lsfm_managed_ucode_img_v0 {
+	struct lsfm_managed_ucode_img_v0 *next;
+	struct lsf_wpr_header_v0 wpr_header;
+	struct lsf_lsb_header_v0 lsb_header;
+	union flcn_bl_generic_desc bl_gen_desc;
+	u32 bl_gen_desc_size;
+	u32 full_ucode_size;
+	struct flcn_ucode_img_v0 ucode_img;
+};
+
+/*
+ * Defines the structure used to contain all generic information related to
+ * the LSFM.
+ *
+ * Contains the Light Secure Falcon Manager (LSFM) feature related data.
+ */
+struct ls_flcn_mgr_v0 {
+	u16 managed_flcn_cnt;
+	u32 wpr_size;
+	struct lsfm_managed_ucode_img_v0 *ucode_img_list;
+};
+
+int nvgpu_acr_lsf_pmu_ucode_details_v0(struct gk20a *g, void *lsf_ucode_img);
+int nvgpu_acr_lsf_fecs_ucode_details_v0(struct gk20a *g, void *lsf_ucode_img);
+int nvgpu_acr_lsf_gpccs_ucode_details_v0(struct gk20a *g, void *lsf_ucode_img);
+
+int nvgpu_acr_prepare_ucode_blob_v0(struct gk20a *g);
+
+#endif /* ACR_BLOB_CONSTRUCT_V0_H */

drivers/gpu/nvgpu/common/acr/acr_bootstrap.c

@@ -0,0 +1,254 @@
+/*
+ * Copyright (c) 2019-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/types.h>
+#include <nvgpu/dma.h>
+#include <nvgpu/timers.h>
+#include <nvgpu/nvgpu_mem.h>
+#include <nvgpu/firmware.h>
+#include <nvgpu/pmu.h>
+#include <nvgpu/falcon.h>
+#include <nvgpu/gk20a.h>
+#include <nvgpu/acr.h>
+#include <nvgpu/bug.h>
+#include <nvgpu/soc.h>
+
+#include "acr_bootstrap.h"
+#include "acr_priv.h"
+
+int nvgpu_acr_wait_for_completion(struct gk20a *g, struct hs_acr *acr_desc,
+	u32 timeout)
+{
+	u32 flcn_id;
+#ifdef CONFIG_NVGPU_FALCON_NON_FUSA
+	u32 sctl, cpuctl;
+#endif
+	int completion = 0;
+	u32 data = 0;
+	u32 bar0_status = 0;
+	u32 error_type;
+
+	nvgpu_log_fn(g, " ");
+
+	flcn_id = nvgpu_falcon_get_id(acr_desc->acr_flcn);
+
+	completion = nvgpu_falcon_wait_for_halt(acr_desc->acr_flcn, timeout);
+	if (completion != 0) {
+		nvgpu_err(g, "flcn-%d: HS ucode boot timed out, limit: %d ms",
+				flcn_id, timeout);
+		error_type = ACR_BOOT_TIMEDOUT;
+		goto exit;
+	}
+
+	if (acr_desc->acr_engine_bus_err_status != NULL) {
+		completion = acr_desc->acr_engine_bus_err_status(g,
+			&bar0_status, &error_type);
+		if (completion != 0) {
+			nvgpu_err(g, "flcn-%d: ACR engine bus error", flcn_id);
+			goto exit;
+		}
+	}
+
+	data = nvgpu_falcon_mailbox_read(acr_desc->acr_flcn, FALCON_MAILBOX_0);
+	if (data != 0U) {
+		nvgpu_err(g, "flcn-%d: HS ucode boot failed, err %x", flcn_id,
+				data);
+		nvgpu_err(g, "flcn-%d: Mailbox-1 : 0x%x", flcn_id,
+				nvgpu_falcon_mailbox_read(acr_desc->acr_flcn,
+				FALCON_MAILBOX_1));
+		completion = -EAGAIN;
+		error_type = ACR_BOOT_FAILED;
+		goto exit;
+	}
+
+	/*
+	 * When engine-falcon is used for ACR bootstrap, validate the integrity
+	 * of falcon IMEM and DMEM.
+	 */
+	if (acr_desc->acr_validate_mem_integrity != NULL) {
+		if (!acr_desc->acr_validate_mem_integrity(g)) {
+			nvgpu_err(g, "flcn-%d: memcheck failed", flcn_id);
+			completion = -EAGAIN;
+			error_type = ACR_BOOT_FAILED;
+		}
+	}
+
+exit:
+
+#ifdef CONFIG_NVGPU_FALCON_NON_FUSA
+	nvgpu_falcon_get_ctls(acr_desc->acr_flcn, &sctl, &cpuctl);
+
+	nvgpu_acr_dbg(g, "flcn-%d: sctl reg %x cpuctl reg %x",
+			flcn_id, sctl, cpuctl);
+#endif
+
+	if (completion != 0) {
+#ifdef CONFIG_NVGPU_FALCON_DEBUG
+		nvgpu_falcon_dump_stats(acr_desc->acr_flcn);
+#endif
+		if (acr_desc->report_acr_engine_bus_err_status != NULL) {
+			acr_desc->report_acr_engine_bus_err_status(g,
+				bar0_status, error_type);
+		}
+	}
+
+	return completion;
+}
+
+/*
+ * Patch signatures into ucode image
+ */
+static void acr_ucode_patch_sig(struct gk20a *g,
+	unsigned int *p_img, unsigned int *p_prod_sig,
+	unsigned int *p_dbg_sig, unsigned int *p_patch_loc,
+	unsigned int *p_patch_ind, u32 sig_size)
+{
+#if defined(CONFIG_NVGPU_NEXT)
+	struct nvgpu_acr *acr = g->acr;
+#endif
+	unsigned int i, j, *p_sig;
+	const u32 dmem_word_size = 4U;
+	nvgpu_acr_dbg(g, " ");
+
+	if (!g->ops.pmu.is_debug_mode_enabled(g)) {
+		p_sig = p_prod_sig;
+		nvgpu_acr_dbg(g, "PRODUCTION MODE\n");
+	} else {
+		p_sig = p_dbg_sig;
+		nvgpu_info(g, "DEBUG MODE\n");
+	}
+
+#if defined(CONFIG_NVGPU_NEXT)
+	if (acr->get_versioned_sig != NULL) {
+		p_sig = acr->get_versioned_sig(g, acr, p_sig, &sig_size);
+	}
+#endif
+
+	/* Patching logic:*/
+	sig_size = sig_size / dmem_word_size;
+	for (i = 0U; i < (sizeof(*p_patch_loc) / dmem_word_size); i++) {
+		for (j = 0U; j < sig_size; j++) {
+			p_img[nvgpu_safe_add_u32(
+				(p_patch_loc[i] / dmem_word_size), j)] =
+				p_sig[nvgpu_safe_add_u32(
+					(p_patch_ind[i] * dmem_word_size), j)];
+		}
+	}
+}
+
+/*
+ * Loads ACR bin to SYSMEM/FB and bootstraps ACR with bootloader code
+ * start and end are addresses of ucode blob in non-WPR region
+ */
+int nvgpu_acr_bootstrap_hs_ucode(struct gk20a *g, struct nvgpu_acr *acr,
+	struct hs_acr *acr_desc)
+{
+	struct nvgpu_firmware *acr_fw = acr_desc->acr_fw;
+	struct bin_hdr *hs_bin_hdr = NULL;
+	struct acr_fw_header *fw_hdr = NULL;
+	u32 *ucode_header = NULL;
+	u32 *ucode = NULL;
+	u32 timeout = 0;
+	int err = 0;
+
+	nvgpu_acr_dbg(g, "ACR TYPE %x ", acr_desc->acr_type);
+
+	if (acr_fw != NULL) {
+		err = acr->patch_wpr_info_to_ucode(g, acr, acr_desc, true);
+		if (err != 0) {
+			nvgpu_err(g, "Falcon ucode patch wpr info failed");
+			return err;
+                }
+	} else {
+		/* Firmware is stored in soc specific path in FMODEL
+		 * Hence NVGPU_REQUEST_FIRMWARE_NO_WARN is used instead
+		 * of NVGPU_REQUEST_FIRMWARE_NO_SOC
+		 */
+#ifdef CONFIG_NVGPU_SIM
+		if (nvgpu_is_enabled(g, NVGPU_IS_FMODEL)) {
+			acr_fw = nvgpu_request_firmware(g,
+					acr_desc->acr_fw_name,
+					NVGPU_REQUEST_FIRMWARE_NO_WARN);
+		} else
+#endif
+		{
+			acr_fw = nvgpu_request_firmware(g,
+					acr_desc->acr_fw_name,
+					NVGPU_REQUEST_FIRMWARE_NO_SOC);
+		}
+		if (acr_fw == NULL) {
+			nvgpu_err(g, "%s ucode get fail for %s",
+				acr_desc->acr_fw_name, g->name);
+			return -ENOENT;
+		}
+
+		acr_desc->acr_fw = acr_fw;
+
+		err = acr->patch_wpr_info_to_ucode(g, acr, acr_desc, false);
+		if (err != 0) {
+			nvgpu_err(g, "Falcon ucode patch wpr info failed");
+			goto err_free_ucode;
+                }
+	}
+
+
+	hs_bin_hdr = (struct bin_hdr *)(void *)acr_fw->data;
+	fw_hdr = (struct acr_fw_header *)(void *)(acr_fw->data +
+			hs_bin_hdr->header_offset);
+	ucode_header = (u32 *)(void *)(acr_fw->data + fw_hdr->hdr_offset);
+	ucode = (u32 *)(void *)(acr_fw->data + hs_bin_hdr->data_offset);
+
+	/* Patch Ucode signatures */
+	acr_ucode_patch_sig(g, ucode,
+		(u32 *)(void *)(acr_fw->data + fw_hdr->sig_prod_offset),
+		(u32 *)(void *)(acr_fw->data + fw_hdr->sig_dbg_offset),
+		(u32 *)(void *)(acr_fw->data + fw_hdr->patch_loc),
+		(u32 *)(void *)(acr_fw->data + fw_hdr->patch_sig),
+		fw_hdr->sig_dbg_size);
+
+	err = nvgpu_falcon_hs_ucode_load_bootstrap(acr_desc->acr_flcn,
+			ucode, ucode_header);
+	if (err != 0) {
+		nvgpu_err(g, "HS ucode load & bootstrap failed");
+		goto err_free_ucode;
+	}
+
+	/* wait for complete & halt */
+	if (nvgpu_platform_is_silicon(g)) {
+		timeout = ACR_COMPLETION_TIMEOUT_SILICON_MS;
+	} else {
+		timeout = ACR_COMPLETION_TIMEOUT_NON_SILICON_MS;
+	}
+	err = nvgpu_acr_wait_for_completion(g, acr_desc, timeout);
+
+	if (err != 0) {
+		nvgpu_err(g, "HS ucode completion err %d", err);
+		goto err_free_ucode;
+	}
+
+	return 0;
+
+err_free_ucode:
+	nvgpu_release_firmware(g, acr_fw);
+	acr_desc->acr_fw = NULL;
+	return err;
+}

drivers/gpu/nvgpu/common/acr/acr_bootstrap.h

@@ -0,0 +1,139 @@
+/*
+ * Copyright (c) 2019-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef ACR_BOOTSTRAP_H
+#define ACR_BOOTSTRAP_H
+
+#include "nvgpu_acr_interface.h"
+#ifdef CONFIG_NVGPU_NEXT
+#include "common/acr/nvgpu_next_acr_bootstrap.h"
+#endif
+
+struct gk20a;
+struct nvgpu_acr;
+
+struct flcn_acr_region_prop_v0 {
+	u32 start_addr;
+	u32 end_addr;
+	u32 region_id;
+	u32 read_mask;
+	u32 write_mask;
+	u32 client_mask;
+};
+
+struct flcn_acr_regions_v0 {
+	u32 no_regions;
+	struct flcn_acr_region_prop_v0 region_props[NVGPU_FLCN_ACR_MAX_REGIONS];
+};
+
+struct flcn_acr_desc_v0 {
+	union {
+		u32 reserved_dmem[(LSF_BOOTSTRAP_OWNER_RESERVED_DMEM_SIZE/4)];
+		u32 signatures[4];
+	} ucode_reserved_space;
+	/*Always 1st*/
+	u32 wpr_region_id;
+	u32 wpr_offset;
+	u32 mmu_mem_range;
+	struct flcn_acr_regions_v0 regions;
+	u32 nonwpr_ucode_blob_size;
+	u64 nonwpr_ucode_blob_start;
+};
+
+struct bin_hdr {
+	/* 0x10de */
+	u32 bin_magic;
+	/* versioning of bin format */
+	u32 bin_ver;
+	/* Entire image size including this header */
+	u32 bin_size;
+	/*
+	 * Header offset of executable binary metadata,
+	 * start @ offset- 0x100 *
+	 */
+	u32 header_offset;
+	/*
+	 * Start of executable binary data, start @
+	 * offset- 0x200
+	 */
+	u32 data_offset;
+	/* Size of executable binary */
+	u32 data_size;
+};
+
+struct acr_fw_header {
+	u32 sig_dbg_offset;
+	u32 sig_dbg_size;
+	u32 sig_prod_offset;
+	u32 sig_prod_size;
+	u32 patch_loc;
+	u32 patch_sig;
+	u32 hdr_offset; /* This header points to acr_ucode_header_t210_load */
+	u32 hdr_size; /* Size of above header */
+};
+
+/* ACR Falcon descriptor's */
+struct hs_acr {
+#define ACR_DEFAULT		0U
+#define ACR_AHESASC_NON_FUSA	1U
+#define ACR_ASB_NON_FUSA	2U
+#define ACR_AHESASC_FUSA	3U
+#define ACR_ASB_FUSA		4U
+	u32 acr_type;
+
+	/* ACR ucode */
+	const char *acr_fw_name;
+	const char *acr_code_name;
+	const char *acr_data_name;
+	const char *acr_manifest_name;
+	struct nvgpu_firmware *code_fw;
+	struct nvgpu_firmware *data_fw;
+	struct nvgpu_firmware *manifest_fw;
+	struct nvgpu_firmware *acr_fw;
+
+	union{
+		struct flcn_acr_desc_v0 *acr_dmem_desc_v0;
+		struct flcn_acr_desc *acr_dmem_desc;
+	};
+
+#if defined(CONFIG_NVGPU_NEXT)
+	struct nvgpu_mem acr_falcon2_sysmem_desc;
+	struct flcn2_acr_desc acr_sysmem_desc;
+	struct nvgpu_mem ls_pmu_desc;
+#endif
+
+	/* Falcon used to execute ACR ucode */
+	struct nvgpu_falcon *acr_flcn;
+
+	void (*report_acr_engine_bus_err_status)(struct gk20a *g,
+		u32 bar0_status, u32 error_type);
+	int (*acr_engine_bus_err_status)(struct gk20a *g, u32 *bar0_status,
+		u32 *error_type);
+	bool (*acr_validate_mem_integrity)(struct gk20a *g);
+};
+
+int nvgpu_acr_wait_for_completion(struct gk20a *g, struct hs_acr *acr_desc,
+	u32 timeout);
+int nvgpu_acr_bootstrap_hs_ucode(struct gk20a *g, struct nvgpu_acr *acr,
+	struct hs_acr *acr_desc);
+
+#endif /* ACR_BOOTSTRAP_H */

drivers/gpu/nvgpu/common/acr/acr_priv.h

@@ -0,0 +1,161 @@
+/*
+ * Copyright (c) 2019-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef ACR_H
+#define ACR_H
+
+#include "acr_bootstrap.h"
+#ifdef CONFIG_NVGPU_ACR_LEGACY
+#include "acr_blob_construct_v0.h"
+#endif
+#include "acr_blob_construct.h"
+
+struct gk20a;
+struct nvgpu_acr;
+struct wpr_carveout_info;
+
+#define nvgpu_acr_dbg(g, fmt, args...) \
+	nvgpu_log(g, gpu_dbg_pmu, fmt, ##args)
+
+/*
+ * Falcon UCODE header index.
+ */
+#define FLCN_NL_UCODE_HDR_OS_CODE_OFF_IND              (0U)
+#define FLCN_NL_UCODE_HDR_OS_CODE_SIZE_IND             (1U)
+#define FLCN_NL_UCODE_HDR_OS_DATA_OFF_IND              (2U)
+#define FLCN_NL_UCODE_HDR_OS_DATA_SIZE_IND             (3U)
+#define FLCN_NL_UCODE_HDR_NUM_APPS_IND                 (4U)
+
+/*
+ * There are total N number of Apps with code and offset defined in UCODE header
+ * This macro provides the CODE and DATA offset and size of Ath application.
+ */
+#define FLCN_NL_UCODE_HDR_APP_CODE_START_IND           (5U)
+#define FLCN_NL_UCODE_HDR_APP_CODE_OFF_IND(N, A) \
+	(FLCN_NL_UCODE_HDR_APP_CODE_START_IND + ((A)*2U))
+#define FLCN_NL_UCODE_HDR_APP_CODE_SIZE_IND(N, A) \
+	(FLCN_NL_UCODE_HDR_APP_CODE_START_IND + ((A)*2U) + 1U)
+#define FLCN_NL_UCODE_HDR_APP_CODE_END_IND(N) \
+	(FLCN_NL_UCODE_HDR_APP_CODE_START_IND + ((N)*2U) - 1U)
+
+#define FLCN_NL_UCODE_HDR_APP_DATA_START_IND(N) \
+	(FLCN_NL_UCODE_HDR_APP_CODE_END_IND(N) + 1U)
+#define FLCN_NL_UCODE_HDR_APP_DATA_OFF_IND(N, A) \
+	(FLCN_NL_UCODE_HDR_APP_DATA_START_IND(N) + ((A)*2U))
+#define FLCN_NL_UCODE_HDR_APP_DATA_SIZE_IND(N, A) \
+	(FLCN_NL_UCODE_HDR_APP_DATA_START_IND(N) + ((A)*2U) + 1U)
+#define FLCN_NL_UCODE_HDR_APP_DATA_END_IND(N) \
+	(FLCN_NL_UCODE_HDR_APP_DATA_START_IND(N) + ((N)*2U) - 1U)
+
+#define FLCN_NL_UCODE_HDR_OS_OVL_OFF_IND(N) \
+	(FLCN_NL_UCODE_HDR_APP_DATA_END_IND(N) + 1U)
+#define FLCN_NL_UCODE_HDR_OS_OVL_SIZE_IND(N) \
+	(FLCN_NL_UCODE_HDR_APP_DATA_END_IND(N) + 2U)
+
+#define GM20B_HSBIN_ACR_PROD_UCODE "nv_acr_ucode_prod.bin"
+#define GM20B_HSBIN_ACR_DBG_UCODE "nv_acr_ucode_dbg.bin"
+#define HSBIN_ACR_BL_UCODE_IMAGE "pmu_bl.bin"
+#define HSBIN_ACR_PROD_UCODE "acr_ucode_prod.bin"
+#define HSBIN_ACR_DBG_UCODE "acr_ucode_dbg.bin"
+#define HSBIN_ACR_AHESASC_NON_FUSA_PROD_UCODE "acr_ahesasc_prod_ucode.bin"
+#define HSBIN_ACR_ASB_NON_FUSA_PROD_UCODE "acr_asb_prod_ucode.bin"
+#define HSBIN_ACR_AHESASC_NON_FUSA_DBG_UCODE "acr_ahesasc_dbg_ucode.bin"
+#define HSBIN_ACR_ASB_NON_FUSA_DBG_UCODE "acr_asb_dbg_ucode.bin"
+
+#define HSBIN_ACR_AHESASC_FUSA_PROD_UCODE "acr_ahesasc_fusa_prod_ucode.bin"
+#define HSBIN_ACR_ASB_FUSA_PROD_UCODE "acr_asb_fusa_prod_ucode.bin"
+#define HSBIN_ACR_AHESASC_FUSA_DBG_UCODE "acr_ahesasc_fusa_dbg_ucode.bin"
+#define HSBIN_ACR_ASB_FUSA_DBG_UCODE "acr_asb_fusa_dbg_ucode.bin"
+
+#define GM20B_FECS_UCODE_SIG "fecs_sig.bin"
+#define T18x_GPCCS_UCODE_SIG "gpccs_sig.bin"
+
+#define TU104_FECS_UCODE_SIG "tu104/fecs_sig.bin"
+#define TU104_GPCCS_UCODE_SIG "tu104/gpccs_sig.bin"
+
+#define LSF_SEC2_UCODE_IMAGE_BIN "sec2_ucode_image.bin"
+#define LSF_SEC2_UCODE_DESC_BIN "sec2_ucode_desc.bin"
+#define LSF_SEC2_UCODE_SIG_BIN "sec2_sig.bin"
+
+#define LSF_SEC2_UCODE_IMAGE_FUSA_BIN "sec2_ucode_fusa_image.bin"
+#define LSF_SEC2_UCODE_DESC_FUSA_BIN "sec2_ucode_fusa_desc.bin"
+#define LSF_SEC2_UCODE_SIG_FUSA_BIN "sec2_fusa_sig.bin"
+
+#define ACR_COMPLETION_TIMEOUT_NON_SILICON_MS 10000U /*in msec */
+#define ACR_COMPLETION_TIMEOUT_SILICON_MS 100 /*in msec */
+
+struct acr_lsf_config {
+	u32 falcon_id;
+	u32 falcon_dma_idx;
+	bool is_lazy_bootstrap;
+	bool is_priv_load;
+
+	int (*get_lsf_ucode_details)(struct gk20a *g, void *lsf_ucode_img);
+	void (*get_cmd_line_args_offset)(struct gk20a *g, u32 *args_offset);
+};
+
+struct nvgpu_acr {
+	struct gk20a *g;
+
+	u32 bootstrap_owner;
+	u32 num_of_sig;
+
+	/* LSF properties */
+	u64 lsf_enable_mask;
+	struct acr_lsf_config lsf[FALCON_ID_END];
+
+	/*
+	 * non-wpr space to hold LSF ucodes,
+	 * ACR does copy ucode from non-wpr to wpr
+	 */
+	struct nvgpu_mem ucode_blob;
+	/*
+	 * Even though this mem_desc wouldn't be used,
+	 * the wpr region needs to be reserved in the
+	 * allocator in dGPU case.
+	 */
+	struct nvgpu_mem wpr_dummy;
+
+	/* ACR member for different types of ucode */
+	/* For older dgpu/tegra ACR cuode */
+	struct hs_acr acr;
+	/* ACR load split feature support */
+	struct hs_acr acr_ahesasc;
+	struct hs_acr acr_asb;
+
+	/* ACR load split feature support for iGPU*/
+	struct hs_acr acr_alsb;
+	struct hs_acr acr_asc;
+
+	int (*prepare_ucode_blob)(struct gk20a *g);
+	int (*alloc_blob_space)(struct gk20a *g, size_t size,
+		struct nvgpu_mem *mem);
+	int (*patch_wpr_info_to_ucode)(struct gk20a *g, struct nvgpu_acr *acr,
+		struct hs_acr *acr_desc, bool is_recovery);
+	int (*bootstrap_hs_acr)(struct gk20a *g, struct nvgpu_acr *acr);
+
+	void (*get_wpr_info)(struct gk20a *g, struct wpr_carveout_info *inf);
+	u32* (*get_versioned_sig)(struct gk20a *g, struct nvgpu_acr *acr,
+		u32 *sig, u32 *sig_size);
+};
+
+#endif /* ACR_H */

drivers/gpu/nvgpu/common/acr/acr_sw_gm20b.c

@@ -0,0 +1,172 @@
+/*
+ * Copyright (c) 2015-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/gk20a.h>
+#include <nvgpu/types.h>
+#include <nvgpu/firmware.h>
+#include <nvgpu/falcon.h>
+#include <nvgpu/bug.h>
+#include <nvgpu/pmu/fw.h>
+
+#include "acr_wpr.h"
+#include "acr_priv.h"
+#include "acr_sw_gm20b.h"
+#include "acr_blob_alloc.h"
+#include "acr_bootstrap.h"
+#include "acr_blob_construct_v0.h"
+
+static int gm20b_bootstrap_hs_acr(struct gk20a *g, struct nvgpu_acr *acr)
+{
+	int err = 0;
+
+	nvgpu_log_fn(g, " ");
+
+	err = nvgpu_acr_bootstrap_hs_ucode(g, g->acr, &g->acr->acr);
+	if (err != 0) {
+		nvgpu_err(g, "ACR bootstrap failed");
+	}
+
+	return err;
+}
+
+static int gm20b_acr_patch_wpr_info_to_ucode(struct gk20a *g,
+	struct nvgpu_acr *acr, struct hs_acr *acr_desc, bool is_recovery)
+{
+	struct nvgpu_firmware *acr_fw = acr_desc->acr_fw;
+	struct acr_fw_header *acr_fw_hdr = NULL;
+	struct bin_hdr *acr_fw_bin_hdr = NULL;
+	struct flcn_acr_desc_v0 *acr_dmem_desc;
+	u32 *acr_ucode_header = NULL;
+	u32 *acr_ucode_data = NULL;
+
+	nvgpu_log_fn(g, " ");
+
+	if (is_recovery) {
+		acr_desc->acr_dmem_desc_v0->nonwpr_ucode_blob_size = 0U;
+	} else {
+		acr_fw_bin_hdr = (struct bin_hdr *)acr_fw->data;
+		acr_fw_hdr = (struct acr_fw_header *)
+			(acr_fw->data + acr_fw_bin_hdr->header_offset);
+
+		acr_ucode_data = (u32 *)(acr_fw->data +
+			acr_fw_bin_hdr->data_offset);
+
+		acr_ucode_header = (u32 *)(acr_fw->data +
+			acr_fw_hdr->hdr_offset);
+
+		/* Patch WPR info to ucode */
+		acr_dmem_desc = (struct flcn_acr_desc_v0 *)
+			&(((u8 *)acr_ucode_data)[acr_ucode_header[2U]]);
+
+		acr_desc->acr_dmem_desc_v0 = acr_dmem_desc;
+
+		acr_dmem_desc->nonwpr_ucode_blob_start =
+			nvgpu_mem_get_addr(g, &g->acr->ucode_blob);
+		nvgpu_assert(g->acr->ucode_blob.size <= U32_MAX);
+		acr_dmem_desc->nonwpr_ucode_blob_size =
+			(u32)g->acr->ucode_blob.size;
+		acr_dmem_desc->regions.no_regions = 1U;
+		acr_dmem_desc->wpr_offset = 0U;
+	}
+
+	return 0;
+}
+
+/* LSF static config functions */
+static u32 gm20b_acr_lsf_pmu(struct gk20a *g,
+		struct acr_lsf_config *lsf)
+{
+	/* PMU LS falcon info */
+	lsf->falcon_id = FALCON_ID_PMU;
+	lsf->falcon_dma_idx = GK20A_PMU_DMAIDX_UCODE;
+	lsf->is_lazy_bootstrap = false;
+	lsf->is_priv_load = false;
+#ifdef CONFIG_NVGPU_LS_PMU
+	lsf->get_lsf_ucode_details = nvgpu_acr_lsf_pmu_ucode_details_v0;
+	lsf->get_cmd_line_args_offset = nvgpu_pmu_fw_get_cmd_line_args_offset;
+#endif
+	return BIT32(lsf->falcon_id);
+}
+
+static u32 gm20b_acr_lsf_fecs(struct gk20a *g,
+		struct acr_lsf_config *lsf)
+{
+	/* FECS LS falcon info */
+	lsf->falcon_id = FALCON_ID_FECS;
+	lsf->falcon_dma_idx = GK20A_PMU_DMAIDX_UCODE;
+	lsf->is_lazy_bootstrap = false;
+	lsf->is_priv_load = false;
+	lsf->get_lsf_ucode_details = nvgpu_acr_lsf_fecs_ucode_details_v0;
+	lsf->get_cmd_line_args_offset = NULL;
+
+	return BIT32(lsf->falcon_id);
+}
+
+static u32 gm20b_acr_lsf_conifg(struct gk20a *g,
+	struct nvgpu_acr *acr)
+{
+	u32 lsf_enable_mask = 0;
+
+	lsf_enable_mask |= gm20b_acr_lsf_pmu(g, &acr->lsf[FALCON_ID_PMU]);
+	lsf_enable_mask |= gm20b_acr_lsf_fecs(g, &acr->lsf[FALCON_ID_FECS]);
+
+	return lsf_enable_mask;
+}
+
+static void gm20b_acr_default_sw_init(struct gk20a *g, struct hs_acr *hs_acr)
+{
+	nvgpu_log_fn(g, " ");
+
+	/* ACR HS ucode type & f/w name*/
+	hs_acr->acr_type = ACR_DEFAULT;
+
+	if (!g->ops.pmu.is_debug_mode_enabled(g)) {
+		hs_acr->acr_fw_name = GM20B_HSBIN_ACR_PROD_UCODE;
+	} else {
+		hs_acr->acr_fw_name = GM20B_HSBIN_ACR_DBG_UCODE;
+	}
+
+	/* set on which falcon ACR need to execute*/
+	hs_acr->acr_flcn = g->pmu->flcn;
+	hs_acr->acr_engine_bus_err_status =
+		g->ops.pmu.bar0_error_status;
+}
+
+void nvgpu_gm20b_acr_sw_init(struct gk20a *g, struct nvgpu_acr *acr)
+{
+	nvgpu_log_fn(g, " ");
+
+	acr->g = g;
+
+	acr->bootstrap_owner = FALCON_ID_PMU;
+
+	acr->lsf_enable_mask = gm20b_acr_lsf_conifg(g, acr);
+
+	gm20b_acr_default_sw_init(g, &acr->acr);
+
+	acr->prepare_ucode_blob = nvgpu_acr_prepare_ucode_blob_v0;
+	acr->get_wpr_info = nvgpu_acr_wpr_info_sys;
+	acr->alloc_blob_space = nvgpu_acr_alloc_blob_space_sys;
+	acr->bootstrap_hs_acr = gm20b_bootstrap_hs_acr;
+	acr->patch_wpr_info_to_ucode =
+		gm20b_acr_patch_wpr_info_to_ucode;
+}

drivers/gpu/nvgpu/common/acr/acr_sw_gm20b.h

@@ -0,0 +1,33 @@
+/*
+ * GM20B ACR
+ *
+ * Copyright (c) 2015-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef ACR_SW_GM20B_H
+#define ACR_SW_GM20B_H
+
+struct gk20a;
+struct nvgpu_acr;
+
+void nvgpu_gm20b_acr_sw_init(struct gk20a *g, struct nvgpu_acr *acr);
+
+#endif /*ACR_SW_GM20B_H*/

drivers/gpu/nvgpu/common/acr/acr_sw_gp10b.c

@@ -0,0 +1,80 @@
+/*
+ * Copyright (c) 2019-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include "acr_sw_gp10b.h"
+
+#include <nvgpu/types.h>
+#include <nvgpu/gk20a.h>
+#include <nvgpu/pmu.h>
+
+#include "acr_blob_construct_v0.h"
+#include "acr_priv.h"
+
+#include "acr_sw_gm20b.h"
+#include "acr_sw_gp10b.h"
+
+/* LSF static config functions */
+static u32 gp10b_acr_lsf_gpccs(struct gk20a *g,
+		struct acr_lsf_config *lsf)
+{
+	/* GPCCS LS falcon info */
+	lsf->falcon_id = FALCON_ID_GPCCS;
+	lsf->falcon_dma_idx = GK20A_PMU_DMAIDX_UCODE;
+	lsf->is_lazy_bootstrap = true;
+	lsf->is_priv_load = true;
+	lsf->get_lsf_ucode_details = nvgpu_acr_lsf_gpccs_ucode_details_v0;
+	lsf->get_cmd_line_args_offset = NULL;
+
+	return BIT32(lsf->falcon_id);
+}
+
+static void gp10b_acr_default_sw_init(struct gk20a *g, struct hs_acr *hs_acr)
+{
+	nvgpu_log_fn(g, " ");
+
+	/* ACR HS ucode type & f/w name*/
+	hs_acr->acr_type = ACR_DEFAULT;
+
+	if (!g->ops.pmu.is_debug_mode_enabled(g)) {
+		hs_acr->acr_fw_name = HSBIN_ACR_PROD_UCODE;
+	} else {
+		hs_acr->acr_fw_name = HSBIN_ACR_DBG_UCODE;
+	}
+
+	/* set on which falcon ACR need to execute*/
+	hs_acr->acr_flcn = g->pmu->flcn;
+	hs_acr->acr_engine_bus_err_status =
+		g->ops.pmu.bar0_error_status;
+}
+
+void nvgpu_gp10b_acr_sw_init(struct gk20a *g, struct nvgpu_acr *acr)
+{
+	nvgpu_log_fn(g, " ");
+
+	/* inherit the gm20b config data */
+	nvgpu_gm20b_acr_sw_init(g, acr);
+	gp10b_acr_default_sw_init(g, &acr->acr);
+
+	/* gp10b supports LSF gpccs bootstrap */
+	acr->lsf_enable_mask |= gp10b_acr_lsf_gpccs(g,
+		&acr->lsf[FALCON_ID_GPCCS]);
+}

drivers/gpu/nvgpu/common/acr/acr_sw_gp10b.h

@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef ACR_SW_GP10B_H
+#define ACR_SW_GP10B_H
+
+struct gk20a;
+struct nvgpu_acr;
+
+void nvgpu_gp10b_acr_sw_init(struct gk20a *g, struct nvgpu_acr *acr);
+
+#endif /* ACR_SW_GP10B_H */

drivers/gpu/nvgpu/common/acr/acr_sw_gv11b.c

@@ -0,0 +1,211 @@
+/*
+ * Copyright (c) 2017-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/types.h>
+#include <nvgpu/firmware.h>
+#include <nvgpu/gk20a.h>
+#include <nvgpu/bug.h>
+#ifdef CONFIG_NVGPU_LS_PMU
+#include <nvgpu/pmu/fw.h>
+#endif
+
+#include "acr_wpr.h"
+#include "acr_priv.h"
+#include "acr_blob_alloc.h"
+#include "acr_blob_construct.h"
+#include "acr_bootstrap.h"
+#include "acr_sw_gv11b.h"
+
+#define RECOVERY_UCODE_BLOB_SIZE	(0U)
+#define WPR_OFFSET			(0U)
+#define ACR_REGIONS			(1U)
+
+static int gv11b_bootstrap_hs_acr(struct gk20a *g, struct nvgpu_acr *acr)
+{
+	int err = 0;
+
+	nvgpu_log_fn(g, " ");
+
+	err = nvgpu_acr_bootstrap_hs_ucode(g, g->acr, &g->acr->acr);
+	if (err != 0) {
+		nvgpu_err(g, "ACR bootstrap failed");
+	}
+
+	return err;
+}
+
+static int gv11b_acr_patch_wpr_info_to_ucode(struct gk20a *g,
+	struct nvgpu_acr *acr, struct hs_acr *acr_desc, bool is_recovery)
+{
+	struct nvgpu_firmware *acr_fw = acr_desc->acr_fw;
+	struct acr_fw_header *acr_fw_hdr = NULL;
+	struct bin_hdr *acr_fw_bin_hdr = NULL;
+	struct flcn_acr_desc *acr_dmem_desc;
+	u32 *acr_ucode_header = NULL;
+	u32 *acr_ucode_data = NULL;
+	const u32 acr_desc_offset = 2U;
+
+	nvgpu_log_fn(g, " ");
+#ifdef CONFIG_NVGPU_NON_FUSA
+	if (is_recovery) {
+		acr_desc->acr_dmem_desc->nonwpr_ucode_blob_size =
+						RECOVERY_UCODE_BLOB_SIZE;
+	} else
+#endif
+	{
+		acr_fw_bin_hdr = (struct bin_hdr *)(void *)acr_fw->data;
+		acr_fw_hdr = (struct acr_fw_header *)(void *)
+			(acr_fw->data + acr_fw_bin_hdr->header_offset);
+
+		acr_ucode_data = (u32 *)(void *)(acr_fw->data +
+			acr_fw_bin_hdr->data_offset);
+		acr_ucode_header = (u32 *)(void *)(acr_fw->data +
+			acr_fw_hdr->hdr_offset);
+
+		/* Patch WPR info to ucode */
+		acr_dmem_desc = (struct flcn_acr_desc *)(void *)
+			&(((u8 *)acr_ucode_data)[acr_ucode_header[acr_desc_offset]]);
+
+		acr_desc->acr_dmem_desc = acr_dmem_desc;
+
+		acr_dmem_desc->nonwpr_ucode_blob_start =
+			nvgpu_mem_get_addr(g, &g->acr->ucode_blob);
+		nvgpu_assert(g->acr->ucode_blob.size <= U32_MAX);
+		acr_dmem_desc->nonwpr_ucode_blob_size =
+			(u32)g->acr->ucode_blob.size;
+		acr_dmem_desc->regions.no_regions = ACR_REGIONS;
+		acr_dmem_desc->wpr_offset = WPR_OFFSET;
+	}
+
+	return 0;
+}
+
+/* LSF static config functions */
+#ifdef CONFIG_NVGPU_LS_PMU
+static u32 gv11b_acr_lsf_pmu(struct gk20a *g,
+		struct acr_lsf_config *lsf)
+{
+	if (!g->support_ls_pmu) {
+		/* skip adding LS PMU ucode to ACR blob */
+		return 0;
+	}
+
+	/* PMU LS falcon info */
+	lsf->falcon_id = FALCON_ID_PMU;
+	lsf->falcon_dma_idx = GK20A_PMU_DMAIDX_UCODE;
+	lsf->is_lazy_bootstrap = false;
+	lsf->is_priv_load = false;
+	lsf->get_lsf_ucode_details = nvgpu_acr_lsf_pmu_ucode_details;
+	lsf->get_cmd_line_args_offset = nvgpu_pmu_fw_get_cmd_line_args_offset;
+
+	return BIT32(lsf->falcon_id);
+}
+#endif
+
+/* LSF init */
+static u32 gv11b_acr_lsf_fecs(struct gk20a *g,
+		struct acr_lsf_config *lsf)
+{
+	/* FECS LS falcon info */
+	lsf->falcon_id = FALCON_ID_FECS;
+	lsf->falcon_dma_idx = GK20A_PMU_DMAIDX_UCODE;
+	/*
+	 * FECS LSF cold/recovery bootstrap is handled by ACR when LS PMU
+	 * not present
+	 */
+	lsf->is_lazy_bootstrap = g->support_ls_pmu ? true : false;
+	lsf->is_priv_load = false;
+	lsf->get_lsf_ucode_details = nvgpu_acr_lsf_fecs_ucode_details;
+	lsf->get_cmd_line_args_offset = NULL;
+
+	return BIT32(lsf->falcon_id);
+}
+
+static u32 gv11b_acr_lsf_gpccs(struct gk20a *g,
+		struct acr_lsf_config *lsf)
+{
+	/* GPCCS LS falcon info */
+	lsf->falcon_id = FALCON_ID_GPCCS;
+	lsf->falcon_dma_idx = GK20A_PMU_DMAIDX_UCODE;
+	/*
+	 * GPCCS LSF cold/recovery bootstrap is handled by ACR when LS PMU
+	 * not present
+	 */
+	lsf->is_lazy_bootstrap = g->support_ls_pmu ? true : false;
+	lsf->is_priv_load = true;
+	lsf->get_lsf_ucode_details = nvgpu_acr_lsf_gpccs_ucode_details;
+	lsf->get_cmd_line_args_offset = NULL;
+
+	return BIT32(lsf->falcon_id);
+}
+
+u32 gv11b_acr_lsf_config(struct gk20a *g,
+	struct nvgpu_acr *acr)
+{
+	u32 lsf_enable_mask = 0;
+#ifdef CONFIG_NVGPU_LS_PMU
+	lsf_enable_mask |= gv11b_acr_lsf_pmu(g, &acr->lsf[FALCON_ID_PMU]);
+#endif
+	lsf_enable_mask |= gv11b_acr_lsf_fecs(g, &acr->lsf[FALCON_ID_FECS]);
+	lsf_enable_mask |= gv11b_acr_lsf_gpccs(g, &acr->lsf[FALCON_ID_GPCCS]);
+
+	return lsf_enable_mask;
+}
+
+static void gv11b_acr_default_sw_init(struct gk20a *g, struct hs_acr *acr_desc)
+{
+	nvgpu_log_fn(g, " ");
+
+	acr_desc->acr_type = ACR_DEFAULT;
+
+	if (!g->ops.pmu.is_debug_mode_enabled(g)) {
+		acr_desc->acr_fw_name = HSBIN_ACR_PROD_UCODE;
+	} else {
+		acr_desc->acr_fw_name = HSBIN_ACR_DBG_UCODE;
+	}
+
+	acr_desc->acr_flcn = g->pmu->flcn;
+	acr_desc->report_acr_engine_bus_err_status =
+		nvgpu_pmu_report_bar0_pri_err_status;
+	acr_desc->acr_engine_bus_err_status =
+		g->ops.pmu.bar0_error_status;
+	acr_desc->acr_validate_mem_integrity = g->ops.pmu.validate_mem_integrity;
+}
+
+void nvgpu_gv11b_acr_sw_init(struct gk20a *g, struct nvgpu_acr *acr)
+{
+	nvgpu_log_fn(g, " ");
+
+	acr->g = g;
+
+	acr->bootstrap_owner = FALCON_ID_PMU;
+
+	acr->lsf_enable_mask = gv11b_acr_lsf_config(g, acr);
+
+	gv11b_acr_default_sw_init(g, &acr->acr);
+
+	acr->prepare_ucode_blob = nvgpu_acr_prepare_ucode_blob;
+	acr->get_wpr_info = nvgpu_acr_wpr_info_sys;
+	acr->alloc_blob_space = nvgpu_acr_alloc_blob_space_sys;
+	acr->bootstrap_hs_acr = gv11b_bootstrap_hs_acr;
+	acr->patch_wpr_info_to_ucode = gv11b_acr_patch_wpr_info_to_ucode;
+}

drivers/gpu/nvgpu/common/acr/acr_sw_gv11b.h

@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2017-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef ACR_SW_GV11B_H
+#define ACR_SW_GV11B_H
+
+struct gk20a;
+struct nvgpu_acr;
+struct hs_acr;
+
+void nvgpu_gv11b_acr_sw_init(struct gk20a *g, struct nvgpu_acr *acr);
+u32 gv11b_acr_lsf_config(struct gk20a *g, struct nvgpu_acr *acr);
+
+#endif /* ACR_SW_GV11B_H */
+

drivers/gpu/nvgpu/common/acr/acr_sw_tu104.c

@@ -0,0 +1,285 @@
+/*
+ * Copyright (c) 2016-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include "acr_sw_tu104.h"
+
+#include <nvgpu/gk20a.h>
+#include <nvgpu/firmware.h>
+
+#include "acr_wpr.h"
+#include "acr_priv.h"
+#include "acr_blob_alloc.h"
+#include "acr_bootstrap.h"
+#include "acr_blob_construct.h"
+#include "acr_sw_gv11b.h"
+#include "acr_sw_tu104.h"
+
+static int tu104_bootstrap_hs_acr(struct gk20a *g, struct nvgpu_acr *acr)
+{
+	int err = 0;
+
+	nvgpu_log_fn(g, " ");
+
+	err = nvgpu_acr_bootstrap_hs_ucode(g, g->acr, &g->acr->acr_ahesasc);
+	if (err != 0) {
+		nvgpu_err(g, "ACR AHESASC bootstrap failed");
+		goto exit;
+	}
+	err = nvgpu_acr_bootstrap_hs_ucode(g, g->acr, &g->acr->acr_asb);
+	if (err != 0) {
+		nvgpu_err(g, "ACR ASB bootstrap failed");
+		goto exit;
+	}
+
+exit:
+	return err;
+}
+
+/* WPR info update */
+static int tu104_acr_patch_wpr_info_to_ucode(struct gk20a *g,
+	struct nvgpu_acr *acr, struct hs_acr *acr_desc,
+	bool is_recovery)
+{
+	struct nvgpu_firmware *acr_fw = acr_desc->acr_fw;
+	struct acr_fw_header *acr_fw_hdr = NULL;
+	struct bin_hdr *acr_fw_bin_hdr = NULL;
+	struct flcn_acr_desc *acr_dmem_desc;
+	struct wpr_carveout_info wpr_inf;
+	u32 *acr_ucode_header = NULL;
+	u32 *acr_ucode_data = NULL;
+	u64 tmp_addr;
+
+	nvgpu_log_fn(g, " ");
+
+	acr_fw_bin_hdr = (struct bin_hdr *)acr_fw->data;
+	acr_fw_hdr = (struct acr_fw_header *)
+		(acr_fw->data + acr_fw_bin_hdr->header_offset);
+
+	acr_ucode_data = (u32 *)(acr_fw->data + acr_fw_bin_hdr->data_offset);
+	acr_ucode_header = (u32 *)(acr_fw->data + acr_fw_hdr->hdr_offset);
+
+	acr->get_wpr_info(g, &wpr_inf);
+
+	acr_dmem_desc = (struct flcn_acr_desc *)
+		&(((u8 *)acr_ucode_data)[acr_ucode_header[2U]]);
+
+	acr_dmem_desc->nonwpr_ucode_blob_start = wpr_inf.nonwpr_base;
+	nvgpu_assert(wpr_inf.size <= U32_MAX);
+	acr_dmem_desc->nonwpr_ucode_blob_size = (u32)wpr_inf.size;
+	acr_dmem_desc->regions.no_regions = 1U;
+	acr_dmem_desc->wpr_offset = 0U;
+
+	acr_dmem_desc->wpr_region_id = 1U;
+	acr_dmem_desc->regions.region_props[0U].region_id = 1U;
+
+	tmp_addr = (wpr_inf.wpr_base) >> 8U;
+	nvgpu_assert(u64_hi32(tmp_addr) == 0U);
+	acr_dmem_desc->regions.region_props[0U].start_addr = U32(tmp_addr);
+
+	tmp_addr = ((wpr_inf.wpr_base) + wpr_inf.size) >> 8U;
+	nvgpu_assert(u64_hi32(tmp_addr) == 0U);
+	acr_dmem_desc->regions.region_props[0U].end_addr = U32(tmp_addr);
+
+	tmp_addr = wpr_inf.nonwpr_base >> 8U;
+	nvgpu_assert(u64_hi32(tmp_addr) == 0U);
+	acr_dmem_desc->regions.region_props[0U].shadowmMem_startaddress =
+		U32(tmp_addr);
+
+	return 0;
+}
+
+/* LSF init */
+static u32 tu104_acr_lsf_sec2(struct gk20a *g,
+		struct acr_lsf_config *lsf)
+{
+	/* SEC2 LS falcon info */
+	lsf->falcon_id = FALCON_ID_SEC2;
+	lsf->falcon_dma_idx = NV_SEC2_DMAIDX_UCODE;
+	lsf->is_lazy_bootstrap = false;
+	lsf->is_priv_load = false;
+	lsf->get_lsf_ucode_details = nvgpu_acr_lsf_sec2_ucode_details;
+	lsf->get_cmd_line_args_offset = NULL;
+
+	return BIT32(lsf->falcon_id);
+}
+
+static u32 tu104_acr_lsf_pmu(struct gk20a *g,
+		struct acr_lsf_config *lsf)
+{
+	/* PMU support not required until PSTATE support is enabled */
+	if (!g->support_ls_pmu) {
+		/* skip adding LS PMU ucode to ACR blob */
+		return 0;
+	}
+
+	/* PMU LS falcon info */
+	lsf->falcon_id = FALCON_ID_PMU;
+	lsf->falcon_dma_idx = GK20A_PMU_DMAIDX_UCODE;
+	lsf->is_lazy_bootstrap = false;
+	lsf->is_priv_load = false;
+#ifdef CONFIG_NVGPU_LS_PMU
+	lsf->get_lsf_ucode_details = nvgpu_acr_lsf_pmu_ucode_details;
+	lsf->get_cmd_line_args_offset = nvgpu_pmu_fw_get_cmd_line_args_offset;
+#endif
+	return BIT32(lsf->falcon_id);
+}
+
+static u32 tu104_acr_lsf_fecs(struct gk20a *g,
+		struct acr_lsf_config *lsf)
+{
+	/* FECS LS falcon info */
+	lsf->falcon_id = FALCON_ID_FECS;
+	lsf->falcon_dma_idx = GK20A_PMU_DMAIDX_UCODE;
+	lsf->is_lazy_bootstrap = true;
+	lsf->is_priv_load = true;
+	lsf->get_lsf_ucode_details = nvgpu_acr_lsf_fecs_ucode_details;
+	lsf->get_cmd_line_args_offset = NULL;
+
+	return BIT32(lsf->falcon_id);
+}
+
+static u32 tu104_acr_lsf_gpccs(struct gk20a *g,
+		struct acr_lsf_config *lsf)
+{
+	/* FECS LS falcon info */
+	lsf->falcon_id = FALCON_ID_GPCCS;
+	lsf->falcon_dma_idx = GK20A_PMU_DMAIDX_UCODE;
+	lsf->is_lazy_bootstrap = true;
+	lsf->is_priv_load = true;
+	lsf->get_lsf_ucode_details = nvgpu_acr_lsf_gpccs_ucode_details;
+	lsf->get_cmd_line_args_offset = NULL;
+
+	return BIT32(lsf->falcon_id);
+}
+
+static u32 tu104_acr_lsf_conifg(struct gk20a *g,
+	struct nvgpu_acr *acr)
+{
+	u32 lsf_enable_mask = 0;
+	lsf_enable_mask |= tu104_acr_lsf_pmu(g, &acr->lsf[FALCON_ID_PMU]);
+	lsf_enable_mask |= tu104_acr_lsf_fecs(g, &acr->lsf[FALCON_ID_FECS]);
+	lsf_enable_mask |= tu104_acr_lsf_gpccs(g, &acr->lsf[FALCON_ID_GPCCS]);
+	lsf_enable_mask |= tu104_acr_lsf_sec2(g, &acr->lsf[FALCON_ID_SEC2]);
+
+	return lsf_enable_mask;
+}
+
+/* fusa signing enable check */
+static bool tu104_acr_is_fusa_enabled(struct gk20a *g)
+{
+	return g->is_fusa_sku;
+}
+
+/* ACR-AHESASC(ACR hub encryption setter and signature checker) init*/
+static void tu104_acr_ahesasc_v0_ucode_select(struct gk20a *g,
+		struct hs_acr *acr_ahesasc)
+{
+	acr_ahesasc->acr_type = ACR_AHESASC_NON_FUSA;
+
+	if (!g->ops.pmu.is_debug_mode_enabled(g)) {
+		acr_ahesasc->acr_fw_name = HSBIN_ACR_AHESASC_NON_FUSA_PROD_UCODE;
+	} else {
+		acr_ahesasc->acr_fw_name = HSBIN_ACR_AHESASC_NON_FUSA_DBG_UCODE;
+	}
+
+}
+
+static void tu104_acr_ahesasc_fusa_ucode_select(struct gk20a *g,
+		struct hs_acr *acr_ahesasc)
+{
+	acr_ahesasc->acr_type = ACR_AHESASC_FUSA;
+
+	if (!g->ops.pmu.is_debug_mode_enabled(g)) {
+		acr_ahesasc->acr_fw_name = HSBIN_ACR_AHESASC_FUSA_PROD_UCODE;
+	} else {
+		acr_ahesasc->acr_fw_name = HSBIN_ACR_AHESASC_FUSA_DBG_UCODE;
+	}
+}
+
+static void tu104_acr_ahesasc_sw_init(struct gk20a *g,
+	struct hs_acr *acr_ahesasc)
+{
+	if (tu104_acr_is_fusa_enabled(g)) {
+		tu104_acr_ahesasc_fusa_ucode_select(g, acr_ahesasc);
+	} else {
+		tu104_acr_ahesasc_v0_ucode_select(g, acr_ahesasc);
+	}
+
+	acr_ahesasc->acr_flcn = &g->sec2.flcn;
+}
+
+/* ACR-ASB(ACR SEC2 booter) init*/
+static void tu104_acr_asb_v0_ucode_select(struct gk20a *g,
+	struct hs_acr *acr_asb)
+{
+	acr_asb->acr_type = ACR_ASB_NON_FUSA;
+
+	if (!g->ops.pmu.is_debug_mode_enabled(g)) {
+		acr_asb->acr_fw_name = HSBIN_ACR_ASB_NON_FUSA_PROD_UCODE;
+	} else {
+		acr_asb->acr_fw_name = HSBIN_ACR_ASB_NON_FUSA_DBG_UCODE;
+	}
+}
+
+static void tu104_acr_asb_fusa_ucode_select(struct gk20a *g,
+	struct hs_acr *acr_asb)
+{
+	acr_asb->acr_type = ACR_ASB_FUSA;
+
+	if (!g->ops.pmu.is_debug_mode_enabled(g)) {
+		acr_asb->acr_fw_name = HSBIN_ACR_ASB_FUSA_PROD_UCODE;
+	} else {
+		acr_asb->acr_fw_name = HSBIN_ACR_ASB_FUSA_DBG_UCODE;
+	}
+}
+
+static void tu104_acr_asb_sw_init(struct gk20a *g,
+	struct hs_acr *acr_asb)
+{
+	if (tu104_acr_is_fusa_enabled(g)) {
+		tu104_acr_asb_fusa_ucode_select(g, acr_asb);
+	} else {
+		tu104_acr_asb_v0_ucode_select(g, acr_asb);
+	}
+
+	acr_asb->acr_flcn = &g->gsp_flcn;
+}
+
+void nvgpu_tu104_acr_sw_init(struct gk20a *g, struct nvgpu_acr *acr)
+{
+	nvgpu_log_fn(g, " ");
+
+	acr->lsf_enable_mask = tu104_acr_lsf_conifg(g, acr);
+
+	acr->prepare_ucode_blob = nvgpu_acr_prepare_ucode_blob;
+	acr->get_wpr_info = nvgpu_acr_wpr_info_vid;
+	acr->alloc_blob_space = nvgpu_acr_alloc_blob_space_vid;
+	acr->bootstrap_owner = FALCON_ID_GSPLITE;
+	acr->bootstrap_hs_acr = tu104_bootstrap_hs_acr;
+	acr->patch_wpr_info_to_ucode = tu104_acr_patch_wpr_info_to_ucode;
+
+	/* Init ACR-AHESASC */
+	tu104_acr_ahesasc_sw_init(g, &acr->acr_ahesasc);
+
+	/* Init ACR-ASB*/
+	tu104_acr_asb_sw_init(g, &acr->acr_asb);
+}

drivers/gpu/nvgpu/common/acr/acr_sw_tu104.h

@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2018-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef ACR_SW_TU104_H
+#define ACR_SW_TU104_H
+
+struct gk20a;
+struct nvgpu_acr;
+
+void nvgpu_tu104_acr_sw_init(struct gk20a *g, struct nvgpu_acr *acr);
+
+#endif /*ACR_SW_TU104_H*/

drivers/gpu/nvgpu/common/acr/acr_wpr.c

@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/gk20a.h>
+#include <nvgpu/types.h>
+#include <nvgpu/dma.h>
+
+#include "acr_wpr.h"
+
+/* Both size and address of WPR need to be 128K-aligned */
+#define DGPU_WPR_SIZE 0x200000U
+
+void nvgpu_acr_wpr_info_sys(struct gk20a *g, struct wpr_carveout_info *inf)
+{
+	g->ops.fb.read_wpr_info(g, &inf->wpr_base, &inf->size);
+}
+#ifdef CONFIG_NVGPU_DGPU
+void nvgpu_acr_wpr_info_vid(struct gk20a *g, struct wpr_carveout_info *inf)
+{
+	inf->wpr_base = g->mm.vidmem.bootstrap_base;
+	inf->nonwpr_base = inf->wpr_base + DGPU_WPR_SIZE;
+	inf->size = DGPU_WPR_SIZE;
+}
+#endif

drivers/gpu/nvgpu/common/acr/acr_wpr.h

@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2018-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef ACR_WPR_H
+#define ACR_WPR_H
+
+struct gk20a;
+struct wpr_carveout_info;
+
+struct wpr_carveout_info {
+	u64 wpr_base;
+	u64 nonwpr_base;
+	u64 size;
+};
+
+void nvgpu_acr_wpr_info_sys(struct gk20a *g, struct wpr_carveout_info *inf);
+#ifdef CONFIG_NVGPU_DGPU
+void nvgpu_acr_wpr_info_vid(struct gk20a *g, struct wpr_carveout_info *inf);
+#endif
+
+#endif /* NVGPU_ACR_WPR_H */

drivers/gpu/nvgpu/common/acr/nvgpu_acr_interface.h

@@ -0,0 +1,609 @@
+/*
+ * Copyright (c) 2019-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef NVGPU_ACR_INTERFACE_H
+#define NVGPU_ACR_INTERFACE_H
+
+/**
+ * @defgroup NVGPURM_BLOB_CONSTRUCT blob construct
+ *
+ * Blob construct interfaces:
+ * NVGPU creates LS ucode blob in system/FB's non-WPR memory. LS ucodes
+ * will be read from filesystem and added to blob for the detected chip.
+ * Below are the structs that need to be filled by NvGPU for each LS Falcon
+ * ucode supported for the detected chip. After filling structures successfully,
+ * NvGPU should copy below structs along with ucode to the non-WPR blob
+ * in below mentioned pattern. LS ucodes blob is required by the ACR HS
+ * ucode to authenticate & load LS ucode on to respective engine's LS Falcon.
+ *
+ * + WPR header struct #lsf_wpr_header.
+ * + LSB header struct #lsf_lsb_header.
+ * + Boot loader struct #flcn_bl_dmem_desc.
+ * + ucode image.
+ *
+ * + BLOB Pattern:
+ *    ---------------------------------------------
+ *   | LSF WPR HDR  | LSF LSB HDR | BL desc | ucode |
+ *    ---------------------------------------------
+ */
+
+/**
+ * @ingroup NVGPURM_BLOB_CONSTRUCT
+ */
+/** @{*/
+
+/**
+ * Light Secure WPR Content Alignments
+ */
+/** WPR header should be aligned to 256 bytes */
+#define LSF_WPR_HEADER_ALIGNMENT        (256U)
+/** SUB WPR header should be aligned to 256 bytes */
+#define LSF_SUB_WPR_HEADER_ALIGNMENT    (256U)
+/** LSB header should be aligned to 256 bytes */
+#define LSF_LSB_HEADER_ALIGNMENT        (256U)
+/** BL DATA should be aligned to 256 bytes */
+#define LSF_BL_DATA_ALIGNMENT           (256U)
+/** BL DATA size should be aligned to 256 bytes */
+#define LSF_BL_DATA_SIZE_ALIGNMENT      (256U)
+/** BL CODE size should be aligned to 256 bytes */
+#define LSF_BL_CODE_SIZE_ALIGNMENT      (256U)
+/** LSF DATA size should be aligned to 256 bytes */
+#define LSF_DATA_SIZE_ALIGNMENT         (256U)
+/** LSF CODE size should be aligned to 256 bytes */
+#define LSF_CODE_SIZE_ALIGNMENT         (256U)
+
+/** UCODE surface should be aligned to 4k PAGE_SIZE */
+#define LSF_UCODE_DATA_ALIGNMENT 4096U
+
+/**
+ * Maximum WPR Header size
+ */
+#define LSF_WPR_HEADERS_TOTAL_SIZE_MAX	\
+	(ALIGN_UP(((u32)sizeof(struct lsf_wpr_header) * FALCON_ID_END), \
+		LSF_WPR_HEADER_ALIGNMENT))
+#define LSF_LSB_HEADER_TOTAL_SIZE_MAX	(\
+	ALIGN_UP(sizeof(struct lsf_lsb_header), LSF_LSB_HEADER_ALIGNMENT))
+
+/** @} */
+
+#ifdef CONFIG_NVGPU_DGPU
+/* Maximum SUB WPR header size */
+#define LSF_SUB_WPR_HEADERS_TOTAL_SIZE_MAX	(ALIGN_UP( \
+	(sizeof(struct lsf_shared_sub_wpr_header) * \
+	LSF_SHARED_DATA_SUB_WPR_USE_CASE_ID_MAX), \
+	LSF_SUB_WPR_HEADER_ALIGNMENT))
+
+/* MMU excepts sub_wpr sizes in units of 4K */
+#define SUB_WPR_SIZE_ALIGNMENT	(4096U)
+
+/* Defined for 1MB alignment */
+#define SHIFT_4KB	(12U)
+
+/* shared sub_wpr use case IDs */
+enum {
+	LSF_SHARED_DATA_SUB_WPR_USE_CASE_ID_FRTS_VBIOS_TABLES	= 1,
+	LSF_SHARED_DATA_SUB_WPR_USE_CASE_ID_PLAYREADY_SHARED_DATA = 2
+};
+
+#define LSF_SHARED_DATA_SUB_WPR_USE_CASE_ID_MAX \
+	LSF_SHARED_DATA_SUB_WPR_USE_CASE_ID_PLAYREADY_SHARED_DATA
+
+#define LSF_SHARED_DATA_SUB_WPR_USE_CASE_ID_INVALID	(0xFFFFFFFFU)
+
+#define MAX_SUPPORTED_SHARED_SUB_WPR_USE_CASES	\
+	LSF_SHARED_DATA_SUB_WPR_USE_CASE_ID_MAX
+
+/* Static sizes of shared subWPRs */
+/* Minimum granularity supported is 4K */
+/* 1MB in 4K */
+#define LSF_SHARED_DATA_SUB_WPR_FRTS_VBIOS_TABLES_SIZE_IN_4K	(0x100U)
+/* 4K */
+#define LSF_SHARED_DATA_SUB_WPR_PLAYREADY_SHARED_DATA_SIZE_IN_4K	(0x1U)
+#endif
+
+/**
+ * @ingroup NVGPURM_BLOB_CONSTRUCT
+ */
+/** @{*/
+
+/**
+ * Image status updated by ACR HS ucode to know the LS
+ * Falcon ucode status.
+ */
+/** IMAGE copied from NON-WPR to WPR BLOB*/
+#define LSF_IMAGE_STATUS_COPY                           (1U)
+/** LS Falcon ucode verification failed*/
+#define LSF_IMAGE_STATUS_VALIDATION_CODE_FAILED         (2U)
+/** LS Falcon data verification failed*/
+#define LSF_IMAGE_STATUS_VALIDATION_DATA_FAILED         (3U)
+/** Both ucode and data validation passed */
+#define LSF_IMAGE_STATUS_VALIDATION_DONE                (4U)
+/**
+ * LS Falcons such as FECS and GPCCS does not have signatures for binaries in
+ * debug environment(fmodel).
+ */
+#define LSF_IMAGE_STATUS_VALIDATION_SKIPPED             (5U)
+/** LS Falcon validation passed & ready for bootstrap */
+#define LSF_IMAGE_STATUS_BOOTSTRAP_READY                (6U)
+
+/**
+ * Light Secure WPR Header
+ * Defines state allowing Light Secure Falcon bootstrapping.
+ */
+struct lsf_wpr_header {
+	 /**
+	 * LS Falcon ID
+	 * FALCON_ID_FECS  - 2
+	 * FALCON_ID_GPCCS - 3
+	 */
+	u32 falcon_id;
+	/**
+	 * LS Falcon LSB header offset from non-WPR base, below equation used
+	 * to get LSB header offset for each managed LS falcon.
+	 * Offset = Non-WPR base + #LSF_LSB_HEADER_ALIGNMENT +
+	 *          ((#LSF_UCODE_DATA_ALIGNMENT + #LSF_BL_DATA_ALIGNMENT) *
+	 *          LS Falcon index)
+	 *
+	 */
+	u32 lsb_offset;
+	/**
+	 * LS Falcon bootstrap owner, which performs bootstrapping of
+	 * supported LS Falcon from ACR HS ucode. Below are the bootstrapping
+	 * supporting Falcon owners.
+	 *  + Falcon #FALCON_ID_PMU
+	 *
+	 * On GV11B, bootstrap_owner set to #FALCON_ID_PMU as ACR HS ucode
+	 * runs on PMU Engine Falcon.
+	 *
+	 */
+	u32 bootstrap_owner;
+	/**
+	 * Skip bootstrapping by ACR HS ucode,
+	 * 1 - skip LS Falcon bootstrapping by ACR HS ucode.
+	 * 0 - LS Falcon bootstrapping is done by ACR HS ucode.
+	 *
+	 * On GV11B, always set 0.
+	 */
+	u32 lazy_bootstrap;
+	/** LS ucode bin version*/
+	u32 bin_version;
+	/**
+	 * Bootstrapping status updated by ACR HS ucode to know the LS
+	 * Falcon ucode status.
+	 */
+	u32 status;
+};
+
+/** @} */
+
+/**
+ * @ingroup NVGPURM_BLOB_CONSTRUCT
+ */
+/** @{*/
+/**
+ * Size in entries of the ucode descriptor's dependency map.
+ */
+#define LSF_FALCON_DEPMAP_SIZE  (11U)
+
+/**
+ * Code/data signature details of LS falcon
+ */
+struct lsf_ucode_desc {
+	/** ucode's production signature */
+	u8  prd_keys[2][16];
+	/** ucode's debug signature */
+	u8  dbg_keys[2][16];
+	/**
+	 * production signature present status,
+	 * 1 - production signature present
+	 * 0 - production signature not present
+	 */
+	u32 b_prd_present;
+	/**
+	 * debug signature present
+	 * 1 - debug signature present
+	 * 0 - debug signature not present
+	 */
+	u32 b_dbg_present;
+	/**
+	 * LS Falcon ID
+	 * FALCON_ID_FECS  - 2
+	 * FALCON_ID_GPCCS - 3
+	 */
+	u32 falcon_id;
+	/**
+	 * include version in signature calculation if supported
+	 * 1 - supported
+	 * 0 - not supported
+	 */
+	u32 bsupports_versioning;
+	/** version to include it in signature calculation if supported */
+	u32 version;
+	/** valid dependency map data to consider from  dep_map array member */
+	u32 dep_map_count;
+	/**
+	 * packed dependency map used to compute the DM hashes on the code and
+	 * data.
+	 */
+	u8  dep_map[LSF_FALCON_DEPMAP_SIZE * 2 * 4];
+	/** Message used to derive key */
+	u8  kdf[16];
+};
+
+/** @} */
+
+/**
+ * @ingroup NVGPURM_BLOB_CONSTRUCT
+ */
+/** @{*/
+
+/**
+ * Light Secure Bootstrap Header
+ * Defines state allowing Light Secure Falcon bootstrapping.
+ */
+/** Load BL at 0th IMEM offset */
+#define NV_FLCN_ACR_LSF_FLAG_LOAD_CODE_AT_0_FALSE       0U
+#define NV_FLCN_ACR_LSF_FLAG_LOAD_CODE_AT_0_TRUE        BIT32(0)
+/** This falcon requires a ctx before issuing DMAs. */
+#define NV_FLCN_ACR_LSF_FLAG_DMACTL_REQ_CTX_FALSE       0U
+#define NV_FLCN_ACR_LSF_FLAG_DMACTL_REQ_CTX_TRUE        BIT32(2)
+/** Use priv loading method instead of bootloader/DMAs */
+#define NV_FLCN_ACR_LSF_FLAG_FORCE_PRIV_LOAD_TRUE       BIT32(3)
+#define NV_FLCN_ACR_LSF_FLAG_FORCE_PRIV_LOAD_FALSE      (0U)
+struct lsf_lsb_header {
+	/** Code/data signature details of each LS falcon */
+	struct lsf_ucode_desc signature;
+	/**
+	 * Offset from non-WPR base where UCODE is located,
+	 * Offset = Non-WPR base + #LSF_LSB_HEADER_ALIGNMENT +
+	 *          #LSF_UCODE_DATA_ALIGNMENT + ( #LSF_BL_DATA_ALIGNMENT *
+	 *          LS Falcon index)
+	 */
+	u32 ucode_off;
+	/**
+	 * Size of LS Falcon ucode, required to perform signature verification
+	 * of LS Falcon ucode by ACR HS.
+	 */
+	u32 ucode_size;
+	/**
+	 * Size of LS Falcon ucode data, required to perform signature
+	 * verification of LS Falcon ucode data by ACR HS.
+	 */
+	u32 data_size;
+	/**
+	 * Size of bootloader that needs to be loaded by bootstrap owner.
+	 *
+	 * On GV11B, respective LS Falcon BL code size should not exceed
+	 * below mentioned size.
+	 * FALCON_ID_FECS IMEM size  - 32k
+	 * FALCON_ID_GPCCS IMEM size - 16k
+	 */
+	u32 bl_code_size;
+	/** BL starting virtual address. Need for tagging */
+	u32 bl_imem_off;
+	/**
+	 * Offset from non-WPR base holding the BL data
+	 * Offset = (Non-WPR base + #LSF_LSB_HEADER_ALIGNMENT +
+	 *          #LSF_UCODE_DATA_ALIGNMENT + #LSF_BL_DATA_ALIGNMENT) *
+	 *          #LS Falcon index
+	 */
+	u32 bl_data_off;
+	/**
+	 * Size of BL data, BL data will be copied to LS Falcon DMEM of
+	 * bl data size
+	 *
+	 * On GV11B, respective LS Falcon BL data size should not exceed
+	 * below mentioned size.
+	 * FALCON_ID_FECS DMEM size  - 8k
+	 * FALCON_ID_GPCCS DMEM size - 5k
+	 */
+	u32 bl_data_size;
+	/**
+	 * Offset from non-WPR base address where UCODE Application code is
+	 * located.
+	 */
+	u32 app_code_off;
+	/**
+	 * Size of UCODE Application code.
+	 *
+	 * On GV11B, FECS/GPCCS LS Falcon app code size should not exceed
+	 * below mentioned size.
+	 * FALCON_ID_FECS IMEM size  - 32k
+	 * FALCON_ID_GPCCS IMEM size - 16k
+	 */
+	u32 app_code_size;
+	/**
+	 * Offset from non-WPR base address where UCODE Application data
+	 * is located
+	 */
+	u32 app_data_off;
+	/**
+	 * Size of UCODE Application data.
+	 *
+	 * On GV11B, respective LS Falcon app data size should not exceed
+	 * below mentioned size.
+	 * FALCON_ID_FECS DMEM size  - 8k
+	 * FALCON_ID_GPCCS DMEM size - 5k
+	 */
+	u32 app_data_size;
+	/**
+	 * NV_FLCN_ACR_LSF_FLAG_LOAD_CODE_AT_0 - Load BL at 0th IMEM offset
+	 * NV_FLCN_ACR_LSF_FLAG_DMACTL_REQ_CTX - This falcon requires a ctx
+	 * before issuing DMAs.
+	 * NV_FLCN_ACR_LSF_FLAG_FORCE_PRIV_LOAD - Use priv loading method
+	 * instead of bootloader/DMAs
+	 */
+	u32 flags;
+};
+
+#define FLCN_SIG_SIZE	(4U)
+/** @} */
+
+/**
+ * @ingroup NVGPURM_BLOB_CONSTRUCT
+ */
+/** @{*/
+/**
+ * Structure used by the boot-loader to load the rest of the LS Falcon code.
+ *
+ * This has to be filled by the GPU driver and copied into WPR region offset
+ * holding the BL data.
+ */
+struct flcn_bl_dmem_desc {
+	/** Should be always first element */
+	u32 reserved[FLCN_SIG_SIZE];
+	/**
+	 * Signature should follow reserved 16B signature for secure code.
+	 * 0s if no secure code
+	 */
+	u32 signature[FLCN_SIG_SIZE];
+	/**
+	 * Type of memory-aperture DMA index used by the bootloader
+	 * while loading code/data.
+	 */
+	u32 ctx_dma;
+	/**
+	 * 256B aligned physical sysmem(iGPU)/FB(dGPU) address where code
+	 * is located.
+	 */
+	struct falc_u64 code_dma_base;
+	/**
+	 * Offset from code_dma_base where the nonSecure code is located.
+	 * The offset must be multiple of 256 to help performance.
+	 */
+	u32 non_sec_code_off;
+	/**
+	 * The size of the non-secure code part.
+	 *
+	 * On GV11B, FECS/GPCCS LS Falcon non-secure + secure code size
+	 * should not exceed below mentioned size.
+	 * FALCON_ID_FECS IMEM size  - 32k
+	 * FALCON_ID_GPCCS IMEM size - 16k
+	 */
+	u32 non_sec_code_size;
+	/**
+	 * Offset from code_dma_base where the secure code is located.
+	 * The offset must be multiple of 256 to help performance.
+	 */
+	u32 sec_code_off;
+	/**
+	 * The size of the secure code part.
+	 *
+	 * On GV11B, FECS/GPCCS LS Falcon non-secure + secure code size
+	 * should not exceed below mentioned size.
+	 * FALCON_ID_FECS IMEM size  - 32k
+	 * FALCON_ID_GPCCS IMEM size - 16k
+	 */
+	u32 sec_code_size;
+	/**
+	 * Code entry point which will be invoked by BL after code is
+	 * loaded.
+	 */
+	u32 code_entry_point;
+	/**
+	 * 256B aligned Physical sysmem(iGPU)/FB(dGPU) Address where data
+	 * is located.
+	 */
+	struct falc_u64 data_dma_base;
+	/**
+	 * Size of data block. Should be multiple of 256B.
+	 *
+	 * On GV11B, respective LS Falcon data size should not exceed
+	 * below mentioned size.
+	 * FALCON_ID_FECS DMEM size  - 8k
+	 * FALCON_ID_GPCCS DMEM size - 5k
+	 */
+	u32 data_size;
+	/** Arguments to be passed to the target firmware being loaded. */
+	u32 argc;
+	/**
+	 * Number of arguments to be passed to the target firmware
+	 * being loaded.
+	 */
+	u32 argv;
+};
+
+/** @} */
+
+/**
+ * @defgroup NVGPURM_ACR_HS_LOAD_BOOTSTRAP ACR HS ucode load & bootstrap
+ *
+ * ACR HS ucode load & bootstrap interfaces:
+ * ACR HS ucode is read from the filesystem based on the chip-id by the ACR
+ * unit. Read ACR HS ucode will be update with below structs by patching at
+ * offset present in struct #struct acr_fw_header member hdr_offset. Read
+ * ACR HS ucode is loaded onto PMU/SEC2/GSP engines Falcon to bootstrap
+ * ACR HS ucode. ACR HS ucode does self-authentication using H/W based
+ * HS authentication methodology. Once authenticated the ACR HS ucode
+ * starts executing on the falcon.
+ */
+
+/**
+ * @ingroup NVGPURM_ACR_HS_LOAD_BOOTSTRAP
+ */
+/** @{*/
+
+/**
+ * Supporting maximum of 2 regions.
+ * This is needed to pre-allocate space in DMEM
+ */
+#define NVGPU_FLCN_ACR_MAX_REGIONS                (2U)
+/** Reserve 512 bytes for bootstrap owner LS ucode data */
+#define LSF_BOOTSTRAP_OWNER_RESERVED_DMEM_SIZE    (0x200U)
+
+/**
+ * The descriptor used by ACR HS ucode to figure out properties of individual
+ * WPR regions.
+ *
+ * On GV11B, this struct members are set to 0x0 by default, reason
+ * to fetch WPR1 details from H/W.
+ */
+struct flcn_acr_region_prop {
+	/** Starting address of WPR region */
+	u32 start_addr;
+	/** Ending address of WPR region */
+	u32 end_addr;
+	/** The ID of the WPR region. 0 for WPR1 and 1 for WPR2  */
+	u32 region_id;
+	/** Read mask associated with this region */
+	u32 read_mask;
+	/** Write mask associated with this region */
+	u32 write_mask;
+	/** Bit map of all clients currently using this region */
+	u32 client_mask;
+	/**
+	 * sysmem(iGPU)/FB(dGPU) location from where contents need to
+	 * be copied to startAddress
+	 */
+	u32 shadowmMem_startaddress;
+};
+
+/**
+ * The descriptor used by ACR HS ucode to figure out supporting regions &
+ * its properties.
+ */
+struct flcn_acr_regions {
+	/**
+	 * Number of regions used by NVGPU from the total number of ACR
+	 * regions supported in chip.
+	 *
+	 * On GV11B, 1 ACR region supported and should always be greater
+	 * than 0.
+	 */
+	u32 no_regions;
+	/** Region properties */
+	struct flcn_acr_region_prop region_props[NVGPU_FLCN_ACR_MAX_REGIONS];
+};
+
+#define DMEM_WORD_SIZE		4U
+#define DUMMY_SPACE_SIZE	4U
+/**
+ * The descriptor used by ACR HS ucode to figure out the
+ * WPR & non-WPR blob details.
+ */
+struct flcn_acr_desc {
+	/*
+	 * The bootstrap owner needs to switch into LS mode when bootstrapping
+	 * other LS Falcons is completed. It needs to have its own actual
+	 * DMEM image copied into DMEM as part of LS setup. If ACR desc is
+	 * at location 0, it will definitely get overwritten causing data
+	 * corruption. Hence need to reserve 0x200 bytes to give room for
+	 * any loading data.
+	 * NOTE: This has to be the first member always.
+	 */
+	union {
+		u32 reserved_dmem[(LSF_BOOTSTRAP_OWNER_RESERVED_DMEM_SIZE/DMEM_WORD_SIZE)];
+	} ucode_reserved_space;
+	/** Signature of ACR ucode. */
+	u32 signatures[FLCN_SIG_SIZE];
+	/**
+	 * WPR Region ID holding the WPR header and its details
+	 *
+	 * on GV11B, wpr_region_id set to 0x0 by default to indicate
+	 * to ACR HS ucode to fetch WPR region details from H/W &
+	 * updating WPR start_addr, end_addr, read_mask & write_mask
+	 * of struct #flcn_acr_region_prop.
+	 */
+	u32 wpr_region_id;
+	/** Offset from the non-WPR base holding the wpr header */
+	u32 wpr_offset;
+	/** usable memory ranges, on GV11B it is not set */
+	u32 mmu_mem_range;
+	/**
+	 * WPR Region descriptors to provide info about WPR.
+	 * on GV11B, no_regions set to 1 & region properties value to 0x0
+	 * to indicate to ACR HS ucode to fetch WPR region details from H/W.
+	 */
+	struct flcn_acr_regions regions;
+	/**
+	 * stores the size of the ucode blob.
+	 *
+	 * On GV11B, size is calculated at runtime & aligned to 256 bytes.
+	 * Size varies based on number of LS falcon supports.
+	 */
+	u32 nonwpr_ucode_blob_size;
+	/**
+	 * stores sysmem(iGPU)/FB's(dGPU) non-WPR start address where
+	 * kernel stores ucode blob
+	 */
+	u64 nonwpr_ucode_blob_start;
+	/** dummy space, not used by iGPU */
+	u32 dummy[DUMMY_SPACE_SIZE];
+};
+
+struct flcn2_acr_desc {
+	/**
+	 * WPR Region ID holding the WPR header and its details
+	 *
+	 * on GPUID_NEXT, wpr_region_id set to 0x0 by default to indicate
+	 * to ACR HS ucode to fetch WPR region details from H/W &
+	 * updating WPR start_addr, end_addr, read_mask & write_mask
+	 * of struct #flcn_acr_region_prop.
+	 */
+	u32 wpr_region_id;
+	/** Offset from the non-WPR base holding the wpr header */
+	u32 wpr_offset;
+	/**
+	 * WPR Region descriptors to provide info about WPR.
+	 * on GPUID_NEXT, no_regions set to 1 & region properties value to 0x0
+	 * to indicate to ACR HS ucode to fetch WPR region details from H/W.
+	 */
+	struct flcn_acr_regions regions;
+	/**
+	 * stores the size of the ucode blob.
+	 *
+	 * On GPUID_NEXT, size is calculated at runtime & aligned to 256 bytes.
+	 * Size varies based on number of LS falcon supports.
+	 */
+	u32 nonwpr_ucode_blob_size;
+	/**
+	 * stores sysmem(iGPU)/FB's(dGPU) non-WPR start address where
+	 * kernel stores ucode blob
+	 */
+	u64 nonwpr_ucode_blob_start;
+
+	u64 ls_pmu_desc;
+};
+
+/** @} */
+
+#endif /* NVGPU_ACR_INTERFACE_H */

drivers/gpu/nvgpu/common/cbc/cbc.c

@@ -0,0 +1,122 @@
+/*
+ * CBC
+ *
+ * Copyright (c) 2019-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+
+#include <nvgpu/gk20a.h>
+#include <nvgpu/cbc.h>
+#include <nvgpu/dma.h>
+#include <nvgpu/log.h>
+#include <nvgpu/nvgpu_mem.h>
+#include <nvgpu/comptags.h>
+
+void nvgpu_cbc_remove_support(struct gk20a *g)
+{
+	struct nvgpu_cbc *cbc = g->cbc;
+
+	nvgpu_log_fn(g, " ");
+
+	if (cbc == NULL) {
+		return;
+	}
+
+	if (nvgpu_mem_is_valid(&cbc->compbit_store.mem)) {
+		nvgpu_dma_free(g, &cbc->compbit_store.mem);
+		(void) memset(&cbc->compbit_store, 0,
+			sizeof(struct compbit_store_desc));
+	}
+	gk20a_comptag_allocator_destroy(g, &cbc->comp_tags);
+
+	nvgpu_kfree(g, cbc);
+	g->cbc = NULL;
+}
+
+/*
+ * This function is triggered during finalize_poweron multiple times.
+ * This function should not return if cbc is not NULL.
+ * cbc.init(), which re-writes HW registers that are reset during suspend,
+ * should be allowed to execute each time.
+ */
+int nvgpu_cbc_init_support(struct gk20a *g)
+{
+	int err = 0;
+	struct nvgpu_cbc *cbc = g->cbc;
+
+	nvgpu_log_fn(g, " ");
+
+	if (cbc == NULL) {
+		cbc = nvgpu_kzalloc(g, sizeof(*cbc));
+		if (cbc == NULL) {
+			return -ENOMEM;
+		}
+		g->cbc = cbc;
+
+		if (g->ops.cbc.alloc_comptags != NULL) {
+			err = g->ops.cbc.alloc_comptags(g, g->cbc);
+			if (err != 0) {
+				nvgpu_err(g, "Failed to allocate comptags");
+				nvgpu_kfree(g, cbc);
+				g->cbc = NULL;
+				return err;
+			}
+		}
+	}
+
+	if (g->ops.cbc.init != NULL) {
+		g->ops.cbc.init(g, g->cbc);
+	}
+
+	return err;
+}
+
+int nvgpu_cbc_alloc(struct gk20a *g, size_t compbit_backing_size,
+			bool vidmem_alloc)
+{
+	struct nvgpu_cbc *cbc = g->cbc;
+
+	if (nvgpu_mem_is_valid(&cbc->compbit_store.mem) != 0) {
+		return 0;
+	}
+
+#ifdef CONFIG_NVGPU_DGPU
+	if (vidmem_alloc == true) {
+		/*
+		 * Backing store MUST be physically contiguous and allocated in
+		 * one chunk
+		 * Vidmem allocation API does not support FORCE_CONTIGUOUS like
+		 * flag to allocate contiguous memory
+		 * But this allocation will happen in vidmem bootstrap allocator
+		 * which always allocates contiguous memory
+		 */
+		return nvgpu_dma_alloc_vid(g,
+					 compbit_backing_size,
+					 &cbc->compbit_store.mem);
+	} else
+#endif
+	{
+		return nvgpu_dma_alloc_flags_sys(g,
+					 NVGPU_DMA_PHYSICALLY_ADDRESSED,
+					 compbit_backing_size,
+					 &cbc->compbit_store.mem);
+	}
+}

drivers/gpu/nvgpu/common/ce/ce.c

@@ -0,0 +1,71 @@
+/*
+ * Copyright (c) 2018-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/types.h>
+#include <nvgpu/gk20a.h>
+#include <nvgpu/engines.h>
+#include <nvgpu/device.h>
+#include <nvgpu/ce.h>
+#include <nvgpu/power_features/cg.h>
+#include <nvgpu/cic.h>
+#include <nvgpu/mc.h>
+
+int nvgpu_ce_init_support(struct gk20a *g)
+{
+	int err = 0;
+
+	if (g->ops.ce.set_pce2lce_mapping != NULL) {
+		g->ops.ce.set_pce2lce_mapping(g);
+	}
+
+	err = nvgpu_mc_reset_devtype(g, NVGPU_DEVTYPE_LCE);
+	if (err != 0) {
+		nvgpu_err(g, "NVGPU_DEVTYPE_LCE reset failed");
+		return err;
+	}
+
+	nvgpu_cg_slcg_ce2_load_enable(g);
+
+	nvgpu_cg_blcg_ce_load_enable(g);
+
+#if defined(CONFIG_NVGPU_HAL_NON_FUSA) && defined(CONFIG_NVGPU_NEXT)
+	nvgpu_cg_elcg_ce_load_enable(g);
+#endif
+
+	if (g->ops.ce.init_prod_values != NULL) {
+		g->ops.ce.init_prod_values(g);
+	}
+
+	if (g->ops.ce.init_hw != NULL) {
+		g->ops.ce.init_hw(g);
+	}
+
+	if (g->ops.ce.intr_enable != NULL) {
+		g->ops.ce.intr_enable(g, true);
+	}
+
+	/** Enable interrupts at MC level */
+	nvgpu_cic_intr_stall_unit_config(g, NVGPU_CIC_INTR_UNIT_CE, NVGPU_CIC_INTR_ENABLE);
+	nvgpu_cic_intr_nonstall_unit_config(g, NVGPU_CIC_INTR_UNIT_CE, NVGPU_CIC_INTR_ENABLE);
+
+	return 0;
+}

drivers/gpu/nvgpu/common/ce/ce_app.c

@@ -0,0 +1,621 @@
+/*
+ * Copyright (c) 2018-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/types.h>
+#include <nvgpu/gk20a.h>
+#include <nvgpu/engines.h>
+#include <nvgpu/os_sched.h>
+#include <nvgpu/channel.h>
+#include <nvgpu/watchdog.h>
+#include <nvgpu/dma.h>
+#include <nvgpu/utils.h>
+#include <nvgpu/fence.h>
+#include <nvgpu/ce_app.h>
+#include <nvgpu/power_features/cg.h>
+
+#include "common/ce/ce_priv.h"
+
+static inline u32 nvgpu_ce_get_valid_launch_flags(struct gk20a *g,
+		u32 launch_flags)
+{
+#ifdef CONFIG_NVGPU_DGPU
+	/*
+	 * there is no local memory available,
+	 * don't allow local memory related CE flags
+	 */
+	if (g->mm.vidmem.size == 0ULL) {
+		launch_flags &= ~(NVGPU_CE_SRC_LOCATION_LOCAL_FB |
+			NVGPU_CE_DST_LOCATION_LOCAL_FB);
+	}
+#endif
+	return launch_flags;
+}
+
+int nvgpu_ce_execute_ops(struct gk20a *g,
+		u32 ce_ctx_id,
+		u64 src_paddr,
+		u64 dst_paddr,
+		u64 size,
+		u32 payload,
+		u32 launch_flags,
+		u32 request_operation,
+		u32 submit_flags,
+		struct nvgpu_fence_type **fence_out)
+{
+	int ret = -EPERM;
+	struct nvgpu_ce_app *ce_app = g->ce_app;
+	struct nvgpu_ce_gpu_ctx *ce_ctx, *ce_ctx_save;
+	bool found = false;
+	u32 *cmd_buf_cpu_va;
+	u64 cmd_buf_gpu_va = 0UL;
+	u32 method_size;
+	u32 cmd_buf_read_offset;
+	u32 dma_copy_class;
+	struct nvgpu_gpfifo_entry gpfifo;
+	struct nvgpu_channel_fence fence = {0U, 0U};
+	struct nvgpu_fence_type *ce_cmd_buf_fence_out = NULL;
+
+	if (!ce_app->initialised || ce_app->app_state != NVGPU_CE_ACTIVE) {
+		goto end;
+	}
+
+	/* This shouldn't happen */
+	if (size == 0ULL) {
+		ret = -EINVAL;
+		goto end;
+	}
+
+	if (request_operation != NVGPU_CE_PHYS_MODE_TRANSFER &&
+	    request_operation != NVGPU_CE_MEMSET) {
+		ret = -EINVAL;
+		goto end;
+	}
+
+	if (src_paddr > NVGPU_CE_MAX_ADDRESS) {
+		ret = -EINVAL;
+		goto end;
+	}
+
+	if (dst_paddr > NVGPU_CE_MAX_ADDRESS) {
+		ret = -EINVAL;
+		goto end;
+	}
+
+	nvgpu_mutex_acquire(&ce_app->app_mutex);
+
+	nvgpu_list_for_each_entry_safe(ce_ctx, ce_ctx_save,
+			&ce_app->allocated_contexts, nvgpu_ce_gpu_ctx, list) {
+		if (ce_ctx->ctx_id == ce_ctx_id) {
+			found = true;
+			break;
+		}
+	}
+
+	nvgpu_mutex_release(&ce_app->app_mutex);
+
+	if (!found) {
+		ret = -EINVAL;
+		goto end;
+	}
+
+	if (ce_ctx->gpu_ctx_state != NVGPU_CE_GPU_CTX_ALLOCATED) {
+		ret = -ENODEV;
+		goto end;
+	}
+
+	nvgpu_mutex_acquire(&ce_ctx->gpu_ctx_mutex);
+
+	ce_ctx->cmd_buf_read_queue_offset %= NVGPU_CE_MAX_INFLIGHT_JOBS;
+
+	cmd_buf_read_offset = (ce_ctx->cmd_buf_read_queue_offset *
+			(NVGPU_CE_MAX_COMMAND_BUFF_BYTES_PER_SUBMIT /
+			U32(sizeof(u32))));
+
+	cmd_buf_cpu_va = (u32 *)ce_ctx->cmd_buf_mem.cpu_va;
+
+	if (ce_ctx->postfences[ce_ctx->cmd_buf_read_queue_offset] != NULL) {
+		struct nvgpu_fence_type **prev_post_fence =
+			&ce_ctx->postfences[ce_ctx->cmd_buf_read_queue_offset];
+
+		ret = nvgpu_fence_wait(g, *prev_post_fence,
+				       nvgpu_get_poll_timeout(g));
+
+		nvgpu_fence_put(*prev_post_fence);
+		*prev_post_fence = NULL;
+		if (ret != 0) {
+			goto noop;
+		}
+	}
+
+	cmd_buf_gpu_va = (ce_ctx->cmd_buf_mem.gpu_va +
+			(u64)(cmd_buf_read_offset * sizeof(u32)));
+
+	dma_copy_class = g->ops.get_litter_value(g, GPU_LIT_DMA_COPY_CLASS);
+	method_size = nvgpu_ce_prepare_submit(src_paddr,
+			dst_paddr,
+			size,
+			&cmd_buf_cpu_va[cmd_buf_read_offset],
+			payload,
+			nvgpu_ce_get_valid_launch_flags(g, launch_flags),
+			request_operation,
+			dma_copy_class);
+	nvgpu_assert(method_size <= NVGPU_CE_MAX_COMMAND_BUFF_BYTES_PER_SUBMIT);
+
+	if (method_size != 0U) {
+		/* store the element into gpfifo */
+		g->ops.pbdma.format_gpfifo_entry(g, &gpfifo,
+				cmd_buf_gpu_va, method_size);
+
+		/*
+		 * take always the postfence as it is needed for protecting the
+		 * ce context
+		 */
+		submit_flags |= NVGPU_SUBMIT_FLAGS_FENCE_GET;
+
+		nvgpu_smp_wmb();
+
+		ret = nvgpu_submit_channel_gpfifo_kernel(ce_ctx->ch, &gpfifo,
+				1, submit_flags, &fence, &ce_cmd_buf_fence_out);
+
+		if (ret == 0) {
+			ce_ctx->postfences[ce_ctx->cmd_buf_read_queue_offset] =
+				ce_cmd_buf_fence_out;
+			if (fence_out != NULL) {
+				nvgpu_fence_get(ce_cmd_buf_fence_out);
+				*fence_out = ce_cmd_buf_fence_out;
+			}
+
+			/* Next available command buffer queue Index */
+			++ce_ctx->cmd_buf_read_queue_offset;
+		}
+	} else {
+		ret = -ENOMEM;
+	}
+noop:
+	nvgpu_mutex_release(&ce_ctx->gpu_ctx_mutex);
+end:
+	return ret;
+}
+
+/* static CE app api */
+static void nvgpu_ce_put_fences(struct nvgpu_ce_gpu_ctx *ce_ctx)
+{
+	u32 i;
+
+	for (i = 0U; i < NVGPU_CE_MAX_INFLIGHT_JOBS; i++) {
+		struct nvgpu_fence_type **fence = &ce_ctx->postfences[i];
+
+		if (*fence != NULL) {
+			nvgpu_fence_put(*fence);
+		}
+		*fence = NULL;
+	}
+}
+
+/* caller must hold ce_app->app_mutex */
+static void nvgpu_ce_delete_gpu_context_locked(struct nvgpu_ce_gpu_ctx *ce_ctx)
+{
+	struct nvgpu_list_node *list = &ce_ctx->list;
+
+	ce_ctx->gpu_ctx_state = NVGPU_CE_GPU_CTX_DELETED;
+	ce_ctx->tsg->abortable = true;
+
+	nvgpu_mutex_acquire(&ce_ctx->gpu_ctx_mutex);
+
+	if (nvgpu_mem_is_valid(&ce_ctx->cmd_buf_mem)) {
+		nvgpu_ce_put_fences(ce_ctx);
+		nvgpu_dma_unmap_free(ce_ctx->vm, &ce_ctx->cmd_buf_mem);
+	}
+
+	/*
+	 * free the channel
+	 * nvgpu_channel_close() will also unbind the channel from TSG
+	 */
+	nvgpu_channel_close(ce_ctx->ch);
+	nvgpu_ref_put(&ce_ctx->tsg->refcount, nvgpu_tsg_release);
+
+	/* housekeeping on app */
+	if ((list->prev != NULL) && (list->next != NULL)) {
+		nvgpu_list_del(list);
+	}
+
+	nvgpu_mutex_release(&ce_ctx->gpu_ctx_mutex);
+	nvgpu_mutex_destroy(&ce_ctx->gpu_ctx_mutex);
+
+	nvgpu_kfree(ce_ctx->g, ce_ctx);
+}
+
+static u32 nvgpu_prepare_ce_op(u32 *cmd_buf_cpu_va,
+		u64 src_paddr, u64 dst_paddr,
+		u32 width, u32 height, u32 payload,
+		bool mode_transfer, u32 launch_flags)
+{
+	u32 launch = 0U;
+	u32 methodSize = 0U;
+
+	if (mode_transfer) {
+		/* setup the source */
+		cmd_buf_cpu_va[methodSize++] = 0x20028100;
+		cmd_buf_cpu_va[methodSize++] = (u64_hi32(src_paddr) &
+			NVGPU_CE_UPPER_ADDRESS_OFFSET_MASK);
+		cmd_buf_cpu_va[methodSize++] = (u64_lo32(src_paddr) &
+			NVGPU_CE_LOWER_ADDRESS_OFFSET_MASK);
+
+		cmd_buf_cpu_va[methodSize++] = 0x20018098;
+		if ((launch_flags &
+		     NVGPU_CE_SRC_LOCATION_LOCAL_FB) != 0U) {
+			cmd_buf_cpu_va[methodSize++] = 0x00000000;
+		} else if ((launch_flags &
+		     NVGPU_CE_SRC_LOCATION_NONCOHERENT_SYSMEM) != 0U) {
+			cmd_buf_cpu_va[methodSize++] = 0x00000002;
+		} else {
+			cmd_buf_cpu_va[methodSize++] = 0x00000001;
+		}
+
+		launch |= 0x00001000U;
+	} else { /* memset */
+		/* Remap from component A on 1 byte wide pixels */
+		cmd_buf_cpu_va[methodSize++] = 0x200181c2;
+		cmd_buf_cpu_va[methodSize++] = 0x00000004;
+
+		cmd_buf_cpu_va[methodSize++] = 0x200181c0;
+		cmd_buf_cpu_va[methodSize++] = payload;
+
+		launch |= 0x00000400U;
+	}
+
+	/* setup the destination/output */
+	cmd_buf_cpu_va[methodSize++] = 0x20068102;
+	cmd_buf_cpu_va[methodSize++] = (u64_hi32(dst_paddr) &
+		NVGPU_CE_UPPER_ADDRESS_OFFSET_MASK);
+	cmd_buf_cpu_va[methodSize++] = (u64_lo32(dst_paddr) &
+		NVGPU_CE_LOWER_ADDRESS_OFFSET_MASK);
+	/* Pitch in/out */
+	cmd_buf_cpu_va[methodSize++] = width;
+	cmd_buf_cpu_va[methodSize++] = width;
+	/* width and line count */
+	cmd_buf_cpu_va[methodSize++] = width;
+	cmd_buf_cpu_va[methodSize++] = height;
+
+	cmd_buf_cpu_va[methodSize++] = 0x20018099;
+	if ((launch_flags & NVGPU_CE_DST_LOCATION_LOCAL_FB) != 0U) {
+		cmd_buf_cpu_va[methodSize++] = 0x00000000;
+	} else if ((launch_flags &
+		   NVGPU_CE_DST_LOCATION_NONCOHERENT_SYSMEM) != 0U) {
+		cmd_buf_cpu_va[methodSize++] = 0x00000002;
+	} else {
+		cmd_buf_cpu_va[methodSize++] = 0x00000001;
+	}
+
+	launch |= 0x00002005U;
+
+	if ((launch_flags &
+	     NVGPU_CE_SRC_MEMORY_LAYOUT_BLOCKLINEAR) != 0U) {
+		launch |= 0x00000000U;
+	} else {
+		launch |= 0x00000080U;
+	}
+
+	if ((launch_flags &
+	     NVGPU_CE_DST_MEMORY_LAYOUT_BLOCKLINEAR) != 0U) {
+		launch |= 0x00000000U;
+	} else {
+		launch |= 0x00000100U;
+	}
+
+	cmd_buf_cpu_va[methodSize++] = 0x200180c0;
+	cmd_buf_cpu_va[methodSize++] = launch;
+
+	return methodSize;
+}
+
+u32 nvgpu_ce_prepare_submit(u64 src_paddr,
+		u64 dst_paddr,
+		u64 size,
+		u32 *cmd_buf_cpu_va,
+		u32 payload,
+		u32 launch_flags,
+		u32 request_operation,
+		u32 dma_copy_class)
+{
+	u32 methodSize = 0;
+	u64 low, hi;
+	bool mode_transfer = (request_operation == NVGPU_CE_PHYS_MODE_TRANSFER);
+
+	/* set the channel object */
+	cmd_buf_cpu_va[methodSize++] = 0x20018000;
+	cmd_buf_cpu_va[methodSize++] = dma_copy_class;
+
+	/*
+	 * The CE can work with 2D rectangles of at most 0xffffffff or 4G-1
+	 * pixels per line. Exactly 2G is a more round number, so we'll use
+	 * that as the base unit to clear large amounts of memory. If the
+	 * requested size is not a multiple of 2G, we'll do one clear first to
+	 * deal with the low bits, followed by another in units of 2G.
+	 *
+	 * We'll use 1 bytes per pixel to do byte aligned sets/copies. The
+	 * maximum number of lines is also 4G-1, so (4G-1) * 2 GB is enough for
+	 * whole vidmem.
+	 */
+
+	/* Lower 2GB */
+	low = size & 0x7fffffffULL;
+	/* Over 2GB */
+	hi  = size >> 31U;
+
+	/*
+	 * Unable to fit this in one submit, but no device should have this
+	 * much memory anyway.
+	 */
+	if (hi > 0xffffffffULL) {
+		/* zero size means error */
+		return 0;
+	}
+
+	if (low != 0U) {
+		/* do the low bytes in one long line */
+		methodSize += nvgpu_prepare_ce_op(&cmd_buf_cpu_va[methodSize],
+				src_paddr, dst_paddr,
+				nvgpu_safe_cast_u64_to_u32(low), 1,
+				payload, mode_transfer, launch_flags);
+	}
+	if (hi != 0U) {
+		/* do the high bytes in many 2G lines */
+		methodSize += nvgpu_prepare_ce_op(&cmd_buf_cpu_va[methodSize],
+				src_paddr + low, dst_paddr + low,
+				0x80000000ULL, nvgpu_safe_cast_u64_to_u32(hi),
+				payload, mode_transfer, launch_flags);
+	}
+
+	return methodSize;
+}
+
+/* global CE app related apis */
+int nvgpu_ce_app_init_support(struct gk20a *g)
+{
+	struct nvgpu_ce_app *ce_app = g->ce_app;
+
+	if (unlikely(ce_app == NULL)) {
+		ce_app = nvgpu_kzalloc(g, sizeof(*ce_app));
+		if (ce_app == NULL) {
+			return -ENOMEM;
+		}
+		g->ce_app = ce_app;
+	}
+
+	if (ce_app->initialised) {
+		/* assume this happen during poweron/poweroff GPU sequence */
+		ce_app->app_state = NVGPU_CE_ACTIVE;
+		return 0;
+	}
+
+	nvgpu_log(g, gpu_dbg_fn, "ce: init");
+
+	nvgpu_mutex_init(&ce_app->app_mutex);
+
+	nvgpu_mutex_acquire(&ce_app->app_mutex);
+
+	nvgpu_init_list_node(&ce_app->allocated_contexts);
+	ce_app->ctx_count = 0;
+	ce_app->next_ctx_id = 0;
+	ce_app->initialised = true;
+	ce_app->app_state = NVGPU_CE_ACTIVE;
+
+	nvgpu_mutex_release(&ce_app->app_mutex);
+
+	nvgpu_log(g, gpu_dbg_cde_ctx, "ce: init finished");
+
+	return 0;
+}
+
+void nvgpu_ce_app_destroy(struct gk20a *g)
+{
+	struct nvgpu_ce_app *ce_app = g->ce_app;
+	struct nvgpu_ce_gpu_ctx *ce_ctx, *ce_ctx_save;
+
+	if (ce_app == NULL) {
+		return;
+	}
+
+	if (ce_app->initialised == false) {
+		goto free;
+	}
+
+	ce_app->app_state = NVGPU_CE_SUSPEND;
+	ce_app->initialised = false;
+
+	nvgpu_mutex_acquire(&ce_app->app_mutex);
+
+	nvgpu_list_for_each_entry_safe(ce_ctx, ce_ctx_save,
+			&ce_app->allocated_contexts, nvgpu_ce_gpu_ctx, list) {
+		nvgpu_ce_delete_gpu_context_locked(ce_ctx);
+	}
+
+	nvgpu_init_list_node(&ce_app->allocated_contexts);
+	ce_app->ctx_count = 0;
+	ce_app->next_ctx_id = 0;
+
+	nvgpu_mutex_release(&ce_app->app_mutex);
+
+	nvgpu_mutex_destroy(&ce_app->app_mutex);
+free:
+	nvgpu_kfree(g, ce_app);
+	g->ce_app = NULL;
+}
+
+void nvgpu_ce_app_suspend(struct gk20a *g)
+{
+	struct nvgpu_ce_app *ce_app = g->ce_app;
+
+	if (ce_app == NULL || !ce_app->initialised) {
+		return;
+	}
+
+	ce_app->app_state = NVGPU_CE_SUSPEND;
+}
+
+/* CE app utility functions */
+u32 nvgpu_ce_app_create_context(struct gk20a *g,
+		u32 runlist_id,
+		int timeslice,
+		int runlist_level)
+{
+	struct nvgpu_ce_gpu_ctx *ce_ctx;
+	struct nvgpu_ce_app *ce_app = g->ce_app;
+	struct nvgpu_setup_bind_args setup_bind_args;
+	u32 ctx_id = NVGPU_CE_INVAL_CTX_ID;
+	int err = 0;
+
+	if (!ce_app->initialised || ce_app->app_state != NVGPU_CE_ACTIVE) {
+		return ctx_id;
+	}
+
+	ce_ctx = nvgpu_kzalloc(g, sizeof(*ce_ctx));
+	if (ce_ctx == NULL) {
+		return ctx_id;
+	}
+
+	nvgpu_mutex_init(&ce_ctx->gpu_ctx_mutex);
+
+	ce_ctx->g = g;
+	ce_ctx->cmd_buf_read_queue_offset = 0;
+	ce_ctx->vm = g->mm.ce.vm;
+
+	/* allocate a tsg if needed */
+	ce_ctx->tsg = nvgpu_tsg_open(g, nvgpu_current_pid(g));
+	if (ce_ctx->tsg == NULL) {
+		nvgpu_err(g, "ce: gk20a tsg not available");
+		goto end;
+	}
+
+	/* this TSG should never be aborted */
+	ce_ctx->tsg->abortable = false;
+
+	/* always kernel client needs privileged channel */
+	ce_ctx->ch = nvgpu_channel_open_new(g, runlist_id, true,
+				nvgpu_current_pid(g), nvgpu_current_tid(g));
+	if (ce_ctx->ch == NULL) {
+		nvgpu_err(g, "ce: gk20a channel not available");
+		goto end;
+	}
+
+	nvgpu_channel_wdt_disable(ce_ctx->ch->wdt);
+
+	/* bind the channel to the vm */
+	err = g->ops.mm.vm_bind_channel(g->mm.ce.vm, ce_ctx->ch);
+	if (err != 0) {
+		nvgpu_err(g, "ce: could not bind vm");
+		goto end;
+	}
+
+	err = nvgpu_tsg_bind_channel(ce_ctx->tsg, ce_ctx->ch);
+	if (err != 0) {
+		nvgpu_err(g, "ce: unable to bind to tsg");
+		goto end;
+	}
+
+	setup_bind_args.num_gpfifo_entries = 1024;
+	setup_bind_args.num_inflight_jobs = 0;
+	setup_bind_args.flags = 0;
+	err = nvgpu_channel_setup_bind(ce_ctx->ch, &setup_bind_args);
+	if (err != 0) {
+		nvgpu_err(g, "ce: unable to setup and bind channel");
+		goto end;
+	}
+
+	/* allocate command buffer from sysmem */
+	err = nvgpu_dma_alloc_map_sys(ce_ctx->vm,
+			NVGPU_CE_MAX_INFLIGHT_JOBS *
+			NVGPU_CE_MAX_COMMAND_BUFF_BYTES_PER_SUBMIT,
+			&ce_ctx->cmd_buf_mem);
+	if (err != 0) {
+		nvgpu_err(g,
+			"ce: alloc command buffer failed");
+		goto end;
+	}
+
+	(void) memset(ce_ctx->cmd_buf_mem.cpu_va, 0x00,
+		ce_ctx->cmd_buf_mem.size);
+
+#ifdef CONFIG_NVGPU_CHANNEL_TSG_SCHEDULING
+	/* -1 means default channel timeslice value */
+	if (timeslice != -1) {
+		err = g->ops.tsg.set_timeslice(ce_ctx->tsg, timeslice);
+		if (err != 0) {
+			nvgpu_err(g, "ce: set timesliced failed for CE context");
+			goto end;
+		}
+	}
+
+	/* -1 means default channel runlist level */
+	if (runlist_level != -1) {
+		err = nvgpu_tsg_set_interleave(ce_ctx->tsg, runlist_level);
+		if (err != 0) {
+			nvgpu_err(g, "ce: set runlist interleave failed");
+			goto end;
+		}
+	}
+#endif
+
+	nvgpu_mutex_acquire(&ce_app->app_mutex);
+	ctx_id = ce_ctx->ctx_id = ce_app->next_ctx_id;
+	nvgpu_list_add(&ce_ctx->list, &ce_app->allocated_contexts);
+	++ce_app->next_ctx_id;
+	++ce_app->ctx_count;
+	nvgpu_mutex_release(&ce_app->app_mutex);
+
+	ce_ctx->gpu_ctx_state = NVGPU_CE_GPU_CTX_ALLOCATED;
+
+end:
+	if (ctx_id == NVGPU_CE_INVAL_CTX_ID) {
+		nvgpu_mutex_acquire(&ce_app->app_mutex);
+		nvgpu_ce_delete_gpu_context_locked(ce_ctx);
+		nvgpu_mutex_release(&ce_app->app_mutex);
+	}
+	return ctx_id;
+
+}
+
+void nvgpu_ce_app_delete_context(struct gk20a *g,
+		u32 ce_ctx_id)
+{
+	struct nvgpu_ce_app *ce_app = g->ce_app;
+	struct nvgpu_ce_gpu_ctx *ce_ctx, *ce_ctx_save;
+
+	if (ce_app == NULL || !ce_app->initialised ||
+		ce_app->app_state != NVGPU_CE_ACTIVE) {
+		return;
+	}
+
+	nvgpu_mutex_acquire(&ce_app->app_mutex);
+
+	nvgpu_list_for_each_entry_safe(ce_ctx, ce_ctx_save,
+			&ce_app->allocated_contexts, nvgpu_ce_gpu_ctx, list) {
+		if (ce_ctx->ctx_id == ce_ctx_id) {
+			nvgpu_ce_delete_gpu_context_locked(ce_ctx);
+			--ce_app->ctx_count;
+			break;
+		}
+	}
+
+	nvgpu_mutex_release(&ce_app->app_mutex);
+}

drivers/gpu/nvgpu/common/ce/ce_priv.h

@@ -0,0 +1,82 @@
+/*
+ * Copyright (c) 2011-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+#ifndef NVGPU_CE_PRIV_H
+#define NVGPU_CE_PRIV_H
+
+#include <nvgpu/types.h>
+#include <nvgpu/nvgpu_mem.h>
+#include <nvgpu/list.h>
+#include <nvgpu/lock.h>
+
+struct gk20a;
+
+/* ce context db */
+struct nvgpu_ce_gpu_ctx {
+	struct gk20a *g;
+	u32 ctx_id;
+	struct nvgpu_mutex gpu_ctx_mutex;
+	int gpu_ctx_state;
+
+	/* tsg related data */
+	struct nvgpu_tsg *tsg;
+
+	/* channel related data */
+	struct nvgpu_channel *ch;
+	struct vm_gk20a *vm;
+
+	/* cmd buf mem_desc */
+	struct nvgpu_mem cmd_buf_mem;
+	struct nvgpu_fence_type *postfences[NVGPU_CE_MAX_INFLIGHT_JOBS];
+
+	struct nvgpu_list_node list;
+
+	u32 cmd_buf_read_queue_offset;
+};
+
+/* global ce app db */
+struct nvgpu_ce_app {
+	bool initialised;
+	struct nvgpu_mutex app_mutex;
+	int app_state;
+
+	struct nvgpu_list_node allocated_contexts;
+	u32 ctx_count;
+	u32 next_ctx_id;
+};
+
+static inline struct nvgpu_ce_gpu_ctx *
+nvgpu_ce_gpu_ctx_from_list(struct nvgpu_list_node *node)
+{
+	return (struct nvgpu_ce_gpu_ctx *)
+		((uintptr_t)node - offsetof(struct nvgpu_ce_gpu_ctx, list));
+};
+
+u32 nvgpu_ce_prepare_submit(u64 src_paddr,
+		u64 dst_paddr,
+		u64 size,
+		u32 *cmd_buf_cpu_va,
+		u32 payload,
+		u32 launch_flags,
+		u32 request_operation,
+		u32 dma_copy_class);
+
+#endif /*NVGPU_CE_PRIV_H*/

drivers/gpu/nvgpu/common/cic/ce_cic.c

@@ -0,0 +1,91 @@
+/*
+ * Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/gk20a.h>
+#include <nvgpu/nvgpu_init.h>
+#include <nvgpu/nvgpu_err.h>
+#include <nvgpu/nvgpu_err_info.h>
+#include <nvgpu/cic.h>
+
+#include "cic_priv.h"
+
+void nvgpu_report_ce_err(struct gk20a *g, u32 hw_unit,
+		u32 inst, u32 err_id, u32 intr_info)
+{
+	int err = 0;
+	struct nvgpu_err_desc *err_desc = NULL;
+	struct nvgpu_err_msg err_pkt;
+
+	if (g->ops.cic.report_err == NULL) {
+		cic_dbg(g, "CIC does not support reporting error "
+			       "to safety services");
+		return;
+	}
+
+	if (hw_unit != NVGPU_ERR_MODULE_CE) {
+		nvgpu_err(g, "invalid hw module (%u)", hw_unit);
+		err = -EINVAL;
+		goto handle_report_failure;
+	}
+
+	err = nvgpu_cic_get_err_desc(g, hw_unit, err_id, &err_desc);
+	if (err != 0) {
+		nvgpu_err(g, "Failed to get err_desc for "
+				"err_id (%u) for hw module (%u)",
+				err_id, hw_unit);
+		goto handle_report_failure;
+	}
+
+	nvgpu_init_ce_err_msg(&err_pkt);
+	err_pkt.hw_unit_id = hw_unit;
+	err_pkt.err_id = err_desc->error_id;
+	err_pkt.is_critical = err_desc->is_critical;
+	err_pkt.err_info.ce_info.header.sub_unit_id = inst;
+	err_pkt.err_desc = err_desc;
+	/* sub_err_type can be decoded using intr_info by referring
+	 * to the interrupt status register definition corresponding
+	 * to the error that is being reported.
+	 */
+	err_pkt.err_info.ce_info.header.sub_err_type = intr_info;
+	err_pkt.err_size = nvgpu_safe_cast_u64_to_u8(
+			sizeof(err_pkt.err_info.ce_info));
+
+	if (g->ops.cic.report_err != NULL) {
+		err = g->ops.cic.report_err(g, (void *)&err_pkt,
+			sizeof(err_pkt), err_desc->is_critical);
+		if (err != 0) {
+			nvgpu_err(g, "Failed to report CE error: "
+				"inst=%u err_id=%u intr_info=%u",
+				inst, err_id, intr_info);
+		}
+	}
+handle_report_failure:
+	if (err != 0) {
+		nvgpu_sw_quiesce(g);
+	}
+}
+
+void nvgpu_inject_ce_swerror(struct gk20a *g, u32 hw_unit,
+		u32 err_index, u32 sub_err_type)
+{
+	nvgpu_report_ce_err(g, hw_unit, 0U, err_index, sub_err_type);
+}

drivers/gpu/nvgpu/common/cic/cic.c

@@ -0,0 +1,161 @@
+/*
+ * Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/gk20a.h>
+#include <nvgpu/kmem.h>
+#include <nvgpu/log.h>
+#include <nvgpu/cic.h>
+#include <nvgpu/nvgpu_err_info.h>
+
+#include "cic_priv.h"
+
+int nvgpu_cic_init_common(struct gk20a *g)
+{
+	struct nvgpu_cic *cic;
+	int err = 0;
+
+	if (g->cic != NULL) {
+		cic_dbg(g, "CIC unit already initialized");
+		return 0;
+	}
+
+	cic = nvgpu_kzalloc(g, sizeof(*cic));
+	if (cic == NULL) {
+		nvgpu_err(g, "Failed to allocate memory "
+				"for struct nvgpu_cic");
+		return -ENOMEM;
+	}
+
+	if (g->ops.cic.init != NULL) {
+		err = g->ops.cic.init(g, cic);
+		if (err != 0) {
+			nvgpu_err(g, "CIC chip specific "
+					"initialization failed.");
+			goto cleanup;
+		}
+	} else {
+		cic->err_lut = NULL;
+		cic->num_hw_modules = 0;
+	}
+
+	g->cic = cic;
+	cic_dbg(g, "CIC unit initialization done.");
+	return 0;
+
+cleanup:
+	if (cic != NULL) {
+		nvgpu_kfree(g, cic);
+	}
+	return err;
+}
+
+int nvgpu_cic_deinit_common(struct gk20a *g)
+{
+	struct nvgpu_cic *cic;
+
+	cic = g->cic;
+
+	if (cic == NULL) {
+		cic_dbg(g, "CIC unit already deinitialized");
+		return 0;
+	}
+
+	cic->err_lut = NULL;
+	cic->num_hw_modules = 0;
+
+	nvgpu_kfree(g, cic);
+	g->cic = NULL;
+
+	return 0;
+}
+
+int nvgpu_cic_check_hw_unit_id(struct gk20a *g, u32 hw_unit_id)
+{
+	if (g->cic == NULL) {
+		nvgpu_err(g, "CIC is not initialized");
+		return -EINVAL;
+	}
+
+	if (g->cic->num_hw_modules == 0U) {
+		cic_dbg(g, "LUT not initialized.");
+		return -EINVAL;
+	}
+
+	if (hw_unit_id >= g->cic->num_hw_modules) {
+		cic_dbg(g, "Invalid input HW unit ID.");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+int nvgpu_cic_check_err_id(struct gk20a *g, u32 hw_unit_id,
+		u32 err_id)
+{
+	int err = 0;
+
+	if ((g->cic == NULL) || (g->cic->err_lut == NULL)) {
+		cic_dbg(g, "CIC/LUT not initialized.");
+		return -EINVAL;
+	}
+
+	err = nvgpu_cic_check_hw_unit_id(g, hw_unit_id);
+	if (err != 0) {
+		return err;
+	}
+
+	if (err_id >= g->cic->err_lut[hw_unit_id].num_errs) {
+		err = -EINVAL;
+	}
+
+	return err;
+}
+
+int nvgpu_cic_get_err_desc(struct gk20a *g, u32 hw_unit_id,
+		u32 err_id, struct nvgpu_err_desc **err_desc)
+{
+	int err = 0;
+
+	/* if (g->cic != NULL) and (g->cic->err_lut != NULL) check
+	 * can be skipped here as it checked as part of
+	 * nvgpu_cic_check_err_id() called below.
+	 */
+
+	err = nvgpu_cic_check_err_id(g, hw_unit_id, err_id);
+	if (err != 0) {
+		return err;
+	}
+
+	*err_desc = &(g->cic->err_lut[hw_unit_id].errs[err_id]);
+
+	return err;
+}
+
+int nvgpu_cic_get_num_hw_modules(struct gk20a *g)
+{
+	if (g->cic == NULL) {
+		nvgpu_err(g, "CIC is not initialized");
+		return -EINVAL;
+	}
+
+	return g->cic->num_hw_modules;
+}

drivers/gpu/nvgpu/common/cic/cic_intr.c

@@ -0,0 +1,251 @@
+/*
+ * Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/cic.h>
+#include <nvgpu/gk20a.h>
+#include <nvgpu/bug.h>
+#include <nvgpu/nvgpu_init.h>
+#include <nvgpu/trace.h>
+
+int nvgpu_cic_wait_for_stall_interrupts(struct gk20a *g, u32 timeout)
+{
+	/* wait until all stalling irqs are handled */
+	return NVGPU_COND_WAIT(&g->mc.sw_irq_stall_last_handled_cond,
+			nvgpu_atomic_read(&g->mc.sw_irq_stall_pending) == 0,
+			timeout);
+}
+
+int nvgpu_cic_wait_for_nonstall_interrupts(struct gk20a *g, u32 timeout)
+{
+	/* wait until all non-stalling irqs are handled */
+	return NVGPU_COND_WAIT(&g->mc.sw_irq_nonstall_last_handled_cond,
+			nvgpu_atomic_read(&g->mc.sw_irq_nonstall_pending) == 0,
+			timeout);
+}
+
+void nvgpu_cic_wait_for_deferred_interrupts(struct gk20a *g)
+{
+	int ret;
+
+	ret = nvgpu_cic_wait_for_stall_interrupts(g, 0U);
+	if (ret != 0) {
+		nvgpu_err(g, "wait for stall interrupts failed %d", ret);
+	}
+
+	ret = nvgpu_cic_wait_for_nonstall_interrupts(g, 0U);
+	if (ret != 0) {
+		nvgpu_err(g, "wait for nonstall interrupts failed %d", ret);
+	}
+}
+
+void nvgpu_cic_intr_mask(struct gk20a *g)
+{
+	unsigned long flags = 0;
+
+	if (g->ops.mc.intr_mask != NULL) {
+		nvgpu_spinlock_irqsave(&g->mc.intr_lock, flags);
+		g->ops.mc.intr_mask(g);
+		nvgpu_spinunlock_irqrestore(&g->mc.intr_lock, flags);
+	}
+}
+
+#ifdef CONFIG_NVGPU_NON_FUSA
+void nvgpu_cic_log_pending_intrs(struct gk20a *g)
+{
+	if (g->ops.mc.log_pending_intrs != NULL) {
+		g->ops.mc.log_pending_intrs(g);
+	}
+}
+
+void nvgpu_cic_intr_enable(struct gk20a *g)
+{
+	unsigned long flags = 0;
+
+	if (g->ops.mc.intr_enable != NULL) {
+		nvgpu_spinlock_irqsave(&g->mc.intr_lock, flags);
+		g->ops.mc.intr_enable(g);
+		nvgpu_spinunlock_irqrestore(&g->mc.intr_lock, flags);
+	}
+}
+#endif
+
+void nvgpu_cic_intr_stall_unit_config(struct gk20a *g, u32 unit, bool enable)
+{
+	unsigned long flags = 0;
+
+	nvgpu_spinlock_irqsave(&g->mc.intr_lock, flags);
+	g->ops.mc.intr_stall_unit_config(g, unit, enable);
+	nvgpu_spinunlock_irqrestore(&g->mc.intr_lock, flags);
+}
+
+void nvgpu_cic_intr_nonstall_unit_config(struct gk20a *g, u32 unit, bool enable)
+{
+	unsigned long flags = 0;
+
+	nvgpu_spinlock_irqsave(&g->mc.intr_lock, flags);
+	g->ops.mc.intr_nonstall_unit_config(g, unit, enable);
+	nvgpu_spinunlock_irqrestore(&g->mc.intr_lock, flags);
+}
+
+void nvgpu_cic_intr_stall_pause(struct gk20a *g)
+{
+	unsigned long flags = 0;
+
+	nvgpu_spinlock_irqsave(&g->mc.intr_lock, flags);
+	g->ops.mc.intr_stall_pause(g);
+	nvgpu_spinunlock_irqrestore(&g->mc.intr_lock, flags);
+}
+
+void nvgpu_cic_intr_stall_resume(struct gk20a *g)
+{
+	unsigned long flags = 0;
+
+	nvgpu_spinlock_irqsave(&g->mc.intr_lock, flags);
+	g->ops.mc.intr_stall_resume(g);
+	nvgpu_spinunlock_irqrestore(&g->mc.intr_lock, flags);
+}
+
+void nvgpu_cic_intr_nonstall_pause(struct gk20a *g)
+{
+	unsigned long flags = 0;
+
+	nvgpu_spinlock_irqsave(&g->mc.intr_lock, flags);
+	g->ops.mc.intr_nonstall_pause(g);
+	nvgpu_spinunlock_irqrestore(&g->mc.intr_lock, flags);
+}
+
+void nvgpu_cic_intr_nonstall_resume(struct gk20a *g)
+{
+	unsigned long flags = 0;
+
+	nvgpu_spinlock_irqsave(&g->mc.intr_lock, flags);
+	g->ops.mc.intr_nonstall_resume(g);
+	nvgpu_spinunlock_irqrestore(&g->mc.intr_lock, flags);
+}
+
+static void nvgpu_cic_intr_nonstall_work(struct gk20a *g, u32 work_ops)
+{
+	bool semaphore_wakeup, post_events;
+
+	semaphore_wakeup =
+		(((work_ops & NVGPU_CIC_NONSTALL_OPS_WAKEUP_SEMAPHORE) != 0U) ?
+					true : false);
+	post_events = (((work_ops & NVGPU_CIC_NONSTALL_OPS_POST_EVENTS) != 0U) ?
+					true : false);
+
+	if (semaphore_wakeup) {
+		g->ops.semaphore_wakeup(g, post_events);
+	}
+}
+
+u32 nvgpu_cic_intr_nonstall_isr(struct gk20a *g)
+{
+	u32 non_stall_intr_val = 0U;
+
+	if (nvgpu_is_powered_off(g)) {
+		return NVGPU_CIC_INTR_UNMASK;
+	}
+
+	/* not from gpu when sharing irq with others */
+	non_stall_intr_val = g->ops.mc.intr_nonstall(g);
+	if (non_stall_intr_val == 0U) {
+		return NVGPU_CIC_INTR_NONE;
+	}
+
+	nvgpu_cic_intr_nonstall_pause(g);
+	if (g->sw_quiesce_pending) {
+		return NVGPU_CIC_INTR_QUIESCE_PENDING;
+	}
+
+	nvgpu_atomic_set(&g->mc.sw_irq_nonstall_pending, 1);
+	return NVGPU_CIC_INTR_HANDLE;
+}
+
+void nvgpu_cic_intr_nonstall_handle(struct gk20a *g)
+{
+	int err;
+	u32 nonstall_ops = 0;
+
+	nonstall_ops = g->ops.mc.isr_nonstall(g);
+	if (nonstall_ops != 0U) {
+		nvgpu_cic_intr_nonstall_work(g, nonstall_ops);
+	}
+
+	/* sync handled irq counter before re-enabling interrupts */
+	nvgpu_atomic_set(&g->mc.sw_irq_nonstall_pending, 0);
+
+	nvgpu_cic_intr_nonstall_resume(g);
+
+	err = nvgpu_cond_broadcast(&g->mc.sw_irq_nonstall_last_handled_cond);
+	if (err != 0) {
+		nvgpu_err(g, "nvgpu_cond_broadcast failed err=%d", err);
+	}
+}
+
+u32 nvgpu_cic_intr_stall_isr(struct gk20a *g)
+{
+	u32 mc_intr_0 = 0U;
+
+	nvgpu_trace_intr_stall_start(g);
+
+	if (nvgpu_is_powered_off(g)) {
+		return NVGPU_CIC_INTR_UNMASK;
+	}
+
+	/* not from gpu when sharing irq with others */
+	mc_intr_0 = g->ops.mc.intr_stall(g);
+	if (mc_intr_0 == 0U) {
+		return NVGPU_CIC_INTR_NONE;
+	}
+
+	nvgpu_cic_intr_stall_pause(g);
+
+	if (g->sw_quiesce_pending) {
+		return NVGPU_CIC_INTR_QUIESCE_PENDING;
+	}
+
+	nvgpu_atomic_set(&g->mc.sw_irq_stall_pending, 1);
+
+	nvgpu_trace_intr_stall_done(g);
+
+	return NVGPU_CIC_INTR_HANDLE;
+}
+
+void nvgpu_cic_intr_stall_handle(struct gk20a *g)
+{
+	int err;
+
+	nvgpu_trace_intr_thread_stall_start(g);
+
+	g->ops.mc.isr_stall(g);
+
+	nvgpu_trace_intr_thread_stall_done(g);
+
+	/* sync handled irq counter before re-enabling interrupts */
+	nvgpu_atomic_set(&g->mc.sw_irq_stall_pending, 0);
+	nvgpu_cic_intr_stall_resume(g);
+
+	err = nvgpu_cond_broadcast(&g->mc.sw_irq_stall_last_handled_cond);
+	if (err != 0) {
+		nvgpu_err(g, "nvgpu_cond_broadcast failed err=%d", err);
+	}
+}

drivers/gpu/nvgpu/common/cic/cic_priv.h

@@ -0,0 +1,291 @@
+/*
+ * Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef CIC_PRIV_H
+#define CIC_PRIV_H
+
+#include <nvgpu/types.h>
+
+struct gk20a;
+struct nvgpu_err_hw_module;
+struct nvgpu_err_msg;
+struct gpu_err_header;
+
+/*
+ * @file
+ *
+ * Declare CIC's private structure to store error-policy LUT and
+ * other data and ops needed during error reporting.
+ */
+
+#define ERR_INJECT_TEST_PATTERN 0xA5
+
+/*
+ * This struct contains members related to error-policy look-up table,
+ * number of units reporting errors.
+ */
+struct nvgpu_cic {
+	/** Pointer for error look-up table. */
+	struct nvgpu_err_hw_module *err_lut;
+
+	/** Total number of GPU HW modules considered in CIC. */
+	u32 num_hw_modules;
+
+};
+
+/**
+ * @brief Inject ECC error.
+ *
+ * @param g [in]		- The GPU driver struct.
+ * @param hw_unit [in]		- Index of HW unit.
+ * @param err_index [in]	- Error index.
+ * @param inst [in]		- Instance ID.
+ *
+ *  - Sets values for error address and error count.
+ *  - Invokes error reporting API with the required set of inputs.
+ *
+ * @return None
+ */
+void nvgpu_inject_ecc_swerror(struct gk20a *g, u32 hw_unit,
+		u32 err_index, u32 inst);
+
+/**
+ * @brief Inject HOST error.
+ *
+ * @param g [in]		- The GPU driver struct.
+ * @param hw_unit [in]		- Index of HW unit.
+ * @param err_index [in]	- Error index.
+ * @param sub_err_type [in]	- Sub error type.
+ *
+ *  - Invokes error reporting API with the required set of inputs.
+ *
+ * @return None
+ */
+void nvgpu_inject_host_swerror(struct gk20a *g, u32 hw_unit,
+		u32 err_index, u32 sub_err_type);
+
+/**
+ * @brief Inject GR error.
+ *
+ * @param g [in]		- The GPU driver struct.
+ * @param hw_unit [in]		- Index of HW unit.
+ * @param err_index [in]	- Error index.
+ * @param sub_err_type [in]	- Sub error type.
+ *
+ *  - Sets values for GR exception and SM machine check error information.
+ *  - Invokes error reporting API with the required set of inputs.
+ *
+ * @return None
+ */
+void nvgpu_inject_gr_swerror(struct gk20a *g, u32 hw_unit,
+		u32 err_index, u32 sub_err_type);
+
+/**
+ * @brief Inject CE error.
+ *
+ * @param g [in]		- The GPU driver struct.
+ * @param hw_unit [in]		- Index of HW unit.
+ * @param err_index [in]	- Error index.
+ * @param sub_err_type [in]	- Sub error type.
+ *
+ *  - Invokes error reporting API with the required set of inputs.
+ *
+ * @return None
+ */
+void nvgpu_inject_ce_swerror(struct gk20a *g, u32 hw_unit,
+		u32 err_index, u32 sub_err_type);
+
+/**
+ * @brief Inject CE error.
+ *
+ * @param g [in]		- The GPU driver struct.
+ * @param hw_unit [in]		- Index of HW unit.
+ * @param err_index [in]	- Error index.
+ * @param err_code [in]		- Error code.
+ *
+ *  - Invokes error reporting API with the required set of inputs.
+ *
+ * @return None
+ */
+void nvgpu_inject_pri_swerror(struct gk20a *g, u32 hw_unit,
+		u32 err_index, u32 err_code);
+
+/**
+ * @brief Inject PMU error.
+ *
+ * @param g [in]		- The GPU driver struct.
+ * @param hw_unit [in]		- Index of HW unit.
+ * @param err_index [in]	- Error index.
+ * @param sub_err_type [in]	- Sub error type.
+ *
+ *  - Sets values for error info.
+ *  - Invokes error reporting API with the required set of inputs.
+ *
+ * @return None
+ */
+void nvgpu_inject_pmu_swerror(struct gk20a *g, u32 hw_unit,
+		u32 err_index, u32 sub_err_type);
+
+/**
+ * @brief Inject CTXSW error.
+ *
+ * @param g [in]		- The GPU driver struct.
+ * @param hw_unit [in]		- Index of HW unit.
+ * @param err_index [in]	- Error index.
+ * @param inst [in]	        - Instance ID.
+ *
+ *  - Sets values for error info.
+ *  - Invokes error reporting API with the required set of inputs.
+ *
+ * @return None
+ */
+void nvgpu_inject_ctxsw_swerror(struct gk20a *g, u32 hw_unit,
+		u32 err_index, u32 inst);
+
+/**
+ * @brief Inject MMU error.
+ *
+ * @param g [in]		- The GPU driver struct.
+ * @param hw_unit [in]		- Index of HW unit.
+ * @param err_index [in]	- Error index.
+ * @param sub_err_type [in]	- Sub error type.
+ *
+ *  - Sets values for mmu page fault info.
+ *  - Invokes error reporting API with the required set of inputs.
+ *
+ * @return None
+ */
+void nvgpu_inject_mmu_swerror(struct gk20a *g, u32 hw_unit,
+		u32 err_index, u32 sub_err_type);
+
+/**
+ * @brief Initialize error message header.
+ *
+ * @param header [in]		- Error message header.
+ *
+ *  This is used to initialize error message header.
+ *
+ * @return None
+ */
+void nvgpu_init_err_msg_header(struct gpu_err_header *header);
+
+/**
+ * @brief Initialize error message.
+ *
+ * @param msg [in]		- Error message.
+ *
+ *  This is used to initialize error message that is common
+ *  for all HW units.
+ *
+ * @return None
+ */
+void nvgpu_init_err_msg(struct nvgpu_err_msg *msg);
+
+/**
+ * @brief Initialize error message for HOST unit.
+ *
+ * @param msg [in]		- Error message.
+ *
+ *  This is used to initialize error message that is specific to HOST unit.
+ *
+ * @return None
+ */
+void nvgpu_init_host_err_msg(struct nvgpu_err_msg *msg);
+
+/**
+ * @brief Initialize ECC error message.
+ *
+ * @param msg [in]		- Error message.
+ *
+ *  This is used to initialize error message that is specific to ECC errors.
+ *
+ * @return None
+ */
+void nvgpu_init_ecc_err_msg(struct nvgpu_err_msg *msg);
+
+/**
+ * @brief Initialize error message for PRI unit.
+ *
+ * @param msg [in]		- Error message.
+ *
+ *  This is used to initialize error message that is specific to PRI unit.
+ *
+ * @return None
+ */
+void nvgpu_init_pri_err_msg(struct nvgpu_err_msg *msg);
+
+/**
+ * @brief Initialize error message for CE unit.
+ *
+ * @param msg [in]		- Error message.
+ *
+ *  This is used to initialize error message that is specific to CE unit.
+ *
+ * @return None
+ */
+void nvgpu_init_ce_err_msg(struct nvgpu_err_msg *msg);
+
+/**
+ * @brief Initialize error message for PMU unit.
+ *
+ * @param msg [in]		- Error message.
+ *
+ *  This is used to initialize error message that is specific to PMU unit.
+ *
+ * @return None
+ */
+void nvgpu_init_pmu_err_msg(struct nvgpu_err_msg *msg);
+
+/**
+ * @brief Initialize error message for GR unit.
+ *
+ * @param msg [in]		- Error message.
+ *
+ *  This is used to initialize error message that is specific to GR unit.
+ *
+ * @return None
+ */
+void nvgpu_init_gr_err_msg(struct nvgpu_err_msg *msg);
+
+/**
+ * @brief Initialize error message for CTXSW.
+ *
+ * @param msg [in]		- Error message.
+ *
+ *  This is used to initialize error message that is specific to CTXSW.
+ *
+ * @return None
+ */
+void nvgpu_init_ctxsw_err_msg(struct nvgpu_err_msg *msg);
+
+/**
+ * @brief Initialize error message for MMU unit.
+ *
+ * @param msg [in]		- Error message.
+ *
+ *  This is used to initialize error message that is specific to MMU unit.
+ *
+ * @return None
+ */
+void nvgpu_init_mmu_err_msg(struct nvgpu_err_msg *msg);
+
+#endif /* CIC_PRIV_H */

drivers/gpu/nvgpu/common/cic/ctxsw_cic.c

@@ -0,0 +1,97 @@
+/*
+ * Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/gk20a.h>
+#include <nvgpu/nvgpu_init.h>
+#include <nvgpu/nvgpu_err.h>
+#include <nvgpu/nvgpu_err_info.h>
+#include <nvgpu/cic.h>
+
+#include "cic_priv.h"
+
+void nvgpu_report_ctxsw_err(struct gk20a *g, u32 hw_unit, u32 err_id,
+		void *data)
+{
+	int err = 0;
+	struct nvgpu_err_desc *err_desc = NULL;
+	struct nvgpu_err_msg err_pkt;
+	u32 inst = 0;
+	struct ctxsw_err_info *err_info = (struct ctxsw_err_info *)data;
+
+	if (g->ops.cic.report_err == NULL) {
+		cic_dbg(g, "CIC does not support reporting error "
+			       "to safety services");
+		return;
+	}
+
+	if (hw_unit != NVGPU_ERR_MODULE_FECS) {
+		nvgpu_err(g, "invalid hw module (%u)", hw_unit);
+		err = -EINVAL;
+		goto handle_report_failure;
+	}
+
+	err = nvgpu_cic_get_err_desc(g, hw_unit, err_id, &err_desc);
+	if (err != 0) {
+		nvgpu_err(g, "Failed to get err_desc for"
+			       " err_id (%u) for hw module (%u)",
+				err_id, hw_unit);
+		goto handle_report_failure;
+	}
+
+	nvgpu_init_ctxsw_err_msg(&err_pkt);
+	err_pkt.hw_unit_id = hw_unit;
+	err_pkt.err_id = err_desc->error_id;
+	err_pkt.is_critical = err_desc->is_critical;
+	err_pkt.err_info.ctxsw_info.header.sub_unit_id = inst;
+	err_pkt.err_info.ctxsw_info.curr_ctx = err_info->curr_ctx;
+	err_pkt.err_info.ctxsw_info.chid = err_info->chid;
+	err_pkt.err_info.ctxsw_info.ctxsw_status0 = err_info->ctxsw_status0;
+	err_pkt.err_info.ctxsw_info.ctxsw_status1 = err_info->ctxsw_status1;
+	err_pkt.err_info.ctxsw_info.mailbox_value = err_info->mailbox_value;
+	err_pkt.err_desc = err_desc;
+	err_pkt.err_size = nvgpu_safe_cast_u64_to_u8(
+			sizeof(err_pkt.err_info.ctxsw_info));
+
+	if (g->ops.cic.report_err != NULL) {
+		err = g->ops.cic.report_err(g, (void *)&err_pkt,
+			sizeof(err_pkt), err_desc->is_critical);
+		if (err != 0) {
+			nvgpu_err(g, "Failed to report CTXSW error: "
+					"err_id=%u, mailbox_val=%u",
+					err_id, err_info->mailbox_value);
+		}
+	}
+handle_report_failure:
+	if (err != 0) {
+		nvgpu_sw_quiesce(g);
+	}
+}
+
+void nvgpu_inject_ctxsw_swerror(struct gk20a *g, u32 hw_unit,
+		u32 err_index, u32 inst)
+{
+	struct ctxsw_err_info err_info;
+
+	(void)memset(&err_info, ERR_INJECT_TEST_PATTERN, sizeof(err_info));
+
+	nvgpu_report_ctxsw_err(g, hw_unit, err_index, (void *)&err_info);
+}

drivers/gpu/nvgpu/common/cic/ecc_cic.c

@@ -0,0 +1,87 @@
+/*
+ * Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/gk20a.h>
+#include <nvgpu/nvgpu_init.h>
+#include <nvgpu/nvgpu_err.h>
+#include <nvgpu/nvgpu_err_info.h>
+#include <nvgpu/cic.h>
+
+#include "cic_priv.h"
+
+void nvgpu_report_ecc_err(struct gk20a *g, u32 hw_unit, u32 inst,
+		u32 err_id, u64 err_addr, u64 err_count)
+{
+	int err = 0;
+	struct nvgpu_err_desc *err_desc = NULL;
+	struct nvgpu_err_msg err_pkt;
+
+	if (g->ops.cic.report_err == NULL) {
+		cic_dbg(g, "CIC does not support reporting error "
+			       "to safety services");
+		return;
+	}
+
+	err = nvgpu_cic_get_err_desc(g, hw_unit, err_id, &err_desc);
+	if (err != 0) {
+		nvgpu_err(g, "Failed to get err_desc for "
+			       "err_id (%u) for hw module (%u)",
+				err_id, hw_unit);
+		goto handle_report_failure;
+	}
+
+	nvgpu_init_ecc_err_msg(&err_pkt);
+	err_pkt.hw_unit_id = hw_unit;
+	err_pkt.err_id = err_desc->error_id;
+	err_pkt.is_critical = err_desc->is_critical;
+	err_pkt.err_info.ecc_info.header.sub_unit_id = inst;
+	err_pkt.err_info.ecc_info.header.address = err_addr;
+	err_pkt.err_info.ecc_info.err_cnt = err_count;
+	err_pkt.err_desc = err_desc;
+	err_pkt.err_size = nvgpu_safe_cast_u64_to_u8(
+			sizeof(err_pkt.err_info.ecc_info));
+
+	if (g->ops.cic.report_err != NULL) {
+		err = g->ops.cic.report_err(g, (void *)&err_pkt,
+			sizeof(err_pkt), err_desc->is_critical);
+		if (err != 0) {
+			nvgpu_err(g, "Failed to report ECC error: hw_unit=%u, inst=%u, "
+				"err_id=%u, err_addr=%llu, err_count=%llu",
+				hw_unit, inst, err_id, err_addr, err_count);
+		}
+	}
+handle_report_failure:
+	if (err != 0) {
+		nvgpu_sw_quiesce(g);
+	}
+}
+
+void nvgpu_inject_ecc_swerror(struct gk20a *g, u32 hw_unit, u32 err_index,
+		u32 inst)
+{
+	u64 err_addr, err_count;
+
+	err_addr = (u64)ERR_INJECT_TEST_PATTERN;
+	err_count = (u64)ERR_INJECT_TEST_PATTERN;
+
+	nvgpu_report_ecc_err(g, hw_unit, inst, err_index, err_addr, err_count);
+}

drivers/gpu/nvgpu/common/cic/gr_cic.c

@@ -0,0 +1,169 @@
+/*
+ * Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/gk20a.h>
+#include <nvgpu/nvgpu_init.h>
+#include <nvgpu/nvgpu_err.h>
+#include <nvgpu/nvgpu_err_info.h>
+#include <nvgpu/cic.h>
+
+#include "cic_priv.h"
+
+static void nvpgu_report_fill_err_info(u32 hw_unit,
+		struct nvgpu_err_msg *err_pkt, struct gr_err_info *err_info)
+{
+	if (hw_unit == NVGPU_ERR_MODULE_SM) {
+		struct gr_sm_mcerr_info *info = err_info->sm_mcerr_info;
+
+		err_pkt->err_info.sm_info.warp_esr_pc =
+			info->hww_warp_esr_pc;
+		err_pkt->err_info.sm_info.warp_esr_status =
+			info->hww_warp_esr_status;
+		err_pkt->err_info.sm_info.curr_ctx =
+			info->curr_ctx;
+		err_pkt->err_info.sm_info.chid =
+			info->chid;
+		err_pkt->err_info.sm_info.tsgid =
+			info->tsgid;
+		err_pkt->err_info.sm_info.gpc =
+			info->gpc;
+		err_pkt->err_info.sm_info.tpc =
+			info->tpc;
+		err_pkt->err_info.sm_info.sm =
+			info->sm;
+	} else {
+		struct gr_exception_info *info  = err_info->exception_info;
+
+		err_pkt->err_info.gr_info.curr_ctx = info->curr_ctx;
+		err_pkt->err_info.gr_info.chid = info->chid;
+		err_pkt->err_info.gr_info.tsgid = info->tsgid;
+		err_pkt->err_info.gr_info.status = info->status;
+	}
+}
+
+void nvgpu_report_gr_err(struct gk20a *g, u32 hw_unit, u32 inst,
+		u32 err_id, struct gr_err_info *err_info, u32 sub_err_type)
+{
+	int err = 0;
+	struct nvgpu_err_desc *err_desc = NULL;
+	struct nvgpu_err_msg err_pkt;
+
+	if (g->ops.cic.report_err == NULL) {
+		cic_dbg(g, "CIC does not support reporting error "
+			       "to safety services");
+		return;
+	}
+
+	if ((hw_unit != NVGPU_ERR_MODULE_SM) &&
+			(hw_unit != NVGPU_ERR_MODULE_PGRAPH)) {
+		nvgpu_err(g, "invalid hw module (%u)", hw_unit);
+		err = -EINVAL;
+		goto handle_report_failure;
+	}
+
+	err = nvgpu_cic_get_err_desc(g, hw_unit, err_id, &err_desc);
+	if (err != 0) {
+		nvgpu_err(g, "Failed to get err_desc for "
+				"err_id (%u) for hw module (%u)",
+				err_id, hw_unit);
+		goto handle_report_failure;
+	}
+
+	nvgpu_init_gr_err_msg(&err_pkt);
+	err_pkt.hw_unit_id = hw_unit;
+	err_pkt.err_id = err_desc->error_id;
+	err_pkt.is_critical = err_desc->is_critical;
+	err_pkt.err_desc = err_desc;
+	err_pkt.err_info.gr_info.header.sub_err_type = sub_err_type;
+	err_pkt.err_info.gr_info.header.sub_unit_id = inst;
+	nvpgu_report_fill_err_info(hw_unit, &err_pkt, err_info);
+	err_pkt.err_size = nvgpu_safe_cast_u64_to_u8(sizeof(err_pkt.err_info));
+
+	if (g->ops.cic.report_err != NULL) {
+		err = g->ops.cic.report_err(g, (void *)&err_pkt,
+			sizeof(err_pkt), err_desc->is_critical);
+		if (err != 0) {
+			if (hw_unit == NVGPU_ERR_MODULE_SM) {
+				nvgpu_err(g, "Failed to report SM exception"
+					"gpc=%u, tpc=%u, sm=%u, esr_status=%x",
+					err_pkt.err_info.sm_info.gpc,
+					err_pkt.err_info.sm_info.tpc,
+					err_pkt.err_info.sm_info.sm,
+					err_pkt.err_info.sm_info.warp_esr_status);
+			}
+			if (hw_unit == NVGPU_ERR_MODULE_PGRAPH) {
+				nvgpu_err(g, "Failed to report PGRAPH"
+					"exception: inst=%u, err_id=%u, "
+					"status=%u", inst, err_id,
+					err_pkt.err_info.gr_info.status);
+			}
+		}
+	}
+handle_report_failure:
+	if (err != 0) {
+		nvgpu_sw_quiesce(g);
+	}
+}
+
+void nvgpu_inject_gr_swerror(struct gk20a *g, u32 hw_unit,
+		u32 err_index, u32 sub_err_type)
+{
+	struct gr_err_info err_info;
+	struct gr_exception_info gr_error_info;
+	struct gr_sm_mcerr_info sm_error_info;
+	int err = 0;
+	u32 inst = 0U;
+
+	/*
+	 * Fill fixed test pattern data for the error message
+	 * payload.
+	 */
+	(void)memset(&gr_error_info, ERR_INJECT_TEST_PATTERN, sizeof(gr_error_info));
+	(void)memset(&sm_error_info, ERR_INJECT_TEST_PATTERN, sizeof(sm_error_info));
+
+	switch (hw_unit) {
+	case NVGPU_ERR_MODULE_PGRAPH:
+		{
+			err_info.exception_info = &gr_error_info;
+		}
+		break;
+
+	case NVGPU_ERR_MODULE_SM:
+		{
+			err_info.sm_mcerr_info = &sm_error_info;
+		}
+		break;
+
+	default:
+		{
+			nvgpu_err(g, "unsupported hw_unit(%u)", hw_unit);
+			err = -EINVAL;
+		}
+		break;
+	}
+	if (err != 0) {
+		return;
+	}
+
+	nvgpu_report_gr_err(g, hw_unit, inst, err_index,
+			&err_info, sub_err_type);
+}

drivers/gpu/nvgpu/common/cic/host_cic.c

@@ -0,0 +1,91 @@
+/*
+ * Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/gk20a.h>
+#include <nvgpu/nvgpu_init.h>
+#include <nvgpu/nvgpu_err.h>
+#include <nvgpu/nvgpu_err_info.h>
+#include <nvgpu/cic.h>
+
+#include "cic_priv.h"
+
+void nvgpu_report_host_err(struct gk20a *g, u32 hw_unit,
+		u32 inst, u32 err_id, u32 intr_info)
+{
+	int err = 0;
+	struct nvgpu_err_desc *err_desc = NULL;
+	struct nvgpu_err_msg err_pkt;
+
+	if (g->ops.cic.report_err == NULL) {
+		cic_dbg(g, "CIC does not support reporting error "
+			       "to safety services");
+		return;
+	}
+
+	if (hw_unit != NVGPU_ERR_MODULE_HOST) {
+		nvgpu_err(g, "invalid hw module (%u)", hw_unit);
+		err = -EINVAL;
+		goto handle_report_failure;
+	}
+
+	err = nvgpu_cic_get_err_desc(g, hw_unit, err_id, &err_desc);
+	if (err != 0) {
+		nvgpu_err(g, "Failed to get err_desc for "
+				"err_id (%u) for hw module (%u)",
+				err_id, hw_unit);
+		goto handle_report_failure;
+	}
+
+	nvgpu_init_host_err_msg(&err_pkt);
+	err_pkt.hw_unit_id = hw_unit;
+	err_pkt.err_id = err_desc->error_id;
+	err_pkt.is_critical = err_desc->is_critical;
+	err_pkt.err_info.host_info.header.sub_unit_id = inst;
+	err_pkt.err_desc = err_desc;
+	/* sub_err_type can be decoded using intr_info by referring
+	 * to the interrupt status register definition corresponding
+	 * to the error that is being reported.
+	 */
+	err_pkt.err_info.host_info.header.sub_err_type = intr_info;
+	err_pkt.err_size = nvgpu_safe_cast_u64_to_u8(
+			sizeof(err_pkt.err_info.host_info));
+
+	if (g->ops.cic.report_err != NULL) {
+		err = g->ops.cic.report_err(g, (void *)&err_pkt,
+			sizeof(err_pkt), err_desc->is_critical);
+		if (err != 0) {
+			nvgpu_err(g, "Failed to report HOST error: "
+					"inst=%u, err_id=%u, intr_info=%u",
+					inst, err_id, intr_info);
+		}
+	}
+handle_report_failure:
+	if (err != 0) {
+		nvgpu_sw_quiesce(g);
+	}
+}
+
+void nvgpu_inject_host_swerror(struct gk20a *g, u32 hw_unit,
+		u32 err_index, u32 sub_err_type)
+{
+	nvgpu_report_host_err(g, hw_unit, 0U, err_index, sub_err_type);
+}

drivers/gpu/nvgpu/common/cic/mmu_cic.c

@@ -0,0 +1,131 @@
+/*
+ * Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/gk20a.h>
+#include <nvgpu/nvgpu_init.h>
+#include <nvgpu/nvgpu_err.h>
+#include <nvgpu/nvgpu_err_info.h>
+#include <nvgpu/cic.h>
+
+#include "cic_priv.h"
+
+void nvgpu_report_mmu_err(struct gk20a *g, u32 hw_unit, u32 err_id,
+		struct mmu_fault_info *fault_info, u32 status, u32 sub_err_type)
+{
+	int err = 0;
+	struct nvgpu_err_desc *err_desc = NULL;
+	struct nvgpu_err_msg err_pkt;
+
+	if (g->ops.cic.report_err == NULL) {
+		cic_dbg(g, "CIC does not support reporting error "
+			       "to safety services");
+		return;
+	}
+
+	if (hw_unit != NVGPU_ERR_MODULE_HUBMMU) {
+		nvgpu_err(g, "invalid hw module (%u)", hw_unit);
+		err = -EINVAL;
+		goto handle_report_failure;
+	}
+
+	err = nvgpu_cic_get_err_desc(g, hw_unit, err_id, &err_desc);
+	if (err != 0) {
+		nvgpu_err(g, "Failed to get err_desc for "
+				"err_id (%u) for hw module (%u)",
+				err_id, hw_unit);
+		goto handle_report_failure;
+	}
+
+	nvgpu_init_mmu_err_msg(&err_pkt);
+	err_pkt.hw_unit_id = hw_unit;
+	err_pkt.err_id = err_desc->error_id;
+	err_pkt.is_critical = err_desc->is_critical;
+	err_pkt.err_info.mmu_info.header.sub_err_type = sub_err_type;
+	err_pkt.err_info.mmu_info.status = status;
+	/* Copy contents of mmu_fault_info */
+	if (fault_info != NULL) {
+		err_pkt.err_info.mmu_info.info.inst_ptr = fault_info->inst_ptr;
+		err_pkt.err_info.mmu_info.info.inst_aperture
+			= fault_info->inst_aperture;
+		err_pkt.err_info.mmu_info.info.fault_addr
+			= fault_info->fault_addr;
+		err_pkt.err_info.mmu_info.info.fault_addr_aperture
+			= fault_info->fault_addr_aperture;
+		err_pkt.err_info.mmu_info.info.timestamp_lo
+			= fault_info->timestamp_lo;
+		err_pkt.err_info.mmu_info.info.timestamp_hi
+			= fault_info->timestamp_hi;
+		err_pkt.err_info.mmu_info.info.mmu_engine_id
+			= fault_info->mmu_engine_id;
+		err_pkt.err_info.mmu_info.info.gpc_id = fault_info->gpc_id;
+		err_pkt.err_info.mmu_info.info.client_type
+			= fault_info->client_type;
+		err_pkt.err_info.mmu_info.info.client_id
+			= fault_info->client_id;
+		err_pkt.err_info.mmu_info.info.fault_type
+			= fault_info->fault_type;
+		err_pkt.err_info.mmu_info.info.access_type
+			= fault_info->access_type;
+		err_pkt.err_info.mmu_info.info.protected_mode
+			= fault_info->protected_mode;
+		err_pkt.err_info.mmu_info.info.replayable_fault
+			= fault_info->replayable_fault;
+		err_pkt.err_info.mmu_info.info.replay_fault_en
+			= fault_info->replay_fault_en;
+		err_pkt.err_info.mmu_info.info.valid = fault_info->valid;
+		err_pkt.err_info.mmu_info.info.faulted_pbdma =
+			fault_info->faulted_pbdma;
+		err_pkt.err_info.mmu_info.info.faulted_engine =
+			fault_info->faulted_engine;
+		err_pkt.err_info.mmu_info.info.faulted_subid =
+			fault_info->faulted_subid;
+		err_pkt.err_info.mmu_info.info.chid = fault_info->chid;
+	}
+	err_pkt.err_desc = err_desc;
+	err_pkt.err_size = nvgpu_safe_cast_u64_to_u8(
+			sizeof(err_pkt.err_info.mmu_info));
+
+	if (g->ops.cic.report_err != NULL) {
+		err = g->ops.cic.report_err(g, (void *)&err_pkt,
+			sizeof(err_pkt), err_desc->is_critical);
+		if (err != 0) {
+			nvgpu_err(g, "Failed to report MMU fault: hw_unit=%u, "
+				"err_id=%u, sub_err_type=%u, status=%u",
+				hw_unit, err_id, sub_err_type, status);
+		}
+	}
+handle_report_failure:
+	if (err != 0) {
+		nvgpu_sw_quiesce(g);
+	}
+}
+
+void nvgpu_inject_mmu_swerror(struct gk20a *g, u32 hw_unit, u32 err_index,
+		u32 sub_err_type)
+{
+	u32 status = 0U;
+	struct mmu_fault_info fault_info;
+
+	(void) memset(&fault_info, ERR_INJECT_TEST_PATTERN, sizeof(fault_info));
+	nvgpu_report_mmu_err(g, hw_unit, err_index,
+			&fault_info, status, sub_err_type);
+}

drivers/gpu/nvgpu/common/cic/msg_cic.c

@@ -0,0 +1,126 @@
+/*
+ * Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/nvgpu_err_info.h>
+#include <nvgpu/string.h>
+
+#include "cic_priv.h"
+
+void nvgpu_init_err_msg_header(struct gpu_err_header *header)
+{
+	header->version.major = (u16)1U;
+	header->version.minor = (u16)0U;
+	header->sub_err_type = 0U;
+	header->sub_unit_id = 0UL;
+	header->address = 0UL;
+	header->timestamp_ns = 0UL;
+}
+
+void nvgpu_init_err_msg(struct nvgpu_err_msg *msg)
+{
+	(void) memset(msg, 0, sizeof(struct nvgpu_err_msg));
+	msg->hw_unit_id = 0U;
+	msg->is_critical = false;
+	msg->err_id = (u8)0U;
+	msg->err_size = (u8)0U;
+}
+
+void nvgpu_init_host_err_msg(struct nvgpu_err_msg *msg)
+{
+	nvgpu_init_err_msg(msg);
+	nvgpu_init_err_msg_header(&msg->err_info.host_info.header);
+}
+
+void nvgpu_init_ecc_err_msg(struct nvgpu_err_msg *msg)
+{
+	nvgpu_init_err_msg(msg);
+	nvgpu_init_err_msg_header(&msg->err_info.ecc_info.header);
+	msg->err_info.ecc_info.err_cnt = 0UL;
+}
+
+void nvgpu_init_pri_err_msg(struct nvgpu_err_msg *msg)
+{
+	nvgpu_init_err_msg(msg);
+	nvgpu_init_err_msg_header(&msg->err_info.pri_info.header);
+}
+
+void nvgpu_init_ce_err_msg(struct nvgpu_err_msg *msg)
+{
+	nvgpu_init_err_msg(msg);
+	nvgpu_init_err_msg_header(&msg->err_info.ce_info.header);
+}
+
+void nvgpu_init_pmu_err_msg(struct nvgpu_err_msg *msg)
+{
+	nvgpu_init_err_msg(msg);
+	nvgpu_init_err_msg_header(&msg->err_info.pmu_err_info.header);
+	msg->err_info.pmu_err_info.status = 0U;
+}
+
+void nvgpu_init_gr_err_msg(struct nvgpu_err_msg *msg)
+{
+	nvgpu_init_err_msg(msg);
+	nvgpu_init_err_msg_header(&msg->err_info.gr_info.header);
+	msg->err_info.gr_info.curr_ctx = 0U;
+	msg->err_info.gr_info.chid = 0U;
+	msg->err_info.gr_info.tsgid = 0U;
+	msg->err_info.gr_info.status = 0U;
+}
+
+void nvgpu_init_ctxsw_err_msg(struct nvgpu_err_msg *msg)
+{
+	nvgpu_init_err_msg(msg);
+	nvgpu_init_err_msg_header(&msg->err_info.ctxsw_info.header);
+	msg->err_info.ctxsw_info.curr_ctx = 0U;
+	msg->err_info.ctxsw_info.tsgid = 0U;
+	msg->err_info.ctxsw_info.chid = 0U;
+	msg->err_info.ctxsw_info.ctxsw_status0 = 0U;
+	msg->err_info.ctxsw_info.ctxsw_status1 = 0U;
+	msg->err_info.ctxsw_info.mailbox_value = 0U;
+}
+
+void nvgpu_init_mmu_err_msg(struct nvgpu_err_msg *msg)
+{
+	nvgpu_init_err_msg(msg);
+	nvgpu_init_err_msg_header(&msg->err_info.mmu_info.header);
+	msg->err_info.mmu_info.info.inst_ptr = 0UL;
+	msg->err_info.mmu_info.info.inst_aperture = 0U;
+	msg->err_info.mmu_info.info.fault_addr = 0UL;
+	msg->err_info.mmu_info.info.fault_addr_aperture = 0U;
+	msg->err_info.mmu_info.info.timestamp_lo = 0U;
+	msg->err_info.mmu_info.info.timestamp_hi = 0U;
+	msg->err_info.mmu_info.info.mmu_engine_id = 0U;
+	msg->err_info.mmu_info.info.gpc_id = 0U;
+	msg->err_info.mmu_info.info.client_type = 0U;
+	msg->err_info.mmu_info.info.client_id = 0U;
+	msg->err_info.mmu_info.info.fault_type = 0U;
+	msg->err_info.mmu_info.info.access_type = 0U;
+	msg->err_info.mmu_info.info.protected_mode = 0U;
+	msg->err_info.mmu_info.info.replayable_fault = false;
+	msg->err_info.mmu_info.info.replay_fault_en = 0U;
+	msg->err_info.mmu_info.info.valid = false;
+	msg->err_info.mmu_info.info.faulted_pbdma = 0U;
+	msg->err_info.mmu_info.info.faulted_engine = 0U;
+	msg->err_info.mmu_info.info.faulted_subid = 0U;
+	msg->err_info.mmu_info.info.chid = 0U;
+	msg->err_info.mmu_info.status = 0U;
+}

drivers/gpu/nvgpu/common/cic/pmu_cic.c

@@ -0,0 +1,91 @@
+/*
+ * Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/gk20a.h>
+#include <nvgpu/nvgpu_init.h>
+#include <nvgpu/nvgpu_err.h>
+#include <nvgpu/nvgpu_err_info.h>
+#include <nvgpu/cic.h>
+
+#include "cic_priv.h"
+
+void nvgpu_report_pmu_err(struct gk20a *g, u32 hw_unit, u32 err_id,
+		u32 sub_err_type, u32 status)
+{
+	int err = 0;
+	struct nvgpu_err_desc *err_desc = NULL;
+	struct nvgpu_err_msg err_pkt;
+
+	if (g->ops.cic.report_err == NULL) {
+		cic_dbg(g, "CIC does not support reporting error "
+			       "to safety services");
+		return;
+	}
+
+	if (hw_unit != NVGPU_ERR_MODULE_PMU) {
+		nvgpu_err(g, "invalid hw module (%u)", hw_unit);
+		err = -EINVAL;
+		goto handle_report_failure;
+	}
+
+	err = nvgpu_cic_get_err_desc(g, hw_unit, err_id, &err_desc);
+	if (err != 0) {
+		nvgpu_err(g, "Failed to get err_desc for "
+				"err_id (%u) for hw module (%u)",
+				err_id, hw_unit);
+		goto handle_report_failure;
+	}
+
+	nvgpu_init_pmu_err_msg(&err_pkt);
+	err_pkt.hw_unit_id = hw_unit;
+	err_pkt.err_id = err_desc->error_id;
+	err_pkt.is_critical = err_desc->is_critical;
+	err_pkt.err_info.pmu_err_info.status = status;
+	err_pkt.err_info.pmu_err_info.header.sub_err_type = sub_err_type;
+	err_pkt.err_desc = err_desc;
+	err_pkt.err_size = nvgpu_safe_cast_u64_to_u8(
+			sizeof(err_pkt.err_info.pmu_err_info));
+
+	if (g->ops.cic.report_err != NULL) {
+		err = g->ops.cic.report_err(g, (void *)&err_pkt,
+			sizeof(err_pkt), err_desc->is_critical);
+		if (err != 0) {
+			nvgpu_err(g, "Failed to report PMU error: "
+				"err_id=%u, sub_err_type=%u, status=%u",
+				err_id, sub_err_type, status);
+		}
+	}
+handle_report_failure:
+	if (err != 0) {
+		nvgpu_sw_quiesce(g);
+	}
+}
+
+void nvgpu_inject_pmu_swerror(struct gk20a *g, u32 hw_unit,
+		u32 err_index, u32 sub_err_type)
+{
+	u32 err_info;
+
+	err_info = (u32)ERR_INJECT_TEST_PATTERN;
+
+	nvgpu_report_pmu_err(g, hw_unit, err_index, sub_err_type, err_info);
+}

drivers/gpu/nvgpu/common/cic/pri_cic.c

@@ -0,0 +1,91 @@
+/*
+ * Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/gk20a.h>
+#include <nvgpu/nvgpu_init.h>
+#include <nvgpu/nvgpu_err.h>
+#include <nvgpu/nvgpu_err_info.h>
+#include <nvgpu/cic.h>
+
+#include "cic_priv.h"
+
+void nvgpu_report_pri_err(struct gk20a *g, u32 hw_unit, u32 inst,
+		u32 err_id, u32 err_addr, u32 err_code)
+{
+	int err = 0;
+	struct nvgpu_err_desc *err_desc = NULL;
+	struct nvgpu_err_msg err_pkt;
+
+	if (g->ops.cic.report_err == NULL) {
+		cic_dbg(g, "CIC does not support reporting error "
+			       "to safety services");
+		return;
+	}
+
+	if (hw_unit != NVGPU_ERR_MODULE_PRI) {
+		nvgpu_err(g, "invalid hw module (%u)", hw_unit);
+		err = -EINVAL;
+		goto handle_report_failure;
+	}
+
+	err = nvgpu_cic_get_err_desc(g, hw_unit, err_id, &err_desc);
+	if (err != 0) {
+		nvgpu_err(g, "Failed to get err_desc for "
+				"err_id (%u) for hw module (%u)",
+				err_id, hw_unit);
+		goto handle_report_failure;
+	}
+
+	nvgpu_init_pri_err_msg(&err_pkt);
+	err_pkt.hw_unit_id = hw_unit;
+	err_pkt.err_id = err_desc->error_id;
+	err_pkt.is_critical = err_desc->is_critical;
+	err_pkt.err_info.pri_info.header.sub_unit_id = inst;
+	err_pkt.err_info.pri_info.header.address = (u64) err_addr;
+	err_pkt.err_desc = err_desc;
+	/* sub_err_type can be decoded using err_code by referring
+	 * to the FECS pri error codes.
+	 */
+	err_pkt.err_info.pri_info.header.sub_err_type = err_code;
+	err_pkt.err_size = nvgpu_safe_cast_u64_to_u8(
+			sizeof(err_pkt.err_info.pri_info));
+
+	if (g->ops.cic.report_err != NULL) {
+		err = g->ops.cic.report_err(g, (void *)&err_pkt,
+			sizeof(err_pkt), err_desc->is_critical);
+		if (err != 0) {
+			nvgpu_err(g, "Failed to report PRI error: "
+					"inst=%u, err_id=%u, err_code=%u",
+					inst, err_id, err_code);
+		}
+	}
+handle_report_failure:
+	if (err != 0) {
+		nvgpu_sw_quiesce(g);
+	}
+}
+
+void nvgpu_inject_pri_swerror(struct gk20a *g, u32 hw_unit,
+		u32 err_index, u32 err_code)
+{
+	nvgpu_report_pri_err(g, hw_unit, 0U, err_index, 0U, err_code);
+}

drivers/gpu/nvgpu/common/clk_arb/clk_arb.c

@@ -0,0 +1,826 @@
+/*
+ * Copyright (c) 2016-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/bitops.h>
+#include <nvgpu/lock.h>
+#include <nvgpu/kmem.h>
+#include <nvgpu/atomic.h>
+#include <nvgpu/bug.h>
+#include <nvgpu/kref.h>
+#include <nvgpu/log.h>
+#include <nvgpu/barrier.h>
+#include <nvgpu/cond.h>
+#include <nvgpu/list.h>
+#include <nvgpu/clk_arb.h>
+#include <nvgpu/timers.h>
+#include <nvgpu/worker.h>
+#include <nvgpu/gk20a.h>
+#ifdef CONFIG_NVGPU_LS_PMU
+#include <nvgpu/pmu/clk/clk.h>
+#include <nvgpu/pmu/perf.h>
+#include <nvgpu/pmu/volt.h>
+#endif
+#include <nvgpu/boardobjgrp_e255.h>
+
+int nvgpu_clk_notification_queue_alloc(struct gk20a *g,
+				struct nvgpu_clk_notification_queue *queue,
+				u32 events_number) {
+	queue->clk_q_notifications = nvgpu_kcalloc(g, events_number,
+		sizeof(struct nvgpu_clk_notification));
+	if (queue->clk_q_notifications == NULL) {
+		return -ENOMEM;
+	}
+	queue->size = events_number;
+
+	nvgpu_atomic_set(&queue->head, 0);
+	nvgpu_atomic_set(&queue->tail, 0);
+
+	return 0;
+}
+
+void nvgpu_clk_notification_queue_free(struct gk20a *g,
+		struct nvgpu_clk_notification_queue *queue) {
+	if (queue->size > 0U) {
+		nvgpu_kfree(g, queue->clk_q_notifications);
+		queue->size = 0;
+		nvgpu_atomic_set(&queue->head, 0);
+		nvgpu_atomic_set(&queue->tail, 0);
+	}
+}
+
+static void nvgpu_clk_arb_queue_notification(struct gk20a *g,
+				struct nvgpu_clk_notification_queue *queue,
+				u32 alarm_mask) {
+
+	u32 queue_index;
+	u64 timestamp = 0U;
+
+	queue_index = U32(nvgpu_atomic_inc_return(&queue->tail)) % queue->size;
+
+#ifdef CONFIG_NVGPU_NON_FUSA
+	/* get current timestamp */
+	timestamp = (u64) nvgpu_hr_timestamp();
+#endif
+
+	queue->clk_q_notifications[queue_index].timestamp = timestamp;
+	queue->clk_q_notifications[queue_index].clk_notification = alarm_mask;
+
+}
+
+void nvgpu_clk_arb_set_global_alarm(struct gk20a *g, u32 alarm)
+{
+	struct nvgpu_clk_arb *arb = g->clk_arb;
+
+	u64 current_mask;
+	u32 refcnt;
+	u32 alarm_mask;
+	u64 new_mask;
+
+	do {
+		current_mask = (u64)nvgpu_atomic64_read(&arb->alarm_mask);
+		/* atomic operations are strong so they do not need masks */
+
+		refcnt = ((u32) (current_mask >> 32)) + 1U;
+		alarm_mask =  (u32) (current_mask &  ~U32(0)) | alarm;
+		new_mask = ((u64) refcnt << 32) | alarm_mask;
+
+	} while (unlikely(current_mask !=
+			(u64)nvgpu_atomic64_cmpxchg(&arb->alarm_mask,
+				(long int)current_mask, (long int)new_mask)));
+
+	nvgpu_clk_arb_queue_notification(g, &arb->notification_queue, alarm);
+}
+
+#ifdef CONFIG_NVGPU_LS_PMU
+int nvgpu_clk_arb_update_vf_table(struct nvgpu_clk_arb *arb)
+{
+	struct gk20a *g = arb->g;
+	struct nvgpu_clk_vf_table *table;
+
+	u32 i, j;
+	int status = -EINVAL;
+	u16 clk_cur;
+	u32 num_points;
+
+	struct nvgpu_pmu_perf_pstate_clk_info *p0_info;
+
+	table = NV_READ_ONCE(arb->current_vf_table);
+	/* make flag visible when all data has resolved in the tables */
+	nvgpu_smp_rmb();
+	table = (table == &arb->vf_table_pool[0]) ? &arb->vf_table_pool[1] :
+		&arb->vf_table_pool[0];
+
+	/* Get allowed memory ranges */
+	if (g->ops.clk_arb.get_arbiter_clk_range(g, CTRL_CLK_DOMAIN_GPCCLK,
+						&arb->gpc2clk_min,
+						&arb->gpc2clk_max) < 0) {
+		nvgpu_err(g, "failed to fetch GPC2CLK range");
+		goto exit_vf_table;
+	}
+
+	if (g->ops.clk_arb.get_arbiter_clk_range(g, CTRL_CLK_DOMAIN_MCLK,
+						&arb->mclk_min,
+						&arb->mclk_max) < 0) {
+		nvgpu_err(g, "failed to fetch MCLK range");
+		goto exit_vf_table;
+	}
+
+	table->gpc2clk_num_points = MAX_F_POINTS;
+	table->mclk_num_points = MAX_F_POINTS;
+	if (g->ops.clk.clk_domain_get_f_points(arb->g, CTRL_CLK_DOMAIN_GPCCLK,
+		&table->gpc2clk_num_points, arb->gpc2clk_f_points)) {
+		nvgpu_err(g, "failed to fetch GPC2CLK frequency points");
+		goto exit_vf_table;
+	}
+	if (!table->gpc2clk_num_points) {
+		nvgpu_err(g, "empty queries to f points gpc2clk %d", table->gpc2clk_num_points);
+		status = -EINVAL;
+		goto exit_vf_table;
+	}
+
+	(void) memset(table->gpc2clk_points, 0,
+		table->gpc2clk_num_points*sizeof(struct nvgpu_clk_vf_point));
+
+	p0_info = nvgpu_pmu_perf_pstate_get_clk_set_info(g,
+			CTRL_PERF_PSTATE_P0, CLKWHICH_GPCCLK);
+	if (!p0_info) {
+		status = -EINVAL;
+		nvgpu_err(g, "failed to get GPC2CLK P0 info");
+		goto exit_vf_table;
+	}
+
+	/* GPC2CLK needs to be checked in two passes. The first determines the
+	 * relationships between GPC2CLK, SYS2CLK and XBAR2CLK, while the
+	 * second verifies that the clocks minimum is satisfied and sets
+	 * the voltages,the later part is done in nvgpu_pmu_perf_changeseq_set_clks
+	 */
+	j = 0; num_points = 0; clk_cur = 0;
+	for (i = 0; i < table->gpc2clk_num_points; i++) {
+		struct nvgpu_clk_slave_freq setfllclk;
+
+		if ((arb->gpc2clk_f_points[i] >= arb->gpc2clk_min) &&
+			(arb->gpc2clk_f_points[i] <= arb->gpc2clk_max) &&
+			(arb->gpc2clk_f_points[i] != clk_cur)) {
+
+			table->gpc2clk_points[j].gpc_mhz =
+				arb->gpc2clk_f_points[i];
+			setfllclk.gpc_mhz = arb->gpc2clk_f_points[i];
+
+			status = clk_get_fll_clks_per_clk_domain(g, &setfllclk);
+			if (status < 0) {
+				nvgpu_err(g,
+					"failed to get GPC2CLK slave clocks");
+				goto exit_vf_table;
+			}
+
+			table->gpc2clk_points[j].sys_mhz =
+				setfllclk.sys_mhz;
+			table->gpc2clk_points[j].xbar_mhz =
+				setfllclk.xbar_mhz;
+			table->gpc2clk_points[j].nvd_mhz =
+				setfllclk.nvd_mhz;
+			table->gpc2clk_points[j].host_mhz =
+				setfllclk.host_mhz;
+
+			clk_cur = table->gpc2clk_points[j].gpc_mhz;
+
+			if ((clk_cur >= p0_info->min_mhz) &&
+					(clk_cur <= p0_info->max_mhz)) {
+				VF_POINT_SET_PSTATE_SUPPORTED(
+					&table->gpc2clk_points[j],
+					CTRL_PERF_PSTATE_P0);
+			}
+
+			j++;
+			num_points++;
+		}
+	}
+	table->gpc2clk_num_points = num_points;
+
+	/* make table visible when all data has resolved in the tables */
+	nvgpu_smp_wmb();
+	arb->current_vf_table = table;
+
+exit_vf_table:
+
+	if (status < 0) {
+		nvgpu_clk_arb_set_global_alarm(g,
+			EVENT(ALARM_VF_TABLE_UPDATE_FAILED));
+	}
+	nvgpu_clk_arb_worker_enqueue(g, &arb->update_arb_work_item);
+
+	return status;
+}
+
+static void nvgpu_clk_arb_run_vf_table_cb(struct nvgpu_clk_arb *arb)
+{
+	struct gk20a *g = arb->g;
+	int err;
+
+	/* get latest vf curve from pmu */
+	err = nvgpu_clk_vf_point_cache(g);
+	if (err != 0) {
+		nvgpu_err(g, "failed to cache VF table");
+		nvgpu_clk_arb_set_global_alarm(g,
+			EVENT(ALARM_VF_TABLE_UPDATE_FAILED));
+		nvgpu_clk_arb_worker_enqueue(g, &arb->update_arb_work_item);
+
+		return;
+	}
+	nvgpu_clk_arb_update_vf_table(arb);
+}
+#endif
+u32 nvgpu_clk_arb_notify(struct nvgpu_clk_dev *dev,
+				struct nvgpu_clk_arb_target *target,
+				u32 alarm) {
+
+	struct nvgpu_clk_session *session = dev->session;
+	struct nvgpu_clk_arb *arb = session->g->clk_arb;
+	struct nvgpu_clk_notification *l_notification;
+
+	u32 queue_alarm_mask = 0;
+	u32 enabled_mask = 0;
+	u32 new_alarms_reported = 0;
+	u32 poll_mask = 0;
+	u32 tail, head, index;
+	u32 queue_index;
+	size_t size;
+
+	enabled_mask = (u32)nvgpu_atomic_read(&dev->enabled_mask);
+	size = arb->notification_queue.size;
+
+	/* queue global arbiter notifications in buffer */
+	do {
+		tail = (u32)nvgpu_atomic_read(&arb->notification_queue.tail);
+		/* copy items to the queue */
+		queue_index = (u32)nvgpu_atomic_read(&dev->queue.tail);
+		head = dev->arb_queue_head;
+		head = (tail - head) < arb->notification_queue.size ?
+			head : tail - arb->notification_queue.size;
+
+		for (index = head; WRAPGTEQ(tail, index); index++) {
+			u32 alarm_detected;
+
+			l_notification = &arb->notification_queue.
+				clk_q_notifications[((u64)index + 1ULL) % size];
+			alarm_detected = NV_READ_ONCE(
+					l_notification->clk_notification);
+
+			if ((enabled_mask & alarm_detected) == 0U) {
+				continue;
+			}
+
+			queue_index++;
+			dev->queue.clk_q_notifications[
+				queue_index % dev->queue.size].timestamp =
+				NV_READ_ONCE(l_notification->timestamp);
+
+			dev->queue.clk_q_notifications[queue_index %
+				dev->queue.size].clk_notification =
+					alarm_detected;
+
+			queue_alarm_mask |= alarm_detected;
+		}
+	} while (unlikely(nvgpu_atomic_read(&arb->notification_queue.tail) !=
+			(int)tail));
+
+	nvgpu_atomic_set(&dev->queue.tail, (int)queue_index);
+	/* update the last notification we processed from global queue */
+
+	dev->arb_queue_head = tail;
+
+	/* Check if current session targets are met */
+	if ((enabled_mask & EVENT(ALARM_LOCAL_TARGET_VF_NOT_POSSIBLE)) != 0U) {
+		if ((target->gpc2clk < session->target->gpc2clk)
+			|| (target->mclk < session->target->mclk)) {
+
+			poll_mask |= (NVGPU_POLLIN | NVGPU_POLLPRI);
+			nvgpu_clk_arb_queue_notification(arb->g, &dev->queue,
+				EVENT(ALARM_LOCAL_TARGET_VF_NOT_POSSIBLE));
+		}
+	}
+
+	/* Check if there is a new VF update */
+	if ((queue_alarm_mask & EVENT(VF_UPDATE)) != 0U) {
+		poll_mask |= (NVGPU_POLLIN | NVGPU_POLLRDNORM);
+	}
+
+	/* Notify sticky alarms that were not reported on previous run*/
+	new_alarms_reported = (queue_alarm_mask |
+			(alarm & ~dev->alarms_reported & queue_alarm_mask));
+
+	if ((new_alarms_reported & ~LOCAL_ALARM_MASK) != 0U) {
+		/* check that we are not re-reporting */
+		if ((new_alarms_reported & EVENT(ALARM_GPU_LOST)) != 0U) {
+			poll_mask |= NVGPU_POLLHUP;
+		}
+
+		poll_mask |= (NVGPU_POLLIN | NVGPU_POLLPRI);
+		/* On next run do not report global alarms that were already
+		 * reported, but report SHUTDOWN always
+		 */
+		dev->alarms_reported = new_alarms_reported & ~LOCAL_ALARM_MASK &
+							~EVENT(ALARM_GPU_LOST);
+	}
+
+	if (poll_mask != 0U) {
+		nvgpu_atomic_set(&dev->poll_mask, (int)poll_mask);
+		nvgpu_clk_arb_event_post_event(dev);
+	}
+
+	return new_alarms_reported;
+}
+
+void nvgpu_clk_arb_clear_global_alarm(struct gk20a *g, u32 alarm)
+{
+	struct nvgpu_clk_arb *arb = g->clk_arb;
+
+	u64 current_mask;
+	u32 refcnt;
+	u32 alarm_mask;
+	u64 new_mask;
+
+	do {
+		current_mask = (u64)nvgpu_atomic64_read(&arb->alarm_mask);
+		/* atomic operations are strong so they do not need masks */
+
+		refcnt = ((u32) (current_mask >> 32)) + 1U;
+		alarm_mask =  (u32) ((u32)current_mask & ~alarm);
+		new_mask = ((u64) refcnt << 32) | alarm_mask;
+
+	} while (unlikely(current_mask !=
+			(u64)nvgpu_atomic64_cmpxchg(&arb->alarm_mask,
+				(long int)current_mask, (long int)new_mask)));
+}
+
+/*
+ * Process one scheduled work item.
+ */
+static void nvgpu_clk_arb_worker_poll_wakeup_process_item(
+		struct nvgpu_list_node *work_item)
+{
+	struct nvgpu_clk_arb_work_item *clk_arb_work_item =
+		nvgpu_clk_arb_work_item_from_worker_item(work_item);
+
+	struct gk20a *g = clk_arb_work_item->arb->g;
+
+	clk_arb_dbg(g, " ");
+
+	if (clk_arb_work_item->item_type == CLK_ARB_WORK_UPDATE_VF_TABLE) {
+#ifdef CONFIG_NVGPU_LS_PMU
+		nvgpu_clk_arb_run_vf_table_cb(clk_arb_work_item->arb);
+#endif
+	} else {
+		if (clk_arb_work_item->item_type == CLK_ARB_WORK_UPDATE_ARB) {
+			g->ops.clk_arb.clk_arb_run_arbiter_cb(
+							clk_arb_work_item->arb);
+		}
+	}
+}
+
+static void nvgpu_clk_arb_worker_poll_init(struct nvgpu_worker *worker)
+{
+	clk_arb_dbg(worker->g, " ");
+}
+
+const struct nvgpu_worker_ops clk_arb_worker_ops = {
+	.pre_process = nvgpu_clk_arb_worker_poll_init,
+	.wakeup_early_exit = nvgpu_worker_should_stop,
+	.wakeup_post_process = NULL,
+	.wakeup_process_item =
+		nvgpu_clk_arb_worker_poll_wakeup_process_item,
+	.wakeup_condition =
+		nvgpu_worker_should_stop,
+	.wakeup_timeout = NULL,
+};
+
+/**
+ * Append a work item to the worker's list.
+ *
+ * This adds work item to the end of the list and wakes the worker
+ * up immediately. If the work item already existed in the list, it's not added,
+ * because in that case it has been scheduled already but has not yet been
+ * processed.
+ */
+void nvgpu_clk_arb_worker_enqueue(struct gk20a *g,
+		struct nvgpu_clk_arb_work_item *work_item)
+{
+	clk_arb_dbg(g, " ");
+
+	(void)nvgpu_worker_enqueue(&g->clk_arb_worker.worker,
+			&work_item->worker_item);
+}
+
+/**
+ * Initialize the clk arb worker's metadata and start the background thread.
+ */
+int nvgpu_clk_arb_worker_init(struct gk20a *g)
+{
+	struct nvgpu_worker *worker = &g->clk_arb_worker.worker;
+
+	nvgpu_worker_init_name(worker, "nvgpu_clk_arb_poll", g->name);
+
+	return nvgpu_worker_init(g, worker, &clk_arb_worker_ops);
+}
+
+int nvgpu_clk_arb_init_arbiter(struct gk20a *g)
+{
+	int err = 0;
+
+	if (g->ops.clk_arb.check_clk_arb_support != NULL) {
+		if (!g->ops.clk_arb.check_clk_arb_support(g)) {
+			return 0;
+		}
+	}
+	else {
+		return 0;
+	}
+
+	nvgpu_mutex_acquire(&g->clk_arb_enable_lock);
+
+	err = g->ops.clk_arb.arbiter_clk_init(g);
+
+	nvgpu_mutex_release(&g->clk_arb_enable_lock);
+
+	return err;
+}
+
+bool nvgpu_clk_arb_has_active_req(struct gk20a *g)
+{
+	return (nvgpu_atomic_read(&g->clk_arb_global_nr) > 0);
+}
+
+static void nvgpu_clk_arb_schedule_alarm(struct gk20a *g, u32 alarm)
+{
+	struct nvgpu_clk_arb *arb = g->clk_arb;
+
+	nvgpu_clk_arb_set_global_alarm(g, alarm);
+	nvgpu_clk_arb_worker_enqueue(g, &arb->update_arb_work_item);
+}
+
+void nvgpu_clk_arb_send_thermal_alarm(struct gk20a *g)
+{
+	struct nvgpu_clk_arb *arb = g->clk_arb;
+
+	if (arb != NULL) {
+		nvgpu_clk_arb_schedule_alarm(g,
+			BIT32(NVGPU_EVENT_ALARM_THERMAL_ABOVE_THRESHOLD));
+	}
+}
+
+void nvgpu_clk_arb_worker_deinit(struct gk20a *g)
+{
+	struct nvgpu_worker *worker = &g->clk_arb_worker.worker;
+
+	nvgpu_worker_deinit(worker);
+}
+
+void nvgpu_clk_arb_cleanup_arbiter(struct gk20a *g)
+{
+	struct nvgpu_clk_arb *arb = g->clk_arb;
+
+	nvgpu_mutex_acquire(&g->clk_arb_enable_lock);
+
+	if (arb != NULL) {
+		g->ops.clk_arb.clk_arb_cleanup(g->clk_arb);
+	}
+
+	nvgpu_mutex_release(&g->clk_arb_enable_lock);
+}
+
+int nvgpu_clk_arb_init_session(struct gk20a *g,
+		struct nvgpu_clk_session **l_session)
+{
+	struct nvgpu_clk_arb *arb = g->clk_arb;
+	struct nvgpu_clk_session *session = *(l_session);
+
+	clk_arb_dbg(g, " ");
+
+	if (g->ops.clk_arb.check_clk_arb_support != NULL) {
+		if (!g->ops.clk_arb.check_clk_arb_support(g)) {
+			return 0;
+		}
+	}
+	else {
+		return 0;
+	}
+
+	session = nvgpu_kzalloc(g, sizeof(struct nvgpu_clk_session));
+	if (session == NULL) {
+		return -ENOMEM;
+	}
+	session->g = g;
+
+	nvgpu_ref_init(&session->refcount);
+
+	session->zombie = false;
+	session->target_pool[0].pstate = CTRL_PERF_PSTATE_P8;
+	/* make sure that the initialization of the pool is visible
+	 * before the update
+	 */
+	nvgpu_smp_wmb();
+	session->target = &session->target_pool[0];
+
+	nvgpu_init_list_node(&session->targets);
+	nvgpu_spinlock_init(&session->session_lock);
+
+	nvgpu_spinlock_acquire(&arb->sessions_lock);
+	nvgpu_list_add_tail(&session->link, &arb->sessions);
+	nvgpu_spinlock_release(&arb->sessions_lock);
+
+	*l_session = session;
+
+	return 0;
+}
+
+static struct nvgpu_clk_dev *
+nvgpu_clk_dev_from_refcount(struct nvgpu_ref *refcount)
+{
+	return (struct nvgpu_clk_dev *)
+	   ((uintptr_t)refcount - offsetof(struct nvgpu_clk_dev, refcount));
+};
+
+void nvgpu_clk_arb_free_fd(struct nvgpu_ref *refcount)
+{
+	struct nvgpu_clk_dev *dev = nvgpu_clk_dev_from_refcount(refcount);
+	struct nvgpu_clk_session *session = dev->session;
+	struct gk20a *g = session->g;
+
+	nvgpu_clk_notification_queue_free(g, &dev->queue);
+
+	nvgpu_atomic_dec(&g->clk_arb_global_nr);
+	nvgpu_kfree(g, dev);
+}
+
+static struct nvgpu_clk_session *
+nvgpu_clk_session_from_refcount(struct nvgpu_ref *refcount)
+{
+	return (struct nvgpu_clk_session *)
+	   ((uintptr_t)refcount - offsetof(struct nvgpu_clk_session, refcount));
+};
+
+void nvgpu_clk_arb_free_session(struct nvgpu_ref *refcount)
+{
+	struct nvgpu_clk_session *session =
+		nvgpu_clk_session_from_refcount(refcount);
+	struct nvgpu_clk_arb *arb = session->g->clk_arb;
+	struct gk20a *g = session->g;
+	struct nvgpu_clk_dev *dev, *tmp;
+
+	clk_arb_dbg(g, " ");
+
+	if (arb != NULL) {
+		nvgpu_spinlock_acquire(&arb->sessions_lock);
+		nvgpu_list_del(&session->link);
+		nvgpu_spinlock_release(&arb->sessions_lock);
+	}
+
+	nvgpu_spinlock_acquire(&session->session_lock);
+	nvgpu_list_for_each_entry_safe(dev, tmp, &session->targets,
+			nvgpu_clk_dev, node) {
+		nvgpu_ref_put(&dev->refcount, nvgpu_clk_arb_free_fd);
+		nvgpu_list_del(&dev->node);
+	}
+	nvgpu_spinlock_release(&session->session_lock);
+
+	nvgpu_kfree(g, session);
+}
+
+void nvgpu_clk_arb_release_session(struct gk20a *g,
+	struct nvgpu_clk_session *session)
+{
+	struct nvgpu_clk_arb *arb = g->clk_arb;
+
+	clk_arb_dbg(g, " ");
+
+	session->zombie = true;
+	nvgpu_ref_put(&session->refcount, nvgpu_clk_arb_free_session);
+	if (arb != NULL) {
+		nvgpu_clk_arb_worker_enqueue(g, &arb->update_arb_work_item);
+	}
+}
+#ifdef CONFIG_NVGPU_LS_PMU
+void nvgpu_clk_arb_schedule_vf_table_update(struct gk20a *g)
+{
+	struct nvgpu_clk_arb *arb = g->clk_arb;
+
+	nvgpu_clk_arb_worker_enqueue(g, &arb->update_vf_table_work_item);
+}
+
+/* This function is inherently unsafe to call while arbiter is running
+ * arbiter must be blocked before calling this function
+ */
+u32 nvgpu_clk_arb_get_current_pstate(struct gk20a *g)
+{
+	return NV_READ_ONCE(g->clk_arb->actual->pstate);
+}
+
+void nvgpu_clk_arb_pstate_change_lock(struct gk20a *g, bool lock)
+{
+	struct nvgpu_clk_arb *arb = g->clk_arb;
+
+	if (lock) {
+		nvgpu_mutex_acquire(&arb->pstate_lock);
+	} else {
+		nvgpu_mutex_release(&arb->pstate_lock);
+	}
+}
+#endif
+bool nvgpu_clk_arb_is_valid_domain(struct gk20a *g, u32 api_domain)
+{
+	u32 clk_domains = g->ops.clk_arb.get_arbiter_clk_domains(g);
+	bool ret_result = false;
+
+	switch (api_domain) {
+	case NVGPU_CLK_DOMAIN_MCLK:
+		ret_result = ((clk_domains & CTRL_CLK_DOMAIN_MCLK) != 0U) ?
+								true : false;
+		break;
+	case NVGPU_CLK_DOMAIN_GPCCLK:
+		ret_result = ((clk_domains & CTRL_CLK_DOMAIN_GPCCLK) != 0U) ?
+								true : false;
+		break;
+	default:
+		ret_result = false;
+		break;
+	}
+	return ret_result;
+}
+
+int nvgpu_clk_arb_get_arbiter_clk_range(struct gk20a *g, u32 api_domain,
+		u16 *min_mhz, u16 *max_mhz)
+{
+	int ret = -EINVAL;
+
+	switch (api_domain) {
+	case NVGPU_CLK_DOMAIN_MCLK:
+		ret = g->ops.clk_arb.get_arbiter_clk_range(g,
+				CTRL_CLK_DOMAIN_MCLK, min_mhz, max_mhz);
+		break;
+
+	case NVGPU_CLK_DOMAIN_GPCCLK:
+		ret = g->ops.clk_arb.get_arbiter_clk_range(g,
+				CTRL_CLK_DOMAIN_GPCCLK, min_mhz, max_mhz);
+		break;
+
+	default:
+		ret = -EINVAL;
+		break;
+	}
+	return ret;
+}
+
+int nvgpu_clk_arb_get_arbiter_clk_f_points(struct gk20a *g,
+	u32 api_domain, u32 *max_points, u16 *fpoints)
+{
+	int err = -EINVAL;
+
+	switch (api_domain) {
+	case NVGPU_CLK_DOMAIN_GPCCLK:
+		err = g->ops.clk_arb.get_arbiter_f_points(g,
+			CTRL_CLK_DOMAIN_GPCCLK, max_points, fpoints);
+		if ((err != 0) || (fpoints == NULL)) {
+			break;
+		}
+		err = 0;
+		break;
+	case NVGPU_CLK_DOMAIN_MCLK:
+		err = g->ops.clk_arb.get_arbiter_f_points(g,
+			CTRL_CLK_DOMAIN_MCLK, max_points, fpoints);
+		break;
+	default:
+		err = -EINVAL;
+		break;
+	}
+	return err;
+}
+
+int nvgpu_clk_arb_get_session_target_mhz(struct nvgpu_clk_session *session,
+		u32 api_domain, u16 *target_mhz)
+{
+	int err = 0;
+	struct nvgpu_clk_arb_target *target = session->target;
+
+	if (!nvgpu_clk_arb_is_valid_domain(session->g, api_domain)) {
+		return -EINVAL;
+	}
+
+	switch (api_domain) {
+		case NVGPU_CLK_DOMAIN_MCLK:
+			*target_mhz = target->mclk;
+			break;
+
+		case NVGPU_CLK_DOMAIN_GPCCLK:
+			*target_mhz = target->gpc2clk;
+			break;
+
+		default:
+			*target_mhz = 0;
+			err = -EINVAL;
+			break;
+	}
+	return err;
+}
+
+int nvgpu_clk_arb_get_arbiter_actual_mhz(struct gk20a *g,
+		u32 api_domain, u16 *actual_mhz)
+{
+	struct nvgpu_clk_arb *arb = g->clk_arb;
+	int err = 0;
+	struct nvgpu_clk_arb_target *actual = arb->actual;
+
+	if (!nvgpu_clk_arb_is_valid_domain(g, api_domain)) {
+		return -EINVAL;
+	}
+
+	switch (api_domain) {
+		case NVGPU_CLK_DOMAIN_MCLK:
+			*actual_mhz = actual->mclk;
+			break;
+
+		case NVGPU_CLK_DOMAIN_GPCCLK:
+			*actual_mhz = actual->gpc2clk;
+			break;
+
+		default:
+			*actual_mhz = 0;
+			err = -EINVAL;
+			break;
+	}
+	return err;
+}
+
+unsigned long nvgpu_clk_measure_freq(struct gk20a *g, u32 api_domain)
+{
+	unsigned long freq = 0UL;
+
+	switch (api_domain) {
+	/*
+	 * Incase of iGPU clocks to each parition (GPC, SYS, LTC, XBAR) are
+	 * generated using 1X GPCCLK and hence should be the same.
+	 */
+	case CTRL_CLK_DOMAIN_GPCCLK:
+	case CTRL_CLK_DOMAIN_SYSCLK:
+	case CTRL_CLK_DOMAIN_XBARCLK:
+		freq = g->ops.clk.get_rate(g, CTRL_CLK_DOMAIN_GPCCLK);
+		break;
+	default:
+		freq = 0UL;
+		break;
+	}
+	return freq;
+}
+
+int nvgpu_clk_arb_get_arbiter_effective_mhz(struct gk20a *g,
+		u32 api_domain, u16 *effective_mhz)
+{
+	u64 freq_mhz_u64;
+	int err = -EINVAL;
+
+	if (!nvgpu_clk_arb_is_valid_domain(g, api_domain)) {
+		return -EINVAL;
+	}
+
+	switch (api_domain) {
+	case NVGPU_CLK_DOMAIN_MCLK:
+		freq_mhz_u64 = g->ops.clk.measure_freq(g,
+					CTRL_CLK_DOMAIN_MCLK) /	1000000ULL;
+		err = 0;
+		break;
+
+	case NVGPU_CLK_DOMAIN_GPCCLK:
+		freq_mhz_u64 = g->ops.clk.measure_freq(g,
+					CTRL_CLK_DOMAIN_GPCCLK) / 1000000ULL;
+		err = 0;
+		break;
+
+	default:
+		err = -EINVAL;
+		break;
+	}
+
+	if (err == 0) {
+		nvgpu_assert(freq_mhz_u64 <= (u64)U16_MAX);
+		*effective_mhz = (u16)freq_mhz_u64;
+	}
+	return err;
+}

drivers/gpu/nvgpu/common/clk_arb/clk_arb_gp10b.c

@@ -0,0 +1,406 @@
+/*
+ * Copyright (c) 2018-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/gk20a.h>
+#include <nvgpu/clk_arb.h>
+#include <nvgpu/pmu/clk/clk.h>
+
+#include "clk_arb_gp10b.h"
+
+bool gp10b_check_clk_arb_support(struct gk20a *g)
+{
+	if (g->ops.clk_arb.get_arbiter_clk_domains != NULL) {
+		return true;
+	}
+	else {
+		return false;
+	}
+}
+
+u32 gp10b_get_arbiter_clk_domains(struct gk20a *g)
+{
+	(void)g;
+	clk_arb_dbg(g, " ");
+	return CTRL_CLK_DOMAIN_GPCCLK;
+}
+
+int gp10b_get_arbiter_f_points(struct gk20a *g,u32 api_domain,
+				u32 *num_points, u16 *freqs_in_mhz)
+{
+	int ret = 0;
+
+	clk_arb_dbg(g, " ");
+
+	switch (api_domain) {
+	case CTRL_CLK_DOMAIN_GPCCLK:
+		ret = g->ops.clk.clk_domain_get_f_points(g, CTRL_CLK_DOMAIN_GPCCLK,
+			num_points, freqs_in_mhz);
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+
+	return ret;
+}
+
+int gp10b_get_arbiter_clk_range(struct gk20a *g, u32 api_domain,
+		u16 *min_mhz, u16 *max_mhz)
+{
+	int ret = 0;
+
+	clk_arb_dbg(g, " ");
+
+	switch (api_domain) {
+	case CTRL_CLK_DOMAIN_GPCCLK:
+		ret = g->ops.clk.get_clk_range(g, CTRL_CLK_DOMAIN_GPCCLK,
+			min_mhz, max_mhz);
+		break;
+
+	default:
+		ret = -EINVAL;
+		break;
+	}
+
+	return ret;
+}
+
+int gp10b_get_arbiter_clk_default(struct gk20a *g, u32 api_domain,
+		u16 *default_mhz)
+{
+	int ret = 0;
+	u16 min_mhz, max_mhz;
+
+	clk_arb_dbg(g, " ");
+
+	switch (api_domain) {
+	case CTRL_CLK_DOMAIN_GPCCLK:
+		ret = gp10b_get_arbiter_clk_range(g, api_domain,
+			&min_mhz, &max_mhz);
+
+		if (ret == 0) {
+			*default_mhz = max_mhz;
+		}
+		break;
+
+	default:
+		ret = -EINVAL;
+		break;
+	}
+
+	return ret;
+}
+
+int gp10b_init_clk_arbiter(struct gk20a *g)
+{
+	struct nvgpu_clk_arb *arb = NULL;
+	u16 default_mhz;
+	int err;
+	int index;
+	struct nvgpu_clk_vf_table *table;
+
+	clk_arb_dbg(g, " ");
+
+	if(g->clk_arb != NULL) {
+		return 0;
+	}
+
+	arb = nvgpu_kzalloc(g, sizeof(struct nvgpu_clk_arb));
+	if (arb == NULL) {
+		return -ENOMEM;
+	}
+
+	nvgpu_mutex_init(&arb->pstate_lock);
+
+	nvgpu_spinlock_init(&arb->sessions_lock);
+	nvgpu_spinlock_init(&arb->users_lock);
+	nvgpu_spinlock_init(&arb->requests_lock);
+
+	arb->gpc2clk_f_points = nvgpu_kcalloc(g, MAX_F_POINTS, sizeof(u16));
+	if (arb->gpc2clk_f_points == NULL) {
+		err = -ENOMEM;
+		goto init_fail;
+	}
+
+	for (index = 0; index < 2; index++) {
+		table = &arb->vf_table_pool[index];
+		table->gpc2clk_num_points = MAX_F_POINTS;
+
+		table->gpc2clk_points = (struct nvgpu_clk_vf_point *)
+			nvgpu_kcalloc(g, MAX_F_POINTS,
+			sizeof(struct nvgpu_clk_vf_point));
+		if (table->gpc2clk_points == NULL) {
+			err = -ENOMEM;
+			goto init_fail;
+		}
+	}
+
+	g->clk_arb = arb;
+	arb->g = g;
+
+	err =  g->ops.clk_arb.get_arbiter_clk_default(g,
+			CTRL_CLK_DOMAIN_GPCCLK, &default_mhz);
+	if (err < 0) {
+		err = -EINVAL;
+		goto init_fail;
+	}
+
+	arb->gpc2clk_default_mhz = default_mhz;
+
+	err = g->ops.clk_arb.get_arbiter_clk_range(g, CTRL_CLK_DOMAIN_GPCCLK,
+		&arb->gpc2clk_min, &arb->gpc2clk_max);
+
+	if (err < 0) {
+		err = -EINVAL;
+		goto init_fail;
+	}
+
+	arb->actual = &arb->actual_pool[0];
+
+	nvgpu_atomic_set(&arb->req_nr, 0);
+
+	nvgpu_atomic64_set(&arb->alarm_mask, 0);
+	err = nvgpu_clk_notification_queue_alloc(g, &arb->notification_queue,
+		DEFAULT_EVENT_NUMBER);
+	if (err < 0) {
+		goto init_fail;
+	}
+
+	nvgpu_init_list_node(&arb->users);
+	nvgpu_init_list_node(&arb->sessions);
+	nvgpu_init_list_node(&arb->requests);
+
+	err = nvgpu_cond_init(&arb->request_wq);
+	if (err < 0) {
+		goto init_fail;
+	}
+
+	nvgpu_init_list_node(&arb->update_arb_work_item.worker_item);
+	arb->update_arb_work_item.arb = arb;
+	arb->update_arb_work_item.item_type = CLK_ARB_WORK_UPDATE_ARB;
+
+	err = nvgpu_clk_arb_worker_init(g);
+	if (err < 0) {
+		goto init_fail;
+	}
+
+	/* This is set for the duration of the default req */
+	nvgpu_atomic_inc(&g->clk_arb_global_nr);
+
+	nvgpu_clk_arb_worker_enqueue(g, &arb->update_arb_work_item);
+
+	do {
+		/* Check that first run is completed */
+		nvgpu_smp_mb();
+		NVGPU_COND_WAIT_INTERRUPTIBLE(&arb->request_wq,
+			nvgpu_atomic_read(&arb->req_nr) != 0, 0U);
+	} while (nvgpu_atomic_read(&arb->req_nr) == 0);
+
+	/* Once the default request is completed, reduce the usage count */
+	nvgpu_atomic_dec(&g->clk_arb_global_nr);
+
+	return arb->status;
+
+init_fail:
+	nvgpu_kfree(g, arb->gpc2clk_f_points);
+
+	for (index = 0; index < 2; index++) {
+		nvgpu_kfree(g, arb->vf_table_pool[index].gpc2clk_points);
+	}
+
+	nvgpu_mutex_destroy(&arb->pstate_lock);
+	nvgpu_kfree(g, arb);
+
+	return err;
+}
+
+void gp10b_clk_arb_run_arbiter_cb(struct nvgpu_clk_arb *arb)
+{
+	struct nvgpu_clk_session *session;
+	struct nvgpu_clk_dev *dev;
+	struct nvgpu_clk_dev *tmp;
+	struct nvgpu_clk_arb_target *target, *actual;
+	struct gk20a *g = arb->g;
+
+	bool gpc2clk_set;
+
+	int status = 0;
+	unsigned long rounded_rate = 0;
+
+	u16 gpc2clk_target, gpc2clk_session_target;
+
+	clk_arb_dbg(g, " ");
+
+	/* Only one arbiter should be running */
+	gpc2clk_target = 0;
+
+	nvgpu_spinlock_acquire(&arb->sessions_lock);
+	nvgpu_list_for_each_entry(session, &arb->sessions,
+			nvgpu_clk_session, link) {
+		if (session->zombie) {
+			continue;
+		}
+		gpc2clk_set = false;
+		target = (session->target == &session->target_pool[0] ?
+				&session->target_pool[1] :
+				&session->target_pool[0]);
+		nvgpu_spinlock_acquire(&session->session_lock);
+		if (!nvgpu_list_empty(&session->targets)) {
+			/* Copy over state */
+			target->gpc2clk = session->target->gpc2clk;
+			/* Query the latest committed request */
+			nvgpu_list_for_each_entry_safe(dev, tmp, &session->targets,
+						nvgpu_clk_dev, node) {
+				if (!gpc2clk_set &&
+					dev->gpc2clk_target_mhz != (u16)0) {
+					target->gpc2clk =
+						dev->gpc2clk_target_mhz;
+					gpc2clk_set = true;
+				}
+				nvgpu_ref_get(&dev->refcount);
+				nvgpu_list_del(&dev->node);
+				nvgpu_spinlock_acquire(&arb->requests_lock);
+				nvgpu_list_add(&dev->node, &arb->requests);
+				nvgpu_spinlock_release(&arb->requests_lock);
+			}
+			session->target = target;
+		}
+		nvgpu_spinlock_release(&session->session_lock);
+
+		gpc2clk_target =
+			gpc2clk_target > session->target->gpc2clk ?
+			gpc2clk_target : session->target->gpc2clk;
+	}
+	nvgpu_spinlock_release(&arb->sessions_lock);
+
+	gpc2clk_target = (gpc2clk_target > (u16)0) ? gpc2clk_target :
+			arb->gpc2clk_default_mhz;
+
+	if (gpc2clk_target < arb->gpc2clk_min) {
+		gpc2clk_target = arb->gpc2clk_min;
+	}
+
+	if (gpc2clk_target > arb->gpc2clk_max) {
+		gpc2clk_target = arb->gpc2clk_max;
+	}
+
+	gpc2clk_session_target = gpc2clk_target;
+
+	if (arb->actual->gpc2clk == gpc2clk_target) {
+		nvgpu_atomic_inc(&arb->req_nr);
+		nvgpu_cond_signal_interruptible(&arb->request_wq);
+		goto exit_arb;
+	}
+
+	nvgpu_mutex_acquire(&arb->pstate_lock);
+
+	status = g->ops.clk.clk_get_round_rate(g,
+		CTRL_CLK_DOMAIN_GPCCLK, gpc2clk_session_target * 1000000UL, &rounded_rate);
+
+	clk_arb_dbg(g, "rounded_rate: %lu\n",
+		rounded_rate);
+
+	if (status < 0) {
+		arb->status = status;
+		nvgpu_mutex_release(&arb->pstate_lock);
+
+		/* make status visible */
+		nvgpu_smp_mb();
+		nvgpu_atomic_inc(&arb->req_nr);
+		nvgpu_cond_signal_interruptible(&arb->request_wq);
+		goto exit_arb;
+	}
+
+	/* the igpu set_rate accepts freq in Hz */
+	status = g->ops.clk.set_rate(g, CTRL_CLK_DOMAIN_GPCCLK, rounded_rate);
+
+	if (status < 0) {
+		arb->status = status;
+		nvgpu_mutex_release(&arb->pstate_lock);
+
+		/* make status visible */
+		nvgpu_smp_mb();
+		nvgpu_atomic_inc(&arb->req_nr);
+		nvgpu_cond_signal_interruptible(&arb->request_wq);
+		goto exit_arb;
+	}
+
+	actual = ((NV_READ_ONCE(arb->actual)) == &arb->actual_pool[0] ?
+			&arb->actual_pool[1] : &arb->actual_pool[0]);
+
+	/* do not reorder this pointer */
+	nvgpu_smp_rmb();
+	actual->gpc2clk = gpc2clk_target;
+	arb->status = 0;
+
+	/* Make changes visible to other threads */
+	nvgpu_smp_wmb();
+	arb->actual = actual;
+
+	/* status must be visible before atomic inc */
+	nvgpu_smp_wmb();
+	nvgpu_atomic_inc(&arb->req_nr);
+
+	/* Unlock pstate change for PG */
+	nvgpu_mutex_release(&arb->pstate_lock);
+
+	nvgpu_cond_signal_interruptible(&arb->request_wq);
+
+exit_arb:
+	if (status < 0) {
+		nvgpu_err(g, "Error in arbiter update");
+	}
+
+	/* notify completion for all requests */
+	nvgpu_spinlock_acquire(&arb->requests_lock);
+	nvgpu_list_for_each_entry_safe(dev, tmp, &arb->requests,
+			nvgpu_clk_dev, node) {
+		u32 tmp_poll_mask = NVGPU_POLLIN | NVGPU_POLLRDNORM;
+		nvgpu_atomic_set(&dev->poll_mask,
+			(int)tmp_poll_mask);
+		nvgpu_clk_arb_event_post_event(dev);
+		nvgpu_ref_put(&dev->refcount, nvgpu_clk_arb_free_fd);
+		nvgpu_list_del(&dev->node);
+	}
+	nvgpu_spinlock_release(&arb->requests_lock);
+
+	clk_arb_dbg(g, "done");
+}
+
+void gp10b_clk_arb_cleanup(struct nvgpu_clk_arb *arb)
+{
+	struct gk20a *g = arb->g;
+	int index;
+
+	nvgpu_clk_arb_worker_deinit(g);
+	nvgpu_kfree(g, arb->gpc2clk_f_points);
+
+	for (index = 0; index < 2; index++) {
+		nvgpu_kfree(g,
+			arb->vf_table_pool[index].gpc2clk_points);
+	}
+
+	nvgpu_mutex_destroy(&g->clk_arb->pstate_lock);
+	nvgpu_kfree(g, g->clk_arb);
+
+	g->clk_arb = NULL;
+}

drivers/gpu/nvgpu/common/clk_arb/clk_arb_gp10b.h

@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2018-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+#ifndef CLK_ARB_GP10B_H
+#define CLK_ARB_GP10B_H
+
+struct nvgpu_clk_session;
+struct nvgpu_clk_arb;
+
+bool gp10b_check_clk_arb_support(struct gk20a *g);
+u32 gp10b_get_arbiter_clk_domains(struct gk20a *g);
+int gp10b_get_arbiter_f_points(struct gk20a *g,u32 api_domain,
+				u32 *num_points, u16 *freqs_in_mhz);
+int gp10b_get_arbiter_clk_range(struct gk20a *g, u32 api_domain,
+		u16 *min_mhz, u16 *max_mhz);
+int gp10b_get_arbiter_clk_default(struct gk20a *g, u32 api_domain,
+		u16 *default_mhz);
+int gp10b_init_clk_arbiter(struct gk20a *g);
+void gp10b_clk_arb_run_arbiter_cb(struct nvgpu_clk_arb *arb);
+void gp10b_clk_arb_cleanup(struct nvgpu_clk_arb *arb);
+
+#endif /* CLK_ARB_GP106_H */

drivers/gpu/nvgpu/common/clk_arb/clk_arb_gv100.c

@@ -0,0 +1,560 @@
+/*
+ * Copyright (c) 2016-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/gk20a.h>
+#include <nvgpu/clk_arb.h>
+#include <nvgpu/pmu/clk/clk.h>
+#include <nvgpu/timers.h>
+#include <nvgpu/boardobjgrp_e255.h>
+#include <nvgpu/pmu/perf.h>
+
+#include "clk_arb_gv100.h"
+
+bool gv100_check_clk_arb_support(struct gk20a *g)
+{
+	if ((g->ops.clk_arb.get_arbiter_clk_domains != NULL) &&
+			nvgpu_is_enabled(g, NVGPU_PMU_PSTATE)){
+		return true;
+	}
+	else {
+		return false;
+	}
+}
+
+u32 gv100_get_arbiter_clk_domains(struct gk20a *g)
+{
+	(void)g;
+	return (CTRL_CLK_DOMAIN_GPCCLK);
+}
+
+int gv100_get_arbiter_f_points(struct gk20a *g,u32 api_domain,
+				u32 *num_points, u16 *freqs_in_mhz)
+{
+	return g->ops.clk.clk_domain_get_f_points(g,
+		api_domain, num_points, freqs_in_mhz);
+}
+
+int gv100_get_arbiter_clk_range(struct gk20a *g, u32 api_domain,
+		u16 *min_mhz, u16 *max_mhz)
+{
+	u32 clkwhich;
+	struct nvgpu_pmu_perf_pstate_clk_info *p0_info;
+	u16 max_min_freq_mhz;
+	u16 limit_min_mhz;
+	u16 gpcclk_cap_mhz;
+	bool error_status = false;
+
+	switch (api_domain) {
+	case CTRL_CLK_DOMAIN_MCLK:
+		clkwhich = CLKWHICH_MCLK;
+		break;
+
+	case CTRL_CLK_DOMAIN_GPCCLK:
+		clkwhich = CLKWHICH_GPCCLK;
+		break;
+
+	default:
+		error_status = true;
+		break;
+	}
+
+	if (error_status == true) {
+		return -EINVAL;
+	}
+
+	p0_info = nvgpu_pmu_perf_pstate_get_clk_set_info(g,
+			CTRL_PERF_PSTATE_P0, clkwhich);
+	if (p0_info == NULL) {
+		return -EINVAL;
+	}
+
+	limit_min_mhz = p0_info->min_mhz;
+	gpcclk_cap_mhz = p0_info->max_mhz;
+
+	max_min_freq_mhz = nvgpu_pmu_clk_fll_get_min_max_freq(g);
+	/*
+	 * When DVCO min is 0 in vbios update it to DVCO_MIN_DEFAULT_MHZ.
+	 */
+	if (max_min_freq_mhz == 0U) {
+		max_min_freq_mhz = DVCO_MIN_DEFAULT_MHZ;
+	}
+
+	/*
+	 * Needed for DVCO min.
+	 */
+	if (api_domain == CTRL_CLK_DOMAIN_GPCCLK) {
+		if ((max_min_freq_mhz != 0U) &&
+			(max_min_freq_mhz >= limit_min_mhz)) {
+			limit_min_mhz = nvgpu_safe_cast_u32_to_u16(
+				nvgpu_safe_add_u32(max_min_freq_mhz, 1U));
+		}
+		if ((g->clk_arb->gpc_cap_clkmhz != 0U) &&
+			(p0_info->max_mhz > g->clk_arb->gpc_cap_clkmhz )) {
+			gpcclk_cap_mhz = g->clk_arb->gpc_cap_clkmhz;
+		}
+	}
+	*min_mhz = limit_min_mhz;
+	*max_mhz = gpcclk_cap_mhz;
+
+	return 0;
+}
+
+int gv100_get_arbiter_clk_default(struct gk20a *g, u32 api_domain,
+		u16 *default_mhz)
+{
+	u32 clkwhich;
+	struct nvgpu_pmu_perf_pstate_clk_info *p0_info;
+	bool error_status = false;
+	u16 gpcclk_cap_mhz;
+
+	switch (api_domain) {
+	case CTRL_CLK_DOMAIN_MCLK:
+		clkwhich = CLKWHICH_MCLK;
+		break;
+
+	case CTRL_CLK_DOMAIN_GPCCLK:
+		clkwhich = CLKWHICH_GPCCLK;
+		break;
+
+	default:
+		error_status = true;
+		break;
+	}
+
+	if (error_status == true) {
+		return -EINVAL;
+	}
+
+	p0_info = nvgpu_pmu_perf_pstate_get_clk_set_info(g,
+			CTRL_PERF_PSTATE_P0, clkwhich);
+	if (p0_info == NULL) {
+		return -EINVAL;
+	}
+
+	gpcclk_cap_mhz = p0_info->max_mhz;
+	if (api_domain == CTRL_CLK_DOMAIN_GPCCLK) {
+		if ((g->clk_arb->gpc_cap_clkmhz != 0U) &&
+			(p0_info->max_mhz > g->clk_arb->gpc_cap_clkmhz )) {
+			gpcclk_cap_mhz = g->clk_arb->gpc_cap_clkmhz;
+		}
+	}
+	*default_mhz = gpcclk_cap_mhz;
+
+	return 0;
+}
+
+int gv100_init_clk_arbiter(struct gk20a *g)
+{
+	struct nvgpu_clk_arb *arb;
+	u16 default_mhz;
+	int err;
+	int index;
+	struct nvgpu_clk_vf_table *table;
+	clk_arb_dbg(g, " ");
+
+	if (g->clk_arb != NULL) {
+		return 0;
+	}
+	arb = nvgpu_kzalloc(g, sizeof(struct nvgpu_clk_arb));
+	if (arb == NULL) {
+		return -ENOMEM;
+	}
+
+	nvgpu_mutex_init(&arb->pstate_lock);
+	nvgpu_spinlock_init(&arb->sessions_lock);
+	nvgpu_spinlock_init(&arb->users_lock);
+	nvgpu_spinlock_init(&arb->requests_lock);
+
+	arb->mclk_f_points = nvgpu_kcalloc(g, MAX_F_POINTS, sizeof(u16));
+	if (arb->mclk_f_points == NULL) {
+		err = -ENOMEM;
+		goto init_fail;
+	}
+
+	arb->gpc2clk_f_points = nvgpu_kcalloc(g, MAX_F_POINTS, sizeof(u16));
+	if (arb->gpc2clk_f_points == NULL) {
+		err = -ENOMEM;
+		goto init_fail;
+	}
+
+	for (index = 0; index < 2; index++) {
+		table = &arb->vf_table_pool[index];
+		table->gpc2clk_num_points = MAX_F_POINTS;
+		table->mclk_num_points = MAX_F_POINTS;
+
+		table->gpc2clk_points = nvgpu_kcalloc(g, MAX_F_POINTS,
+			sizeof(struct nvgpu_clk_vf_point));
+		if (table->gpc2clk_points == NULL) {
+			err = -ENOMEM;
+			goto init_fail;
+		}
+
+
+		table->mclk_points = nvgpu_kcalloc(g, MAX_F_POINTS,
+			sizeof(struct nvgpu_clk_vf_point));
+		if (table->mclk_points == NULL) {
+			err = -ENOMEM;
+			goto init_fail;
+		}
+	}
+
+	g->clk_arb = arb;
+	arb->g = g;
+
+	err =  g->ops.clk_arb.get_arbiter_clk_default(g,
+			CTRL_CLK_DOMAIN_MCLK, &default_mhz);
+	if (err < 0) {
+		err = -EINVAL;
+		goto init_fail;
+	}
+
+	arb->mclk_default_mhz = default_mhz;
+
+	err =  g->ops.clk_arb.get_arbiter_clk_default(g,
+			CTRL_CLK_DOMAIN_GPCCLK, &default_mhz);
+	if (err < 0) {
+		err = -EINVAL;
+		goto init_fail;
+	}
+
+	arb->gpc2clk_default_mhz = default_mhz;
+
+	arb->actual = &arb->actual_pool[0];
+
+	nvgpu_atomic_set(&arb->req_nr, 0);
+
+	nvgpu_atomic64_set(&arb->alarm_mask, 0);
+	err = nvgpu_clk_notification_queue_alloc(g, &arb->notification_queue,
+		DEFAULT_EVENT_NUMBER);
+	if (err < 0) {
+		goto init_fail;
+	}
+	nvgpu_init_list_node(&arb->users);
+	nvgpu_init_list_node(&arb->sessions);
+	nvgpu_init_list_node(&arb->requests);
+
+	(void)nvgpu_cond_init(&arb->request_wq);
+
+	nvgpu_init_list_node(&arb->update_vf_table_work_item.worker_item);
+	nvgpu_init_list_node(&arb->update_arb_work_item.worker_item);
+	arb->update_vf_table_work_item.arb = arb;
+	arb->update_arb_work_item.arb = arb;
+	arb->update_vf_table_work_item.item_type = CLK_ARB_WORK_UPDATE_VF_TABLE;
+	arb->update_arb_work_item.item_type = CLK_ARB_WORK_UPDATE_ARB;
+	err = nvgpu_clk_arb_worker_init(g);
+	if (err < 0) {
+		goto init_fail;
+	}
+
+	if (g->dgpu_max_clk != 0U) {
+		g->dgpu_max_clk = (g->dgpu_max_clk /
+			FREQ_STEP_SIZE_MHZ) * FREQ_STEP_SIZE_MHZ;
+		arb->gpc_cap_clkmhz = g->dgpu_max_clk;
+	}
+#ifdef CONFIG_DEBUG_FS
+	arb->debug = &arb->debug_pool[0];
+
+	if (!arb->debugfs_set) {
+		if (nvgpu_clk_arb_debugfs_init(g))
+			arb->debugfs_set = true;
+	}
+#endif
+	err = nvgpu_clk_vf_point_cache(g);
+	if (err < 0) {
+		goto init_fail;
+	}
+
+	err = nvgpu_clk_arb_update_vf_table(arb);
+	if (err < 0) {
+		goto init_fail;
+	}
+
+	do {
+		/* Check that first run is completed */
+		nvgpu_smp_mb();
+		NVGPU_COND_WAIT_INTERRUPTIBLE(&arb->request_wq,
+			nvgpu_atomic_read(&arb->req_nr), 0U);
+	} while (nvgpu_atomic_read(&arb->req_nr) == 0);
+	return arb->status;
+
+init_fail:
+	nvgpu_kfree(g, arb->gpc2clk_f_points);
+	nvgpu_kfree(g, arb->mclk_f_points);
+
+	for (index = 0; index < 2; index++) {
+		nvgpu_kfree(g, arb->vf_table_pool[index].gpc2clk_points);
+		nvgpu_kfree(g, arb->vf_table_pool[index].mclk_points);
+	}
+
+	nvgpu_mutex_destroy(&arb->pstate_lock);
+	nvgpu_kfree(g, arb);
+
+	return err;
+}
+
+void gv100_clk_arb_run_arbiter_cb(struct nvgpu_clk_arb *arb)
+{
+	struct nvgpu_clk_session *session;
+	struct nvgpu_clk_dev *dev;
+	struct nvgpu_clk_dev *tmp;
+	struct nvgpu_clk_arb_target *target, *actual;
+	struct gk20a *g = arb->g;
+
+	u32 current_pstate = VF_POINT_INVALID_PSTATE;
+	u32 voltuv = 0;
+	bool mclk_set, gpc2clk_set;
+	u32 alarms_notified = 0;
+	u32 current_alarm;
+	int status = 0;
+	/* Temporary variables for checking target frequency */
+	u16 gpc2clk_target, mclk_target;
+	struct nvgpu_clk_slave_freq vf_point;
+
+#ifdef CONFIG_DEBUG_FS
+	s64 t0, t1;
+	struct nvgpu_clk_arb_debug *debug;
+
+#endif
+
+	clk_arb_dbg(g, " ");
+
+	/* bail out if gpu is down */
+	if (nvgpu_atomic64_read(&arb->alarm_mask) & EVENT(ALARM_GPU_LOST)) {
+		goto exit_arb;
+	}
+
+#ifdef CONFIG_DEBUG_FS
+	t0 = nvgpu_current_time_ns();
+#endif
+
+	/* Only one arbiter should be running */
+	gpc2clk_target = 0;
+	mclk_target = 0;
+	nvgpu_spinlock_acquire(&arb->sessions_lock);
+	nvgpu_list_for_each_entry(session, &arb->sessions,
+			nvgpu_clk_session, link) {
+		if (!session->zombie) {
+			mclk_set = false;
+			gpc2clk_set = false;
+			target = (session->target == &session->target_pool[0] ?
+					&session->target_pool[1] :
+					&session->target_pool[0]);
+			nvgpu_spinlock_acquire(&session->session_lock);
+			if (!nvgpu_list_empty(&session->targets)) {
+				/* Copy over state */
+				target->mclk = session->target->mclk;
+				target->gpc2clk = session->target->gpc2clk;
+				/* Query the latest committed request */
+				nvgpu_list_for_each_entry_safe(dev, tmp,
+				 &session->targets, nvgpu_clk_dev, node) {
+					if ((mclk_set == false) && (dev->mclk_target_mhz != 0U)) {
+						target->mclk =
+							dev->mclk_target_mhz;
+						mclk_set = true;
+					}
+					if ((gpc2clk_set == false) &&
+						(dev->gpc2clk_target_mhz != 0U)) {
+						target->gpc2clk =
+							dev->gpc2clk_target_mhz;
+						gpc2clk_set = true;
+					}
+					nvgpu_ref_get(&dev->refcount);
+					nvgpu_list_del(&dev->node);
+					nvgpu_spinlock_acquire(
+						&arb->requests_lock);
+					nvgpu_list_add(
+						&dev->node, &arb->requests);
+					nvgpu_spinlock_release(&arb->requests_lock);
+				}
+				session->target = target;
+			}
+			nvgpu_spinlock_release(
+				&session->session_lock);
+
+			mclk_target = mclk_target > session->target->mclk ?
+				mclk_target : session->target->mclk;
+
+			gpc2clk_target =
+				gpc2clk_target > session->target->gpc2clk ?
+				gpc2clk_target : session->target->gpc2clk;
+		}
+	}
+	nvgpu_spinlock_release(&arb->sessions_lock);
+
+	gpc2clk_target = (gpc2clk_target > 0U) ? gpc2clk_target :
+			arb->gpc2clk_default_mhz;
+
+	if (gpc2clk_target < arb->gpc2clk_min) {
+		gpc2clk_target = arb->gpc2clk_min;
+	}
+
+	if (gpc2clk_target > arb->gpc2clk_max) {
+		gpc2clk_target = arb->gpc2clk_max;
+	}
+
+	mclk_target = (mclk_target > 0U) ? mclk_target :
+			arb->mclk_default_mhz;
+
+	if (mclk_target < arb->mclk_min) {
+		mclk_target = arb->mclk_min;
+	}
+
+	if (mclk_target > arb->mclk_max) {
+		mclk_target = arb->mclk_max;
+	}
+
+	if ((arb->gpc_cap_clkmhz != 0U) &&
+			(gpc2clk_target > arb->gpc_cap_clkmhz)) {
+		gpc2clk_target = arb->gpc_cap_clkmhz;
+	}
+
+	vf_point.gpc_mhz = gpc2clk_target;
+	(void)nvgpu_clk_arb_find_slave_points(arb, &vf_point);
+	if (status != 0) {
+		nvgpu_err(g, "Unable to get slave frequency");
+		goto exit_arb;
+	}
+
+	status = nvgpu_pmu_perf_changeseq_set_clks(g, &vf_point);
+	if (status != 0) {
+		nvgpu_err(g, "Unable to program frequency");
+		goto exit_arb;
+	}
+
+	actual = NV_READ_ONCE(arb->actual) == &arb->actual_pool[0] ?
+			&arb->actual_pool[1] : &arb->actual_pool[0];
+
+	/* do not reorder this pointer */
+	nvgpu_smp_rmb();
+	actual->gpc2clk = gpc2clk_target;
+	actual->mclk = mclk_target;
+	arb->voltuv_actual = voltuv;
+	actual->pstate = current_pstate;
+	arb->status = status;
+
+	/* Make changes visible to other threads */
+	nvgpu_smp_wmb();
+	arb->actual = actual;
+
+	/* status must be visible before atomic inc */
+	nvgpu_smp_wmb();
+	nvgpu_atomic_inc(&arb->req_nr);
+
+	/* VF Update complete */
+	nvgpu_clk_arb_set_global_alarm(g, EVENT(VF_UPDATE));
+
+	nvgpu_cond_signal_interruptible(&arb->request_wq);
+#ifdef CONFIG_DEBUG_FS
+	t1 = nvgpu_current_time_ns();
+
+	debug = arb->debug == &arb->debug_pool[0] ?
+		&arb->debug_pool[1] : &arb->debug_pool[0];
+
+	memcpy(debug, arb->debug, sizeof(arb->debug_pool[0]));
+	debug->switch_num++;
+
+	if (debug->switch_num == 1) {
+		debug->switch_max = debug->switch_min =
+			debug->switch_avg = (t1-t0)/1000;
+		debug->switch_std = 0;
+	} else {
+		s64 prev_avg;
+		s64 curr = (t1-t0)/1000;
+
+		debug->switch_max = curr > debug->switch_max ?
+			curr : debug->switch_max;
+		debug->switch_min = debug->switch_min ?
+			(curr < debug->switch_min ?
+				curr : debug->switch_min) : curr;
+		prev_avg = debug->switch_avg;
+		debug->switch_avg = (curr +
+			(debug->switch_avg * (debug->switch_num-1))) /
+			debug->switch_num;
+		debug->switch_std +=
+			(curr - debug->switch_avg) * (curr - prev_avg);
+	}
+	/* commit changes before exchanging debug pointer */
+	nvgpu_smp_wmb();
+	arb->debug = debug;
+#endif
+
+exit_arb:
+	if (status < 0) {
+		nvgpu_err(g, "Error in arbiter update");
+		nvgpu_clk_arb_set_global_alarm(g,
+			EVENT(ALARM_CLOCK_ARBITER_FAILED));
+	}
+
+	current_alarm = (u32) nvgpu_atomic64_read(&arb->alarm_mask);
+	/* notify completion for all requests */
+	nvgpu_spinlock_acquire(&arb->requests_lock);
+	nvgpu_list_for_each_entry_safe(dev, tmp, &arb->requests,
+			nvgpu_clk_dev, node) {
+		/* avoid casting composite expression below */
+		u32 tmp_mask = NVGPU_POLLIN | NVGPU_POLLRDNORM;
+
+		nvgpu_atomic_set(&dev->poll_mask, (int)tmp_mask);
+		nvgpu_clk_arb_event_post_event(dev);
+		nvgpu_ref_put(&dev->refcount, nvgpu_clk_arb_free_fd);
+		nvgpu_list_del(&dev->node);
+	}
+	nvgpu_spinlock_release(&arb->requests_lock);
+
+	nvgpu_atomic_set(&arb->notification_queue.head,
+		nvgpu_atomic_read(&arb->notification_queue.tail));
+	/* notify event for all users */
+	nvgpu_spinlock_acquire(&arb->users_lock);
+	nvgpu_list_for_each_entry(dev, &arb->users, nvgpu_clk_dev, link) {
+		alarms_notified |=
+			nvgpu_clk_arb_notify(dev, arb->actual, current_alarm);
+	}
+	nvgpu_spinlock_release(&arb->users_lock);
+
+	/* clear alarms */
+	nvgpu_clk_arb_clear_global_alarm(g, alarms_notified &
+		~EVENT(ALARM_GPU_LOST));
+}
+
+void gv100_clk_arb_cleanup(struct nvgpu_clk_arb *arb)
+{
+	struct gk20a *g = arb->g;
+	int index;
+
+	nvgpu_kfree(g, arb->gpc2clk_f_points);
+	nvgpu_kfree(g, arb->mclk_f_points);
+
+	for (index = 0; index < 2; index++) {
+		nvgpu_kfree(g,
+			arb->vf_table_pool[index].gpc2clk_points);
+		nvgpu_kfree(g, arb->vf_table_pool[index].mclk_points);
+	}
+
+	nvgpu_mutex_destroy(&g->clk_arb->pstate_lock);
+	nvgpu_kfree(g, g->clk_arb);
+
+	g->clk_arb = NULL;
+}
+
+void gv100_stop_clk_arb_threads(struct gk20a *g)
+{
+	nvgpu_clk_arb_worker_deinit(g);
+}

drivers/gpu/nvgpu/common/clk_arb/clk_arb_gv100.h

@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2016-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+#ifndef CLK_ARB_GV100_H
+#define CLK_ARB_GV100_H
+
+struct nvgpu_clk_session;
+struct nvgpu_clk_arb;
+
+#define DVCO_MIN_DEFAULT_MHZ	405
+
+bool gv100_check_clk_arb_support(struct gk20a *g);
+u32 gv100_get_arbiter_clk_domains(struct gk20a *g);
+int gv100_get_arbiter_f_points(struct gk20a *g,u32 api_domain,
+				u32 *num_points, u16 *freqs_in_mhz);
+int gv100_get_arbiter_clk_range(struct gk20a *g, u32 api_domain,
+		u16 *min_mhz, u16 *max_mhz);
+int gv100_get_arbiter_clk_default(struct gk20a *g, u32 api_domain,
+		u16 *default_mhz);
+int gv100_init_clk_arbiter(struct gk20a *g);
+void gv100_clk_arb_run_arbiter_cb(struct nvgpu_clk_arb *arb);
+void gv100_clk_arb_cleanup(struct nvgpu_clk_arb *arb);
+void gv100_stop_clk_arb_threads(struct gk20a *g);
+#endif /* CLK_ARB_GV100_H */

drivers/gpu/nvgpu/common/cyclestats/cyclestats.c

@@ -0,0 +1,169 @@
+/*
+ * Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/gk20a.h>
+#include <nvgpu/regops.h>
+#include <nvgpu/log.h>
+#include <nvgpu/io.h>
+#include <nvgpu/lock.h>
+#include <nvgpu/channel.h>
+#include <nvgpu/cyclestats.h>
+
+#include "cyclestats_priv.h"
+
+static inline bool is_valid_cyclestats_bar0_offset_gk20a(struct gk20a *g,
+							 u32 offset)
+{
+	/* support only 24-bit 4-byte aligned offsets */
+	bool valid = !(offset & 0xFF000003U);
+
+	if (g->allow_all) {
+		return true;
+	}
+
+	/* whitelist check */
+	valid = valid &&
+		is_bar0_global_offset_whitelisted_gk20a(g, offset);
+	/* resource size check in case there was a problem
+	 * with allocating the assumed size of bar0 */
+	valid = valid && nvgpu_io_valid_reg(g, offset);
+	return valid;
+}
+
+void nvgpu_cyclestats_exec(struct gk20a *g,
+		struct nvgpu_channel *ch, u32 offset)
+{
+	void *virtual_address;
+	u32 buffer_size;
+	bool exit;
+
+	/* GL will never use payload 0 for cycle state */
+	if ((ch->cyclestate.cyclestate_buffer == NULL) || (offset == 0U)) {
+		return;
+	}
+
+	nvgpu_mutex_acquire(&ch->cyclestate.cyclestate_buffer_mutex);
+
+	virtual_address = ch->cyclestate.cyclestate_buffer;
+	buffer_size = ch->cyclestate.cyclestate_buffer_size;
+	exit = false;
+
+	while (!exit) {
+		struct share_buffer_head *sh_hdr;
+		u32 min_element_size;
+
+		/* validate offset */
+		if (offset + sizeof(struct share_buffer_head) > buffer_size ||
+		    offset + sizeof(struct share_buffer_head) < offset) {
+			nvgpu_err(g,
+				  "cyclestats buffer overrun at offset 0x%x",
+				  offset);
+			break;
+		}
+
+		sh_hdr = (struct share_buffer_head *)
+			 ((char *)virtual_address + offset);
+
+		min_element_size =
+			U32(sh_hdr->operation == OP_END ?
+			 sizeof(struct share_buffer_head) :
+			 sizeof(struct nvgpu_cyclestate_buffer_elem));
+
+		/* validate sh_hdr->size */
+		if (sh_hdr->size < min_element_size ||
+		    offset + sh_hdr->size > buffer_size ||
+		    offset + sh_hdr->size < offset) {
+			nvgpu_err(g,
+				  "bad cyclestate buffer header size at offset 0x%x",
+				  offset);
+			sh_hdr->failed = U32(true);
+			break;
+		}
+
+		switch (sh_hdr->operation) {
+		case OP_END:
+			exit = true;
+			break;
+
+		case BAR0_READ32:
+		case BAR0_WRITE32:
+		{
+			struct nvgpu_cyclestate_buffer_elem *op_elem =
+				(struct nvgpu_cyclestate_buffer_elem *)sh_hdr;
+			bool valid = is_valid_cyclestats_bar0_offset_gk20a(
+						g, op_elem->offset_bar0);
+			u32 raw_reg;
+			u64 mask_orig;
+			u64 v;
+
+			if (!valid) {
+				nvgpu_err(g,
+					   "invalid cycletstats op offset: 0x%x",
+					   op_elem->offset_bar0);
+
+				exit = true;
+				sh_hdr->failed = U32(exit);
+				break;
+			}
+
+			mask_orig =
+				((1ULL << (op_elem->last_bit + 1)) - 1) &
+				~((1ULL << op_elem->first_bit) - 1);
+
+			raw_reg = nvgpu_readl(g, op_elem->offset_bar0);
+
+			switch (sh_hdr->operation) {
+			case BAR0_READ32:
+				op_elem->data =	((raw_reg & mask_orig)
+							>> op_elem->first_bit);
+				break;
+
+			case BAR0_WRITE32:
+				v = 0;
+				if ((unsigned int)mask_orig !=
+							~((unsigned int)0)) {
+					v = (unsigned int)
+						(raw_reg & ~mask_orig);
+				}
+
+				v |= ((op_elem->data << op_elem->first_bit)
+							& mask_orig);
+				nvgpu_writel(g,op_elem->offset_bar0,
+					     (unsigned int)v);
+				break;
+			default:
+				/* nop ok?*/
+				break;
+			}
+		}
+		break;
+
+		default:
+			/* no operation content case */
+			exit = true;
+			break;
+		}
+		sh_hdr->completed = U32(true);
+		offset += sh_hdr->size;
+	}
+	nvgpu_mutex_release(&ch->cyclestate.cyclestate_buffer_mutex);
+}

drivers/gpu/nvgpu/common/cyclestats/cyclestats_priv.h

@@ -0,0 +1,60 @@
+/*
+ * Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef NVGPU_CYCLESTATS_PRIV_H
+#define NVGPU_CYCLESTATS_PRIV_H
+
+#include <nvgpu/types.h>
+
+#define MULTICHAR_TAG(a, b, c, d) (((a) << 24) | ((b) << 16) | ((c) << 8) | (d))
+
+enum BAR0_DEBUG_OPERATION {
+	BARO_ZERO_NOP = 0,
+	OP_END = MULTICHAR_TAG('D', 'O', 'N', 'E'),
+	BAR0_READ32 = MULTICHAR_TAG('0', 'R', '3', '2'),
+	BAR0_WRITE32 = MULTICHAR_TAG('0', 'W', '3', '2'),
+};
+
+struct share_buffer_head {
+	enum BAR0_DEBUG_OPERATION operation;
+/* size of the operation item */
+	u32 size;
+	u32 completed;
+	u32 failed;
+	u64 context;
+	u64 completion_callback;
+};
+
+struct nvgpu_cyclestate_buffer_elem {
+	struct share_buffer_head	head;
+/* in */
+	u64 p_data;
+	u64 p_done;
+	u32 offset_bar0;
+	u16 first_bit;
+	u16 last_bit;
+/* out */
+/* keep 64 bits to be consistent */
+	u64 data;
+};
+
+#endif /* NVGPU_CYCLESTATS_PRIV_H */

drivers/gpu/nvgpu/common/debugger.c

@@ -0,0 +1,241 @@
+/*
+ * Tegra GK20A GPU Debugger/Profiler Driver
+ *
+ * Copyright (c) 2013-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/kmem.h>
+#include <nvgpu/log.h>
+#include <nvgpu/vm.h>
+#include <nvgpu/atomic.h>
+#include <nvgpu/mm.h>
+#include <nvgpu/bug.h>
+#include <nvgpu/io.h>
+#include <nvgpu/utils.h>
+#include <nvgpu/channel.h>
+#include <nvgpu/gk20a.h>
+#include <nvgpu/nvgpu_init.h>
+#include <nvgpu/debugger.h>
+#include <nvgpu/profiler.h>
+#include <nvgpu/power_features/power_features.h>
+
+/*
+ * API to get first channel from the list of all channels
+ * bound to the debug session
+ */
+struct nvgpu_channel *
+nvgpu_dbg_gpu_get_session_channel(struct dbg_session_gk20a *dbg_s)
+{
+	struct dbg_session_channel_data *ch_data;
+	struct nvgpu_channel *ch;
+	struct gk20a *g = dbg_s->g;
+
+	nvgpu_mutex_acquire(&dbg_s->ch_list_lock);
+	if (nvgpu_list_empty(&dbg_s->ch_list)) {
+		nvgpu_mutex_release(&dbg_s->ch_list_lock);
+		return NULL;
+	}
+
+	ch_data = nvgpu_list_first_entry(&dbg_s->ch_list,
+				   dbg_session_channel_data,
+				   ch_entry);
+	ch = g->fifo.channel + ch_data->chid;
+
+	nvgpu_mutex_release(&dbg_s->ch_list_lock);
+
+	return ch;
+}
+
+void nvgpu_dbg_gpu_post_events(struct nvgpu_channel *ch)
+{
+	struct dbg_session_data *session_data;
+	struct dbg_session_gk20a *dbg_s;
+	struct gk20a *g = ch->g;
+
+	nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg, " ");
+
+	/* guard against the session list being modified */
+	nvgpu_mutex_acquire(&ch->dbg_s_lock);
+
+	nvgpu_list_for_each_entry(session_data, &ch->dbg_s_list,
+				dbg_session_data, dbg_s_entry) {
+		dbg_s = session_data->dbg_s;
+		if (dbg_s->dbg_events.events_enabled) {
+			nvgpu_log(g, gpu_dbg_gpu_dbg, "posting event on session id %d",
+					dbg_s->id);
+			nvgpu_log(g, gpu_dbg_gpu_dbg, "%d events pending",
+					dbg_s->dbg_events.num_pending_events);
+
+			dbg_s->dbg_events.num_pending_events++;
+
+			nvgpu_dbg_session_post_event(dbg_s);
+		}
+	}
+
+	nvgpu_mutex_release(&ch->dbg_s_lock);
+}
+
+bool nvgpu_dbg_gpu_broadcast_stop_trigger(struct nvgpu_channel *ch)
+{
+	struct dbg_session_data *session_data;
+	struct dbg_session_gk20a *dbg_s;
+	bool broadcast = false;
+	struct gk20a *g = ch->g;
+
+	nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg | gpu_dbg_intr, " ");
+
+	/* guard against the session list being modified */
+	nvgpu_mutex_acquire(&ch->dbg_s_lock);
+
+	nvgpu_list_for_each_entry(session_data, &ch->dbg_s_list,
+				dbg_session_data, dbg_s_entry) {
+		dbg_s = session_data->dbg_s;
+		if (dbg_s->broadcast_stop_trigger) {
+			nvgpu_log(g, gpu_dbg_gpu_dbg | gpu_dbg_fn | gpu_dbg_intr,
+					"stop trigger broadcast enabled");
+			broadcast = true;
+			break;
+		}
+	}
+
+	nvgpu_mutex_release(&ch->dbg_s_lock);
+
+	return broadcast;
+}
+
+void nvgpu_dbg_gpu_clear_broadcast_stop_trigger(struct nvgpu_channel *ch)
+{
+	struct dbg_session_data *session_data;
+	struct dbg_session_gk20a *dbg_s;
+	struct gk20a *g = ch->g;
+
+	nvgpu_log(g, gpu_dbg_fn | gpu_dbg_gpu_dbg | gpu_dbg_intr, " ");
+
+	/* guard against the session list being modified */
+	nvgpu_mutex_acquire(&ch->dbg_s_lock);
+
+	nvgpu_list_for_each_entry(session_data, &ch->dbg_s_list,
+				dbg_session_data, dbg_s_entry) {
+		dbg_s = session_data->dbg_s;
+		if (dbg_s->broadcast_stop_trigger) {
+			nvgpu_log(g, gpu_dbg_gpu_dbg | gpu_dbg_fn | gpu_dbg_intr,
+					"stop trigger broadcast disabled");
+			dbg_s->broadcast_stop_trigger = false;
+		}
+	}
+
+	nvgpu_mutex_release(&ch->dbg_s_lock);
+}
+
+u32 nvgpu_set_powergate_locked(struct dbg_session_gk20a *dbg_s,
+				bool mode)
+{
+	u32 err = 0U;
+	struct gk20a *g = dbg_s->g;
+
+	if (dbg_s->is_pg_disabled != mode) {
+		if (mode == false) {
+			g->dbg_powergating_disabled_refcount--;
+		}
+
+		/*
+		 * Allow powergate disable or enable only if
+		 * the global pg disabled refcount is zero
+		 */
+		if (g->dbg_powergating_disabled_refcount == 0) {
+			err = g->ops.debugger.dbg_set_powergate(dbg_s,
+									mode);
+		}
+
+		if (mode) {
+			g->dbg_powergating_disabled_refcount++;
+		}
+
+		dbg_s->is_pg_disabled = mode;
+	}
+
+	return err;
+}
+
+int nvgpu_dbg_set_powergate(struct dbg_session_gk20a *dbg_s, bool disable_powergate)
+{
+	int err = 0;
+	struct gk20a *g = dbg_s->g;
+
+	 /* This function must be called with g->dbg_sessions_lock held */
+
+	nvgpu_log(g, gpu_dbg_fn|gpu_dbg_gpu_dbg, "%s powergate mode = %s",
+		   g->name, disable_powergate ? "disable" : "enable");
+
+	/*
+	 * Powergate mode here refers to railgate+powergate+clockgate
+	 * so in case slcg/blcg/elcg are disabled and railgating is enabled,
+	 * disable railgating and then set is_pg_disabled = true
+	 * Similarly re-enable railgating and not other features if they are not
+	 * enabled when powermode=MODE_ENABLE
+	 */
+	if (disable_powergate) {
+		/* save off current powergate, clk state.
+		 * set gpu module's can_powergate = 0.
+		 * set gpu module's clk to max.
+		 * while *a* debug session is active there will be no power or
+		 * clocking state changes allowed from mainline code (but they
+		 * should be saved).
+		 */
+
+		nvgpu_log(g, gpu_dbg_gpu_dbg | gpu_dbg_fn,
+						"module busy");
+		err = gk20a_busy(g);
+		if (err != 0) {
+			return err;
+		}
+
+#ifdef CONFIG_NVGPU_NON_FUSA
+		err = nvgpu_cg_pg_disable(g);
+#endif
+		if (err == 0) {
+			dbg_s->is_pg_disabled = true;
+			nvgpu_log(g, gpu_dbg_gpu_dbg | gpu_dbg_fn,
+					"pg disabled");
+		}
+	} else {
+		/* restore (can) powergate, clk state */
+		/* release pending exceptions to fault/be handled as usual */
+		/*TBD: ordering of these? */
+
+#ifdef CONFIG_NVGPU_NON_FUSA
+		err = nvgpu_cg_pg_enable(g);
+#endif
+		if (err == 0) {
+			dbg_s->is_pg_disabled = false;
+			nvgpu_log(g, gpu_dbg_gpu_dbg | gpu_dbg_fn,
+					"pg enabled");
+		}
+
+		nvgpu_log(g, gpu_dbg_gpu_dbg | gpu_dbg_fn, "module idle");
+
+		gk20a_idle(g);
+	}
+
+	nvgpu_log(g, gpu_dbg_fn|gpu_dbg_gpu_dbg, "%s powergate mode = %s done",
+		   g->name, disable_powergate ? "disable" : "enable");
+	return err;
+}

drivers/gpu/nvgpu/common/device.c

@@ -0,0 +1,338 @@
+/*
+ * Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/gk20a.h>
+#include <nvgpu/device.h>
+#include <nvgpu/list.h>
+#include <nvgpu/kmem.h>
+#include <nvgpu/string.h>
+#include <nvgpu/log.h>
+
+#define device_dbg(g, fmt, args...)					\
+	do {								\
+		nvgpu_log(g, gpu_dbg_device, fmt, ##args);		\
+	} while (0)
+
+static inline const char *nvgpu_device_type_to_str(const struct nvgpu_device *dev)
+{
+	const char *str = "Unknown";
+
+	switch (dev->type) {
+	case NVGPU_DEVTYPE_GRAPHICS:
+		str = "GFX";
+		break;
+	case NVGPU_DEVTYPE_COPY0:
+		str = "CE0";
+		break;
+	case NVGPU_DEVTYPE_COPY1:
+		str = "CE1";
+		break;
+	case NVGPU_DEVTYPE_COPY2:
+		str = "CE2";
+		break;
+	case NVGPU_DEVTYPE_IOCTRL:
+		str = "IOCTRL";
+		break;
+	case NVGPU_DEVTYPE_LCE:
+		str = "LCE";
+		break;
+	default:
+		break;
+	}
+
+	return str;
+}
+
+void nvgpu_device_dump_dev(struct gk20a *g, const struct nvgpu_device *dev)
+{
+	device_dbg(g, "Device %s:%d",
+		   nvgpu_device_type_to_str(dev), dev->inst_id);
+	device_dbg(g, "  EngineID:  %2u  FaultID: %2u",
+		   dev->engine_id, dev->fault_id);
+	device_dbg(g, "  RunlistID: %2u  IntrID:  %2u  ResetID: %u",
+		   dev->runlist_id, dev->intr_id, dev->reset_id);
+	device_dbg(g, "  PRI Base: 0x%x", dev->pri_base);
+}
+
+/*
+ * Faciliate the parsing of the TOP array describing the devices present in the
+ * GPU.
+ */
+static int nvgpu_device_parse_hw_table(struct gk20a *g)
+{
+	int ret = 0;
+	u32 token = NVGPU_DEVICE_TOKEN_INIT;
+	struct nvgpu_device *dev;
+	struct nvgpu_list_node *devlist;
+
+	while (true) {
+		dev = g->ops.top.parse_next_device(g, &token);
+		if (dev == NULL) {
+			break;
+		}
+
+		nvgpu_device_dump_dev(g, dev);
+
+		/*
+		 * Otherwise we have a device - let's add it to the right device
+		 * list.
+		 */
+		devlist = &g->devs->devlist_heads[dev->type];
+
+		nvgpu_list_add_tail(&dev->dev_list_node, devlist);
+		g->devs->dev_counts[dev->type] += 1;
+	}
+
+	return ret;
+}
+
+/*
+ * Faciliate reading the HW register table into a software abstraction. This is
+ * done only on the first boot as the table will never change dynamically.
+ */
+int nvgpu_device_init(struct gk20a *g)
+{
+	u32 i;
+
+	device_dbg(g, "Initializating GPU device list");
+
+	/*
+	 * Ground work - make sure we aren't doing this again and that we have
+	 * all the necessary data structures.
+	 */
+	if (g->devs != NULL) {
+		device_dbg(g, "  GPU device list already present. Done.");
+		return 0;
+	}
+
+	g->devs = nvgpu_kzalloc(g, sizeof(*g->devs));
+	if (g->devs == NULL) {
+		return -ENOMEM;
+	}
+
+	for (i = 0; i < NVGPU_MAX_DEVTYPE; i++) {
+		nvgpu_init_list_node(&g->devs->devlist_heads[i]);
+	}
+
+	return nvgpu_device_parse_hw_table(g);
+}
+
+static void nvgpu_device_cleanup_devtype(struct gk20a *g,
+					 struct nvgpu_list_node *list)
+{
+	struct nvgpu_device *dev;
+
+	while (!nvgpu_list_empty(list)) {
+		dev = nvgpu_list_first_entry(list,
+					     nvgpu_device,
+					     dev_list_node);
+		nvgpu_list_del(&dev->dev_list_node);
+		nvgpu_kfree(g, dev);
+	}
+}
+
+void nvgpu_device_cleanup(struct gk20a *g)
+{
+	u32 i;
+	struct nvgpu_list_node *devlist;
+
+	device_dbg(g, "Releasing GPU device list");
+
+	/*
+	 * Make unit testing a bit easier.
+	 */
+	if (g->devs == NULL) {
+		device_dbg(g, "  Already done.");
+		return;
+	}
+
+	for (i = 0; i < NVGPU_MAX_DEVTYPE; i++) {
+		devlist = &g->devs->devlist_heads[i];
+
+		if (devlist == NULL) {
+			continue;
+		}
+
+		nvgpu_device_cleanup_devtype(g, devlist);
+	}
+
+	nvgpu_kfree(g, g->devs);
+	g->devs = NULL;
+}
+
+/*
+ * Find the instance passed. Do this by simply traversing the linked list; it's
+ * not particularly efficient, but we aren't expecting there to ever be _that_
+ * many devices.
+ *
+ * Return a pointer to the device or NULL of the inst ID is out of range.
+ */
+static const struct nvgpu_device *dev_instance_from_devlist(
+	struct nvgpu_list_node *devlist, u32 inst_id)
+{
+	struct nvgpu_device *dev;
+
+	nvgpu_list_for_each_entry(dev, devlist, nvgpu_device, dev_list_node) {
+		if (dev->inst_id == inst_id) {
+			return dev;
+		}
+	}
+
+	return NULL;
+}
+
+const struct nvgpu_device *nvgpu_device_get(struct gk20a *g,
+		     u32 type, u32 inst_id)
+{
+	const struct nvgpu_device *dev;
+	struct nvgpu_list_node *device_list;
+
+	if (type >= NVGPU_MAX_DEVTYPE) {
+		return NULL;
+	}
+
+	device_list = &g->devs->devlist_heads[type];
+	dev = dev_instance_from_devlist(device_list, inst_id);
+
+	if (dev == NULL) {
+		return NULL;
+	}
+
+	return dev;
+}
+
+u32 nvgpu_device_count(struct gk20a *g, u32 type)
+{
+	if (type >= NVGPU_MAX_DEVTYPE) {
+		return 0U;
+	}
+
+	return g->devs->dev_counts[type];
+}
+
+/*
+ * Internal function to query copy engines; async_only specifies whether
+ * this function should or should not include the GR copy engines (CEs that
+ * share a runlist with the GR engine(s)).
+ *
+ * This function basically iterates over two distinct copy engine lists:
+ * first the COPY0-2 (the old way of describing copy engines) and the LCE
+ * list (the new in Pascal way of describing copy engines).
+ */
+static u32 nvgpu_device_do_get_copies(struct gk20a *g,
+				      bool async_only,
+				      const struct nvgpu_device **ces,
+				      u32 max)
+{
+	u32 i;
+	u32 copies = 0U;
+	const struct nvgpu_device *dev;
+	const struct nvgpu_device *gr_dev;
+
+	if (max == 0U) {
+		return 0U;
+	}
+
+	gr_dev = nvgpu_device_get(g, NVGPU_DEVTYPE_GRAPHICS, 0U);
+	nvgpu_assert(gr_dev != NULL);
+
+	/*
+	 * Start with the COPY0-2 engines. Note the awkward instance ID.
+	 */
+	for (i = NVGPU_DEVTYPE_COPY0; i <= NVGPU_DEVTYPE_COPY2; i++) {
+		dev = nvgpu_device_get(g, i, i - NVGPU_DEVTYPE_COPY0);
+		if (dev == NULL) {
+			continue;
+		}
+
+		if (async_only &&
+		    dev->runlist_id == gr_dev->runlist_id) {
+			/* It's a GRCE, skip it per async_only. */
+			continue;
+		}
+
+		ces[copies] = dev;
+		copies = nvgpu_safe_add_u32(copies, 1U);
+		if (copies == max) {
+			return copies;
+		}
+	}
+
+	for (i = 0; i < nvgpu_device_count(g, NVGPU_DEVTYPE_LCE); i++) {
+		dev = nvgpu_device_get(g, NVGPU_DEVTYPE_LCE, i);
+		nvgpu_assert(dev != NULL);
+
+		if (async_only &&
+		    dev->runlist_id == gr_dev->runlist_id) {
+			/* It's a GRCE, skip it per async_only. */
+			continue;
+		}
+
+		ces[copies] = dev;
+		copies = nvgpu_safe_add_u32(copies, 1U);
+		if (copies == max) {
+			return copies;
+		}
+	}
+
+	return copies;
+}
+
+u32 nvgpu_device_get_async_copies(struct gk20a *g,
+				  const struct nvgpu_device **ces,
+				  u32 max)
+{
+	return nvgpu_device_do_get_copies(g, true, ces, max);
+}
+
+u32 nvgpu_device_get_copies(struct gk20a *g,
+			    const struct nvgpu_device **ces,
+			    u32 max)
+{
+	return nvgpu_device_do_get_copies(g, false, ces, max);
+}
+
+/*
+ * Note: this kind of bleeds HW details into the core code. Eventually this
+ * should be handled by a translation table. However, for now, HW has kept the
+ * device type values consistent across chips and nvgpu already has this present
+ * in core code.
+ *
+ * Once a per-chip translation table exists we can translate and then do a
+ * comparison.
+ */
+bool nvgpu_device_is_ce(struct gk20a *g, const struct nvgpu_device *dev)
+{
+	if (dev->type == NVGPU_DEVTYPE_COPY0 ||
+	    dev->type == NVGPU_DEVTYPE_COPY1 ||
+	    dev->type == NVGPU_DEVTYPE_COPY2 ||
+	    dev->type == NVGPU_DEVTYPE_LCE) {
+		return true;
+	}
+
+	return false;
+}
+
+bool nvgpu_device_is_graphics(struct gk20a *g, const struct nvgpu_device *dev)
+{
+	return dev->type == NVGPU_DEVTYPE_GRAPHICS;
+}

drivers/gpu/nvgpu/common/ecc.c

@@ -0,0 +1,167 @@
+/*
+ * Copyright (c) 2018-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/gk20a.h>
+#include <nvgpu/gr/gr_ecc.h>
+#include <nvgpu/ltc.h>
+
+void nvgpu_ecc_stat_add(struct gk20a *g, struct nvgpu_ecc_stat *stat)
+{
+	struct nvgpu_ecc *ecc = &g->ecc;
+
+	nvgpu_init_list_node(&stat->node);
+
+	nvgpu_mutex_acquire(&ecc->stats_lock);
+
+	nvgpu_list_add_tail(&stat->node, &ecc->stats_list);
+	ecc->stats_count = nvgpu_safe_add_s32(ecc->stats_count, 1);
+
+	nvgpu_mutex_release(&ecc->stats_lock);
+}
+
+void nvgpu_ecc_stat_del(struct gk20a *g, struct nvgpu_ecc_stat *stat)
+{
+	struct nvgpu_ecc *ecc = &g->ecc;
+
+	nvgpu_mutex_acquire(&ecc->stats_lock);
+
+	nvgpu_list_del(&stat->node);
+	ecc->stats_count = nvgpu_safe_sub_s32(ecc->stats_count, 1);
+
+	nvgpu_mutex_release(&ecc->stats_lock);
+}
+
+int nvgpu_ecc_counter_init(struct gk20a *g,
+		struct nvgpu_ecc_stat **statp, const char *name)
+{
+	struct nvgpu_ecc_stat *stat;
+
+	stat = nvgpu_kzalloc(g, sizeof(*stat));
+	if (stat == NULL) {
+		nvgpu_err(g, "ecc counter alloc failed");
+		return -ENOMEM;
+	}
+
+	(void)strncpy(stat->name, name, NVGPU_ECC_STAT_NAME_MAX_SIZE - 1U);
+	nvgpu_ecc_stat_add(g, stat);
+	*statp = stat;
+	return 0;
+}
+
+void nvgpu_ecc_counter_deinit(struct gk20a *g, struct nvgpu_ecc_stat **statp)
+{
+	struct nvgpu_ecc_stat *stat;
+
+	if (*statp == NULL) {
+		return;
+	}
+
+	stat = *statp;
+
+	nvgpu_ecc_stat_del(g, stat);
+	nvgpu_kfree(g, stat);
+	*statp = NULL;
+}
+
+/* release all ecc_stat */
+void nvgpu_ecc_free(struct gk20a *g)
+{
+	struct nvgpu_ecc *ecc = &g->ecc;
+
+	nvgpu_gr_ecc_free(g);
+	nvgpu_ltc_ecc_free(g);
+
+	if (g->ops.fb.ecc.free != NULL) {
+		g->ops.fb.ecc.free(g);
+	}
+
+#ifdef CONFIG_NVGPU_DGPU
+	if (g->ops.fb.fbpa_ecc_free != NULL) {
+		g->ops.fb.fbpa_ecc_free(g);
+	}
+#endif
+
+	if (g->ops.pmu.ecc_free != NULL) {
+		g->ops.pmu.ecc_free(g);
+	}
+
+	nvgpu_mutex_acquire(&ecc->stats_lock);
+	WARN_ON(!nvgpu_list_empty(&ecc->stats_list));
+	nvgpu_mutex_release(&ecc->stats_lock);
+
+	(void)memset(ecc, 0, sizeof(*ecc));
+}
+
+int nvgpu_ecc_init_support(struct gk20a *g)
+{
+	struct nvgpu_ecc *ecc = &g->ecc;
+
+	if (ecc->initialized) {
+		return 0;
+	}
+
+	nvgpu_mutex_init(&ecc->stats_lock);
+	nvgpu_init_list_node(&ecc->stats_list);
+
+	return 0;
+}
+
+/**
+ * Note that this function is to be called after all units requiring ecc stats
+ * have added entries to ecc->stats_list.
+ */
+int nvgpu_ecc_finalize_support(struct gk20a *g)
+{
+#ifdef CONFIG_NVGPU_SYSFS
+	int err;
+#endif
+
+	if (g->ecc.initialized) {
+		return 0;
+	}
+
+#ifdef CONFIG_NVGPU_SYSFS
+	err = nvgpu_ecc_sysfs_init(g);
+	if (err != 0) {
+		nvgpu_ecc_free(g);
+		return err;
+	}
+#endif
+
+	g->ecc.initialized = true;
+
+	return 0;
+}
+
+void nvgpu_ecc_remove_support(struct gk20a *g)
+{
+	if (!g->ecc.initialized) {
+		return;
+	}
+
+#ifdef CONFIG_NVGPU_SYSFS
+	nvgpu_ecc_sysfs_remove(g);
+#endif
+	nvgpu_ecc_free(g);
+
+	nvgpu_mutex_destroy(&g->ecc.stats_lock);
+}

drivers/gpu/nvgpu/common/engine_queues/engine_dmem_queue.c

@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/falcon.h>
+#include <nvgpu/log.h>
+
+#include "engine_mem_queue_priv.h"
+#include "engine_dmem_queue.h"
+
+/* DMEM-Q specific ops */
+static int engine_dmem_queue_push(struct nvgpu_falcon *flcn,
+	struct nvgpu_engine_mem_queue *queue, u32 dst, void *data, u32 size)
+{
+	struct gk20a *g = queue->g;
+	int err = 0;
+
+	err = nvgpu_falcon_copy_to_dmem(flcn, dst, data, size, 0);
+	if (err != 0) {
+		nvgpu_err(g, "flcn-%d, queue-%d", queue->flcn_id, queue->id);
+		nvgpu_err(g, "dmem queue write failed");
+		goto exit;
+	}
+
+exit:
+	return err;
+}
+
+static int engine_dmem_queue_pop(struct nvgpu_falcon *flcn,
+	struct nvgpu_engine_mem_queue *queue, u32 src, void *data, u32 size)
+{
+	struct gk20a *g = queue->g;
+	int err = 0;
+
+	err = nvgpu_falcon_copy_from_dmem(flcn, src, data, size, 0);
+	if (err != 0) {
+		nvgpu_err(g, "flcn-%d, queue-%d", queue->flcn_id, queue->id);
+		nvgpu_err(g, "dmem queue read failed");
+		goto exit;
+	}
+
+exit:
+	return err;
+}
+
+/* assign DMEM queue type specific ops */
+void engine_dmem_queue_init(struct nvgpu_engine_mem_queue *queue)
+{
+	queue->push = engine_dmem_queue_push;
+	queue->pop = engine_dmem_queue_pop;
+}

drivers/gpu/nvgpu/common/engine_queues/engine_dmem_queue.h

@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef NVGPU_ENGINE_DMEM_QUEUE_H
+#define NVGPU_ENGINE_DMEM_QUEUE_H
+
+void engine_dmem_queue_init(struct nvgpu_engine_mem_queue *queue);
+
+#endif /* NVGPU_ENGINE_DMEM_QUEUE_H */

drivers/gpu/nvgpu/common/engine_queues/engine_emem_queue.c

@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/falcon.h>
+#include <nvgpu/log.h>
+
+#include "engine_mem_queue_priv.h"
+#include "engine_emem_queue.h"
+
+/* EMEM-Q specific ops */
+static int engine_emem_queue_push(struct nvgpu_falcon *flcn,
+	struct nvgpu_engine_mem_queue *queue, u32 dst, void *data, u32 size)
+{
+	struct gk20a *g = queue->g;
+	int err = 0;
+
+	err = nvgpu_falcon_copy_to_emem(flcn, dst, data, size, 0);
+	if (err != 0) {
+		nvgpu_err(g, "flcn-%d, queue-%d", queue->flcn_id, queue->id);
+		nvgpu_err(g, "emem queue write failed");
+		goto exit;
+	}
+
+exit:
+	return err;
+}
+
+static int engine_emem_queue_pop(struct nvgpu_falcon *flcn,
+	struct nvgpu_engine_mem_queue *queue, u32 src, void *data, u32 size)
+{
+	struct gk20a *g = queue->g;
+	int err = 0;
+
+	err = nvgpu_falcon_copy_from_emem(flcn, src, data, size, 0);
+	if (err != 0) {
+		nvgpu_err(g, "flcn-%d, queue-%d", queue->flcn_id, queue->id);
+		nvgpu_err(g, "emem queue read failed");
+		goto exit;
+	}
+
+exit:
+	return err;
+}
+
+/* assign EMEM queue type specific ops */
+void engine_emem_queue_init(struct nvgpu_engine_mem_queue *queue)
+{
+	queue->push = engine_emem_queue_push;
+	queue->pop = engine_emem_queue_pop;
+}

drivers/gpu/nvgpu/common/engine_queues/engine_emem_queue.h

@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef NVGPU_ENGINE_EMEM_QUEUE_H
+#define NVGPU_ENGINE_EMEM_QUEUE_H
+
+void engine_emem_queue_init(struct nvgpu_engine_mem_queue *queue);
+
+#endif /* NVGPU_ENGINE_EMEM_QUEUE_H */

drivers/gpu/nvgpu/common/engine_queues/engine_fb_queue.c

@@ -0,0 +1,603 @@
+/*
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/log.h>
+#include <nvgpu/errno.h>
+#include <nvgpu/types.h>
+#include <nvgpu/flcnif_cmn.h>
+#include <nvgpu/nvgpu_mem.h>
+#include <nvgpu/string.h>
+#include <nvgpu/kmem.h>
+#include <nvgpu/engine_queue.h>
+#include <nvgpu/engine_fb_queue.h>
+#include <nvgpu/pmu/pmuif/cmn.h>
+
+#include "engine_fb_queue_priv.h"
+
+/* FB-Q ops */
+static int engine_fb_queue_head(struct nvgpu_engine_fb_queue *queue,
+				u32 *head, bool set)
+{
+	return queue->queue_head(queue->g, queue->id, queue->index, head, set);
+}
+
+static int engine_fb_queue_tail(struct nvgpu_engine_fb_queue *queue,
+				u32 *tail, bool set)
+{
+	struct gk20a *g = queue->g;
+	int err;
+
+	if (set == false && PMU_IS_COMMAND_QUEUE(queue->id)) {
+		*tail = queue->fbq.tail;
+		err = 0;
+	} else {
+		err = queue->queue_tail(g, queue->id, queue->index, tail, set);
+	}
+
+	return err;
+}
+
+static inline u32 engine_fb_queue_get_next(struct nvgpu_engine_fb_queue *queue,
+					   u32 head)
+{
+		return (head + 1U) % queue->size;
+}
+
+static bool engine_fb_queue_has_room(struct nvgpu_engine_fb_queue *queue,
+	u32 size)
+{
+	u32 head = 0;
+	u32 tail = 0;
+	u32 next_head = 0;
+	int err = 0;
+
+	err = queue->head(queue, &head, QUEUE_GET);
+	if (err != 0) {
+		nvgpu_err(queue->g, "queue head GET failed");
+		goto exit;
+	}
+
+	err = queue->tail(queue, &tail, QUEUE_GET);
+	if (err != 0) {
+		nvgpu_err(queue->g, "queue tail GET failed");
+		goto exit;
+	}
+
+	next_head = engine_fb_queue_get_next(queue, head);
+
+exit:
+	return next_head != tail;
+}
+
+static int engine_fb_queue_write(struct nvgpu_engine_fb_queue *queue,
+	u32 offset, u8 *src, u32 size)
+{
+	struct gk20a *g = queue->g;
+	struct nv_falcon_fbq_hdr *fb_q_hdr = (struct nv_falcon_fbq_hdr *)
+		(void *)queue->fbq.work_buffer;
+	u32 entry_offset = 0U;
+	int err = 0;
+
+	if (queue->fbq.work_buffer == NULL) {
+		nvgpu_err(g, "Invalid/Unallocated work buffer");
+		err = -EINVAL;
+		goto exit;
+	}
+
+	/* Fill out FBQ hdr, that is in the work buffer */
+	fb_q_hdr->element_index = (u8)offset;
+
+	/* check queue entry size */
+	if (fb_q_hdr->heap_size >= (u16)queue->fbq.element_size) {
+		err = -EINVAL;
+		goto exit;
+	}
+
+	/* get offset to this element entry */
+	entry_offset = offset * queue->fbq.element_size;
+
+	/* copy cmd to super-surface */
+	nvgpu_mem_wr_n(g, queue->fbq.super_surface_mem,
+		queue->fbq.fb_offset + entry_offset,
+		queue->fbq.work_buffer, queue->fbq.element_size);
+
+exit:
+	return err;
+}
+
+static int engine_fb_queue_set_element_use_state(
+	struct nvgpu_engine_fb_queue *queue, u32 queue_pos, bool set)
+{
+	int err = 0;
+
+	if (queue_pos >= queue->size) {
+		err = -EINVAL;
+		goto exit;
+	}
+
+	if (nvgpu_test_bit(queue_pos,
+		(void *)&queue->fbq.element_in_use) && set) {
+		nvgpu_err(queue->g,
+			"FBQ last received queue element not processed yet"
+			" queue_pos %d", queue_pos);
+		err = -EINVAL;
+		goto exit;
+	}
+
+	if (set) {
+		nvgpu_set_bit(queue_pos, (void *)&queue->fbq.element_in_use);
+	} else {
+		nvgpu_clear_bit(queue_pos, (void *)&queue->fbq.element_in_use);
+	}
+
+exit:
+	return err;
+}
+
+static int engine_fb_queue_is_element_in_use(
+	struct nvgpu_engine_fb_queue *queue,
+	u32 queue_pos, bool *in_use)
+{
+	int err = 0;
+
+	if (queue_pos >= queue->size) {
+		err = -EINVAL;
+		goto exit;
+	}
+
+	*in_use = nvgpu_test_bit(queue_pos, (void *)&queue->fbq.element_in_use);
+
+exit:
+	return err;
+}
+
+static int engine_fb_queue_sweep(struct nvgpu_engine_fb_queue *queue)
+{
+	u32 head;
+	u32 tail;
+	bool in_use = false;
+	int err = 0;
+
+	tail = queue->fbq.tail;
+	err = queue->head(queue, &head, QUEUE_GET);
+	if (err != 0) {
+		nvgpu_err(queue->g, "flcn-%d queue-%d, position GET failed",
+			queue->flcn_id, queue->id);
+		goto exit;
+	}
+
+	/*
+	 * Step from tail forward in the queue,
+	 * to see how many consecutive entries
+	 * can be made available.
+	 */
+	while (tail != head) {
+		if (engine_fb_queue_is_element_in_use(queue,
+			tail, &in_use) != 0) {
+			break;
+		}
+
+		if (in_use) {
+			break;
+		}
+
+		tail = engine_fb_queue_get_next(queue, tail);
+	}
+
+	/* Update tail */
+	queue->fbq.tail = tail;
+
+exit:
+	return err;
+}
+
+u32 nvgpu_engine_fb_queue_get_position(struct nvgpu_engine_fb_queue *queue)
+{
+	return queue->position;
+}
+
+/* return the queue element size */
+u32 nvgpu_engine_fb_queue_get_element_size(struct nvgpu_engine_fb_queue *queue)
+{
+	return queue->fbq.element_size;
+}
+
+/* return the queue offset from super surface FBQ's */
+u32 nvgpu_engine_fb_queue_get_offset(struct nvgpu_engine_fb_queue *queue)
+{
+	return queue->fbq.fb_offset;
+}
+
+/* lock work buffer of queue */
+void nvgpu_engine_fb_queue_lock_work_buffer(struct nvgpu_engine_fb_queue *queue)
+{
+	/* acquire work buffer mutex */
+	nvgpu_mutex_acquire(&queue->fbq.work_buffer_mutex);
+}
+
+/* unlock work buffer of queue */
+void nvgpu_engine_fb_queue_unlock_work_buffer(
+					struct nvgpu_engine_fb_queue *queue)
+{
+	/* release work buffer mutex */
+	nvgpu_mutex_release(&queue->fbq.work_buffer_mutex);
+}
+
+/* return a pointer of queue work buffer */
+u8 *nvgpu_engine_fb_queue_get_work_buffer(struct nvgpu_engine_fb_queue *queue)
+{
+	return queue->fbq.work_buffer;
+}
+
+int nvgpu_engine_fb_queue_free_element(struct nvgpu_engine_fb_queue *queue,
+			u32 queue_pos)
+{
+	int err = 0;
+
+	err = engine_fb_queue_set_element_use_state(queue,
+		queue_pos, false);
+	if (err != 0) {
+		nvgpu_err(queue->g, "fb queue elelment %d free failed",
+			queue_pos);
+		goto exit;
+	}
+
+	err = engine_fb_queue_sweep(queue);
+
+exit:
+	return err;
+}
+
+/* queue is_empty check with lock */
+bool nvgpu_engine_fb_queue_is_empty(struct nvgpu_engine_fb_queue *queue)
+{
+	u32 q_head = 0;
+	u32 q_tail = 0;
+	int err = 0;
+
+	if (queue == NULL) {
+		return true;
+	}
+
+	/* acquire mutex */
+	nvgpu_mutex_acquire(&queue->mutex);
+
+	err = queue->head(queue, &q_head, QUEUE_GET);
+	if (err != 0) {
+		nvgpu_err(queue->g, "flcn-%d queue-%d, head GET failed",
+			queue->flcn_id, queue->id);
+		goto exit;
+	}
+
+	err = queue->tail(queue, &q_tail, QUEUE_GET);
+	if (err != 0) {
+		nvgpu_err(queue->g, "flcn-%d queue-%d, tail GET failed",
+			queue->flcn_id, queue->id);
+		goto exit;
+	}
+
+exit:
+	/* release mutex */
+	nvgpu_mutex_release(&queue->mutex);
+
+	return q_head == q_tail;
+}
+
+static int engine_fb_queue_prepare_write(struct nvgpu_engine_fb_queue *queue,
+				u32 size)
+{
+	int err = 0;
+
+	/* make sure there's enough free space for the write */
+	if (!engine_fb_queue_has_room(queue, size)) {
+		nvgpu_log_info(queue->g, "queue full: queue-id %d: index %d",
+			       queue->id, queue->index);
+		err = -EAGAIN;
+		goto exit;
+	}
+
+	err = queue->head(queue, &queue->position, QUEUE_GET);
+	if (err != 0) {
+		nvgpu_err(queue->g, "flcn-%d queue-%d, position GET failed",
+			queue->flcn_id, queue->id);
+		goto exit;
+	}
+
+exit:
+	return err;
+}
+
+/* queue push operation with lock */
+int nvgpu_engine_fb_queue_push(struct nvgpu_engine_fb_queue *queue,
+			void *data, u32 size)
+{
+	struct gk20a *g;
+	int err = 0;
+
+	if (queue == NULL) {
+		return -EINVAL;
+	}
+
+	g = queue->g;
+
+	nvgpu_log_fn(g, " ");
+
+	if (queue->oflag != OFLAG_WRITE) {
+		nvgpu_err(queue->g, "flcn-%d, queue-%d not opened for write",
+			queue->flcn_id, queue->id);
+		err = -EINVAL;
+		goto exit;
+	}
+
+	/* acquire mutex */
+	nvgpu_mutex_acquire(&queue->mutex);
+
+	err = engine_fb_queue_prepare_write(queue, size);
+	if (err != 0) {
+		goto unlock_mutex;
+	}
+
+	/* Bounds check size */
+	if (size > queue->fbq.element_size) {
+		nvgpu_err(g, "size too large size=0x%x", size);
+		goto unlock_mutex;
+	}
+
+	/* Set queue element in use */
+	if (engine_fb_queue_set_element_use_state(queue,
+		queue->position, true) != 0) {
+		nvgpu_err(g,
+			"fb-queue element in use map is in invalid state");
+		err = -EINVAL;
+		goto unlock_mutex;
+	}
+
+	/* write data to FB */
+	err = engine_fb_queue_write(queue, queue->position, data, size);
+	if (err != 0) {
+		nvgpu_err(g, "write to fb-queue failed");
+		goto unlock_mutex;
+	}
+
+	queue->position = engine_fb_queue_get_next(queue,
+			queue->position);
+
+	err = queue->head(queue, &queue->position, QUEUE_SET);
+	if (err != 0) {
+		nvgpu_err(queue->g, "flcn-%d queue-%d, position SET failed",
+			queue->flcn_id, queue->id);
+		goto unlock_mutex;
+	}
+
+unlock_mutex:
+	/* release mutex */
+	nvgpu_mutex_release(&queue->mutex);
+exit:
+	if (err != 0) {
+		nvgpu_err(queue->g, "falcon id-%d, queue id-%d, failed",
+			queue->flcn_id, queue->id);
+	}
+
+	return err;
+}
+
+/* queue pop operation with lock */
+int nvgpu_engine_fb_queue_pop(struct nvgpu_engine_fb_queue *queue,
+	void *data, u32 size, u32 *bytes_read)
+{
+	struct gk20a *g;
+	struct pmu_hdr *hdr;
+	u32 entry_offset = 0U;
+	int err = 0;
+
+	if (queue == NULL) {
+		return -EINVAL;
+	}
+
+	g = queue->g;
+	hdr = (struct pmu_hdr *) (void *) (queue->fbq.work_buffer +
+			sizeof(struct nv_falcon_fbq_msgq_hdr));
+
+	nvgpu_log_fn(g, " ");
+
+	if (queue->oflag != OFLAG_READ) {
+		nvgpu_err(g, "flcn-%d, queue-%d, not opened for read",
+			queue->flcn_id, queue->id);
+		err = -EINVAL;
+		goto exit;
+	}
+
+	/* acquire mutex */
+	nvgpu_mutex_acquire(&queue->mutex);
+
+	err = queue->tail(queue, &queue->position, QUEUE_GET);
+	if (err != 0) {
+		nvgpu_err(g, "flcn-%d queue-%d, position GET failed",
+			queue->flcn_id, queue->id);
+		goto unlock_mutex;
+	}
+
+	*bytes_read = 0U;
+
+	/* Check size */
+	if ((size + queue->fbq.read_position) >= queue->fbq.element_size) {
+		nvgpu_err(g,
+			"Attempt to read > than queue element size "
+			"for queue id-%d", queue->id);
+		err = -EINVAL;
+		goto unlock_mutex;
+	}
+
+	entry_offset = queue->position * queue->fbq.element_size;
+
+	/*
+	 * If first read for this queue element then read whole queue
+	 * element into work buffer.
+	 */
+	if (queue->fbq.read_position == 0U) {
+		nvgpu_mem_rd_n(g, queue->fbq.super_surface_mem,
+			/* source (FBQ data) offset*/
+			queue->fbq.fb_offset + entry_offset,
+			/* destination buffer */
+			(void *)queue->fbq.work_buffer,
+			/* copy size */
+			queue->fbq.element_size);
+
+		/* Check size in hdr of MSG just read */
+		if (hdr->size >= queue->fbq.element_size) {
+			nvgpu_err(g, "Super Surface read failed");
+			err = -ERANGE;
+			goto unlock_mutex;
+		}
+	}
+
+	nvgpu_memcpy((u8 *)data, (u8 *)queue->fbq.work_buffer +
+		queue->fbq.read_position +
+		sizeof(struct nv_falcon_fbq_msgq_hdr),
+		size);
+
+	/* update current position */
+	queue->fbq.read_position += size;
+
+	/* If reached end of this queue element, move on to next. */
+	if (queue->fbq.read_position >= hdr->size) {
+		queue->fbq.read_position = 0U;
+		/* Increment queue index. */
+		queue->position = engine_fb_queue_get_next(queue,
+			queue->position);
+	}
+
+	*bytes_read = size;
+
+	err = queue->tail(queue, &queue->position, QUEUE_SET);
+	if (err != 0) {
+		nvgpu_err(g, "flcn-%d queue-%d, position SET failed",
+			queue->flcn_id, queue->id);
+		goto unlock_mutex;
+	}
+
+unlock_mutex:
+	/* release mutex */
+	nvgpu_mutex_release(&queue->mutex);
+exit:
+	if (err != 0) {
+		nvgpu_err(g, "falcon id-%d, queue id-%d, failed",
+			queue->flcn_id, queue->id);
+	}
+
+	return err;
+}
+
+void nvgpu_engine_fb_queue_free(struct nvgpu_engine_fb_queue **queue_p)
+{
+	struct nvgpu_engine_fb_queue *queue = NULL;
+	struct gk20a *g;
+
+	if ((queue_p == NULL) || (*queue_p == NULL)) {
+		return;
+	}
+
+	queue = *queue_p;
+
+	g = queue->g;
+
+	nvgpu_log_info(g, "flcn id-%d q-id %d: index %d ",
+		      queue->flcn_id, queue->id, queue->index);
+
+	nvgpu_kfree(g, queue->fbq.work_buffer);
+	nvgpu_mutex_destroy(&queue->fbq.work_buffer_mutex);
+
+	/* destroy mutex */
+	nvgpu_mutex_destroy(&queue->mutex);
+
+	nvgpu_kfree(g, queue);
+	*queue_p = NULL;
+}
+
+int nvgpu_engine_fb_queue_init(struct nvgpu_engine_fb_queue **queue_p,
+	struct nvgpu_engine_fb_queue_params params)
+{
+	struct nvgpu_engine_fb_queue *queue = NULL;
+	struct gk20a *g = params.g;
+	int err = 0;
+
+	if (queue_p == NULL) {
+		return -EINVAL;
+	}
+
+	queue = (struct nvgpu_engine_fb_queue *)
+		   nvgpu_kmalloc(g, sizeof(struct nvgpu_engine_fb_queue));
+
+	if (queue == NULL) {
+		return -ENOMEM;
+	}
+
+	queue->g = params.g;
+	queue->flcn_id = params.flcn_id;
+	queue->id = params.id;
+	queue->index = params.index;
+	queue->size = params.size;
+	queue->oflag = params.oflag;
+
+	queue->fbq.tail = 0U;
+	queue->fbq.element_in_use = 0U;
+	queue->fbq.read_position = 0U;
+	queue->fbq.super_surface_mem = params.super_surface_mem;
+	queue->fbq.element_size = params.fbq_element_size;
+	queue->fbq.fb_offset = params.fbq_offset;
+
+	queue->position = 0U;
+
+	queue->queue_head = params.queue_head;
+	queue->queue_tail = params.queue_tail;
+
+	queue->head = engine_fb_queue_head;
+	queue->tail = engine_fb_queue_tail;
+
+	/* init mutex */
+	nvgpu_mutex_init(&queue->mutex);
+
+	/* init mutex */
+	nvgpu_mutex_init(&queue->fbq.work_buffer_mutex);
+
+	queue->fbq.work_buffer = nvgpu_kzalloc(g, queue->fbq.element_size);
+	if (queue->fbq.work_buffer == NULL) {
+		err = -ENOMEM;
+		goto free_work_mutex;
+	}
+
+	nvgpu_log_info(g,
+		"flcn id-%d q-id %d: index %d, size 0x%08x",
+		queue->flcn_id, queue->id, queue->index,
+		queue->size);
+
+	*queue_p = queue;
+
+	return 0;
+
+free_work_mutex:
+	nvgpu_mutex_destroy(&queue->fbq.work_buffer_mutex);
+	nvgpu_mutex_destroy(&queue->mutex);
+	nvgpu_kfree(g, queue);
+
+	return err;
+}

drivers/gpu/nvgpu/common/engine_queues/engine_fb_queue_priv.h

@@ -0,0 +1,108 @@
+/*
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef NVGPU_ENGINE_FB_QUEUE_PRIV_H
+#define NVGPU_ENGINE_FB_QUEUE_PRIV_H
+
+#include <nvgpu/lock.h>
+
+struct nvgpu_engine_fb_queue {
+	struct gk20a *g;
+	u32 flcn_id;
+
+	/* used by nvgpu, for command LPQ/HPQ */
+	struct nvgpu_mutex mutex;
+
+	/* current write position */
+	u32 position;
+	/* logical queue identifier */
+	u32 id;
+	/* physical queue index */
+	u32 index;
+	/* in bytes */
+	u32 size;
+	/* open-flag */
+	u32 oflag;
+
+	/* members unique to the FB version of the falcon queues */
+	struct {
+		/* Holds super surface base address */
+		struct nvgpu_mem *super_surface_mem;
+
+		/*
+		 * Holds the offset of queue data (0th element).
+		 * This is used for FB Queues to hold a offset of
+		 * Super Surface for this queue.
+		 */
+		 u32 fb_offset;
+
+		/*
+		 * Define the size of a single queue element.
+		 * queues_size above is used for the number of
+		 * queue elements.
+		 */
+		u32 element_size;
+
+		/* To keep track of elements in use */
+		u64 element_in_use;
+
+		/*
+		 * Define a pointer to a local (SYSMEM) allocated
+		 * buffer to hold a single queue element
+		 * it is being assembled.
+		 */
+		 u8 *work_buffer;
+		 struct nvgpu_mutex work_buffer_mutex;
+
+		/*
+		 * Tracks how much of the current FB Queue MSG queue
+		 * entry have been read. This is needed as functions read
+		 * the MSG queue as a byte stream, rather
+		 * than reading a whole MSG at a time.
+		 */
+		u32 read_position;
+
+		/*
+		 * Tail as tracked on the nvgpu "side".  Because the queue
+		 * elements and its associated payload (which is also moved
+		 * PMU->nvgpu through the FB CMD Queue) can't be free-ed until
+		 * the command is complete, response is received and any "out"
+		 * payload delivered to the client, it is necessary for the
+		 * nvgpu to track it's own version of "tail".  This one is
+		 * incremented as commands and completed entries are found
+		 * following tail.
+		 */
+		u32 tail;
+	} fbq;
+
+	/* engine and queue specific ops */
+	int (*tail)(struct nvgpu_engine_fb_queue *queue, u32 *tail, bool set);
+	int (*head)(struct nvgpu_engine_fb_queue *queue, u32 *head, bool set);
+
+	/* engine specific ops */
+	int (*queue_head)(struct gk20a *g, u32 queue_id, u32 queue_index,
+		u32 *head, bool set);
+	int (*queue_tail)(struct gk20a *g, u32 queue_id, u32 queue_index,
+		u32 *tail, bool set);
+};
+
+#endif /* NVGPU_ENGINE_FB_QUEUE_PRIV_H */

drivers/gpu/nvgpu/common/engine_queues/engine_mem_queue.c

@@ -0,0 +1,438 @@
+/*
+ * Copyright (c) 2017-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/errno.h>
+#include <nvgpu/kmem.h>
+#include <nvgpu/lock.h>
+#include <nvgpu/log.h>
+#include <nvgpu/flcnif_cmn.h>
+#include <nvgpu/pmu/pmuif/nvgpu_cmdif.h>
+#include <nvgpu/engine_queue.h>
+#include <nvgpu/engine_mem_queue.h>
+#include <nvgpu/pmu/cmd.h>
+
+#include "engine_mem_queue_priv.h"
+#include "engine_dmem_queue.h"
+#include "engine_emem_queue.h"
+
+static int mem_queue_get_head_tail(struct nvgpu_engine_mem_queue *queue,
+				   u32 *q_head, u32 *q_tail)
+{
+	int err = 0;
+
+	err = queue->head(queue->g, queue->id, queue->index,
+			  q_head, QUEUE_GET);
+	if (err != 0) {
+		nvgpu_err(queue->g, "flcn-%d, queue-%d, head GET failed",
+			  queue->flcn_id, queue->id);
+		goto exit;
+	}
+
+	err = queue->tail(queue->g, queue->id, queue->index,
+			  q_tail, QUEUE_GET);
+	if (err != 0) {
+		nvgpu_err(queue->g, "flcn-%d, queue-%d, tail GET failed",
+			  queue->flcn_id, queue->id);
+		goto exit;
+	}
+
+	return 0;
+exit:
+	return err;
+}
+
+/* common falcon queue ops */
+static bool engine_mem_queue_has_room(struct nvgpu_engine_mem_queue *queue,
+			u32 size, bool *need_rewind)
+{
+	u32 q_head = 0;
+	u32 q_tail = 0;
+	u32 q_free = 0;
+	bool q_rewind = false;
+	int err = 0;
+
+	size = NVGPU_ALIGN(size, QUEUE_ALIGNMENT);
+
+	err = mem_queue_get_head_tail(queue, &q_head, &q_tail);
+	if (err != 0) {
+		goto exit;
+	}
+
+	if (q_head >= q_tail) {
+		q_free = queue->offset + queue->size - q_head;
+		q_free -= (u32)PMU_CMD_HDR_SIZE;
+
+		if (size > q_free) {
+			q_rewind = true;
+			q_head = queue->offset;
+		}
+	}
+
+	if (q_head < q_tail) {
+		q_free = q_tail - q_head - 1U;
+	}
+
+	if (need_rewind != NULL) {
+		*need_rewind = q_rewind;
+	}
+
+exit:
+	return size <= q_free;
+}
+
+static int engine_mem_queue_rewind(struct nvgpu_falcon *flcn,
+	struct nvgpu_engine_mem_queue *queue)
+{
+	struct gk20a *g = queue->g;
+	struct pmu_cmd cmd;
+	int err = 0;
+
+	if (queue->oflag == OFLAG_WRITE) {
+		cmd.hdr.unit_id = PMU_UNIT_REWIND;
+		cmd.hdr.size = (u8)PMU_CMD_HDR_SIZE;
+		err = queue->push(flcn, queue, queue->position,
+				  &cmd, cmd.hdr.size);
+		if (err != 0) {
+			nvgpu_err(g, "flcn-%d queue-%d, rewind request failed",
+				queue->flcn_id, queue->id);
+			goto exit;
+		} else {
+			queue->position += nvgpu_safe_cast_u32_to_u8(
+				NVGPU_ALIGN(U32(cmd.hdr.size), QUEUE_ALIGNMENT));
+			nvgpu_log_info(g, "flcn-%d queue-%d, rewinded",
+			queue->flcn_id, queue->id);
+		}
+	}
+
+	/* update queue position */
+	queue->position = queue->offset;
+
+	if (queue->oflag == OFLAG_READ) {
+		err = queue->tail(g, queue->id, queue->index, &queue->position,
+			QUEUE_SET);
+		if (err != 0) {
+			nvgpu_err(g, "flcn-%d queue-%d, position SET failed",
+				queue->flcn_id, queue->id);
+			goto exit;
+		}
+	}
+
+exit:
+	return err;
+}
+
+static int engine_mem_queue_prepare_write(struct nvgpu_falcon *flcn,
+	struct nvgpu_engine_mem_queue *queue, u32 size)
+{
+	bool q_rewind = false;
+	int err = 0;
+
+	/* make sure there's enough free space for the write */
+	if (!engine_mem_queue_has_room(queue, size, &q_rewind)) {
+		nvgpu_log_info(queue->g, "queue full: queue-id %d: index %d",
+			queue->id, queue->index);
+		err = -EAGAIN;
+		goto exit;
+	}
+
+	err = queue->head(queue->g, queue->id, queue->index,
+			  &queue->position, QUEUE_GET);
+	if (err != 0) {
+		nvgpu_err(queue->g, "flcn-%d queue-%d, position GET failed",
+			queue->flcn_id, queue->id);
+		goto exit;
+	}
+
+	if (q_rewind) {
+		err = engine_mem_queue_rewind(flcn, queue);
+	}
+
+exit:
+	return err;
+}
+
+/* queue public functions */
+
+/* queue push operation with lock */
+int nvgpu_engine_mem_queue_push(struct nvgpu_falcon *flcn,
+	struct nvgpu_engine_mem_queue *queue, void *data, u32 size)
+{
+	struct gk20a *g;
+	int err = 0;
+
+	if ((flcn == NULL) || (queue == NULL)) {
+		return -EINVAL;
+	}
+
+	g = queue->g;
+
+	if (queue->oflag != OFLAG_WRITE) {
+		nvgpu_err(g, "flcn-%d, queue-%d not opened for write",
+			queue->flcn_id, queue->id);
+		err = -EINVAL;
+		goto exit;
+	}
+
+	/* acquire mutex */
+	nvgpu_mutex_acquire(&queue->mutex);
+
+	err = engine_mem_queue_prepare_write(flcn, queue, size);
+	if (err != 0) {
+		goto unlock_mutex;
+	}
+
+	err = queue->push(flcn, queue, queue->position, data, size);
+	if (err != 0) {
+		nvgpu_err(g, "flcn-%d queue-%d, fail to write",
+			queue->flcn_id, queue->id);
+		goto unlock_mutex;
+	}
+
+	queue->position += NVGPU_ALIGN(size, QUEUE_ALIGNMENT);
+
+	err = queue->head(g, queue->id, queue->index,
+			  &queue->position, QUEUE_SET);
+	if (err != 0) {
+		nvgpu_err(g, "flcn-%d queue-%d, position SET failed",
+			queue->flcn_id, queue->id);
+	}
+
+unlock_mutex:
+	/* release mutex */
+	nvgpu_mutex_release(&queue->mutex);
+exit:
+	return err;
+}
+
+/* queue pop operation with lock */
+int nvgpu_engine_mem_queue_pop(struct nvgpu_falcon *flcn,
+	struct nvgpu_engine_mem_queue *queue, void *data, u32 size,
+	u32 *bytes_read)
+{
+	struct gk20a *g;
+	u32 q_tail = 0;
+	u32 q_head = 0;
+	u32 used = 0;
+	int err = 0;
+
+	*bytes_read = 0;
+
+	if ((flcn == NULL) || (queue == NULL)) {
+		return -EINVAL;
+	}
+
+	g = queue->g;
+
+	if (queue->oflag != OFLAG_READ) {
+		nvgpu_err(g, "flcn-%d, queue-%d, not opened for read",
+			queue->flcn_id, queue->id);
+		err = -EINVAL;
+		goto exit;
+	}
+
+	/* acquire mutex */
+	nvgpu_mutex_acquire(&queue->mutex);
+
+	err = mem_queue_get_head_tail(queue, &q_head, &queue->position);
+	if (err != 0) {
+		goto unlock_mutex;
+	}
+
+	q_tail = queue->position;
+
+	if (q_head == q_tail) {
+		goto unlock_mutex;
+	} else if (q_head > q_tail) {
+		used = q_head - q_tail;
+	} else {
+		used = queue->offset + queue->size - q_tail;
+	}
+
+	if (size > used) {
+		nvgpu_warn(g, "queue size smaller than request read");
+		size = used;
+	}
+
+	err = queue->pop(flcn, queue, q_tail, data, size);
+	if (err != 0) {
+		nvgpu_err(g, "flcn-%d queue-%d, fail to read",
+			queue->flcn_id, queue->id);
+		goto unlock_mutex;
+	}
+
+	queue->position += NVGPU_ALIGN(size, QUEUE_ALIGNMENT);
+
+	err = queue->tail(g, queue->id, queue->index,
+			  &queue->position, QUEUE_SET);
+	if (err != 0) {
+		nvgpu_err(g, "flcn-%d queue-%d, position SET failed",
+			queue->flcn_id, queue->id);
+		goto unlock_mutex;
+	}
+
+	*bytes_read = size;
+
+unlock_mutex:
+	/* release mutex */
+	nvgpu_mutex_release(&queue->mutex);
+exit:
+	return err;
+}
+
+int nvgpu_engine_mem_queue_rewind(struct nvgpu_falcon *flcn,
+	struct nvgpu_engine_mem_queue *queue)
+{
+	int err = 0;
+
+	if ((flcn == NULL) || (queue == NULL)) {
+		return -EINVAL;
+	}
+
+	/* acquire mutex */
+	nvgpu_mutex_acquire(&queue->mutex);
+
+	err = engine_mem_queue_rewind(flcn, queue);
+
+	/* release mutex */
+	nvgpu_mutex_release(&queue->mutex);
+
+	return err;
+}
+
+/* queue is_empty check with lock */
+bool nvgpu_engine_mem_queue_is_empty(struct nvgpu_engine_mem_queue *queue)
+{
+	u32 q_head = 0;
+	u32 q_tail = 0;
+	int err = 0;
+
+	if (queue == NULL) {
+		return true;
+	}
+
+	/* acquire mutex */
+	nvgpu_mutex_acquire(&queue->mutex);
+
+	err = mem_queue_get_head_tail(queue, &q_head, &q_tail);
+	if (err != 0) {
+		goto exit;
+	}
+
+exit:
+	/* release mutex */
+	nvgpu_mutex_release(&queue->mutex);
+
+	return q_head == q_tail;
+}
+
+void nvgpu_engine_mem_queue_free(struct nvgpu_engine_mem_queue **queue_p)
+{
+	struct nvgpu_engine_mem_queue *queue = NULL;
+	struct gk20a *g;
+
+	if ((queue_p == NULL) || (*queue_p == NULL)) {
+		return;
+	}
+
+	queue = *queue_p;
+
+	g = queue->g;
+
+	nvgpu_log_info(g, "flcn id-%d q-id %d: index %d ",
+		       queue->flcn_id, queue->id, queue->index);
+
+	/* destroy mutex */
+	nvgpu_mutex_destroy(&queue->mutex);
+
+	nvgpu_kfree(g, queue);
+	*queue_p = NULL;
+}
+
+u32 nvgpu_engine_mem_queue_get_size(struct nvgpu_engine_mem_queue *queue)
+{
+	return queue->size;
+}
+
+int nvgpu_engine_mem_queue_init(struct nvgpu_engine_mem_queue **queue_p,
+	struct nvgpu_engine_mem_queue_params params)
+{
+	struct nvgpu_engine_mem_queue *queue = NULL;
+	struct gk20a *g = params.g;
+	int err = 0;
+
+	if (queue_p == NULL) {
+		return -EINVAL;
+	}
+
+	queue = (struct nvgpu_engine_mem_queue *)
+		   nvgpu_kmalloc(g, sizeof(struct nvgpu_engine_mem_queue));
+
+	if (queue == NULL) {
+		return -ENOMEM;
+	}
+
+	queue->g = params.g;
+	queue->flcn_id = params.flcn_id;
+	queue->id = params.id;
+	queue->index = params.index;
+	queue->offset = params.offset;
+	queue->position = params.position;
+	queue->size = params.size;
+	queue->oflag = params.oflag;
+	queue->queue_type = params.queue_type;
+
+	queue->head = params.queue_head;
+	queue->tail = params.queue_tail;
+
+	nvgpu_log_info(g,
+		"flcn id-%d q-id %d: index %d, offset 0x%08x, size 0x%08x",
+		queue->flcn_id, queue->id, queue->index,
+		queue->offset, queue->size);
+
+	switch (queue->queue_type) {
+	case QUEUE_TYPE_DMEM:
+		engine_dmem_queue_init(queue);
+		break;
+#ifdef CONFIG_NVGPU_DGPU
+	case QUEUE_TYPE_EMEM:
+		engine_emem_queue_init(queue);
+		break;
+#endif
+	default:
+		err = -EINVAL;
+		break;
+	}
+
+	if (err != 0) {
+		nvgpu_err(g, "flcn-%d queue-%d, init failed",
+			queue->flcn_id, queue->id);
+		nvgpu_kfree(g, queue);
+		goto exit;
+	}
+
+	/* init mutex */
+	nvgpu_mutex_init(&queue->mutex);
+
+	*queue_p = queue;
+exit:
+	return err;
+}

drivers/gpu/nvgpu/common/engine_queues/engine_mem_queue_priv.h

@@ -0,0 +1,70 @@
+/*
+ * Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef NVGPU_ENGINE_MEM_QUEUE_PRIV_H
+#define NVGPU_ENGINE_MEM_QUEUE_PRIV_H
+
+#include <nvgpu/lock.h>
+#include <nvgpu/types.h>
+
+struct gk20a;
+struct nvgpu_falcon;
+
+struct nvgpu_engine_mem_queue {
+	struct gk20a *g;
+	u32 flcn_id;
+
+	/* Queue Type (queue_type) */
+	u8 queue_type;
+
+	/* used by nvgpu, for command LPQ/HPQ */
+	struct nvgpu_mutex mutex;
+
+	/* current write position */
+	u32 position;
+	/* physical dmem offset where this queue begins */
+	u32 offset;
+	/* logical queue identifier */
+	u32 id;
+	/* physical queue index */
+	u32 index;
+	/* in bytes */
+	u32 size;
+	/* open-flag */
+	u32 oflag;
+
+	/* queue type(DMEM-Q/EMEM-Q) specific ops */
+	int (*push)(struct nvgpu_falcon *flcn,
+		    struct nvgpu_engine_mem_queue *queue,
+		    u32 dst, void *data, u32 size);
+	int (*pop)(struct nvgpu_falcon *flcn,
+		   struct nvgpu_engine_mem_queue *queue,
+		   u32 src, void *data, u32 size);
+
+	/* engine specific ops */
+	int (*head)(struct gk20a *g, u32 queue_id, u32 queue_index,
+		u32 *head, bool set);
+	int (*tail)(struct gk20a *g, u32 queue_id, u32 queue_index,
+		u32 *tail, bool set);
+};
+
+#endif /* NVGPU_ENGINE_MEM_QUEUE_PRIV_H */

drivers/gpu/nvgpu/common/falcon/falcon.c

@@ -0,0 +1,825 @@
+/*
+ * Copyright (c) 2017-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+#include <nvgpu/gk20a.h>
+#include <nvgpu/timers.h>
+#include <nvgpu/falcon.h>
+#include <nvgpu/io.h>
+#include <nvgpu/static_analysis.h>
+
+#include "falcon_sw_gk20a.h"
+#ifdef CONFIG_NVGPU_DGPU
+#include "falcon_sw_tu104.h"
+#endif
+
+#if defined(CONFIG_NVGPU_NEXT) && defined(CONFIG_NVGPU_NON_FUSA)
+#include "nvgpu_next_gpuid.h"
+#endif
+
+static bool is_falcon_valid(struct nvgpu_falcon *flcn)
+{
+	if (flcn == NULL) {
+		return false;
+	}
+
+	if (!flcn->is_falcon_supported) {
+		nvgpu_err(flcn->g, "Falcon %d not supported", flcn->flcn_id);
+		return false;
+	}
+
+	return true;
+}
+
+u32 nvgpu_falcon_readl(struct nvgpu_falcon *flcn, u32 offset)
+{
+	return nvgpu_readl(flcn->g,
+			   nvgpu_safe_add_u32(flcn->flcn_base, offset));
+}
+
+void nvgpu_falcon_writel(struct nvgpu_falcon *flcn,
+				       u32 offset, u32 val)
+{
+	nvgpu_writel(flcn->g, nvgpu_safe_add_u32(flcn->flcn_base, offset), val);
+}
+
+int nvgpu_falcon_reset(struct nvgpu_falcon *flcn)
+{
+	struct gk20a *g;
+	int status = 0;
+
+	if (!is_falcon_valid(flcn)) {
+		return -EINVAL;
+	}
+
+	g = flcn->g;
+
+	if (flcn->flcn_engine_dep_ops.reset_eng != NULL) {
+		/* falcon & engine reset */
+		status = flcn->flcn_engine_dep_ops.reset_eng(g);
+	} else {
+		g->ops.falcon.reset(flcn);
+	}
+
+	if (status == 0) {
+		status = nvgpu_falcon_mem_scrub_wait(flcn);
+	}
+
+	return status;
+}
+
+int nvgpu_falcon_wait_for_halt(struct nvgpu_falcon *flcn, unsigned int timeout)
+{
+	struct nvgpu_timeout to;
+	struct gk20a *g;
+	int status;
+
+	if (!is_falcon_valid(flcn)) {
+		return -EINVAL;
+	}
+
+	g = flcn->g;
+
+	status = nvgpu_timeout_init(g, &to, timeout, NVGPU_TIMER_CPU_TIMER);
+	if (status != 0) {
+		return status;
+	}
+
+	do {
+		if (g->ops.falcon.is_falcon_cpu_halted(flcn)) {
+			break;
+		}
+
+		nvgpu_udelay(10);
+	} while (nvgpu_timeout_expired(&to) == 0);
+
+	if (nvgpu_timeout_peek_expired(&to)) {
+		status = -ETIMEDOUT;
+	}
+
+	return status;
+}
+
+int nvgpu_falcon_wait_idle(struct nvgpu_falcon *flcn)
+{
+	struct nvgpu_timeout timeout;
+	struct gk20a *g;
+	int status;
+
+	if (!is_falcon_valid(flcn)) {
+		return -EINVAL;
+	}
+
+	g = flcn->g;
+
+	status = nvgpu_timeout_init(g, &timeout, 2000, NVGPU_TIMER_RETRY_TIMER);
+	if (status != 0) {
+		return status;
+	}
+
+	/* wait for falcon idle */
+	do {
+		if (g->ops.falcon.is_falcon_idle(flcn)) {
+			break;
+		}
+
+		if (nvgpu_timeout_expired_msg(&timeout,
+			"waiting for falcon idle") != 0) {
+			return -ETIMEDOUT;
+		}
+
+		nvgpu_usleep_range(100, 200);
+	} while (true);
+
+	return 0;
+}
+
+int nvgpu_falcon_mem_scrub_wait(struct nvgpu_falcon *flcn)
+{
+	struct nvgpu_timeout timeout;
+	/**
+	 * Delay depends on memory size and pwr_clk
+	 * delay = (MAX {IMEM_SIZE, DMEM_SIZE} * 64 + 1) / pwr_clk
+	 * Timeout set is 1msec & status check at interval 10usec
+	 */
+	const u32 mem_scrubbing_max_timeout = 1000U;
+	const u32 mem_scrubbing_default_timeout = 10U;
+	struct gk20a *g;
+	int status;
+
+	if (!is_falcon_valid(flcn)) {
+		return -EINVAL;
+	}
+
+	g = flcn->g;
+
+	/* check IMEM/DMEM scrubbing complete status */
+	status = nvgpu_timeout_init(g, &timeout,
+				    mem_scrubbing_max_timeout /
+					mem_scrubbing_default_timeout,
+				    NVGPU_TIMER_RETRY_TIMER);
+	if (status != 0) {
+		return status;
+	}
+
+	do {
+		if (g->ops.falcon.is_falcon_scrubbing_done(flcn)) {
+			break;
+		}
+		nvgpu_udelay(mem_scrubbing_default_timeout);
+	} while (nvgpu_timeout_expired(&timeout) == 0);
+
+	if (nvgpu_timeout_peek_expired(&timeout)) {
+		status = -ETIMEDOUT;
+	}
+
+	return status;
+}
+
+static int falcon_memcpy_params_check(struct nvgpu_falcon *flcn,
+		u32 offset, u32 size, enum falcon_mem_type mem_type, u8 port)
+{
+	struct gk20a *g = flcn->g;
+	u32 mem_size = 0;
+	int ret = -EINVAL;
+
+	if (size == 0U) {
+		nvgpu_err(g, "size is zero");
+		goto exit;
+	}
+
+	if ((offset & 0x3U) != 0U) {
+		nvgpu_err(g, "offset (0x%08x) not 4-byte aligned", offset);
+		goto exit;
+	}
+
+	if (port >= g->ops.falcon.get_ports_count(flcn, mem_type)) {
+		nvgpu_err(g, "invalid port %u", (u32) port);
+		goto exit;
+	}
+
+	mem_size = g->ops.falcon.get_mem_size(flcn, mem_type);
+
+	if (!((offset < mem_size) && ((offset + size) <= mem_size))) {
+		nvgpu_err(g, "flcn-id 0x%x, copy overflow ",
+			flcn->flcn_id);
+		nvgpu_err(g, "total size 0x%x, offset 0x%x, copy size 0x%x",
+			mem_size, offset, size);
+		ret = -EINVAL;
+		goto exit;
+	}
+
+	ret = 0;
+
+exit:
+	return ret;
+}
+
+int nvgpu_falcon_copy_to_dmem(struct nvgpu_falcon *flcn,
+	u32 dst, u8 *src, u32 size, u8 port)
+{
+	int status = -EINVAL;
+	struct gk20a *g;
+
+	if (!is_falcon_valid(flcn)) {
+		return -EINVAL;
+	}
+
+	g = flcn->g;
+
+	if (falcon_memcpy_params_check(flcn, dst, size, MEM_DMEM, port) != 0) {
+		nvgpu_err(g, "incorrect parameters");
+		goto exit;
+	}
+
+	nvgpu_mutex_acquire(&flcn->dmem_lock);
+	status = g->ops.falcon.copy_to_dmem(flcn, dst, src, size, port);
+	nvgpu_mutex_release(&flcn->dmem_lock);
+
+exit:
+	return status;
+}
+
+int nvgpu_falcon_copy_to_imem(struct nvgpu_falcon *flcn,
+	u32 dst, u8 *src, u32 size, u8 port, bool sec, u32 tag)
+{
+	int status = -EINVAL;
+	struct gk20a *g;
+
+	if (!is_falcon_valid(flcn)) {
+		return -EINVAL;
+	}
+
+	g = flcn->g;
+
+	if (falcon_memcpy_params_check(flcn, dst, size, MEM_IMEM, port) != 0) {
+		nvgpu_err(g, "incorrect parameters");
+		goto exit;
+	}
+
+	nvgpu_mutex_acquire(&flcn->imem_lock);
+	status = g->ops.falcon.copy_to_imem(flcn, dst, src,
+					    size, port, sec, tag);
+	nvgpu_mutex_release(&flcn->imem_lock);
+
+exit:
+	return status;
+}
+
+u32 nvgpu_falcon_mailbox_read(struct nvgpu_falcon *flcn, u32 mailbox_index)
+{
+	struct gk20a *g;
+	u32 data = 0;
+
+	if (!is_falcon_valid(flcn)) {
+		return 0;
+	}
+
+	g = flcn->g;
+
+	if (mailbox_index >= FALCON_MAILBOX_COUNT) {
+		nvgpu_err(g, "incorrect mailbox id %d", mailbox_index);
+		goto exit;
+	}
+
+	data = g->ops.falcon.mailbox_read(flcn, mailbox_index);
+
+exit:
+	return data;
+}
+
+void nvgpu_falcon_mailbox_write(struct nvgpu_falcon *flcn, u32 mailbox_index,
+		u32 data)
+{
+	struct gk20a *g;
+
+	if (!is_falcon_valid(flcn)) {
+		return;
+	}
+
+	g = flcn->g;
+
+	if (mailbox_index >= FALCON_MAILBOX_COUNT) {
+		nvgpu_err(g, "incorrect mailbox id %d", mailbox_index);
+		goto exit;
+	}
+
+	g->ops.falcon.mailbox_write(flcn, mailbox_index, data);
+
+exit:
+	return;
+}
+
+int nvgpu_falcon_hs_ucode_load_bootstrap(struct nvgpu_falcon *flcn, u32 *ucode,
+	u32 *ucode_header)
+{
+	struct gk20a *g;
+	u32 sec_imem_dest = 0U;
+	int err = 0;
+
+	if (!is_falcon_valid(flcn)) {
+		return -EINVAL;
+	}
+
+	g = flcn->g;
+
+	/* falcon reset */
+	err = nvgpu_falcon_reset(flcn);
+	if (err != 0) {
+		nvgpu_err(g, "nvgpu_falcon_reset() failed err=%d", err);
+		return err;
+	}
+
+	/* setup falcon apertures, boot-config */
+	if (flcn->flcn_engine_dep_ops.setup_bootstrap_config != NULL) {
+		flcn->flcn_engine_dep_ops.setup_bootstrap_config(flcn->g);
+	}
+
+	/* Copy Non Secure IMEM code */
+	err = nvgpu_falcon_copy_to_imem(flcn, 0U,
+		(u8 *)&ucode[ucode_header[OS_CODE_OFFSET] >> 2U],
+		ucode_header[OS_CODE_SIZE], 0U, false,
+		GET_IMEM_TAG(ucode_header[OS_CODE_OFFSET]));
+	if (err != 0) {
+		nvgpu_err(g, "HS ucode non-secure code to IMEM failed");
+		goto exit;
+	}
+
+	/* Put secure code after non-secure block */
+	sec_imem_dest = GET_NEXT_BLOCK(ucode_header[OS_CODE_SIZE]);
+
+	err = nvgpu_falcon_copy_to_imem(flcn, sec_imem_dest,
+		(u8 *)&ucode[ucode_header[APP_0_CODE_OFFSET] >> 2U],
+		ucode_header[APP_0_CODE_SIZE], 0U, true,
+		GET_IMEM_TAG(ucode_header[APP_0_CODE_OFFSET]));
+	if (err != 0) {
+		nvgpu_err(g, "HS ucode secure code to IMEM failed");
+		goto exit;
+	}
+
+	/* load DMEM: ensure that signatures are patched */
+	err = nvgpu_falcon_copy_to_dmem(flcn, 0U, (u8 *)&ucode[
+		ucode_header[OS_DATA_OFFSET] >> 2U],
+		ucode_header[OS_DATA_SIZE], 0U);
+	if (err != 0) {
+		nvgpu_err(g, "HS ucode data copy to DMEM failed");
+		goto exit;
+	}
+
+	/*
+	 * Write non-zero value to mailbox register which is updated by
+	 * HS bin to denote its return status.
+	 */
+	nvgpu_falcon_mailbox_write(flcn, FALCON_MAILBOX_0, 0xdeadbeefU);
+
+	/* set BOOTVEC to start of non-secure code */
+	g->ops.falcon.bootstrap(flcn, 0U);
+
+exit:
+	return err;
+}
+
+u32 nvgpu_falcon_get_id(struct nvgpu_falcon *flcn)
+{
+	return flcn->flcn_id;
+}
+
+#if defined(CONFIG_NVGPU_NEXT)
+bool nvgpu_falcon_is_falcon2_enabled(struct nvgpu_falcon *flcn)
+{
+	return flcn->is_falcon2_enabled ? true : false;
+}
+
+bool nvgpu_falcon_is_feature_supported(struct nvgpu_falcon *flcn,
+		u32 feature)
+{
+	return nvgpu_test_bit(feature, (void *)&flcn->fuse_settings);
+}
+#endif
+
+struct nvgpu_falcon *nvgpu_falcon_get_instance(struct gk20a *g, u32 flcn_id)
+{
+	struct nvgpu_falcon *flcn = NULL;
+
+	switch (flcn_id) {
+	case FALCON_ID_PMU:
+		flcn = &g->pmu_flcn;
+		break;
+	case FALCON_ID_FECS:
+		flcn = &g->fecs_flcn;
+		break;
+	case FALCON_ID_GPCCS:
+		flcn = &g->gpccs_flcn;
+		break;
+	case FALCON_ID_GSPLITE:
+		flcn = &g->gsp_flcn;
+		break;
+	case FALCON_ID_NVDEC:
+		flcn = &g->nvdec_flcn;
+		break;
+	case FALCON_ID_SEC2:
+		flcn = &g->sec2.flcn;
+		break;
+	case FALCON_ID_MINION:
+		flcn = &g->minion_flcn;
+		break;
+	default:
+		nvgpu_err(g, "Invalid/Unsupported falcon ID %x", flcn_id);
+		break;
+	};
+
+	return flcn;
+}
+
+static int falcon_sw_chip_init(struct gk20a *g, struct nvgpu_falcon *flcn)
+{
+	u32 ver = nvgpu_safe_add_u32(g->params.gpu_arch, g->params.gpu_impl);
+	int err = 0;
+
+	switch (ver) {
+#ifdef CONFIG_NVGPU_NON_FUSA
+	case GK20A_GPUID_GM20B:
+	case GK20A_GPUID_GM20B_B:
+		gk20a_falcon_sw_init(flcn);
+		break;
+	case NVGPU_GPUID_GP10B:
+		gk20a_falcon_sw_init(flcn);
+		break;
+#if defined(CONFIG_NVGPU_NEXT)
+	case NVGPU_NEXT_GPUID:
+		nvgpu_next_falcon_sw_init(flcn);
+		break;
+#endif
+#endif
+	case NVGPU_GPUID_GV11B:
+		gk20a_falcon_sw_init(flcn);
+		break;
+#ifdef CONFIG_NVGPU_DGPU
+	case NVGPU_GPUID_TU104:
+#if defined(CONFIG_NVGPU_NEXT)
+	case NVGPU_NEXT_DGPU_GPUID:
+#endif
+		tu104_falcon_sw_init(flcn);
+		break;
+#endif
+	default:
+		err = -EINVAL;
+		nvgpu_err(g, "no support for GPUID %x", ver);
+		break;
+	}
+
+	return err;
+}
+
+int nvgpu_falcon_sw_init(struct gk20a *g, u32 flcn_id)
+{
+	struct nvgpu_falcon *flcn = NULL;
+	int err = 0;
+
+	flcn = nvgpu_falcon_get_instance(g, flcn_id);
+	if (flcn == NULL) {
+		return -ENODEV;
+	}
+
+	flcn->flcn_id = flcn_id;
+	flcn->g = g;
+
+	/* call SW init methods to assign flcn base & support of a falcon */
+	err = falcon_sw_chip_init(g, flcn);
+	if (err != 0) {
+		nvgpu_err(g, "Chip specific falcon sw init failed %d", err);
+		return err;
+	}
+
+	nvgpu_mutex_init(&flcn->imem_lock);
+	nvgpu_mutex_init(&flcn->dmem_lock);
+
+#ifdef CONFIG_NVGPU_DGPU
+	if (flcn->emem_supported) {
+		nvgpu_mutex_init(&flcn->emem_lock);
+	}
+#endif
+
+	return 0;
+}
+
+void nvgpu_falcon_sw_free(struct gk20a *g, u32 flcn_id)
+{
+	struct nvgpu_falcon *flcn = NULL;
+
+	flcn = nvgpu_falcon_get_instance(g, flcn_id);
+	if (flcn == NULL) {
+		return;
+	}
+
+	if (flcn->is_falcon_supported) {
+		flcn->is_falcon_supported = false;
+	} else {
+		nvgpu_log_info(g, "falcon 0x%x not supported on %s",
+			flcn->flcn_id, g->name);
+		return;
+	}
+
+#ifdef CONFIG_NVGPU_DGPU
+	if (flcn->emem_supported) {
+		nvgpu_mutex_destroy(&flcn->emem_lock);
+	}
+#endif
+	nvgpu_mutex_destroy(&flcn->dmem_lock);
+	nvgpu_mutex_destroy(&flcn->imem_lock);
+}
+
+void nvgpu_falcon_set_irq(struct nvgpu_falcon *flcn, bool enable,
+	u32 intr_mask, u32 intr_dest)
+{
+	struct gk20a *g;
+
+	if (!is_falcon_valid(flcn)) {
+		return;
+	}
+
+	g = flcn->g;
+
+	if (!flcn->is_interrupt_enabled) {
+		nvgpu_warn(g, "Interrupt not supported on flcn 0x%x ",
+			flcn->flcn_id);
+		return;
+	}
+
+	g->ops.falcon.set_irq(flcn, enable, intr_mask, intr_dest);
+}
+
+#ifdef CONFIG_NVGPU_DGPU
+int nvgpu_falcon_copy_from_emem(struct nvgpu_falcon *flcn,
+	u32 src, u8 *dst, u32 size, u8 port)
+{
+	struct nvgpu_falcon_engine_dependency_ops *flcn_dops;
+	int status = -EINVAL;
+	struct gk20a *g;
+
+	if (!is_falcon_valid(flcn)) {
+		return -EINVAL;
+	}
+
+	g = flcn->g;
+	flcn_dops = &flcn->flcn_engine_dep_ops;
+
+	if (flcn_dops->copy_from_emem != NULL) {
+		nvgpu_mutex_acquire(&flcn->emem_lock);
+		status = flcn_dops->copy_from_emem(g, src, dst, size, port);
+		nvgpu_mutex_release(&flcn->emem_lock);
+	} else {
+		nvgpu_warn(g, "Invalid op on falcon 0x%x ",
+			flcn->flcn_id);
+		goto exit;
+	}
+
+exit:
+	return status;
+}
+
+int nvgpu_falcon_copy_to_emem(struct nvgpu_falcon *flcn,
+	u32 dst, u8 *src, u32 size, u8 port)
+{
+	struct nvgpu_falcon_engine_dependency_ops *flcn_dops;
+	int status = -EINVAL;
+	struct gk20a *g;
+
+	if (!is_falcon_valid(flcn)) {
+		return -EINVAL;
+	}
+
+	g = flcn->g;
+	flcn_dops = &flcn->flcn_engine_dep_ops;
+
+	if (flcn_dops->copy_to_emem != NULL) {
+		nvgpu_mutex_acquire(&flcn->emem_lock);
+		status = flcn_dops->copy_to_emem(g, dst, src, size, port);
+		nvgpu_mutex_release(&flcn->emem_lock);
+	} else {
+		nvgpu_warn(g, "Invalid op on falcon 0x%x ",
+			flcn->flcn_id);
+		goto exit;
+	}
+
+exit:
+	return status;
+}
+#endif
+
+#ifdef CONFIG_NVGPU_FALCON_DEBUG
+void nvgpu_falcon_dump_stats(struct nvgpu_falcon *flcn)
+{
+	if (!is_falcon_valid(flcn)) {
+		return;
+	}
+
+	flcn->g->ops.falcon.dump_falcon_stats(flcn);
+}
+#endif
+
+#ifdef CONFIG_NVGPU_FALCON_NON_FUSA
+int nvgpu_falcon_bootstrap(struct nvgpu_falcon *flcn, u32 boot_vector)
+{
+	if (!is_falcon_valid(flcn)) {
+		return -EINVAL;
+	}
+
+	flcn->g->ops.falcon.bootstrap(flcn, boot_vector);
+
+	return 0;
+}
+
+int nvgpu_falcon_get_mem_size(struct nvgpu_falcon *flcn,
+			      enum falcon_mem_type type, u32 *size)
+{
+	if (!is_falcon_valid(flcn)) {
+		return -EINVAL;
+	}
+
+	*size = flcn->g->ops.falcon.get_mem_size(flcn, type);
+
+	return 0;
+}
+
+int nvgpu_falcon_clear_halt_intr_status(struct nvgpu_falcon *flcn,
+	unsigned int timeout)
+{
+	struct nvgpu_timeout to;
+	struct gk20a *g;
+	int status;
+
+	if (!is_falcon_valid(flcn)) {
+		return -EINVAL;
+	}
+
+	g = flcn->g;
+
+	status = nvgpu_timeout_init(g, &to, timeout, NVGPU_TIMER_CPU_TIMER);
+	if (status != 0) {
+		return status;
+	}
+
+	do {
+		if (g->ops.falcon.clear_halt_interrupt_status(flcn)) {
+			break;
+		}
+
+		nvgpu_udelay(1);
+	} while (nvgpu_timeout_expired(&to) == 0);
+
+	if (nvgpu_timeout_peek_expired(&to)) {
+		status = -ETIMEDOUT;
+	}
+
+	return status;
+}
+
+int nvgpu_falcon_copy_from_dmem(struct nvgpu_falcon *flcn,
+	u32 src, u8 *dst, u32 size, u8 port)
+{
+	int status = -EINVAL;
+	struct gk20a *g;
+
+	if (!is_falcon_valid(flcn)) {
+		return -EINVAL;
+	}
+
+	g = flcn->g;
+
+	if (falcon_memcpy_params_check(flcn, src, size, MEM_DMEM, port) != 0) {
+		nvgpu_err(g, "incorrect parameters");
+		goto exit;
+	}
+
+	nvgpu_mutex_acquire(&flcn->dmem_lock);
+	status = g->ops.falcon.copy_from_dmem(flcn, src, dst, size, port);
+	nvgpu_mutex_release(&flcn->dmem_lock);
+
+exit:
+	return status;
+}
+
+int nvgpu_falcon_copy_from_imem(struct nvgpu_falcon *flcn,
+	u32 src, u8 *dst, u32 size, u8 port)
+{
+	int status = -EINVAL;
+	struct gk20a *g;
+
+	if (!is_falcon_valid(flcn)) {
+		return -EINVAL;
+	}
+
+	g = flcn->g;
+
+	if (falcon_memcpy_params_check(flcn, src, size, MEM_IMEM, port) != 0) {
+		nvgpu_err(g, "incorrect parameters");
+		goto exit;
+	}
+
+	nvgpu_mutex_acquire(&flcn->imem_lock);
+	status = g->ops.falcon.copy_from_imem(flcn, src, dst, size, port);
+	nvgpu_mutex_release(&flcn->imem_lock);
+
+exit:
+	return status;
+}
+
+static void falcon_print_mem(struct nvgpu_falcon *flcn, u32 src,
+	u32 size, enum falcon_mem_type mem_type)
+{
+	u32 buff[64] = {0};
+	u32 total_block_read = 0;
+	u32 byte_read_count = 0;
+	struct gk20a *g;
+	u32 i = 0;
+	int status = 0;
+
+	g = flcn->g;
+
+	if (falcon_memcpy_params_check(flcn, src, size, mem_type, 0) != 0) {
+		nvgpu_err(g, "incorrect parameters");
+		return;
+	}
+
+	nvgpu_info(g, " offset 0x%x  size %d bytes", src, size);
+
+	total_block_read = size >> 8;
+	do {
+		byte_read_count =
+			(total_block_read != 0U) ? (u32)sizeof(buff) : size;
+
+		if (byte_read_count == 0U) {
+			break;
+		}
+
+		if (mem_type == MEM_DMEM) {
+			status = nvgpu_falcon_copy_from_dmem(flcn, src,
+				(u8 *)buff, byte_read_count, 0);
+		} else {
+			status = nvgpu_falcon_copy_from_imem(flcn, src,
+				(u8 *)buff, byte_read_count, 0);
+		}
+
+		if (status != 0) {
+			nvgpu_err(g, "MEM print failed");
+			break;
+		}
+
+		for (i = 0U; i < (byte_read_count >> 2U); i += 4U) {
+			nvgpu_info(g, "0x%04x: 0x%08x 0x%08x 0x%08x 0x%08x",
+				src + (i << 2U), buff[i], buff[i+1U],
+				buff[i+2U], buff[i+3U]);
+		}
+
+		src += byte_read_count;
+		size -= byte_read_count;
+	} while (total_block_read-- != 0U);
+}
+
+void nvgpu_falcon_print_dmem(struct nvgpu_falcon *flcn, u32 src, u32 size)
+{
+	if (!is_falcon_valid(flcn)) {
+		return;
+	}
+
+	nvgpu_info(flcn->g, " PRINT DMEM ");
+	falcon_print_mem(flcn, src, size, MEM_DMEM);
+}
+
+void nvgpu_falcon_print_imem(struct nvgpu_falcon *flcn, u32 src, u32 size)
+{
+	if (!is_falcon_valid(flcn)) {
+		return;
+	}
+
+	nvgpu_info(flcn->g, " PRINT IMEM ");
+	falcon_print_mem(flcn, src, size, MEM_IMEM);
+}
+
+void nvgpu_falcon_get_ctls(struct nvgpu_falcon *flcn, u32 *sctl, u32 *cpuctl)
+{
+	if (!is_falcon_valid(flcn)) {
+		return;
+	}
+
+	flcn->g->ops.falcon.get_falcon_ctls(flcn, sctl, cpuctl);
+}
+#endif

drivers/gpu/nvgpu/common/falcon/falcon_sw_gk20a.c

@@ -0,0 +1,80 @@
+/*
+ * Copyright (c) 2017-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+#include <nvgpu/gk20a.h>
+#include <nvgpu/falcon.h>
+
+#include "falcon_sw_gk20a.h"
+
+void gk20a_falcon_engine_dependency_ops(struct nvgpu_falcon *flcn)
+{
+	struct gk20a *g = flcn->g;
+	struct nvgpu_falcon_engine_dependency_ops *flcn_eng_dep_ops =
+			&flcn->flcn_engine_dep_ops;
+
+	switch (flcn->flcn_id) {
+	case FALCON_ID_PMU:
+		flcn_eng_dep_ops->reset_eng = g->ops.pmu.pmu_reset;
+		flcn_eng_dep_ops->setup_bootstrap_config =
+			g->ops.pmu.flcn_setup_boot_config;
+		break;
+	default:
+		/* NULL assignment make sure
+		 * CPU hard reset in gk20a_falcon_reset() gets execute
+		 * if falcon doesn't need specific reset implementation
+		 */
+		flcn_eng_dep_ops->reset_eng = NULL;
+		break;
+	}
+}
+
+void gk20a_falcon_sw_init(struct nvgpu_falcon *flcn)
+{
+	struct gk20a *g = flcn->g;
+
+	switch (flcn->flcn_id) {
+	case FALCON_ID_PMU:
+		flcn->flcn_base = g->ops.pmu.falcon_base_addr();
+		flcn->is_falcon_supported = true;
+		flcn->is_interrupt_enabled = true;
+		break;
+	case FALCON_ID_FECS:
+		flcn->flcn_base = g->ops.gr.falcon.fecs_base_addr();
+		flcn->is_falcon_supported = true;
+		flcn->is_interrupt_enabled = false;
+		break;
+	case FALCON_ID_GPCCS:
+		flcn->flcn_base = g->ops.gr.falcon.gpccs_base_addr();
+		flcn->is_falcon_supported = true;
+		flcn->is_interrupt_enabled = false;
+		break;
+	default:
+		flcn->is_falcon_supported = false;
+		break;
+	}
+
+	if (flcn->is_falcon_supported) {
+		gk20a_falcon_engine_dependency_ops(flcn);
+	} else {
+		nvgpu_log_info(g, "falcon 0x%x not supported on %s",
+			flcn->flcn_id, g->name);
+	}
+}

drivers/gpu/nvgpu/common/falcon/falcon_sw_gk20a.h

@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+#ifndef NVGPU_FALCON_SW_GK20A_H
+#define NVGPU_FALCON_SW_GK20A_H
+
+void gk20a_falcon_engine_dependency_ops(struct nvgpu_falcon *flcn);
+void gk20a_falcon_sw_init(struct nvgpu_falcon *flcn);
+
+#endif /* NVGPU_FALCON_SW_GK20A_H */

drivers/gpu/nvgpu/common/falcon/falcon_sw_tu104.c

@@ -0,0 +1,102 @@
+/*
+ * Copyright (c) 2018-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+#include <nvgpu/gk20a.h>
+#include <nvgpu/falcon.h>
+
+#include "falcon_sw_gk20a.h"
+#include "falcon_sw_tu104.h"
+
+void tu104_falcon_engine_dependency_ops(struct nvgpu_falcon *flcn)
+{
+	struct nvgpu_falcon_engine_dependency_ops *flcn_eng_dep_ops =
+			&flcn->flcn_engine_dep_ops;
+	struct gk20a *g = flcn->g;
+
+	gk20a_falcon_engine_dependency_ops(flcn);
+
+	switch (flcn->flcn_id) {
+	case FALCON_ID_GSPLITE:
+		flcn_eng_dep_ops->reset_eng = g->ops.gsp.gsp_reset;
+		flcn_eng_dep_ops->setup_bootstrap_config =
+			g->ops.gsp.falcon_setup_boot_config;
+		break;
+	case FALCON_ID_SEC2:
+		flcn_eng_dep_ops->reset_eng = g->ops.sec2.sec2_reset;
+		flcn_eng_dep_ops->setup_bootstrap_config =
+			g->ops.sec2.flcn_setup_boot_config;
+		flcn_eng_dep_ops->copy_to_emem = g->ops.sec2.sec2_copy_to_emem;
+		flcn_eng_dep_ops->copy_from_emem =
+						g->ops.sec2.sec2_copy_from_emem;
+		break;
+	default:
+		flcn_eng_dep_ops->reset_eng = NULL;
+		break;
+	}
+}
+
+void tu104_falcon_sw_init(struct nvgpu_falcon *flcn)
+{
+	struct gk20a *g = flcn->g;
+
+	switch (flcn->flcn_id) {
+	case FALCON_ID_GSPLITE:
+		flcn->flcn_base = g->ops.gsp.falcon_base_addr();
+		flcn->is_falcon_supported = true;
+		flcn->is_interrupt_enabled = false;
+		break;
+	case FALCON_ID_SEC2:
+		flcn->flcn_base = g->ops.sec2.falcon_base_addr();
+		flcn->is_falcon_supported = true;
+		flcn->is_interrupt_enabled = true;
+		flcn->emem_supported = true;
+		break;
+	case FALCON_ID_MINION:
+		flcn->flcn_base = g->ops.nvlink.minion.base_addr(g);
+		flcn->is_falcon_supported = true;
+		flcn->is_interrupt_enabled = true;
+		break;
+	case FALCON_ID_NVDEC:
+		flcn->flcn_base = g->ops.nvdec.falcon_base_addr();
+		flcn->is_falcon_supported = true;
+		flcn->is_interrupt_enabled = true;
+		break;
+	default:
+		/*
+		 * set false to inherit falcon support
+		 * from previous chips HAL
+		 */
+		flcn->is_falcon_supported = false;
+		break;
+	}
+
+	if (flcn->is_falcon_supported) {
+		tu104_falcon_engine_dependency_ops(flcn);
+	} else {
+		/*
+		 * Forward call to previous chip's SW init
+		 * to fetch info for requested
+		 * falcon as no changes between
+		 * current & previous chips.
+		 */
+		gk20a_falcon_sw_init(flcn);
+	}
+}

drivers/gpu/nvgpu/common/falcon/falcon_sw_tu104.h

@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef NVGPU_FALCON_SW_TU104_H
+#define NVGPU_FALCON_SW_TU104_H
+
+void tu104_falcon_engine_dependency_ops(struct nvgpu_falcon *flcn);
+void tu104_falcon_sw_init(struct nvgpu_falcon *flcn);
+
+#endif /* NVGPU_FALCON_SW_TU104_H */

drivers/gpu/nvgpu/common/fb/fb.c

@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/gk20a.h>
+#include <nvgpu/power_features/cg.h>
+#include <nvgpu/fb.h>
+
+int nvgpu_init_fb_support(struct gk20a *g)
+{
+	if (g->ops.mc.fb_reset != NULL) {
+		g->ops.mc.fb_reset(g);
+	}
+
+	nvgpu_cg_slcg_fb_ltc_load_enable(g);
+
+	nvgpu_cg_blcg_fb_ltc_load_enable(g);
+
+	if (g->ops.fb.init_fs_state != NULL) {
+		g->ops.fb.init_fs_state(g);
+	}
+	return 0;
+}

drivers/gpu/nvgpu/common/fbp/fbp.c

@@ -0,0 +1,130 @@
+/*
+ * Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/gk20a.h>
+#include <nvgpu/kmem.h>
+#include <nvgpu/log.h>
+#include <nvgpu/fbp.h>
+#include <nvgpu/static_analysis.h>
+
+#include "fbp_priv.h"
+
+int nvgpu_fbp_init_support(struct gk20a *g)
+{
+	struct nvgpu_fbp *fbp;
+	u32 fbp_en_mask;
+#ifdef CONFIG_NVGPU_NON_FUSA
+	u32 max_ltc_per_fbp;
+	u32 rop_l2_all_en;
+	unsigned long i;
+	unsigned long fbp_en_mask_tmp;
+	u32 tmp;
+#endif
+
+	if (g->fbp != NULL) {
+		return 0;
+	}
+
+	fbp = nvgpu_kzalloc(g, sizeof(*fbp));
+	if (fbp == NULL) {
+		return -ENOMEM;
+	}
+
+#ifdef CONFIG_NVGPU_NON_FUSA
+	fbp->num_fbps = g->ops.priv_ring.get_fbp_count(g);
+	nvgpu_log_info(g, "fbps: %d", fbp->num_fbps);
+#endif
+
+	fbp->max_fbps_count = g->ops.top.get_max_fbps_count(g);
+	nvgpu_log_info(g, "max_fbps_count: %d", fbp->max_fbps_count);
+
+	/*
+	 * Read active fbp mask from fuse
+	 * Note that 0:enable and 1:disable in value read from fuse so we've to
+	 * flip the bits.
+	 * Also set unused bits to zero
+	 */
+	fbp_en_mask = g->ops.fuse.fuse_status_opt_fbp(g);
+	fbp_en_mask = ~fbp_en_mask;
+	fbp_en_mask = fbp_en_mask &
+		nvgpu_safe_sub_u32(BIT32(fbp->max_fbps_count), 1U);
+	fbp->fbp_en_mask = fbp_en_mask;
+
+#ifdef CONFIG_NVGPU_NON_FUSA
+	fbp->fbp_rop_l2_en_mask =
+		nvgpu_kzalloc(g,
+			nvgpu_safe_mult_u64(fbp->max_fbps_count, sizeof(u32)));
+	if (fbp->fbp_rop_l2_en_mask == NULL) {
+		nvgpu_kfree(g, fbp);
+		return -ENOMEM;
+	}
+
+	fbp_en_mask_tmp = fbp_en_mask;
+	max_ltc_per_fbp = g->ops.top.get_max_ltc_per_fbp(g);
+	rop_l2_all_en = nvgpu_safe_sub_u32(BIT32(max_ltc_per_fbp), 1U);
+
+	/* mask of Rop_L2 for each FBP */
+	for_each_set_bit(i, &fbp_en_mask_tmp, fbp->max_fbps_count) {
+		tmp = g->ops.fuse.fuse_status_opt_rop_l2_fbp(g, i);
+		fbp->fbp_rop_l2_en_mask[i] = rop_l2_all_en ^ tmp;
+	}
+#endif
+
+	g->fbp = fbp;
+
+	return 0;
+}
+
+void nvgpu_fbp_remove_support(struct gk20a *g)
+{
+	struct nvgpu_fbp *fbp = g->fbp;
+
+	if (fbp != NULL) {
+		nvgpu_kfree(g, fbp->fbp_rop_l2_en_mask);
+		nvgpu_kfree(g, fbp);
+	}
+
+	g->fbp = NULL;
+}
+
+u32 nvgpu_fbp_get_max_fbps_count(struct nvgpu_fbp *fbp)
+{
+	return fbp->max_fbps_count;
+}
+
+u32 nvgpu_fbp_get_fbp_en_mask(struct nvgpu_fbp *fbp)
+{
+	return fbp->fbp_en_mask;
+}
+
+#ifdef CONFIG_NVGPU_NON_FUSA
+u32 nvgpu_fbp_get_num_fbps(struct nvgpu_fbp *fbp)
+{
+	return fbp->num_fbps;
+}
+
+u32 *nvgpu_fbp_get_rop_l2_en_mask(struct nvgpu_fbp *fbp)
+{
+	return fbp->fbp_rop_l2_en_mask;
+}
+#endif
+

drivers/gpu/nvgpu/common/fbp/fbp_priv.h

@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef NVGPU_FBP_PRIV_H
+#define NVGPU_FBP_PRIV_H
+
+struct nvgpu_fbp {
+	u32 num_fbps;
+	u32 max_fbps_count;
+	u32 fbp_en_mask;
+	u32 *fbp_rop_l2_en_mask;
+};
+
+#endif /* NVGPU_FBP_PRIV_H */

drivers/gpu/nvgpu/common/fence/fence.c

@@ -0,0 +1,120 @@
+/*
+ * Copyright (c) 2014-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/soc.h>
+#include <nvgpu/os_fence.h>
+#include <nvgpu/gk20a.h>
+#include <nvgpu/fence.h>
+#include <nvgpu/user_fence.h>
+#include "fence_priv.h"
+
+static struct nvgpu_fence_type *nvgpu_fence_from_ref(struct nvgpu_ref *ref)
+{
+	return (struct nvgpu_fence_type *)((uintptr_t)ref -
+				offsetof(struct nvgpu_fence_type, priv.ref));
+}
+
+static void nvgpu_fence_release(struct nvgpu_ref *ref)
+{
+	struct nvgpu_fence_type *f = nvgpu_fence_from_ref(ref);
+	struct nvgpu_fence_type_priv *pf = &f->priv;
+
+	if (nvgpu_os_fence_is_initialized(&pf->os_fence)) {
+		pf->os_fence.ops->drop_ref(&pf->os_fence);
+	}
+
+	pf->ops->release(f);
+}
+
+void nvgpu_fence_put(struct nvgpu_fence_type *f)
+{
+	struct nvgpu_fence_type_priv *pf = &f->priv;
+
+	nvgpu_ref_put(&pf->ref, nvgpu_fence_release);
+}
+
+struct nvgpu_fence_type *nvgpu_fence_get(struct nvgpu_fence_type *f)
+{
+	struct nvgpu_fence_type_priv *pf = &f->priv;
+
+	nvgpu_ref_get(&pf->ref);
+	return f;
+}
+
+/*
+ * Extract an object to be passed to the userspace as a result of a submitted
+ * job. This must be balanced with a call to nvgpu_user_fence_release().
+ */
+struct nvgpu_user_fence nvgpu_fence_extract_user(struct nvgpu_fence_type *f)
+{
+	struct nvgpu_fence_type_priv *pf = &f->priv;
+
+	struct nvgpu_user_fence uf = (struct nvgpu_user_fence) {
+#ifdef CONFIG_TEGRA_GK20A_NVHOST
+		.syncpt_id = pf->syncpt_id,
+		.syncpt_value = pf->syncpt_value,
+#endif
+		.os_fence = pf->os_fence,
+	};
+
+	/*
+	 * The os fence member has to live so it can be signaled when the job
+	 * completes. The returned user fence may live longer than that before
+	 * being safely attached to an fd if the job completes before a
+	 * submission ioctl finishes, or if it's stored for cde job state
+	 * tracking.
+	 */
+	if (nvgpu_os_fence_is_initialized(&pf->os_fence)) {
+		pf->os_fence.ops->dup(&pf->os_fence);
+	}
+
+	return uf;
+}
+
+int nvgpu_fence_wait(struct gk20a *g, struct nvgpu_fence_type *f,
+							u32 timeout)
+{
+	struct nvgpu_fence_type_priv *pf = &f->priv;
+
+	if (!nvgpu_platform_is_silicon(g)) {
+		timeout = U32_MAX;
+	}
+	return pf->ops->wait(f, timeout);
+}
+
+bool nvgpu_fence_is_expired(struct nvgpu_fence_type *f)
+{
+	struct nvgpu_fence_type_priv *pf = &f->priv;
+
+	return pf->ops->is_expired(f);
+}
+
+void nvgpu_fence_init(struct nvgpu_fence_type *f,
+		const struct nvgpu_fence_ops *ops,
+		struct nvgpu_os_fence os_fence)
+{
+	struct nvgpu_fence_type_priv *pf = &f->priv;
+
+	nvgpu_ref_init(&pf->ref);
+	pf->ops = ops;
+	pf->os_fence = os_fence;
+}

drivers/gpu/nvgpu/common/fence/fence_priv.h

@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2014-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+#ifndef NVGPU_FENCE_PRIV_H
+#define NVGPU_FENCE_PRIV_H
+
+#include <nvgpu/os_fence.h>
+
+struct nvgpu_fence_type;
+
+struct nvgpu_fence_ops {
+	int (*wait)(struct nvgpu_fence_type *f, u32 timeout);
+	bool (*is_expired)(struct nvgpu_fence_type *f);
+	void (*release)(struct nvgpu_fence_type *f);
+};
+
+void nvgpu_fence_init(struct nvgpu_fence_type *f,
+		const struct nvgpu_fence_ops *ops,
+		struct nvgpu_os_fence os_fence);
+
+#endif

drivers/gpu/nvgpu/common/fence/fence_sema.c

@@ -0,0 +1,78 @@
+/*
+ * Copyright (c) 2014-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/semaphore.h>
+#include <nvgpu/cond.h>
+#include <nvgpu/fence.h>
+#include <nvgpu/fence_sema.h>
+#include "fence_priv.h"
+
+static int nvgpu_fence_semaphore_wait(struct nvgpu_fence_type *f, u32 timeout)
+{
+	struct nvgpu_fence_type_priv *pf = &f->priv;
+
+	if (!nvgpu_semaphore_is_acquired(pf->semaphore)) {
+		return 0;
+	}
+
+	return NVGPU_COND_WAIT_INTERRUPTIBLE(
+		pf->semaphore_wq,
+		!nvgpu_semaphore_is_acquired(pf->semaphore),
+		timeout);
+}
+
+static bool nvgpu_fence_semaphore_is_expired(struct nvgpu_fence_type *f)
+{
+	struct nvgpu_fence_type_priv *pf = &f->priv;
+
+	return !nvgpu_semaphore_is_acquired(pf->semaphore);
+}
+
+static void nvgpu_fence_semaphore_release(struct nvgpu_fence_type *f)
+{
+	struct nvgpu_fence_type_priv *pf = &f->priv;
+
+	if (pf->semaphore != NULL) {
+		nvgpu_semaphore_put(pf->semaphore);
+	}
+}
+
+static const struct nvgpu_fence_ops nvgpu_fence_semaphore_ops = {
+	.wait = nvgpu_fence_semaphore_wait,
+	.is_expired = nvgpu_fence_semaphore_is_expired,
+	.release = nvgpu_fence_semaphore_release,
+};
+
+/* This function takes ownership of the semaphore as well as the os_fence */
+void nvgpu_fence_from_semaphore(
+		struct nvgpu_fence_type *f,
+		struct nvgpu_semaphore *semaphore,
+		struct nvgpu_cond *semaphore_wq,
+		struct nvgpu_os_fence os_fence)
+{
+	struct nvgpu_fence_type_priv *pf = &f->priv;
+
+	nvgpu_fence_init(f, &nvgpu_fence_semaphore_ops, os_fence);
+
+	pf->semaphore = semaphore;
+	pf->semaphore_wq = semaphore_wq;
+}

drivers/gpu/nvgpu/common/fence/fence_syncpt.c

@@ -0,0 +1,89 @@
+/*
+ * Copyright (c) 2014-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/nvhost.h>
+#include <nvgpu/fence.h>
+#include <nvgpu/fence_syncpt.h>
+#include "fence_priv.h"
+
+static int nvgpu_fence_syncpt_wait(struct nvgpu_fence_type *f, u32 timeout)
+{
+	struct nvgpu_fence_type_priv *pf = &f->priv;
+
+	return nvgpu_nvhost_syncpt_wait_timeout_ext(
+			pf->nvhost_device, pf->syncpt_id, pf->syncpt_value,
+			timeout, NVGPU_NVHOST_DEFAULT_WAITER);
+}
+
+static bool nvgpu_fence_syncpt_is_expired(struct nvgpu_fence_type *f)
+{
+	struct nvgpu_fence_type_priv *pf = &f->priv;
+
+	/*
+	 * In cases we don't register a notifier, we can't expect the
+	 * syncpt value to be updated. For this case, we force a read
+	 * of the value from HW, and then check for expiration.
+	 */
+	if (!nvgpu_nvhost_syncpt_is_expired_ext(pf->nvhost_device,
+				pf->syncpt_id, pf->syncpt_value)) {
+		int err;
+		u32 val;
+
+		err = nvgpu_nvhost_syncpt_read_ext_check(pf->nvhost_device,
+				pf->syncpt_id, &val);
+		WARN(err != 0, "syncpt read failed??");
+		if (err == 0) {
+			return nvgpu_nvhost_syncpt_is_expired_ext(
+					pf->nvhost_device,
+					pf->syncpt_id, pf->syncpt_value);
+		} else {
+			return false;
+		}
+	}
+
+	return true;
+}
+
+static void nvgpu_fence_syncpt_release(struct nvgpu_fence_type *f)
+{
+}
+
+static const struct nvgpu_fence_ops nvgpu_fence_syncpt_ops = {
+	.wait = nvgpu_fence_syncpt_wait,
+	.is_expired = nvgpu_fence_syncpt_is_expired,
+	.release = nvgpu_fence_syncpt_release,
+};
+
+/* This function takes the ownership of the os_fence */
+void nvgpu_fence_from_syncpt(
+		struct nvgpu_fence_type *f,
+		struct nvgpu_nvhost_dev *nvhost_device,
+		u32 id, u32 value, struct nvgpu_os_fence os_fence)
+{
+	struct nvgpu_fence_type_priv *pf = &f->priv;
+
+	nvgpu_fence_init(f, &nvgpu_fence_syncpt_ops, os_fence);
+
+	pf->nvhost_device = nvhost_device;
+	pf->syncpt_id = id;
+	pf->syncpt_value = value;
+}

drivers/gpu/nvgpu/common/fifo/channel_wdt.c

@@ -0,0 +1,199 @@
+/*
+ * Copyright (c) 2015-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include "channel_wdt.h"
+#include "channel_worker.h"
+
+#include <nvgpu/watchdog.h>
+#include <nvgpu/channel.h>
+#include <nvgpu/error_notifier.h>
+#include <nvgpu/gk20a.h>
+
+void nvgpu_channel_set_wdt_debug_dump(struct nvgpu_channel *ch, bool dump)
+{
+	ch->wdt_debug_dump = dump;
+}
+
+static struct nvgpu_channel_wdt_state nvgpu_channel_collect_wdt_state(
+		struct nvgpu_channel *ch)
+{
+	struct gk20a *g = ch->g;
+	struct nvgpu_channel_wdt_state state = { 0, 0 };
+
+	/*
+	 * Note: just checking for nvgpu_channel_wdt_enabled() is not enough at
+	 * the moment because system suspend puts g->regs away but doesn't stop
+	 * the worker thread that runs the watchdog. This might need to be
+	 * cleared up in the future.
+	 */
+	if (nvgpu_channel_wdt_running(ch->wdt)) {
+		/*
+		 * Read the state only if the wdt is on to avoid unnecessary
+		 * accesses. The kernel mem for userd may not even exist; this
+		 * channel could be in usermode submit mode.
+		 */
+		state.gp_get = g->ops.userd.gp_get(g, ch);
+		state.pb_get = g->ops.userd.pb_get(g, ch);
+	}
+
+	return state;
+}
+
+void nvgpu_channel_launch_wdt(struct nvgpu_channel *ch)
+{
+	struct nvgpu_channel_wdt_state state = nvgpu_channel_collect_wdt_state(ch);
+
+	/*
+	 * FIXME: channel recovery can race the submit path and can start even
+	 * after this, but this check is the best we can do for now.
+	 */
+	if (!nvgpu_channel_check_unserviceable(ch)) {
+		nvgpu_channel_wdt_start(ch->wdt, &state);
+	}
+}
+
+void nvgpu_channel_restart_all_wdts(struct gk20a *g)
+{
+	struct nvgpu_fifo *f = &g->fifo;
+	u32 chid;
+
+	for (chid = 0; chid < f->num_channels; chid++) {
+		struct nvgpu_channel *ch = nvgpu_channel_from_id(g, chid);
+
+		if (ch != NULL) {
+			if ((ch->wdt != NULL) &&
+			    !nvgpu_channel_check_unserviceable(ch)) {
+				struct nvgpu_channel_wdt_state state =
+					nvgpu_channel_collect_wdt_state(ch);
+
+				nvgpu_channel_wdt_rewind(ch->wdt, &state);
+			}
+			nvgpu_channel_put(ch);
+		}
+	}
+}
+
+static void nvgpu_channel_recover_from_wdt(struct nvgpu_channel *ch)
+{
+	struct gk20a *g = ch->g;
+
+	nvgpu_log_fn(g, " ");
+
+	if (nvgpu_channel_check_unserviceable(ch)) {
+		/* channel is already recovered */
+		nvgpu_info(g, "chid: %d unserviceable but wdt was ON", ch->chid);
+		return;
+	}
+
+	nvgpu_err(g, "Job on channel %d timed out", ch->chid);
+
+	/* force reset calls gk20a_debug_dump but not this */
+	if (ch->wdt_debug_dump) {
+		gk20a_gr_debug_dump(g);
+	}
+
+#ifdef CONFIG_NVGPU_CHANNEL_TSG_CONTROL
+	if (g->ops.tsg.force_reset(ch,
+	    NVGPU_ERR_NOTIFIER_FIFO_ERROR_IDLE_TIMEOUT,
+	    ch->wdt_debug_dump) != 0) {
+		nvgpu_err(g, "failed tsg force reset for chid: %d", ch->chid);
+	}
+#endif
+}
+
+/*
+ * Test the watchdog progress. If the channel is stuck, reset it.
+ *
+ * The gpu is implicitly on at this point because the watchdog can only run on
+ * channels that have submitted jobs pending for cleanup.
+ */
+static void nvgpu_channel_check_wdt(struct nvgpu_channel *ch)
+{
+	struct nvgpu_channel_wdt_state state = nvgpu_channel_collect_wdt_state(ch);
+
+	if (nvgpu_channel_wdt_check(ch->wdt, &state)) {
+		nvgpu_channel_recover_from_wdt(ch);
+	}
+}
+
+void nvgpu_channel_worker_poll_init(struct nvgpu_worker *worker)
+{
+	struct nvgpu_channel_worker *ch_worker =
+		nvgpu_channel_worker_from_worker(worker);
+	int ret;
+
+	ch_worker->watchdog_interval = 100U;
+
+	ret = nvgpu_timeout_init(worker->g, &ch_worker->timeout,
+			ch_worker->watchdog_interval, NVGPU_TIMER_CPU_TIMER);
+	if (ret != 0) {
+		nvgpu_err(worker->g, "timeout_init failed: %d", ret);
+	}
+}
+
+/**
+ * Loop every living channel, check timeouts and handle stuck channels.
+ */
+static void nvgpu_channel_poll_wdt(struct gk20a *g)
+{
+	unsigned int chid;
+
+	for (chid = 0; chid < g->fifo.num_channels; chid++) {
+		struct nvgpu_channel *ch = nvgpu_channel_from_id(g, chid);
+
+		if (ch != NULL) {
+			if (!nvgpu_channel_check_unserviceable(ch)) {
+				nvgpu_channel_check_wdt(ch);
+			}
+			nvgpu_channel_put(ch);
+		}
+	}
+}
+
+void nvgpu_channel_worker_poll_wakeup_post_process_item(
+		struct nvgpu_worker *worker)
+{
+	struct gk20a *g = worker->g;
+
+	struct nvgpu_channel_worker *ch_worker =
+		nvgpu_channel_worker_from_worker(worker);
+	int ret;
+
+	if (nvgpu_timeout_peek_expired(&ch_worker->timeout)) {
+		nvgpu_channel_poll_wdt(g);
+		ret = nvgpu_timeout_init(g, &ch_worker->timeout,
+				ch_worker->watchdog_interval,
+				NVGPU_TIMER_CPU_TIMER);
+		if (ret != 0) {
+			nvgpu_err(g, "timeout_init failed: %d", ret);
+		}
+	}
+}
+
+u32 nvgpu_channel_worker_poll_wakeup_condition_get_timeout(
+		struct nvgpu_worker *worker)
+{
+	struct nvgpu_channel_worker *ch_worker =
+		nvgpu_channel_worker_from_worker(worker);
+
+	return ch_worker->watchdog_interval;
+}

drivers/gpu/nvgpu/common/fifo/channel_wdt.h

@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2015-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+#ifndef NVGPU_COMMON_FIFO_CHANNEL_WDT_H
+#define NVGPU_COMMON_FIFO_CHANNEL_WDT_H
+
+#include <nvgpu/types.h>
+
+struct nvgpu_channel;
+
+#ifdef CONFIG_NVGPU_CHANNEL_WDT
+struct nvgpu_worker;
+
+void nvgpu_channel_launch_wdt(struct nvgpu_channel *ch);
+void nvgpu_channel_worker_poll_init(struct nvgpu_worker *worker);
+void nvgpu_channel_worker_poll_wakeup_post_process_item(
+		struct nvgpu_worker *worker);
+u32 nvgpu_channel_worker_poll_wakeup_condition_get_timeout(
+		struct nvgpu_worker *worker);
+#else
+static inline void nvgpu_channel_launch_wdt(struct nvgpu_channel *ch) {}
+#endif /* CONFIG_NVGPU_CHANNEL_WDT */
+
+#endif /* NVGPU_COMMON_FIFO_CHANNEL_WDT_H */

drivers/gpu/nvgpu/common/fifo/channel_worker.c

@@ -0,0 +1,118 @@
+/*
+ * Copyright (c) 2017-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include "channel_worker.h"
+#include "channel_wdt.h"
+
+#include <nvgpu/worker.h>
+#include <nvgpu/channel.h>
+
+static inline struct nvgpu_channel *
+nvgpu_channel_from_worker_item(struct nvgpu_list_node *node)
+{
+	return (struct nvgpu_channel *)
+	   ((uintptr_t)node - offsetof(struct nvgpu_channel, worker_item));
+};
+
+static void nvgpu_channel_worker_poll_wakeup_process_item(
+		struct nvgpu_list_node *work_item)
+{
+	struct nvgpu_channel *ch = nvgpu_channel_from_worker_item(work_item);
+
+	nvgpu_assert(ch != NULL);
+
+	nvgpu_log_fn(ch->g, " ");
+
+	nvgpu_channel_clean_up_jobs(ch);
+
+	/* ref taken when enqueued */
+	nvgpu_channel_put(ch);
+}
+
+static const struct nvgpu_worker_ops channel_worker_ops = {
+#ifdef CONFIG_NVGPU_CHANNEL_WDT
+	.pre_process = nvgpu_channel_worker_poll_init,
+	.wakeup_post_process =
+		nvgpu_channel_worker_poll_wakeup_post_process_item,
+	.wakeup_timeout =
+		nvgpu_channel_worker_poll_wakeup_condition_get_timeout,
+#endif
+	.wakeup_early_exit = NULL,
+	.wakeup_process_item =
+		nvgpu_channel_worker_poll_wakeup_process_item,
+	.wakeup_condition = NULL,
+};
+
+/**
+ * Initialize the channel worker's metadata and start the background thread.
+ */
+int nvgpu_channel_worker_init(struct gk20a *g)
+{
+	struct nvgpu_worker *worker = &g->channel_worker.worker;
+
+	nvgpu_worker_init_name(worker, "nvgpu_channel_poll", g->name);
+
+	return nvgpu_worker_init(g, worker, &channel_worker_ops);
+}
+
+void nvgpu_channel_worker_deinit(struct gk20a *g)
+{
+	struct nvgpu_worker *worker = &g->channel_worker.worker;
+
+	nvgpu_worker_deinit(worker);
+}
+
+/**
+ * Append a channel to the worker's list, if not there already.
+ *
+ * The worker thread processes work items (channels in its work list) and polls
+ * for other things. This adds @ch to the end of the list and wakes the worker
+ * up immediately. If the channel already existed in the list, it's not added,
+ * because in that case it has been scheduled already but has not yet been
+ * processed.
+ */
+void nvgpu_channel_worker_enqueue(struct nvgpu_channel *ch)
+{
+	struct gk20a *g = ch->g;
+	int ret;
+
+	nvgpu_log_fn(g, " ");
+
+	/*
+	 * Ref released when this item gets processed. The caller should hold
+	 * one ref already, so normally shouldn't fail, but the channel could
+	 * end up being freed between the time the caller got its reference and
+	 * the time we end up here (e.g., if the client got killed); if so, just
+	 * return.
+	 */
+	if (nvgpu_channel_get(ch) == NULL) {
+		nvgpu_info(g, "cannot get ch ref for worker!");
+		return;
+	}
+
+	ret = nvgpu_worker_enqueue(&g->channel_worker.worker,
+			&ch->worker_item);
+	if (ret != 0) {
+		nvgpu_channel_put(ch);
+		return;
+	}
+}

drivers/gpu/nvgpu/common/fifo/channel_worker.h

@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2017-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef NVGPU_COMMON_FIFO_CHANNEL_WORKER_H
+#define NVGPU_COMMON_FIFO_CHANNEL_WORKER_H
+
+#include <nvgpu/gk20a.h>
+
+void nvgpu_channel_worker_enqueue(struct nvgpu_channel *ch);
+
+static inline struct nvgpu_channel_worker *
+nvgpu_channel_worker_from_worker(struct nvgpu_worker *worker)
+{
+	return (struct nvgpu_channel_worker *)
+	   ((uintptr_t)worker - offsetof(struct nvgpu_channel_worker, worker));
+};
+
+#endif /* NVGPU_COMMON_FIFO_CHANNEL_WORKER_H */

drivers/gpu/nvgpu/common/fifo/engine_status.c

@@ -0,0 +1,88 @@
+/*
+ * Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/io.h>
+#include <nvgpu/engine_status.h>
+
+bool nvgpu_engine_status_is_ctxsw_switch(struct nvgpu_engine_status_info
+		*engine_status)
+{
+	return engine_status->ctxsw_status == NVGPU_CTX_STATUS_CTXSW_SWITCH;
+}
+
+bool nvgpu_engine_status_is_ctxsw_load(struct nvgpu_engine_status_info
+		*engine_status)
+{
+	return engine_status->ctxsw_status == NVGPU_CTX_STATUS_CTXSW_LOAD;
+}
+
+bool nvgpu_engine_status_is_ctxsw_save(struct nvgpu_engine_status_info
+		*engine_status)
+{
+	return	engine_status->ctxsw_status == NVGPU_CTX_STATUS_CTXSW_SAVE;
+}
+
+bool nvgpu_engine_status_is_ctxsw(struct nvgpu_engine_status_info
+		*engine_status)
+{
+	return (nvgpu_engine_status_is_ctxsw_switch(engine_status) ||
+		nvgpu_engine_status_is_ctxsw_load(engine_status) ||
+		nvgpu_engine_status_is_ctxsw_save(engine_status));
+}
+
+bool nvgpu_engine_status_is_ctxsw_invalid(struct nvgpu_engine_status_info
+		*engine_status)
+{
+	return engine_status->ctxsw_status == NVGPU_CTX_STATUS_INVALID;
+}
+
+bool nvgpu_engine_status_is_ctxsw_valid(struct nvgpu_engine_status_info
+		*engine_status)
+{
+	return engine_status->ctxsw_status == NVGPU_CTX_STATUS_VALID;
+}
+bool nvgpu_engine_status_is_ctx_type_tsg(struct nvgpu_engine_status_info
+		*engine_status)
+{
+	return engine_status->ctx_id_type == ENGINE_STATUS_CTX_ID_TYPE_TSGID;
+}
+bool nvgpu_engine_status_is_next_ctx_type_tsg(struct nvgpu_engine_status_info
+		*engine_status)
+{
+	return engine_status->ctx_next_id_type ==
+		ENGINE_STATUS_CTX_NEXT_ID_TYPE_TSGID;
+}
+
+void nvgpu_engine_status_get_ctx_id_type(struct nvgpu_engine_status_info
+		*engine_status, u32 *ctx_id, u32 *ctx_type)
+{
+	*ctx_id = engine_status->ctx_id;
+	*ctx_type = engine_status->ctx_id_type;
+}
+
+void nvgpu_engine_status_get_next_ctx_id_type(struct nvgpu_engine_status_info
+		*engine_status, u32 *ctx_next_id,
+		u32 *ctx_next_type)
+{
+	*ctx_next_id = engine_status->ctx_next_id;
+	*ctx_next_type = engine_status->ctx_next_id_type;
+}

drivers/gpu/nvgpu/common/fifo/engines.c

@@ -0,0 +1,960 @@
+/*
+ * Copyright (c) 2019-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+
+#include <nvgpu/log.h>
+#include <nvgpu/errno.h>
+#include <nvgpu/timers.h>
+#include <nvgpu/bitops.h>
+#ifdef CONFIG_NVGPU_LS_PMU
+#include <nvgpu/pmu.h>
+#include <nvgpu/pmu/mutex.h>
+#endif
+#include <nvgpu/runlist.h>
+#include <nvgpu/engines.h>
+#include <nvgpu/engine_status.h>
+#include <nvgpu/gk20a.h>
+#include <nvgpu/pbdma_status.h>
+#include <nvgpu/power_features/pg.h>
+#include <nvgpu/channel.h>
+#include <nvgpu/soc.h>
+#include <nvgpu/device.h>
+#include <nvgpu/gr/gr_falcon.h>
+#include <nvgpu/gr/gr.h>
+#include <nvgpu/gr/gr_instances.h>
+#include <nvgpu/fifo.h>
+#include <nvgpu/static_analysis.h>
+#include <nvgpu/swprofile.h>
+
+#include <nvgpu/fifo/swprofile.h>
+
+#define FECS_METHOD_WFI_RESTORE	0x80000U
+
+enum nvgpu_fifo_engine nvgpu_engine_enum_from_dev(struct gk20a *g,
+			const struct nvgpu_device *dev)
+{
+	enum nvgpu_fifo_engine ret = NVGPU_ENGINE_INVAL;
+
+	if (nvgpu_device_is_graphics(g, dev)) {
+		ret = NVGPU_ENGINE_GR;
+	} else if (nvgpu_device_is_ce(g, dev)) {
+		/* For now, all CE engines have separate runlists. We can
+		 * identify the NVGPU_ENGINE_GRCE type CE using runlist_id
+		 * comparsion logic with GR runlist_id in init_info()
+		 */
+		ret = NVGPU_ENGINE_ASYNC_CE;
+	} else {
+		ret = NVGPU_ENGINE_INVAL;
+	}
+
+	return ret;
+}
+
+const struct nvgpu_device *nvgpu_engine_get_active_eng_info(
+	struct gk20a *g, u32 engine_id)
+{
+	struct nvgpu_fifo *f = &g->fifo;
+
+	if (engine_id >= f->max_engines) {
+		return NULL;
+	}
+
+	return f->host_engines[engine_id];
+}
+
+bool nvgpu_engine_check_valid_id(struct gk20a *g, u32 engine_id)
+{
+	struct nvgpu_fifo *f = &g->fifo;
+
+	if (engine_id >= f->max_engines) {
+		return false;
+	}
+
+	return f->host_engines[engine_id] != NULL;
+}
+
+u32 nvgpu_engine_get_gr_id_for_inst(struct gk20a *g, u32 inst_id)
+{
+	const struct nvgpu_device *dev;
+
+	dev = nvgpu_device_get(g, NVGPU_DEVTYPE_GRAPHICS, inst_id);
+	if (dev == NULL) {
+		nvgpu_warn(g, "No GR devices on this GPU for inst[%u]?!",
+			inst_id);
+		return NVGPU_INVALID_ENG_ID;
+	}
+
+	return dev->engine_id;
+}
+
+u32 nvgpu_engine_get_gr_id(struct gk20a *g)
+{
+	/* Consider 1st available GR engine */
+	return nvgpu_engine_get_gr_id_for_inst(g, 0U);
+}
+
+u32 nvgpu_engine_act_interrupt_mask(struct gk20a *g, u32 engine_id)
+{
+	const struct nvgpu_device *dev = NULL;
+
+	dev = nvgpu_engine_get_active_eng_info(g, engine_id);
+	if (dev == NULL) {
+		return 0;
+	}
+
+	return BIT32(dev->intr_id);
+}
+
+u32 nvgpu_gr_engine_interrupt_mask(struct gk20a *g)
+{
+	const struct nvgpu_device *dev;
+	u32 intr_mask = 0U;
+	u32 i;
+
+	for (i = 0U; i < g->num_gr_instances; i++) {
+		dev = nvgpu_device_get(g, NVGPU_DEVTYPE_GRAPHICS,
+				nvgpu_gr_get_syspipe_id(g, i));
+		if (dev == NULL) {
+			continue;
+		}
+
+		intr_mask |= BIT32(dev->intr_id);
+	}
+
+	return intr_mask;
+}
+
+u32 nvgpu_ce_engine_interrupt_mask(struct gk20a *g)
+{
+	const struct nvgpu_device *dev;
+	u32 i;
+	u32 mask = 0U;
+
+	/*
+	 * For old chips - pre-Pascal - we have COPY[0-2], for new chips we
+	 * have some number of LCE instances. For the purpose of this code we
+	 * imagine a system that could have both; in reality that'll never be
+	 * the case.
+	 *
+	 * This can be cleaned up in the future by defining a SW type for CE and
+	 * hiding this ugliness in the device management code.
+	 */
+	for (i = NVGPU_DEVTYPE_COPY0;  i <= NVGPU_DEVTYPE_COPY2; i++) {
+		dev = nvgpu_device_get(g, i, i - NVGPU_DEVTYPE_COPY0);
+		if (dev == NULL) {
+			continue;
+		}
+
+		mask |= BIT32(dev->intr_id);
+	}
+
+	/*
+	 * Now take care of LCEs.
+	 */
+	for (i = 0U; i < nvgpu_device_count(g, NVGPU_DEVTYPE_LCE); i++) {
+		dev = nvgpu_device_get(g, NVGPU_DEVTYPE_LCE, i);
+		nvgpu_assert(dev != NULL);
+
+		mask |= BIT32(dev->intr_id);
+	}
+
+	return mask;
+}
+
+#ifdef CONFIG_NVGPU_FIFO_ENGINE_ACTIVITY
+
+static void nvgpu_engine_enable_activity(struct gk20a *g,
+					 const struct nvgpu_device *dev)
+{
+	nvgpu_runlist_set_state(g, BIT32(dev->runlist_id), RUNLIST_ENABLED);
+}
+
+void nvgpu_engine_enable_activity_all(struct gk20a *g)
+{
+	u32 i;
+
+	for (i = 0; i < g->fifo.num_engines; i++) {
+		nvgpu_engine_enable_activity(g, g->fifo.active_engines[i]);
+	}
+}
+
+int nvgpu_engine_disable_activity(struct gk20a *g,
+				const struct nvgpu_device *dev,
+				bool wait_for_idle)
+{
+	u32 pbdma_chid = NVGPU_INVALID_CHANNEL_ID;
+	u32 engine_chid = NVGPU_INVALID_CHANNEL_ID;
+#ifdef CONFIG_NVGPU_LS_PMU
+	u32 token = PMU_INVALID_MUTEX_OWNER_ID;
+	int mutex_ret = -EINVAL;
+#endif
+	int err = 0;
+	struct nvgpu_channel *ch = NULL;
+	struct nvgpu_engine_status_info engine_status;
+	struct nvgpu_pbdma_status_info pbdma_status;
+	unsigned long runlist_served_pbdmas;
+	unsigned long bit;
+	u32 pbdma_id;
+	struct nvgpu_fifo *f = &g->fifo;
+
+	nvgpu_log_fn(g, " ");
+
+	g->ops.engine_status.read_engine_status_info(g, dev->engine_id,
+		 &engine_status);
+	if (engine_status.is_busy && !wait_for_idle) {
+		return -EBUSY;
+	}
+
+#ifdef CONFIG_NVGPU_LS_PMU
+	if (g->ops.pmu.is_pmu_supported(g)) {
+		mutex_ret = nvgpu_pmu_lock_acquire(g, g->pmu,
+						PMU_MUTEX_ID_FIFO, &token);
+	}
+#endif
+
+	nvgpu_runlist_set_state(g, BIT32(dev->runlist_id),
+			RUNLIST_DISABLED);
+
+	runlist_served_pbdmas = f->runlists[dev->runlist_id]->pbdma_bitmask;
+
+	for_each_set_bit(bit, &runlist_served_pbdmas,
+			 nvgpu_get_litter_value(g, GPU_LIT_HOST_NUM_PBDMA)) {
+		pbdma_id = U32(bit);
+		/* chid from pbdma status */
+		g->ops.pbdma_status.read_pbdma_status_info(g,
+						pbdma_id,
+						&pbdma_status);
+		if (nvgpu_pbdma_status_is_chsw_valid(&pbdma_status) ||
+			nvgpu_pbdma_status_is_chsw_save(&pbdma_status)) {
+			pbdma_chid = pbdma_status.id;
+		} else if (nvgpu_pbdma_status_is_chsw_load(&pbdma_status) ||
+			nvgpu_pbdma_status_is_chsw_switch(&pbdma_status)) {
+			pbdma_chid = pbdma_status.next_id;
+		} else {
+			/* Nothing to do here */
+		}
+
+		if (pbdma_chid != NVGPU_INVALID_CHANNEL_ID) {
+			ch = nvgpu_channel_from_id(g, pbdma_chid);
+			if (ch != NULL) {
+				err = g->ops.fifo.preempt_channel(g, ch);
+				nvgpu_channel_put(ch);
+			}
+			if (err != 0) {
+				goto clean_up;
+			}
+		}
+	}
+
+	/* chid from engine status */
+	g->ops.engine_status.read_engine_status_info(g, dev->engine_id,
+		 &engine_status);
+	if (nvgpu_engine_status_is_ctxsw_valid(&engine_status) ||
+	    nvgpu_engine_status_is_ctxsw_save(&engine_status)) {
+		engine_chid = engine_status.ctx_id;
+	} else if (nvgpu_engine_status_is_ctxsw_switch(&engine_status) ||
+	    nvgpu_engine_status_is_ctxsw_load(&engine_status)) {
+		engine_chid = engine_status.ctx_next_id;
+	} else {
+		/* Nothing to do here */
+	}
+
+	if (engine_chid != NVGPU_INVALID_ENG_ID && engine_chid != pbdma_chid) {
+		ch = nvgpu_channel_from_id(g, engine_chid);
+		if (ch != NULL) {
+			err = g->ops.fifo.preempt_channel(g, ch);
+			nvgpu_channel_put(ch);
+		}
+		if (err != 0) {
+			goto clean_up;
+		}
+	}
+
+clean_up:
+#ifdef CONFIG_NVGPU_LS_PMU
+	if (mutex_ret == 0) {
+		if (nvgpu_pmu_lock_release(g, g->pmu,
+			PMU_MUTEX_ID_FIFO, &token) != 0){
+			nvgpu_err(g, "failed to release PMU lock");
+		}
+	}
+#endif
+	if (err != 0) {
+		nvgpu_log_fn(g, "failed");
+		nvgpu_engine_enable_activity(g, dev);
+	} else {
+		nvgpu_log_fn(g, "done");
+	}
+	return err;
+}
+
+int nvgpu_engine_disable_activity_all(struct gk20a *g,
+					   bool wait_for_idle)
+{
+	unsigned int i;
+	int err = 0, ret = 0;
+
+	for (i = 0; i < g->fifo.num_engines; i++) {
+		err = nvgpu_engine_disable_activity(g,
+				g->fifo.active_engines[i],
+				wait_for_idle);
+		if (err != 0) {
+			nvgpu_err(g, "failed to disable engine %d activity",
+				  g->fifo.active_engines[i]->engine_id);
+			ret = err;
+			break;
+		}
+	}
+
+	if (err != 0) {
+		while (i-- != 0U) {
+			nvgpu_engine_enable_activity(g,
+					g->fifo.active_engines[i]);
+		}
+	}
+
+	return ret;
+}
+
+int nvgpu_engine_wait_for_idle(struct gk20a *g)
+{
+	struct nvgpu_timeout timeout;
+	u32 delay = POLL_DELAY_MIN_US;
+	int ret = 0, err = 0;
+	u32 i, host_num_engines;
+	struct nvgpu_engine_status_info engine_status;
+
+	nvgpu_log_fn(g, " ");
+
+	host_num_engines =
+		 nvgpu_get_litter_value(g, GPU_LIT_HOST_NUM_ENGINES);
+
+	err = nvgpu_timeout_init(g, &timeout, nvgpu_get_poll_timeout(g),
+			   NVGPU_TIMER_CPU_TIMER);
+	if (err != 0) {
+		return -EINVAL;
+	}
+
+	for (i = 0; i < host_num_engines; i++) {
+		if (!nvgpu_engine_check_valid_id(g, i)) {
+			continue;
+		}
+
+		ret = -ETIMEDOUT;
+		do {
+			g->ops.engine_status.read_engine_status_info(g, i,
+				&engine_status);
+			if (!engine_status.is_busy) {
+				ret = 0;
+				break;
+			}
+
+			nvgpu_usleep_range(delay, delay * 2U);
+			delay = min_t(u32,
+					delay << 1U, POLL_DELAY_MAX_US);
+		} while (nvgpu_timeout_expired(&timeout) == 0);
+
+		if (ret != 0) {
+			/* possible causes:
+			 * check register settings programmed in hal set by
+			 * elcg_init_idle_filters and init_therm_setup_hw
+			 */
+			nvgpu_err(g, "cannot idle engine: %u "
+					"engine_status: 0x%08x", i,
+					engine_status.reg_data);
+			break;
+		}
+	}
+
+	nvgpu_log_fn(g, "done");
+
+	return ret;
+}
+
+#endif /* CONFIG_NVGPU_FIFO_ENGINE_ACTIVITY */
+
+int nvgpu_engine_setup_sw(struct gk20a *g)
+{
+	struct nvgpu_fifo *f = &g->fifo;
+	int err = 0;
+	size_t size;
+
+	f->max_engines = nvgpu_get_litter_value(g, GPU_LIT_HOST_NUM_ENGINES);
+	size = nvgpu_safe_mult_u64(f->max_engines,
+				   sizeof(struct nvgpu_device *));
+
+	/*
+	 * Allocate the two device lists for host devices.
+	 */
+	f->host_engines = nvgpu_kzalloc(g, size);
+	if (f->host_engines == NULL) {
+		nvgpu_err(g, "OOM allocating host engine list");
+		return -ENOMEM;
+	}
+	f->active_engines = nvgpu_kzalloc(g, size);
+	if (f->active_engines == NULL) {
+		nvgpu_err(g, "no mem for active engine list");
+		err = -ENOMEM;
+		goto clean_up_engine_info;
+	}
+
+	err = nvgpu_engine_init_info(f);
+	if (err != 0) {
+		nvgpu_err(g, "init engine info failed");
+		goto clean_up;
+	}
+
+	return 0;
+
+clean_up:
+	nvgpu_kfree(g, f->active_engines);
+	f->active_engines = NULL;
+
+clean_up_engine_info:
+	nvgpu_kfree(g, f->host_engines);
+	f->host_engines = NULL;
+
+	return err;
+}
+
+void nvgpu_engine_cleanup_sw(struct gk20a *g)
+{
+	struct nvgpu_fifo *f = &g->fifo;
+
+	f->num_engines = 0;
+	nvgpu_kfree(g, f->host_engines);
+	f->host_engines = NULL;
+	nvgpu_kfree(g, f->active_engines);
+	f->active_engines = NULL;
+}
+
+#ifdef CONFIG_NVGPU_ENGINE_RESET
+static void nvgpu_engine_gr_reset(struct gk20a *g)
+{
+	struct nvgpu_swprofiler *prof = &g->fifo.eng_reset_profiler;
+	int err = 0;
+
+	nvgpu_swprofile_snapshot(prof, PROF_ENG_RESET_PREAMBLE);
+
+#ifdef CONFIG_NVGPU_POWER_PG
+	if (nvgpu_pg_elpg_disable(g) != 0 ) {
+		nvgpu_err(g, "failed to set disable elpg");
+	}
+#endif
+	nvgpu_swprofile_snapshot(prof, PROF_ENG_RESET_ELPG_DISABLE);
+
+#ifdef CONFIG_NVGPU_FECS_TRACE
+	/*
+	 * Resetting engine will alter read/write index. Need to flush
+	 * circular buffer before re-enabling FECS.
+	 */
+	if (g->ops.gr.fecs_trace.reset != NULL) {
+		if (g->ops.gr.fecs_trace.reset(g) != 0) {
+			nvgpu_warn(g, "failed to reset fecs traces");
+		}
+	}
+#endif
+
+	nvgpu_swprofile_snapshot(prof, PROF_ENG_RESET_FECS_TRACE_RESET);
+
+	/*
+	 * HALT_PIPELINE method and gr reset during recovery is supported
+	 * starting nvgpu-next simulation.
+	 */
+	err = g->ops.gr.falcon.ctrl_ctxsw(g,
+			NVGPU_GR_FALCON_METHOD_HALT_PIPELINE, 0U, NULL);
+	if (err != 0) {
+		nvgpu_err(g, "failed to halt gr pipe");
+	}
+
+	nvgpu_swprofile_snapshot(prof, PROF_ENG_RESET_HALT_PIPELINE);
+
+	/*
+	 * resetting only engine is not
+	 * enough, we do full init sequence
+	 */
+	nvgpu_log(g, gpu_dbg_rec, "resetting gr engine");
+
+	err = nvgpu_gr_reset(g);
+	if (err != 0) {
+		nvgpu_err(g, "failed to reset gr engine");
+	}
+
+#ifdef CONFIG_NVGPU_POWER_PG
+	if (nvgpu_pg_elpg_enable(g) != 0) {
+		nvgpu_err(g, "failed to set enable elpg");
+	}
+	nvgpu_swprofile_snapshot(prof, PROF_ENG_RESET_ELPG_REENABLE);
+#endif
+}
+
+void nvgpu_engine_reset(struct gk20a *g, u32 engine_id)
+{
+	struct nvgpu_swprofiler *prof = &g->fifo.eng_reset_profiler;
+	const struct nvgpu_device *dev;
+	int err = 0;
+	u32 gr_instance_id;
+
+	nvgpu_log_fn(g, " ");
+
+	if (g == NULL) {
+		return;
+	}
+
+	nvgpu_swprofile_begin_sample(prof);
+
+	dev = nvgpu_engine_get_active_eng_info(g, engine_id);
+	if (dev == NULL) {
+		nvgpu_err(g, "unsupported engine_id %d", engine_id);
+		return;
+	}
+
+	if (!nvgpu_device_is_ce(g, dev) &&
+	    !nvgpu_device_is_graphics(g, dev)) {
+		nvgpu_warn(g, "Ignoring reset for non-host engine.");
+		return;
+	}
+
+	/*
+	 * Simple case first: reset a copy engine.
+	 */
+	if (nvgpu_device_is_ce(g, dev)) {
+		err = nvgpu_mc_reset_dev(g, dev);
+		if (err != 0) {
+			nvgpu_log_info(g, "CE engine [id:%u] reset failed",
+				dev->engine_id);
+		}
+		return;
+	}
+
+	/*
+	 * Now reset a GR engine.
+	 */
+	gr_instance_id =
+		nvgpu_grmgr_get_gr_instance_id_for_syspipe(
+			g, dev->inst_id);
+
+	nvgpu_gr_exec_for_instance(g,
+		gr_instance_id, nvgpu_engine_gr_reset(g));
+}
+#endif
+
+u32 nvgpu_engine_get_fast_ce_runlist_id(struct gk20a *g)
+{
+	const struct nvgpu_device *dev;
+	u32 nr_lces;
+	u32 i;
+
+	/*
+	 * Obtain a runlist ID for the fastest available CE. The priority order
+	 * is:
+	 *
+	 *   1. Last available LCE
+	 *   2. Last available COPY[0-2]
+	 *   3. GRAPHICS runlist as a last resort.
+	 */
+	nr_lces = nvgpu_device_count(g, NVGPU_DEVTYPE_LCE);
+	if (nr_lces > 0U) {
+		dev = nvgpu_device_get(g,
+				       NVGPU_DEVTYPE_LCE,
+				       nr_lces - 1U);
+		nvgpu_assert(dev != NULL);
+
+		return dev->runlist_id;
+	}
+
+	/*
+	 * Note: this only works since NVGPU_DEVTYPE_GRAPHICS is 0 and the COPYx
+	 * are all > 0.
+	 */
+	for (i = NVGPU_DEVTYPE_COPY2; i >= NVGPU_DEVTYPE_COPY0; i--) {
+		dev = nvgpu_device_get(g, i, i - NVGPU_DEVTYPE_COPY0);
+		if (dev != NULL) {
+			return dev->runlist_id;
+		}
+	}
+
+	/*
+	 * Fall back to GR.
+	 */
+	dev = nvgpu_device_get(g, NVGPU_DEVTYPE_GRAPHICS, 0);
+	nvgpu_assert(dev != NULL);
+
+	return dev->runlist_id;
+}
+
+u32 nvgpu_engine_get_gr_runlist_id(struct gk20a *g)
+{
+	const struct nvgpu_device *dev;
+
+	dev = nvgpu_device_get(g, NVGPU_DEVTYPE_GRAPHICS, 0);
+	if (dev == NULL) {
+		nvgpu_warn(g, "No GR device on this GPU?!");
+		return NVGPU_INVALID_RUNLIST_ID;
+	}
+
+	return dev->runlist_id;
+}
+
+bool nvgpu_engine_is_valid_runlist_id(struct gk20a *g, u32 runlist_id)
+{
+	u32 i;
+	struct nvgpu_fifo *f = &g->fifo;
+
+	for (i = 0U; i < f->num_engines; i++) {
+		const struct nvgpu_device *dev = f->active_engines[i];
+
+		if (dev->runlist_id == runlist_id) {
+			return true;
+		}
+	}
+
+	return false;
+}
+
+/*
+ * Link engine IDs to MMU IDs and vice versa.
+ */
+u32 nvgpu_engine_id_to_mmu_fault_id(struct gk20a *g, u32 engine_id)
+{
+	const struct nvgpu_device *dev;
+
+	dev = nvgpu_engine_get_active_eng_info(g, engine_id);
+
+	if (dev == NULL) {
+		nvgpu_err(g,
+			  "engine_id: %u is not in active list",
+			  engine_id);
+		return NVGPU_INVALID_ENG_ID;
+	}
+
+	return dev->fault_id;
+}
+
+u32 nvgpu_engine_mmu_fault_id_to_engine_id(struct gk20a *g, u32 fault_id)
+{
+	u32 i;
+	const struct nvgpu_device *dev;
+	struct nvgpu_fifo *f = &g->fifo;
+
+	for (i = 0U; i < f->num_engines; i++) {
+		dev = f->active_engines[i];
+
+		if (dev->fault_id == fault_id) {
+			return dev->engine_id;
+		}
+	}
+
+	return NVGPU_INVALID_ENG_ID;
+}
+
+u32 nvgpu_engine_get_mask_on_id(struct gk20a *g, u32 id, bool is_tsg)
+{
+	unsigned int i;
+	u32 engines = 0;
+	struct nvgpu_engine_status_info engine_status;
+	u32 ctx_id;
+	u32 type;
+	bool busy;
+
+	for (i = 0; i < g->fifo.num_engines; i++) {
+		const struct nvgpu_device *dev = g->fifo.active_engines[i];
+
+		g->ops.engine_status.read_engine_status_info(g,
+			dev->engine_id, &engine_status);
+
+		if (nvgpu_engine_status_is_ctxsw_load(
+			&engine_status)) {
+			nvgpu_engine_status_get_next_ctx_id_type(
+				&engine_status, &ctx_id, &type);
+		} else {
+			nvgpu_engine_status_get_ctx_id_type(
+				&engine_status, &ctx_id, &type);
+		}
+
+		busy = engine_status.is_busy;
+
+		if (!busy || !(ctx_id == id)) {
+			continue;
+		}
+
+		if ((is_tsg  && (type == ENGINE_STATUS_CTX_ID_TYPE_TSGID)) ||
+		    (!is_tsg && (type == ENGINE_STATUS_CTX_ID_TYPE_CHID))) {
+			engines |= BIT32(dev->engine_id);
+		}
+	}
+
+	return engines;
+}
+
+static int nvgpu_engine_init_one_dev(struct nvgpu_fifo *f,
+				     const struct nvgpu_device *dev)
+{
+	bool found;
+	struct nvgpu_device *dev_rw;
+	struct gk20a *g = f->g;
+
+	dev_rw = (struct nvgpu_device *)dev;
+
+	/*
+	 * Populate the PBDMA info for this device; ideally it'd be done
+	 * during device init, but the FIFO unit is not out of reset that
+	 * early in the nvgpu_finalize_poweron() sequence.
+	 *
+	 * We only need to do this for native; vGPU already has pbdma_id
+	 * populated during device initialization.
+	 */
+	if (g->ops.fifo.find_pbdma_for_runlist != NULL) {
+		found = g->ops.fifo.find_pbdma_for_runlist(g,
+							   dev->runlist_id,
+							   &dev_rw->pbdma_id);
+		if (!found) {
+			nvgpu_err(g, "busted pbdma map");
+			return -EINVAL;
+		}
+	}
+
+#if defined(CONFIG_NVGPU_NEXT)
+	{
+		int err = nvgpu_next_engine_init_one_dev(g, dev);
+		if (err != 0) {
+			return err;
+		}
+	}
+#endif
+
+	f->host_engines[dev->engine_id] = dev;
+	f->active_engines[f->num_engines] = dev;
+	++f->num_engines;
+
+	return 0;
+}
+
+int nvgpu_engine_init_info(struct nvgpu_fifo *f)
+{
+	int err;
+	struct gk20a *g = f->g;
+	const struct nvgpu_device *dev;
+
+	f->num_engines = 0;
+
+	nvgpu_log(g, gpu_dbg_device, "Loading host engines from device list");
+	nvgpu_log(g, gpu_dbg_device, "  GFX devices: %u",
+		  nvgpu_device_count(g, NVGPU_DEVTYPE_GRAPHICS));
+
+	nvgpu_device_for_each(g, dev, NVGPU_DEVTYPE_GRAPHICS) {
+		err = nvgpu_engine_init_one_dev(f, dev);
+		if (err != 0) {
+			return err;
+		}
+	}
+
+	return g->ops.engine.init_ce_info(f);
+}
+
+void nvgpu_engine_get_id_and_type(struct gk20a *g, u32 engine_id,
+					  u32 *id, u32 *type)
+{
+	struct nvgpu_engine_status_info engine_status;
+
+	g->ops.engine_status.read_engine_status_info(g, engine_id,
+		&engine_status);
+
+	/* use next_id if context load is failing */
+	if (nvgpu_engine_status_is_ctxsw_load(
+		&engine_status)) {
+		nvgpu_engine_status_get_next_ctx_id_type(
+			&engine_status, id, type);
+	} else {
+		nvgpu_engine_status_get_ctx_id_type(
+			&engine_status, id, type);
+	}
+}
+
+u32 nvgpu_engine_find_busy_doing_ctxsw(struct gk20a *g,
+			u32 *id_ptr, bool *is_tsg_ptr)
+{
+	u32 i;
+	u32 id = U32_MAX;
+	bool is_tsg = false;
+	u32 mailbox2;
+	struct nvgpu_engine_status_info engine_status;
+	const struct nvgpu_device *dev = NULL;
+
+	for (i = 0U; i < g->fifo.num_engines; i++) {
+		dev = g->fifo.active_engines[i];
+
+		g->ops.engine_status.read_engine_status_info(g, dev->engine_id,
+			&engine_status);
+
+		/*
+		 * we are interested in busy engines that
+		 * are doing context switch
+		 */
+		if (!engine_status.is_busy ||
+		    !nvgpu_engine_status_is_ctxsw(&engine_status)) {
+			continue;
+		}
+
+		if (nvgpu_engine_status_is_ctxsw_load(&engine_status)) {
+			id = engine_status.ctx_next_id;
+			is_tsg = nvgpu_engine_status_is_next_ctx_type_tsg(
+					&engine_status);
+		} else if (nvgpu_engine_status_is_ctxsw_switch(&engine_status)) {
+			mailbox2 = g->ops.gr.falcon.read_fecs_ctxsw_mailbox(g,
+					NVGPU_GR_FALCON_FECS_CTXSW_MAILBOX2);
+			if ((mailbox2 & FECS_METHOD_WFI_RESTORE) != 0U) {
+				id = engine_status.ctx_next_id;
+				is_tsg = nvgpu_engine_status_is_next_ctx_type_tsg(
+						&engine_status);
+			} else {
+				id = engine_status.ctx_id;
+				is_tsg = nvgpu_engine_status_is_ctx_type_tsg(
+						&engine_status);
+			}
+		} else {
+			id = engine_status.ctx_id;
+			is_tsg = nvgpu_engine_status_is_ctx_type_tsg(
+					&engine_status);
+		}
+		break;
+	}
+
+	*id_ptr = id;
+	*is_tsg_ptr = is_tsg;
+
+	return dev->engine_id;
+}
+
+u32 nvgpu_engine_get_runlist_busy_engines(struct gk20a *g, u32 runlist_id)
+{
+	struct nvgpu_fifo *f = &g->fifo;
+	u32 i, eng_bitmask = 0U;
+	struct nvgpu_engine_status_info engine_status;
+
+	for (i = 0U; i < f->num_engines; i++) {
+		const struct nvgpu_device *dev = f->active_engines[i];
+
+		g->ops.engine_status.read_engine_status_info(g, dev->engine_id,
+			&engine_status);
+
+		if (engine_status.is_busy && (dev->runlist_id == runlist_id)) {
+			eng_bitmask |= BIT32(dev->engine_id);
+		}
+	}
+
+	return eng_bitmask;
+}
+
+#ifdef CONFIG_NVGPU_DEBUGGER
+bool nvgpu_engine_should_defer_reset(struct gk20a *g, u32 engine_id,
+		u32 engine_subid, bool fake_fault)
+{
+	const struct nvgpu_device *dev;
+
+	dev = nvgpu_engine_get_active_eng_info(g, engine_id);
+	if (dev == NULL) {
+		return false;
+	}
+
+	/*
+	 * channel recovery is only deferred if an sm debugger
+	 * is attached and has MMU debug mode is enabled
+	 */
+	if (!g->ops.gr.sm_debugger_attached(g) ||
+	    !g->ops.fb.is_debug_mode_enabled(g)) {
+		return false;
+	}
+
+	/* if this fault is fake (due to RC recovery), don't defer recovery */
+	if (fake_fault) {
+		return false;
+	}
+
+	if (dev->type != NVGPU_DEVTYPE_GRAPHICS) {
+		return false;
+	}
+
+	return g->ops.engine.is_fault_engine_subid_gpc(g, engine_subid);
+}
+#endif
+
+u32 nvgpu_engine_mmu_fault_id_to_veid(struct gk20a *g, u32 mmu_fault_id,
+			u32 gr_eng_fault_id)
+{
+	struct nvgpu_fifo *f = &g->fifo;
+	u32 num_subctx;
+	u32 veid = INVAL_ID;
+
+	num_subctx = f->max_subctx_count;
+
+	if ((mmu_fault_id >= gr_eng_fault_id) &&
+		(mmu_fault_id < nvgpu_safe_add_u32(gr_eng_fault_id,
+						num_subctx))) {
+		veid = mmu_fault_id - gr_eng_fault_id;
+	}
+
+	return veid;
+}
+
+static u32 nvgpu_engine_mmu_fault_id_to_eng_id_and_veid(struct gk20a *g,
+			 u32 mmu_fault_id, u32 *veid)
+{
+	u32 i;
+	u32 engine_id = INVAL_ID;
+	const struct nvgpu_device *dev;
+	struct nvgpu_fifo *f = &g->fifo;
+
+	for (i = 0U; i < f->num_engines; i++) {
+		dev = f->active_engines[i];
+
+		if (dev->type == NVGPU_DEVTYPE_GRAPHICS) {
+			*veid = nvgpu_engine_mmu_fault_id_to_veid(g,
+					mmu_fault_id, dev->fault_id);
+			if (*veid != INVAL_ID) {
+				engine_id = dev->engine_id;
+				break;
+			}
+		} else {
+			if (dev->fault_id == mmu_fault_id) {
+				engine_id = dev->engine_id;
+				*veid = INVAL_ID;
+				break;
+			}
+		}
+	}
+	return engine_id;
+}
+
+void nvgpu_engine_mmu_fault_id_to_eng_ve_pbdma_id(struct gk20a *g,
+	u32 mmu_fault_id, u32 *engine_id, u32 *veid, u32 *pbdma_id)
+{
+	*engine_id = nvgpu_engine_mmu_fault_id_to_eng_id_and_veid(g,
+				 mmu_fault_id, veid);
+
+	if (*engine_id == INVAL_ID) {
+		*pbdma_id = g->ops.fifo.mmu_fault_id_to_pbdma_id(g,
+				mmu_fault_id);
+	} else {
+		*pbdma_id = INVAL_ID;
+	}
+}

drivers/gpu/nvgpu/common/fifo/fifo.c

@@ -0,0 +1,315 @@
+/*
+ * FIFO
+ *
+ * Copyright (c) 2011-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/trace.h>
+#include <nvgpu/dma.h>
+#include <nvgpu/fifo.h>
+#include <nvgpu/engines.h>
+#include <nvgpu/runlist.h>
+#include <nvgpu/preempt.h>
+#include <nvgpu/gk20a.h>
+#include <nvgpu/channel.h>
+#include <nvgpu/pbdma.h>
+#include <nvgpu/tsg.h>
+#include <nvgpu/vm_area.h>
+#include <nvgpu/nvgpu_err.h>
+#include <nvgpu/cic.h>
+#include <nvgpu/mc.h>
+#include <nvgpu/swprofile.h>
+#include <nvgpu/fifo/swprofile.h>
+
+static const char *nvgpu_fifo_kickoff_profile_events[] = {
+	NVGPU_FIFO_KICKOFF_PROFILE_EVENTS,
+};
+
+static const char *nvgpu_fifo_recovery_profile_events[] = {
+	NVGPU_FIFO_RECOVERY_PROFILE_EVENTS,
+};
+
+static const char *nvgpu_fifo_engine_reset_events[] = {
+	NVGPU_FIFO_ENGINE_RESET_EVENTS,
+};
+
+void nvgpu_fifo_cleanup_sw_common(struct gk20a *g)
+{
+	struct nvgpu_fifo *f = &g->fifo;
+
+	nvgpu_log_fn(g, " ");
+
+#ifdef CONFIG_NVGPU_USERD
+	g->ops.userd.cleanup_sw(g);
+#endif
+	nvgpu_channel_cleanup_sw(g);
+	nvgpu_tsg_cleanup_sw(g);
+	nvgpu_runlist_cleanup_sw(g);
+	nvgpu_engine_cleanup_sw(g);
+	if (g->ops.pbdma.cleanup_sw != NULL) {
+		g->ops.pbdma.cleanup_sw(g);
+	}
+
+#ifdef CONFIG_NVGPU_DEBUGGER
+	f->deferred_reset_pending = false;
+	nvgpu_mutex_destroy(&f->deferred_reset_mutex);
+#endif
+	nvgpu_mutex_destroy(&f->engines_reset_mutex);
+	nvgpu_mutex_destroy(&f->intr.isr.mutex);
+
+	f->sw_ready = false;
+}
+
+void nvgpu_fifo_cleanup_sw(struct gk20a *g)
+{
+#ifdef CONFIG_NVGPU_KERNEL_MODE_SUBMIT
+	nvgpu_channel_worker_deinit(g);
+#endif
+	nvgpu_fifo_cleanup_sw_common(g);
+}
+
+static void nvgpu_fifo_remove_support(struct nvgpu_fifo *f)
+{
+	struct gk20a *g = f->g;
+
+	g->ops.fifo.cleanup_sw(g);
+}
+
+int nvgpu_fifo_setup_sw_common(struct gk20a *g)
+{
+	struct nvgpu_fifo *f = &g->fifo;
+	int err = 0;
+
+	nvgpu_log_fn(g, " ");
+
+	f->g = g;
+
+	nvgpu_mutex_init(&f->intr.isr.mutex);
+	nvgpu_mutex_init(&f->engines_reset_mutex);
+#ifdef CONFIG_NVGPU_DEBUGGER
+	nvgpu_mutex_init(&f->deferred_reset_mutex);
+#endif
+
+	nvgpu_swprofile_initialize(g, &f->kickoff_profiler,
+				 nvgpu_fifo_kickoff_profile_events);
+	nvgpu_swprofile_initialize(g, &f->recovery_profiler,
+				 nvgpu_fifo_recovery_profile_events);
+	nvgpu_swprofile_initialize(g, &f->eng_reset_profiler,
+				 nvgpu_fifo_engine_reset_events);
+
+
+	err = nvgpu_channel_setup_sw(g);
+	if (err != 0) {
+		nvgpu_err(g, "failed to init channel support");
+		goto clean_up;
+	}
+
+	err = nvgpu_tsg_setup_sw(g);
+	if (err != 0) {
+		nvgpu_err(g, "failed to init tsg support");
+		goto clean_up_channel;
+	}
+
+	if (g->ops.pbdma.setup_sw != NULL) {
+		err = g->ops.pbdma.setup_sw(g);
+		if (err != 0) {
+			nvgpu_err(g, "failed to init pbdma support");
+			goto clean_up_tsg;
+		}
+	}
+
+	err = nvgpu_engine_setup_sw(g);
+	if (err != 0) {
+		nvgpu_err(g, "failed to init engine support");
+		goto clean_up_pbdma;
+	}
+
+	err = nvgpu_runlist_setup_sw(g);
+	if (err != 0) {
+		nvgpu_err(g, "failed to init runlist support");
+		goto clean_up_engine;
+	}
+
+#ifdef CONFIG_NVGPU_USERD
+	err = g->ops.userd.setup_sw(g);
+	if (err != 0) {
+		nvgpu_err(g, "failed to init userd support");
+		goto clean_up_runlist;
+	}
+#endif
+
+	f->remove_support = nvgpu_fifo_remove_support;
+
+	nvgpu_log_fn(g, "done");
+	return 0;
+
+#ifdef CONFIG_NVGPU_USERD
+clean_up_runlist:
+	nvgpu_runlist_cleanup_sw(g);
+#endif
+
+clean_up_engine:
+	nvgpu_engine_cleanup_sw(g);
+
+clean_up_pbdma:
+	if (g->ops.pbdma.cleanup_sw != NULL) {
+		g->ops.pbdma.cleanup_sw(g);
+	}
+
+clean_up_tsg:
+	nvgpu_tsg_cleanup_sw(g);
+
+clean_up_channel:
+	nvgpu_channel_cleanup_sw(g);
+
+clean_up:
+	nvgpu_err(g, "init fifo support failed");
+	return err;
+}
+
+int nvgpu_fifo_setup_sw(struct gk20a *g)
+{
+	struct nvgpu_fifo *f = &g->fifo;
+	int err = 0;
+
+	nvgpu_log_fn(g, " ");
+
+	if (f->sw_ready) {
+		nvgpu_log_fn(g, "skip init");
+		return 0;
+	}
+
+	err = nvgpu_fifo_setup_sw_common(g);
+	if (err != 0) {
+		nvgpu_err(g, "fifo common sw setup failed, err=%d", err);
+		return err;
+	}
+
+#ifdef CONFIG_NVGPU_KERNEL_MODE_SUBMIT
+	err = nvgpu_channel_worker_init(g);
+	if (err != 0) {
+		nvgpu_err(g, "worker init fail, err=%d", err);
+		goto clean_up;
+	}
+#endif
+
+	f->sw_ready = true;
+
+	nvgpu_log_fn(g, "done");
+	return 0;
+
+#ifdef CONFIG_NVGPU_KERNEL_MODE_SUBMIT
+clean_up:
+	nvgpu_fifo_cleanup_sw_common(g);
+
+	return err;
+#endif
+}
+
+int nvgpu_fifo_init_support(struct gk20a *g)
+{
+	int err;
+
+	err = g->ops.fifo.setup_sw(g);
+	if (err != 0) {
+		nvgpu_err(g, "fifo sw setup failed, err=%d", err);
+		return err;
+	}
+
+	if (g->ops.fifo.init_fifo_setup_hw != NULL) {
+		err = g->ops.fifo.init_fifo_setup_hw(g);
+		if (err != 0) {
+			nvgpu_err(g, "fifo hw setup failed, err=%d", err);
+			goto clean_up;
+		}
+	}
+
+	return 0;
+
+clean_up:
+	nvgpu_fifo_cleanup_sw_common(g);
+
+	return err;
+}
+
+static const char * const pbdma_ch_eng_status_str[] = {
+	"invalid",
+	"valid",
+	"NA",
+	"NA",
+	"NA",
+	"load",
+	"save",
+	"switch",
+};
+
+static const char * const not_found_str[] = {
+	"NOT FOUND"
+};
+
+const char *nvgpu_fifo_decode_pbdma_ch_eng_status(u32 index)
+{
+	if (index >= ARRAY_SIZE(pbdma_ch_eng_status_str)) {
+		return not_found_str[0];
+	} else {
+		return pbdma_ch_eng_status_str[index];
+	}
+}
+
+static void disable_fifo_interrupts(struct gk20a *g)
+{
+	/** Disable fifo intr */
+	g->ops.fifo.intr_0_enable(g, false);
+	g->ops.fifo.intr_1_enable(g, false);
+
+	if (g->ops.fifo.intr_top_enable == NULL) {
+		nvgpu_cic_intr_stall_unit_config(g, NVGPU_CIC_INTR_UNIT_FIFO,
+					NVGPU_CIC_INTR_DISABLE);
+		nvgpu_cic_intr_nonstall_unit_config(g, NVGPU_CIC_INTR_UNIT_FIFO,
+					   NVGPU_CIC_INTR_DISABLE);
+	} else {
+		g->ops.fifo.intr_top_enable(g, NVGPU_CIC_INTR_DISABLE);
+	}
+}
+
+int nvgpu_fifo_suspend(struct gk20a *g)
+{
+	nvgpu_log_fn(g, " ");
+
+	if (g->ops.mm.is_bar1_supported(g)) {
+		g->ops.fifo.bar1_snooping_disable(g);
+	}
+
+	disable_fifo_interrupts(g);
+
+	nvgpu_log_fn(g, "done");
+	return 0;
+}
+
+void nvgpu_fifo_sw_quiesce(struct gk20a *g)
+{
+	u32 runlist_mask = U32_MAX;
+
+	g->ops.runlist.write_state(g, runlist_mask, RUNLIST_DISABLED);
+
+	/* Preempt all runlists */
+	nvgpu_fifo_preempt_runlists_for_rc(g, runlist_mask);
+}

drivers/gpu/nvgpu/common/fifo/job.c

@@ -0,0 +1,149 @@
+/*
+ * Copyright (c) 2011-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/log.h>
+#include <nvgpu/lock.h>
+#include <nvgpu/kmem.h>
+#include <nvgpu/barrier.h>
+#include <nvgpu/circ_buf.h>
+#include <nvgpu/channel.h>
+#include <nvgpu/job.h>
+#include <nvgpu/priv_cmdbuf.h>
+#include <nvgpu/fence.h>
+
+static inline struct nvgpu_channel_job *
+channel_gk20a_job_from_list(struct nvgpu_list_node *node)
+{
+	return (struct nvgpu_channel_job *)
+	((uintptr_t)node - offsetof(struct nvgpu_channel_job, list));
+};
+
+int nvgpu_channel_alloc_job(struct nvgpu_channel *c,
+		struct nvgpu_channel_job **job_out)
+{
+	unsigned int put = c->joblist.pre_alloc.put;
+	unsigned int get = c->joblist.pre_alloc.get;
+	unsigned int next = (put + 1) % c->joblist.pre_alloc.length;
+	bool full = next == get;
+
+	if (full) {
+		return -EAGAIN;
+	}
+
+	*job_out = &c->joblist.pre_alloc.jobs[put];
+	(void) memset(*job_out, 0, sizeof(**job_out));
+
+	return 0;
+}
+
+void nvgpu_channel_free_job(struct nvgpu_channel *c,
+		struct nvgpu_channel_job *job)
+{
+	/*
+	 * Nothing needed for now. The job contents are preallocated. The
+	 * completion fence may briefly outlive the job, but the job memory is
+	 * reclaimed only when a new submit comes in and the ringbuffer has ran
+	 * out of space.
+	 */
+}
+
+void nvgpu_channel_joblist_lock(struct nvgpu_channel *c)
+{
+	nvgpu_mutex_acquire(&c->joblist.pre_alloc.read_lock);
+}
+
+void nvgpu_channel_joblist_unlock(struct nvgpu_channel *c)
+{
+	nvgpu_mutex_release(&c->joblist.pre_alloc.read_lock);
+}
+
+struct nvgpu_channel_job *nvgpu_channel_joblist_peek(struct nvgpu_channel *c)
+{
+	unsigned int get = c->joblist.pre_alloc.get;
+	unsigned int put = c->joblist.pre_alloc.put;
+	bool empty = get == put;
+
+	return empty ? NULL : &c->joblist.pre_alloc.jobs[get];
+}
+
+void nvgpu_channel_joblist_add(struct nvgpu_channel *c,
+		struct nvgpu_channel_job *job)
+{
+	c->joblist.pre_alloc.put = (c->joblist.pre_alloc.put + 1U) %
+			(c->joblist.pre_alloc.length);
+}
+
+void nvgpu_channel_joblist_delete(struct nvgpu_channel *c,
+		struct nvgpu_channel_job *job)
+{
+	c->joblist.pre_alloc.get = (c->joblist.pre_alloc.get + 1U) %
+			(c->joblist.pre_alloc.length);
+}
+
+int nvgpu_channel_joblist_init(struct nvgpu_channel *c, u32 num_jobs)
+{
+	int err;
+	u32 size;
+
+	size = (u32)sizeof(struct nvgpu_channel_job);
+	if (num_jobs > nvgpu_safe_sub_u32(U32_MAX / size, 1U)) {
+		err = -ERANGE;
+		goto clean_up;
+	}
+
+	/*
+	 * The max capacity of this ring buffer is the alloc size minus one (in
+	 * units of item slot), so allocate a size of (num_jobs + 1) * size
+	 * bytes.
+	 */
+	c->joblist.pre_alloc.jobs = nvgpu_vzalloc(c->g,
+			nvgpu_safe_mult_u32(
+				nvgpu_safe_add_u32(num_jobs, 1U),
+				size));
+	if (c->joblist.pre_alloc.jobs == NULL) {
+		err = -ENOMEM;
+		goto clean_up;
+	}
+
+	/*
+	 * length is the allocation size of the ringbuffer; the number of jobs
+	 * that fit is one less.
+	 */
+	c->joblist.pre_alloc.length = nvgpu_safe_add_u32(num_jobs, 1U);
+	c->joblist.pre_alloc.put = 0;
+	c->joblist.pre_alloc.get = 0;
+
+	return 0;
+
+clean_up:
+	nvgpu_vfree(c->g, c->joblist.pre_alloc.jobs);
+	(void) memset(&c->joblist.pre_alloc, 0, sizeof(c->joblist.pre_alloc));
+	return err;
+}
+
+void nvgpu_channel_joblist_deinit(struct nvgpu_channel *c)
+{
+	if (c->joblist.pre_alloc.jobs != NULL) {
+		nvgpu_vfree(c->g, c->joblist.pre_alloc.jobs);
+		c->joblist.pre_alloc.jobs = NULL;
+	}
+}

drivers/gpu/nvgpu/common/fifo/pbdma.c

@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2011-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/gk20a.h>
+#include <nvgpu/pbdma.h>
+
+static void nvgpu_pbdma_init_intr_descs(struct gk20a *g)
+{
+	struct nvgpu_fifo *f = &g->fifo;
+
+	if (g->ops.pbdma.device_fatal_0_intr_descs != NULL) {
+		f->intr.pbdma.device_fatal_0 =
+			g->ops.pbdma.device_fatal_0_intr_descs();
+	}
+
+	if (g->ops.pbdma.channel_fatal_0_intr_descs != NULL) {
+		f->intr.pbdma.channel_fatal_0 =
+			g->ops.pbdma.channel_fatal_0_intr_descs();
+	}
+	if (g->ops.pbdma.restartable_0_intr_descs != NULL) {
+		f->intr.pbdma.restartable_0 =
+			g->ops.pbdma.restartable_0_intr_descs();
+	}
+}
+
+int nvgpu_pbdma_setup_sw(struct gk20a *g)
+{
+	nvgpu_pbdma_init_intr_descs(g);
+
+	return 0;
+}
+
+void nvgpu_pbdma_cleanup_sw(struct gk20a *g)
+{
+	return;
+}

drivers/gpu/nvgpu/common/fifo/pbdma_status.c

@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/pbdma_status.h>
+
+bool nvgpu_pbdma_status_is_chsw_switch(struct nvgpu_pbdma_status_info
+		*pbdma_status)
+{
+	return pbdma_status->chsw_status == NVGPU_PBDMA_CHSW_STATUS_SWITCH;
+}
+bool nvgpu_pbdma_status_is_chsw_load(struct nvgpu_pbdma_status_info
+		*pbdma_status)
+{
+	return pbdma_status->chsw_status == NVGPU_PBDMA_CHSW_STATUS_LOAD;
+}
+bool nvgpu_pbdma_status_is_chsw_save(struct nvgpu_pbdma_status_info
+		*pbdma_status)
+{
+	return pbdma_status->chsw_status == NVGPU_PBDMA_CHSW_STATUS_SAVE;
+}
+bool nvgpu_pbdma_status_is_chsw_valid(struct nvgpu_pbdma_status_info
+		*pbdma_status)
+{
+	return pbdma_status->chsw_status == NVGPU_PBDMA_CHSW_STATUS_VALID;
+}
+bool nvgpu_pbdma_status_is_id_type_tsg(struct nvgpu_pbdma_status_info
+		*pbdma_status)
+{
+	return pbdma_status->id_type == PBDMA_STATUS_ID_TYPE_TSGID;
+}
+bool nvgpu_pbdma_status_is_next_id_type_tsg(struct nvgpu_pbdma_status_info
+		*pbdma_status)
+{
+	return pbdma_status->next_id_type == PBDMA_STATUS_NEXT_ID_TYPE_TSGID;
+}

drivers/gpu/nvgpu/common/fifo/preempt.c

@@ -0,0 +1,220 @@
+/*
+ * Copyright (c) 2011-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/soc.h>
+#include <nvgpu/gk20a.h>
+#include <nvgpu/errata.h>
+#include <nvgpu/runlist.h>
+#include <nvgpu/types.h>
+#include <nvgpu/channel.h>
+#include <nvgpu/tsg.h>
+#include <nvgpu/preempt.h>
+#include <nvgpu/nvgpu_err.h>
+#include <nvgpu/cic.h>
+#include <nvgpu/rc.h>
+#ifdef CONFIG_NVGPU_LS_PMU
+#include <nvgpu/pmu/mutex.h>
+#endif
+
+u32 nvgpu_preempt_get_timeout(struct gk20a *g)
+{
+	return g->ctxsw_timeout_period_ms;
+}
+
+int nvgpu_fifo_preempt_tsg(struct gk20a *g, struct nvgpu_tsg *tsg)
+{
+	int ret = 0;
+	u32 preempt_retry_count = 10U;
+	u32 preempt_retry_timeout =
+			nvgpu_preempt_get_timeout(g) / preempt_retry_count;
+#ifdef CONFIG_NVGPU_LS_PMU
+	u32 token = PMU_INVALID_MUTEX_OWNER_ID;
+	int mutex_ret = 0;
+#endif
+
+	nvgpu_log_fn(g, "tsgid: %d", tsg->tsgid);
+
+	if (tsg->runlist == NULL) {
+		return 0;
+	}
+
+	do {
+		nvgpu_mutex_acquire(&tsg->runlist->runlist_lock);
+
+		if (nvgpu_is_errata_present(g, NVGPU_ERRATA_2016608)) {
+			nvgpu_runlist_set_state(g, BIT32(tsg->runlist->id),
+						RUNLIST_DISABLED);
+		}
+
+#ifdef CONFIG_NVGPU_LS_PMU
+		mutex_ret = nvgpu_pmu_lock_acquire(g, g->pmu,
+						   PMU_MUTEX_ID_FIFO, &token);
+#endif
+		g->ops.fifo.preempt_trigger(g, tsg->tsgid, ID_TYPE_TSG);
+
+		/*
+		 * Poll for preempt done. if stalling interrupts are pending
+		 * while preempt is in progress we poll for stalling interrupts
+		 * to finish based on return value from this function and
+		 * retry preempt again.
+		 * If HW is hung, on the last retry instance we try to identify
+		 * the engines hung and set the runlist reset_eng_bitmask
+		 * and mark preemption completion.
+		 */
+		ret = g->ops.fifo.is_preempt_pending(g, tsg->tsgid,
+					ID_TYPE_TSG, preempt_retry_count > 1U);
+
+#ifdef CONFIG_NVGPU_LS_PMU
+		if (mutex_ret == 0) {
+			int err = nvgpu_pmu_lock_release(g, g->pmu,
+						PMU_MUTEX_ID_FIFO, &token);
+			if (err != 0) {
+				nvgpu_err(g, "PMU_MUTEX_ID_FIFO not released err=%d", err);
+			}
+		}
+#endif
+		if (nvgpu_is_errata_present(g, NVGPU_ERRATA_2016608)) {
+			nvgpu_runlist_set_state(g, BIT32(tsg->runlist->id),
+						RUNLIST_ENABLED);
+		}
+
+		nvgpu_mutex_release(&tsg->runlist->runlist_lock);
+
+		if (ret != -EAGAIN) {
+			break;
+		}
+
+		ret = nvgpu_cic_wait_for_stall_interrupts(g, preempt_retry_timeout);
+		if (ret != 0) {
+			nvgpu_log_info(g, "wait for stall interrupts failed %d", ret);
+		}
+	} while (--preempt_retry_count != 0U);
+
+	if (ret != 0) {
+		if (nvgpu_platform_is_silicon(g)) {
+			nvgpu_err(g, "preempt timed out for tsgid: %u, "
+			"ctxsw timeout will trigger recovery if needed",
+			tsg->tsgid);
+		} else {
+			nvgpu_rc_preempt_timeout(g, tsg);
+		}
+	}
+	return ret;
+}
+
+int nvgpu_preempt_channel(struct gk20a *g, struct nvgpu_channel *ch)
+{
+	int err;
+	struct nvgpu_tsg *tsg = nvgpu_tsg_from_ch(ch);
+
+	if (tsg != NULL) {
+		err = g->ops.fifo.preempt_tsg(ch->g, tsg);
+	} else {
+		err = g->ops.fifo.preempt_channel(ch->g, ch);
+	}
+
+	return err;
+}
+
+/* called from rc */
+int nvgpu_preempt_poll_tsg_on_pbdma(struct gk20a *g,
+		struct nvgpu_tsg *tsg)
+{
+	unsigned long runlist_served_pbdmas;
+	unsigned long pbdma_id_bit;
+	u32 tsgid, pbdma_id;
+
+	if (g->ops.fifo.preempt_poll_pbdma == NULL) {
+		return 0;
+	}
+
+	tsgid = tsg->tsgid;
+	runlist_served_pbdmas = tsg->runlist->pbdma_bitmask;
+
+	for_each_set_bit(pbdma_id_bit, &runlist_served_pbdmas,
+			 nvgpu_get_litter_value(g, GPU_LIT_HOST_NUM_PBDMA)) {
+		pbdma_id = U32(pbdma_id_bit);
+		/*
+		 * If pbdma preempt fails the only option is to reset
+		 * GPU. Any sort of hang indicates the entire GPU’s
+		 * memory system would be blocked.
+		 */
+		if (g->ops.fifo.preempt_poll_pbdma(g, tsgid, pbdma_id) != 0) {
+			nvgpu_err(g, "PBDMA preempt failed");
+			return -EBUSY;
+		}
+	}
+	return 0;
+}
+
+/*
+ * This should be called with runlist_lock held for all the
+ * runlists set in runlists_mask
+ */
+void nvgpu_fifo_preempt_runlists_for_rc(struct gk20a *g, u32 runlists_bitmask)
+{
+	struct nvgpu_fifo *f = &g->fifo;
+	u32 i;
+#ifdef CONFIG_NVGPU_LS_PMU
+	u32 token = PMU_INVALID_MUTEX_OWNER_ID;
+	int mutex_ret = 0;
+#endif
+
+	/* runlist_lock are locked by teardown and sched are disabled too */
+	nvgpu_log_fn(g, "preempt runlists_bitmask:0x%08x", runlists_bitmask);
+#ifdef CONFIG_NVGPU_LS_PMU
+	mutex_ret = nvgpu_pmu_lock_acquire(g, g->pmu,
+			PMU_MUTEX_ID_FIFO, &token);
+#endif
+
+	for (i = 0U; i < f->num_runlists; i++) {
+		struct nvgpu_runlist *runlist;
+
+		runlist = &f->active_runlists[i];
+
+		if ((BIT32(runlist->id) & runlists_bitmask) == 0U) {
+			continue;
+		}
+		/* issue runlist preempt */
+		g->ops.fifo.preempt_trigger(g, runlist->id,
+					ID_TYPE_RUNLIST);
+#ifdef CONFIG_NVGPU_RECOVERY
+		/*
+		 * Preemption will never complete in RC due to some
+		 * fatal condition. Do not poll for preemption to
+		 * complete. Reset engines served by runlists.
+		 */
+		runlist->reset_eng_bitmask = runlist->eng_bitmask;
+#endif
+	}
+
+#ifdef CONFIG_NVGPU_LS_PMU
+	if (mutex_ret == 0) {
+		int err = nvgpu_pmu_lock_release(g, g->pmu, PMU_MUTEX_ID_FIFO,
+				&token);
+		if (err != 0) {
+			nvgpu_err(g, "PMU_MUTEX_ID_FIFO not released err=%d",
+					err);
+		}
+	}
+#endif
+}

drivers/gpu/nvgpu/common/fifo/priv_cmdbuf.c

@@ -0,0 +1,333 @@
+/*
+ * Copyright (c) 2011-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/log.h>
+#include <nvgpu/utils.h>
+#include <nvgpu/log2.h>
+#include <nvgpu/barrier.h>
+#include <nvgpu/dma.h>
+#include <nvgpu/nvgpu_mem.h>
+#include <nvgpu/kmem.h>
+#include <nvgpu/vm.h>
+#include <nvgpu/priv_cmdbuf.h>
+#include <nvgpu/gk20a.h>
+#include <nvgpu/trace.h>
+#include <nvgpu/circ_buf.h>
+
+struct priv_cmd_entry {
+	struct nvgpu_mem *mem;
+	u32 off;	/* offset in mem, in u32 entries */
+	u32 fill_off;	/* write offset from off, in u32 entries */
+	u32 size;	/* in words */
+	u32 alloc_size;
+};
+
+struct priv_cmd_queue {
+	struct vm_gk20a *vm;
+	struct nvgpu_mem mem; /* pushbuf */
+	u32 size;	/* allocated length in words */
+	u32 put;	/* next entry will begin here */
+	u32 get;	/* next entry to free begins here */
+
+	/* an entry is a fragment of the pushbuf memory */
+	struct priv_cmd_entry *entries;
+	u32 entries_len; /* allocated length */
+	u32 entry_put;
+	u32 entry_get;
+};
+
+/* allocate private cmd buffer queue.
+   used for inserting commands before/after user submitted buffers. */
+int nvgpu_priv_cmdbuf_queue_alloc(struct vm_gk20a *vm,
+	u32 job_count, struct priv_cmd_queue **queue)
+{
+	struct gk20a *g = vm->mm->g;
+	struct priv_cmd_queue *q;
+	u64 size, tmp_size;
+	int err = 0;
+	u32 wait_size, incr_size;
+	u32 mem_per_job;
+
+	/*
+	 * sema size is at least as much as syncpt size, but semas may not be
+	 * enabled in the build. If neither semas nor syncpts are enabled, priv
+	 * cmdbufs and as such kernel mode submits with job tracking won't be
+	 * supported.
+	 */
+#ifdef CONFIG_NVGPU_SW_SEMAPHORE
+	wait_size = g->ops.sync.sema.get_wait_cmd_size();
+	incr_size = g->ops.sync.sema.get_incr_cmd_size();
+#else
+	wait_size = g->ops.sync.syncpt.get_wait_cmd_size();
+	incr_size = g->ops.sync.syncpt.get_incr_cmd_size(true);
+#endif
+
+	/*
+	 * Compute the amount of priv_cmdbuf space we need. In general the
+	 * worst case is the kernel inserts both a semaphore pre-fence and
+	 * post-fence. Any sync-pt fences will take less memory so we can
+	 * ignore them unless they're the only supported type. Jobs can also
+	 * have more than one pre-fence but that's abnormal and we'll -EAGAIN
+	 * if such jobs would fill the queue.
+	 *
+	 * A semaphore ACQ (fence-wait) is 8 words: semaphore_a, semaphore_b,
+	 * semaphore_c, and semaphore_d. A semaphore INCR (fence-get) will be
+	 * 10 words: all the same as an ACQ plus a non-stalling intr which is
+	 * another 2 words. In reality these numbers vary by chip but we'll use
+	 * 8 and 10 as examples.
+	 *
+	 * Given the job count, cmdbuf space is allocated such that each job
+	 * can get one wait command and one increment command:
+	 *
+	 *   job_count * (8 + 10) * 4 bytes
+	 *
+	 * These cmdbufs are inserted as gpfifo entries right before and after
+	 * the user submitted gpfifo entries per submit.
+	 *
+	 * One extra slot is added to the queue length so that the requested
+	 * job count can actually be allocated. This ring buffer implementation
+	 * is full when the number of consumed entries is one less than the
+	 * allocation size:
+	 *
+	 * alloc bytes = job_count * (wait + incr + 1) * slot in bytes
+	 */
+	mem_per_job = nvgpu_safe_mult_u32(
+			nvgpu_safe_add_u32(
+				nvgpu_safe_add_u32(wait_size, incr_size),
+				1U),
+			(u32)sizeof(u32));
+	/* both 32 bit and mem_per_job is small */
+	size = nvgpu_safe_mult_u64((u64)job_count, (u64)mem_per_job);
+
+	tmp_size = PAGE_ALIGN(roundup_pow_of_two(size));
+	if (tmp_size > U32_MAX) {
+		return -ERANGE;
+	}
+	size = (u32)tmp_size;
+
+	q = nvgpu_kzalloc(g, sizeof(*q));
+	if (q == NULL) {
+		return -ENOMEM;
+	}
+
+	q->vm = vm;
+
+	if (job_count > U32_MAX / 2U - 1U) {
+		err = -ERANGE;
+		goto err_free_queue;
+	}
+
+	/* One extra to account for the full condition: 2 * job_count + 1 */
+	q->entries_len = nvgpu_safe_mult_u32(2U,
+			nvgpu_safe_add_u32(job_count, 1U));
+	q->entries = nvgpu_vzalloc(g,
+			nvgpu_safe_mult_u64((u64)q->entries_len,
+				sizeof(*q->entries)));
+	if (q->entries == NULL) {
+		err = -ENOMEM;
+		goto err_free_queue;
+	}
+
+	err = nvgpu_dma_alloc_map_sys(vm, size, &q->mem);
+	if (err != 0) {
+		nvgpu_err(g, "%s: memory allocation failed", __func__);
+		goto err_free_entries;
+	}
+
+	tmp_size = q->mem.size / sizeof(u32);
+	nvgpu_assert(tmp_size <= U32_MAX);
+	q->size = (u32)tmp_size;
+
+	*queue = q;
+	return 0;
+err_free_entries:
+	nvgpu_vfree(g, q->entries);
+err_free_queue:
+	nvgpu_kfree(g, q);
+	return err;
+}
+
+void nvgpu_priv_cmdbuf_queue_free(struct priv_cmd_queue *q)
+{
+	struct vm_gk20a *vm = q->vm;
+	struct gk20a *g = vm->mm->g;
+
+	nvgpu_dma_unmap_free(vm, &q->mem);
+	nvgpu_vfree(g, q->entries);
+	nvgpu_kfree(g, q);
+}
+
+/* allocate a cmd buffer with given size. size is number of u32 entries */
+static int nvgpu_priv_cmdbuf_alloc_buf(struct priv_cmd_queue *q, u32 orig_size,
+			     struct priv_cmd_entry *e)
+{
+	struct gk20a *g = q->vm->mm->g;
+	u32 size = orig_size;
+	u32 free_count;
+
+	nvgpu_log_fn(g, "size %d", orig_size);
+
+	/*
+	 * If free space in the end is less than requested, increase the size
+	 * to make the real allocated space start from beginning. The hardware
+	 * expects each cmdbuf to be contiguous in the dma space.
+	 *
+	 * This too small extra space in the end may happen because the
+	 * requested wait and incr command buffers do not necessarily align
+	 * with the whole buffer capacity. They don't always align because the
+	 * buffer size is rounded to the next power of two and because not all
+	 * jobs necessarily use exactly one wait command.
+	 */
+	if (nvgpu_safe_add_u32(q->put, size) > q->size) {
+		size = orig_size + (q->size - q->put);
+	}
+
+	nvgpu_log_info(g, "priv cmd queue get:put %d:%d",
+			q->get, q->put);
+
+	nvgpu_assert(q->put < q->size);
+	nvgpu_assert(q->get < q->size);
+	nvgpu_assert(q->size > 0U);
+	free_count = (q->size - q->put + q->get - 1U) & (q->size - 1U);
+
+	if (size > free_count) {
+		return -EAGAIN;
+	}
+
+	e->fill_off = 0;
+	e->size = orig_size;
+	e->alloc_size = size;
+	e->mem = &q->mem;
+
+	/*
+	 * if we have increased size to skip free space in the end, set put
+	 * to beginning of cmd buffer + size, as if the prev put was at
+	 * position 0.
+	 */
+	if (size != orig_size) {
+		e->off = 0;
+		q->put = orig_size;
+	} else {
+		e->off = q->put;
+		q->put = (q->put + orig_size) & (q->size - 1U);
+	}
+
+	/* we already handled q->put + size > q->size so BUG_ON this */
+	BUG_ON(q->put > q->size);
+
+	nvgpu_log_fn(g, "done");
+
+	return 0;
+}
+
+int nvgpu_priv_cmdbuf_alloc(struct priv_cmd_queue *q, u32 size,
+			     struct priv_cmd_entry **e)
+{
+	u32 next_put = nvgpu_safe_add_u32(q->entry_put, 1U) % q->entries_len;
+	struct priv_cmd_entry *entry;
+	int err;
+
+	if (next_put == q->entry_get) {
+		return -EAGAIN;
+	}
+	entry = &q->entries[q->entry_put];
+
+	err = nvgpu_priv_cmdbuf_alloc_buf(q, size, entry);
+	if (err != 0) {
+		return err;
+	}
+
+	q->entry_put = next_put;
+	*e = entry;
+
+	return 0;
+}
+
+void nvgpu_priv_cmdbuf_rollback(struct priv_cmd_queue *q,
+		struct priv_cmd_entry *e)
+{
+	nvgpu_assert(q->put < q->size);
+	nvgpu_assert(q->size > 0U);
+	nvgpu_assert(e->alloc_size <= q->size);
+	q->put = (q->put + q->size - e->alloc_size) & (q->size - 1U);
+
+	(void)memset(e, 0, sizeof(*e));
+
+	nvgpu_assert(q->entry_put < q->entries_len);
+	nvgpu_assert(q->entries_len > 0U);
+	q->entry_put = (q->entry_put + q->entries_len - 1U)
+		% q->entries_len;
+}
+
+void nvgpu_priv_cmdbuf_free(struct priv_cmd_queue *q, struct priv_cmd_entry *e)
+{
+	struct gk20a *g = q->vm->mm->g;
+
+	if ((q->get != e->off) && e->off != 0U) {
+		nvgpu_err(g, "priv cmdbuf requests out-of-order");
+	}
+	nvgpu_assert(q->size > 0U);
+	q->get = nvgpu_safe_add_u32(e->off, e->size) & (q->size - 1U);
+	q->entry_get = nvgpu_safe_add_u32(q->entry_get, 1U) % q->entries_len;
+
+	(void)memset(e, 0, sizeof(*e));
+}
+
+void nvgpu_priv_cmdbuf_append(struct gk20a *g, struct priv_cmd_entry *e,
+		u32 *data, u32 entries)
+{
+	nvgpu_assert(e->fill_off + entries <= e->size);
+	nvgpu_mem_wr_n(g, e->mem, (e->off + e->fill_off) * sizeof(u32),
+			data, entries * sizeof(u32));
+	e->fill_off += entries;
+}
+
+void nvgpu_priv_cmdbuf_append_zeros(struct gk20a *g, struct priv_cmd_entry *e,
+		u32 entries)
+{
+	nvgpu_assert(e->fill_off + entries <= e->size);
+	nvgpu_memset(g, e->mem, (e->off + e->fill_off) * sizeof(u32),
+			0, entries * sizeof(u32));
+	e->fill_off += entries;
+}
+
+void nvgpu_priv_cmdbuf_finish(struct gk20a *g, struct priv_cmd_entry *e,
+		u64 *gva, u32 *size)
+{
+	/*
+	 * The size is written to the pushbuf entry, so make sure this buffer
+	 * is complete at this point. The responsibility of the channel sync is
+	 * to be consistent in allocation and usage, and the matching size and
+	 * add gops (e.g., get_wait_cmd_size, add_wait_cmd) help there.
+	 */
+	nvgpu_assert(e->fill_off == e->size);
+
+#ifdef CONFIG_NVGPU_TRACE
+	if (e->mem->aperture == APERTURE_SYSMEM) {
+		trace_gk20a_push_cmdbuf(g->name, 0, e->size, 0,
+				(u32 *)e->mem->cpu_va + e->off);
+	}
+#endif
+	*gva = nvgpu_safe_add_u64(e->mem->gpu_va,
+			nvgpu_safe_mult_u64((u64)e->off, sizeof(u32)));
+	*size = e->size;
+}

drivers/gpu/nvgpu/common/fifo/runlist.c

@@ -0,0 +1,914 @@
+/*
+ * Copyright (c) 2011-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/gk20a.h>
+#include <nvgpu/channel.h>
+#include <nvgpu/fifo.h>
+#include <nvgpu/engines.h>
+#include <nvgpu/device.h>
+#include <nvgpu/runlist.h>
+#include <nvgpu/ptimer.h>
+#include <nvgpu/bug.h>
+#include <nvgpu/dma.h>
+#include <nvgpu/rc.h>
+#include <nvgpu/static_analysis.h>
+#ifdef CONFIG_NVGPU_LS_PMU
+#include <nvgpu/pmu/mutex.h>
+#endif
+
+void nvgpu_runlist_lock_active_runlists(struct gk20a *g)
+{
+	struct nvgpu_fifo *f = &g->fifo;
+	struct nvgpu_runlist *runlist;
+	u32 i;
+
+	nvgpu_log_info(g, "acquire runlist_lock for active runlists");
+	for (i = 0; i < g->fifo.num_runlists; i++) {
+		runlist = &f->active_runlists[i];
+		nvgpu_mutex_acquire(&runlist->runlist_lock);
+	}
+}
+
+void nvgpu_runlist_unlock_active_runlists(struct gk20a *g)
+{
+	struct nvgpu_fifo *f = &g->fifo;
+	struct nvgpu_runlist *runlist;
+	u32 i;
+
+	nvgpu_log_info(g, "release runlist_lock for active runlists");
+	for (i = 0; i < g->fifo.num_runlists; i++) {
+		runlist = &f->active_runlists[i];
+		nvgpu_mutex_release(&runlist->runlist_lock);
+	}
+}
+
+static u32 nvgpu_runlist_append_tsg(struct gk20a *g,
+		struct nvgpu_runlist *runlist,
+		u32 **runlist_entry,
+		u32 *entries_left,
+		struct nvgpu_tsg *tsg)
+{
+	struct nvgpu_fifo *f = &g->fifo;
+	u32 runlist_entry_words = f->runlist_entry_size / (u32)sizeof(u32);
+	struct nvgpu_channel *ch;
+	u32 count = 0;
+	u32 timeslice;
+
+	nvgpu_log_fn(f->g, " ");
+
+	if (*entries_left == 0U) {
+		return RUNLIST_APPEND_FAILURE;
+	}
+
+	/* add TSG entry */
+	nvgpu_log_info(g, "add TSG %d to runlist", tsg->tsgid);
+
+	/*
+	 * timeslice is measured with PTIMER.
+	 * On some platforms, PTIMER is lower than 1GHz.
+	 */
+	timeslice = scale_ptimer(tsg->timeslice_us,
+			ptimer_scalingfactor10x(g->ptimer_src_freq));
+
+	g->ops.runlist.get_tsg_entry(tsg, *runlist_entry, timeslice);
+
+	nvgpu_log_info(g, "tsg rl entries left %d runlist [0] %x [1] %x",
+			*entries_left,
+			(*runlist_entry)[0], (*runlist_entry)[1]);
+	*runlist_entry += runlist_entry_words;
+	count++;
+	(*entries_left)--;
+
+	nvgpu_rwsem_down_read(&tsg->ch_list_lock);
+	/* add runnable channels bound to this TSG */
+	nvgpu_list_for_each_entry(ch, &tsg->ch_list,
+			nvgpu_channel, ch_entry) {
+		if (!nvgpu_test_bit(ch->chid,
+			      runlist->active_channels)) {
+			continue;
+		}
+
+		if (*entries_left == 0U) {
+			nvgpu_rwsem_up_read(&tsg->ch_list_lock);
+			return RUNLIST_APPEND_FAILURE;
+		}
+
+		nvgpu_log_info(g, "add channel %d to runlist",
+			ch->chid);
+		g->ops.runlist.get_ch_entry(ch, *runlist_entry);
+		nvgpu_log_info(g, "rl entries left %d runlist [0] %x [1] %x",
+			*entries_left,
+			(*runlist_entry)[0], (*runlist_entry)[1]);
+		count = nvgpu_safe_add_u32(count, 1U);
+		*runlist_entry += runlist_entry_words;
+		(*entries_left)--;
+	}
+	nvgpu_rwsem_up_read(&tsg->ch_list_lock);
+
+	return count;
+}
+
+
+static u32 nvgpu_runlist_append_prio(struct nvgpu_fifo *f,
+				struct nvgpu_runlist *runlist,
+				u32 **runlist_entry,
+				u32 *entries_left,
+				u32 interleave_level)
+{
+	u32 count = 0;
+	unsigned long tsgid;
+
+	nvgpu_log_fn(f->g, " ");
+
+	for_each_set_bit(tsgid, runlist->active_tsgs, f->num_channels) {
+		struct nvgpu_tsg *tsg = nvgpu_tsg_get_from_id(f->g, (u32)tsgid);
+		u32 entries;
+
+		if (tsg->interleave_level == interleave_level) {
+			entries = nvgpu_runlist_append_tsg(f->g, runlist,
+					runlist_entry, entries_left, tsg);
+			if (entries == RUNLIST_APPEND_FAILURE) {
+				return RUNLIST_APPEND_FAILURE;
+			}
+			count += entries;
+		}
+	}
+
+	return count;
+}
+
+static u32 nvgpu_runlist_append_hi(struct nvgpu_fifo *f,
+				struct nvgpu_runlist *runlist,
+				u32 **runlist_entry,
+				u32 *entries_left)
+{
+	nvgpu_log_fn(f->g, " ");
+
+	/*
+	 * No higher levels - this is where the "recursion" ends; just add all
+	 * active TSGs at this level.
+	 */
+	return nvgpu_runlist_append_prio(f, runlist, runlist_entry,
+			entries_left,
+			NVGPU_FIFO_RUNLIST_INTERLEAVE_LEVEL_HIGH);
+}
+
+static u32 nvgpu_runlist_append_med(struct nvgpu_fifo *f,
+				struct nvgpu_runlist *runlist,
+				u32 **runlist_entry,
+				u32 *entries_left)
+{
+	u32 count = 0;
+	unsigned long tsgid;
+
+	nvgpu_log_fn(f->g, " ");
+
+	for_each_set_bit(tsgid, runlist->active_tsgs, f->num_channels) {
+		struct nvgpu_tsg *tsg = nvgpu_tsg_get_from_id(f->g, (u32)tsgid);
+		u32 entries;
+
+		if (tsg->interleave_level !=
+				NVGPU_FIFO_RUNLIST_INTERLEAVE_LEVEL_MEDIUM) {
+			continue;
+		}
+
+		/* LEVEL_MEDIUM list starts with a LEVEL_HIGH, if any */
+
+		entries = nvgpu_runlist_append_hi(f, runlist,
+				runlist_entry, entries_left);
+		if (entries == RUNLIST_APPEND_FAILURE) {
+			return RUNLIST_APPEND_FAILURE;
+		}
+		count += entries;
+
+		entries = nvgpu_runlist_append_tsg(f->g, runlist,
+				runlist_entry, entries_left, tsg);
+		if (entries == RUNLIST_APPEND_FAILURE) {
+			return RUNLIST_APPEND_FAILURE;
+		}
+		count += entries;
+	}
+
+	return count;
+}
+
+static u32 nvgpu_runlist_append_low(struct nvgpu_fifo *f,
+				struct nvgpu_runlist *runlist,
+				u32 **runlist_entry,
+				u32 *entries_left)
+{
+	u32 count = 0;
+	unsigned long tsgid;
+
+	nvgpu_log_fn(f->g, " ");
+
+	for_each_set_bit(tsgid, runlist->active_tsgs, f->num_channels) {
+		struct nvgpu_tsg *tsg = nvgpu_tsg_get_from_id(f->g, (u32)tsgid);
+		u32 entries;
+
+		if (tsg->interleave_level !=
+				NVGPU_FIFO_RUNLIST_INTERLEAVE_LEVEL_LOW) {
+			continue;
+		}
+
+		/* The medium level starts with the highs, if any. */
+
+		entries = nvgpu_runlist_append_med(f, runlist,
+				runlist_entry, entries_left);
+		if (entries == RUNLIST_APPEND_FAILURE) {
+			return RUNLIST_APPEND_FAILURE;
+		}
+		count += entries;
+
+		entries = nvgpu_runlist_append_hi(f, runlist,
+				runlist_entry, entries_left);
+		if (entries == RUNLIST_APPEND_FAILURE) {
+			return RUNLIST_APPEND_FAILURE;
+		}
+		count += entries;
+
+		entries = nvgpu_runlist_append_tsg(f->g, runlist,
+				runlist_entry, entries_left, tsg);
+		if (entries == RUNLIST_APPEND_FAILURE) {
+			return RUNLIST_APPEND_FAILURE;
+		}
+		count += entries;
+	}
+
+	if (count == 0U) {
+		/*
+		 * No transitions to fill with higher levels, so add
+		 * the next level once. If that's empty too, we have only
+		 * LEVEL_HIGH jobs.
+		 */
+		count = nvgpu_runlist_append_med(f, runlist,
+				runlist_entry, entries_left);
+		if (count == 0U) {
+			count = nvgpu_runlist_append_hi(f, runlist,
+					runlist_entry, entries_left);
+		}
+	}
+
+	return count;
+}
+
+static u32 nvgpu_runlist_append_flat(struct nvgpu_fifo *f,
+				struct nvgpu_runlist *runlist,
+				u32 **runlist_entry,
+				u32 *entries_left)
+{
+	u32 count = 0, entries, i;
+
+	nvgpu_log_fn(f->g, " ");
+
+	/* Group by priority but don't interleave. High comes first. */
+
+	for (i = 0; i < NVGPU_FIFO_RUNLIST_INTERLEAVE_NUM_LEVELS; i++) {
+		u32 level = NVGPU_FIFO_RUNLIST_INTERLEAVE_LEVEL_HIGH - i;
+
+		entries = nvgpu_runlist_append_prio(f, runlist, runlist_entry,
+				entries_left, level);
+		if (entries == RUNLIST_APPEND_FAILURE) {
+			return RUNLIST_APPEND_FAILURE;
+		}
+		count += entries;
+	}
+
+	return count;
+}
+
+u32 nvgpu_runlist_construct_locked(struct nvgpu_fifo *f,
+				struct nvgpu_runlist *runlist,
+				u32 buf_id,
+				u32 max_entries)
+{
+	u32 *runlist_entry_base = runlist->mem[buf_id].cpu_va;
+
+	/*
+	 * The entry pointer and capacity counter that live on the stack here
+	 * keep track of the current position and the remaining space when tsg
+	 * and channel entries are ultimately appended.
+	 */
+	if (f->g->runlist_interleave) {
+		return nvgpu_runlist_append_low(f, runlist,
+				&runlist_entry_base, &max_entries);
+	} else {
+		return nvgpu_runlist_append_flat(f, runlist,
+				&runlist_entry_base, &max_entries);
+	}
+}
+
+static bool nvgpu_runlist_modify_active_locked(struct gk20a *g,
+					       struct nvgpu_runlist *runlist,
+					       struct nvgpu_channel *ch, bool add)
+{
+	struct nvgpu_tsg *tsg = NULL;
+
+	tsg = nvgpu_tsg_from_ch(ch);
+
+	if (tsg == NULL) {
+		/*
+		 * Unsupported condition, but shouldn't break anything. Warn
+		 * and tell the caller that nothing has changed.
+		 */
+		nvgpu_warn(g, "Bare channel in runlist update");
+		return false;
+	}
+
+	if (add) {
+		if (nvgpu_test_and_set_bit(ch->chid,
+				runlist->active_channels)) {
+			/* was already there */
+			return false;
+		} else {
+			/* new, and belongs to a tsg */
+			nvgpu_set_bit(tsg->tsgid, runlist->active_tsgs);
+			tsg->num_active_channels = nvgpu_safe_add_u32(
+					tsg->num_active_channels, 1U);
+		}
+	} else {
+		if (!nvgpu_test_and_clear_bit(ch->chid,
+				runlist->active_channels)) {
+			/* wasn't there */
+			return false;
+		} else {
+			tsg->num_active_channels = nvgpu_safe_sub_u32(
+				tsg->num_active_channels, 1U);
+			if (tsg->num_active_channels == 0U) {
+				/* was the only member of this tsg */
+				nvgpu_clear_bit(tsg->tsgid,
+						runlist->active_tsgs);
+			}
+		}
+	}
+
+	return true;
+}
+
+static int nvgpu_runlist_reconstruct_locked(struct gk20a *g,
+					    struct nvgpu_runlist *runlist,
+					    u32 buf_id, bool add_entries)
+{
+	u32 num_entries;
+	struct nvgpu_fifo *f = &g->fifo;
+
+	rl_dbg(g, "[%u] switch to new buffer 0x%16llx",
+		runlist->id, (u64)nvgpu_mem_get_addr(g, &runlist->mem[buf_id]));
+
+	if (!add_entries) {
+		runlist->count = 0;
+		return 0;
+	}
+
+	num_entries = nvgpu_runlist_construct_locked(f, runlist, buf_id,
+						f->num_runlist_entries);
+	if (num_entries == RUNLIST_APPEND_FAILURE) {
+		return -E2BIG;
+	}
+	runlist->count = num_entries;
+NVGPU_COV_WHITELIST_BLOCK_BEGIN(false_positive, 1, NVGPU_MISRA(Rule, 10_3), "Bug 2277532")
+NVGPU_COV_WHITELIST_BLOCK_BEGIN(false_positive, 1, NVGPU_MISRA(Rule, 14_4), "Bug 2277532")
+NVGPU_COV_WHITELIST_BLOCK_BEGIN(false_positive, 1, NVGPU_MISRA(Rule, 15_6), "Bug 2277532")
+	WARN_ON(runlist->count > f->num_runlist_entries);
+NVGPU_COV_WHITELIST_BLOCK_END(NVGPU_MISRA(Rule, 10_3))
+NVGPU_COV_WHITELIST_BLOCK_END(NVGPU_MISRA(Rule, 14_4))
+NVGPU_COV_WHITELIST_BLOCK_END(NVGPU_MISRA(Rule, 15_6))
+
+	return 0;
+}
+
+int nvgpu_runlist_update_locked(struct gk20a *g, struct nvgpu_runlist *rl,
+				struct nvgpu_channel *ch, bool add,
+				bool wait_for_finish)
+{
+	int ret = 0;
+	u32 buf_id;
+	bool add_entries;
+
+	if (ch != NULL) {
+		bool update = nvgpu_runlist_modify_active_locked(g, rl, ch, add);
+		if (!update) {
+			/* no change in runlist contents */
+			return 0;
+		}
+		/* had a channel to update, so reconstruct */
+		add_entries = true;
+	} else {
+		/* no channel; add means update all, !add means clear all */
+		add_entries = add;
+	}
+
+	/* double buffering, swap to next */
+	buf_id = (rl->cur_buffer == 0U) ? 1U : 0U;
+
+	ret = nvgpu_runlist_reconstruct_locked(g, rl, buf_id, add_entries);
+	if (ret != 0) {
+		return ret;
+	}
+
+	g->ops.runlist.hw_submit(g, rl->id, rl->count, buf_id);
+
+	if (wait_for_finish) {
+		ret = g->ops.runlist.wait_pending(g, rl->id);
+
+		if (ret == -ETIMEDOUT) {
+			nvgpu_err(g, "runlist %d update timeout", rl->id);
+			/* trigger runlist update timeout recovery */
+			return ret;
+
+		} else {
+			if (ret == -EINTR) {
+				nvgpu_err(g, "runlist update interrupted");
+			}
+		}
+	}
+
+	rl->cur_buffer = buf_id;
+
+	return ret;
+}
+
+#ifdef CONFIG_NVGPU_CHANNEL_TSG_SCHEDULING
+/* trigger host to expire current timeslice and reschedule runlist from front */
+int nvgpu_runlist_reschedule(struct nvgpu_channel *ch, bool preempt_next,
+		bool wait_preempt)
+{
+	struct gk20a *g = ch->g;
+	struct nvgpu_runlist *runlist;
+#ifdef CONFIG_NVGPU_LS_PMU
+	u32 token = PMU_INVALID_MUTEX_OWNER_ID;
+	int mutex_ret = 0;
+#endif
+	int ret = 0;
+
+	runlist = ch->runlist;
+	if (nvgpu_mutex_tryacquire(&runlist->runlist_lock) == 0) {
+		return -EBUSY;
+	}
+#ifdef CONFIG_NVGPU_LS_PMU
+	mutex_ret = nvgpu_pmu_lock_acquire(
+		g, g->pmu, PMU_MUTEX_ID_FIFO, &token);
+#endif
+
+	g->ops.runlist.hw_submit(
+		g, runlist->id, runlist->count, runlist->cur_buffer);
+
+	if (preempt_next) {
+		if (g->ops.runlist.reschedule_preempt_next_locked(ch,
+				wait_preempt) != 0) {
+			nvgpu_err(g, "reschedule preempt next failed");
+		}
+	}
+
+	if (g->ops.runlist.wait_pending(g, runlist->id) != 0) {
+		nvgpu_err(g, "wait pending failed for runlist %u",
+				runlist->id);
+	}
+#ifdef CONFIG_NVGPU_LS_PMU
+	if (mutex_ret == 0) {
+		if (nvgpu_pmu_lock_release(g, g->pmu,
+				PMU_MUTEX_ID_FIFO, &token) != 0) {
+			nvgpu_err(g, "failed to release PMU lock");
+		}
+	}
+#endif
+	nvgpu_mutex_release(&runlist->runlist_lock);
+
+	return ret;
+}
+#endif
+
+/* add/remove a channel from runlist
+   special cases below: runlist->active_channels will NOT be changed.
+   (ch == NULL && !add) means remove all active channels from runlist.
+   (ch == NULL &&  add) means restore all active channels on runlist. */
+static int nvgpu_runlist_do_update(struct gk20a *g, struct nvgpu_runlist *rl,
+				   struct nvgpu_channel *ch,
+				   bool add, bool wait_for_finish)
+{
+#ifdef CONFIG_NVGPU_LS_PMU
+	u32 token = PMU_INVALID_MUTEX_OWNER_ID;
+	int mutex_ret = 0;
+#endif
+	int ret = 0;
+
+	nvgpu_log_fn(g, " ");
+
+	nvgpu_mutex_acquire(&rl->runlist_lock);
+#ifdef CONFIG_NVGPU_LS_PMU
+	mutex_ret = nvgpu_pmu_lock_acquire(g, g->pmu,
+		PMU_MUTEX_ID_FIFO, &token);
+#endif
+	ret = nvgpu_runlist_update_locked(g, rl, ch, add, wait_for_finish);
+#ifdef CONFIG_NVGPU_LS_PMU
+	if (mutex_ret == 0) {
+		if (nvgpu_pmu_lock_release(g, g->pmu,
+				PMU_MUTEX_ID_FIFO, &token) != 0) {
+			nvgpu_err(g, "failed to release PMU lock");
+		}
+	}
+#endif
+	nvgpu_mutex_release(&rl->runlist_lock);
+
+	if (ret == -ETIMEDOUT) {
+		nvgpu_rc_runlist_update(g, rl->id);
+	}
+
+	return ret;
+}
+
+int nvgpu_runlist_update(struct gk20a *g, struct nvgpu_runlist *rl,
+			 struct nvgpu_channel *ch,
+			 bool add, bool wait_for_finish)
+{
+	nvgpu_assert(ch != NULL);
+
+	return nvgpu_runlist_do_update(g, rl, ch, add, wait_for_finish);
+}
+
+int nvgpu_runlist_reload(struct gk20a *g, struct nvgpu_runlist *rl,
+			      bool add, bool wait_for_finish)
+{
+	return nvgpu_runlist_do_update(g, rl, NULL, add, wait_for_finish);
+}
+
+int nvgpu_runlist_reload_ids(struct gk20a *g, u32 runlist_ids, bool add)
+{
+	struct nvgpu_fifo *f = &g->fifo;
+	int ret = -EINVAL;
+	unsigned long runlist_id = 0;
+	int errcode;
+	unsigned long ulong_runlist_ids = (unsigned long)runlist_ids;
+
+	if (g == NULL) {
+		goto end;
+	}
+
+	ret = 0;
+	for_each_set_bit(runlist_id, &ulong_runlist_ids, 32U) {
+		/* Capture the last failure error code */
+		errcode = g->ops.runlist.reload(g,
+						f->runlists[runlist_id], add, true);
+		if (errcode != 0) {
+			nvgpu_err(g,
+				"failed to update_runlist %lu %d",
+				runlist_id, errcode);
+			ret = errcode;
+		}
+	}
+end:
+	return ret;
+}
+
+const char *nvgpu_runlist_interleave_level_name(u32 interleave_level)
+{
+	const char *ret_string = NULL;
+
+	switch (interleave_level) {
+	case NVGPU_FIFO_RUNLIST_INTERLEAVE_LEVEL_LOW:
+		ret_string = "LOW";
+		break;
+
+	case NVGPU_FIFO_RUNLIST_INTERLEAVE_LEVEL_MEDIUM:
+		ret_string = "MEDIUM";
+		break;
+
+	case NVGPU_FIFO_RUNLIST_INTERLEAVE_LEVEL_HIGH:
+		ret_string = "HIGH";
+		break;
+
+	default:
+		ret_string = "?";
+		break;
+	}
+
+	return ret_string;
+}
+
+void nvgpu_runlist_set_state(struct gk20a *g, u32 runlists_mask,
+		u32 runlist_state)
+{
+#ifdef CONFIG_NVGPU_LS_PMU
+	u32 token = PMU_INVALID_MUTEX_OWNER_ID;
+	int mutex_ret = 0;
+#endif
+	nvgpu_log(g, gpu_dbg_info, "runlist mask = 0x%08x state = 0x%08x",
+			runlists_mask, runlist_state);
+
+#ifdef CONFIG_NVGPU_LS_PMU
+	mutex_ret = nvgpu_pmu_lock_acquire(g, g->pmu,
+		PMU_MUTEX_ID_FIFO, &token);
+#endif
+	g->ops.runlist.write_state(g, runlists_mask, runlist_state);
+#ifdef CONFIG_NVGPU_LS_PMU
+	if (mutex_ret == 0) {
+		if (nvgpu_pmu_lock_release(g, g->pmu,
+				PMU_MUTEX_ID_FIFO, &token) != 0) {
+			nvgpu_err(g, "failed to release PMU lock");
+		}
+	}
+#endif
+}
+
+void nvgpu_runlist_cleanup_sw(struct gk20a *g)
+{
+	struct nvgpu_fifo *f = &g->fifo;
+	u32 i, j;
+	struct nvgpu_runlist *runlist;
+
+	if ((f->runlists == NULL) || (f->active_runlists == NULL)) {
+		return;
+	}
+
+	g = f->g;
+
+	for (i = 0; i < f->num_runlists; i++) {
+		runlist = &f->active_runlists[i];
+		for (j = 0; j < MAX_RUNLIST_BUFFERS; j++) {
+			nvgpu_dma_free(g, &runlist->mem[j]);
+		}
+
+		nvgpu_kfree(g, runlist->active_channels);
+		runlist->active_channels = NULL;
+
+		nvgpu_kfree(g, runlist->active_tsgs);
+		runlist->active_tsgs = NULL;
+
+		nvgpu_mutex_destroy(&runlist->runlist_lock);
+		f->runlists[runlist->id] = NULL;
+	}
+
+	nvgpu_kfree(g, f->active_runlists);
+	f->active_runlists = NULL;
+	f->num_runlists = 0;
+	nvgpu_kfree(g, f->runlists);
+	f->runlists = NULL;
+	f->max_runlists = 0;
+}
+
+void nvgpu_runlist_init_enginfo(struct gk20a *g, struct nvgpu_fifo *f)
+{
+	struct nvgpu_runlist *runlist;
+	const struct nvgpu_device *dev;
+	u32 i, j;
+
+	nvgpu_log_fn(g, " ");
+
+	if (g->is_virtual) {
+		return;
+	}
+
+	for (i = 0; i < f->num_runlists; i++) {
+		runlist = &f->active_runlists[i];
+
+		(void) g->ops.fifo.find_pbdma_for_runlist(g,
+						runlist->id,
+						&runlist->pbdma_bitmask);
+		nvgpu_log(g, gpu_dbg_info, "runlist %d: pbdma bitmask 0x%x",
+				 runlist->id, runlist->pbdma_bitmask);
+
+		for (j = 0; j < f->num_engines; j++) {
+			dev = f->active_engines[j];
+
+			if (dev->runlist_id == runlist->id) {
+				runlist->eng_bitmask |= BIT32(dev->engine_id);
+			}
+		}
+		nvgpu_log(g, gpu_dbg_info, "runlist %d: act eng bitmask 0x%x",
+				 runlist->id, runlist->eng_bitmask);
+	}
+
+	nvgpu_log_fn(g, "done");
+}
+
+static int nvgpu_init_active_runlist_mapping(struct gk20a *g)
+{
+	struct nvgpu_runlist *runlist;
+	struct nvgpu_fifo *f = &g->fifo;
+	unsigned int runlist_id;
+	size_t runlist_size;
+	u32 i, j;
+	int err = 0;
+
+	rl_dbg(g, "Building active runlist map.");
+
+	/*
+	 * In most case we want to loop through active runlists only. Here
+	 * we need to loop through all possible runlists, to build the mapping
+	 * between runlists[runlist_id] and active_runlists[i].
+	 */
+	i = 0U;
+	for (runlist_id = 0; runlist_id < f->max_runlists; runlist_id++) {
+		if (!nvgpu_engine_is_valid_runlist_id(g, runlist_id)) {
+			/* skip inactive runlist */
+			rl_dbg(g, "  Skipping invalid runlist: %d", runlist_id);
+			continue;
+		}
+
+		rl_dbg(g, "  Configuring HW runlist: %u", runlist_id);
+		rl_dbg(g, "  SW runlist index to HW: %u -> %u", i, runlist_id);
+
+		runlist = &f->active_runlists[i];
+		runlist->id = runlist_id;
+		f->runlists[runlist_id] = runlist;
+		i = nvgpu_safe_add_u32(i, 1U);
+
+		runlist->active_channels =
+			nvgpu_kzalloc(g, DIV_ROUND_UP(f->num_channels,
+						      BITS_PER_BYTE));
+		if (runlist->active_channels == NULL) {
+			err = -ENOMEM;
+			goto clean_up_runlist;
+		}
+
+		runlist->active_tsgs =
+			nvgpu_kzalloc(g, DIV_ROUND_UP(f->num_channels,
+						      BITS_PER_BYTE));
+		if (runlist->active_tsgs == NULL) {
+			err = -ENOMEM;
+			goto clean_up_runlist;
+		}
+
+		runlist_size = (size_t)f->runlist_entry_size *
+				(size_t)f->num_runlist_entries;
+		rl_dbg(g, "    RL entries: %d", f->num_runlist_entries);
+		rl_dbg(g, "    RL size %zu", runlist_size);
+
+		for (j = 0; j < MAX_RUNLIST_BUFFERS; j++) {
+			err = nvgpu_dma_alloc_flags_sys(g,
+					g->is_virtual ?
+					  0ULL : NVGPU_DMA_PHYSICALLY_ADDRESSED,
+					runlist_size,
+					&runlist->mem[j]);
+			if (err != 0) {
+				nvgpu_err(g, "memory allocation failed");
+				err = -ENOMEM;
+				goto clean_up_runlist;
+			}
+		}
+
+		nvgpu_mutex_init(&runlist->runlist_lock);
+
+		/*
+                 * None of buffers is pinned if this value doesn't change.
+		 * Otherwise, one of them (cur_buffer) must have been pinned.
+                 */
+		runlist->cur_buffer = MAX_RUNLIST_BUFFERS;
+	}
+
+	return 0;
+
+clean_up_runlist:
+	return err;
+}
+
+int nvgpu_runlist_setup_sw(struct gk20a *g)
+{
+	struct nvgpu_fifo *f = &g->fifo;
+	u32 num_runlists = 0U;
+	unsigned int runlist_id;
+	int err = 0;
+
+	rl_dbg(g, "Initializing Runlists");
+
+	nvgpu_spinlock_init(&f->runlist_submit_lock);
+
+	f->runlist_entry_size = g->ops.runlist.entry_size(g);
+	f->num_runlist_entries = g->ops.runlist.length_max(g);
+	f->max_runlists = g->ops.runlist.count_max(g);
+
+	f->runlists = nvgpu_kzalloc(g, nvgpu_safe_mult_u64(
+				sizeof(*f->runlists), f->max_runlists));
+	if (f->runlists == NULL) {
+		err = -ENOMEM;
+		goto clean_up_runlist;
+	}
+
+	for (runlist_id = 0; runlist_id < f->max_runlists; runlist_id++) {
+		if (nvgpu_engine_is_valid_runlist_id(g, runlist_id)) {
+			num_runlists = nvgpu_safe_add_u32(num_runlists, 1U);
+		}
+	}
+	f->num_runlists = num_runlists;
+
+	f->active_runlists = nvgpu_kzalloc(g, nvgpu_safe_mult_u64(
+			 sizeof(*f->active_runlists), num_runlists));
+	if (f->active_runlists == NULL) {
+		err = -ENOMEM;
+		goto clean_up_runlist;
+	}
+
+
+	rl_dbg(g, "  Max runlists:    %u", f->max_runlists);
+	rl_dbg(g, "  Active runlists: %u", f->num_runlists);
+	rl_dbg(g, "  RL entry size:   %u bytes", f->runlist_entry_size);
+	rl_dbg(g, "  Max RL entries:  %u", f->num_runlist_entries);
+
+	err = nvgpu_init_active_runlist_mapping(g);
+	if (err != 0) {
+		goto clean_up_runlist;
+	}
+
+	g->ops.runlist.init_enginfo(g, f);
+	return 0;
+
+clean_up_runlist:
+	nvgpu_runlist_cleanup_sw(g);
+	rl_dbg(g, "fail");
+	return err;
+}
+
+u32 nvgpu_runlist_get_runlists_mask(struct gk20a *g, u32 id,
+	unsigned int id_type, u32 act_eng_bitmask, u32 pbdma_bitmask)
+{
+	u32 i, runlists_mask = 0;
+	struct nvgpu_fifo *f = &g->fifo;
+	struct nvgpu_runlist *runlist;
+
+	bool bitmask_disabled = ((act_eng_bitmask == 0U) &&
+				(pbdma_bitmask == 0U));
+
+	/* engine and/or pbdma ids are known */
+	if (!bitmask_disabled) {
+		for (i = 0U; i < f->num_runlists; i++) {
+			runlist = &f->active_runlists[i];
+
+			if ((runlist->eng_bitmask & act_eng_bitmask) != 0U) {
+				runlists_mask |= BIT32(runlist->id);
+			}
+
+			if ((runlist->pbdma_bitmask & pbdma_bitmask) != 0U) {
+				runlists_mask |= BIT32(runlist->id);
+			}
+		}
+	}
+
+	if (id_type != ID_TYPE_UNKNOWN) {
+		if (id_type == ID_TYPE_TSG) {
+			runlist = f->tsg[id].runlist;
+		} else {
+			runlist = f->channel[id].runlist;
+		}
+
+		if (runlist == NULL) {
+			/* Warning on Linux, real assert on QNX. */
+			nvgpu_assert(runlist != NULL);
+		} else {
+			runlists_mask |= BIT32(runlist->id);
+		}
+	} else {
+		if (bitmask_disabled) {
+			nvgpu_log(g, gpu_dbg_info, "id_type_unknown, engine "
+				"and pbdma ids are unknown");
+
+			for (i = 0U; i < f->num_runlists; i++) {
+				runlist = &f->active_runlists[i];
+
+				runlists_mask |= BIT32(runlist->id);
+			}
+		} else {
+			nvgpu_log(g, gpu_dbg_info, "id_type_unknown, engine "
+				"and/or pbdma ids are known");
+		}
+	}
+
+	nvgpu_log(g, gpu_dbg_info, "runlists_mask = 0x%08x", runlists_mask);
+	return runlists_mask;
+}
+
+void nvgpu_runlist_unlock_runlists(struct gk20a *g, u32 runlists_mask)
+{
+	struct nvgpu_fifo *f = &g->fifo;
+	struct nvgpu_runlist *runlist;
+	u32 i;
+
+	nvgpu_log_info(g, "release runlist_lock for runlists set in "
+				"runlists_mask: 0x%08x", runlists_mask);
+
+	for (i = 0U; i < f->num_runlists; i++) {
+		runlist = &f->active_runlists[i];
+
+		if ((BIT32(i) & runlists_mask) != 0U) {
+			nvgpu_mutex_release(&runlist->runlist_lock);
+		}
+	}
+}