Open source GPL/LGPL release

drivers/gpu/nvgpu/hal/bios/bios_tu104.c

@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include "nvgpu/hw/tu104/hw_gc6_tu104.h"
+
+#include "bios_tu104.h"
+
+u32 tu104_get_aon_secure_scratch_reg(struct gk20a *g, u32 i)
+{
+	return gc6_aon_secure_scratch_group_05_r(i);
+}
+

drivers/gpu/nvgpu/hal/bios/bios_tu104.h

@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef NVGPU_BIOS_TU104_H
+#define NVGPU_BIOS_TU104_H
+
+#include <nvgpu/types.h>
+
+struct gk20a;
+
+u32 tu104_get_aon_secure_scratch_reg(struct gk20a *g, u32 i);
+
+#endif /*NVGPU_BIOS_TU104_H */

drivers/gpu/nvgpu/hal/bus/bus_gk20a.c

@@ -0,0 +1,67 @@
+/*
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/log.h>
+#include <nvgpu/soc.h>
+#include <nvgpu/mm.h>
+#include <nvgpu/io.h>
+#include <nvgpu/bug.h>
+#include <nvgpu/gk20a.h>
+#include <nvgpu/nvgpu_sgt.h>
+#include <nvgpu/nvgpu_err.h>
+
+#include "bus_gk20a.h"
+
+#include <nvgpu/hw/gk20a/hw_bus_gk20a.h>
+
+#ifdef CONFIG_NVGPU_DGPU
+u32 gk20a_bus_set_bar0_window(struct gk20a *g, struct nvgpu_mem *mem,
+		       struct nvgpu_sgt *sgt, void *sgl, u32 w)
+{
+	u64 bufbase = nvgpu_sgt_get_phys(g, sgt, sgl);
+	u64 addr = bufbase + w * sizeof(u32);
+	u32 hi = (u32)((addr & ~(u64)0xfffff)
+		>> bus_bar0_window_target_bar0_window_base_shift_v());
+	u32 lo = U32(addr & 0xfffffULL);
+	u32 win = nvgpu_aperture_mask(g, mem,
+			bus_bar0_window_target_sys_mem_noncoherent_f(),
+			bus_bar0_window_target_sys_mem_coherent_f(),
+			bus_bar0_window_target_vid_mem_f()) |
+		bus_bar0_window_base_f(hi);
+
+	nvgpu_log(g, gpu_dbg_mem,
+			"0x%08x:%08x begin for %p,%p at [%llx,%llx] (sz %llx)",
+			hi, lo, mem, sgl, bufbase,
+			bufbase + nvgpu_sgt_get_phys(g, sgt, sgl),
+			nvgpu_sgt_get_length(sgt, sgl));
+
+	WARN_ON(bufbase == 0ULL);
+
+	if (g->mm.pramin_window != win) {
+		gk20a_writel(g, bus_bar0_window_r(), win);
+		(void) gk20a_readl(g, bus_bar0_window_r());
+		g->mm.pramin_window = win;
+	}
+
+	return lo;
+}
+#endif

drivers/gpu/nvgpu/hal/bus/bus_gk20a.h

@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+#ifndef BUS_GK20A_H
+#define BUS_GK20A_H
+
+#include <nvgpu/types.h>
+
+struct gk20a;
+struct nvgpu_mem;
+struct nvgpu_sgt;
+
+void gk20a_bus_isr(struct gk20a *g);
+int gk20a_bus_init_hw(struct gk20a *g);
+#ifdef CONFIG_NVGPU_DGPU
+u32 gk20a_bus_set_bar0_window(struct gk20a *g, struct nvgpu_mem *mem,
+			struct nvgpu_sgt *sgt, void *sgl, u32 w);
+#endif
+
+#endif /* BUS_GK20A_H */

drivers/gpu/nvgpu/hal/bus/bus_gk20a_fusa.c

@@ -0,0 +1,95 @@
+/*
+ * Copyright (c) 2017-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/log.h>
+#include <nvgpu/soc.h>
+#include <nvgpu/mm.h>
+#include <nvgpu/io.h>
+#include <nvgpu/bug.h>
+#include <nvgpu/gk20a.h>
+#include <nvgpu/nvgpu_sgt.h>
+#include <nvgpu/nvgpu_err.h>
+#include <nvgpu/cic.h>
+#include <nvgpu/mc.h>
+
+#include "bus_gk20a.h"
+
+#include <nvgpu/hw/gk20a/hw_bus_gk20a.h>
+
+int gk20a_bus_init_hw(struct gk20a *g)
+{
+	u32 intr_en_mask = 0U;
+
+	nvgpu_cic_intr_nonstall_unit_config(g, NVGPU_CIC_INTR_UNIT_BUS, NVGPU_CIC_INTR_ENABLE);
+
+	/*
+	 * Note: bus_intr_en_0 is for routing intr to stall tree (mc_intr_0)
+	 * bus_intr_en_1 is for routing bus intr to nostall tree (mc_intr_1)
+	 */
+	if (nvgpu_platform_is_silicon(g) || nvgpu_platform_is_fpga(g)) {
+		intr_en_mask = bus_intr_en_1_pri_squash_m() |
+				bus_intr_en_1_pri_fecserr_m() |
+				bus_intr_en_1_pri_timeout_m();
+	}
+
+	nvgpu_writel(g, bus_intr_en_1_r(), intr_en_mask);
+
+	if (g->ops.bus.configure_debug_bus != NULL) {
+		g->ops.bus.configure_debug_bus(g);
+	}
+
+	return 0;
+}
+
+void gk20a_bus_isr(struct gk20a *g)
+{
+	u32 val;
+	u32 err_type = GPU_HOST_INVALID_ERROR;
+
+	val = nvgpu_readl(g, bus_intr_0_r());
+
+	if ((val & (bus_intr_0_pri_squash_m() |
+			bus_intr_0_pri_fecserr_m() |
+			bus_intr_0_pri_timeout_m())) != 0U) {
+		if ((val & bus_intr_0_pri_squash_m()) != 0U) {
+			err_type = GPU_HOST_PBUS_SQUASH_ERROR;
+		}
+		if ((val & bus_intr_0_pri_fecserr_m()) != 0U) {
+			err_type = GPU_HOST_PBUS_FECS_ERROR;
+		}
+		if ((val & bus_intr_0_pri_timeout_m()) != 0U) {
+			err_type = GPU_HOST_PBUS_TIMEOUT_ERROR;
+		}
+		g->ops.ptimer.isr(g);
+	} else {
+		nvgpu_err(g, "Unhandled NV_PBUS_INTR_0: 0x%08x", val);
+		/* We group following errors as part of PBUS_TIMEOUT_ERROR:
+		 * FB_REQ_TIMEOUT, FB_ACK_TIMEOUT, FB_ACK_EXTRA,
+		 * FB_RDATA_TIMEOUT, FB_RDATA_EXTRA, POSTED_DEADLOCK_TIMEOUT,
+		 * ACCESS_TIMEOUT.
+		 */
+		err_type = GPU_HOST_PBUS_TIMEOUT_ERROR;
+	}
+	nvgpu_report_host_err(g, NVGPU_ERR_MODULE_HOST,
+			0, err_type, val);
+	nvgpu_writel(g, bus_intr_0_r(), val);
+}

drivers/gpu/nvgpu/hal/bus/bus_gm20b.h

@@ -0,0 +1,33 @@
+/*
+ * GM20B BUS
+ *
+ * Copyright (c) 2017, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef NVGPU_GM20B_BUS
+#define NVGPU_GM20B_BUS
+
+struct gk20a;
+struct nvgpu_mem;
+
+int gm20b_bus_bar1_bind(struct gk20a *g, struct nvgpu_mem *bar1_inst);
+
+#endif

drivers/gpu/nvgpu/hal/bus/bus_gm20b_fusa.c

@@ -0,0 +1,75 @@
+/*
+ * GM20B MMU
+ *
+ * Copyright (c) 2014-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/timers.h>
+#include <nvgpu/mm.h>
+#include <nvgpu/enabled.h>
+#include <nvgpu/io.h>
+#include <nvgpu/gk20a.h>
+#include <nvgpu/static_analysis.h>
+
+#include "bus_gm20b.h"
+
+#include <nvgpu/hw/gm20b/hw_bus_gm20b.h>
+
+int gm20b_bus_bar1_bind(struct gk20a *g, struct nvgpu_mem *bar1_inst)
+{
+	struct nvgpu_timeout timeout;
+	int err = 0;
+	u64 iova = nvgpu_inst_block_addr(g, bar1_inst);
+	u32 ptr_v = nvgpu_safe_cast_u64_to_u32(iova
+					>> bus_bar1_block_ptr_shift_v());
+
+	nvgpu_log_info(g, "bar1 inst block ptr: 0x%08x", ptr_v);
+
+	gk20a_writel(g, bus_bar1_block_r(),
+		     nvgpu_aperture_mask(g, bar1_inst,
+					 bus_bar1_block_target_sys_mem_ncoh_f(),
+					 bus_bar1_block_target_sys_mem_coh_f(),
+					 bus_bar1_block_target_vid_mem_f()) |
+		     bus_bar1_block_mode_virtual_f() |
+		     bus_bar1_block_ptr_f(ptr_v));
+	err = nvgpu_timeout_init(g, &timeout, 1000, NVGPU_TIMER_RETRY_TIMER);
+	if (err != 0) {
+		nvgpu_err(g, "nvgpu_timeout_init failed err=%d", err);
+		return err;
+	}
+
+	do {
+		u32 val = gk20a_readl(g, bus_bind_status_r());
+		u32 pending = bus_bind_status_bar1_pending_v(val);
+		u32 outstanding = bus_bind_status_bar1_outstanding_v(val);
+		if ((pending == 0U) && (outstanding == 0U)) {
+			break;
+		}
+
+		nvgpu_udelay(5);
+	} while (nvgpu_timeout_expired(&timeout) == 0);
+
+	if (nvgpu_timeout_peek_expired(&timeout)) {
+		err = -EINVAL;
+	}
+
+	return err;
+}

drivers/gpu/nvgpu/hal/bus/bus_gp10b.h

@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef NVGPU_BUS_GP10B
+#define NVGPU_BUS_GP10B
+
+struct gk20a;
+struct nvgpu_mem;
+
+int gp10b_bus_bar2_bind(struct gk20a *g, struct nvgpu_mem *bar2_inst);
+
+#endif

drivers/gpu/nvgpu/hal/bus/bus_gp10b_fusa.c

@@ -0,0 +1,72 @@
+/*
+ * Copyright (c) 2014-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/timers.h>
+#include <nvgpu/mm.h>
+#include <nvgpu/io.h>
+#include <nvgpu/gk20a.h>
+#include <nvgpu/static_analysis.h>
+
+#include "bus_gp10b.h"
+
+#include <nvgpu/hw/gp10b/hw_bus_gp10b.h>
+
+int gp10b_bus_bar2_bind(struct gk20a *g, struct nvgpu_mem *bar2_inst)
+{
+	struct nvgpu_timeout timeout;
+	int err = 0;
+	u64 iova = nvgpu_inst_block_addr(g, bar2_inst);
+	u32 ptr_v = nvgpu_safe_cast_u64_to_u32(iova
+					>> bus_bar2_block_ptr_shift_v());
+
+	nvgpu_log_info(g, "bar2 inst block ptr: 0x%08x", ptr_v);
+
+	gk20a_writel(g, bus_bar2_block_r(),
+		     nvgpu_aperture_mask(g, bar2_inst,
+					 bus_bar2_block_target_sys_mem_ncoh_f(),
+					 bus_bar2_block_target_sys_mem_coh_f(),
+					 bus_bar2_block_target_vid_mem_f()) |
+		     bus_bar2_block_mode_virtual_f() |
+		     bus_bar2_block_ptr_f(ptr_v));
+	err = nvgpu_timeout_init(g, &timeout, 1000, NVGPU_TIMER_RETRY_TIMER);
+	if (err != 0) {
+		nvgpu_err(g, "nvgpu_timeout_init failed err=%d", err);
+		return err;
+	}
+
+	do {
+		u32 val = gk20a_readl(g, bus_bind_status_r());
+		u32 pending = bus_bind_status_bar2_pending_v(val);
+		u32 outstanding = bus_bind_status_bar2_outstanding_v(val);
+		if ((pending == 0U) && (outstanding == 0U)) {
+			break;
+		}
+
+		nvgpu_udelay(5);
+	} while (nvgpu_timeout_expired(&timeout) == 0);
+
+	if (nvgpu_timeout_peek_expired(&timeout)) {
+		err = -EINVAL;
+	}
+
+	return err;
+}

drivers/gpu/nvgpu/hal/bus/bus_gv100.c

@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/timers.h>
+#include <nvgpu/mm.h>
+#include <nvgpu/io.h>
+#include <nvgpu/gk20a.h>
+
+#include "bus_gv100.h"
+
+#include <nvgpu/hw/gv100/hw_bus_gv100.h>
+
+u32 gv100_bus_read_sw_scratch(struct gk20a *g, u32 index)
+{
+	return gk20a_readl(g, bus_sw_scratch_r(index));
+}
+
+void gv100_bus_write_sw_scratch(struct gk20a *g, u32 index, u32 val)
+{
+	gk20a_writel(g, bus_sw_scratch_r(index), val);
+}

drivers/gpu/nvgpu/hal/bus/bus_gv100.h

@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef NVGPU_GV100_BUS
+#define NVGPU_GV100_BUS
+
+#include <nvgpu/types.h>
+
+struct gk20a;
+
+u32 gv100_bus_read_sw_scratch(struct gk20a *g, u32 index);
+void gv100_bus_write_sw_scratch(struct gk20a *g, u32 index, u32 val);
+
+#endif

drivers/gpu/nvgpu/hal/bus/bus_gv11b.h

@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+#ifndef BUS_GV11B_H
+#define BUS_GV11B_H
+
+struct gk20a;
+
+void gv11b_bus_configure_debug_bus(struct gk20a *g);
+
+#endif /* BUS_GV11B_H */

drivers/gpu/nvgpu/hal/bus/bus_gv11b_fusa.c

@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/gk20a.h>
+#include <nvgpu/io.h>
+
+#include "bus_gv11b.h"
+
+#include <nvgpu/hw/gv11b/hw_bus_gv11b.h>
+
+void gv11b_bus_configure_debug_bus(struct gk20a *g)
+{
+#if !defined(CONFIG_NVGPU_DEBUGGER)
+	nvgpu_writel(g, bus_debug_sel_0_r(), 0U);
+	nvgpu_writel(g, bus_debug_sel_1_r(), 0U);
+	nvgpu_writel(g, bus_debug_sel_2_r(), 0U);
+	nvgpu_writel(g, bus_debug_sel_3_r(), 0U);
+#endif
+}

drivers/gpu/nvgpu/hal/bus/bus_tu104.c

@@ -0,0 +1,101 @@
+/*
+ * Copyright (c) 2018-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/timers.h>
+#include <nvgpu/soc.h>
+#include <nvgpu/io.h>
+#include <nvgpu/mm.h>
+#include <nvgpu/cic.h>
+#include <nvgpu/gk20a.h>
+
+#include "bus_tu104.h"
+
+#include <nvgpu/hw/tu104/hw_bus_tu104.h>
+#include <nvgpu/hw/tu104/hw_func_tu104.h>
+
+int tu104_bus_init_hw(struct gk20a *g)
+{
+	u32 intr_en_mask = 0U;
+
+	nvgpu_cic_intr_stall_unit_config(g, NVGPU_CIC_INTR_UNIT_BUS, NVGPU_CIC_INTR_ENABLE);
+
+	/*
+	 * Note: bus_intr_en_0 is for routing intr to stall tree (mc_intr_0)
+	 * bus_intr_en_1 is for routing bus intr to nostall tree (mc_intr_1)
+	 */
+	if (nvgpu_platform_is_silicon(g) || nvgpu_platform_is_fpga(g)) {
+		intr_en_mask = bus_intr_en_0_pri_squash_m() |
+				bus_intr_en_0_pri_fecserr_m() |
+				bus_intr_en_0_pri_timeout_m();
+	}
+
+	nvgpu_writel(g, bus_intr_en_0_r(), intr_en_mask);
+
+	if (g->ops.bus.configure_debug_bus != NULL) {
+		g->ops.bus.configure_debug_bus(g);
+	}
+
+	return 0;
+}
+
+int bus_tu104_bar2_bind(struct gk20a *g, struct nvgpu_mem *bar2_inst)
+{
+	struct nvgpu_timeout timeout;
+	int err = 0;
+	u64 iova = nvgpu_inst_block_addr(g, bar2_inst);
+	u32 ptr_v = (u32)(iova >> bus_bar2_block_ptr_shift_v());
+
+	nvgpu_log_info(g, "bar2 inst block ptr: 0x%08x", ptr_v);
+
+	err = nvgpu_timeout_init(g, &timeout, 1000, NVGPU_TIMER_RETRY_TIMER);
+	if (err != 0) {
+		return err;
+	}
+
+	nvgpu_func_writel(g, func_priv_bar2_block_r(),
+		     nvgpu_aperture_mask(g, bar2_inst,
+					 bus_bar2_block_target_sys_mem_ncoh_f(),
+					 bus_bar2_block_target_sys_mem_coh_f(),
+					 bus_bar2_block_target_vid_mem_f()) |
+		     bus_bar2_block_mode_virtual_f() |
+		     bus_bar2_block_ptr_f(ptr_v));
+
+	do {
+		u32 val = nvgpu_func_readl(g,
+				func_priv_bind_status_r());
+		bool pending = (bus_bind_status_bar2_pending_v(val) ==
+				bus_bind_status_bar2_pending_busy_v());
+		bool outstanding = (bus_bind_status_bar2_outstanding_v(val) ==
+				  bus_bind_status_bar2_outstanding_true_v());
+		if (!pending && !outstanding) {
+			break;
+		}
+
+		nvgpu_udelay(5);
+	} while (nvgpu_timeout_expired(&timeout) == 0);
+
+	if (nvgpu_timeout_peek_expired(&timeout)) {
+		err = -EINVAL;
+	}
+
+	return err;
+}

drivers/gpu/nvgpu/hal/bus/bus_tu104.h

@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2018-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef NVGPU_BUS_TU104_H
+#define NVGPU_BUS_TU104_H
+
+struct gk20a;
+struct nvgpu_mem;
+
+int tu104_bus_init_hw(struct gk20a *g);
+int bus_tu104_bar2_bind(struct gk20a *g, struct nvgpu_mem *bar2_inst);
+
+#endif /* NVGPU_BUS_TU104_H */

drivers/gpu/nvgpu/hal/cbc/cbc_gm20b.c

@@ -0,0 +1,289 @@
+/*
+ * GM20B CBC
+ *
+ * Copyright (c) 2019-2020 NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/trace.h>
+#include <nvgpu/timers.h>
+#include <nvgpu/enabled.h>
+#include <nvgpu/bug.h>
+#include <nvgpu/ltc.h>
+#include <nvgpu/cbc.h>
+#include <nvgpu/io.h>
+#include <nvgpu/utils.h>
+#include <nvgpu/gk20a.h>
+
+#include <nvgpu/hw/gm20b/hw_ltc_gm20b.h>
+
+#include "cbc_gm20b.h"
+
+int gm20b_cbc_alloc_comptags(struct gk20a *g, struct nvgpu_cbc *cbc)
+{
+	/* max memory size (MB) to cover */
+	u32 max_size = g->max_comptag_mem;
+	/* one tag line covers 128KB */
+	u32 max_comptag_lines = max_size << 3U;
+
+	u32 hw_max_comptag_lines =
+		ltc_ltcs_ltss_cbc_ctrl3_clear_upper_bound_init_v();
+
+	u32 cbc_param =
+		gk20a_readl(g, ltc_ltcs_ltss_cbc_param_r());
+	u32 comptags_per_cacheline =
+		ltc_ltcs_ltss_cbc_param_comptags_per_cache_line_v(cbc_param);
+
+	u32 compbit_backing_size;
+
+	int err;
+
+	nvgpu_log_fn(g, " ");
+
+	if (max_comptag_lines == 0U) {
+		return 0;
+	}
+
+	/* Already initialized */
+	if (cbc->max_comptag_lines != 0U) {
+		return 0;
+	}
+
+	if (max_comptag_lines > hw_max_comptag_lines) {
+		max_comptag_lines = hw_max_comptag_lines;
+	}
+
+	compbit_backing_size =
+		DIV_ROUND_UP(max_comptag_lines, comptags_per_cacheline) *
+			(nvgpu_ltc_get_ltc_count(g) *
+			nvgpu_ltc_get_slices_per_ltc(g) *
+			nvgpu_ltc_get_cacheline_size(g));
+
+	/* aligned to 2KB * ltc_count */
+	compbit_backing_size +=
+		nvgpu_ltc_get_ltc_count(g) <<
+			ltc_ltcs_ltss_cbc_base_alignment_shift_v();
+
+	/* must be a multiple of 64KB */
+	compbit_backing_size = round_up(compbit_backing_size,
+					U32(64) * U32(1024));
+
+	max_comptag_lines =
+		(compbit_backing_size * comptags_per_cacheline) /
+			(nvgpu_ltc_get_ltc_count(g) *
+			nvgpu_ltc_get_slices_per_ltc(g) *
+			nvgpu_ltc_get_cacheline_size(g));
+
+	if (max_comptag_lines > hw_max_comptag_lines) {
+		max_comptag_lines = hw_max_comptag_lines;
+	}
+
+	nvgpu_log_info(g, "compbit backing store size : %d",
+		compbit_backing_size);
+	nvgpu_log_info(g, "max comptag lines : %d",
+		max_comptag_lines);
+
+	err = nvgpu_cbc_alloc(g, compbit_backing_size, false);
+	if (err != 0) {
+		return err;
+	}
+
+	err = gk20a_comptag_allocator_init(g, &cbc->comp_tags,
+						max_comptag_lines);
+	if (err != 0) {
+		return err;
+	}
+
+	cbc->max_comptag_lines = max_comptag_lines;
+	cbc->comptags_per_cacheline = comptags_per_cacheline;
+	cbc->compbit_backing_size = compbit_backing_size;
+
+	return 0;
+}
+
+int gm20b_cbc_ctrl(struct gk20a *g, enum nvgpu_cbc_op op,
+		       u32 min, u32 max)
+{
+	struct nvgpu_timeout timeout;
+	int err = 0;
+	u32 ltc, slice, ctrl1, val, hw_op = 0U;
+	u32 slices_per_ltc = ltc_ltcs_ltss_cbc_param_slices_per_ltc_v(
+				gk20a_readl(g, ltc_ltcs_ltss_cbc_param_r()));
+	u32 ltc_stride = nvgpu_get_litter_value(g, GPU_LIT_LTC_STRIDE);
+	u32 lts_stride = nvgpu_get_litter_value(g, GPU_LIT_LTS_STRIDE);
+	const u32 max_lines = 16384U;
+
+	nvgpu_log_fn(g, " ");
+
+#ifdef CONFIG_NVGPU_TRACE
+	trace_gk20a_ltc_cbc_ctrl_start(g->name, op, min, max);
+#endif
+
+	if (g->cbc->compbit_store.mem.size == 0ULL) {
+		return 0;
+	}
+
+	while (true) {
+		const u32 iter_max = min(min + max_lines - 1U, max);
+		bool full_cache_op = true;
+
+		nvgpu_mutex_acquire(&g->mm.l2_op_lock);
+
+		nvgpu_log_info(g, "clearing CBC lines %u..%u", min, iter_max);
+
+		if (op == nvgpu_cbc_op_clear) {
+			gk20a_writel(
+				g, ltc_ltcs_ltss_cbc_ctrl2_r(),
+				ltc_ltcs_ltss_cbc_ctrl2_clear_lower_bound_f(
+					min));
+			gk20a_writel(
+				g, ltc_ltcs_ltss_cbc_ctrl3_r(),
+				ltc_ltcs_ltss_cbc_ctrl3_clear_upper_bound_f(
+					iter_max));
+			hw_op = ltc_ltcs_ltss_cbc_ctrl1_clear_active_f();
+			full_cache_op = false;
+		} else if (op == nvgpu_cbc_op_clean) {
+			/* this is full-cache op */
+			hw_op = ltc_ltcs_ltss_cbc_ctrl1_clean_active_f();
+		} else if (op == nvgpu_cbc_op_invalidate) {
+			/* this is full-cache op */
+			hw_op = ltc_ltcs_ltss_cbc_ctrl1_invalidate_active_f();
+		} else {
+			nvgpu_err(g, "Unknown op: %u", (unsigned)op);
+			err = -EINVAL;
+			goto out;
+		}
+		gk20a_writel(g, ltc_ltcs_ltss_cbc_ctrl1_r(),
+			     gk20a_readl(g,
+					 ltc_ltcs_ltss_cbc_ctrl1_r()) | hw_op);
+
+		for (ltc = 0; ltc < nvgpu_ltc_get_ltc_count(g); ltc++) {
+			for (slice = 0; slice < slices_per_ltc; slice++) {
+
+				ctrl1 = ltc_ltc0_lts0_cbc_ctrl1_r() +
+					ltc * ltc_stride + slice * lts_stride;
+
+				nvgpu_timeout_init(g, &timeout, 2000,
+						   NVGPU_TIMER_RETRY_TIMER);
+				do {
+					val = gk20a_readl(g, ctrl1);
+					if ((val & hw_op) == 0U) {
+						break;
+					}
+					nvgpu_udelay(5);
+				} while (nvgpu_timeout_expired(&timeout) == 0);
+
+				if (nvgpu_timeout_peek_expired(&timeout)) {
+					nvgpu_err(g, "comp tag clear timeout");
+					err = -EBUSY;
+					goto out;
+				}
+			}
+		}
+
+		/* are we done? */
+		if (full_cache_op || iter_max == max) {
+			break;
+		}
+
+		/* note: iter_max is inclusive upper bound */
+		min = iter_max + 1U;
+
+		/* give a chance for higher-priority threads to progress */
+		nvgpu_mutex_release(&g->mm.l2_op_lock);
+	}
+out:
+#ifdef CONFIG_NVGPU_TRACE
+	trace_gk20a_ltc_cbc_ctrl_done(g->name);
+#endif
+	nvgpu_mutex_release(&g->mm.l2_op_lock);
+	return err;
+}
+
+u32 gm20b_cbc_fix_config(struct gk20a *g, int base)
+{
+	u32 val = gk20a_readl(g, ltc_ltcs_ltss_cbc_num_active_ltcs_r());
+
+	if (val == 2U) {
+		return base * 2;
+	} else if (val != 1U) {
+		nvgpu_err(g, "Invalid number of active ltcs: %08x", val);
+	}
+
+	return base;
+}
+
+
+void gm20b_cbc_init(struct gk20a *g, struct nvgpu_cbc *cbc)
+{
+	u32 max_size = g->max_comptag_mem;
+	u32 max_comptag_lines = max_size << 3U;
+
+	u32 compbit_base_post_divide;
+	u64 compbit_base_post_multiply64;
+	u64 compbit_store_iova;
+	u64 compbit_base_post_divide64;
+
+#ifdef CONFIG_NVGPU_SIM
+	if (nvgpu_is_enabled(g, NVGPU_IS_FMODEL)) {
+		compbit_store_iova = nvgpu_mem_get_phys_addr(g,
+						&cbc->compbit_store.mem);
+	} else
+#endif
+	{
+		compbit_store_iova = nvgpu_mem_get_addr(g,
+						&cbc->compbit_store.mem);
+	}
+
+	compbit_base_post_divide64 = compbit_store_iova >>
+		ltc_ltcs_ltss_cbc_base_alignment_shift_v();
+
+	do_div(compbit_base_post_divide64, nvgpu_ltc_get_ltc_count(g));
+	compbit_base_post_divide = u64_lo32(compbit_base_post_divide64);
+
+	compbit_base_post_multiply64 = ((u64)compbit_base_post_divide *
+		nvgpu_ltc_get_ltc_count(g)) <<
+			ltc_ltcs_ltss_cbc_base_alignment_shift_v();
+
+	if (compbit_base_post_multiply64 < compbit_store_iova) {
+		compbit_base_post_divide++;
+	}
+
+	/* Bug 1477079 indicates sw adjustment on the posted divided base. */
+	if (g->ops.cbc.fix_config != NULL) {
+		compbit_base_post_divide =
+			g->ops.cbc.fix_config(g, compbit_base_post_divide);
+	}
+
+	gk20a_writel(g, ltc_ltcs_ltss_cbc_base_r(),
+		compbit_base_post_divide);
+
+	nvgpu_log(g, gpu_dbg_info | gpu_dbg_map_v | gpu_dbg_pte,
+		   "compbit base.pa: 0x%x,%08x cbc_base:0x%08x\n",
+		   (u32)(compbit_store_iova >> 32),
+		   (u32)(compbit_store_iova & 0xffffffffU),
+		   compbit_base_post_divide);
+
+	cbc->compbit_store.base_hw = compbit_base_post_divide;
+
+	g->ops.cbc.ctrl(g, nvgpu_cbc_op_invalidate,
+			    0, max_comptag_lines - 1U);
+
+}

drivers/gpu/nvgpu/hal/cbc/cbc_gm20b.h

@@ -0,0 +1,48 @@
+/*
+ * GM20B CBC
+ *
+ * Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef NVGPU_CBC_GM20B
+#define NVGPU_CBC_GM20B
+
+#ifdef CONFIG_NVGPU_COMPRESSION
+
+#include <nvgpu/types.h>
+
+struct gk20a;
+struct gpu_ops;
+struct nvgpu_cbc;
+enum nvgpu_cbc_op;
+
+int gm20b_cbc_alloc_comptags(struct gk20a *g, struct nvgpu_cbc *cbc);
+void gm20b_cbc_init(struct gk20a *g, struct nvgpu_cbc *cbc);
+int gm20b_cbc_ctrl(struct gk20a *g, enum nvgpu_cbc_op op,
+		       u32 min, u32 max);
+u32 gm20b_cbc_fix_config(struct gk20a *g, int base);
+int gm20b_cbc_alloc_phys(struct gk20a *g,
+			     size_t compbit_backing_size);
+int gm20b_cbc_alloc_virtc(struct gk20a *g,
+			     size_t compbit_backing_size);
+
+#endif
+#endif

drivers/gpu/nvgpu/hal/cbc/cbc_gp10b.c

@@ -0,0 +1,218 @@
+/*
+ * GP10B CBC
+ *
+ * Copyright (c) 2019-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/trace.h>
+#include <nvgpu/ltc.h>
+#include <nvgpu/cbc.h>
+#include <nvgpu/log.h>
+#include <nvgpu/enabled.h>
+#include <nvgpu/io.h>
+#include <nvgpu/timers.h>
+#include <nvgpu/gk20a.h>
+
+#include <nvgpu/hw/gp10b/hw_ltc_gp10b.h>
+
+#include "cbc_gp10b.h"
+
+int gp10b_cbc_alloc_comptags(struct gk20a *g, struct nvgpu_cbc *cbc)
+{
+	/* max memory size (MB) to cover */
+	u32 max_size = g->max_comptag_mem;
+	/* one tag line covers 64KB */
+	u32 max_comptag_lines = max_size << 4U;
+
+	u32 hw_max_comptag_lines =
+		ltc_ltcs_ltss_cbc_ctrl3_clear_upper_bound_init_v();
+
+	u32 cbc_param =
+		gk20a_readl(g, ltc_ltcs_ltss_cbc_param_r());
+	u32 comptags_per_cacheline =
+		ltc_ltcs_ltss_cbc_param_comptags_per_cache_line_v(cbc_param);
+	u32 cbc_param2 =
+		gk20a_readl(g, ltc_ltcs_ltss_cbc_param2_r());
+	u32 gobs_per_comptagline_per_slice =
+		ltc_ltcs_ltss_cbc_param2_gobs_per_comptagline_per_slice_v(
+								cbc_param2);
+
+	u32 compbit_backing_size;
+
+	int err;
+
+	nvgpu_log_fn(g, " ");
+
+	if (max_comptag_lines == 0U) {
+		return 0;
+	}
+
+	/* Already initialized */
+	if (cbc->max_comptag_lines != 0U) {
+		return 0;
+	}
+
+	if (max_comptag_lines > hw_max_comptag_lines) {
+		max_comptag_lines = hw_max_comptag_lines;
+	}
+
+	compbit_backing_size =
+		round_up(max_comptag_lines * gobs_per_comptagline_per_slice,
+			nvgpu_ltc_get_cacheline_size(g));
+	compbit_backing_size = round_up(
+		compbit_backing_size * nvgpu_ltc_get_slices_per_ltc(g) *
+			nvgpu_ltc_get_ltc_count(g),
+			g->ops.fb.compressible_page_size(g));
+
+	/* aligned to 2KB * ltc_count */
+	compbit_backing_size +=
+		nvgpu_ltc_get_ltc_count(g) <<
+			ltc_ltcs_ltss_cbc_base_alignment_shift_v();
+
+	/* must be a multiple of 64KB */
+	compbit_backing_size = round_up(compbit_backing_size,
+					U32(64) * U32(1024));
+
+	nvgpu_log_info(g, "compbit backing store size : %d",
+		compbit_backing_size);
+	nvgpu_log_info(g, "max comptag lines : %d",
+		max_comptag_lines);
+	nvgpu_log_info(g, "gobs_per_comptagline_per_slice: %d",
+		gobs_per_comptagline_per_slice);
+
+	err = nvgpu_cbc_alloc(g, compbit_backing_size, false);
+	if (err != 0) {
+		return err;
+	}
+
+	err = gk20a_comptag_allocator_init(g, &cbc->comp_tags,
+						max_comptag_lines);
+	if (err != 0) {
+		return err;
+	}
+
+	cbc->max_comptag_lines = max_comptag_lines;
+	cbc->comptags_per_cacheline = comptags_per_cacheline;
+	cbc->gobs_per_comptagline_per_slice = gobs_per_comptagline_per_slice;
+	cbc->compbit_backing_size = compbit_backing_size;
+
+	return 0;
+}
+
+int gp10b_cbc_ctrl(struct gk20a *g, enum nvgpu_cbc_op op,
+		       u32 min, u32 max)
+{
+	struct nvgpu_timeout timeout;
+	int err = 0;
+	u32 ltc, slice, ctrl1, val, hw_op = 0U;
+	u32 ltc_stride = nvgpu_get_litter_value(g, GPU_LIT_LTC_STRIDE);
+	u32 lts_stride = nvgpu_get_litter_value(g, GPU_LIT_LTS_STRIDE);
+	const u32 max_lines = 16384U;
+
+	nvgpu_log_fn(g, " ");
+
+#ifdef CONFIG_NVGPU_TRACE
+	trace_gk20a_ltc_cbc_ctrl_start(g->name, op, min, max);
+#endif
+
+	if (g->cbc->compbit_store.mem.size == 0U) {
+		return 0;
+	}
+
+	while (true) {
+		const u32 iter_max = min(min + max_lines - 1U, max);
+		bool full_cache_op = true;
+
+		nvgpu_mutex_acquire(&g->mm.l2_op_lock);
+
+		nvgpu_log_info(g, "clearing CBC lines %u..%u", min, iter_max);
+
+		if (op == nvgpu_cbc_op_clear) {
+			nvgpu_writel(
+				g, ltc_ltcs_ltss_cbc_ctrl2_r(),
+				ltc_ltcs_ltss_cbc_ctrl2_clear_lower_bound_f(
+					min));
+
+			nvgpu_writel(
+				g, ltc_ltcs_ltss_cbc_ctrl3_r(),
+				ltc_ltcs_ltss_cbc_ctrl3_clear_upper_bound_f(
+					iter_max));
+
+			hw_op = ltc_ltcs_ltss_cbc_ctrl1_clear_active_f();
+			full_cache_op = false;
+		} else if (op == nvgpu_cbc_op_clean) {
+			/* this is full-cache op */
+			hw_op = ltc_ltcs_ltss_cbc_ctrl1_clean_active_f();
+		} else if (op == nvgpu_cbc_op_invalidate) {
+			/* this is full-cache op */
+			hw_op = ltc_ltcs_ltss_cbc_ctrl1_invalidate_active_f();
+		} else {
+			nvgpu_err(g, "Unknown op: %u", (unsigned)op);
+			err = -EINVAL;
+			goto out;
+		}
+		gk20a_writel(g, ltc_ltcs_ltss_cbc_ctrl1_r(),
+			     gk20a_readl(g,
+					 ltc_ltcs_ltss_cbc_ctrl1_r()) | hw_op);
+
+		for (ltc = 0; ltc < nvgpu_ltc_get_ltc_count(g); ltc++) {
+			for (slice = 0; slice <
+				nvgpu_ltc_get_slices_per_ltc(g); slice++) {
+
+				ctrl1 = ltc_ltc0_lts0_cbc_ctrl1_r() +
+					ltc * ltc_stride + slice * lts_stride;
+
+				nvgpu_timeout_init(g, &timeout, 2000,
+						   NVGPU_TIMER_RETRY_TIMER);
+				do {
+					val = gk20a_readl(g, ctrl1);
+					if ((val & hw_op) == 0U) {
+						break;
+					}
+					nvgpu_udelay(5);
+				} while (nvgpu_timeout_expired(&timeout) == 0);
+
+				if (nvgpu_timeout_peek_expired(&timeout)) {
+					nvgpu_err(g, "comp tag clear timeout");
+					err = -EBUSY;
+					goto out;
+				}
+			}
+		}
+
+		/* are we done? */
+		if (full_cache_op || iter_max == max) {
+			break;
+		}
+
+		/* note: iter_max is inclusive upper bound */
+		min = iter_max + 1U;
+
+		/* give a chance for higher-priority threads to progress */
+		nvgpu_mutex_release(&g->mm.l2_op_lock);
+	}
+out:
+#ifdef CONFIG_NVGPU_TRACE
+	trace_gk20a_ltc_cbc_ctrl_done(g->name);
+#endif
+	nvgpu_mutex_release(&g->mm.l2_op_lock);
+	return err;
+}

drivers/gpu/nvgpu/hal/cbc/cbc_gp10b.h

@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef CBC_GP10B_H
+#define CBC_GP10B_H
+
+#ifdef CONFIG_NVGPU_COMPRESSION
+
+struct gk20a;
+struct nvgpu_cbc;
+
+#include <nvgpu/types.h>
+
+int gp10b_cbc_alloc_comptags(struct gk20a *g, struct nvgpu_cbc *cbc);
+int gp10b_cbc_ctrl(struct gk20a *g, enum nvgpu_cbc_op op, u32 min, u32 max);
+
+#endif
+#endif

drivers/gpu/nvgpu/hal/cbc/cbc_gv11b.c

@@ -0,0 +1,45 @@
+/*
+ * GV11B CBC
+ *
+ * Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+
+#include <nvgpu/cbc.h>
+#include <nvgpu/log.h>
+#include <nvgpu/gk20a.h>
+
+#include "cbc_gv11b.h"
+
+void gv11b_cbc_init(struct gk20a *g, struct nvgpu_cbc *cbc)
+{
+	u32 max_size = g->max_comptag_mem;
+	/* one tag line covers 64KB */
+	u32 max_comptag_lines = max_size << 4;
+
+	nvgpu_log_fn(g, " ");
+
+	g->ops.fb.cbc_configure(g, cbc);
+
+	g->ops.cbc.ctrl(g, nvgpu_cbc_op_invalidate,
+			0, max_comptag_lines - 1U);
+
+}

drivers/gpu/nvgpu/hal/cbc/cbc_gv11b.h

@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef CBC_GV11B_H
+#define CBC_GV11B_H
+
+#ifdef CONFIG_NVGPU_COMPRESSION
+
+struct gk20a;
+struct nvgpu_cbc;
+
+void gv11b_cbc_init(struct gk20a *g, struct nvgpu_cbc *cbc);
+
+#endif
+#endif

drivers/gpu/nvgpu/hal/cbc/cbc_tu104.c

@@ -0,0 +1,231 @@
+/*
+ * TU104 CBC
+ *
+ * Copyright (c) 2019-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/types.h>
+#include <nvgpu/ltc.h>
+#include <nvgpu/cbc.h>
+#include <nvgpu/comptags.h>
+#include <nvgpu/io.h>
+#include <nvgpu/timers.h>
+#include <nvgpu/gk20a.h>
+#include <nvgpu/trace.h>
+#include <nvgpu/hw/tu104/hw_ltc_tu104.h>
+
+#include "cbc_tu104.h"
+
+
+u64 tu104_cbc_get_base_divisor(struct gk20a *g)
+{
+	return (u64)nvgpu_ltc_get_ltc_count(g) <<
+		       ltc_ltcs_ltss_cbc_base_alignment_shift_v();
+}
+
+int tu104_cbc_alloc_comptags(struct gk20a *g, struct nvgpu_cbc *cbc)
+{
+	/* max memory size (MB) to cover */
+	u32 max_size = g->max_comptag_mem;
+	/* one tag line covers 64KB */
+	u32 max_comptag_lines = max_size << 4U;
+	u32 compbit_backing_size;
+	u32 hw_max_comptag_lines;
+	u32 cbc_param;
+	u32 ctags_size;
+	u32 ctags_per_cacheline;
+	u32 amap_divide_rounding, amap_swizzle_rounding;
+	int err;
+
+	nvgpu_log_fn(g, " ");
+
+	if (max_comptag_lines == 0U) {
+		return 0;
+	}
+
+	/* Already initialized */
+	if (cbc->max_comptag_lines != 0U) {
+		return 0;
+	}
+
+	hw_max_comptag_lines =
+		ltc_ltcs_ltss_cbc_ctrl3_clear_upper_bound_init_v();
+	if (max_comptag_lines > hw_max_comptag_lines) {
+		max_comptag_lines = hw_max_comptag_lines;
+	}
+
+	cbc_param = nvgpu_readl(g, ltc_ltcs_ltss_cbc_param_r());
+
+	ctags_size = ltc_ltcs_ltss_cbc_param_bytes_per_comptagline_per_slice_v(
+						cbc_param);
+
+	amap_divide_rounding = (U32(2U) * U32(1024U)) <<
+		ltc_ltcs_ltss_cbc_param_amap_divide_rounding_v(cbc_param);
+	amap_swizzle_rounding = (U32(64U) * U32(1024U)) <<
+		ltc_ltcs_ltss_cbc_param_amap_swizzle_rounding_v(cbc_param);
+	ctags_per_cacheline = nvgpu_ltc_get_cacheline_size(g) / ctags_size;
+
+	compbit_backing_size =
+		round_up(max_comptag_lines * ctags_size,
+				nvgpu_ltc_get_cacheline_size(g));
+	compbit_backing_size =
+		compbit_backing_size * nvgpu_ltc_get_slices_per_ltc(g) *
+					nvgpu_ltc_get_ltc_count(g);
+
+	compbit_backing_size += nvgpu_ltc_get_ltc_count(g) *
+					amap_divide_rounding;
+	compbit_backing_size += amap_swizzle_rounding;
+
+	/* must be a multiple of 64KB */
+	compbit_backing_size = round_up(compbit_backing_size,
+					U32(64) * U32(1024));
+
+	err = nvgpu_cbc_alloc(g, compbit_backing_size, true);
+	if (err != 0) {
+		return err;
+	}
+
+	err = gk20a_comptag_allocator_init(g, &cbc->comp_tags,
+						max_comptag_lines);
+	if (err != 0) {
+		return err;
+	}
+
+	cbc->max_comptag_lines = max_comptag_lines;
+	cbc->comptags_per_cacheline = ctags_per_cacheline;
+	cbc->gobs_per_comptagline_per_slice = ctags_size;
+	cbc->compbit_backing_size = compbit_backing_size;
+
+	nvgpu_log_info(g, "compbit backing store size : %d",
+		compbit_backing_size);
+	nvgpu_log_info(g, "max comptag lines : %d",
+		max_comptag_lines);
+	nvgpu_log_info(g, "gobs_per_comptagline_per_slice: %d",
+		cbc->gobs_per_comptagline_per_slice);
+
+	return 0;
+}
+
+int tu104_cbc_ctrl(struct gk20a *g, enum nvgpu_cbc_op op,
+		       u32 min, u32 max)
+{
+	struct nvgpu_timeout timeout;
+	int err = 0;
+	u32 ltc, slice, ctrl1, val, hw_op = 0U;
+	u32 ltc_stride = nvgpu_get_litter_value(g, GPU_LIT_LTC_STRIDE);
+	u32 lts_stride = nvgpu_get_litter_value(g, GPU_LIT_LTS_STRIDE);
+	const u32 max_lines = 16384U;
+
+	nvgpu_log_fn(g, " ");
+
+#ifdef CONFIG_NVGPU_TRACE
+	trace_gk20a_ltc_cbc_ctrl_start(g->name, op, min, max);
+#endif
+
+	if (g->cbc->compbit_store.mem.size == 0U) {
+		return 0;
+	}
+
+	while (true) {
+		const u32 iter_max = min(min + max_lines - 1U, max);
+		bool full_cache_op = true;
+
+		nvgpu_mutex_acquire(&g->mm.l2_op_lock);
+
+		nvgpu_log_info(g, "clearing CBC lines %u..%u", min, iter_max);
+
+		if (op == nvgpu_cbc_op_clear) {
+			nvgpu_writel(
+				g, ltc_ltcs_ltss_cbc_ctrl2_r(),
+				ltc_ltcs_ltss_cbc_ctrl2_clear_lower_bound_f(
+					min));
+			nvgpu_writel(
+				g, ltc_ltcs_ltss_cbc_ctrl3_r(),
+				ltc_ltcs_ltss_cbc_ctrl3_clear_upper_bound_f(
+					iter_max));
+			hw_op = ltc_ltcs_ltss_cbc_ctrl1_clear_active_f();
+			full_cache_op = false;
+		} else if (op == nvgpu_cbc_op_clean) {
+			/* this is full-cache op */
+			hw_op = ltc_ltcs_ltss_cbc_ctrl1_clean_active_f();
+		} else if (op == nvgpu_cbc_op_invalidate) {
+			/* this is full-cache op */
+			hw_op = ltc_ltcs_ltss_cbc_ctrl1_invalidate_active_f();
+		} else {
+			nvgpu_err(g, "Unknown op: %u", (unsigned)op);
+			err = -EINVAL;
+			goto out;
+		}
+
+		nvgpu_writel(g, ltc_ltcs_ltss_cbc_ctrl1_r(),
+			     nvgpu_readl(g,
+					 ltc_ltcs_ltss_cbc_ctrl1_r()) | hw_op);
+
+		for (ltc = 0; ltc < nvgpu_ltc_get_ltc_count(g); ltc++) {
+			for (slice = 0; slice <
+				nvgpu_ltc_get_slices_per_ltc(g); slice++) {
+
+				ctrl1 = ltc_ltc0_lts0_cbc_ctrl1_r() +
+					ltc * ltc_stride + slice * lts_stride;
+
+				nvgpu_timeout_init(g, &timeout, 2000,
+						   NVGPU_TIMER_RETRY_TIMER);
+				do {
+					val = nvgpu_readl(g, ctrl1);
+					if ((val & hw_op) == 0U) {
+						break;
+					}
+					nvgpu_udelay(5);
+				} while (nvgpu_timeout_expired(&timeout) == 0);
+
+				if (nvgpu_timeout_peek_expired(&timeout)) {
+					nvgpu_err(g, "comp tag clear timeout");
+					err = -EBUSY;
+					goto out;
+				}
+			}
+		}
+
+		/* are we done? */
+		if (full_cache_op || iter_max == max) {
+			break;
+		}
+
+		/* note: iter_max is inclusive upper bound */
+		min = iter_max + 1U;
+
+		/* give a chance for higher-priority threads to progress */
+		nvgpu_mutex_release(&g->mm.l2_op_lock);
+	}
+out:
+#ifdef CONFIG_NVGPU_TRACE
+	trace_gk20a_ltc_cbc_ctrl_done(g->name);
+#endif
+	nvgpu_mutex_release(&g->mm.l2_op_lock);
+	return err;
+}
+
+void tu104_cbc_init(struct gk20a *g, struct nvgpu_cbc *cbc)
+{
+	g->ops.fb.cbc_configure(g, cbc);
+	g->ops.cbc.ctrl(g, nvgpu_cbc_op_invalidate,
+			0, cbc->max_comptag_lines - 1U);
+}

drivers/gpu/nvgpu/hal/cbc/cbc_tu104.h

@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef CBC_TU104_H
+#define CBC_TU104_H
+
+#ifdef CONFIG_NVGPU_COMPRESSION
+
+#include <nvgpu/types.h>
+
+enum nvgpu_cbc_op;
+struct gk20a;
+struct nvgpu_cbc;
+
+u64 tu104_cbc_get_base_divisor(struct gk20a *g);
+int tu104_cbc_alloc_comptags(struct gk20a *g, struct nvgpu_cbc *cbc);
+int tu104_cbc_ctrl(struct gk20a *g, enum nvgpu_cbc_op op,
+		       u32 min, u32 max);
+void tu104_cbc_init(struct gk20a *g, struct nvgpu_cbc *cbc);
+
+#endif
+#endif

drivers/gpu/nvgpu/hal/ce/ce2_gk20a.c

@@ -0,0 +1,77 @@
+/*
+ * GK20A Graphics Copy Engine  (gr host)
+ *
+ * Copyright (c) 2011-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/kmem.h>
+#include <nvgpu/dma.h>
+#include <nvgpu/log.h>
+#include <nvgpu/enabled.h>
+#include <nvgpu/io.h>
+#include <nvgpu/utils.h>
+#include <nvgpu/gk20a.h>
+#include <nvgpu/cic.h>
+#include <nvgpu/mc.h>
+#include <nvgpu/channel.h>
+#include <nvgpu/engines.h>
+
+#include "ce2_gk20a.h"
+
+#include <nvgpu/hw/gk20a/hw_ce2_gk20a.h>
+
+void gk20a_ce2_stall_isr(struct gk20a *g, u32 inst_id, u32 pri_base)
+{
+	u32 ce2_intr = nvgpu_readl(g, ce2_intr_status_r());
+	u32 clear_intr = 0U;
+
+	nvgpu_log(g, gpu_dbg_intr, "ce2 isr %08x", ce2_intr);
+
+	/* clear blocking interrupts: they exibit broken behavior */
+	if ((ce2_intr & ce2_intr_status_blockpipe_pending_f()) != 0U) {
+		nvgpu_log(g, gpu_dbg_intr, "ce2 blocking pipe interrupt");
+		clear_intr |= ce2_intr_status_blockpipe_pending_f();
+	}
+
+	if ((ce2_intr & ce2_intr_status_launcherr_pending_f()) != 0U) {
+		nvgpu_log(g, gpu_dbg_intr, "ce2 launch error interrupt");
+		clear_intr |= ce2_intr_status_launcherr_pending_f();
+	}
+
+	nvgpu_writel(g, ce2_intr_status_r(), clear_intr);
+}
+
+u32 gk20a_ce2_nonstall_isr(struct gk20a *g, u32 inst_id, u32 pri_base)
+{
+	u32 ops = 0U;
+	u32 ce2_intr = nvgpu_readl(g, ce2_intr_status_r());
+
+	nvgpu_log(g, gpu_dbg_intr, "ce2 nonstall isr %08x", ce2_intr);
+
+	if ((ce2_intr & ce2_intr_status_nonblockpipe_pending_f()) != 0U) {
+		nvgpu_log(g, gpu_dbg_intr, "ce2 non-blocking pipe interrupt");
+		nvgpu_writel(g, ce2_intr_status_r(),
+			ce2_intr_status_nonblockpipe_pending_f());
+		ops |= (NVGPU_CIC_NONSTALL_OPS_WAKEUP_SEMAPHORE |
+			NVGPU_CIC_NONSTALL_OPS_POST_EVENTS);
+	}
+	return ops;
+}

drivers/gpu/nvgpu/hal/ce/ce2_gk20a.h

@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2011-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+#ifndef NVGPU_CE2_GK20A_H
+#define NVGPU_CE2_GK20A_H
+
+#include <nvgpu/types.h>
+
+struct gk20a;
+
+void gk20a_ce2_stall_isr(struct gk20a *g, u32 inst_id, u32 pri_base);
+u32 gk20a_ce2_nonstall_isr(struct gk20a *g, u32 inst_id, u32 pri_base);
+
+#endif /*NVGPU_CE2_GK20A_H*/

drivers/gpu/nvgpu/hal/ce/ce_gp10b.h

@@ -0,0 +1,34 @@
+/*
+ * Pascal GPU series Copy Engine.
+ *
+ * Copyright (c) 2011-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+#ifndef NVGPU_CE_GP10B_H
+#define NVGPU_CE_GP10B_H
+
+#include <nvgpu/types.h>
+
+struct gk20a;
+
+void gp10b_ce_stall_isr(struct gk20a *g, u32 inst_id, u32 pri_base);
+u32 gp10b_ce_nonstall_isr(struct gk20a *g, u32 inst_id, u32 pri_base);
+
+#endif /* NVGPU_CE_GP10B_H */

drivers/gpu/nvgpu/hal/ce/ce_gp10b_fusa.c

@@ -0,0 +1,77 @@
+/*
+ * Pascal GPU series Copy Engine.
+ *
+ * Copyright (c) 2011-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/io.h>
+#include <nvgpu/gk20a.h>
+#include <nvgpu/mc.h>
+#include <nvgpu/cic.h>
+#include <nvgpu/nvgpu_err.h>
+
+#include "ce_gp10b.h"
+
+#include <nvgpu/hw/gp10b/hw_ce_gp10b.h>
+
+void gp10b_ce_stall_isr(struct gk20a *g, u32 inst_id, u32 pri_base)
+{
+	u32 ce_intr = nvgpu_readl(g, ce_intr_status_r(inst_id));
+	u32 clear_intr = 0U;
+
+	nvgpu_log(g, gpu_dbg_intr, "ce isr %08x %08x", ce_intr, inst_id);
+
+	/* clear blocking interrupts: they exibit broken behavior */
+	if ((ce_intr & ce_intr_status_blockpipe_pending_f()) != 0U) {
+		nvgpu_report_ce_err(g, NVGPU_ERR_MODULE_CE, inst_id,
+				GPU_CE_BLOCK_PIPE, ce_intr);
+		nvgpu_err(g, "ce blocking pipe interrupt");
+		clear_intr |= ce_intr_status_blockpipe_pending_f();
+	}
+
+	if ((ce_intr & ce_intr_status_launcherr_pending_f()) != 0U) {
+		nvgpu_report_ce_err(g, NVGPU_ERR_MODULE_CE, inst_id,
+				GPU_CE_LAUNCH_ERROR, ce_intr);
+		nvgpu_err(g, "ce launch error interrupt");
+		clear_intr |= ce_intr_status_launcherr_pending_f();
+	}
+
+	nvgpu_writel(g, ce_intr_status_r(inst_id), clear_intr);
+	return;
+}
+
+u32 gp10b_ce_nonstall_isr(struct gk20a *g, u32 inst_id, u32 pri_base)
+{
+	u32 nonstall_ops = 0U;
+	u32 ce_intr = nvgpu_readl(g, ce_intr_status_r(inst_id));
+
+	nvgpu_log(g, gpu_dbg_intr, "ce nonstall isr %08x %08x",
+			ce_intr, inst_id);
+
+	if ((ce_intr & ce_intr_status_nonblockpipe_pending_f()) != 0U) {
+		nvgpu_writel(g, ce_intr_status_r(inst_id),
+			ce_intr_status_nonblockpipe_pending_f());
+		nonstall_ops |= (NVGPU_CIC_NONSTALL_OPS_WAKEUP_SEMAPHORE |
+			NVGPU_CIC_NONSTALL_OPS_POST_EVENTS);
+	}
+
+	return nonstall_ops;
+}

drivers/gpu/nvgpu/hal/ce/ce_gv11b.h

@@ -0,0 +1,35 @@
+/*
+ *
+ * Volta GPU series copy engine
+ *
+ * Copyright (c) 2016-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+#ifndef NVGPU_CE_GV11B_H
+#define NVGPU_CE_GV11B_H
+
+struct gk20a;
+
+void gv11b_ce_mthd_buffer_fault_in_bar2_fault(struct gk20a *g);
+u32 gv11b_ce_get_num_pce(struct gk20a *g);
+void gv11b_ce_stall_isr(struct gk20a *g, u32 inst_id, u32 pri_base);
+void gv11b_ce_init_prod_values(struct gk20a *g);
+
+#endif /* NVGPU_CE_GV11B_H */

drivers/gpu/nvgpu/hal/ce/ce_gv11b_fusa.c

@@ -0,0 +1,118 @@
+/*
+ * Volta GPU series Copy Engine.
+ *
+ * Copyright (c) 2016-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/io.h>
+#include <nvgpu/log.h>
+#include <nvgpu/bitops.h>
+#include <nvgpu/gk20a.h>
+#include <nvgpu/nvgpu_err.h>
+
+#include "ce_gp10b.h"
+#include "ce_gv11b.h"
+
+#include <nvgpu/hw/gv11b/hw_ce_gv11b.h>
+
+u32 gv11b_ce_get_num_pce(struct gk20a *g)
+{
+	/*
+	 * register contains a bitmask indicating which physical copy
+	 * engines are present (and not floorswept).
+	 */
+	u32 num_pce;
+	u32 ce_pce_map = nvgpu_readl(g, ce_pce_map_r());
+
+	num_pce = U32(hweight32(ce_pce_map));
+	nvgpu_log_info(g, "num PCE: %d", num_pce);
+	return num_pce;
+}
+
+void gv11b_ce_stall_isr(struct gk20a *g, u32 inst_id, u32 pri_base)
+{
+	u32 ce_intr = nvgpu_readl(g, ce_intr_status_r(inst_id));
+	u32 clear_intr = 0U;
+
+	nvgpu_log(g, gpu_dbg_intr, "ce isr 0x%08x 0x%08x", ce_intr, inst_id);
+
+	/*
+	 * An INVALID_CONFIG interrupt will be generated if a floorswept
+	 * PCE is assigned to a valid LCE in the NV_CE_PCE2LCE_CONFIG
+	 * registers. This is a fatal error and the LCE will have to be
+	 * reset to get back to a working state.
+	 */
+	if ((ce_intr & ce_intr_status_invalid_config_pending_f()) != 0U) {
+		nvgpu_report_ce_err(g, NVGPU_ERR_MODULE_CE, inst_id,
+				GPU_CE_INVALID_CONFIG, ce_intr);
+		nvgpu_err(g, "ce: inst %d: invalid config", inst_id);
+		clear_intr |= ce_intr_status_invalid_config_reset_f();
+	}
+
+	/*
+	 * A MTHD_BUFFER_FAULT interrupt will be triggered if any access
+	 * to a method buffer during context load or save encounters a fault.
+	 * This is a fatal interrupt and will require at least the LCE to be
+	 * reset before operations can start again, if not the entire GPU.
+	 */
+	if ((ce_intr & ce_intr_status_mthd_buffer_fault_pending_f()) != 0U) {
+		nvgpu_report_ce_err(g, NVGPU_ERR_MODULE_CE, inst_id,
+				GPU_CE_METHOD_BUFFER_FAULT, ce_intr);
+		nvgpu_err(g, "ce: inst %d: mthd buffer fault", inst_id);
+		clear_intr |= ce_intr_status_mthd_buffer_fault_reset_f();
+	}
+
+	nvgpu_writel(g, ce_intr_status_r(inst_id), clear_intr);
+
+	gp10b_ce_stall_isr(g, inst_id, pri_base);
+}
+
+void gv11b_ce_mthd_buffer_fault_in_bar2_fault(struct gk20a *g)
+{
+	u32 reg_val, num_lce, lce, clear_intr;
+
+	num_lce = g->ops.top.get_num_lce(g);
+
+	for (lce = 0U; lce < num_lce; lce++) {
+		reg_val = nvgpu_readl(g, ce_intr_status_r(lce));
+		if ((reg_val &
+			ce_intr_status_mthd_buffer_fault_pending_f()) != 0U) {
+			nvgpu_log(g, gpu_dbg_intr,
+			"ce: lce %d: mthd buffer fault", lce);
+			clear_intr = ce_intr_status_mthd_buffer_fault_reset_f();
+			nvgpu_writel(g, ce_intr_status_r(lce), clear_intr);
+		}
+	}
+}
+
+void gv11b_ce_init_prod_values(struct gk20a *g)
+{
+	u32 reg_val;
+	u32 num_lce, lce;
+
+	num_lce = g->ops.top.get_num_lce(g);
+
+	for (lce = 0U; lce < num_lce; lce++) {
+		reg_val = nvgpu_readl(g, ce_lce_opt_r(lce));
+		reg_val |= ce_lce_opt_force_barriers_npl__prod_f();
+		nvgpu_writel(g, ce_lce_opt_r(lce), reg_val);
+	}
+}

drivers/gpu/nvgpu/hal/ce/ce_tu104.c

@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/log.h>
+#include <nvgpu/io.h>
+
+#include "ce_tu104.h"
+
+#include <nvgpu/hw/tu104/hw_ce_tu104.h>
+
+void tu104_ce_set_pce2lce_mapping(struct gk20a *g)
+{
+	/*
+	 * By default GRCE0 and GRCE1 share PCE0.
+	 * Do not change PCE0 config until GRCEs are remapped to PCE1/PCE3.
+	 */
+
+	/* PCE1 (HSHUB) is assigned to LCE4 */
+	nvgpu_writel(g, ce_pce2lce_config_r(1),
+		ce_pce2lce_config_pce_assigned_lce_f(4));
+	/* GRCE1 shares with LCE4 */
+	nvgpu_writel(g, ce_grce_config_r(1),
+		ce_grce_config_shared_lce_f(4) |
+		ce_grce_config_shared_f(1));
+
+	/* PCE2 (FBHUB) is assigned to LCE2 */
+	nvgpu_writel(g, ce_pce2lce_config_r(2),
+		ce_pce2lce_config_pce_assigned_lce_f(2));
+
+	/* PCE3 (FBHUB) is assigned to LCE3 */
+	nvgpu_writel(g, ce_pce2lce_config_r(3),
+		ce_pce2lce_config_pce_assigned_lce_f(3));
+	/* GRCE0 shares with LCE3 */
+	nvgpu_writel(g, ce_grce_config_r(0),
+		ce_grce_config_shared_lce_f(3) |
+		ce_grce_config_shared_f(1));
+
+	/* PCE0 (HSHUB) is unconnected */
+	nvgpu_writel(g, ce_pce2lce_config_r(0),
+		ce_pce2lce_config_pce_assigned_lce_none_f());
+
+}

drivers/gpu/nvgpu/hal/ce/ce_tu104.h

@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+#ifndef NVGPU_CE_TU104_H
+#define NVGPU_CE_TU104_H
+
+struct gk20a;
+
+void tu104_ce_set_pce2lce_mapping(struct gk20a *g);
+
+#endif /* NVGPU_CE_TU104_H */

drivers/gpu/nvgpu/hal/cic/cic_gv11b.h

@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef CIC_GV11B_H
+#define CIC_GV11B_H
+
+#include <nvgpu/nvgpu_err_info.h>
+
+struct gk20a;
+struct nvgpu_cic;
+
+extern struct nvgpu_err_hw_module gv11b_err_lut[];
+extern u32 size_of_gv11b_lut;
+
+int gv11b_cic_init(struct gk20a *g, struct nvgpu_cic *cic);
+
+#endif /* CIC_GV11B_H */

drivers/gpu/nvgpu/hal/cic/cic_gv11b_fusa.c

@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/log.h>
+
+#include "common/cic/cic_priv.h"
+#include "cic_gv11b.h"
+
+int gv11b_cic_init(struct gk20a *g, struct nvgpu_cic *cic)
+{
+	if (cic == NULL) {
+		nvgpu_err(g, "Invalid CIC reference pointer.");
+		return -EINVAL;
+	}
+
+	cic->err_lut = gv11b_err_lut;
+	cic->num_hw_modules = size_of_gv11b_lut;
+	return 0;
+}

drivers/gpu/nvgpu/hal/cic/cic_lut_gv11b_fusa.c

@@ -0,0 +1,599 @@
+/*
+ * Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/nvgpu_err.h>
+#include <nvgpu/nvgpu_err_info.h>
+#include <nvgpu/static_analysis.h>
+#include "common/cic/cic_priv.h"
+#include "cic_gv11b.h"
+
+/*
+ * A flag to enable/disable hw error injection.
+ */
+#ifdef CONFIG_NVGPU_INJECT_HWERR
+#define INJECT_TYPE	(INJECT_HW)
+#else
+#define INJECT_TYPE	(INJECT_SW)
+#endif
+
+/* This look-up table initializes the list of hw units and their errors.
+ * It also specifies the error injection mechanism supported, for each error.
+ * In case of hw error injection support, this initialization will be overriden
+ * by the values provided from the hal layes of corresponding hw units.
+ */
+struct nvgpu_err_hw_module gv11b_err_lut[] = {
+	{
+		.name = "host",
+		.hw_unit = (u32)NVGPU_ERR_MODULE_HOST,
+		.num_instances = 1U,
+		.num_errs = 17U,
+		.errs = (struct nvgpu_err_desc[]) {
+			GPU_CRITERR("pfifo_bind_error",
+					GPU_HOST_PFIFO_BIND_ERROR, INJECT_SW,
+					NULL, nvgpu_inject_host_swerror,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("pfifo_sched_error",
+					GPU_HOST_PFIFO_SCHED_ERROR, INJECT_SW,
+					NULL, nvgpu_inject_host_swerror,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("pfifo_chsw_error",
+					GPU_HOST_PFIFO_CHSW_ERROR, INJECT_SW,
+					NULL, nvgpu_inject_host_swerror,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("pfifo_memop_error",
+					GPU_HOST_PFIFO_MEMOP_TIMEOUT_ERROR,
+					INJECT_SW,
+					NULL, nvgpu_inject_host_swerror,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("pfifo_lb_error",
+					GPU_HOST_PFIFO_LB_ERROR, INJECT_SW,
+					NULL, nvgpu_inject_host_swerror,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("pbus_squash_error",
+					GPU_HOST_PBUS_SQUASH_ERROR,
+					INJECT_SW,
+					NULL, nvgpu_inject_host_swerror,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("pbus_fecs_error",
+					GPU_HOST_PBUS_FECS_ERROR,
+					INJECT_SW,
+					NULL, nvgpu_inject_host_swerror,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("pbus_timeout_error",
+					GPU_HOST_PBUS_TIMEOUT_ERROR,
+					INJECT_SW,
+					NULL, nvgpu_inject_host_swerror,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("pbdma_timeout_error",
+					GPU_HOST_PBDMA_TIMEOUT_ERROR,
+					INJECT_SW,
+					NULL, nvgpu_inject_host_swerror,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("pbdma_extra_error",
+					GPU_HOST_PBDMA_EXTRA_ERROR,
+					INJECT_SW,
+					NULL, nvgpu_inject_host_swerror,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("pbdma_gpfifo_pb_error",
+					GPU_HOST_PBDMA_GPFIFO_PB_ERROR,
+					INJECT_SW,
+					NULL, nvgpu_inject_host_swerror,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("pbdma_method_error",
+					GPU_HOST_PBDMA_METHOD_ERROR,
+					INJECT_SW,
+					NULL, nvgpu_inject_host_swerror,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("pbdma_signature_error",
+					GPU_HOST_PBDMA_SIGNATURE_ERROR,
+					INJECT_SW,
+					NULL, nvgpu_inject_host_swerror,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("pbdma_hce_error",
+					GPU_HOST_PBDMA_HCE_ERROR,
+					INJECT_SW,
+					NULL, nvgpu_inject_host_swerror,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("pbdma_preempt_error",
+					GPU_HOST_PBDMA_PREEMPT_ERROR,
+					INJECT_SW,
+					NULL, nvgpu_inject_host_swerror,
+					NULL, NULL, 0, 0),
+			GPU_NONCRITERR("pfifo_ctxsw_timeout",
+					GPU_HOST_PFIFO_CTXSW_TIMEOUT_ERROR,
+					INJECT_SW,
+					NULL, nvgpu_inject_host_swerror,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("pfifo_fb_flush_timeout",
+					GPU_HOST_PFIFO_FB_FLUSH_TIMEOUT_ERROR,
+					INJECT_SW,
+					NULL, nvgpu_inject_host_swerror,
+					NULL, NULL, 0, 0),
+		},
+	},
+	{
+		.name = "sm",
+		.hw_unit = (u32)NVGPU_ERR_MODULE_SM,
+		.num_instances = 8U,
+		.num_errs = 21U,
+		.errs = (struct nvgpu_err_desc[]) {
+			GPU_NONCRITERR("l1_tag_ecc_corrected",
+					GPU_SM_L1_TAG_ECC_CORRECTED,
+					INJECT_TYPE,
+					NULL, nvgpu_inject_ecc_swerror,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("l1_tag_ecc_uncorrected",
+					GPU_SM_L1_TAG_ECC_UNCORRECTED,
+					INJECT_TYPE,
+					NULL, nvgpu_inject_ecc_swerror,
+					NULL, NULL, 0, 0),
+			GPU_NONCRITERR("cbu_ecc_corrected",
+					0, INJECT_NONE,
+					NULL, NULL,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("cbu_ecc_uncorrected",
+					GPU_SM_CBU_ECC_UNCORRECTED,
+					INJECT_TYPE,
+					NULL, nvgpu_inject_ecc_swerror,
+					NULL, NULL, 0, 0),
+			GPU_NONCRITERR("lrf_ecc_corrected",
+					0, INJECT_NONE,
+					NULL, NULL,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("lrf_ecc_uncorrected",
+					GPU_SM_LRF_ECC_UNCORRECTED,
+					INJECT_TYPE,
+					NULL, nvgpu_inject_ecc_swerror,
+					NULL, NULL, 0, 0),
+			GPU_NONCRITERR("l1_data_ecc_corrected",
+					0, INJECT_NONE,
+					NULL, NULL,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("l1_data_ecc_uncorrected",
+					GPU_SM_L1_DATA_ECC_UNCORRECTED,
+					INJECT_TYPE,
+					NULL, nvgpu_inject_ecc_swerror,
+					NULL, NULL, 0, 0),
+			GPU_NONCRITERR("icache_l0_data_ecc_corrected",
+					0, INJECT_NONE,
+					NULL, NULL,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("icache_l0_data_ecc_uncorrected",
+					GPU_SM_ICACHE_L0_DATA_ECC_UNCORRECTED,
+					INJECT_TYPE,
+					NULL, nvgpu_inject_ecc_swerror,
+					NULL, NULL, 0, 0),
+			GPU_NONCRITERR("icache_l1_data_ecc_corrected",
+					0, INJECT_NONE,
+					NULL, NULL,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("icache_l1_data_ecc_uncorrected",
+					GPU_SM_ICACHE_L1_DATA_ECC_UNCORRECTED,
+					INJECT_SW,
+					NULL, nvgpu_inject_ecc_swerror,
+					NULL, NULL, 0, 0),
+			GPU_NONCRITERR("icache_l0_predecode_ecc_corrected",
+					0, INJECT_NONE,
+					NULL, NULL,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("icache_l0_predecode_ecc_uncorrected",
+					GPU_SM_ICACHE_L0_PREDECODE_ECC_UNCORRECTED,
+					INJECT_SW,
+					NULL, nvgpu_inject_ecc_swerror,
+					NULL, NULL, 0, 0),
+			GPU_NONCRITERR("l1_tag_miss_fifo_ecc_corrected",
+					0, INJECT_NONE,
+					NULL, NULL,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("l1_tag_miss_fifo_ecc_uncorrected",
+					GPU_SM_L1_TAG_MISS_FIFO_ECC_UNCORRECTED,
+					INJECT_SW,
+					NULL, nvgpu_inject_ecc_swerror,
+					NULL, NULL, 0, 0),
+			GPU_NONCRITERR("l1_tag_s2r_pixprf_ecc_corrected",
+					0, INJECT_NONE,
+					NULL, NULL,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("l1_tag_s2r_pixprf_ecc_uncorrected",
+					GPU_SM_L1_TAG_S2R_PIXPRF_ECC_UNCORRECTED,
+					INJECT_SW,
+					NULL, nvgpu_inject_ecc_swerror,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("machine_check_error",
+					GPU_SM_MACHINE_CHECK_ERROR,
+					INJECT_SW,
+					NULL, nvgpu_inject_gr_swerror,
+					NULL, NULL, 0, 0),
+			GPU_NONCRITERR("icache_l1_predecode_ecc_corrected",
+					0, INJECT_NONE,
+					NULL, NULL,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("icache_l1_predecode_ecc_uncorrected",
+					GPU_SM_ICACHE_L1_PREDECODE_ECC_UNCORRECTED,
+					INJECT_SW,
+					NULL, nvgpu_inject_ecc_swerror,
+					NULL, NULL, 0, 0),
+		},
+	},
+	{
+		.name = "fecs",
+		.hw_unit = (u32)NVGPU_ERR_MODULE_FECS,
+		.num_instances = 1U,
+		.num_errs = 8U,
+		.errs = (struct nvgpu_err_desc[]) {
+			GPU_NONCRITERR("falcon_imem_ecc_corrected",
+					GPU_FECS_FALCON_IMEM_ECC_CORRECTED,
+					INJECT_TYPE,
+					NULL, nvgpu_inject_ecc_swerror,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("falcon_imem_ecc_uncorrected",
+					GPU_FECS_FALCON_IMEM_ECC_UNCORRECTED,
+					INJECT_TYPE,
+					NULL, nvgpu_inject_ecc_swerror,
+					NULL, NULL, 0, 0),
+			GPU_NONCRITERR("falcon_dmem_ecc_corrected",
+					0, INJECT_NONE,
+					NULL, NULL,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("falcon_dmem_ecc_uncorrected",
+					GPU_FECS_FALCON_DMEM_ECC_UNCORRECTED,
+					INJECT_SW,
+					NULL, nvgpu_inject_ecc_swerror,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("ctxsw_watchdog_timeout",
+					GPU_FECS_CTXSW_WATCHDOG_TIMEOUT,
+					INJECT_SW,
+					NULL, nvgpu_inject_ctxsw_swerror,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("ctxsw_crc_mismatch",
+					GPU_FECS_CTXSW_CRC_MISMATCH,
+					INJECT_SW,
+					NULL, nvgpu_inject_ctxsw_swerror,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("fault_during_ctxsw",
+					GPU_FECS_FAULT_DURING_CTXSW,
+					INJECT_SW,
+					NULL, nvgpu_inject_ctxsw_swerror,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("ctxsw_init_error",
+					GPU_FECS_CTXSW_INIT_ERROR,
+					INJECT_SW,
+					NULL, nvgpu_inject_ctxsw_swerror,
+					NULL, NULL, 0, 0),
+		},
+	},
+	{
+		.name = "gpccs",
+		.hw_unit = (u32)NVGPU_ERR_MODULE_GPCCS,
+		.num_instances = 1U,
+		.num_errs = 4U,
+		.errs = (struct nvgpu_err_desc[]) {
+			GPU_NONCRITERR("falcon_imem_ecc_corrected",
+					GPU_GPCCS_FALCON_IMEM_ECC_CORRECTED,
+					INJECT_TYPE,
+					NULL, nvgpu_inject_ecc_swerror,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("falcon_imem_ecc_uncorrected",
+					GPU_GPCCS_FALCON_IMEM_ECC_UNCORRECTED,
+					INJECT_TYPE,
+					NULL, nvgpu_inject_ecc_swerror,
+					NULL, NULL, 0, 0),
+			GPU_NONCRITERR("falcon_dmem_ecc_corrected",
+					0, INJECT_NONE,
+					NULL, NULL,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("falcon_dmem_ecc_uncorrected",
+					GPU_GPCCS_FALCON_DMEM_ECC_UNCORRECTED,
+					INJECT_SW,
+					NULL, nvgpu_inject_ecc_swerror,
+					NULL, NULL, 0, 0),
+		},
+	},
+	{
+		.name = "mmu",
+		.hw_unit = (u32)NVGPU_ERR_MODULE_MMU,
+		.num_instances = 1U,
+		.num_errs = 4U,
+		.errs = (struct nvgpu_err_desc[]) {
+			GPU_NONCRITERR("l1tlb_sa_data_ecc_corrected",
+					0, INJECT_NONE,
+					NULL, NULL,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("l1tlb_sa_data_ecc_uncorrected",
+					GPU_MMU_L1TLB_SA_DATA_ECC_UNCORRECTED,
+					INJECT_TYPE,
+					NULL, nvgpu_inject_ecc_swerror,
+					NULL, NULL, 0, 0),
+			GPU_NONCRITERR("l1tlb_fa_data_ecc_corrected",
+					0, INJECT_NONE,
+					NULL, NULL,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("l1tlb_fa_data_ecc_uncorrected",
+					GPU_MMU_L1TLB_FA_DATA_ECC_UNCORRECTED,
+					INJECT_SW,
+					NULL, nvgpu_inject_ecc_swerror,
+					NULL, NULL, 0, 0),
+		},
+	},
+	{
+		.name = "gcc",
+		.hw_unit = (u32)NVGPU_ERR_MODULE_GCC,
+		.num_instances = 1U,
+		.num_errs = 2U,
+		.errs = (struct nvgpu_err_desc[]) {
+			GPU_NONCRITERR("l15_ecc_corrected",
+					0, INJECT_NONE,
+					NULL, NULL,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("l15_ecc_uncorrected",
+					GPU_GCC_L15_ECC_UNCORRECTED,
+					INJECT_TYPE,
+					NULL, nvgpu_inject_ecc_swerror,
+					NULL, NULL, 0, 0),
+		},
+	},
+	{
+		.name = "pmu",
+		.hw_unit = (u32)NVGPU_ERR_MODULE_PMU,
+		.num_instances = 1U,
+		.num_errs = 5U,
+		.errs = (struct nvgpu_err_desc[]) {
+			GPU_NONCRITERR("falcon_imem_ecc_corrected",
+					GPU_PMU_FALCON_IMEM_ECC_CORRECTED,
+					INJECT_SW,
+					NULL, nvgpu_inject_ecc_swerror,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("falcon_imem_ecc_uncorrected",
+					GPU_PMU_FALCON_IMEM_ECC_UNCORRECTED,
+					INJECT_SW,
+					NULL, nvgpu_inject_ecc_swerror,
+					NULL, NULL, 0, 0),
+			GPU_NONCRITERR("falcon_dmem_ecc_corrected",
+					0, INJECT_NONE,
+					NULL, NULL,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("falcon_dmem_ecc_uncorrected",
+					GPU_PMU_FALCON_DMEM_ECC_UNCORRECTED,
+					INJECT_SW,
+					NULL, nvgpu_inject_ecc_swerror,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("bar0_error_timeout",
+					GPU_PMU_BAR0_ERROR_TIMEOUT, INJECT_SW,
+					NULL, nvgpu_inject_pmu_swerror,
+					NULL, NULL, 0, 0),
+		},
+	},
+	{
+		.name = "pgraph",
+		.hw_unit = (u32)NVGPU_ERR_MODULE_PGRAPH,
+		.num_instances = 1U,
+		.num_errs = 12U,
+		.errs = (struct nvgpu_err_desc[]) {
+			GPU_CRITERR("fe_exception",
+					GPU_PGRAPH_FE_EXCEPTION,
+					INJECT_SW,
+					NULL, nvgpu_inject_gr_swerror,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("memfmt_exception",
+					GPU_PGRAPH_MEMFMT_EXCEPTION,
+					INJECT_SW,
+					NULL, nvgpu_inject_gr_swerror,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("pd_exception",
+					GPU_PGRAPH_PD_EXCEPTION,
+					INJECT_SW,
+					NULL, nvgpu_inject_gr_swerror,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("scc_exception",
+					GPU_PGRAPH_SCC_EXCEPTION,
+					INJECT_SW,
+					NULL, nvgpu_inject_gr_swerror,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("ds_exception",
+					GPU_PGRAPH_DS_EXCEPTION,
+					INJECT_SW,
+					NULL, nvgpu_inject_gr_swerror,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("ssync_exception",
+					GPU_PGRAPH_SSYNC_EXCEPTION,
+					INJECT_SW,
+					NULL, nvgpu_inject_gr_swerror,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("mme_exception",
+					GPU_PGRAPH_MME_EXCEPTION,
+					INJECT_SW,
+					NULL, nvgpu_inject_gr_swerror,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("sked_exception",
+					GPU_PGRAPH_SKED_EXCEPTION,
+					INJECT_SW,
+					NULL, nvgpu_inject_gr_swerror,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("be_exception",
+					GPU_PGRAPH_BE_EXCEPTION,
+					INJECT_SW,
+					NULL, nvgpu_inject_gr_swerror,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("mpc_exception",
+					GPU_PGRAPH_MPC_EXCEPTION,
+					INJECT_SW,
+					NULL, nvgpu_inject_gr_swerror,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("illegal_error",
+					GPU_PGRAPH_ILLEGAL_ERROR,
+					INJECT_SW,
+					NULL, nvgpu_inject_gr_swerror,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("gpc_gfx_exception",
+					GPU_PGRAPH_GPC_GFX_EXCEPTION,
+					INJECT_SW,
+					NULL, nvgpu_inject_gr_swerror,
+					NULL, NULL, 0, 0),
+		},
+	},
+	{
+		.name = "ltc",
+		.hw_unit = (u32)NVGPU_ERR_MODULE_LTC,
+		.num_instances = 1U,
+		.num_errs = 8U,
+		.errs = (struct nvgpu_err_desc[]) {
+			GPU_NONCRITERR("cache_dstg_ecc_corrected",
+					GPU_LTC_CACHE_DSTG_ECC_CORRECTED,
+					INJECT_SW,
+					NULL, nvgpu_inject_ecc_swerror,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("cache_dstg_ecc_uncorrected",
+					GPU_LTC_CACHE_DSTG_ECC_UNCORRECTED,
+					INJECT_SW,
+					NULL, nvgpu_inject_ecc_swerror,
+					NULL, NULL, 0, 0),
+			GPU_NONCRITERR("cache_tstg_ecc_corrected",
+					0, INJECT_NONE,
+					NULL, NULL,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("cache_tstg_ecc_uncorrected",
+					GPU_LTC_CACHE_TSTG_ECC_UNCORRECTED,
+					INJECT_SW,
+					NULL, nvgpu_inject_ecc_swerror,
+					NULL, NULL, 0, 0),
+			GPU_NONCRITERR("cache_rstg_ecc_corrected",
+					0, INJECT_NONE,
+					NULL, NULL,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("cache_rstg_ecc_uncorrected",
+					0, INJECT_NONE,
+					NULL, NULL,
+					NULL, NULL, 0, 0),
+			GPU_NONCRITERR("cache_dstg_be_ecc_corrected",
+					0, INJECT_NONE,
+					NULL, NULL,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("cache_dstg_be_ecc_uncorrected",
+					GPU_LTC_CACHE_DSTG_BE_ECC_UNCORRECTED,
+					INJECT_SW,
+					NULL, nvgpu_inject_ecc_swerror,
+					NULL, NULL, 0, 0),
+		},
+	},
+	{
+		.name = "hubmmu",
+		.hw_unit = (u32)NVGPU_ERR_MODULE_HUBMMU,
+		.num_instances = 1U,
+		.num_errs = 9U,
+		.errs = (struct nvgpu_err_desc[]) {
+			GPU_NONCRITERR("hubmmu_l2tlb_sa_data_ecc_corrected",
+					0, INJECT_NONE,
+					NULL, NULL,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("hubmmu_l2tlb_sa_data_ecc_uncorrected",
+					GPU_HUBMMU_L2TLB_SA_DATA_ECC_UNCORRECTED,
+					INJECT_TYPE,
+					NULL, nvgpu_inject_ecc_swerror,
+					NULL, NULL, 0, 0),
+			GPU_NONCRITERR("hubmmu_tlb_sa_data_ecc_corrected",
+					0, INJECT_NONE,
+					NULL, NULL,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("hubmmu_tlb_sa_data_ecc_uncorrected",
+					GPU_HUBMMU_TLB_SA_DATA_ECC_UNCORRECTED,
+					INJECT_TYPE,
+					NULL, nvgpu_inject_ecc_swerror,
+					NULL, NULL, 0, 0),
+			GPU_NONCRITERR("hubmmu_pte_data_ecc_corrected",
+					0, INJECT_NONE,
+					NULL, NULL,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("hubmmu_pte_data_ecc_uncorrected",
+					GPU_HUBMMU_PTE_DATA_ECC_UNCORRECTED,
+					INJECT_TYPE,
+					NULL, nvgpu_inject_ecc_swerror,
+					NULL, NULL, 0, 0),
+			GPU_NONCRITERR("hubmmu_pde0_data_ecc_corrected",
+					0, INJECT_NONE,
+					NULL, NULL,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("hubmmu_pde0_data_ecc_uncorrected",
+					GPU_HUBMMU_PDE0_DATA_ECC_UNCORRECTED,
+					INJECT_SW,
+					NULL, nvgpu_inject_ecc_swerror,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("hubmmu_page_fault_error",
+					GPU_HUBMMU_PAGE_FAULT_ERROR,
+					INJECT_SW,
+					NULL, nvgpu_inject_mmu_swerror,
+					NULL, NULL, 0, 0),
+		},
+	},
+	{
+		.name = "pri",
+		.hw_unit = (u32)NVGPU_ERR_MODULE_PRI,
+		.num_instances = 1U,
+		.num_errs = 2U,
+		.errs = (struct nvgpu_err_desc[]) {
+			GPU_CRITERR("pri_timeout_error",
+					GPU_PRI_TIMEOUT_ERROR,
+					INJECT_SW,
+					NULL, nvgpu_inject_pri_swerror,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("pri_access_violation",
+					GPU_PRI_ACCESS_VIOLATION,
+					INJECT_SW,
+					NULL, nvgpu_inject_pri_swerror,
+					NULL, NULL, 0, 0),
+		},
+	},
+	{
+		.name = "ce",
+		.hw_unit = (u32)NVGPU_ERR_MODULE_CE,
+		.num_instances = 1U,
+		.num_errs = 5U,
+		.errs = (struct nvgpu_err_desc[]) {
+			GPU_CRITERR("ce_launch_error",
+					GPU_CE_LAUNCH_ERROR,
+					INJECT_SW,
+					NULL, nvgpu_inject_ce_swerror,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("ce_block_pipe",
+					GPU_CE_BLOCK_PIPE,
+					INJECT_SW,
+					NULL, nvgpu_inject_ce_swerror,
+					NULL, NULL, 0, 0),
+			GPU_NONCRITERR("ce_nonblock_pipe",
+					0, INJECT_NONE,
+					NULL, NULL,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("ce_invalid_config",
+					GPU_CE_INVALID_CONFIG,
+					INJECT_SW,
+					NULL, nvgpu_inject_ce_swerror,
+					NULL, NULL, 0, 0),
+			GPU_CRITERR("ce_method_buffer_fault",
+					GPU_CE_METHOD_BUFFER_FAULT,
+					INJECT_SW,
+					NULL, nvgpu_inject_ce_swerror,
+					NULL, NULL, 0, 0),
+		},
+	},
+};
+
+u32 size_of_gv11b_lut = sizeof(gv11b_err_lut) /
+					sizeof(struct nvgpu_err_hw_module);

drivers/gpu/nvgpu/hal/class/class_gm20b.c

@@ -0,0 +1,72 @@
+/*
+ * Copyright (c) 2019-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/class.h>
+
+#include "class_gm20b.h"
+
+bool gm20b_class_is_valid_gfx(u32 class_num)
+{
+	if (class_num == MAXWELL_B) {
+		return true;
+	} else {
+		return false;
+	}
+}
+
+bool gm20b_class_is_valid_compute(u32 class_num)
+{
+	if (class_num == MAXWELL_COMPUTE_B) {
+		return true;
+	} else {
+		return false;
+	}
+}
+
+bool gm20b_class_is_valid(u32 class_num)
+{
+	bool valid;
+
+	switch (class_num) {
+	case KEPLER_DMA_COPY_A:
+	case KEPLER_INLINE_TO_MEMORY_B:
+	case MAXWELL_DMA_COPY_A:
+	case MAXWELL_CHANNEL_GPFIFO_A:
+		valid = true;
+		break;
+
+	case MAXWELL_COMPUTE_B:
+		valid = true;
+		break;
+#ifdef CONFIG_NVGPU_GRAPHICS
+	case MAXWELL_B:
+	case FERMI_TWOD_A:
+		valid = true;
+		break;
+#endif
+	default:
+		valid = false;
+		break;
+	}
+
+	return valid;
+}

drivers/gpu/nvgpu/hal/class/class_gm20b.h

@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef NVGPU_CLASS_GM20B
+#define NVGPU_CLASS_GM20B
+
+#include <nvgpu/types.h>
+
+bool gm20b_class_is_valid(u32 class_num);
+bool gm20b_class_is_valid_gfx(u32 class_num);
+bool gm20b_class_is_valid_compute(u32 class_num);
+
+#endif /* NVGPU_CLASS_GM20B */

drivers/gpu/nvgpu/hal/class/class_gp10b.c

@@ -0,0 +1,72 @@
+/*
+ * Copyright (c) 2019 NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/class.h>
+#include <nvgpu/barrier.h>
+
+#include "class_gm20b.h"
+#include "class_gp10b.h"
+
+bool gp10b_class_is_valid(u32 class_num)
+{
+	bool valid;
+
+	nvgpu_speculation_barrier();
+	switch (class_num) {
+	case PASCAL_DMA_COPY_A:
+	case PASCAL_CHANNEL_GPFIFO_A:
+		valid = true;
+		break;
+#ifdef CONFIG_NVGPU_GRAPHICS
+	case PASCAL_A:
+		valid = true;
+		break;
+#endif
+	case PASCAL_COMPUTE_A:
+		valid = true;
+		break;
+	default:
+		valid = gm20b_class_is_valid(class_num);
+		break;
+	}
+	return valid;
+}
+
+#ifdef CONFIG_NVGPU_GRAPHICS
+bool gp10b_class_is_valid_gfx(u32 class_num)
+{
+	if (class_num == PASCAL_A ||  class_num == MAXWELL_B) {
+		return true;
+	} else {
+		return false;
+	}
+}
+#endif
+
+bool gp10b_class_is_valid_compute(u32 class_num)
+{
+	if (class_num == PASCAL_COMPUTE_A ||  class_num == MAXWELL_COMPUTE_B) {
+		return true;
+	} else {
+		return false;
+	}
+}

drivers/gpu/nvgpu/hal/class/class_gp10b.h

@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef NVGPU_CLASS_GP10B
+#define NVGPU_CLASS_GP10B
+
+#include <nvgpu/types.h>
+
+bool gp10b_class_is_valid(u32 class_num);
+bool gp10b_class_is_valid_compute(u32 class_num);
+
+#ifdef CONFIG_NVGPU_GRAPHICS
+bool gp10b_class_is_valid_gfx(u32 class_num);
+#endif
+
+#endif /* NVGPU_CLASS_GP10B */

drivers/gpu/nvgpu/hal/class/class_gv11b.h

@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef NVGPU_CLASS_GV11B
+#define NVGPU_CLASS_GV11B
+
+#include <nvgpu/types.h>
+
+bool gv11b_class_is_valid(u32 class_num);
+bool gv11b_class_is_valid_compute(u32 class_num);
+
+#ifdef CONFIG_NVGPU_GRAPHICS
+bool gv11b_class_is_valid_gfx(u32 class_num);
+#endif
+
+#endif /* NVGPU_CLASS_GV11B */

drivers/gpu/nvgpu/hal/class/class_gv11b_fusa.c

@@ -0,0 +1,89 @@
+/*
+ * Copyright (c) 2019-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/class.h>
+#include <nvgpu/barrier.h>
+
+#ifdef CONFIG_NVGPU_HAL_NON_FUSA
+#include "class_gp10b.h"
+#endif
+#include "class_gv11b.h"
+
+bool gv11b_class_is_valid(u32 class_num)
+{
+	bool valid;
+
+	nvgpu_speculation_barrier();
+
+	switch (class_num) {
+	case VOLTA_COMPUTE_A:
+	case VOLTA_DMA_COPY_A:
+	case VOLTA_CHANNEL_GPFIFO_A:
+		valid = true;
+		break;
+#ifdef CONFIG_NVGPU_GRAPHICS
+	case VOLTA_A:
+		valid = true;
+		break;
+#endif
+	default:
+#ifdef CONFIG_NVGPU_HAL_NON_FUSA
+		valid = gp10b_class_is_valid(class_num);
+#else
+		valid = false;
+#endif
+		break;
+	}
+	return valid;
+}
+
+#ifdef CONFIG_NVGPU_GRAPHICS
+bool gv11b_class_is_valid_gfx(u32 class_num)
+{
+	bool valid;
+
+	nvgpu_speculation_barrier();
+
+	switch (class_num) {
+	case VOLTA_A:
+		valid = true;
+		break;
+	default:
+#ifdef CONFIG_NVGPU_HAL_NON_FUSA
+		valid = gp10b_class_is_valid_gfx(class_num);
+#else
+		valid = false;
+#endif
+		break;
+	}
+	return valid;
+}
+#endif
+
+bool gv11b_class_is_valid_compute(u32 class_num)
+{
+	if (class_num == VOLTA_COMPUTE_A) {
+		return true;
+	} else {
+		return false;
+	}
+}

drivers/gpu/nvgpu/hal/class/class_tu104.c

@@ -0,0 +1,85 @@
+/*
+ * Copyright (c) 2019 NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/class.h>
+#include <nvgpu/barrier.h>
+
+#include "class_gv11b.h"
+#include "class_tu104.h"
+
+bool tu104_class_is_valid(u32 class_num)
+{
+	bool valid;
+
+	nvgpu_speculation_barrier();
+
+	switch (class_num) {
+	case TURING_CHANNEL_GPFIFO_A:
+	case TURING_COMPUTE_A:
+	case TURING_DMA_COPY_A:
+#ifdef CONFIG_NVGPU_GRAPHICS
+	case TURING_A:
+#endif
+		valid = true;
+		break;
+	default:
+		valid = gv11b_class_is_valid(class_num);
+		break;
+	}
+	return valid;
+};
+
+#ifdef CONFIG_NVGPU_GRAPHICS
+bool tu104_class_is_valid_gfx(u32 class_num)
+{
+	bool valid;
+
+	nvgpu_speculation_barrier();
+
+	switch (class_num) {
+	case TURING_A:
+		valid = true;
+		break;
+	default:
+		valid = gv11b_class_is_valid_gfx(class_num);
+		break;
+	}
+	return valid;
+}
+#endif
+
+bool tu104_class_is_valid_compute(u32 class_num)
+{
+	bool valid;
+
+	nvgpu_speculation_barrier();
+
+	switch (class_num) {
+	case TURING_COMPUTE_A:
+		valid = true;
+		break;
+	default:
+		valid = gv11b_class_is_valid_compute(class_num);
+		break;
+	}
+	return valid;
+}

drivers/gpu/nvgpu/hal/class/class_tu104.h

@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef NVGPU_CLASS_TU104
+#define NVGPU_CLASS_TU104
+
+#include <nvgpu/types.h>
+
+bool tu104_class_is_valid(u32 class_num);
+bool tu104_class_is_valid_compute(u32 class_num);
+
+#ifdef CONFIG_NVGPU_GRAPHICS
+bool tu104_class_is_valid_gfx(u32 class_num);
+#endif
+
+#endif /* NVGPU_CLASS_TU104 */

drivers/gpu/nvgpu/hal/clk/clk_gk20a.h

@@ -0,0 +1,132 @@
+/*
+ * Copyright (c) 2011 - 2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+#ifndef CLK_GK20A_H
+#define CLK_GK20A_H
+
+#include <nvgpu/lock.h>
+
+#if defined(CONFIG_COMMON_CLK)
+#include <linux/clk-provider.h>
+#endif
+
+#define GPUFREQ_TABLE_END     ~(u32)1
+enum gpc_pll_id {
+	/* only one PLL for gk20a */
+	GK20A_GPC_PLL = 0,
+	/* 2 PLL revisions for gm20b */
+	GM20B_GPC_PLL_B1,
+	GM20B_GPC_PLL_C1,
+};
+
+enum gpc_pll_mode {
+	GPC_PLL_MODE_F = 0,	/* fixed frequency mode a.k.a legacy mode */
+	GPC_PLL_MODE_DVFS,	/* DVFS mode a.k.a NA mode */
+};
+
+struct na_dvfs {
+	u32 n_int;
+	u32 sdm_din;
+	int dfs_coeff;
+	int dfs_det_max;
+	int dfs_ext_cal;
+	int uv_cal;
+	int mv;
+};
+
+struct pll {
+	enum gpc_pll_id id;
+	u32 clk_in;	/* KHz */
+	u32 M;
+	u32 N;
+	u32 PL;
+	u32 freq;	/* KHz */
+	bool enabled;
+	enum gpc_pll_mode mode;
+	struct na_dvfs dvfs;
+};
+
+struct pll_parms {
+	u32 min_freq, max_freq;	/* KHz */
+	u32 min_vco, max_vco;	/* KHz */
+	u32 min_u,   max_u;	/* KHz */
+	u32 min_M,   max_M;
+	u32 min_N,   max_N;
+	u32 min_PL,  max_PL;
+	/* NA mode parameters*/
+	int coeff_slope, coeff_offs; /* coeff = slope * V + offs */
+	int uvdet_slope, uvdet_offs; /* uV = slope * det + offs */
+	u32 vco_ctrl;
+	/*
+	 * Timing parameters in us. Lock timeout is applied to locking in fixed
+	 * frequency mode and to dynamic ramp in any mode; does not affect lock
+	 * latency, since lock/ramp done status bit is polled. NA mode lock and
+	 * and IDDQ exit delays set the time of the respective opertaions with
+	 * no status polling.
+	 */
+	u32 lock_timeout;
+	u32 na_lock_delay;
+	u32 iddq_exit_delay;
+	/* NA mode DFS control */
+	u32 dfs_ctrl;
+};
+
+struct namemap_cfg;
+struct clk_domains_mon_status_params;
+
+struct clk_gk20a {
+	struct gk20a *g;
+#if defined(CONFIG_COMMON_CLK)
+	struct clk *tegra_clk;
+	struct clk *tegra_clk_parent;
+	struct clk_hw hw;
+#endif
+	struct pll gpc_pll;
+	struct pll gpc_pll_last;
+	struct nvgpu_mutex clk_mutex;
+	struct namemap_cfg *clk_namemap;
+	struct clk_domains_mon_status_params *clk_mon_status;
+	u32 namemap_num;
+	u32 *namemap_xlat_table;
+	bool sw_ready;
+	bool clk_hw_on;
+	bool debugfs_set;
+	int pll_poweron_uv;
+	unsigned long dvfs_safe_max_freq;
+};
+
+struct gpu_ops;
+
+#define KHZ (1000U)
+#define MHZ (1000000U)
+
+static inline unsigned long rate_gpc2clk_to_gpu(unsigned long rate)
+{
+	/* convert the kHz gpc2clk frequency to Hz gpcpll frequency */
+	return (rate * KHZ) / 2U;
+}
+static inline unsigned long rate_gpu_to_gpc2clk(unsigned long rate)
+{
+	/* convert the Hz gpcpll frequency to kHz gpc2clk frequency */
+	return (rate * 2U) / KHZ;
+}
+
+#endif /* CLK_GK20A_H */

drivers/gpu/nvgpu/hal/clk/clk_gm20b.h

@@ -0,0 +1,93 @@
+/*
+ * GM20B Graphics
+ *
+ * Copyright (c) 2014-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+#ifndef NVGPU_GM20B_CLK_GM20B_H
+#define NVGPU_GM20B_CLK_GM20B_H
+
+#include <nvgpu/lock.h>
+
+struct gk20a;
+struct clk_gk20a;
+
+struct nvgpu_clk_pll_debug_data {
+	u32 trim_sys_sel_vco_reg;
+	u32 trim_sys_sel_vco_val;
+
+	u32 trim_sys_gpc2clk_out_reg;
+	u32 trim_sys_gpc2clk_out_val;
+
+	u32 trim_sys_bypassctrl_reg;
+	u32 trim_sys_bypassctrl_val;
+
+	u32 trim_sys_gpcpll_cfg_reg;
+	u32 trim_sys_gpcpll_dvfs2_reg;
+	u32 trim_bcast_gpcpll_dvfs2_reg;
+
+	u32 trim_sys_gpcpll_cfg_val;
+	bool trim_sys_gpcpll_cfg_enabled;
+	bool trim_sys_gpcpll_cfg_locked;
+	bool trim_sys_gpcpll_cfg_sync_on;
+
+	u32 trim_sys_gpcpll_coeff_val;
+	u32 trim_sys_gpcpll_coeff_mdiv;
+	u32 trim_sys_gpcpll_coeff_ndiv;
+	u32 trim_sys_gpcpll_coeff_pldiv;
+
+	u32 trim_sys_gpcpll_dvfs0_val;
+	u32 trim_sys_gpcpll_dvfs0_dfs_coeff;
+	u32 trim_sys_gpcpll_dvfs0_dfs_det_max;
+	u32 trim_sys_gpcpll_dvfs0_dfs_dc_offset;
+};
+
+int gm20b_init_clk_setup_sw(struct gk20a *g);
+
+int gm20b_clk_prepare(struct clk_gk20a *clk);
+void gm20b_clk_unprepare(struct clk_gk20a *clk);
+int gm20b_clk_is_prepared(struct clk_gk20a *clk);
+unsigned long gm20b_recalc_rate(struct clk_gk20a *clk, unsigned long parent_rate);
+int gm20b_gpcclk_set_rate(struct clk_gk20a *clk, unsigned long rate,
+		unsigned long parent_rate);
+long gm20b_round_rate(struct clk_gk20a *clk, unsigned long rate,
+		unsigned long *parent_rate);
+struct pll_parms *gm20b_get_gpc_pll_parms(void);
+
+int gm20b_clk_pll_reg_write(struct gk20a *g, u32 reg, u32 val);
+int gm20b_init_clk_support(struct gk20a *g);
+void gm20b_suspend_clk_support(struct gk20a *g);
+int gm20b_clk_get_voltage(struct clk_gk20a *clk, u64 *val);
+int gm20b_clk_get_gpcclk_clock_counter(struct clk_gk20a *clk, u64 *val);
+int gm20b_clk_get_pll_debug_data(struct gk20a *g,
+			struct nvgpu_clk_pll_debug_data *d);
+
+/* 1:1 match between post divider settings and divisor value */
+static inline u32 nvgpu_pl_to_div(u32 pl)
+{
+	return pl;
+}
+
+static inline u32 nvgpu_div_to_pl(u32 divisor)
+{
+	return divisor;
+}
+
+#endif /* NVGPU_GM20B_CLK_GM20B_H */

drivers/gpu/nvgpu/hal/clk/clk_mon_tu104.c

@@ -0,0 +1,298 @@
+/*
+ * TU104 Clocks Monitor
+ *
+ * Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/kmem.h>
+#include <nvgpu/io.h>
+#include <nvgpu/list.h>
+#include <nvgpu/soc.h>
+#include <nvgpu/gk20a.h>
+#include <nvgpu/clk_mon.h>
+#include <nvgpu/hw/tu104/hw_trim_tu104.h>
+
+#include "clk_mon_tu104.h"
+/**
+ * Mapping between the clk domain and the various clock monitor registers
+ * The rows represent clock domains starting from index 0 and column represent
+ * the various registers each domain has, non available domains are set to 0
+ * for easy accessing, refer nvgpu_clk_mon_init_domains() for valid domains.
+ */
+static  u32 clock_mon_map_tu104[CLK_CLOCK_MON_DOMAIN_COUNT]
+				[CLK_CLOCK_MON_REG_TYPE_COUNT] = {
+	{
+		trim_gpcclk_fault_threshold_high_r(),
+		trim_gpcclk_fault_threshold_low_r(),
+		trim_gpcclk_fault_status_r(),
+		trim_gpcclk_fault_priv_level_mask_r(),
+	},
+	{
+		trim_xbarclk_fault_threshold_high_r(),
+		trim_xbarclk_fault_threshold_low_r(),
+		trim_xbarclk_fault_status_r(),
+		trim_xbarclk_fault_priv_level_mask_r(),
+	},
+	{
+		trim_sysclk_fault_threshold_high_r(),
+		trim_sysclk_fault_threshold_low_r(),
+		trim_sysclk_fault_status_r(),
+		trim_sysclk_fault_priv_level_mask_r(),
+	},
+	{
+		trim_hubclk_fault_threshold_high_r(),
+		trim_hubclk_fault_threshold_low_r(),
+		trim_hubclk_fault_status_r(),
+		trim_hubclk_fault_priv_level_mask_r(),
+	},
+	{
+		trim_dramclk_fault_threshold_high_r(),
+		trim_dramclk_fault_threshold_low_r(),
+		trim_dramclk_fault_status_r(),
+		trim_dramclk_fault_priv_level_mask_r(),
+	},
+	{
+		trim_hostclk_fault_threshold_high_r(),
+		trim_hostclk_fault_threshold_low_r(),
+		trim_hostclk_fault_status_r(),
+		trim_hostclk_fault_priv_level_mask_r(),
+	},
+	{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0},
+	{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0},
+	{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0},
+	{
+		trim_utilsclk_fault_threshold_high_r(),
+		trim_utilsclk_fault_threshold_low_r(),
+		trim_utilsclk_fault_status_r(),
+		trim_utilsclk_fault_priv_level_mask_r(),
+	},
+	{
+		trim_pwrclk_fault_threshold_high_r(),
+		trim_pwrclk_fault_threshold_low_r(),
+		trim_pwrclk_fault_status_r(),
+		trim_pwrclk_fault_priv_level_mask_r(),
+	},
+	{
+		trim_nvdclk_fault_threshold_high_r(),
+		trim_nvdclk_fault_threshold_low_r(),
+		trim_nvdclk_fault_status_r(),
+		trim_nvdclk_fault_priv_level_mask_r(),
+	},
+	{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0},
+	{
+		trim_xclk_fault_threshold_high_r(),
+		trim_xclk_fault_threshold_low_r(),
+		trim_xclk_fault_status_r(),
+		trim_xclk_fault_priv_level_mask_r(),
+	},
+	{
+		trim_nvl_commonclk_fault_threshold_high_r(),
+		trim_nvl_commonclk_fault_threshold_low_r(),
+		trim_nvl_commonclk_fault_status_r(),
+		trim_nvl_commonclk_fault_priv_level_mask_r(),
+	},
+	{
+		trim_pex_refclk_fault_threshold_high_r(),
+		trim_pex_refclk_fault_threshold_low_r(),
+		trim_pex_refclk_fault_status_r(),
+		trim_pex_refclk_fault_priv_level_mask_r(),
+	},
+	{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}
+};
+
+static u32 nvgpu_check_for_dc_fault(u32 data)
+{
+	return (trim_fault_status_dc_v(data) ==
+		trim_fault_status_dc_true_v()) ?
+				trim_fault_status_dc_m() : 0U;
+}
+
+static u32 nvgpu_check_for_lower_threshold_fault(u32 data)
+{
+	return (trim_fault_status_lower_threshold_v(data) ==
+		trim_fault_status_lower_threshold_true_v()) ?
+				trim_fault_status_lower_threshold_m() : 0U;
+}
+
+static u32 nvgpu_check_for_higher_threshold_fault(u32 data)
+{
+	return (trim_fault_status_higher_threshold_v(data) ==
+		trim_fault_status_higher_threshold_true_v()) ?
+				trim_fault_status_higher_threshold_m() : 0U;
+}
+
+static u32 nvgpu_check_for_overflow_err(u32 data)
+{
+	return (trim_fault_status_overflow_v(data) ==
+		trim_fault_status_overflow_true_v()) ?
+				trim_fault_status_overflow_m() : 0U;
+}
+
+static int nvgpu_clk_mon_get_fault(struct gk20a *g, u32 i, u32 data,
+		struct clk_domains_mon_status_params *clk_mon_status)
+{
+	u32 reg_address;
+	int status = 0;
+
+	/* Fields for faults are same for all clock domains */
+	clk_mon_status->clk_mon_list[i].clk_domain_fault_status =
+		((nvgpu_check_for_dc_fault(data)) |
+		(nvgpu_check_for_lower_threshold_fault(data)) |
+		(nvgpu_check_for_higher_threshold_fault(data)) |
+		(nvgpu_check_for_overflow_err(data)));
+	nvgpu_err(g, "FMON faulted domain 0x%x value 0x%x",
+		clk_mon_status->clk_mon_list[i].clk_api_domain,
+		clk_mon_status->clk_mon_list[i].
+		clk_domain_fault_status);
+
+	/* Get the low threshold limit */
+	reg_address = clock_mon_map_tu104[i][FMON_THRESHOLD_LOW];
+	data = nvgpu_readl(g, reg_address);
+	clk_mon_status->clk_mon_list[i].low_threshold =
+		trim_fault_threshold_low_count_v(data);
+
+	/* Get the high threshold limit */
+	reg_address = clock_mon_map_tu104[i][FMON_THRESHOLD_HIGH];
+	data = nvgpu_readl(g, reg_address);
+	clk_mon_status->clk_mon_list[i].high_threshold =
+		trim_fault_threshold_high_count_v(data);
+
+	return status;
+}
+
+bool tu104_clk_mon_check_master_fault_status(struct gk20a *g)
+{
+	u32 fmon_master_status = nvgpu_readl(g, trim_fmon_master_status_r());
+
+	if (trim_fmon_master_status_fault_out_v(fmon_master_status) ==
+			trim_fmon_master_status_fault_out_true_v()) {
+		return true;
+	}
+	return false;
+}
+
+int nvgpu_clk_mon_alloc_memory(struct gk20a *g)
+{
+	struct clk_gk20a *clk = &g->clk;
+
+	/* If already allocated, do not re-allocate */
+	if (clk->clk_mon_status != NULL) {
+		return 0;
+	}
+
+	clk->clk_mon_status = nvgpu_kzalloc(g,
+			sizeof(struct clk_domains_mon_status_params));
+	if (clk->clk_mon_status == NULL) {
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+int tu104_clk_mon_check_status(struct gk20a *g, u32 domain_mask)
+{
+	u32 reg_address, bit_pos;
+	u32 data;
+	int status;
+	struct clk_domains_mon_status_params *clk_mon_status;
+
+	clk_mon_status = g->clk.clk_mon_status;
+	clk_mon_status->clk_mon_domain_mask = domain_mask;
+	/*
+	 * Parse through each domain and check for faults, each bit set
+	 * represents a domain here
+	 */
+	for (bit_pos = 0U; bit_pos < (sizeof(domain_mask) * BITS_PER_BYTE);
+			bit_pos++) {
+		if (nvgpu_test_bit(bit_pos, (void *)&domain_mask)) {
+			clk_mon_status->clk_mon_list[bit_pos].clk_api_domain =
+					BIT(bit_pos);
+
+			reg_address = clock_mon_map_tu104[bit_pos]
+					[FMON_FAULT_STATUS];
+			data = nvgpu_readl(g, reg_address);
+
+			clk_mon_status->clk_mon_list[bit_pos].
+				clk_domain_fault_status = 0U;
+			/* Check FMON fault status, field is same for all */
+			if (trim_fault_status_fault_out_v(data) ==
+				trim_fault_status_fault_out_true_v()) {
+				status = nvgpu_clk_mon_get_fault(g, bit_pos,
+						data, clk_mon_status);
+				if (status != 0) {
+					nvgpu_err(g, "Failed to get status");
+					return -EINVAL;
+				}
+			}
+		}
+	}
+	return 0;
+}
+
+bool tu104_clk_mon_check_clk_good(struct gk20a *g)
+{
+	u32 clk_status = nvgpu_readl(g, trim_xtal4x_cfg5_r());
+
+	if (trim_xtal4x_cfg5_curr_state_v(clk_status) !=
+			trim_xtal4x_cfg5_curr_state_good_v()) {
+		return true;
+	}
+	return false;
+}
+
+bool tu104_clk_mon_check_pll_lock(struct gk20a *g)
+{
+	u32 clk_status = nvgpu_readl(g, trim_xtal4x_cfg_r());
+
+	/* check xtal4 */
+	if (trim_xtal4x_cfg_pll_lock_v(clk_status) !=
+			trim_xtal4x_cfg_pll_lock_true_v()) {
+		return true;
+	}
+
+	/* check mem pll */
+	clk_status = nvgpu_readl(g, trim_mem_pll_status_r());
+
+	if (trim_mem_pll_status_dram_curr_state_v(clk_status) !=
+			trim_mem_pll_status_dram_curr_state_good_v()) {
+		return true;
+	}
+	if (trim_mem_pll_status_refm_curr_state_v(clk_status) !=
+			trim_mem_pll_status_refm_curr_state_good_v()) {
+		return true;
+	}
+
+	/* check sppll0,1 */
+	clk_status = nvgpu_readl(g, trim_sppll0_cfg_r());
+
+	if (trim_sppll0_cfg_curr_state_v(clk_status) !=
+			trim_sppll0_cfg_curr_state_good_v()) {
+		return true;
+	}
+	clk_status = nvgpu_readl(g, trim_sppll1_cfg_r());
+
+	if (trim_sppll1_cfg_curr_state_v(clk_status) !=
+			trim_sppll1_cfg_curr_state_good_v()) {
+		return true;
+	}
+	return false;
+}
+

drivers/gpu/nvgpu/hal/clk/clk_mon_tu104.h

@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+#ifndef CLK_MON_TU104_H
+#define CLK_MON_TU104_H
+
+/**
+ * FMON register types
+ */
+#define FMON_THRESHOLD_HIGH				0x0U
+#define FMON_THRESHOLD_LOW				0x1U
+#define FMON_FAULT_STATUS				0x2U
+#define FMON_FAULT_STATUS_PRIV_MASK			0x3U
+#define CLK_CLOCK_MON_REG_TYPE_COUNT			0x4U
+#define CLK_MON_BITS_PER_BYTE				0x8U
+/*
+ * The Maximum count of clock domains supported
+ */
+#define CLK_CLOCK_MON_DOMAIN_COUNT			0x32U
+
+
+struct clk_domain_mon_status {
+	u32 clk_api_domain;
+	u32 low_threshold;
+	u32 high_threshold;
+	u32 clk_domain_fault_status;
+};
+
+struct clk_domains_mon_status_params {
+	u32 clk_mon_domain_mask;
+	struct clk_domain_mon_status
+		clk_mon_list[CLK_CLOCK_MON_DOMAIN_COUNT];
+};
+
+bool tu104_clk_mon_check_master_fault_status(struct gk20a *g);
+int tu104_clk_mon_check_status(struct gk20a *g, u32 domain_mask);
+bool tu104_clk_mon_check_clk_good(struct gk20a *g);
+bool tu104_clk_mon_check_pll_lock(struct gk20a *g);
+
+#endif /* CLK_MON_TU104_H */

drivers/gpu/nvgpu/hal/clk/clk_tu104.c

@@ -0,0 +1,388 @@
+/*
+ * TU104 Clocks
+ *
+ * Copyright (c) 2016-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifdef CONFIG_DEBUG_FS
+#include <linux/debugfs.h>
+#include "os/linux/os_linux.h"
+#endif
+
+#include <nvgpu/kmem.h>
+#include <nvgpu/io.h>
+#include <nvgpu/list.h>
+#include <nvgpu/pmu/clk/clk.h>
+#include <nvgpu/soc.h>
+#include <nvgpu/gk20a.h>
+#include <nvgpu/pmu/perf.h>
+#include <nvgpu/clk_arb.h>
+#include <nvgpu/pmu/volt.h>
+#include <nvgpu/hw/tu104/hw_trim_tu104.h>
+
+#include "clk_tu104.h"
+
+
+
+#define CLK_NAMEMAP_INDEX_GPCCLK	0x00
+#define CLK_NAMEMAP_INDEX_XBARCLK	0x02
+#define CLK_NAMEMAP_INDEX_SYSCLK	0x07	/* SYSPLL */
+#define CLK_NAMEMAP_INDEX_DRAMCLK	0x20	/* DRAMPLL */
+
+#define CLK_DEFAULT_CNTRL_SETTLE_RETRIES 10
+#define CLK_DEFAULT_CNTRL_SETTLE_USECS   5
+
+#define XTAL_CNTR_CLKS		27000	/* 1000usec at 27KHz XTAL */
+#define XTAL_CNTR_DELAY		10000	/* we need acuracy up to the 10ms   */
+#define XTAL_SCALE_TO_KHZ	1
+#define NUM_NAMEMAPS    (3U)
+#define XTAL4X_KHZ 108000
+#define BOOT_GPCCLK_MHZ 645U
+
+#ifdef CONFIG_NVGPU_CLK_ARB
+u32 tu104_crystal_clk_hz(struct gk20a *g)
+{
+	return (XTAL4X_KHZ * 1000);
+}
+
+unsigned long tu104_clk_measure_freq(struct gk20a *g, u32 api_domain)
+{
+	struct clk_gk20a *clk = &g->clk;
+	u32 freq_khz;
+	u32 i;
+	struct namemap_cfg *c = NULL;
+
+	for (i = 0; i < clk->namemap_num; i++) {
+		if (api_domain == clk->namemap_xlat_table[i]) {
+			c = &clk->clk_namemap[i];
+			break;
+		}
+	}
+
+	if (c == NULL) {
+		return 0;
+	}
+	if (c->is_counter != 0U) {
+		freq_khz = c->scale * tu104_get_rate_cntr(g, c);
+	} else {
+		freq_khz = 0U;
+		 /* TODO: PLL read */
+	}
+
+	/* Convert to HZ */
+	return (freq_khz * 1000UL);
+}
+
+static void nvgpu_gpu_gpcclk_counter_init(struct gk20a *g)
+{
+	u32 data;
+
+	data = gk20a_readl(g, trim_gpc_bcast_fr_clk_cntr_ncgpcclk_cfg_r());
+	data |= trim_gpc_bcast_fr_clk_cntr_ncgpcclk_cfg_update_cycle_init_f() |
+		trim_gpc_bcast_fr_clk_cntr_ncgpcclk_cfg_cont_update_enabled_f() |
+		trim_gpc_bcast_fr_clk_cntr_ncgpcclk_cfg_start_count_disabled_f() |
+		trim_gpc_bcast_fr_clk_cntr_ncgpcclk_cfg_reset_asserted_f() |
+		trim_gpc_bcast_fr_clk_cntr_ncgpcclk_cfg_source_gpcclk_noeg_f();
+	gk20a_writel(g,trim_gpc_bcast_fr_clk_cntr_ncgpcclk_cfg_r(), data);
+	/*
+	* Based on the clock counter design, it takes 16 clock cycles of the
+	* "counted clock" for the counter to completely reset. Considering
+	* 27MHz as the slowest clock during boot time, delay of 16/27us (~1us)
+	* should be sufficient. See Bug 1953217.
+	*/
+	nvgpu_udelay(1);
+	data = gk20a_readl(g, trim_gpc_bcast_fr_clk_cntr_ncgpcclk_cfg_r());
+	data = set_field(data, trim_gpc_bcast_fr_clk_cntr_ncgpcclk_cfg_reset_m(),
+			trim_gpc_bcast_fr_clk_cntr_ncgpcclk_cfg_reset_deasserted_f());
+	gk20a_writel(g,trim_gpc_bcast_fr_clk_cntr_ncgpcclk_cfg_r(), data);
+	/*
+	* Enable clock counter.
+	* Note : Need to write un-reset and enable signal in different
+	* register writes as the source (register block) and destination
+	* (FR counter) are on the same clock and far away from each other,
+	* so the signals can not reach in the same clock cycle hence some
+	* delay is required between signals.
+	*/
+	data = gk20a_readl(g, trim_gpc_bcast_fr_clk_cntr_ncgpcclk_cfg_r());
+	data |= trim_gpc_bcast_fr_clk_cntr_ncgpcclk_cfg_start_count_enabled_f();
+	gk20a_writel(g,trim_gpc_bcast_fr_clk_cntr_ncgpcclk_cfg_r(), data);
+}
+
+u32 tu104_clk_get_cntr_sysclk_source(struct gk20a *g)
+{
+	return trim_sys_fr_clk_cntr_sysclk_cfg_source_sys_noeg_f();
+}
+
+static void nvgpu_gpu_sysclk_counter_init(struct gk20a *g)
+{
+	u32 data;
+
+	data = gk20a_readl(g, trim_sys_fr_clk_cntr_sysclk_cfg_r());
+	data |= trim_sys_fr_clk_cntr_sysclk_cfg_update_cycle_init_f() |
+		trim_sys_fr_clk_cntr_sysclk_cfg_cont_update_enabled_f() |
+		trim_sys_fr_clk_cntr_sysclk_cfg_start_count_disabled_f() |
+		trim_sys_fr_clk_cntr_sysclk_cfg_reset_asserted_f() |
+		g->ops.clk.get_cntr_sysclk_source(g);
+	gk20a_writel(g,trim_sys_fr_clk_cntr_sysclk_cfg_r(), data);
+
+	nvgpu_udelay(1);
+
+	data = gk20a_readl(g, trim_sys_fr_clk_cntr_sysclk_cfg_r());
+	data = set_field(data, trim_sys_fr_clk_cntr_sysclk_cfg_reset_m(),
+			trim_sys_fr_clk_cntr_sysclk_cfg_reset_deasserted_f());
+	gk20a_writel(g,trim_sys_fr_clk_cntr_sysclk_cfg_r(), data);
+
+	data = gk20a_readl(g, trim_sys_fr_clk_cntr_sysclk_cfg_r());
+	data |= trim_sys_fr_clk_cntr_sysclk_cfg_start_count_enabled_f();
+	gk20a_writel(g,trim_sys_fr_clk_cntr_sysclk_cfg_r(), data);
+}
+
+u32 tu104_clk_get_cntr_xbarclk_source(struct gk20a *g)
+{
+	return trim_sys_fll_fr_clk_cntr_xbarclk_cfg_source_xbar_nobg_f();
+}
+
+static void nvgpu_gpu_xbarclk_counter_init(struct gk20a *g)
+{
+	u32 data;
+
+	data = gk20a_readl(g, trim_sys_fll_fr_clk_cntr_xbarclk_cfg_r());
+	data |= trim_sys_fll_fr_clk_cntr_xbarclk_cfg_update_cycle_init_f() |
+		trim_sys_fll_fr_clk_cntr_xbarclk_cfg_cont_update_enabled_f() |
+		trim_sys_fll_fr_clk_cntr_xbarclk_cfg_start_count_disabled_f() |
+		trim_sys_fll_fr_clk_cntr_xbarclk_cfg_reset_asserted_f() |
+		g->ops.clk.get_cntr_xbarclk_source(g);
+	gk20a_writel(g,trim_sys_fll_fr_clk_cntr_xbarclk_cfg_r(), data);
+
+	nvgpu_udelay(1);
+
+	data = gk20a_readl(g, trim_sys_fll_fr_clk_cntr_xbarclk_cfg_r());
+	data = set_field(data, trim_sys_fll_fr_clk_cntr_xbarclk_cfg_reset_m(),
+			trim_sys_fll_fr_clk_cntr_xbarclk_cfg_reset_deasserted_f());
+	gk20a_writel(g,trim_sys_fll_fr_clk_cntr_xbarclk_cfg_r(), data);
+
+	data = gk20a_readl(g, trim_sys_fll_fr_clk_cntr_xbarclk_cfg_r());
+	data |= trim_sys_fll_fr_clk_cntr_xbarclk_cfg_start_count_enabled_f();
+	gk20a_writel(g,trim_sys_fll_fr_clk_cntr_xbarclk_cfg_r(), data);
+}
+
+int tu104_init_clk_support(struct gk20a *g)
+{
+	struct clk_gk20a *clk = &g->clk;
+
+
+	nvgpu_log_fn(g, " ");
+
+	nvgpu_mutex_init(&clk->clk_mutex);
+
+	clk->clk_namemap = (struct namemap_cfg *)
+		nvgpu_kzalloc(g, sizeof(struct namemap_cfg) * NUM_NAMEMAPS);
+
+	if (clk->clk_namemap == NULL) {
+		nvgpu_mutex_destroy(&clk->clk_mutex);
+		return -ENOMEM;
+	}
+
+	clk->namemap_xlat_table = nvgpu_kcalloc(g, NUM_NAMEMAPS, sizeof(u32));
+
+	if (clk->namemap_xlat_table == NULL) {
+		nvgpu_kfree(g, clk->clk_namemap);
+		nvgpu_mutex_destroy(&clk->clk_mutex);
+		return -ENOMEM;
+	}
+
+	clk->clk_namemap[0] = (struct namemap_cfg) {
+		.namemap = CLK_NAMEMAP_INDEX_GPCCLK,
+		.is_enable = 1,
+		.is_counter = 1,
+		.g = g,
+		.cntr = {
+			.reg_ctrl_addr = trim_gpc_bcast_fr_clk_cntr_ncgpcclk_cfg_r(),
+			.reg_ctrl_idx  = trim_gpc_bcast_fr_clk_cntr_ncgpcclk_cfg_source_gpcclk_noeg_f(),
+			.reg_cntr_addr[0] = trim_gpc_bcast_fr_clk_cntr_ncgpcclk_cnt0_r(),
+			.reg_cntr_addr[1] = trim_gpc_bcast_fr_clk_cntr_ncgpcclk_cnt1_r()
+		},
+		.name = "gpcclk",
+		.scale = 1
+	};
+
+	nvgpu_gpu_gpcclk_counter_init(g);
+	clk->namemap_xlat_table[0] = CTRL_CLK_DOMAIN_GPCCLK;
+
+	clk->clk_namemap[1] = (struct namemap_cfg) {
+		.namemap = CLK_NAMEMAP_INDEX_SYSCLK,
+		.is_enable = 1,
+		.is_counter = 1,
+		.g = g,
+		.cntr = {
+			.reg_ctrl_addr = trim_sys_fr_clk_cntr_sysclk_cfg_r(),
+			.reg_ctrl_idx  = g->ops.clk.get_cntr_sysclk_source(g),
+			.reg_cntr_addr[0] = trim_sys_fr_clk_cntr_sysclk_cntr0_r(),
+			.reg_cntr_addr[1] = trim_sys_fr_clk_cntr_sysclk_cntr1_r()
+		},
+		.name = "sysclk",
+		.scale = 1
+	};
+
+	nvgpu_gpu_sysclk_counter_init(g);
+	clk->namemap_xlat_table[1] = CTRL_CLK_DOMAIN_SYSCLK;
+
+	clk->clk_namemap[2] = (struct namemap_cfg) {
+		.namemap = CLK_NAMEMAP_INDEX_XBARCLK,
+		.is_enable = 1,
+		.is_counter = 1,
+		.g = g,
+		.cntr = {
+			.reg_ctrl_addr = trim_sys_fll_fr_clk_cntr_xbarclk_cfg_r(),
+			.reg_ctrl_idx  = g->ops.clk.get_cntr_xbarclk_source(g),
+			.reg_cntr_addr[0] = trim_sys_fll_fr_clk_cntr_xbarclk_cntr0_r(),
+			.reg_cntr_addr[1] = trim_sys_fll_fr_clk_cntr_xbarclk_cntr1_r()
+		},
+		.name = "xbarclk",
+		.scale = 1
+	};
+
+	nvgpu_gpu_xbarclk_counter_init(g);
+	clk->namemap_xlat_table[2] = CTRL_CLK_DOMAIN_XBARCLK;
+
+	clk->namemap_num = NUM_NAMEMAPS;
+
+	clk->g = g;
+
+	return 0;
+}
+
+u32 tu104_get_rate_cntr(struct gk20a *g, struct namemap_cfg *c) {
+#ifdef CONFIG_NVGPU_NON_FUSA
+	u32 cntr = 0;
+	u64 cntr_start = 0;
+	u64 cntr_stop = 0;
+	u64 start_time, stop_time;
+	const int max_iterations = 3;
+	int i = 0;
+
+	struct clk_gk20a *clk = &g->clk;
+
+	if ((c == NULL) || (c->cntr.reg_ctrl_addr == 0U) ||
+		(c->cntr.reg_cntr_addr[0] == 0U) ||
+		(c->cntr.reg_cntr_addr[1]) == 0U) {
+			return 0;
+	}
+
+	nvgpu_mutex_acquire(&clk->clk_mutex);
+
+	for (i = 0; i < max_iterations; i++) {
+		/*
+		 * Read the counter values. Counter is 36 bits, 32
+		 * bits on addr[0] and 4 lsb on addr[1] others zero.
+		 */
+		cntr_start = (u64)nvgpu_readl(g,
+				c->cntr.reg_cntr_addr[0]);
+		cntr_start += ((u64)nvgpu_readl(g,
+				c->cntr.reg_cntr_addr[1]) << 32);
+		start_time = (u64)nvgpu_hr_timestamp_us();
+		nvgpu_udelay(XTAL_CNTR_DELAY);
+		stop_time = (u64)nvgpu_hr_timestamp_us();
+		cntr_stop = (u64)nvgpu_readl(g,
+				c->cntr.reg_cntr_addr[0]);
+		cntr_stop += ((u64)nvgpu_readl(g,
+				c->cntr.reg_cntr_addr[1]) << 32);
+
+		if (cntr_stop > cntr_start) {
+			/*
+			 * Calculate the difference with Acutal time
+			 * and convert to KHz
+			 */
+			cntr = (u32)(((cntr_stop - cntr_start) /
+				(stop_time - start_time)) * 1000U);
+			nvgpu_mutex_release(&clk->clk_mutex);
+			return cntr;
+		}
+		/* Else wrap around detected. Hence, retry. */
+	}
+
+	nvgpu_mutex_release(&clk->clk_mutex);
+#endif
+	/* too many iterations, bail out */
+	nvgpu_err(g, "failed to get clk rate");
+	return -EBUSY;
+}
+
+int tu104_clk_domain_get_f_points(
+	struct gk20a *g,
+	u32 clkapidomain,
+	u32 *pfpointscount,
+	u16 *pfreqpointsinmhz)
+{
+	int status = -EINVAL;
+
+	if (pfpointscount == NULL) {
+		return -EINVAL;
+	}
+
+	if ((pfreqpointsinmhz == NULL) && (*pfpointscount != 0U)) {
+		return -EINVAL;
+	}
+
+	status = nvgpu_pmu_clk_domain_get_f_points(g,
+			clkapidomain, pfpointscount, pfreqpointsinmhz);
+	if (status != 0) {
+		nvgpu_err(g, "Unable to get frequency points");
+	}
+	return status;
+}
+
+void tu104_suspend_clk_support(struct gk20a *g)
+{
+	nvgpu_mutex_destroy(&g->clk.clk_mutex);
+}
+
+unsigned long tu104_clk_maxrate(struct gk20a *g, u32 api_domain)
+{
+	u16 min_mhz = 0, max_mhz = 0;
+	int status;
+
+	if (nvgpu_is_enabled(g, NVGPU_PMU_PSTATE)) {
+		status = nvgpu_clk_arb_get_arbiter_clk_range(g, api_domain,
+				&min_mhz, &max_mhz);
+		if (status != 0) {
+			nvgpu_err(g, "failed to fetch clock range");
+			return 0U;
+		}
+	} else {
+		if (api_domain == NVGPU_CLK_DOMAIN_GPCCLK) {
+			max_mhz = BOOT_GPCCLK_MHZ;
+		}
+	}
+
+	return (max_mhz * 1000UL * 1000UL);
+}
+
+void tu104_get_change_seq_time(struct gk20a *g, s64 *change_time)
+{
+	nvgpu_perf_change_seq_execute_time(g, change_time);
+}
+#endif
+void tu104_change_host_clk_source(struct gk20a *g)
+{
+	nvgpu_writel(g, trim_sys_ind_clk_sys_core_clksrc_r(),
+			trim_sys_ind_clk_sys_core_clksrc_hostclk_fll_f());
+}

drivers/gpu/nvgpu/hal/clk/clk_tu104.h

@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2016-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+#ifndef CLK_TU104_H
+#define CLK_TU104_H
+
+#include <nvgpu/lock.h>
+#include <nvgpu/gk20a.h>
+
+#define CLK_NAME_MAX			24
+#define CLK_MAX_CNTRL_REGISTERS		2
+
+struct namemap_cfg {
+	u32 namemap;
+	u32 is_enable;
+	u32 is_counter;
+	struct gk20a *g;
+	struct {
+		u32 reg_ctrl_addr;
+		u32 reg_ctrl_idx;
+		u32 reg_cntr_addr[CLK_MAX_CNTRL_REGISTERS];
+	} cntr;
+	u32 scale;
+	char name[CLK_NAME_MAX];
+};
+
+u32 tu104_get_rate_cntr(struct gk20a *g, struct namemap_cfg *c);
+int tu104_init_clk_support(struct gk20a *g);
+u32 tu104_crystal_clk_hz(struct gk20a *g);
+u32 tu104_clk_get_cntr_xbarclk_source(struct gk20a *g);
+u32 tu104_clk_get_cntr_sysclk_source(struct gk20a *g);
+unsigned long tu104_clk_measure_freq(struct gk20a *g, u32 api_domain);
+void tu104_suspend_clk_support(struct gk20a *g);
+int tu104_clk_domain_get_f_points(
+	struct gk20a *g,
+	u32 clkapidomain,
+	u32 *pfpointscount,
+	u16 *pfreqpointsinmhz);
+unsigned long tu104_clk_maxrate(struct gk20a *g, u32 api_domain);
+void tu104_get_change_seq_time(struct gk20a *g, s64 *change_time);
+void tu104_change_host_clk_source(struct gk20a *g);
+
+#endif /* CLK_TU104_H */

drivers/gpu/nvgpu/hal/falcon/falcon_gk20a.c

@@ -0,0 +1,132 @@
+/*
+ * Copyright (c) 2017-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+#include <nvgpu/io.h>
+#include <nvgpu/gk20a.h>
+#include <nvgpu/falcon.h>
+#include <nvgpu/string.h>
+
+#include "falcon_gk20a.h"
+
+#include <nvgpu/hw/gm20b/hw_falcon_gm20b.h>
+
+bool gk20a_falcon_clear_halt_interrupt_status(struct nvgpu_falcon *flcn)
+{
+	struct gk20a *g = flcn->g;
+	u32 base_addr = flcn->flcn_base;
+	u32 data = 0;
+	bool status = false;
+
+	gk20a_writel(g, base_addr + falcon_falcon_irqsclr_r(),
+		gk20a_readl(g, base_addr + falcon_falcon_irqsclr_r()) |
+		0x10U);
+	data = gk20a_readl(g, (base_addr + falcon_falcon_irqstat_r()));
+
+	if ((data & falcon_falcon_irqstat_halt_true_f()) !=
+		falcon_falcon_irqstat_halt_true_f()) {
+		/*halt irq is clear*/
+		status = true;
+	}
+
+	return status;
+}
+
+int gk20a_falcon_copy_from_dmem(struct nvgpu_falcon *flcn,
+		u32 src, u8 *dst, u32 size, u8 port)
+{
+	struct gk20a *g = flcn->g;
+	u32 base_addr = flcn->flcn_base;
+	u32 i, words, bytes;
+	u32 data, addr_mask;
+	u32 *dst_u32 = (u32 *)dst;
+
+	nvgpu_log_fn(g, " src dmem offset - %x, size - %x", src, size);
+
+	words = size >> 2U;
+	bytes = size & 0x3U;
+
+	addr_mask = falcon_falcon_dmemc_offs_m() |
+				g->ops.falcon.dmemc_blk_mask();
+
+	src &= addr_mask;
+
+	nvgpu_writel(g, base_addr + falcon_falcon_dmemc_r(port),
+		src | falcon_falcon_dmemc_aincr_f(1));
+
+	for (i = 0; i < words; i++) {
+		dst_u32[i] = nvgpu_readl(g,
+			base_addr + falcon_falcon_dmemd_r(port));
+	}
+
+	if (bytes > 0U) {
+		data = nvgpu_readl(g, base_addr + falcon_falcon_dmemd_r(port));
+		nvgpu_memcpy(&dst[words << 2U], (u8 *)&data, bytes);
+	}
+
+	return 0;
+}
+
+int gk20a_falcon_copy_from_imem(struct nvgpu_falcon *flcn, u32 src,
+	u8 *dst, u32 size, u8 port)
+{
+	struct gk20a *g = flcn->g;
+	u32 base_addr = flcn->flcn_base;
+	u32 *dst_u32 = (u32 *)dst;
+	u32 words = 0;
+	u32 bytes = 0;
+	u32 data = 0;
+	u32 blk = 0;
+	u32 i = 0;
+
+	nvgpu_log_info(g, "download %d bytes from 0x%x", size, src);
+
+	words = size >> 2U;
+	bytes = size & 0x3U;
+	blk = src >> 8;
+
+	nvgpu_log_info(g, "download %d words from 0x%x block %d",
+			words, src, blk);
+
+	nvgpu_writel(g, base_addr + falcon_falcon_imemc_r(port),
+		falcon_falcon_imemc_offs_f(src >> 2) |
+		g->ops.falcon.imemc_blk_field(blk) |
+		falcon_falcon_dmemc_aincr_f(1));
+
+	for (i = 0; i < words; i++) {
+		dst_u32[i] = nvgpu_readl(g,
+			base_addr + falcon_falcon_imemd_r(port));
+	}
+
+	if (bytes > 0U) {
+		data = nvgpu_readl(g, base_addr + falcon_falcon_imemd_r(port));
+		nvgpu_memcpy(&dst[words << 2U], (u8 *)&data, bytes);
+	}
+
+	return 0;
+}
+
+void gk20a_falcon_get_ctls(struct nvgpu_falcon *flcn, u32 *sctl,
+				  u32 *cpuctl)
+{
+	*sctl = gk20a_readl(flcn->g, flcn->flcn_base + falcon_falcon_sctl_r());
+	*cpuctl = gk20a_readl(flcn->g, flcn->flcn_base +
+					falcon_falcon_cpuctl_r());
+}

drivers/gpu/nvgpu/hal/falcon/falcon_gk20a.h

@@ -0,0 +1,101 @@
+/*
+ * Copyright (c) 2017-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+#ifndef NVGPU_FALCON_GK20A_H
+#define NVGPU_FALCON_GK20A_H
+
+#include <nvgpu/falcon.h>
+
+/* Falcon Register index */
+#define FALCON_REG_R0		(0U)
+#define FALCON_REG_R1		(1U)
+#define FALCON_REG_R2		(2U)
+#define FALCON_REG_R3		(3U)
+#define FALCON_REG_R4		(4U)
+#define FALCON_REG_R5		(5U)
+#define FALCON_REG_R6		(6U)
+#define FALCON_REG_R7		(7U)
+#define FALCON_REG_R8		(8U)
+#define FALCON_REG_R9		(9U)
+#define FALCON_REG_R10		(10U)
+#define FALCON_REG_R11		(11U)
+#define FALCON_REG_R12		(12U)
+#define FALCON_REG_R13		(13U)
+#define FALCON_REG_R14		(14U)
+#define FALCON_REG_R15		(15U)
+#define FALCON_REG_IV0		(16U)
+#define FALCON_REG_IV1		(17U)
+#define FALCON_REG_UNDEFINED	(18U)
+#define FALCON_REG_EV		(19U)
+#define FALCON_REG_SP		(20U)
+#define FALCON_REG_PC		(21U)
+#define FALCON_REG_IMB		(22U)
+#define FALCON_REG_DMB		(23U)
+#define FALCON_REG_CSW		(24U)
+#define FALCON_REG_CCR		(25U)
+#define FALCON_REG_SEC		(26U)
+#define FALCON_REG_CTX		(27U)
+#define FALCON_REG_EXCI		(28U)
+#define FALCON_REG_RSVD0	(29U)
+#define FALCON_REG_RSVD1	(30U)
+#define FALCON_REG_RSVD2	(31U)
+#define FALCON_REG_SIZE		(32U)
+
+#define FALCON_DMEM_BLKSIZE2	8U
+
+u32 gk20a_falcon_dmemc_blk_mask(void);
+u32 gk20a_falcon_imemc_blk_field(u32 blk);
+void gk20a_falcon_reset(struct nvgpu_falcon *flcn);
+bool gk20a_is_falcon_cpu_halted(struct nvgpu_falcon *flcn);
+bool gk20a_is_falcon_idle(struct nvgpu_falcon *flcn);
+bool gk20a_is_falcon_scrubbing_done(struct nvgpu_falcon *flcn);
+u32 gk20a_falcon_get_mem_size(struct nvgpu_falcon *flcn,
+		enum falcon_mem_type mem_type);
+u8 gk20a_falcon_get_ports_count(struct nvgpu_falcon *flcn,
+		enum falcon_mem_type mem_type);
+int gk20a_falcon_copy_to_dmem(struct nvgpu_falcon *flcn,
+		u32 dst, u8 *src, u32 size, u8 port);
+int gk20a_falcon_copy_to_imem(struct nvgpu_falcon *flcn, u32 dst,
+		u8 *src, u32 size, u8 port, bool sec, u32 tag);
+void gk20a_falcon_bootstrap(struct nvgpu_falcon *flcn,
+	u32 boot_vector);
+u32 gk20a_falcon_mailbox_read(struct nvgpu_falcon *flcn,
+		u32 mailbox_index);
+void gk20a_falcon_mailbox_write(struct nvgpu_falcon *flcn,
+		u32 mailbox_index, u32 data);
+void gk20a_falcon_set_irq(struct nvgpu_falcon *flcn, bool enable,
+				 u32 intr_mask, u32 intr_dest);
+
+#ifdef CONFIG_NVGPU_FALCON_DEBUG
+void gk20a_falcon_dump_stats(struct nvgpu_falcon *flcn);
+#endif
+
+#ifdef CONFIG_NVGPU_FALCON_NON_FUSA
+bool gk20a_falcon_clear_halt_interrupt_status(struct nvgpu_falcon *flcn);
+int gk20a_falcon_copy_from_dmem(struct nvgpu_falcon *flcn,
+		u32 src, u8 *dst, u32 size, u8 port);
+int gk20a_falcon_copy_from_imem(struct nvgpu_falcon *flcn, u32 src,
+	u8 *dst, u32 size, u8 port);
+void gk20a_falcon_get_ctls(struct nvgpu_falcon *flcn, u32 *sctl,
+				  u32 *cpuctl);
+#endif
+
+#endif /* NVGPU_FALCON_GK20A_H */

drivers/gpu/nvgpu/hal/falcon/falcon_gk20a_fusa.c

@@ -0,0 +1,560 @@
+/*
+ * Copyright (c) 2017-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+#include <nvgpu/gk20a.h>
+#include <nvgpu/falcon.h>
+#include <nvgpu/string.h>
+#include <nvgpu/static_analysis.h>
+
+#include "falcon_gk20a.h"
+
+#include <nvgpu/hw/gm20b/hw_falcon_gm20b.h>
+
+u32 gk20a_falcon_dmemc_blk_mask(void)
+{
+	return falcon_falcon_dmemc_blk_m();
+}
+
+u32 gk20a_falcon_imemc_blk_field(u32 blk)
+{
+	return falcon_falcon_imemc_blk_f(blk);
+}
+
+void gk20a_falcon_reset(struct nvgpu_falcon *flcn)
+{
+	u32 unit_status = 0U;
+
+	/* do falcon CPU hard reset */
+	unit_status = nvgpu_falcon_readl(flcn, falcon_falcon_cpuctl_r());
+	nvgpu_falcon_writel(flcn, falcon_falcon_cpuctl_r(),
+			    (unit_status | falcon_falcon_cpuctl_hreset_f(1)));
+}
+
+bool gk20a_is_falcon_cpu_halted(struct nvgpu_falcon *flcn)
+{
+	return ((nvgpu_falcon_readl(flcn, falcon_falcon_cpuctl_r()) &
+			falcon_falcon_cpuctl_halt_intr_m()) != 0U);
+}
+
+bool gk20a_is_falcon_idle(struct nvgpu_falcon *flcn)
+{
+	u32 unit_status = 0U;
+	bool status = false;
+
+	unit_status = nvgpu_falcon_readl(flcn, falcon_falcon_idlestate_r());
+
+	if ((falcon_falcon_idlestate_falcon_busy_v(unit_status) == 0U) &&
+		(falcon_falcon_idlestate_ext_busy_v(unit_status) == 0U)) {
+		status = true;
+	} else {
+		status = false;
+	}
+
+	return status;
+}
+
+bool gk20a_is_falcon_scrubbing_done(struct nvgpu_falcon *flcn)
+{
+	u32 unit_status = 0U;
+	bool status = false;
+
+	unit_status = nvgpu_falcon_readl(flcn, falcon_falcon_dmactl_r());
+
+	if ((unit_status &
+		(falcon_falcon_dmactl_dmem_scrubbing_m() |
+		 falcon_falcon_dmactl_imem_scrubbing_m())) != 0U) {
+		status = false;
+	} else {
+		status = true;
+	}
+
+	return status;
+}
+
+u32 gk20a_falcon_get_mem_size(struct nvgpu_falcon *flcn,
+		enum falcon_mem_type mem_type)
+{
+	u32 mem_size = 0U;
+	u32 hwcfg_val = 0U;
+
+	hwcfg_val = nvgpu_falcon_readl(flcn, falcon_falcon_hwcfg_r());
+
+	if (mem_type == MEM_DMEM) {
+		mem_size = falcon_falcon_hwcfg_dmem_size_v(hwcfg_val)
+			<< FALCON_DMEM_BLKSIZE2;
+	} else {
+		mem_size = falcon_falcon_hwcfg_imem_size_v(hwcfg_val)
+			<< FALCON_DMEM_BLKSIZE2;
+	}
+
+	return mem_size;
+}
+
+u8 gk20a_falcon_get_ports_count(struct nvgpu_falcon *flcn,
+		enum falcon_mem_type mem_type)
+{
+	u8 ports = 0U;
+	u32 hwcfg1_val = 0U;
+
+	hwcfg1_val = nvgpu_falcon_readl(flcn, falcon_falcon_hwcfg1_r());
+
+	if (mem_type == MEM_DMEM) {
+		ports = (u8) falcon_falcon_hwcfg1_dmem_ports_v(hwcfg1_val);
+	} else {
+		ports = (u8) falcon_falcon_hwcfg1_imem_ports_v(hwcfg1_val);
+	}
+
+	return ports;
+}
+
+#define FALCON_UNALIGNED_MEMCPY_BLOCK_SIZE	256U
+
+static void falcon_copy_to_dmem_unaligned_src(struct nvgpu_falcon *flcn,
+					      u8 *src, u32 size, u8 port)
+{
+	u32 src_tmp[FALCON_UNALIGNED_MEMCPY_BLOCK_SIZE];
+	u32 bytes_extra = 0U;
+	u32 elem_size = 0U;
+	u32 offset = 0U;
+	u32 elems = 0U;
+	u32 i = 0U;
+
+	while ((offset + sizeof(src_tmp)) <= size) {
+		nvgpu_memcpy((u8 *)&src_tmp[0], &src[offset],
+			     sizeof(src_tmp));
+		for (i = 0; i < ARRAY_SIZE(src_tmp); i++) {
+			nvgpu_falcon_writel(flcn,
+					    falcon_falcon_dmemd_r(port),
+					    src_tmp[i]);
+		}
+		offset += (u32) sizeof(src_tmp);
+	}
+
+	if (offset < size) {
+		bytes_extra = size - offset;
+		elem_size =
+			nvgpu_safe_cast_u64_to_u32(sizeof(src_tmp[0]));
+		elems = bytes_extra / elem_size;
+
+		nvgpu_memcpy((u8 *)&src_tmp[0], &src[offset],
+			     (u64)elems * elem_size);
+		for (i = 0; i < elems; i++) {
+			nvgpu_falcon_writel(flcn,
+					    falcon_falcon_dmemd_r(port),
+					    src_tmp[i]);
+		}
+	}
+}
+
+int gk20a_falcon_copy_to_dmem(struct nvgpu_falcon *flcn,
+		u32 dst, u8 *src, u32 size, u8 port)
+{
+	struct gk20a *g = flcn->g;
+	u32 i = 0U, words = 0U, bytes = 0U;
+	u32 data = 0U, addr_mask = 0U;
+	u32 *src_u32 = NULL;
+
+	nvgpu_log_fn(flcn->g, "dest dmem offset - %x, size - %x", dst, size);
+
+	words = size >> 2U;
+	bytes = size & 0x3U;
+
+	addr_mask = falcon_falcon_dmemc_offs_m() |
+		g->ops.falcon.dmemc_blk_mask();
+
+	dst &= addr_mask;
+
+	nvgpu_falcon_writel(flcn, falcon_falcon_dmemc_r(port),
+			    dst | falcon_falcon_dmemc_aincw_f(1));
+
+	if (likely(nvgpu_mem_is_word_aligned(flcn->g, src))) {
+NVGPU_COV_WHITELIST(deviate, NVGPU_MISRA(Rule, 11_3), "TID-415")
+		src_u32 = (u32 *)src;
+
+		for (i = 0; i < words; i++) {
+			nvgpu_falcon_writel(flcn, falcon_falcon_dmemd_r(port),
+					    src_u32[i]);
+		}
+	} else {
+		falcon_copy_to_dmem_unaligned_src(flcn, src, size, port);
+	}
+
+	if (bytes > 0U) {
+		data = 0;
+		nvgpu_memcpy((u8 *)&data, &src[words << 2U], bytes);
+		nvgpu_falcon_writel(flcn, falcon_falcon_dmemd_r(port), data);
+	}
+
+	size = NVGPU_ALIGN(size, 4U);
+	data = nvgpu_falcon_readl(flcn, falcon_falcon_dmemc_r(port)) &
+		addr_mask;
+	if (data != (nvgpu_safe_add_u32(dst, size) & addr_mask)) {
+		nvgpu_warn(flcn->g, "copy failed. bytes written %d, expected %d",
+			data - dst, size);
+		return -EIO;
+	}
+
+	return 0;
+}
+
+static void falcon_copy_to_imem_unaligned_src(struct nvgpu_falcon *flcn,
+					      u8 *src, u32 size, u8 port,
+					      u32 tag)
+{
+	u32 src_tmp[FALCON_UNALIGNED_MEMCPY_BLOCK_SIZE];
+	u32 bytes_extra = 0U;
+	u32 elem_size = 0U;
+	u32 offset = 0U;
+	u32 elems = 0U;
+	u32 i = 0U;
+	u32 j = 0U;
+
+	while ((offset + sizeof(src_tmp)) <= size) {
+		nvgpu_memcpy((u8 *)&src_tmp[0], &src[offset],
+			     sizeof(src_tmp));
+		for (i = 0; i < ARRAY_SIZE(src_tmp); i++) {
+			if ((j++ % 64U) == 0U) {
+				/* tag is always 256B aligned */
+				nvgpu_falcon_writel(flcn,
+					falcon_falcon_imemt_r(port), tag);
+				tag = nvgpu_safe_add_u32(tag, 1U);
+			}
+
+			nvgpu_falcon_writel(flcn,
+					    falcon_falcon_imemd_r(port),
+					    src_tmp[i]);
+		}
+		offset += (u32) sizeof(src_tmp);
+	}
+
+	if (offset < size) {
+		bytes_extra = size - offset;
+		elem_size =
+			nvgpu_safe_cast_u64_to_u32(sizeof(src_tmp[0]));
+		elems = bytes_extra / elem_size;
+
+		nvgpu_memcpy((u8 *)&src_tmp[0], &src[offset],
+			     (u64)elems * elem_size);
+		for (i = 0; i < elems; i++) {
+			if ((j++ % 64U) == 0U) {
+				/* tag is always 256B aligned */
+				nvgpu_falcon_writel(flcn,
+					falcon_falcon_imemt_r(port), tag);
+				tag = nvgpu_safe_add_u32(tag, 1U);
+			}
+
+			nvgpu_falcon_writel(flcn,
+					    falcon_falcon_imemd_r(port),
+					    src_tmp[i]);
+		}
+	}
+
+	/* WARNING : setting remaining bytes in block to 0x0 */
+	while ((j % 64U) != 0U) {
+		nvgpu_falcon_writel(flcn,
+				    falcon_falcon_imemd_r(port), 0);
+		j++;
+	}
+}
+
+int gk20a_falcon_copy_to_imem(struct nvgpu_falcon *flcn, u32 dst,
+		u8 *src, u32 size, u8 port, bool sec, u32 tag)
+{
+	struct gk20a *g = flcn->g;
+	u32 *src_u32 = NULL;
+	u32 words = 0U;
+	u32 blk = 0U;
+	u32 i = 0U;
+
+	nvgpu_log_info(flcn->g, "upload %d bytes to 0x%x", size, dst);
+
+	words = size >> 2U;
+	blk = dst >> 8;
+
+	nvgpu_log_info(flcn->g, "upload %d words to 0x%x block %d, tag 0x%x",
+			words, dst, blk, tag);
+
+	nvgpu_falcon_writel(flcn, falcon_falcon_imemc_r(port),
+			falcon_falcon_imemc_offs_f(dst >> 2) |
+			g->ops.falcon.imemc_blk_field(blk) |
+			/* Set Auto-Increment on write */
+			falcon_falcon_imemc_aincw_f(1) |
+			falcon_falcon_imemc_secure_f(sec ? 1U : 0U));
+
+	if (likely(nvgpu_mem_is_word_aligned(flcn->g, src))) {
+NVGPU_COV_WHITELIST(deviate, NVGPU_MISRA(Rule, 11_3), "TID-415")
+		src_u32 = (u32 *)src;
+
+		for (i = 0U; i < words; i++) {
+			if ((i % 64U) == 0U) {
+				/* tag is always 256B aligned */
+				nvgpu_falcon_writel(flcn,
+					falcon_falcon_imemt_r(port), tag);
+				tag = nvgpu_safe_add_u32(tag, 1U);
+			}
+
+			nvgpu_falcon_writel(flcn, falcon_falcon_imemd_r(port),
+					    src_u32[i]);
+		}
+
+		/* WARNING : setting remaining bytes in block to 0x0 */
+		while ((i % 64U) != 0U) {
+			nvgpu_falcon_writel(flcn,
+					    falcon_falcon_imemd_r(port), 0);
+			i++;
+		}
+	} else {
+		falcon_copy_to_imem_unaligned_src(flcn, src, size,
+						  port, tag);
+	}
+
+	return 0;
+}
+
+void gk20a_falcon_bootstrap(struct nvgpu_falcon *flcn,
+	u32 boot_vector)
+{
+	nvgpu_log_info(flcn->g, "boot vec 0x%x", boot_vector);
+
+	nvgpu_falcon_writel(flcn, falcon_falcon_dmactl_r(),
+		falcon_falcon_dmactl_require_ctx_f(0));
+
+	nvgpu_falcon_writel(flcn, falcon_falcon_bootvec_r(),
+		falcon_falcon_bootvec_vec_f(boot_vector));
+
+	nvgpu_falcon_writel(flcn, falcon_falcon_cpuctl_r(),
+		falcon_falcon_cpuctl_startcpu_f(1));
+}
+
+u32 gk20a_falcon_mailbox_read(struct nvgpu_falcon *flcn,
+		u32 mailbox_index)
+{
+	return nvgpu_falcon_readl(flcn, (mailbox_index != 0U) ?
+					falcon_falcon_mailbox1_r() :
+					falcon_falcon_mailbox0_r());
+}
+
+void gk20a_falcon_mailbox_write(struct nvgpu_falcon *flcn,
+		u32 mailbox_index, u32 data)
+{
+	nvgpu_falcon_writel(flcn, (mailbox_index != 0U) ?
+					falcon_falcon_mailbox1_r() :
+					falcon_falcon_mailbox0_r(), data);
+}
+
+void gk20a_falcon_set_irq(struct nvgpu_falcon *flcn, bool enable,
+	u32 intr_mask, u32 intr_dest)
+{
+	if (enable) {
+		nvgpu_falcon_writel(flcn, falcon_falcon_irqmset_r(), intr_mask);
+		nvgpu_falcon_writel(flcn, falcon_falcon_irqdest_r(), intr_dest);
+	} else {
+		nvgpu_falcon_writel(flcn, falcon_falcon_irqmclr_r(),
+				    0xffffffffU);
+	}
+}
+
+#ifdef CONFIG_NVGPU_FALCON_DEBUG
+static void gk20a_falcon_dump_imblk(struct nvgpu_falcon *flcn)
+{
+	struct gk20a *g = NULL;
+	u32 i = 0U, j = 0U;
+	u32 data[8] = {0U};
+	u32 block_count = 0U;
+
+	g = flcn->g;
+
+	block_count = falcon_falcon_hwcfg_imem_size_v(
+				nvgpu_falcon_readl(flcn,
+						   falcon_falcon_hwcfg_r()));
+
+	/* block_count must be multiple of 8 */
+	block_count &= ~0x7U;
+	nvgpu_err(g, "FALCON IMEM BLK MAPPING (PA->VA) (%d TOTAL):",
+		block_count);
+
+	for (i = 0U; i < block_count; i += 8U) {
+		for (j = 0U; j < 8U; j++) {
+			nvgpu_falcon_writel(flcn, falcon_falcon_imctl_debug_r(),
+			falcon_falcon_imctl_debug_cmd_f(0x2) |
+			falcon_falcon_imctl_debug_addr_blk_f(i + j));
+
+			data[j] = nvgpu_falcon_readl(flcn,
+						     falcon_falcon_imstat_r());
+		}
+
+		nvgpu_err(g, " %#04x: %#010x %#010x %#010x %#010x",
+				i, data[0], data[1], data[2], data[3]);
+		nvgpu_err(g, " %#04x: %#010x %#010x %#010x %#010x",
+				i + 4U, data[4], data[5], data[6], data[7]);
+	}
+}
+
+static void gk20a_falcon_dump_pc_trace(struct nvgpu_falcon *flcn)
+{
+	struct gk20a *g = NULL;
+	u32 trace_pc_count = 0U;
+	u32 pc = 0U;
+	u32 i = 0U;
+
+	g = flcn->g;
+
+	if ((nvgpu_falcon_readl(flcn, falcon_falcon_sctl_r()) & 0x02U) != 0U) {
+		nvgpu_err(g, " falcon is in HS mode, PC TRACE dump not supported");
+		return;
+	}
+
+	trace_pc_count = falcon_falcon_traceidx_maxidx_v(
+				nvgpu_falcon_readl(flcn,
+						   falcon_falcon_traceidx_r()));
+	nvgpu_err(g,
+		"PC TRACE (TOTAL %d ENTRIES. entry 0 is the most recent branch):",
+		trace_pc_count);
+
+	for (i = 0; i < trace_pc_count; i++) {
+		nvgpu_falcon_writel(flcn, falcon_falcon_traceidx_r(),
+				    falcon_falcon_traceidx_idx_f(i));
+
+		pc = falcon_falcon_tracepc_pc_v(
+			nvgpu_falcon_readl(flcn, falcon_falcon_tracepc_r()));
+		nvgpu_err(g, "FALCON_TRACEPC(%d)  :  %#010x", i, pc);
+	}
+}
+
+void gk20a_falcon_dump_stats(struct nvgpu_falcon *flcn)
+{
+	struct gk20a *g = NULL;
+	unsigned int i;
+
+	g = flcn->g;
+
+	nvgpu_err(g, "<<< FALCON id-%d DEBUG INFORMATION - START >>>",
+			flcn->flcn_id);
+
+	/* imblk dump */
+	gk20a_falcon_dump_imblk(flcn);
+	/* PC trace dump */
+	gk20a_falcon_dump_pc_trace(flcn);
+
+	nvgpu_err(g, "FALCON ICD REGISTERS DUMP");
+
+	for (i = 0U; i < 4U; i++) {
+		nvgpu_falcon_writel(flcn,
+			falcon_falcon_icd_cmd_r(),
+			falcon_falcon_icd_cmd_opc_rreg_f() |
+			falcon_falcon_icd_cmd_idx_f(FALCON_REG_PC));
+		nvgpu_err(g, "FALCON_REG_PC : 0x%x",
+			nvgpu_falcon_readl(flcn, falcon_falcon_icd_rdata_r()));
+
+		nvgpu_falcon_writel(flcn, falcon_falcon_icd_cmd_r(),
+			falcon_falcon_icd_cmd_opc_rreg_f() |
+			falcon_falcon_icd_cmd_idx_f(FALCON_REG_SP));
+		nvgpu_err(g, "FALCON_REG_SP : 0x%x",
+			nvgpu_falcon_readl(flcn, falcon_falcon_icd_rdata_r()));
+	}
+
+	nvgpu_falcon_writel(flcn, falcon_falcon_icd_cmd_r(),
+		falcon_falcon_icd_cmd_opc_rreg_f() |
+		falcon_falcon_icd_cmd_idx_f(FALCON_REG_IMB));
+	nvgpu_err(g, "FALCON_REG_IMB : 0x%x",
+		nvgpu_falcon_readl(flcn, falcon_falcon_icd_rdata_r()));
+
+	nvgpu_falcon_writel(flcn, falcon_falcon_icd_cmd_r(),
+		falcon_falcon_icd_cmd_opc_rreg_f() |
+		falcon_falcon_icd_cmd_idx_f(FALCON_REG_DMB));
+	nvgpu_err(g, "FALCON_REG_DMB : 0x%x",
+		nvgpu_falcon_readl(flcn, falcon_falcon_icd_rdata_r()));
+
+	nvgpu_falcon_writel(flcn, falcon_falcon_icd_cmd_r(),
+		falcon_falcon_icd_cmd_opc_rreg_f() |
+		falcon_falcon_icd_cmd_idx_f(FALCON_REG_CSW));
+	nvgpu_err(g, "FALCON_REG_CSW : 0x%x",
+		nvgpu_falcon_readl(flcn, falcon_falcon_icd_rdata_r()));
+
+	nvgpu_falcon_writel(flcn, falcon_falcon_icd_cmd_r(),
+		falcon_falcon_icd_cmd_opc_rreg_f() |
+		falcon_falcon_icd_cmd_idx_f(FALCON_REG_CTX));
+	nvgpu_err(g, "FALCON_REG_CTX : 0x%x",
+		nvgpu_falcon_readl(flcn, falcon_falcon_icd_rdata_r()));
+
+	nvgpu_falcon_writel(flcn, falcon_falcon_icd_cmd_r(),
+		falcon_falcon_icd_cmd_opc_rreg_f() |
+		falcon_falcon_icd_cmd_idx_f(FALCON_REG_EXCI));
+	nvgpu_err(g, "FALCON_REG_EXCI : 0x%x",
+		nvgpu_falcon_readl(flcn, falcon_falcon_icd_rdata_r()));
+
+	for (i = 0U; i < 6U; i++) {
+		nvgpu_falcon_writel(flcn, falcon_falcon_icd_cmd_r(),
+			falcon_falcon_icd_cmd_opc_rreg_f() |
+			falcon_falcon_icd_cmd_idx_f(
+			falcon_falcon_icd_cmd_opc_rstat_f()));
+		nvgpu_err(g, "FALCON_REG_RSTAT[%d] : 0x%x", i,
+			nvgpu_falcon_readl(flcn, falcon_falcon_icd_rdata_r()));
+	}
+
+	nvgpu_err(g, " FALCON REGISTERS DUMP");
+	nvgpu_err(g, "falcon_falcon_os_r : %d",
+		nvgpu_falcon_readl(flcn, falcon_falcon_os_r()));
+	nvgpu_err(g, "falcon_falcon_cpuctl_r : 0x%x",
+		nvgpu_falcon_readl(flcn, falcon_falcon_cpuctl_r()));
+	nvgpu_err(g, "falcon_falcon_idlestate_r : 0x%x",
+		nvgpu_falcon_readl(flcn, falcon_falcon_idlestate_r()));
+	nvgpu_err(g, "falcon_falcon_mailbox0_r : 0x%x",
+		nvgpu_falcon_readl(flcn, falcon_falcon_mailbox0_r()));
+	nvgpu_err(g, "falcon_falcon_mailbox1_r : 0x%x",
+		nvgpu_falcon_readl(flcn, falcon_falcon_mailbox1_r()));
+	nvgpu_err(g, "falcon_falcon_irqstat_r : 0x%x",
+		nvgpu_falcon_readl(flcn, falcon_falcon_irqstat_r()));
+	nvgpu_err(g, "falcon_falcon_irqmode_r : 0x%x",
+		nvgpu_falcon_readl(flcn, falcon_falcon_irqmode_r()));
+	nvgpu_err(g, "falcon_falcon_irqmask_r : 0x%x",
+		nvgpu_falcon_readl(flcn, falcon_falcon_irqmask_r()));
+	nvgpu_err(g, "falcon_falcon_irqdest_r : 0x%x",
+		nvgpu_falcon_readl(flcn, falcon_falcon_irqdest_r()));
+	nvgpu_err(g, "falcon_falcon_debug1_r : 0x%x",
+		nvgpu_falcon_readl(flcn, falcon_falcon_debug1_r()));
+	nvgpu_err(g, "falcon_falcon_debuginfo_r : 0x%x",
+		nvgpu_falcon_readl(flcn, falcon_falcon_debuginfo_r()));
+	nvgpu_err(g, "falcon_falcon_bootvec_r : 0x%x",
+		nvgpu_falcon_readl(flcn, falcon_falcon_bootvec_r()));
+	nvgpu_err(g, "falcon_falcon_hwcfg_r : 0x%x",
+		nvgpu_falcon_readl(flcn, falcon_falcon_hwcfg_r()));
+	nvgpu_err(g, "falcon_falcon_engctl_r : 0x%x",
+		nvgpu_falcon_readl(flcn, falcon_falcon_engctl_r()));
+	nvgpu_err(g, "falcon_falcon_curctx_r : 0x%x",
+		nvgpu_falcon_readl(flcn, falcon_falcon_curctx_r()));
+	nvgpu_err(g, "falcon_falcon_nxtctx_r : 0x%x",
+		nvgpu_falcon_readl(flcn, falcon_falcon_nxtctx_r()));
+	/*
+	 * Common Falcon code accesses each engine's falcon registers
+	 * using engine's falcon base address + offset.
+	 * So generate offset for falcon_falcon_exterrstat_r()
+	 * and falcon_falcon_exterraddr_r() registers by applying
+	 * the mask 0xFFF
+	 */
+	nvgpu_err(g, "falcon_falcon_exterrstat_r : 0x%x",
+		nvgpu_falcon_readl(flcn,
+			(falcon_falcon_exterrstat_r() & 0x0FFF)));
+	nvgpu_err(g, "falcon_falcon_exterraddr_r : 0x%x",
+		nvgpu_falcon_readl(flcn,
+			(falcon_falcon_exterraddr_r() & 0x0FFF)));
+}
+#endif

drivers/gpu/nvgpu/hal/fb/ecc/fb_ecc_gv11b.h

@@ -0,0 +1,47 @@
+/*
+ * GV11B FB ECC
+ *
+ * Copyright (c) 2020-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef NVGPU_FB_ECC_GV11B_H
+#define NVGPU_FB_ECC_GV11B_H
+
+struct gk20a;
+
+/*
+ * @brief Allocate and initialize counters for memories within FB.
+ *
+ * @param stat [in] Address of pointer to struct nvgpu_ecc_stat.
+ *
+ */
+#define NVGPU_ECC_COUNTER_INIT_FB(stat) \
+	nvgpu_ecc_counter_init(g, &g->ecc.fb.stat, #stat)
+
+#define NVGPU_ECC_COUNTER_FREE_FB(stat)	\
+	nvgpu_ecc_counter_deinit(g, &g->ecc.fb.stat)
+
+int gv11b_fb_ecc_init(struct gk20a *g);
+void gv11b_fb_ecc_free(struct gk20a *g);
+void gv11b_fb_ecc_l2tlb_error_mask(u32 *corrected_error_mask,
+		u32 *uncorrected_error_mask);
+
+#endif /* NVGPU_FB_ECC_GV11B_H */

drivers/gpu/nvgpu/hal/fb/ecc/fb_ecc_gv11b_fusa.c

@@ -0,0 +1,93 @@
+/*
+ * GV11B FB ECC
+ *
+ * Copyright (c) 2020-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+#include <nvgpu/log.h>
+#include <nvgpu/io.h>
+#include <nvgpu/gk20a.h>
+#include <nvgpu/nvgpu_err.h>
+
+#include "fb_ecc_gv11b.h"
+
+#include <nvgpu/hw/gv11b/hw_fb_gv11b.h>
+
+int gv11b_fb_ecc_init(struct gk20a *g)
+{
+	int err = 0;
+
+	err = NVGPU_ECC_COUNTER_INIT_FB(mmu_l2tlb_ecc_uncorrected_err_count);
+	if (err != 0) {
+		goto init_fb_ecc_err;
+	}
+	err = NVGPU_ECC_COUNTER_INIT_FB(mmu_l2tlb_ecc_corrected_err_count);
+	if (err != 0) {
+		goto init_fb_ecc_err;
+	}
+	err = NVGPU_ECC_COUNTER_INIT_FB(mmu_hubtlb_ecc_uncorrected_err_count);
+	if (err != 0) {
+		goto init_fb_ecc_err;
+	}
+	err = NVGPU_ECC_COUNTER_INIT_FB(mmu_hubtlb_ecc_corrected_err_count);
+	if (err != 0) {
+		goto init_fb_ecc_err;
+	}
+	err = NVGPU_ECC_COUNTER_INIT_FB(
+			mmu_fillunit_ecc_uncorrected_err_count);
+	if (err != 0) {
+		goto init_fb_ecc_err;
+	}
+	err = NVGPU_ECC_COUNTER_INIT_FB(
+			mmu_fillunit_ecc_corrected_err_count);
+	if (err != 0) {
+		goto init_fb_ecc_err;
+	}
+
+init_fb_ecc_err:
+
+	if (err != 0) {
+		nvgpu_err(g, "ecc counter allocate failed, err=%d", err);
+		gv11b_fb_ecc_free(g);
+	}
+
+	return err;
+}
+
+void gv11b_fb_ecc_free(struct gk20a *g)
+{
+	NVGPU_ECC_COUNTER_FREE_FB(mmu_l2tlb_ecc_corrected_err_count);
+	NVGPU_ECC_COUNTER_FREE_FB(mmu_l2tlb_ecc_uncorrected_err_count);
+	NVGPU_ECC_COUNTER_FREE_FB(mmu_hubtlb_ecc_corrected_err_count);
+	NVGPU_ECC_COUNTER_FREE_FB(mmu_hubtlb_ecc_uncorrected_err_count);
+	NVGPU_ECC_COUNTER_FREE_FB(mmu_fillunit_ecc_corrected_err_count);
+	NVGPU_ECC_COUNTER_FREE_FB(mmu_fillunit_ecc_uncorrected_err_count);
+}
+
+void gv11b_fb_ecc_l2tlb_error_mask(u32 *corrected_error_mask,
+		u32 *uncorrected_error_mask)
+{
+	*corrected_error_mask =
+		fb_mmu_l2tlb_ecc_status_corrected_err_l2tlb_sa_data_m();
+	*uncorrected_error_mask =
+		fb_mmu_l2tlb_ecc_status_uncorrected_err_l2tlb_sa_data_m();
+
+	return;
+}

drivers/gpu/nvgpu/hal/fb/fb_gm20b.c

@@ -0,0 +1,124 @@
+/*
+ * GM20B GPC MMU
+ *
+ * Copyright (c) 2014-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/trace.h>
+#include <nvgpu/sizes.h>
+#include <nvgpu/utils.h>
+#include <nvgpu/gk20a.h>
+#include <nvgpu/bug.h>
+#include <nvgpu/ltc.h>
+
+#include "fb_gm20b.h"
+
+#include <nvgpu/io.h>
+#include <nvgpu/timers.h>
+
+#include <nvgpu/hw/gm20b/hw_fb_gm20b.h>
+
+#define VPR_INFO_FETCH_WAIT	(5)
+#define WPR_INFO_ADDR_ALIGNMENT 0x0000000c
+
+void fb_gm20b_init_fs_state(struct gk20a *g)
+{
+	nvgpu_log_info(g, "initialize gm20b fb");
+
+	gk20a_writel(g, fb_fbhub_num_active_ltcs_r(),
+			g->ops.priv_ring.enum_ltc(g));
+
+	if (!nvgpu_is_enabled(g, NVGPU_SEC_PRIVSECURITY)) {
+		/* Bypass MMU check for non-secure boot. For
+		 * secure-boot,this register write has no-effect */
+		gk20a_writel(g, fb_priv_mmu_phy_secure_r(), 0xffffffffU);
+	}
+}
+
+void gm20b_fb_set_mmu_page_size(struct gk20a *g)
+{
+	/* set large page size in fb */
+	u32 fb_mmu_ctrl = gk20a_readl(g, fb_mmu_ctrl_r());
+	fb_mmu_ctrl |= fb_mmu_ctrl_use_pdb_big_page_size_true_f();
+	gk20a_writel(g, fb_mmu_ctrl_r(), fb_mmu_ctrl);
+}
+
+#ifdef CONFIG_NVGPU_COMPRESSION
+bool gm20b_fb_set_use_full_comp_tag_line(struct gk20a *g)
+{
+	/* set large page size in fb */
+	u32 fb_mmu_ctrl = gk20a_readl(g, fb_mmu_ctrl_r());
+
+	fb_mmu_ctrl |= fb_mmu_ctrl_use_full_comp_tag_line_true_f();
+	gk20a_writel(g, fb_mmu_ctrl_r(), fb_mmu_ctrl);
+
+	return true;
+}
+
+u64 gm20b_fb_compression_page_size(struct gk20a *g)
+{
+	return SZ_128K;
+}
+
+unsigned int gm20b_fb_compressible_page_size(struct gk20a *g)
+{
+	return (unsigned int)SZ_64K;
+}
+
+u64 gm20b_fb_compression_align_mask(struct gk20a *g)
+{
+	return SZ_64K - 1UL;
+}
+#endif
+
+#ifdef CONFIG_NVGPU_DEBUGGER
+bool gm20b_fb_debug_mode_enabled(struct gk20a *g)
+{
+	u32 debug_ctrl = gk20a_readl(g, fb_mmu_debug_ctrl_r());
+
+	return fb_mmu_debug_ctrl_debug_v(debug_ctrl) ==
+			fb_mmu_debug_ctrl_debug_enabled_v();
+}
+
+void gm20b_fb_set_mmu_debug_mode(struct gk20a *g, bool enable)
+{
+	u32 reg_val, fb_debug_ctrl;
+
+	if (enable) {
+		fb_debug_ctrl = fb_mmu_debug_ctrl_debug_enabled_f();
+		g->mmu_debug_ctrl = true;
+	} else {
+		fb_debug_ctrl = fb_mmu_debug_ctrl_debug_disabled_f();
+		g->mmu_debug_ctrl = false;
+	}
+
+	reg_val = nvgpu_readl(g, fb_mmu_debug_ctrl_r());
+	reg_val = set_field(reg_val,
+			fb_mmu_debug_ctrl_debug_m(), fb_debug_ctrl);
+	nvgpu_writel(g, fb_mmu_debug_ctrl_r(), reg_val);
+}
+
+void gm20b_fb_set_debug_mode(struct gk20a *g, bool enable)
+{
+	gm20b_fb_set_mmu_debug_mode(g, enable);
+	g->ops.gr.set_debug_mode(g, enable);
+}
+#endif

drivers/gpu/nvgpu/hal/fb/fb_gm20b.h

@@ -0,0 +1,59 @@
+/*
+ * GM20B FB
+ *
+ * Copyright (c) 2014-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef NVGPU_FB_GM20B
+#define NVGPU_FB_GM20B
+
+#include <nvgpu/types.h>
+
+struct gk20a;
+struct nvgpu_mem;
+
+void gm20b_fb_init_hw(struct gk20a *g);
+int gm20b_fb_tlb_invalidate(struct gk20a *g, struct nvgpu_mem *pdb);
+#ifdef CONFIG_NVGPU_HAL_NON_FUSA
+void fb_gm20b_init_fs_state(struct gk20a *g);
+void gm20b_fb_set_mmu_page_size(struct gk20a *g);
+#endif
+u32 gm20b_fb_mmu_ctrl(struct gk20a *g);
+u32 gm20b_fb_mmu_debug_ctrl(struct gk20a *g);
+u32 gm20b_fb_mmu_debug_wr(struct gk20a *g);
+u32 gm20b_fb_mmu_debug_rd(struct gk20a *g);
+#ifdef CONFIG_NVGPU_COMPRESSION
+bool gm20b_fb_set_use_full_comp_tag_line(struct gk20a *g);
+u64 gm20b_fb_compression_page_size(struct gk20a *g);
+unsigned int gm20b_fb_compressible_page_size(struct gk20a *g);
+u64 gm20b_fb_compression_align_mask(struct gk20a *g);
+#endif
+void gm20b_fb_dump_vpr_info(struct gk20a *g);
+void gm20b_fb_dump_wpr_info(struct gk20a *g);
+void gm20b_fb_read_wpr_info(struct gk20a *g, u64 *wpr_base, u64 *wpr_size);
+int gm20b_fb_vpr_info_fetch(struct gk20a *g);
+#ifdef CONFIG_NVGPU_DEBUGGER
+bool gm20b_fb_debug_mode_enabled(struct gk20a *g);
+void gm20b_fb_set_debug_mode(struct gk20a *g, bool enable);
+void gm20b_fb_set_mmu_debug_mode(struct gk20a *g, bool enable);
+#endif
+
+#endif

drivers/gpu/nvgpu/hal/fb/fb_gm20b_fusa.c

@@ -0,0 +1,300 @@
+/*
+ * GM20B GPC MMU
+ *
+ * Copyright (c) 2014-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/trace.h>
+#include <nvgpu/sizes.h>
+#include <nvgpu/utils.h>
+#include <nvgpu/gk20a.h>
+#include <nvgpu/bug.h>
+#include <nvgpu/ltc.h>
+#include <nvgpu/nvgpu_init.h>
+
+#include "fb_gm20b.h"
+
+#include <nvgpu/io.h>
+#include <nvgpu/timers.h>
+
+#include <nvgpu/hw/gm20b/hw_fb_gm20b.h>
+
+#define VPR_INFO_FETCH_WAIT	(5)
+#define WPR_INFO_ADDR_ALIGNMENT 0x0000000c
+
+void gm20b_fb_init_hw(struct gk20a *g)
+{
+	u64 addr = nvgpu_mem_get_addr(g, &g->mm.sysmem_flush) >> 8;
+
+	nvgpu_assert(u64_hi32(addr) == 0U);
+	gk20a_writel(g, fb_niso_flush_sysmem_addr_r(), U32(addr));
+
+	/* init mmu debug buffer */
+	addr = nvgpu_mem_get_addr(g, &g->mm.mmu_wr_mem);
+	addr >>= fb_mmu_debug_wr_addr_alignment_v();
+
+	nvgpu_assert(u64_hi32(addr) == 0U);
+	gk20a_writel(g, fb_mmu_debug_wr_r(),
+		     nvgpu_aperture_mask(g, &g->mm.mmu_wr_mem,
+				fb_mmu_debug_wr_aperture_sys_mem_ncoh_f(),
+				fb_mmu_debug_wr_aperture_sys_mem_coh_f(),
+				fb_mmu_debug_wr_aperture_vid_mem_f()) |
+		     fb_mmu_debug_wr_vol_false_f() |
+		     fb_mmu_debug_wr_addr_f(U32(addr)));
+
+	addr = nvgpu_mem_get_addr(g, &g->mm.mmu_rd_mem);
+	addr >>= fb_mmu_debug_rd_addr_alignment_v();
+
+	nvgpu_assert(u64_hi32(addr) == 0U);
+	gk20a_writel(g, fb_mmu_debug_rd_r(),
+		     nvgpu_aperture_mask(g, &g->mm.mmu_rd_mem,
+				fb_mmu_debug_wr_aperture_sys_mem_ncoh_f(),
+				fb_mmu_debug_wr_aperture_sys_mem_coh_f(),
+				fb_mmu_debug_rd_aperture_vid_mem_f()) |
+		     fb_mmu_debug_rd_vol_false_f() |
+		     fb_mmu_debug_rd_addr_f(U32(addr)));
+}
+
+int gm20b_fb_tlb_invalidate(struct gk20a *g, struct nvgpu_mem *pdb)
+{
+	struct nvgpu_timeout timeout;
+	u32 addr_lo;
+	u32 data;
+	int err = 0;
+
+	nvgpu_log_fn(g, " ");
+
+	/* pagetables are considered sw states which are preserved after
+	   prepare_poweroff. When gk20a deinit releases those pagetables,
+	   common code in vm unmap path calls tlb invalidate that touches
+	   hw. Use the power_on flag to skip tlb invalidation when gpu
+	   power is turned off */
+
+	if (nvgpu_is_powered_off(g)) {
+		return err;
+	}
+
+	addr_lo = u64_lo32(nvgpu_mem_get_addr(g, pdb) >> 12);
+
+	nvgpu_mutex_acquire(&g->mm.tlb_lock);
+
+#ifdef CONFIG_NVGPU_TRACE
+	trace_gk20a_mm_tlb_invalidate(g->name);
+#endif
+
+	err = nvgpu_timeout_init(g, &timeout, 1000, NVGPU_TIMER_RETRY_TIMER);
+	if (err != 0) {
+		nvgpu_err(g, "nvgpu_timeout_init(mmu fifo space) failed err=%d",
+			err);
+		goto out;
+	}
+
+	do {
+		data = gk20a_readl(g, fb_mmu_ctrl_r());
+		if (fb_mmu_ctrl_pri_fifo_space_v(data) != 0U) {
+			break;
+		}
+		nvgpu_udelay(2);
+	} while (nvgpu_timeout_expired_msg(&timeout,
+					 "wait mmu fifo space") == 0);
+
+	if (nvgpu_timeout_peek_expired(&timeout)) {
+		err = -ETIMEDOUT;
+		goto out;
+	}
+
+	err = nvgpu_timeout_init(g, &timeout, 1000, NVGPU_TIMER_RETRY_TIMER);
+	if (err != 0) {
+		nvgpu_err(g, "nvgpu_timeout_init(mmu invalidate) failed err=%d",
+			err);
+		goto out;
+	}
+
+	gk20a_writel(g, fb_mmu_invalidate_pdb_r(),
+		fb_mmu_invalidate_pdb_addr_f(addr_lo) |
+		nvgpu_aperture_mask(g, pdb,
+				fb_mmu_invalidate_pdb_aperture_sys_mem_f(),
+				fb_mmu_invalidate_pdb_aperture_sys_mem_f(),
+				fb_mmu_invalidate_pdb_aperture_vid_mem_f()));
+
+	gk20a_writel(g, fb_mmu_invalidate_r(),
+		fb_mmu_invalidate_all_va_true_f() |
+		fb_mmu_invalidate_trigger_true_f());
+
+	do {
+		data = gk20a_readl(g, fb_mmu_ctrl_r());
+		if (fb_mmu_ctrl_pri_fifo_empty_v(data) !=
+			fb_mmu_ctrl_pri_fifo_empty_false_f()) {
+			break;
+		}
+		nvgpu_udelay(2);
+	} while (nvgpu_timeout_expired_msg(&timeout,
+					 "wait mmu invalidate") == 0);
+
+#ifdef CONFIG_NVGPU_TRACE
+	trace_gk20a_mm_tlb_invalidate_done(g->name);
+#endif
+
+out:
+	nvgpu_mutex_release(&g->mm.tlb_lock);
+	return err;
+}
+
+u32 gm20b_fb_mmu_ctrl(struct gk20a *g)
+{
+	return gk20a_readl(g, fb_mmu_ctrl_r());
+}
+
+u32 gm20b_fb_mmu_debug_ctrl(struct gk20a *g)
+{
+	return gk20a_readl(g, fb_mmu_debug_ctrl_r());
+}
+
+u32 gm20b_fb_mmu_debug_wr(struct gk20a *g)
+{
+	return gk20a_readl(g, fb_mmu_debug_wr_r());
+}
+
+u32 gm20b_fb_mmu_debug_rd(struct gk20a *g)
+{
+	return gk20a_readl(g, fb_mmu_debug_rd_r());
+}
+
+void gm20b_fb_dump_vpr_info(struct gk20a *g)
+{
+	u32 val;
+	u32 addr_lo, addr_hi, cya_lo, cya_hi;
+
+	/* print vpr info */
+	val = gk20a_readl(g, fb_mmu_vpr_info_r());
+	val &= ~0x3U;
+	val |= fb_mmu_vpr_info_index_addr_lo_v();
+	gk20a_writel(g, fb_mmu_vpr_info_r(), val);
+
+	addr_lo = gk20a_readl(g, fb_mmu_vpr_info_r());
+	addr_hi = gk20a_readl(g, fb_mmu_vpr_info_r());
+	cya_lo = gk20a_readl(g, fb_mmu_vpr_info_r());
+	cya_hi = gk20a_readl(g, fb_mmu_vpr_info_r());
+
+	nvgpu_err(g, "VPR: %08x %08x %08x %08x",
+		addr_lo, addr_hi, cya_lo, cya_hi);
+}
+
+void gm20b_fb_dump_wpr_info(struct gk20a *g)
+{
+	u32 val;
+	u32 allow_read, allow_write;
+	u32 wpr1_addr_lo, wpr1_addr_hi;
+	u32 wpr2_addr_lo, wpr2_addr_hi;
+
+	/* print wpr info */
+	val = gk20a_readl(g, fb_mmu_wpr_info_r());
+	val &= ~0xfU;
+	val |= (fb_mmu_wpr_info_index_allow_read_v());
+	gk20a_writel(g, fb_mmu_wpr_info_r(), val);
+
+	allow_read = gk20a_readl(g, fb_mmu_wpr_info_r());
+	allow_write = gk20a_readl(g, fb_mmu_wpr_info_r());
+	wpr1_addr_lo = gk20a_readl(g, fb_mmu_wpr_info_r());
+	wpr1_addr_hi = gk20a_readl(g, fb_mmu_wpr_info_r());
+	wpr2_addr_lo = gk20a_readl(g, fb_mmu_wpr_info_r());
+	wpr2_addr_hi = gk20a_readl(g, fb_mmu_wpr_info_r());
+
+	nvgpu_err(g, "WPR: %08x %08x %08x %08x %08x %08x",
+		allow_read, allow_write,
+		wpr1_addr_lo, wpr1_addr_hi,
+		wpr2_addr_lo, wpr2_addr_hi);
+}
+
+static int gm20b_fb_vpr_info_fetch_wait(struct gk20a *g,
+					    unsigned int msec)
+{
+	struct nvgpu_timeout timeout;
+	int err = 0;
+
+	err = nvgpu_timeout_init(g, &timeout, msec, NVGPU_TIMER_CPU_TIMER);
+	if (err != 0) {
+		nvgpu_err(g, "nvgpu_timeout_init failed err=%d", err);
+		return err;
+	}
+
+	do {
+		u32 val;
+
+		val = gk20a_readl(g, fb_mmu_vpr_info_r());
+		if (fb_mmu_vpr_info_fetch_v(val) ==
+		    fb_mmu_vpr_info_fetch_false_v()) {
+			return 0;
+		}
+
+	} while (nvgpu_timeout_expired(&timeout) == 0);
+
+	return -ETIMEDOUT;
+}
+
+int gm20b_fb_vpr_info_fetch(struct gk20a *g)
+{
+	int err;
+
+	err = gm20b_fb_vpr_info_fetch_wait(g, VPR_INFO_FETCH_WAIT);
+	if (err != 0) {
+		return err;
+	}
+
+	gk20a_writel(g, fb_mmu_vpr_info_r(),
+			fb_mmu_vpr_info_fetch_true_v());
+
+	err = gm20b_fb_vpr_info_fetch_wait(g, VPR_INFO_FETCH_WAIT);
+	if (err != 0) {
+		nvgpu_err(g, "gm20b_fb_vpr_info_fetch_wait failed!");
+	}
+	return err;
+}
+
+void gm20b_fb_read_wpr_info(struct gk20a *g, u64 *wpr_base, u64 *wpr_size)
+{
+	u32 val = 0;
+	u64 wpr_start = 0;
+	u64 wpr_end = 0;
+
+	val = gk20a_readl(g, fb_mmu_wpr_info_r());
+	val &= ~0xFU;
+	val |= fb_mmu_wpr_info_index_wpr1_addr_lo_v();
+	gk20a_writel(g, fb_mmu_wpr_info_r(), val);
+
+	val = gk20a_readl(g, fb_mmu_wpr_info_r()) >> 0x4;
+	wpr_start = hi32_lo32_to_u64(
+			(val >> (32 - WPR_INFO_ADDR_ALIGNMENT)),
+			(val << WPR_INFO_ADDR_ALIGNMENT));
+
+	val = gk20a_readl(g, fb_mmu_wpr_info_r());
+	val &= ~0xFU;
+	val |= fb_mmu_wpr_info_index_wpr1_addr_hi_v();
+	gk20a_writel(g, fb_mmu_wpr_info_r(), val);
+
+	val = gk20a_readl(g, fb_mmu_wpr_info_r()) >> 0x4;
+	wpr_end = hi32_lo32_to_u64(
+			(val >> (32 - WPR_INFO_ADDR_ALIGNMENT)),
+			(val << WPR_INFO_ADDR_ALIGNMENT));
+
+	*wpr_base = wpr_start;
+	*wpr_size = nvgpu_safe_sub_u64(wpr_end, wpr_start);
+}

drivers/gpu/nvgpu/hal/fb/fb_gp106.c

@@ -0,0 +1,71 @@
+/*
+ * Copyright (c) 2016-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/io.h>
+#include <nvgpu/timers.h>
+#include <nvgpu/gk20a.h>
+
+#include "fb_gp10b.h"
+#include "fb_gp106.h"
+
+#include <nvgpu/hw/gp106/hw_fb_gp106.h>
+
+#define HW_SCRUB_TIMEOUT_DEFAULT	100 /* usec */
+#define HW_SCRUB_TIMEOUT_MAX		2000000 /* usec */
+
+void gp106_fb_init_fs_state(struct gk20a *g)
+{
+	u32 val;
+
+	int retries = HW_SCRUB_TIMEOUT_MAX / HW_SCRUB_TIMEOUT_DEFAULT;
+	/* wait for memory to be accessible */
+	do {
+		u32 w = gk20a_readl(g, fb_niso_scrub_status_r());
+		if (fb_niso_scrub_status_flag_v(w) != 0U) {
+			nvgpu_log_fn(g, "done");
+			break;
+		}
+		nvgpu_udelay(HW_SCRUB_TIMEOUT_DEFAULT);
+		--retries;
+	} while (retries != 0);
+
+	val = gk20a_readl(g, fb_mmu_priv_level_mask_r());
+	val &= ~fb_mmu_priv_level_mask_write_violation_m();
+	gk20a_writel(g, fb_mmu_priv_level_mask_r(), val);
+}
+
+#ifdef CONFIG_NVGPU_DGPU
+size_t gp106_fb_get_vidmem_size(struct gk20a *g)
+{
+	u32 range = gk20a_readl(g, fb_mmu_local_memory_range_r());
+	u32 mag = fb_mmu_local_memory_range_lower_mag_v(range);
+	u32 scale = fb_mmu_local_memory_range_lower_scale_v(range);
+	u32 ecc = fb_mmu_local_memory_range_ecc_mode_v(range);
+	size_t bytes = ((size_t)mag << scale) * SZ_1M;
+
+	if (ecc != 0U) {
+		bytes = bytes / 16U * 15U;
+	}
+
+	return bytes;
+}
+#endif

drivers/gpu/nvgpu/hal/fb/fb_gp106.h

@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2016-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef FB_GP106_H
+#define FB_GP106_H
+struct gpu_ops;
+
+void gp106_fb_init_fs_state(struct gk20a *g);
+#ifdef CONFIG_NVGPU_DGPU
+size_t gp106_fb_get_vidmem_size(struct gk20a *g);
+#endif
+#endif

drivers/gpu/nvgpu/hal/fb/fb_gp10b.c

@@ -0,0 +1,40 @@
+/*
+ * GP10B FB
+ *
+ * Copyright (c) 2014-2019, NVIDIA CORPORATION.  All rights reserved.
+*
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/sizes.h>
+#include <nvgpu/gk20a.h>
+
+#include "fb_gp10b.h"
+
+#ifdef CONFIG_NVGPU_COMPRESSION
+u64 gp10b_fb_compression_page_size(struct gk20a *g)
+{
+	return SZ_64K;
+}
+
+unsigned int gp10b_fb_compressible_page_size(struct gk20a *g)
+{
+	return (unsigned int)SZ_4K;
+}
+#endif

drivers/gpu/nvgpu/hal/fb/fb_gp10b.h

@@ -0,0 +1,34 @@
+/*
+ * GP10B FB
+ *
+ * Copyright (c) 2014-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef NVGPU_FB_GP10B_H
+#define NVGPU_FB_GP10B_H
+struct gk20a;
+
+#ifdef CONFIG_NVGPU_COMPRESSION
+u64 gp10b_fb_compression_page_size(struct gk20a *g);
+unsigned int gp10b_fb_compressible_page_size(struct gk20a *g);
+#endif
+
+#endif /* NVGPU_FB_GP10B_H */

drivers/gpu/nvgpu/hal/fb/fb_gv100.c

@@ -0,0 +1,309 @@
+/*
+ * GV100 FB
+ *
+ * Copyright (c) 2017-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/types.h>
+
+#include <nvgpu/dma.h>
+#include <nvgpu/log.h>
+#include <nvgpu/enabled.h>
+#include <nvgpu/gmmu.h>
+#include <nvgpu/nvgpu_common.h>
+#include <nvgpu/kmem.h>
+#include <nvgpu/nvgpu_mem.h>
+#include <nvgpu/firmware.h>
+#include <nvgpu/pmu.h>
+#include <nvgpu/falcon.h>
+#include <nvgpu/io.h>
+#include <nvgpu/utils.h>
+#include <nvgpu/timers.h>
+#include <nvgpu/gk20a.h>
+#include <nvgpu/firmware.h>
+#include <nvgpu/mc.h>
+
+#include "fb_gv100.h"
+
+#include <nvgpu/hw/gv100/hw_fb_gv100.h>
+
+#define HW_SCRUB_TIMEOUT_DEFAULT	100 /* usec */
+#define HW_SCRUB_TIMEOUT_MAX		2000000 /* usec */
+#define MEM_UNLOCK_TIMEOUT			3500 /* msec */
+
+#define MEM_UNLOCK_PROD_BIN		"mem_unlock.bin"
+#define MEM_UNLOCK_DBG_BIN		"mem_unlock_dbg.bin"
+
+struct mem_unlock_bin_hdr {
+	u32 bin_magic;
+	u32 bin_ver;
+	u32 bin_size;
+	u32 header_offset;
+	u32 data_offset;
+	u32 data_size;
+};
+
+struct mem_unlock_fw_header {
+	u32 sig_dbg_offset;
+	u32 sig_dbg_size;
+	u32 sig_prod_offset;
+	u32 sig_prod_size;
+	u32 patch_loc;
+	u32 patch_sig;
+	u32 hdr_offset;
+	u32 hdr_size;
+};
+
+void gv100_fb_reset(struct gk20a *g)
+{
+	u32 val;
+	int retries = HW_SCRUB_TIMEOUT_MAX / HW_SCRUB_TIMEOUT_DEFAULT;
+
+	nvgpu_log_info(g, "reset gv100 fb");
+
+	/* wait for memory to be accessible */
+	do {
+		u32 w = gk20a_readl(g, fb_niso_scrub_status_r());
+		if (fb_niso_scrub_status_flag_v(w) != 0U) {
+			nvgpu_log_info(g, "done");
+			break;
+		}
+		nvgpu_udelay(HW_SCRUB_TIMEOUT_DEFAULT);
+		--retries;
+	} while (retries != 0);
+
+	val = gk20a_readl(g, fb_mmu_priv_level_mask_r());
+	val &= ~fb_mmu_priv_level_mask_write_violation_m();
+	gk20a_writel(g, fb_mmu_priv_level_mask_r(), val);
+}
+
+/*
+ * Patch signatures into ucode image
+ */
+static int fb_ucode_patch_sig(struct gk20a *g,
+	unsigned int *p_img, unsigned int *p_prod_sig,
+	unsigned int *p_dbg_sig, unsigned int *p_patch_loc,
+	unsigned int *p_patch_ind, u32 sig_size)
+{
+	unsigned int i, j, *p_sig;
+
+	if (!g->ops.pmu.is_debug_mode_enabled(g)) {
+		p_sig = p_prod_sig;
+	} else {
+		p_sig = p_dbg_sig;
+	}
+
+	/* Patching logic:*/
+	sig_size = sig_size / 4U;
+	for (i = 0U; i < sizeof(*p_patch_loc)>>2U; i++) {
+		for (j = 0U; j < sig_size; j++) {
+			p_img[nvgpu_safe_add_u32((p_patch_loc[i]>>2U), j)] =
+				p_sig[nvgpu_safe_add_u32((p_patch_ind[i]<<2U),
+					j)];
+		}
+	}
+
+	return 0;
+}
+
+int gv100_fb_memory_unlock(struct gk20a *g)
+{
+	struct nvgpu_firmware *mem_unlock_fw = NULL;
+	struct mem_unlock_bin_hdr *hs_bin_hdr = NULL;
+	struct mem_unlock_fw_header *fw_hdr = NULL;
+	u32 *ucode_header = NULL;
+	u32 *ucode = NULL;
+	u32 data = 0;
+	int err = 0;
+
+	nvgpu_log_fn(g, " ");
+
+	/*
+	 * mem_unlock.bin should be written to install
+	 * traps even if VPR isn’t actually supported
+	 */
+	if (!g->ops.pmu.is_debug_mode_enabled(g)) {
+		mem_unlock_fw = nvgpu_request_firmware(g, MEM_UNLOCK_PROD_BIN, 0);
+	} else {
+		mem_unlock_fw = nvgpu_request_firmware(g, MEM_UNLOCK_DBG_BIN, 0);
+	}
+	if (mem_unlock_fw == NULL) {
+		nvgpu_err(g, "mem unlock ucode get fail");
+		err = -ENOENT;
+		goto exit;
+	}
+
+	/* Enable nvdec */
+	err = nvgpu_mc_reset_units(g, NVGPU_UNIT_NVDEC);
+	if (err != 0) {
+		nvgpu_err(g, "Failed to reset NVDEC unit");
+	}
+
+	hs_bin_hdr = (struct mem_unlock_bin_hdr *)(void *)mem_unlock_fw->data;
+	fw_hdr = (struct mem_unlock_fw_header *)(void *)(mem_unlock_fw->data +
+			hs_bin_hdr->header_offset);
+	ucode_header = (u32 *)(void *)(mem_unlock_fw->data +
+					fw_hdr->hdr_offset);
+	ucode = (u32 *)(void *)(mem_unlock_fw->data + hs_bin_hdr->data_offset);
+
+	/* Patch Ucode signatures */
+	if (fb_ucode_patch_sig(g, ucode,
+		(u32 *)(void *)(mem_unlock_fw->data + fw_hdr->sig_prod_offset),
+		(u32 *)(void *)(mem_unlock_fw->data + fw_hdr->sig_dbg_offset),
+		(u32 *)(void *)(mem_unlock_fw->data + fw_hdr->patch_loc),
+		(u32 *)(void *)(mem_unlock_fw->data + fw_hdr->patch_sig),
+		fw_hdr->sig_dbg_size) < 0) {
+		nvgpu_err(g, "mem unlock ucode patch signatures fail");
+		err = -EPERM;
+		goto exit;
+	}
+
+	err = nvgpu_falcon_hs_ucode_load_bootstrap(&g->nvdec_flcn, ucode,
+			ucode_header);
+	if (err != 0) {
+		nvgpu_err(g, "mem unlock ucode load & bootstrap failed");
+		goto exit;
+	}
+
+	if (nvgpu_falcon_wait_for_halt(&g->nvdec_flcn,
+		MEM_UNLOCK_TIMEOUT) != 0) {
+		nvgpu_err(g, "mem unlock ucode boot timed out");
+#ifdef CONFIG_NVGPU_FALCON_DEBUG
+		nvgpu_falcon_dump_stats(&g->nvdec_flcn);
+#endif
+		goto exit;
+	}
+
+	data = nvgpu_falcon_mailbox_read(&g->nvdec_flcn, FALCON_MAILBOX_0);
+	if (data != 0U) {
+		nvgpu_err(g, "mem unlock ucode boot failed, err %x", data);
+		goto exit;
+	}
+
+exit:
+	if (mem_unlock_fw != NULL) {
+		nvgpu_release_firmware(g, mem_unlock_fw);
+	}
+
+	nvgpu_log_fn(g, "done, status - %d", err);
+
+	return err;
+}
+
+int gv100_fb_init_nvlink(struct gk20a *g)
+{
+	u32 data;
+	u32 mask = g->nvlink.enabled_links;
+
+	/* Map enabled link to SYSMEM */
+	data = nvgpu_readl(g, fb_hshub_config0_r());
+	data = set_field(data, fb_hshub_config0_sysmem_nvlink_mask_m(),
+			fb_hshub_config0_sysmem_nvlink_mask_f(mask));
+	nvgpu_writel(g, fb_hshub_config0_r(), data);
+
+	return 0;
+}
+
+int gv100_fb_enable_nvlink(struct gk20a *g)
+{
+	u32 data;
+
+	nvgpu_log(g, gpu_dbg_nvlink|gpu_dbg_info, "enabling nvlink");
+
+	/* Enable nvlink for NISO FBHUB */
+	data = nvgpu_readl(g, fb_niso_cfg1_r());
+	data = set_field(data, fb_niso_cfg1_sysmem_nvlink_m(),
+		fb_niso_cfg1_sysmem_nvlink_enabled_f());
+	nvgpu_writel(g, fb_niso_cfg1_r(), data);
+
+	return 0;
+}
+
+int gv100_fb_set_atomic_mode(struct gk20a *g)
+{
+	u32 data;
+
+	/* Setup atomics */
+	data = nvgpu_readl(g, fb_mmu_ctrl_r());
+	data = set_field(data, fb_mmu_ctrl_atomic_capability_mode_m(),
+		fb_mmu_ctrl_atomic_capability_mode_rmw_f());
+	nvgpu_writel(g, fb_mmu_ctrl_r(), data);
+
+	data = nvgpu_readl(g, fb_hsmmu_pri_mmu_ctrl_r());
+	data = set_field(data, fb_hsmmu_pri_mmu_ctrl_atomic_capability_mode_m(),
+		    fb_hsmmu_pri_mmu_ctrl_atomic_capability_mode_rmw_f());
+	nvgpu_writel(g, fb_hsmmu_pri_mmu_ctrl_r(), data);
+
+	data = nvgpu_readl(g, fb_fbhub_num_active_ltcs_r());
+	data = set_field(data, fb_fbhub_num_active_ltcs_hub_sys_atomic_mode_m(),
+		    fb_fbhub_num_active_ltcs_hub_sys_atomic_mode_use_rmw_f());
+	nvgpu_writel(g, fb_fbhub_num_active_ltcs_r(), data);
+
+	data = nvgpu_readl(g, fb_hshub_num_active_ltcs_r());
+	data = set_field(data, fb_hshub_num_active_ltcs_hub_sys_atomic_mode_m(),
+		    fb_hshub_num_active_ltcs_hub_sys_atomic_mode_use_rmw_f());
+	nvgpu_writel(g, fb_hshub_num_active_ltcs_r(), data);
+
+	return 0;
+}
+
+#ifdef CONFIG_NVGPU_DGPU
+size_t gv100_fb_get_vidmem_size(struct gk20a *g)
+{
+	u32 range = gk20a_readl(g, fb_mmu_local_memory_range_r());
+	u32 mag = fb_mmu_local_memory_range_lower_mag_v(range);
+	u32 scale = fb_mmu_local_memory_range_lower_scale_v(range);
+	u32 ecc = fb_mmu_local_memory_range_ecc_mode_v(range);
+	size_t bytes = ((size_t)mag << scale) * SZ_1M;
+
+	if (ecc != 0U) {
+		bytes = bytes / 16U * 15U;
+	}
+
+	return bytes;
+}
+#endif
+
+#ifdef CONFIG_NVGPU_DEBUGGER
+void gv100_fb_set_mmu_debug_mode(struct gk20a *g, bool enable)
+{
+	u32 data, fb_ctrl, hsmmu_ctrl;
+
+	if (enable) {
+		fb_ctrl = fb_mmu_debug_ctrl_debug_enabled_f();
+		hsmmu_ctrl = fb_hsmmu_pri_mmu_debug_ctrl_debug_enabled_f();
+		g->mmu_debug_ctrl = true;
+	} else {
+		fb_ctrl = fb_mmu_debug_ctrl_debug_disabled_f();
+		hsmmu_ctrl = fb_hsmmu_pri_mmu_debug_ctrl_debug_disabled_f();
+		g->mmu_debug_ctrl = false;
+	}
+
+	data = nvgpu_readl(g, fb_mmu_debug_ctrl_r());
+	data = set_field(data, fb_mmu_debug_ctrl_debug_m(), fb_ctrl);
+	nvgpu_writel(g, fb_mmu_debug_ctrl_r(), data);
+
+	data = nvgpu_readl(g, fb_hsmmu_pri_mmu_debug_ctrl_r());
+	data = set_field(data,
+			fb_hsmmu_pri_mmu_debug_ctrl_debug_m(), hsmmu_ctrl);
+	nvgpu_writel(g, fb_hsmmu_pri_mmu_debug_ctrl_r(), data);
+}
+#endif

drivers/gpu/nvgpu/hal/fb/fb_gv100.h

@@ -0,0 +1,40 @@
+/*
+ * GV100 FB
+ *
+ * Copyright (c) 2017-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef NVGPU_FB_GV100_H
+#define NVGPU_FB_GV100_H
+
+struct gk20a;
+
+void gv100_fb_reset(struct gk20a *g);
+int gv100_fb_memory_unlock(struct gk20a *g);
+int gv100_fb_init_nvlink(struct gk20a *g);
+int gv100_fb_enable_nvlink(struct gk20a *g);
+int gv100_fb_set_atomic_mode(struct gk20a *g);
+#ifdef CONFIG_NVGPU_DGPU
+size_t gv100_fb_get_vidmem_size(struct gk20a *g);
+#endif
+void gv100_fb_set_mmu_debug_mode(struct gk20a *g, bool enable);
+
+#endif /* NVGPU_FB_GV100_H */

drivers/gpu/nvgpu/hal/fb/fb_gv11b.c

@@ -0,0 +1,103 @@
+/*
+ * GV11B FB
+ *
+ * Copyright (c) 2016-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/dma.h>
+#include <nvgpu/log.h>
+#include <nvgpu/enabled.h>
+#include <nvgpu/gmmu.h>
+#include <nvgpu/barrier.h>
+#include <nvgpu/bug.h>
+#include <nvgpu/soc.h>
+#include <nvgpu/ptimer.h>
+#include <nvgpu/io.h>
+#include <nvgpu/utils.h>
+#include <nvgpu/timers.h>
+#include <nvgpu/fifo.h>
+#include <nvgpu/gk20a.h>
+#include <nvgpu/nvgpu_err.h>
+#include <nvgpu/ltc.h>
+#include <nvgpu/rc.h>
+#include <nvgpu/engines.h>
+
+#include "fb_gm20b.h"
+#include "fb_gp10b.h"
+#include "fb_gv11b.h"
+
+#include <nvgpu/hw/gv11b/hw_fb_gv11b.h>
+
+#ifdef CONFIG_NVGPU_COMPRESSION
+void gv11b_fb_cbc_configure(struct gk20a *g, struct nvgpu_cbc *cbc)
+{
+	u32 compbit_base_post_divide;
+	u64 compbit_base_post_multiply64;
+	u64 compbit_store_iova;
+	u64 compbit_base_post_divide64;
+
+#ifdef CONFIG_NVGPU_SIM
+	if (nvgpu_is_enabled(g, NVGPU_IS_FMODEL)) {
+		compbit_store_iova = nvgpu_mem_get_phys_addr(g,
+						&cbc->compbit_store.mem);
+	} else
+#endif
+	{
+		compbit_store_iova = nvgpu_mem_get_addr(g,
+						&cbc->compbit_store.mem);
+	}
+	/* must be aligned to 64 KB */
+	compbit_store_iova = round_up(compbit_store_iova, (u64)SZ_64K);
+
+	compbit_base_post_divide64 = compbit_store_iova >>
+		fb_mmu_cbc_base_address_alignment_shift_v();
+
+	do_div(compbit_base_post_divide64, nvgpu_ltc_get_ltc_count(g));
+	compbit_base_post_divide = u64_lo32(compbit_base_post_divide64);
+
+	compbit_base_post_multiply64 = ((u64)compbit_base_post_divide *
+		nvgpu_ltc_get_ltc_count(g))
+			<< fb_mmu_cbc_base_address_alignment_shift_v();
+
+	if (compbit_base_post_multiply64 < compbit_store_iova) {
+		compbit_base_post_divide++;
+	}
+
+	if (g->ops.cbc.fix_config != NULL) {
+		compbit_base_post_divide =
+			g->ops.cbc.fix_config(g, compbit_base_post_divide);
+	}
+
+	nvgpu_writel(g, fb_mmu_cbc_base_r(),
+		fb_mmu_cbc_base_address_f(compbit_base_post_divide));
+
+	nvgpu_log(g, gpu_dbg_info | gpu_dbg_map_v | gpu_dbg_pte,
+		"compbit base.pa: 0x%x,%08x cbc_base:0x%08x\n",
+		(u32)(compbit_store_iova >> 32),
+		(u32)(compbit_store_iova & U32_MAX),
+		compbit_base_post_divide);
+	nvgpu_log(g, gpu_dbg_fn, "cbc base %x",
+		nvgpu_readl(g, fb_mmu_cbc_base_r()));
+
+	cbc->compbit_store.base_hw = compbit_base_post_divide;
+
+}
+#endif

drivers/gpu/nvgpu/hal/fb/fb_gv11b.h

@@ -0,0 +1,39 @@
+/*
+ * GV11B FB
+ *
+ * Copyright (c) 2016-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef NVGPU_FB_GV11B_H
+#define NVGPU_FB_GV11B_H
+
+struct gk20a;
+
+void gv11b_fb_init_hw(struct gk20a *g);
+void gv11b_fb_init_fs_state(struct gk20a *g);
+int gv11b_fb_set_atomic_mode(struct gk20a *g);
+
+#ifdef CONFIG_NVGPU_COMPRESSION
+struct nvgpu_cbc;
+void gv11b_fb_cbc_configure(struct gk20a *g, struct nvgpu_cbc *cbc);
+#endif
+
+#endif /* NVGPU_FB_GV11B_H */

drivers/gpu/nvgpu/hal/fb/fb_gv11b_fusa.c

@@ -0,0 +1,117 @@
+/*
+ * GV11B FB
+ *
+ * Copyright (c) 2016-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/dma.h>
+#include <nvgpu/log.h>
+#include <nvgpu/enabled.h>
+#include <nvgpu/gmmu.h>
+#include <nvgpu/barrier.h>
+#include <nvgpu/bug.h>
+#include <nvgpu/soc.h>
+#include <nvgpu/ptimer.h>
+#include <nvgpu/io.h>
+#include <nvgpu/utils.h>
+#include <nvgpu/timers.h>
+#include <nvgpu/fifo.h>
+#include <nvgpu/gk20a.h>
+#include <nvgpu/nvgpu_err.h>
+#include <nvgpu/ltc.h>
+#include <nvgpu/rc.h>
+#include <nvgpu/engines.h>
+
+#include "fb_gm20b.h"
+#include "fb_gp10b.h"
+#include "fb_gv11b.h"
+
+#include <nvgpu/hw/gv11b/hw_fb_gv11b.h>
+
+static void gv11b_init_nvlink_soc_credits(struct gk20a *g)
+{
+#ifndef __NVGPU_POSIX__
+	if (nvgpu_is_bpmp_running(g) && (!nvgpu_platform_is_simulation(g))) {
+		nvgpu_log(g, gpu_dbg_info, "nvlink soc credits init done by bpmp");
+	} else {
+#ifdef CONFIG_NVGPU_NVLINK
+		nvgpu_mss_nvlink_init_credits(g);
+#endif
+	}
+#endif
+}
+
+int gv11b_fb_set_atomic_mode(struct gk20a *g)
+{
+	u32 reg_val;
+
+	/*
+	 * NV_PFB_PRI_MMU_CTRL_ATOMIC_CAPABILITY_MODE to RMW MODE
+	 * NV_PFB_PRI_MMU_CTRL_ATOMIC_CAPABILITY_SYS_NCOH_MODE to L2
+	 */
+	reg_val = nvgpu_readl(g, fb_mmu_ctrl_r());
+	reg_val = set_field(reg_val, fb_mmu_ctrl_atomic_capability_mode_m(),
+		fb_mmu_ctrl_atomic_capability_mode_rmw_f());
+	reg_val = set_field(reg_val, fb_mmu_ctrl_atomic_capability_sys_ncoh_mode_m(),
+		fb_mmu_ctrl_atomic_capability_sys_ncoh_mode_l2_f());
+	nvgpu_writel(g, fb_mmu_ctrl_r(), reg_val);
+
+	/* NV_PFB_HSHUB_NUM_ACTIVE_LTCS_HUB_SYS_ATOMIC_MODE to USE_RMW */
+	reg_val = nvgpu_readl(g, fb_hshub_num_active_ltcs_r());
+	reg_val = set_field(reg_val, fb_hshub_num_active_ltcs_hub_sys_atomic_mode_m(),
+		    fb_hshub_num_active_ltcs_hub_sys_atomic_mode_use_rmw_f());
+	nvgpu_writel(g, fb_hshub_num_active_ltcs_r(), reg_val);
+
+	nvgpu_log(g,  gpu_dbg_info, "fb_mmu_ctrl_r 0x%x",
+					nvgpu_readl(g, fb_mmu_ctrl_r()));
+
+	nvgpu_log(g,   gpu_dbg_info, "fb_hshub_num_active_ltcs_r 0x%x",
+			nvgpu_readl(g, fb_hshub_num_active_ltcs_r()));
+
+	return 0;
+}
+
+void gv11b_fb_init_hw(struct gk20a *g)
+{
+	gm20b_fb_init_hw(g);
+
+	g->ops.fb.intr.enable(g);
+}
+
+void gv11b_fb_init_fs_state(struct gk20a *g)
+{
+	nvgpu_log(g, gpu_dbg_fn, "initialize gv11b fb");
+
+	gv11b_init_nvlink_soc_credits(g);
+
+	nvgpu_log(g, gpu_dbg_info, "fbhub active ltcs %x",
+			nvgpu_readl(g, fb_fbhub_num_active_ltcs_r()));
+
+	nvgpu_log(g, gpu_dbg_info, "mmu active ltcs %u",
+			fb_mmu_num_active_ltcs_count_v(
+			nvgpu_readl(g, fb_mmu_num_active_ltcs_r())));
+
+	if (!nvgpu_is_enabled(g, NVGPU_SEC_PRIVSECURITY)) {
+		/* Bypass MMU check for non-secure boot. For
+		 * secure-boot,this register write has no-effect */
+		nvgpu_writel(g, fb_priv_mmu_phy_secure_r(), U32_MAX);
+	}
+}

drivers/gpu/nvgpu/hal/fb/fb_mmu_fault_gv11b.h

@@ -0,0 +1,82 @@
+/*
+ * Copyright (c) 2016-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef NVGPU_FB_MMU_FAULT_GV11B_H
+#define NVGPU_FB_MMU_FAULT_GV11B_H
+
+#include <nvgpu/types.h>
+
+struct gk20a;
+struct mmu_fault_info;
+
+bool gv11b_fb_is_fault_buf_enabled(struct gk20a *g, u32 index);
+void gv11b_fb_fault_buffer_get_ptr_update(struct gk20a *g,
+		 u32 index, u32 next);
+u32  gv11b_fb_fault_buffer_size_val(struct gk20a *g, u32 index);
+bool gv11b_fb_is_fault_buffer_empty(struct gk20a *g,
+		 u32 index, u32 *get_idx);
+void gv11b_fb_fault_buf_set_state_hw(struct gk20a *g,
+		 u32 index, u32 state);
+void gv11b_fb_fault_buf_configure_hw(struct gk20a *g, u32 index);
+
+void gv11b_mm_copy_from_fault_snap_reg(struct gk20a *g,
+		u32 fault_status, struct mmu_fault_info *mmufault);
+void gv11b_fb_handle_mmu_fault(struct gk20a *g, u32 niso_intr);
+void gv11b_fb_handle_dropped_mmu_fault(struct gk20a *g, u32 fault_status);
+void gv11b_fb_handle_nonreplay_fault_overflow(struct gk20a *g,
+		 u32 fault_status);
+void gv11b_fb_handle_bar2_fault(struct gk20a *g,
+		struct mmu_fault_info *mmufault, u32 fault_status);
+
+void gv11b_fb_mmu_fault_info_dump(struct gk20a *g,
+		struct mmu_fault_info *mmufault);
+
+void gv11b_fb_write_mmu_fault_buffer_lo_hi(struct gk20a *g, u32 index,
+		u32 addr_lo, u32 addr_hi);
+u32  gv11b_fb_read_mmu_fault_buffer_get(struct gk20a *g, u32 index);
+void fb_gv11b_write_mmu_fault_buffer_get(struct gk20a *g, u32 index,
+		u32 reg_val);
+u32  gv11b_fb_read_mmu_fault_buffer_put(struct gk20a *g, u32 index);
+u32  gv11b_fb_read_mmu_fault_buffer_size(struct gk20a *g, u32 index);
+void gv11b_fb_write_mmu_fault_buffer_size(struct gk20a *g, u32 index,
+		u32 reg_val);
+void gv11b_fb_read_mmu_fault_addr_lo_hi(struct gk20a *g,
+		u32 *addr_lo, u32 *addr_hi);
+void gv11b_fb_read_mmu_fault_inst_lo_hi(struct gk20a *g,
+		u32 *inst_lo, u32 *inst_hi);
+u32  gv11b_fb_read_mmu_fault_info(struct gk20a *g);
+u32  gv11b_fb_read_mmu_fault_status(struct gk20a *g);
+void gv11b_fb_write_mmu_fault_status(struct gk20a *g, u32 reg_val);
+
+#ifdef CONFIG_NVGPU_REPLAYABLE_FAULT
+void gv11b_fb_handle_replay_fault_overflow(struct gk20a *g,
+		 u32 fault_status);
+void gv11b_fb_handle_replayable_mmu_fault(struct gk20a *g);
+int  gv11b_fb_mmu_invalidate_replay(struct gk20a *g,
+		u32 invalidate_replay_val);
+int  gv11b_fb_replay_or_cancel_faults(struct gk20a *g,
+		u32 invalidate_replay_val);
+u32 gv11b_fb_get_replay_cancel_global_val(void);
+u32 gv11b_fb_get_replay_start_ack_all(void);
+#endif
+
+#endif /* NVGPU_FB_MMU_FAULT_GV11B_H */

drivers/gpu/nvgpu/hal/fb/fb_mmu_fault_gv11b_fusa.c

@@ -0,0 +1,723 @@
+/*
+ * Copyright (c) 2016-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/dma.h>
+#include <nvgpu/log.h>
+#include <nvgpu/enabled.h>
+#include <nvgpu/gmmu.h>
+#include <nvgpu/barrier.h>
+#include <nvgpu/bug.h>
+#include <nvgpu/soc.h>
+#include <nvgpu/ptimer.h>
+#include <nvgpu/io.h>
+#include <nvgpu/utils.h>
+#include <nvgpu/timers.h>
+#include <nvgpu/fifo.h>
+#include <nvgpu/gk20a.h>
+#include <nvgpu/engines.h>
+#include <nvgpu/channel.h>
+#include <nvgpu/tsg.h>
+#include <nvgpu/nvgpu_err.h>
+#include <nvgpu/ltc.h>
+#include <nvgpu/rc.h>
+
+#include "hal/fb/fb_mmu_fault_gv11b.h"
+#include "hal/mm/mmu_fault/mmu_fault_gv11b.h"
+
+#include "fb_gm20b.h"
+#include "fb_gp10b.h"
+#include "fb_gv11b.h"
+
+#include <nvgpu/hw/gv11b/hw_fb_gv11b.h>
+
+static const char *const gv11b_fault_access_type_descs[] = {
+	"virt read",
+	"virt write",
+	"virt atomic strong",
+	"virt prefetch",
+	"virt atomic weak",
+	"xxx",
+	"xxx",
+	"xxx",
+	"phys read",
+	"phys write",
+	"phys atomic",
+	"phys prefetch",
+};
+
+bool gv11b_fb_is_fault_buf_enabled(struct gk20a *g, u32 index)
+{
+	u32 reg_val;
+
+	reg_val = g->ops.fb.read_mmu_fault_buffer_size(g, index);
+	return fb_mmu_fault_buffer_size_enable_v(reg_val) != 0U;
+}
+
+void gv11b_fb_fault_buffer_get_ptr_update(struct gk20a *g,
+				 u32 index, u32 next)
+{
+	u32 reg_val;
+
+	nvgpu_log(g, gpu_dbg_intr, "updating get index with = %d", next);
+
+	reg_val = g->ops.fb.read_mmu_fault_buffer_get(g, index);
+	reg_val = set_field(reg_val, fb_mmu_fault_buffer_get_ptr_m(),
+			 fb_mmu_fault_buffer_get_ptr_f(next));
+
+	/*
+	 * while the fault is being handled it is possible for overflow
+	 * to happen,
+	 */
+	if ((reg_val & fb_mmu_fault_buffer_get_overflow_m()) != 0U) {
+		reg_val |= fb_mmu_fault_buffer_get_overflow_clear_f();
+	}
+
+	g->ops.fb.write_mmu_fault_buffer_get(g, index, reg_val);
+
+	/*
+	 * make sure get ptr update is visible to everyone to avoid
+	 * reading already read entry
+	 */
+	nvgpu_mb();
+}
+
+static u32 gv11b_fb_fault_buffer_get_index(struct gk20a *g, u32 index)
+{
+	u32 reg_val;
+
+	reg_val = g->ops.fb.read_mmu_fault_buffer_get(g, index);
+	return fb_mmu_fault_buffer_get_ptr_v(reg_val);
+}
+
+static u32 gv11b_fb_fault_buffer_put_index(struct gk20a *g, u32 index)
+{
+	u32 reg_val;
+
+	reg_val = g->ops.fb.read_mmu_fault_buffer_put(g, index);
+	return fb_mmu_fault_buffer_put_ptr_v(reg_val);
+}
+
+u32 gv11b_fb_fault_buffer_size_val(struct gk20a *g, u32 index)
+{
+	u32 reg_val;
+
+	reg_val = g->ops.fb.read_mmu_fault_buffer_size(g, index);
+	return fb_mmu_fault_buffer_size_val_v(reg_val);
+}
+
+bool gv11b_fb_is_fault_buffer_empty(struct gk20a *g,
+		 u32 index, u32 *get_idx)
+{
+	u32 put_idx;
+
+	*get_idx = gv11b_fb_fault_buffer_get_index(g, index);
+	put_idx = gv11b_fb_fault_buffer_put_index(g, index);
+
+	return *get_idx == put_idx;
+}
+
+static bool gv11b_fb_is_fault_buffer_full(struct gk20a *g, u32 index)
+{
+	u32 get_idx, put_idx, entries;
+
+
+	get_idx = gv11b_fb_fault_buffer_get_index(g, index);
+
+	put_idx = gv11b_fb_fault_buffer_put_index(g, index);
+
+	entries = gv11b_fb_fault_buffer_size_val(g, index);
+
+	return get_idx == ((put_idx + 1U) % entries);
+}
+
+void gv11b_fb_fault_buf_set_state_hw(struct gk20a *g,
+		 u32 index, u32 state)
+{
+	u32 fault_status;
+	u32 reg_val;
+
+	nvgpu_log_fn(g, " ");
+
+	reg_val = g->ops.fb.read_mmu_fault_buffer_size(g, index);
+	if (state == NVGPU_MMU_FAULT_BUF_ENABLED) {
+		if (gv11b_fb_is_fault_buf_enabled(g, index)) {
+			nvgpu_log_info(g, "fault buffer is already enabled");
+		} else {
+			reg_val |= fb_mmu_fault_buffer_size_enable_true_f();
+			g->ops.fb.write_mmu_fault_buffer_size(g, index,
+				reg_val);
+		}
+
+	} else {
+		struct nvgpu_timeout timeout;
+		u32 delay = POLL_DELAY_MIN_US;
+		int err;
+
+		err = nvgpu_timeout_init(g, &timeout, nvgpu_get_poll_timeout(g),
+			   NVGPU_TIMER_CPU_TIMER);
+		if (err != 0) {
+			nvgpu_err(g, "nvgpu_timeout_init() failed err=%d", err);
+			return;
+		}
+
+		reg_val &= (~(fb_mmu_fault_buffer_size_enable_m()));
+		g->ops.fb.write_mmu_fault_buffer_size(g, index, reg_val);
+
+		fault_status = g->ops.fb.read_mmu_fault_status(g);
+
+		do {
+			if ((fault_status &
+			     fb_mmu_fault_status_busy_true_f()) == 0U) {
+				break;
+			}
+			/*
+			 * Make sure fault buffer is disabled.
+			 * This is to avoid accessing fault buffer by hw
+			 * during the window BAR2 is being unmapped by s/w
+			 */
+			nvgpu_log_info(g, "fault status busy set, check again");
+			fault_status = g->ops.fb.read_mmu_fault_status(g);
+
+			nvgpu_usleep_range(delay, delay * 2U);
+			delay = min_t(u32, delay << 1, POLL_DELAY_MAX_US);
+		} while (nvgpu_timeout_expired_msg(&timeout,
+				"fault status busy set") == 0);
+	}
+}
+
+void gv11b_fb_fault_buf_configure_hw(struct gk20a *g, u32 index)
+{
+	u32 addr_lo;
+	u32 addr_hi;
+
+	nvgpu_log_fn(g, " ");
+
+	gv11b_fb_fault_buf_set_state_hw(g, index,
+					 NVGPU_MMU_FAULT_BUF_DISABLED);
+	addr_lo = u64_lo32(g->mm.hw_fault_buf[index].gpu_va >>
+					fb_mmu_fault_buffer_lo_addr_b());
+	addr_hi = u64_hi32(g->mm.hw_fault_buf[index].gpu_va);
+
+	g->ops.fb.write_mmu_fault_buffer_lo_hi(g, index,
+		fb_mmu_fault_buffer_lo_addr_f(addr_lo),
+		fb_mmu_fault_buffer_hi_addr_f(addr_hi));
+
+	g->ops.fb.write_mmu_fault_buffer_size(g, index,
+		fb_mmu_fault_buffer_size_val_f(g->ops.channel.count(g)) |
+		fb_mmu_fault_buffer_size_overflow_intr_enable_f());
+
+	gv11b_fb_fault_buf_set_state_hw(g, index, NVGPU_MMU_FAULT_BUF_ENABLED);
+}
+
+void gv11b_fb_write_mmu_fault_buffer_lo_hi(struct gk20a *g, u32 index,
+	u32 addr_lo, u32 addr_hi)
+{
+	nvgpu_writel(g, fb_mmu_fault_buffer_lo_r(index), addr_lo);
+	nvgpu_writel(g, fb_mmu_fault_buffer_hi_r(index), addr_hi);
+}
+
+u32 gv11b_fb_read_mmu_fault_buffer_get(struct gk20a *g, u32 index)
+{
+	return nvgpu_readl(g, fb_mmu_fault_buffer_get_r(index));
+}
+
+void fb_gv11b_write_mmu_fault_buffer_get(struct gk20a *g, u32 index,
+	u32 reg_val)
+{
+	nvgpu_writel(g, fb_mmu_fault_buffer_get_r(index), reg_val);
+}
+
+u32 gv11b_fb_read_mmu_fault_buffer_put(struct gk20a *g, u32 index)
+{
+	return nvgpu_readl(g, fb_mmu_fault_buffer_put_r(index));
+}
+
+u32 gv11b_fb_read_mmu_fault_buffer_size(struct gk20a *g, u32 index)
+{
+	return nvgpu_readl(g, fb_mmu_fault_buffer_size_r(index));
+}
+
+void gv11b_fb_write_mmu_fault_buffer_size(struct gk20a *g, u32 index,
+	u32 reg_val)
+{
+	nvgpu_writel(g, fb_mmu_fault_buffer_size_r(index), reg_val);
+}
+
+void gv11b_fb_read_mmu_fault_addr_lo_hi(struct gk20a *g,
+	u32 *addr_lo, u32 *addr_hi)
+{
+	*addr_lo = nvgpu_readl(g, fb_mmu_fault_addr_lo_r());
+	*addr_hi = nvgpu_readl(g, fb_mmu_fault_addr_hi_r());
+}
+
+void gv11b_fb_read_mmu_fault_inst_lo_hi(struct gk20a *g,
+	u32 *inst_lo, u32 *inst_hi)
+{
+	*inst_lo = nvgpu_readl(g, fb_mmu_fault_inst_lo_r());
+	*inst_hi = nvgpu_readl(g, fb_mmu_fault_inst_hi_r());
+}
+
+u32 gv11b_fb_read_mmu_fault_info(struct gk20a *g)
+{
+	return nvgpu_readl(g, fb_mmu_fault_info_r());
+}
+
+u32 gv11b_fb_read_mmu_fault_status(struct gk20a *g)
+{
+	return nvgpu_readl(g, fb_mmu_fault_status_r());
+}
+
+void gv11b_fb_write_mmu_fault_status(struct gk20a *g, u32 reg_val)
+{
+
+	nvgpu_writel(g, fb_mmu_fault_status_r(), reg_val);
+}
+
+void gv11b_fb_mmu_fault_info_dump(struct gk20a *g,
+			 struct mmu_fault_info *mmufault)
+{
+	if ((mmufault != NULL) && mmufault->valid) {
+		nvgpu_err(g, "[MMU FAULT] "
+			"mmu engine id:  %d, "
+			"ch id:  %d, "
+			"fault addr: 0x%llx, "
+			"fault addr aperture: %d, "
+			"fault type: %s, "
+			"access type: %s, ",
+			mmufault->mmu_engine_id,
+			mmufault->chid,
+			mmufault->fault_addr,
+			mmufault->fault_addr_aperture,
+			mmufault->fault_type_desc,
+			gv11b_fault_access_type_descs[mmufault->access_type]);
+		nvgpu_err(g, "[MMU FAULT] "
+			"protected mode: %d, "
+			"client type: %s, "
+			"client id:  %s, "
+			"gpc id if client type is gpc: %d, ",
+			mmufault->protected_mode,
+			mmufault->client_type_desc,
+			mmufault->client_id_desc,
+			mmufault->gpc_id);
+
+		nvgpu_log(g, gpu_dbg_intr, "[MMU FAULT] "
+			"faulted act eng id if any: 0x%x, "
+			"faulted veid if any: 0x%x, "
+			"faulted pbdma id if any: 0x%x, ",
+			mmufault->faulted_engine,
+			mmufault->faulted_subid,
+			mmufault->faulted_pbdma);
+		nvgpu_log(g, gpu_dbg_intr, "[MMU FAULT] "
+			"inst ptr: 0x%llx, "
+			"inst ptr aperture: %d, "
+			"replayable fault: %d, "
+			"replayable fault en:  %d "
+			"timestamp hi:lo 0x%08x:0x%08x, ",
+			mmufault->inst_ptr,
+			mmufault->inst_aperture,
+			mmufault->replayable_fault,
+			mmufault->replay_fault_en,
+			mmufault->timestamp_hi, mmufault->timestamp_lo);
+	}
+}
+
+void gv11b_mm_copy_from_fault_snap_reg(struct gk20a *g,
+		u32 fault_status, struct mmu_fault_info *mmufault)
+{
+	u32 reg_val;
+	u32 addr_lo, addr_hi;
+	u64 inst_ptr;
+	u32 chid = NVGPU_INVALID_CHANNEL_ID;
+	struct nvgpu_channel *refch;
+
+	(void) memset(mmufault, 0, sizeof(*mmufault));
+
+	if ((fault_status & fb_mmu_fault_status_valid_set_f()) == 0U) {
+
+		nvgpu_log(g, gpu_dbg_intr, "mmu fault status valid not set");
+		return;
+	}
+
+	g->ops.fb.read_mmu_fault_inst_lo_hi(g, &reg_val, &addr_hi);
+
+	addr_lo = fb_mmu_fault_inst_lo_addr_v(reg_val);
+	addr_lo = addr_lo << fb_mmu_fault_inst_lo_addr_b();
+
+	addr_hi = fb_mmu_fault_inst_hi_addr_v(addr_hi);
+	inst_ptr = hi32_lo32_to_u64(addr_hi, addr_lo);
+
+	/* refch will be put back after fault is handled */
+	refch = nvgpu_channel_refch_from_inst_ptr(g, inst_ptr);
+	if (refch != NULL) {
+		chid = refch->chid;
+	}
+
+	/* It is still ok to continue if refch is NULL */
+	mmufault->refch = refch;
+	mmufault->chid = chid;
+	mmufault->inst_ptr = inst_ptr;
+	mmufault->inst_aperture = fb_mmu_fault_inst_lo_aperture_v(reg_val);
+	mmufault->mmu_engine_id = fb_mmu_fault_inst_lo_engine_id_v(reg_val);
+
+	nvgpu_engine_mmu_fault_id_to_eng_ve_pbdma_id(g, mmufault->mmu_engine_id,
+		 &mmufault->faulted_engine, &mmufault->faulted_subid,
+		 &mmufault->faulted_pbdma);
+
+	g->ops.fb.read_mmu_fault_addr_lo_hi(g, &reg_val, &addr_hi);
+
+	addr_lo = fb_mmu_fault_addr_lo_addr_v(reg_val);
+	addr_lo = addr_lo << fb_mmu_fault_addr_lo_addr_b();
+
+	mmufault->fault_addr_aperture =
+			 fb_mmu_fault_addr_lo_phys_aperture_v(reg_val);
+
+	addr_hi = fb_mmu_fault_addr_hi_addr_v(addr_hi);
+	mmufault->fault_addr = hi32_lo32_to_u64(addr_hi, addr_lo);
+
+	reg_val = g->ops.fb.read_mmu_fault_info(g);
+	mmufault->fault_type = fb_mmu_fault_info_fault_type_v(reg_val);
+	mmufault->replayable_fault =
+			(fb_mmu_fault_info_replayable_fault_v(reg_val) == 1U);
+	mmufault->client_id = fb_mmu_fault_info_client_v(reg_val);
+	mmufault->access_type = fb_mmu_fault_info_access_type_v(reg_val);
+	mmufault->client_type = fb_mmu_fault_info_client_type_v(reg_val);
+	mmufault->gpc_id = fb_mmu_fault_info_gpc_id_v(reg_val);
+	mmufault->protected_mode =
+			 fb_mmu_fault_info_protected_mode_v(reg_val);
+	mmufault->replay_fault_en =
+			fb_mmu_fault_info_replayable_fault_en_v(reg_val);
+
+	mmufault->valid = (fb_mmu_fault_info_valid_v(reg_val) == 1U);
+
+	fault_status &= ~(fb_mmu_fault_status_valid_m());
+	g->ops.fb.write_mmu_fault_status(g, fault_status);
+
+	g->ops.mm.mmu_fault.parse_mmu_fault_info(mmufault);
+
+}
+
+void gv11b_fb_handle_nonreplay_fault_overflow(struct gk20a *g,
+			 u32 fault_status)
+{
+	u32 reg_val;
+	u32 index = NVGPU_MMU_FAULT_NONREPLAY_REG_INDX;
+
+	reg_val = g->ops.fb.read_mmu_fault_buffer_get(g, index);
+
+	if ((fault_status &
+	     fb_mmu_fault_status_non_replayable_getptr_corrupted_m()) != 0U) {
+
+		nvgpu_err(g, "non replayable getptr corrupted set");
+
+		gv11b_fb_fault_buf_configure_hw(g, index);
+
+		reg_val = set_field(reg_val,
+			fb_mmu_fault_buffer_get_getptr_corrupted_m(),
+			fb_mmu_fault_buffer_get_getptr_corrupted_clear_f());
+	}
+
+	if ((fault_status &
+	     fb_mmu_fault_status_non_replayable_overflow_m()) != 0U) {
+
+		bool buffer_full = gv11b_fb_is_fault_buffer_full(g, index);
+
+		nvgpu_err(g, "non replayable overflow: buffer full:%s",
+				buffer_full?"true":"false");
+
+		reg_val = set_field(reg_val,
+			fb_mmu_fault_buffer_get_overflow_m(),
+			fb_mmu_fault_buffer_get_overflow_clear_f());
+	}
+
+	g->ops.fb.write_mmu_fault_buffer_get(g, index, reg_val);
+}
+
+void gv11b_fb_handle_bar2_fault(struct gk20a *g,
+			struct mmu_fault_info *mmufault, u32 fault_status)
+{
+	int err;
+
+	if ((fault_status &
+	     fb_mmu_fault_status_non_replayable_error_m()) != 0U) {
+		if (gv11b_fb_is_fault_buf_enabled(g,
+				NVGPU_MMU_FAULT_NONREPLAY_REG_INDX)) {
+			gv11b_fb_fault_buf_configure_hw(g,
+				NVGPU_MMU_FAULT_NONREPLAY_REG_INDX);
+		}
+	}
+
+#ifdef CONFIG_NVGPU_REPLAYABLE_FAULT
+	if ((fault_status & fb_mmu_fault_status_replayable_error_m()) != 0U) {
+		if (gv11b_fb_is_fault_buf_enabled(g,
+				NVGPU_MMU_FAULT_REPLAY_REG_INDX)) {
+			gv11b_fb_fault_buf_configure_hw(g,
+				NVGPU_MMU_FAULT_REPLAY_REG_INDX);
+		}
+	}
+#endif
+	g->ops.ce.mthd_buffer_fault_in_bar2_fault(g);
+
+	err = g->ops.bus.bar2_bind(g, &g->mm.bar2.inst_block);
+	if (err != 0) {
+		nvgpu_err(g, "bar2_bind failed!");
+	}
+
+	if (mmufault->refch != NULL) {
+		nvgpu_channel_put(mmufault->refch);
+		mmufault->refch = NULL;
+	}
+}
+
+void gv11b_fb_handle_dropped_mmu_fault(struct gk20a *g, u32 fault_status)
+{
+	u32 dropped_faults = 0;
+
+	dropped_faults = fb_mmu_fault_status_dropped_bar1_phys_set_f() |
+			fb_mmu_fault_status_dropped_bar1_virt_set_f() |
+			fb_mmu_fault_status_dropped_bar2_phys_set_f() |
+			fb_mmu_fault_status_dropped_bar2_virt_set_f() |
+			fb_mmu_fault_status_dropped_ifb_phys_set_f() |
+			fb_mmu_fault_status_dropped_ifb_virt_set_f() |
+			fb_mmu_fault_status_dropped_other_phys_set_f()|
+			fb_mmu_fault_status_dropped_other_virt_set_f();
+
+	if ((fault_status & dropped_faults) != 0U) {
+		nvgpu_err(g, "dropped mmu fault (0x%08x)",
+				 fault_status & dropped_faults);
+		g->ops.fb.write_mmu_fault_status(g, dropped_faults);
+	}
+}
+
+void gv11b_fb_handle_mmu_fault(struct gk20a *g, u32 niso_intr)
+{
+	u32 fault_status = g->ops.fb.read_mmu_fault_status(g);
+
+	nvgpu_log(g, gpu_dbg_intr, "mmu_fault_status = 0x%08x", fault_status);
+
+	if ((niso_intr &
+	     fb_niso_intr_mmu_other_fault_notify_m()) != 0U) {
+
+		nvgpu_report_mmu_err(g, NVGPU_ERR_MODULE_HUBMMU,
+			GPU_HUBMMU_PAGE_FAULT_ERROR,
+			NULL,
+			fault_status,
+			GPU_HUBMMU_OTHER_FAULT_NOTIFY);
+
+		gv11b_fb_handle_dropped_mmu_fault(g, fault_status);
+
+		gv11b_mm_mmu_fault_handle_other_fault_notify(g, fault_status);
+	}
+
+	if (gv11b_fb_is_fault_buf_enabled(g, NVGPU_MMU_FAULT_NONREPLAY_REG_INDX)) {
+
+		if ((niso_intr &
+		     fb_niso_intr_mmu_nonreplayable_fault_notify_m()) != 0U) {
+
+			gv11b_mm_mmu_fault_handle_nonreplay_replay_fault(g,
+					fault_status,
+					NVGPU_MMU_FAULT_NONREPLAY_REG_INDX);
+
+			/*
+			 * When all the faults are processed,
+			 * GET and PUT will have same value and mmu fault status
+			 * bit will be reset by HW
+			 */
+		}
+		if ((niso_intr &
+		     fb_niso_intr_mmu_nonreplayable_fault_overflow_m()) != 0U) {
+
+			nvgpu_report_mmu_err(g, NVGPU_ERR_MODULE_HUBMMU,
+				GPU_HUBMMU_PAGE_FAULT_ERROR,
+				NULL,
+				fault_status,
+				GPU_HUBMMU_NONREPLAYABLE_FAULT_OVERFLOW);
+
+			gv11b_fb_handle_nonreplay_fault_overflow(g,
+				 fault_status);
+		}
+
+	}
+
+#ifdef CONFIG_NVGPU_SUPPORT_MMU_REPLAYABLE_FAULT
+	if (gv11b_fb_is_fault_buf_enabled(g, NVGPU_MMU_FAULT_REPLAY_REG_INDX)) {
+
+		if ((niso_intr &
+		     fb_niso_intr_mmu_replayable_fault_notify_m()) != 0U) {
+
+			gv11b_mm_mmu_fault_handle_nonreplay_replay_fault(g,
+					fault_status,
+					NVGPU_MMU_FAULT_REPLAY_REG_INDX);
+		}
+		if ((niso_intr &
+		     fb_niso_intr_mmu_replayable_fault_overflow_m()) != 0U) {
+
+			nvgpu_report_mmu_err(g, NVGPU_ERR_MODULE_HUBMMU,
+				GPU_HUBMMU_PAGE_FAULT_ERROR,
+				NULL,
+				fault_status,
+				GPU_HUBMMU_REPLAYABLE_FAULT_OVERFLOW);
+
+			gv11b_fb_handle_replay_fault_overflow(g,
+				 fault_status);
+		}
+
+	}
+#endif
+
+	nvgpu_log(g, gpu_dbg_intr, "clear mmu fault status");
+	g->ops.fb.write_mmu_fault_status(g,
+			fb_mmu_fault_status_valid_clear_f());
+}
+
+#ifdef CONFIG_NVGPU_REPLAYABLE_FAULT
+void gv11b_fb_handle_replayable_mmu_fault(struct gk20a *g)
+{
+	u32 fault_status = nvgpu_readl(g, fb_mmu_fault_status_r());
+
+	if ((fault_status & fb_mmu_fault_status_replayable_m()) == 0U) {
+		return;
+	}
+
+	if (gv11b_fb_is_fault_buf_enabled(g,
+			NVGPU_MMU_FAULT_NONREPLAY_REG_INDX)) {
+		gv11b_mm_mmu_fault_handle_nonreplay_replay_fault(g,
+				fault_status,
+				NVGPU_MMU_FAULT_REPLAY_REG_INDX);
+	}
+}
+
+void gv11b_fb_handle_replay_fault_overflow(struct gk20a *g,
+			 u32 fault_status)
+{
+	u32 reg_val;
+	u32 index = NVGPU_MMU_FAULT_REPLAY_REG_INDX;
+
+	reg_val = g->ops.fb.read_mmu_fault_buffer_get(g, index);
+
+	if ((fault_status &
+	     fb_mmu_fault_status_replayable_getptr_corrupted_m()) != 0U) {
+
+		nvgpu_err(g, "replayable getptr corrupted set");
+
+		gv11b_fb_fault_buf_configure_hw(g, index);
+
+		reg_val = set_field(reg_val,
+			fb_mmu_fault_buffer_get_getptr_corrupted_m(),
+			fb_mmu_fault_buffer_get_getptr_corrupted_clear_f());
+	}
+
+	if ((fault_status &
+	     fb_mmu_fault_status_replayable_overflow_m()) != 0U) {
+		bool buffer_full = gv11b_fb_is_fault_buffer_full(g, index);
+
+		nvgpu_err(g, "replayable overflow: buffer full:%s",
+				buffer_full?"true":"false");
+
+		reg_val = set_field(reg_val,
+			fb_mmu_fault_buffer_get_overflow_m(),
+			fb_mmu_fault_buffer_get_overflow_clear_f());
+	}
+
+	g->ops.fb.write_mmu_fault_buffer_get(g, index, reg_val);
+}
+
+int gv11b_fb_replay_or_cancel_faults(struct gk20a *g,
+			 u32 invalidate_replay_val)
+{
+	int err = 0;
+
+	nvgpu_log_fn(g, " ");
+
+	if ((invalidate_replay_val &
+	     fb_mmu_invalidate_replay_cancel_global_f()) != 0U) {
+		/*
+		 * cancel faults so that next time it faults as
+		 * replayable faults and channel recovery can be done
+		 */
+		err = g->ops.fb.mmu_invalidate_replay(g,
+			fb_mmu_invalidate_replay_cancel_global_f());
+	} else if ((invalidate_replay_val &
+		    fb_mmu_invalidate_replay_start_ack_all_f()) != 0U) {
+		/* pte valid is fixed. replay faulting request */
+		err = g->ops.fb.mmu_invalidate_replay(g,
+			fb_mmu_invalidate_replay_start_ack_all_f());
+	}
+
+	return err;
+}
+
+u32 gv11b_fb_get_replay_cancel_global_val(void)
+{
+	return fb_mmu_invalidate_replay_cancel_global_f();
+}
+
+u32 gv11b_fb_get_replay_start_ack_all(void)
+{
+	return fb_mmu_invalidate_replay_start_ack_all_f();
+}
+
+int gv11b_fb_mmu_invalidate_replay(struct gk20a *g,
+			 u32 invalidate_replay_val)
+{
+	int err = -ETIMEDOUT;
+	u32 reg_val;
+	struct nvgpu_timeout timeout;
+
+	nvgpu_log_fn(g, " ");
+
+	nvgpu_mutex_acquire(&g->mm.tlb_lock);
+
+	reg_val = nvgpu_readl(g, fb_mmu_invalidate_r());
+
+	reg_val |= fb_mmu_invalidate_all_va_true_f() |
+		fb_mmu_invalidate_all_pdb_true_f() |
+		invalidate_replay_val |
+		fb_mmu_invalidate_trigger_true_f();
+
+	nvgpu_writel(g, fb_mmu_invalidate_r(), reg_val);
+
+	/* retry 200 times */
+	err = nvgpu_timeout_init(g, &timeout, 200U, NVGPU_TIMER_RETRY_TIMER);
+	if (err != 0) {
+		nvgpu_err(g, "nvgpu_timeout_init() failed err=%d", err);
+		return err;
+	}
+	do {
+		reg_val = nvgpu_readl(g, fb_mmu_ctrl_r());
+		if (fb_mmu_ctrl_pri_fifo_empty_v(reg_val) !=
+			fb_mmu_ctrl_pri_fifo_empty_false_f()) {
+			err = 0;
+			break;
+		}
+		nvgpu_udelay(5);
+	} while (nvgpu_timeout_expired_msg(&timeout,
+			"invalidate replay failed 0x%x",
+			invalidate_replay_val) == 0);
+	if (err != 0) {
+		nvgpu_err(g, "invalidate replay timedout");
+	}
+
+	nvgpu_mutex_release(&g->mm.tlb_lock);
+
+	return err;
+}
+#endif

drivers/gpu/nvgpu/hal/fb/fb_mmu_fault_tu104.c

@@ -0,0 +1,247 @@
+/*
+ * Copyright (c) 2018-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/log.h>
+#include <nvgpu/types.h>
+#include <nvgpu/timers.h>
+#include <nvgpu/nvgpu_mem.h>
+#include <nvgpu/io.h>
+#include <nvgpu/utils.h>
+#include <nvgpu/gk20a.h>
+#include <nvgpu/bug.h>
+#include <nvgpu/mmu_fault.h>
+
+#include "hal/mc/mc_tu104.h"
+#include "hal/fb/fb_mmu_fault_gv11b.h"
+#include "hal/fb/fb_mmu_fault_tu104.h"
+#include "hal/mm/mmu_fault/mmu_fault_gv11b.h"
+
+#include "nvgpu/hw/tu104/hw_fb_tu104.h"
+#include "nvgpu/hw/tu104/hw_func_tu104.h"
+
+void tu104_fb_handle_mmu_fault(struct gk20a *g)
+{
+	u32 info_fault = nvgpu_readl(g, fb_mmu_int_vector_info_fault_r());
+	u32 nonreplay_fault = nvgpu_readl(g,
+		fb_mmu_int_vector_fault_r(NVGPU_MMU_FAULT_NONREPLAY_REG_INDX));
+#ifdef CONFIG_NVGPU_REPLAYABLE_FAULT
+	u32 replay_fault = nvgpu_readl(g,
+		fb_mmu_int_vector_fault_r(NVGPU_MMU_FAULT_REPLAY_REG_INDX));
+#endif
+	u32 fault_status = g->ops.fb.read_mmu_fault_status(g);
+
+	nvgpu_log(g, gpu_dbg_intr, "mmu_fault_status = 0x%08x", fault_status);
+
+	if (intr_tu104_vector_intr_pending(g,
+			fb_mmu_int_vector_info_fault_vector_v(info_fault))) {
+		intr_tu104_intr_clear_leaf_vector(g,
+			fb_mmu_int_vector_info_fault_vector_v(info_fault));
+
+		gv11b_fb_handle_dropped_mmu_fault(g, fault_status);
+		gv11b_mm_mmu_fault_handle_other_fault_notify(g, fault_status);
+	}
+
+	if (gv11b_fb_is_fault_buf_enabled(g,
+			NVGPU_MMU_FAULT_NONREPLAY_REG_INDX)) {
+		if (intr_tu104_vector_intr_pending(g,
+				fb_mmu_int_vector_fault_notify_v(
+					nonreplay_fault))) {
+			intr_tu104_intr_clear_leaf_vector(g,
+				fb_mmu_int_vector_fault_notify_v(
+					nonreplay_fault));
+
+			gv11b_mm_mmu_fault_handle_nonreplay_replay_fault(g,
+					fault_status,
+					NVGPU_MMU_FAULT_NONREPLAY_REG_INDX);
+
+			/*
+			 * When all the faults are processed,
+			 * GET and PUT will have same value and mmu fault status
+			 * bit will be reset by HW
+			 */
+		}
+
+		if (intr_tu104_vector_intr_pending(g,
+				fb_mmu_int_vector_fault_error_v(nonreplay_fault))) {
+			intr_tu104_intr_clear_leaf_vector(g,
+				fb_mmu_int_vector_fault_error_v(nonreplay_fault));
+
+			gv11b_fb_handle_nonreplay_fault_overflow(g,
+				 fault_status);
+		}
+	}
+
+#ifdef CONFIG_NVGPU_REPLAYABLE_FAULT
+	if (gv11b_fb_is_fault_buf_enabled(g,
+			NVGPU_MMU_FAULT_REPLAY_REG_INDX)) {
+		if (intr_tu104_vector_intr_pending(g,
+				fb_mmu_int_vector_fault_notify_v(replay_fault))) {
+			intr_tu104_intr_clear_leaf_vector(g,
+				fb_mmu_int_vector_fault_notify_v(replay_fault));
+
+			gv11b_mm_mmu_fault_handle_nonreplay_replay_fault(g,
+					fault_status,
+					NVGPU_MMU_FAULT_REPLAY_REG_INDX);
+		}
+
+		if (intr_tu104_vector_intr_pending(g,
+				fb_mmu_int_vector_fault_error_v(replay_fault))) {
+			intr_tu104_intr_clear_leaf_vector(g,
+				fb_mmu_int_vector_fault_error_v(replay_fault));
+
+			gv11b_fb_handle_replay_fault_overflow(g,
+				 fault_status);
+		}
+	}
+#endif
+
+	nvgpu_log(g, gpu_dbg_intr, "clear mmu fault status");
+	g->ops.fb.write_mmu_fault_status(g,
+				fb_mmu_fault_status_valid_clear_f());
+}
+
+void tu104_fb_write_mmu_fault_buffer_lo_hi(struct gk20a *g, u32 index,
+	u32 addr_lo, u32 addr_hi)
+{
+	nvgpu_func_writel(g,
+		func_priv_mmu_fault_buffer_lo_r(index), addr_lo);
+	nvgpu_func_writel(g,
+		func_priv_mmu_fault_buffer_hi_r(index), addr_hi);
+}
+
+u32 tu104_fb_read_mmu_fault_buffer_get(struct gk20a *g, u32 index)
+{
+	return nvgpu_func_readl(g,
+		func_priv_mmu_fault_buffer_get_r(index));
+}
+
+void tu104_fb_write_mmu_fault_buffer_get(struct gk20a *g, u32 index,
+	u32 reg_val)
+{
+	nvgpu_func_writel(g,
+		func_priv_mmu_fault_buffer_get_r(index),
+			reg_val);
+}
+
+u32 tu104_fb_read_mmu_fault_buffer_put(struct gk20a *g, u32 index)
+{
+	return nvgpu_func_readl(g,
+		func_priv_mmu_fault_buffer_put_r(index));
+}
+
+u32 tu104_fb_read_mmu_fault_buffer_size(struct gk20a *g, u32 index)
+{
+	return nvgpu_func_readl(g,
+		func_priv_mmu_fault_buffer_size_r(index));
+}
+
+void tu104_fb_write_mmu_fault_buffer_size(struct gk20a *g, u32 index,
+	u32 reg_val)
+{
+	nvgpu_func_writel(g,
+		func_priv_mmu_fault_buffer_size_r(index),
+			reg_val);
+}
+
+void tu104_fb_read_mmu_fault_addr_lo_hi(struct gk20a *g,
+	u32 *addr_lo, u32 *addr_hi)
+{
+	*addr_lo = nvgpu_func_readl(g,
+			func_priv_mmu_fault_addr_lo_r());
+	*addr_hi = nvgpu_func_readl(g,
+			func_priv_mmu_fault_addr_hi_r());
+}
+
+void tu104_fb_read_mmu_fault_inst_lo_hi(struct gk20a *g,
+	u32 *inst_lo, u32 *inst_hi)
+{
+	*inst_lo = nvgpu_func_readl(g,
+			func_priv_mmu_fault_inst_lo_r());
+	*inst_hi = nvgpu_func_readl(g,
+			func_priv_mmu_fault_inst_hi_r());
+}
+
+u32 tu104_fb_read_mmu_fault_info(struct gk20a *g)
+{
+	return nvgpu_func_readl(g,
+			func_priv_mmu_fault_info_r());
+}
+
+u32 tu104_fb_read_mmu_fault_status(struct gk20a *g)
+{
+	return nvgpu_func_readl(g,
+			func_priv_mmu_fault_status_r());
+}
+
+void tu104_fb_write_mmu_fault_status(struct gk20a *g, u32 reg_val)
+{
+	nvgpu_func_writel(g, func_priv_mmu_fault_status_r(),
+			   reg_val);
+}
+
+#ifdef CONFIG_NVGPU_REPLAYABLE_FAULT
+int tu104_fb_mmu_invalidate_replay(struct gk20a *g,
+			 u32 invalidate_replay_val)
+{
+	int err = -ETIMEDOUT;
+	u32 reg_val;
+	struct nvgpu_timeout timeout;
+
+	nvgpu_log_fn(g, " ");
+
+	/* retry 200 times */
+	err = nvgpu_timeout_init(g, &timeout, 200U, NVGPU_TIMER_RETRY_TIMER);
+	if (err != 0) {
+		return err;
+	}
+
+	nvgpu_mutex_acquire(&g->mm.tlb_lock);
+
+	reg_val = nvgpu_func_readl(g, func_priv_mmu_invalidate_r());
+
+	reg_val |= fb_mmu_invalidate_all_va_true_f() |
+		fb_mmu_invalidate_all_pdb_true_f() |
+		invalidate_replay_val |
+		fb_mmu_invalidate_trigger_true_f();
+
+	nvgpu_func_writel(g, func_priv_mmu_invalidate_r(), reg_val);
+
+	do {
+		reg_val = nvgpu_func_readl(g,
+				func_priv_mmu_invalidate_r());
+		if (fb_mmu_invalidate_trigger_v(reg_val) !=
+				fb_mmu_invalidate_trigger_true_v()) {
+			err = 0;
+			break;
+		}
+		nvgpu_udelay(5);
+	} while (nvgpu_timeout_expired_msg(&timeout,
+			"invalidate replay failed on 0x%x",
+			invalidate_replay_val) == 0);
+	if (err != 0) {
+		nvgpu_err(g, "invalidate replay timedout");
+	}
+
+	nvgpu_mutex_release(&g->mm.tlb_lock);
+	return err;
+}
+#endif

drivers/gpu/nvgpu/hal/fb/fb_mmu_fault_tu104.h

@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2018-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef NVGPU_FB_MMU_FAULT_TU104_H
+#define NVGPU_FB_MMU_FAULT_TU104_H
+
+#include <nvgpu/types.h>
+
+struct gk20a;
+
+void tu104_fb_handle_mmu_fault(struct gk20a *g);
+
+void tu104_fb_write_mmu_fault_buffer_lo_hi(struct gk20a *g, u32 index,
+	u32 addr_lo, u32 addr_hi);
+u32  tu104_fb_read_mmu_fault_buffer_get(struct gk20a *g, u32 index);
+void tu104_fb_write_mmu_fault_buffer_get(struct gk20a *g, u32 index,
+	u32 reg_val);
+u32  tu104_fb_read_mmu_fault_buffer_put(struct gk20a *g, u32 index);
+u32  tu104_fb_read_mmu_fault_buffer_size(struct gk20a *g, u32 index);
+void tu104_fb_write_mmu_fault_buffer_size(struct gk20a *g, u32 index,
+	u32 reg_val);
+void tu104_fb_read_mmu_fault_addr_lo_hi(struct gk20a *g,
+	u32 *addr_lo, u32 *addr_hi);
+void tu104_fb_read_mmu_fault_inst_lo_hi(struct gk20a *g,
+	u32 *inst_lo, u32 *inst_hi);
+u32  tu104_fb_read_mmu_fault_info(struct gk20a *g);
+u32  tu104_fb_read_mmu_fault_status(struct gk20a *g);
+void tu104_fb_write_mmu_fault_status(struct gk20a *g, u32 reg_val);
+
+#ifdef CONFIG_NVGPU_REPLAYABLE_FAULT
+int  tu104_fb_mmu_invalidate_replay(struct gk20a *g,
+	u32 invalidate_replay_val);
+#endif
+
+#endif /* NVGPU_FB_MMU_FAULT_TU104_H */
+

drivers/gpu/nvgpu/hal/fb/fb_tu104.c

@@ -0,0 +1,313 @@
+/*
+ * Copyright (c) 2018-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/trace.h>
+#include <nvgpu/log.h>
+#include <nvgpu/types.h>
+#include <nvgpu/timers.h>
+#include <nvgpu/nvgpu_mem.h>
+#include <nvgpu/io.h>
+#include <nvgpu/utils.h>
+#include <nvgpu/gk20a.h>
+#include <nvgpu/bug.h>
+#include <nvgpu/nvgpu_init.h>
+
+#include "hal/fb/fb_gv11b.h"
+#include "hal/fb/fb_gv100.h"
+#include "hal/mc/mc_tu104.h"
+
+#include "fb_tu104.h"
+
+#include "nvgpu/hw/tu104/hw_fb_tu104.h"
+#include "nvgpu/hw/tu104/hw_func_tu104.h"
+
+int fb_tu104_tlb_invalidate(struct gk20a *g, struct nvgpu_mem *pdb)
+{
+	struct nvgpu_timeout timeout;
+	u32 addr_lo;
+	u32 data;
+	int err = 0;
+
+	nvgpu_log_fn(g, " ");
+
+	/*
+	 * pagetables are considered sw states which are preserved after
+	 * prepare_poweroff. When gk20a deinit releases those pagetables,
+	 * common code in vm unmap path calls tlb invalidate that touches
+	 * hw. Use the power_on flag to skip tlb invalidation when gpu
+	 * power is turned off
+	 */
+	if (nvgpu_is_powered_off(g)) {
+		return 0;
+	}
+
+	addr_lo = u64_lo32(nvgpu_mem_get_addr(g, pdb) >> 12);
+
+	err = nvgpu_timeout_init(g, &timeout, 1000, NVGPU_TIMER_RETRY_TIMER);
+	if (err != 0) {
+		return err;
+	}
+
+	nvgpu_mutex_acquire(&g->mm.tlb_lock);
+
+#ifdef CONFIG_NVGPU_TRACE
+	trace_gk20a_mm_tlb_invalidate(g->name);
+#endif
+
+	nvgpu_func_writel(g, func_priv_mmu_invalidate_pdb_r(),
+		fb_mmu_invalidate_pdb_addr_f(addr_lo) |
+		nvgpu_aperture_mask(g, pdb,
+				fb_mmu_invalidate_pdb_aperture_sys_mem_f(),
+				fb_mmu_invalidate_pdb_aperture_sys_mem_f(),
+				fb_mmu_invalidate_pdb_aperture_vid_mem_f()));
+
+	nvgpu_func_writel(g, func_priv_mmu_invalidate_r(),
+		fb_mmu_invalidate_all_va_true_f() |
+		fb_mmu_invalidate_trigger_true_f());
+
+	do {
+		data = nvgpu_func_readl(g,
+				func_priv_mmu_invalidate_r());
+		if (fb_mmu_invalidate_trigger_v(data) !=
+				fb_mmu_invalidate_trigger_true_v()) {
+			break;
+		}
+		nvgpu_udelay(2);
+	} while (nvgpu_timeout_expired_msg(&timeout,
+					 "wait mmu invalidate") == 0);
+
+#ifdef CONFIG_NVGPU_TRACE
+	trace_gk20a_mm_tlb_invalidate_done(g->name);
+#endif
+
+	nvgpu_mutex_release(&g->mm.tlb_lock);
+	return err;
+}
+
+#ifdef CONFIG_NVGPU_COMPRESSION
+void tu104_fb_cbc_configure(struct gk20a *g, struct nvgpu_cbc *cbc)
+{
+	u64 base_divisor;
+	u64 compbit_store_base;
+	u64 compbit_store_pa;
+	u64 cbc_start_addr, cbc_end_addr;
+	u64 cbc_top;
+	u64 cbc_top_size;
+	u32 cbc_max;
+
+	compbit_store_pa = nvgpu_mem_get_addr(g, &cbc->compbit_store.mem);
+	base_divisor = g->ops.cbc.get_base_divisor(g);
+	compbit_store_base = DIV_ROUND_UP(compbit_store_pa, base_divisor);
+
+	cbc_start_addr = (u64)nvgpu_ltc_get_ltc_count(g) *
+			(compbit_store_base <<
+			 fb_mmu_cbc_base_address_alignment_shift_v());
+	cbc_end_addr = cbc_start_addr + cbc->compbit_backing_size;
+
+	cbc_top = (cbc_end_addr / nvgpu_ltc_get_ltc_count(g)) >>
+		  fb_mmu_cbc_base_address_alignment_shift_v();
+	cbc_top_size = u64_lo32(cbc_top) - compbit_store_base;
+
+	nvgpu_assert(cbc_top_size < U64(U32_MAX));
+	nvgpu_writel(g, fb_mmu_cbc_top_r(),
+			fb_mmu_cbc_top_size_f(U32(cbc_top_size)));
+
+	cbc_max = nvgpu_readl(g, fb_mmu_cbc_max_r());
+	cbc_max = set_field(cbc_max,
+		  fb_mmu_cbc_max_comptagline_m(),
+		  fb_mmu_cbc_max_comptagline_f(cbc->max_comptag_lines));
+	nvgpu_writel(g, fb_mmu_cbc_max_r(), cbc_max);
+
+	nvgpu_assert(compbit_store_base < U64(U32_MAX));
+	nvgpu_writel(g, fb_mmu_cbc_base_r(),
+		fb_mmu_cbc_base_address_f(U32(compbit_store_base)));
+
+	nvgpu_log(g, gpu_dbg_info | gpu_dbg_map_v | gpu_dbg_pte,
+		"compbit base.pa: 0x%x,%08x cbc_base:0x%llx\n",
+		(u32)(compbit_store_pa >> 32),
+		(u32)(compbit_store_pa & 0xffffffffU),
+		compbit_store_base);
+
+	cbc->compbit_store.base_hw = compbit_store_base;
+
+}
+#endif
+
+static int tu104_fb_wait_mmu_bind(struct gk20a *g)
+{
+	struct nvgpu_timeout timeout;
+	u32 val;
+	int err;
+
+	err = nvgpu_timeout_init(g, &timeout, 1000, NVGPU_TIMER_RETRY_TIMER);
+	if (err != 0) {
+		return err;
+	}
+
+	do {
+		val = nvgpu_readl(g, fb_mmu_bind_r());
+		if ((val & fb_mmu_bind_trigger_true_f()) !=
+			   fb_mmu_bind_trigger_true_f()) {
+			return 0;
+		}
+		nvgpu_udelay(2);
+	} while (nvgpu_timeout_expired_msg(&timeout, "mmu bind timedout") == 0);
+
+	return -ETIMEDOUT;
+}
+
+int tu104_fb_apply_pdb_cache_errata(struct gk20a *g)
+{
+	u64 inst_blk_base_addr;
+	u32 inst_blk_addr;
+	u32 i;
+	int err;
+
+	if (!nvgpu_mem_is_valid(&g->pdb_cache_errata_mem)) {
+		return -EINVAL;
+	}
+
+	inst_blk_base_addr = nvgpu_mem_get_addr(g, &g->pdb_cache_errata_mem);
+
+	/* Bind 256 instance blocks to unused engine ID 0x0 */
+	for (i = 0U; i < 256U; i++) {
+		inst_blk_addr = u64_lo32((inst_blk_base_addr +
+						(U64(i) * U64(NVGPU_CPU_PAGE_SIZE)))
+				>> fb_mmu_bind_imb_addr_alignment_v());
+
+		nvgpu_writel(g, fb_mmu_bind_imb_r(),
+			fb_mmu_bind_imb_addr_f(inst_blk_addr) |
+			nvgpu_aperture_mask(g, &g->pdb_cache_errata_mem,
+				fb_mmu_bind_imb_aperture_sys_mem_nc_f(),
+				fb_mmu_bind_imb_aperture_sys_mem_c_f(),
+				fb_mmu_bind_imb_aperture_vid_mem_f()));
+
+		nvgpu_writel(g, fb_mmu_bind_r(),
+			fb_mmu_bind_engine_id_f(0x0U) |
+			fb_mmu_bind_trigger_true_f());
+
+		err = tu104_fb_wait_mmu_bind(g);
+		if (err != 0) {
+			return err;
+		}
+	}
+
+	/* first unbind */
+	nvgpu_writel(g, fb_mmu_bind_imb_r(),
+		fb_mmu_bind_imb_aperture_f(0x1U) |
+		fb_mmu_bind_imb_addr_f(0x0U));
+
+	nvgpu_writel(g, fb_mmu_bind_r(),
+		fb_mmu_bind_engine_id_f(0x0U) |
+		fb_mmu_bind_trigger_true_f());
+
+	err = tu104_fb_wait_mmu_bind(g);
+	if (err != 0) {
+		return err;
+	}
+
+	/* second unbind */
+	nvgpu_writel(g, fb_mmu_bind_r(),
+		fb_mmu_bind_engine_id_f(0x0U) |
+		fb_mmu_bind_trigger_true_f());
+
+	err = tu104_fb_wait_mmu_bind(g);
+	if (err != 0) {
+		return err;
+	}
+
+	/* Bind 257th (last) instance block that reserves PDB cache entry 255 */
+	inst_blk_addr = u64_lo32((inst_blk_base_addr + (256ULL * U64(NVGPU_CPU_PAGE_SIZE)))
+			>> U64(fb_mmu_bind_imb_addr_alignment_v()));
+
+	nvgpu_writel(g, fb_mmu_bind_imb_r(),
+		fb_mmu_bind_imb_addr_f(inst_blk_addr) |
+		nvgpu_aperture_mask(g, &g->pdb_cache_errata_mem,
+			fb_mmu_bind_imb_aperture_sys_mem_nc_f(),
+			fb_mmu_bind_imb_aperture_sys_mem_c_f(),
+			fb_mmu_bind_imb_aperture_vid_mem_f()));
+
+	nvgpu_writel(g, fb_mmu_bind_r(),
+		fb_mmu_bind_engine_id_f(0x0U) |
+		fb_mmu_bind_trigger_true_f());
+
+	err = tu104_fb_wait_mmu_bind(g);
+	if (err != 0) {
+		return err;
+	}
+
+	return 0;
+}
+
+#ifdef CONFIG_NVGPU_DGPU
+size_t tu104_fb_get_vidmem_size(struct gk20a *g)
+{
+	u32 range = gk20a_readl(g, fb_mmu_local_memory_range_r());
+	u32 mag = fb_mmu_local_memory_range_lower_mag_v(range);
+	u32 scale = fb_mmu_local_memory_range_lower_scale_v(range);
+	u32 ecc = fb_mmu_local_memory_range_ecc_mode_v(range);
+	size_t bytes = ((size_t)mag << scale) * SZ_1M;
+
+#ifdef CONFIG_NVGPU_SIM
+	if (nvgpu_is_enabled(g, NVGPU_IS_FMODEL) && (bytes == 0)) {
+		/* 192 MB */
+		bytes = 192*1024*1024;
+	}
+#endif
+
+	if (ecc != 0U) {
+		bytes = bytes / 16U * 15U;
+	}
+
+	return bytes;
+}
+#endif
+
+int tu104_fb_enable_nvlink(struct gk20a *g)
+{
+	nvgpu_log(g, gpu_dbg_nvlink|gpu_dbg_info, "enabling nvlink");
+
+	return gv100_fb_enable_nvlink(g);
+
+}
+
+int tu104_fb_set_atomic_mode(struct gk20a *g)
+{
+	u32 data;
+
+	gv100_fb_set_atomic_mode(g);
+
+	/* NV_PFB_PRI_MMU_CTRL_ATOMIC_CAPABILITY_SYS_NCOH_MODE to L2 */
+	data = nvgpu_readl(g, fb_mmu_ctrl_r());
+	data = set_field(data, fb_mmu_ctrl_atomic_capability_sys_ncoh_mode_m(),
+		fb_mmu_ctrl_atomic_capability_sys_ncoh_mode_l2_f());
+	nvgpu_writel(g, fb_mmu_ctrl_r(), data);
+
+	/* NV_PFB_FBHUB_NUM_ACTIVE_LTCS_HUB_SYS_NCOH_ATOMIC_MODE to USE_READ */
+	data = nvgpu_readl(g, fb_fbhub_num_active_ltcs_r());
+	data = set_field(data,
+		fb_fbhub_num_active_ltcs_hub_sys_ncoh_atomic_mode_m(),
+		fb_fbhub_num_active_ltcs_hub_sys_ncoh_atomic_mode_use_read_f());
+	nvgpu_writel(g, fb_fbhub_num_active_ltcs_r(), data);
+
+	return 0;
+}

drivers/gpu/nvgpu/hal/fb/fb_tu104.h

@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2018-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef NVGPU_FB_TU104_H
+#define NVGPU_FB_TU104_H
+
+#include <nvgpu/types.h>
+
+struct gk20a;
+struct nvgpu_mem;
+
+int  fb_tu104_tlb_invalidate(struct gk20a *g, struct nvgpu_mem *pdb);
+#ifdef CONFIG_NVGPU_COMPRESSION
+struct nvgpu_cbc;
+void tu104_fb_cbc_configure(struct gk20a *g, struct nvgpu_cbc *cbc);
+#endif
+int  tu104_fb_apply_pdb_cache_errata(struct gk20a *g);
+#ifdef CONFIG_NVGPU_DGPU
+size_t tu104_fb_get_vidmem_size(struct gk20a *g);
+#endif
+int  tu104_fb_enable_nvlink(struct gk20a *g);
+int tu104_fb_set_atomic_mode(struct gk20a *g);
+
+#endif /* NVGPU_FB_TU104_H */

drivers/gpu/nvgpu/hal/fb/intr/fb_intr_ecc_gv11b.c

@@ -0,0 +1,102 @@
+/*
+ * GV11B ECC INTR
+ *
+ * Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/log.h>
+#include <nvgpu/io.h>
+#include <nvgpu/gk20a.h>
+#include <nvgpu/nvgpu_err.h>
+
+#include "fb_intr_ecc_gv11b.h"
+
+#include <nvgpu/hw/gv11b/hw_fb_gv11b.h>
+
+#ifdef CONFIG_NVGPU_INJECT_HWERR
+void gv11b_fb_intr_inject_hubmmu_ecc_error(struct gk20a *g,
+		struct nvgpu_hw_err_inject_info *err,
+		u32 error_info)
+{
+	unsigned int reg_addr = err->get_reg_addr();
+
+	nvgpu_info(g, "Injecting HUBMMU fault %s", err->name);
+	nvgpu_writel(g, reg_addr, err->get_reg_val(1U));
+}
+
+static inline u32 l2tlb_ecc_control_r(void)
+{
+	return fb_mmu_l2tlb_ecc_control_r();
+}
+
+static inline u32 l2tlb_ecc_control_inject_uncorrected_err_f(u32 v)
+{
+	return fb_mmu_l2tlb_ecc_control_inject_uncorrected_err_f(v);
+}
+
+static inline u32 hubtlb_ecc_control_r(void)
+{
+	return fb_mmu_hubtlb_ecc_control_r();
+}
+
+static inline u32 hubtlb_ecc_control_inject_uncorrected_err_f(u32 v)
+{
+	return fb_mmu_hubtlb_ecc_control_inject_uncorrected_err_f(v);
+}
+
+static inline u32 fillunit_ecc_control_r(void)
+{
+	return fb_mmu_fillunit_ecc_control_r();
+}
+
+static inline u32 fillunit_ecc_control_inject_uncorrected_err_f(u32 v)
+{
+	return fb_mmu_fillunit_ecc_control_inject_uncorrected_err_f(v);
+}
+
+static struct nvgpu_hw_err_inject_info hubmmu_ecc_err_desc[] = {
+	NVGPU_ECC_ERR("hubmmu_l2tlb_sa_data_ecc_uncorrected",
+			gv11b_fb_intr_inject_hubmmu_ecc_error,
+			l2tlb_ecc_control_r,
+			l2tlb_ecc_control_inject_uncorrected_err_f),
+	NVGPU_ECC_ERR("hubmmu_tlb_sa_data_ecc_uncorrected",
+			gv11b_fb_intr_inject_hubmmu_ecc_error,
+			hubtlb_ecc_control_r,
+			hubtlb_ecc_control_inject_uncorrected_err_f),
+	NVGPU_ECC_ERR("hubmmu_pte_data_ecc_uncorrected",
+			gv11b_fb_intr_inject_hubmmu_ecc_error,
+			fillunit_ecc_control_r,
+			fillunit_ecc_control_inject_uncorrected_err_f),
+};
+
+static struct nvgpu_hw_err_inject_info_desc hubmmu_err_desc;
+
+struct nvgpu_hw_err_inject_info_desc *
+gv11b_fb_intr_get_hubmmu_err_desc(struct gk20a *g)
+{
+	hubmmu_err_desc.info_ptr = hubmmu_ecc_err_desc;
+	hubmmu_err_desc.info_size = nvgpu_safe_cast_u64_to_u32(
+			sizeof(hubmmu_ecc_err_desc) /
+			sizeof(struct nvgpu_hw_err_inject_info));
+
+	return &hubmmu_err_desc;
+}
+#endif /* CONFIG_NVGPU_INJECT_HWERR */

drivers/gpu/nvgpu/hal/fb/intr/fb_intr_ecc_gv11b.h

@@ -0,0 +1,47 @@
+/*
+ * GV11B FB INTR ECC
+ *
+ * Copyright (c) 2016-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef NVGPU_FB_INTR_ECC_GV11B_H
+#define NVGPU_FB_INTR_ECC_GV11B_H
+
+#include <nvgpu/types.h>
+#include <nvgpu/nvgpu_err.h>
+
+struct gk20a;
+struct nvgpu_hw_err_inject_info;
+struct nvgpu_hw_err_inject_info_desc;
+
+void gv11b_fb_intr_handle_ecc(struct gk20a *g);
+void gv11b_fb_intr_handle_ecc_l2tlb(struct gk20a *g, u32 ecc_status);
+void gv11b_fb_intr_handle_ecc_fillunit(struct gk20a *g, u32 ecc_status);
+void gv11b_fb_intr_handle_ecc_hubtlb(struct gk20a *g, u32 ecc_status);
+
+#ifdef CONFIG_NVGPU_INJECT_HWERR
+struct nvgpu_hw_err_inject_info_desc *
+		gv11b_fb_intr_get_hubmmu_err_desc(struct gk20a *g);
+void gv11b_fb_intr_inject_hubmmu_ecc_error(struct gk20a *g,
+		struct nvgpu_hw_err_inject_info *err, u32 error_info);
+#endif
+
+#endif /* NVGPU_FB_INTR_ECC_GV11B_H */

drivers/gpu/nvgpu/hal/fb/intr/fb_intr_ecc_gv11b_fusa.c

@@ -0,0 +1,344 @@
+/*
+ * GV11B ECC INTR
+ *
+ * Copyright (c) 2016-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/log.h>
+#include <nvgpu/io.h>
+#include <nvgpu/gk20a.h>
+#include <nvgpu/nvgpu_err.h>
+
+#include "fb_intr_ecc_gv11b.h"
+
+#include <nvgpu/hw/gv11b/hw_fb_gv11b.h>
+
+static void gv11b_fb_intr_handle_ecc_l2tlb_errs(struct gk20a *g,
+		u32 ecc_status, u32 ecc_addr)
+{
+	u32 corrected_error_mask = 0U;
+	u32 uncorrected_error_mask = 0U;
+
+	g->ops.fb.ecc.l2tlb_error_mask(&corrected_error_mask,
+			&uncorrected_error_mask);
+
+	if ((ecc_status & corrected_error_mask) != 0U) {
+		nvgpu_log(g, gpu_dbg_intr, "corrected ecc sa data error");
+		/* This error is not expected to occur in gv11b and hence,
+		 * this scenario is considered as a fatal error.
+		 */
+		nvgpu_mutex_release(&g->mm.hub_isr_mutex);
+		BUG();
+	}
+	if ((ecc_status & uncorrected_error_mask) != 0U) {
+		nvgpu_report_fb_ecc_err(g,
+			GPU_HUBMMU_L2TLB_SA_DATA_ECC_UNCORRECTED,
+			ecc_addr,
+			g->ecc.fb.mmu_l2tlb_ecc_uncorrected_err_count[0].counter);
+		nvgpu_log(g, gpu_dbg_intr, "uncorrected ecc sa data error");
+	}
+}
+
+void gv11b_fb_intr_handle_ecc_l2tlb(struct gk20a *g, u32 ecc_status)
+{
+	u32 ecc_addr, corrected_cnt, uncorrected_cnt;
+	u32 corrected_delta, uncorrected_delta;
+	u32 corrected_overflow, uncorrected_overflow;
+
+	ecc_addr = nvgpu_readl(g, fb_mmu_l2tlb_ecc_address_r());
+	corrected_cnt = nvgpu_readl(g,
+		fb_mmu_l2tlb_ecc_corrected_err_count_r());
+	uncorrected_cnt = nvgpu_readl(g,
+		fb_mmu_l2tlb_ecc_uncorrected_err_count_r());
+
+	corrected_delta = fb_mmu_l2tlb_ecc_corrected_err_count_total_v(
+							corrected_cnt);
+	uncorrected_delta = fb_mmu_l2tlb_ecc_uncorrected_err_count_total_v(
+							uncorrected_cnt);
+	corrected_overflow = ecc_status &
+		fb_mmu_l2tlb_ecc_status_corrected_err_total_counter_overflow_m();
+
+	uncorrected_overflow = ecc_status &
+		fb_mmu_l2tlb_ecc_status_uncorrected_err_total_counter_overflow_m();
+
+	/* clear the interrupt */
+	if ((corrected_delta > 0U) || (corrected_overflow != 0U)) {
+		nvgpu_writel(g, fb_mmu_l2tlb_ecc_corrected_err_count_r(), 0);
+	}
+	if ((uncorrected_delta > 0U) || (uncorrected_overflow != 0U)) {
+		nvgpu_writel(g, fb_mmu_l2tlb_ecc_uncorrected_err_count_r(), 0);
+	}
+
+	nvgpu_writel(g, fb_mmu_l2tlb_ecc_status_r(),
+				fb_mmu_l2tlb_ecc_status_reset_clear_f());
+
+	/* Handle overflow */
+	if (corrected_overflow != 0U) {
+		corrected_delta +=
+			BIT32(fb_mmu_l2tlb_ecc_corrected_err_count_total_s());
+	}
+	if (uncorrected_overflow != 0U) {
+		uncorrected_delta +=
+			BIT32(fb_mmu_l2tlb_ecc_uncorrected_err_count_total_s());
+	}
+
+	g->ecc.fb.mmu_l2tlb_ecc_corrected_err_count[0].counter =
+		nvgpu_safe_add_u32(
+			g->ecc.fb.mmu_l2tlb_ecc_corrected_err_count[0].counter,
+			corrected_delta);
+	g->ecc.fb.mmu_l2tlb_ecc_uncorrected_err_count[0].counter =
+		nvgpu_safe_add_u32(
+		       g->ecc.fb.mmu_l2tlb_ecc_uncorrected_err_count[0].counter,
+		       uncorrected_delta);
+
+	gv11b_fb_intr_handle_ecc_l2tlb_errs(g, ecc_status, ecc_addr);
+
+	if ((corrected_overflow != 0U) || (uncorrected_overflow != 0U)) {
+		nvgpu_info(g, "mmu l2tlb ecc counter overflow!");
+	}
+
+	nvgpu_log(g, gpu_dbg_intr,
+		"ecc error address: 0x%x", ecc_addr);
+	nvgpu_log(g, gpu_dbg_intr,
+		"ecc error count corrected: %d, uncorrected %d",
+		g->ecc.fb.mmu_l2tlb_ecc_corrected_err_count[0].counter,
+		g->ecc.fb.mmu_l2tlb_ecc_uncorrected_err_count[0].counter);
+}
+
+static void gv11b_fb_intr_handle_ecc_hubtlb_errs(struct gk20a *g,
+		u32 ecc_status, u32 ecc_addr)
+{
+	if ((ecc_status &
+	     fb_mmu_hubtlb_ecc_status_corrected_err_sa_data_m()) != 0U) {
+		nvgpu_log(g, gpu_dbg_intr, "corrected ecc sa data error");
+		/* This error is not expected to occur in gv11b and hence,
+		 * this scenario is considered as a fatal error.
+		 */
+		nvgpu_mutex_release(&g->mm.hub_isr_mutex);
+		BUG();
+	}
+	if ((ecc_status &
+	     fb_mmu_hubtlb_ecc_status_uncorrected_err_sa_data_m()) != 0U) {
+		nvgpu_report_fb_ecc_err(g,
+				GPU_HUBMMU_TLB_SA_DATA_ECC_UNCORRECTED,
+				ecc_addr,
+				g->ecc.fb.mmu_hubtlb_ecc_uncorrected_err_count[0].counter);
+		nvgpu_log(g, gpu_dbg_intr, "uncorrected ecc sa data error");
+	}
+
+}
+
+void gv11b_fb_intr_handle_ecc_hubtlb(struct gk20a *g, u32 ecc_status)
+{
+	u32 ecc_addr, corrected_cnt, uncorrected_cnt;
+	u32 corrected_delta, uncorrected_delta;
+	u32 corrected_overflow, uncorrected_overflow;
+
+	ecc_addr = nvgpu_readl(g, fb_mmu_hubtlb_ecc_address_r());
+	corrected_cnt = nvgpu_readl(g,
+		fb_mmu_hubtlb_ecc_corrected_err_count_r());
+	uncorrected_cnt = nvgpu_readl(g,
+		fb_mmu_hubtlb_ecc_uncorrected_err_count_r());
+
+	corrected_delta = fb_mmu_hubtlb_ecc_corrected_err_count_total_v(
+							corrected_cnt);
+	uncorrected_delta = fb_mmu_hubtlb_ecc_uncorrected_err_count_total_v(
+							uncorrected_cnt);
+	corrected_overflow = ecc_status &
+		fb_mmu_hubtlb_ecc_status_corrected_err_total_counter_overflow_m();
+
+	uncorrected_overflow = ecc_status &
+		fb_mmu_hubtlb_ecc_status_uncorrected_err_total_counter_overflow_m();
+
+	/* clear the interrupt */
+	if ((corrected_delta > 0U) || (corrected_overflow != 0U)) {
+		nvgpu_writel(g, fb_mmu_hubtlb_ecc_corrected_err_count_r(), 0);
+	}
+	if ((uncorrected_delta > 0U) || (uncorrected_overflow != 0U)) {
+		nvgpu_writel(g, fb_mmu_hubtlb_ecc_uncorrected_err_count_r(), 0);
+	}
+
+	nvgpu_writel(g, fb_mmu_hubtlb_ecc_status_r(),
+				fb_mmu_hubtlb_ecc_status_reset_clear_f());
+
+	/* Handle overflow */
+	if (corrected_overflow != 0U) {
+		corrected_delta +=
+			BIT32(fb_mmu_hubtlb_ecc_corrected_err_count_total_s());
+	}
+	if (uncorrected_overflow != 0U) {
+		uncorrected_delta +=
+			BIT32(fb_mmu_hubtlb_ecc_uncorrected_err_count_total_s());
+	}
+
+	g->ecc.fb.mmu_hubtlb_ecc_corrected_err_count[0].counter =
+		nvgpu_safe_add_u32(
+			g->ecc.fb.mmu_hubtlb_ecc_corrected_err_count[0].counter,
+			corrected_delta);
+	g->ecc.fb.mmu_hubtlb_ecc_uncorrected_err_count[0].counter =
+		nvgpu_safe_add_u32(
+		      g->ecc.fb.mmu_hubtlb_ecc_uncorrected_err_count[0].counter,
+		      uncorrected_delta);
+
+
+	gv11b_fb_intr_handle_ecc_hubtlb_errs(g, ecc_status, ecc_addr);
+
+	if ((corrected_overflow != 0U) || (uncorrected_overflow != 0U)) {
+		nvgpu_info(g, "mmu hubtlb ecc counter overflow!");
+	}
+
+	nvgpu_log(g, gpu_dbg_intr,
+		"ecc error address: 0x%x", ecc_addr);
+	nvgpu_log(g, gpu_dbg_intr,
+		"ecc error count corrected: %d, uncorrected %d",
+		g->ecc.fb.mmu_hubtlb_ecc_corrected_err_count[0].counter,
+		g->ecc.fb.mmu_hubtlb_ecc_uncorrected_err_count[0].counter);
+}
+
+static void gv11b_fb_intr_handle_ecc_fillunit_errors(struct gk20a *g,
+		u32 ecc_status, u32 ecc_addr)
+{
+	if ((ecc_status &
+		fb_mmu_fillunit_ecc_status_corrected_err_pte_data_m()) != 0U) {
+		nvgpu_log(g, gpu_dbg_intr, "corrected ecc pte data error");
+		/* This error is not expected to occur in gv11b and hence,
+		 * this scenario is considered as a fatal error.
+		 */
+		nvgpu_mutex_release(&g->mm.hub_isr_mutex);
+		BUG();
+	}
+	if ((ecc_status &
+		fb_mmu_fillunit_ecc_status_uncorrected_err_pte_data_m())
+									!= 0U) {
+		nvgpu_report_fb_ecc_err(g,
+			GPU_HUBMMU_PTE_DATA_ECC_UNCORRECTED,
+			ecc_addr,
+			g->ecc.fb.mmu_fillunit_ecc_uncorrected_err_count[0].counter);
+		nvgpu_log(g, gpu_dbg_intr, "uncorrected ecc pte data error");
+	}
+	if ((ecc_status &
+		fb_mmu_fillunit_ecc_status_corrected_err_pde0_data_m()) != 0U) {
+		nvgpu_log(g, gpu_dbg_intr, "corrected ecc pde0 data error");
+		/* This error is not expected to occur in gv11b and hence,
+		 * this scenario is considered as a fatal error.
+		 */
+		nvgpu_mutex_release(&g->mm.hub_isr_mutex);
+		BUG();
+	}
+	if ((ecc_status &
+		fb_mmu_fillunit_ecc_status_uncorrected_err_pde0_data_m())
+									!= 0U) {
+		nvgpu_report_fb_ecc_err(g,
+			GPU_HUBMMU_PDE0_DATA_ECC_UNCORRECTED,
+			ecc_addr,
+			g->ecc.fb.mmu_fillunit_ecc_uncorrected_err_count[0].counter);
+		nvgpu_log(g, gpu_dbg_intr, "uncorrected ecc pde0 data error");
+	}
+}
+
+void gv11b_fb_intr_handle_ecc_fillunit(struct gk20a *g, u32 ecc_status)
+{
+	u32 ecc_addr, corrected_cnt, uncorrected_cnt;
+	u32 corrected_delta, uncorrected_delta;
+	u32 corrected_overflow, uncorrected_overflow;
+
+	ecc_addr = nvgpu_readl(g, fb_mmu_fillunit_ecc_address_r());
+	corrected_cnt = nvgpu_readl(g,
+		fb_mmu_fillunit_ecc_corrected_err_count_r());
+	uncorrected_cnt = nvgpu_readl(g,
+		fb_mmu_fillunit_ecc_uncorrected_err_count_r());
+
+	corrected_delta = fb_mmu_fillunit_ecc_corrected_err_count_total_v(
+							corrected_cnt);
+	uncorrected_delta = fb_mmu_fillunit_ecc_uncorrected_err_count_total_v(
+							uncorrected_cnt);
+	corrected_overflow = ecc_status &
+		fb_mmu_fillunit_ecc_status_corrected_err_total_counter_overflow_m();
+
+	uncorrected_overflow = ecc_status &
+		fb_mmu_fillunit_ecc_status_uncorrected_err_total_counter_overflow_m();
+
+	/* clear the interrupt */
+	if ((corrected_delta > 0U) || (corrected_overflow != 0U)) {
+		nvgpu_writel(g, fb_mmu_fillunit_ecc_corrected_err_count_r(), 0);
+	}
+	if ((uncorrected_delta > 0U) || (uncorrected_overflow != 0U)) {
+		nvgpu_writel(g,
+			fb_mmu_fillunit_ecc_uncorrected_err_count_r(), 0);
+	}
+
+	nvgpu_writel(g, fb_mmu_fillunit_ecc_status_r(),
+				fb_mmu_fillunit_ecc_status_reset_clear_f());
+
+	/* Handle overflow */
+	if (corrected_overflow != 0U) {
+		corrected_delta +=
+			BIT32(fb_mmu_fillunit_ecc_corrected_err_count_total_s());
+	}
+	if (uncorrected_overflow != 0U) {
+		uncorrected_delta +=
+			BIT32(fb_mmu_fillunit_ecc_uncorrected_err_count_total_s());
+	}
+
+	g->ecc.fb.mmu_fillunit_ecc_corrected_err_count[0].counter =
+		nvgpu_safe_add_u32(
+		      g->ecc.fb.mmu_fillunit_ecc_corrected_err_count[0].counter,
+		      corrected_delta);
+	g->ecc.fb.mmu_fillunit_ecc_uncorrected_err_count[0].counter =
+		nvgpu_safe_add_u32(
+		    g->ecc.fb.mmu_fillunit_ecc_uncorrected_err_count[0].counter,
+		    uncorrected_delta);
+
+	gv11b_fb_intr_handle_ecc_fillunit_errors(g, ecc_status, ecc_addr);
+
+	if ((corrected_overflow != 0U) || (uncorrected_overflow != 0U)) {
+		nvgpu_info(g, "mmu fillunit ecc counter overflow!");
+	}
+
+	nvgpu_log(g, gpu_dbg_intr,
+		"ecc error address: 0x%x", ecc_addr);
+	nvgpu_log(g, gpu_dbg_intr,
+		"ecc error count corrected: %d, uncorrected %d",
+		g->ecc.fb.mmu_fillunit_ecc_corrected_err_count[0].counter,
+		g->ecc.fb.mmu_fillunit_ecc_uncorrected_err_count[0].counter);
+}
+
+void gv11b_fb_intr_handle_ecc(struct gk20a *g)
+{
+	u32 status;
+
+	nvgpu_info(g, "ecc uncorrected error notify");
+
+	status = nvgpu_readl(g, fb_mmu_l2tlb_ecc_status_r());
+	if (status != 0U) {
+		g->ops.fb.intr.handle_ecc_l2tlb(g, status);
+	}
+
+	status = nvgpu_readl(g, fb_mmu_hubtlb_ecc_status_r());
+	if (status != 0U) {
+		g->ops.fb.intr.handle_ecc_hubtlb(g, status);
+	}
+
+	status = nvgpu_readl(g, fb_mmu_fillunit_ecc_status_r());
+	if (status != 0U) {
+		g->ops.fb.intr.handle_ecc_fillunit(g, status);
+	}
+}

drivers/gpu/nvgpu/hal/fb/intr/fb_intr_gv100.c

@@ -0,0 +1,56 @@
+/*
+ * GV100 FB INTR
+ *
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/gk20a.h>
+#include <nvgpu/io.h>
+
+#include "fb_intr_gv100.h"
+
+#include <nvgpu/hw/gv100/hw_fb_gv100.h>
+
+void gv100_fb_intr_enable(struct gk20a *g)
+{
+	u32 mask;
+
+	mask = fb_niso_intr_en_set_mmu_other_fault_notify_m() |
+		fb_niso_intr_en_set_mmu_nonreplayable_fault_notify_m() |
+		fb_niso_intr_en_set_mmu_nonreplayable_fault_overflow_m() |
+		fb_niso_intr_en_set_mmu_replayable_fault_notify_m() |
+		fb_niso_intr_en_set_mmu_replayable_fault_overflow_m();
+
+	nvgpu_writel(g, fb_niso_intr_en_set_r(0), mask);
+}
+
+void gv100_fb_intr_disable(struct gk20a *g)
+{
+	u32 mask;
+
+	mask = fb_niso_intr_en_set_mmu_other_fault_notify_m() |
+		fb_niso_intr_en_set_mmu_nonreplayable_fault_notify_m() |
+		fb_niso_intr_en_set_mmu_nonreplayable_fault_overflow_m() |
+		fb_niso_intr_en_set_mmu_replayable_fault_notify_m() |
+		fb_niso_intr_en_set_mmu_replayable_fault_overflow_m();
+
+	nvgpu_writel(g, fb_niso_intr_en_clr_r(0), mask);
+}

drivers/gpu/nvgpu/hal/fb/intr/fb_intr_gv100.h

@@ -0,0 +1,33 @@
+/*
+ * GV100 FB INTR
+ *
+ * Copyright (c) 2017-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef NVGPU_FB_INTR_GV100_H
+#define NVGPU_FB_INTR_GV100_H
+
+struct gk20a;
+
+void gv100_fb_intr_enable(struct gk20a *g);
+void gv100_fb_intr_disable(struct gk20a *g);
+
+#endif /* NVGPU_FB_INTR_GV100_H */

drivers/gpu/nvgpu/hal/fb/intr/fb_intr_gv11b.h

@@ -0,0 +1,35 @@
+/*
+ * GV11B FB INTR
+ *
+ * Copyright (c) 2016-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef NVGPU_FB_INTR_GV11B_H
+#define NVGPU_FB_INTR_GV11B_H
+
+struct gk20a;
+
+void gv11b_fb_intr_enable(struct gk20a *g);
+void gv11b_fb_intr_disable(struct gk20a *g);
+void gv11b_fb_intr_isr(struct gk20a *g, u32 intr_unit_bitmask);
+bool gv11b_fb_intr_is_mmu_fault_pending(struct gk20a *g);
+
+#endif /* NVGPU_FB_INTR_GV11B_H */

drivers/gpu/nvgpu/hal/fb/intr/fb_intr_gv11b_fusa.c

@@ -0,0 +1,126 @@
+/*
+ * GV11B FB
+ *
+ * Copyright (c) 2016-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/log.h>
+#include <nvgpu/io.h>
+#include <nvgpu/gk20a.h>
+#include <nvgpu/cic.h>
+#include <nvgpu/mc.h>
+
+#include "hal/fb/fb_gv11b.h"
+#include "hal/fb/fb_mmu_fault_gv11b.h"
+
+#include "fb_intr_gv11b.h"
+
+#include <nvgpu/hw/gv11b/hw_fb_gv11b.h>
+
+void gv11b_fb_intr_enable(struct gk20a *g)
+{
+	u32 mask;
+
+	mask = fb_niso_intr_en_set_mmu_other_fault_notify_m() |
+		fb_niso_intr_en_set_mmu_nonreplayable_fault_notify_m() |
+		fb_niso_intr_en_set_mmu_nonreplayable_fault_overflow_m() |
+#ifdef CONFIG_NVGPU_REPLAYABLE_FAULT
+		fb_niso_intr_en_set_mmu_replayable_fault_notify_m() |
+		fb_niso_intr_en_set_mmu_replayable_fault_overflow_m() |
+#endif
+		fb_niso_intr_en_set_mmu_ecc_uncorrected_error_notify_m();
+
+	nvgpu_cic_intr_stall_unit_config(g, NVGPU_CIC_INTR_UNIT_HUB, NVGPU_CIC_INTR_ENABLE);
+
+	nvgpu_writel(g, fb_niso_intr_en_set_r(0), mask);
+}
+
+void gv11b_fb_intr_disable(struct gk20a *g)
+{
+	u32 mask;
+
+	mask = fb_niso_intr_en_set_mmu_other_fault_notify_m() |
+		fb_niso_intr_en_set_mmu_nonreplayable_fault_notify_m() |
+		fb_niso_intr_en_set_mmu_nonreplayable_fault_overflow_m() |
+#ifdef CONFIG_NVGPU_REPLAYABLE_FAULT
+		fb_niso_intr_en_set_mmu_replayable_fault_notify_m() |
+		fb_niso_intr_en_set_mmu_replayable_fault_overflow_m() |
+#endif
+		fb_niso_intr_en_set_mmu_ecc_uncorrected_error_notify_m();
+
+	nvgpu_writel(g, fb_niso_intr_en_clr_r(0), mask);
+
+	nvgpu_cic_intr_stall_unit_config(g, NVGPU_CIC_INTR_UNIT_HUB, NVGPU_CIC_INTR_DISABLE);
+}
+
+void gv11b_fb_intr_isr(struct gk20a *g, u32 intr_unit_bitmask)
+{
+	u32 niso_intr;
+
+	nvgpu_mutex_acquire(&g->mm.hub_isr_mutex);
+
+	niso_intr = nvgpu_readl(g, fb_niso_intr_r());
+
+	nvgpu_log(g, gpu_dbg_intr, "enter hub isr, niso_intr = 0x%08x",
+			niso_intr);
+
+	if ((niso_intr &
+		 (fb_niso_intr_hub_access_counter_notify_m() |
+		  fb_niso_intr_hub_access_counter_error_m())) != 0U) {
+
+		nvgpu_info(g, "hub access counter notify/error");
+	}
+	if ((niso_intr &
+	     fb_niso_intr_mmu_ecc_uncorrected_error_notify_pending_f()) != 0U) {
+		g->ops.fb.intr.handle_ecc(g);
+	}
+	if ((niso_intr &
+		(fb_niso_intr_mmu_other_fault_notify_m() |
+#ifdef CONFIG_NVGPU_REPLAYABLE_FAULT
+		fb_niso_intr_mmu_replayable_fault_notify_m() |
+		fb_niso_intr_mmu_replayable_fault_overflow_m() |
+#endif
+		fb_niso_intr_mmu_nonreplayable_fault_notify_m() |
+		fb_niso_intr_mmu_nonreplayable_fault_overflow_m())) != 0U) {
+
+		nvgpu_log(g, gpu_dbg_intr, "MMU Fault");
+		gv11b_fb_handle_mmu_fault(g, niso_intr);
+	}
+
+	nvgpu_mutex_release(&g->mm.hub_isr_mutex);
+}
+
+bool gv11b_fb_intr_is_mmu_fault_pending(struct gk20a *g)
+{
+	if ((gk20a_readl(g, fb_niso_intr_r()) &
+		(fb_niso_intr_mmu_other_fault_notify_m() |
+		 fb_niso_intr_mmu_ecc_uncorrected_error_notify_m() |
+#ifdef CONFIG_NVGPU_REPLAYABLE_FAULT
+		 fb_niso_intr_mmu_replayable_fault_notify_m() |
+		 fb_niso_intr_mmu_replayable_fault_overflow_m() |
+#endif
+		 fb_niso_intr_mmu_nonreplayable_fault_notify_m() |
+		 fb_niso_intr_mmu_nonreplayable_fault_overflow_m())) != 0U) {
+		return true;
+	}
+
+	return false;
+}

drivers/gpu/nvgpu/hal/fb/intr/fb_intr_tu104.c

@@ -0,0 +1,142 @@
+/*
+ * Copyright (c) 2018-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/log.h>
+#include <nvgpu/io.h>
+#include <nvgpu/gk20a.h>
+
+#include "hal/fb/fb_tu104.h"
+#include "hal/fb/fb_mmu_fault_tu104.h"
+#include "hal/mc/mc_tu104.h"
+
+#include "fb_intr_tu104.h"
+
+#include "nvgpu/hw/tu104/hw_fb_tu104.h"
+
+void tu104_fb_intr_enable(struct gk20a *g)
+{
+	u32 info_fault = nvgpu_readl(g, fb_mmu_int_vector_info_fault_r());
+	u32 nonreplay_fault = nvgpu_readl(g,
+		fb_mmu_int_vector_fault_r(
+				NVGPU_MMU_FAULT_NONREPLAY_REG_INDX));
+	u32 replay_fault = nvgpu_readl(g,
+		fb_mmu_int_vector_fault_r(NVGPU_MMU_FAULT_REPLAY_REG_INDX));
+	u32 ecc_error = nvgpu_readl(g, fb_mmu_int_vector_ecc_error_r());
+
+	intr_tu104_vector_en_set(g,
+		fb_mmu_int_vector_info_fault_vector_v(info_fault));
+	intr_tu104_vector_en_set(g,
+		fb_mmu_int_vector_fault_notify_v(nonreplay_fault));
+	intr_tu104_vector_en_set(g,
+		fb_mmu_int_vector_fault_error_v(nonreplay_fault));
+	intr_tu104_vector_en_set(g,
+		fb_mmu_int_vector_fault_notify_v(replay_fault));
+	intr_tu104_vector_en_set(g,
+		fb_mmu_int_vector_fault_error_v(replay_fault));
+	intr_tu104_vector_en_set(g,
+		fb_mmu_int_vector_ecc_error_vector_v(ecc_error));
+}
+
+void tu104_fb_intr_disable(struct gk20a *g)
+{
+	u32 info_fault = nvgpu_readl(g, fb_mmu_int_vector_info_fault_r());
+	u32 nonreplay_fault = nvgpu_readl(g,
+		fb_mmu_int_vector_fault_r(
+				NVGPU_MMU_FAULT_NONREPLAY_REG_INDX));
+	u32 replay_fault = nvgpu_readl(g,
+		fb_mmu_int_vector_fault_r(NVGPU_MMU_FAULT_REPLAY_REG_INDX));
+	u32 ecc_error = nvgpu_readl(g, fb_mmu_int_vector_ecc_error_r());
+
+	intr_tu104_vector_en_clear(g,
+		fb_mmu_int_vector_info_fault_vector_v(info_fault));
+	intr_tu104_vector_en_clear(g,
+		fb_mmu_int_vector_fault_notify_v(nonreplay_fault));
+	intr_tu104_vector_en_clear(g,
+		fb_mmu_int_vector_fault_error_v(nonreplay_fault));
+	intr_tu104_vector_en_clear(g,
+		fb_mmu_int_vector_fault_notify_v(replay_fault));
+	intr_tu104_vector_en_clear(g,
+		fb_mmu_int_vector_fault_error_v(replay_fault));
+	intr_tu104_vector_en_clear(g,
+		fb_mmu_int_vector_ecc_error_vector_v(ecc_error));
+}
+
+void tu104_fb_intr_isr(struct gk20a *g, u32 intr_unit_bitmask)
+{
+	u32 info_fault = nvgpu_readl(g, fb_mmu_int_vector_info_fault_r());
+	u32 nonreplay_fault = nvgpu_readl(g,
+		fb_mmu_int_vector_fault_r(
+				NVGPU_MMU_FAULT_NONREPLAY_REG_INDX));
+	u32 replay_fault = nvgpu_readl(g,
+		fb_mmu_int_vector_fault_r(NVGPU_MMU_FAULT_REPLAY_REG_INDX));
+	u32 ecc_error = nvgpu_readl(g, fb_mmu_int_vector_ecc_error_r());
+
+	nvgpu_mutex_acquire(&g->mm.hub_isr_mutex);
+
+	if (intr_tu104_vector_intr_pending(g,
+			fb_mmu_int_vector_ecc_error_vector_v(ecc_error))) {
+		g->ops.fb.intr.handle_ecc(g);
+	}
+
+	if (intr_tu104_vector_intr_pending(g,
+			fb_mmu_int_vector_fault_notify_v(replay_fault)) ||
+	    intr_tu104_vector_intr_pending(g,
+			fb_mmu_int_vector_fault_error_v(replay_fault)) ||
+	    intr_tu104_vector_intr_pending(g,
+			fb_mmu_int_vector_fault_notify_v(nonreplay_fault)) ||
+	    intr_tu104_vector_intr_pending(g,
+			fb_mmu_int_vector_fault_error_v(nonreplay_fault)) ||
+	    intr_tu104_vector_intr_pending(g,
+			fb_mmu_int_vector_info_fault_vector_v(info_fault))) {
+		nvgpu_log(g, gpu_dbg_intr, "MMU Fault");
+		tu104_fb_handle_mmu_fault(g);
+	}
+
+	nvgpu_mutex_release(&g->mm.hub_isr_mutex);
+}
+
+bool tu104_fb_intr_is_mmu_fault_pending(struct gk20a *g)
+{
+	u32 info_fault = nvgpu_readl(g, fb_mmu_int_vector_info_fault_r());
+	u32 nonreplay_fault = nvgpu_readl(g,
+		fb_mmu_int_vector_fault_r(NVGPU_MMU_FAULT_NONREPLAY_REG_INDX));
+	u32 replay_fault = nvgpu_readl(g,
+		fb_mmu_int_vector_fault_r(NVGPU_MMU_FAULT_REPLAY_REG_INDX));
+	u32 ecc_error = nvgpu_readl(g, fb_mmu_int_vector_ecc_error_r());
+
+	if (intr_tu104_vector_intr_pending(g,
+		fb_mmu_int_vector_fault_notify_v(replay_fault)) ||
+	    intr_tu104_vector_intr_pending(g,
+		fb_mmu_int_vector_fault_error_v(replay_fault)) ||
+	    intr_tu104_vector_intr_pending(g,
+		fb_mmu_int_vector_fault_notify_v(nonreplay_fault)) ||
+	    intr_tu104_vector_intr_pending(g,
+		fb_mmu_int_vector_fault_error_v(nonreplay_fault)) ||
+	    intr_tu104_vector_intr_pending(g,
+		fb_mmu_int_vector_info_fault_vector_v(info_fault)) ||
+	    intr_tu104_vector_intr_pending(g,
+		fb_mmu_int_vector_ecc_error_vector_v(ecc_error))) {
+		return true;
+	}
+
+	return false;
+}

drivers/gpu/nvgpu/hal/fb/intr/fb_intr_tu104.h

@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2018-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef NVGPU_FB_INTR_TU104_H
+#define NVGPU_FB_INTR_TU104_H
+
+#include <nvgpu/types.h>
+
+struct gk20a;
+
+void tu104_fb_intr_enable(struct gk20a *g);
+void tu104_fb_intr_disable(struct gk20a *g);
+void tu104_fb_intr_isr(struct gk20a *g, u32 intr_unit_bitmask);
+bool tu104_fb_intr_is_mmu_fault_pending(struct gk20a *g);
+
+#endif /* NVGPU_FB_INTR_TU104_H */

drivers/gpu/nvgpu/hal/fbpa/fbpa_tu104.c

@@ -0,0 +1,202 @@
+/*
+ * TU104 FBPA
+ *
+ * Copyright (c) 2018-2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/string.h>
+#include <nvgpu/types.h>
+#include <nvgpu/io.h>
+#include <nvgpu/gk20a.h>
+#include <nvgpu/hw/tu104/hw_fbpa_tu104.h>
+#include <nvgpu/mc.h>
+#include <nvgpu/cic.h>
+#include <nvgpu/nvgpu_init.h>
+
+#include "hal/fbpa/fbpa_tu104.h"
+
+int tu104_fbpa_init(struct gk20a *g)
+{
+	u32 val;
+
+	val = nvgpu_readl(g, fbpa_ecc_intr_ctrl_r());
+	val |= fbpa_ecc_intr_ctrl_sec_intr_en_enabled_f() |
+		fbpa_ecc_intr_ctrl_ded_intr_en_enabled_f();
+
+	nvgpu_cic_intr_stall_unit_config(g, NVGPU_CIC_INTR_UNIT_FBPA, NVGPU_CIC_INTR_ENABLE);
+
+	nvgpu_writel(g, fbpa_ecc_intr_ctrl_r(), val);
+	/* read back broadcast register */
+	(void) nvgpu_readl(g, fbpa_ecc_intr_ctrl_r());
+
+	return 0;
+}
+
+static void tu104_fbpa_handle_ecc_intr(struct gk20a *g,
+					u32 fbpa_id, u32 subp_id)
+{
+	u32 status, sec_cnt, ded_cnt;
+	u32 offset = nvgpu_get_litter_value(g, GPU_LIT_FBPA_STRIDE) * fbpa_id;
+	u32 cnt_idx = fbpa_id * 2U + subp_id;
+
+	status = nvgpu_readl(g, offset + fbpa_0_ecc_status_r(subp_id));
+
+	if ((status & fbpa_0_ecc_status_sec_counter_overflow_pending_f()) != 0U) {
+		nvgpu_err(g, "fbpa %u subp %u ecc sec counter overflow",
+				fbpa_id, subp_id);
+	}
+
+	if ((status & fbpa_0_ecc_status_ded_counter_overflow_pending_f()) != 0U) {
+		nvgpu_err(g, "fbpa %u subp %u ecc ded counter overflow",
+				fbpa_id, subp_id);
+	}
+
+	if ((status & fbpa_0_ecc_status_sec_intr_pending_f()) != 0U) {
+		sec_cnt = nvgpu_readl(g,
+				offset + fbpa_0_ecc_sec_count_r(subp_id));
+		nvgpu_writel(g, offset + fbpa_0_ecc_sec_count_r(subp_id), 0u);
+		g->ecc.fbpa.fbpa_ecc_sec_err_count[cnt_idx].counter += sec_cnt;
+	}
+
+	if ((status & fbpa_0_ecc_status_ded_intr_pending_f()) != 0U) {
+		ded_cnt = nvgpu_readl(g,
+				offset + fbpa_0_ecc_ded_count_r(subp_id));
+		nvgpu_writel(g, offset + fbpa_0_ecc_ded_count_r(subp_id), 0u);
+		g->ecc.fbpa.fbpa_ecc_ded_err_count[cnt_idx].counter += ded_cnt;
+	}
+
+	nvgpu_writel(g, offset + fbpa_0_ecc_status_r(subp_id), status);
+}
+
+void tu104_fbpa_handle_intr(struct gk20a *g, u32 fbpa_id)
+{
+	u32 offset, status;
+	u32 ecc_subp0_mask = fbpa_0_intr_status_sec_subp0_pending_f() |
+				fbpa_0_intr_status_ded_subp0_pending_f();
+	u32 ecc_subp1_mask = fbpa_0_intr_status_sec_subp1_pending_f() |
+				fbpa_0_intr_status_ded_subp1_pending_f();
+
+	offset = nvgpu_get_litter_value(g, GPU_LIT_FBPA_STRIDE) * fbpa_id;
+
+	status = nvgpu_readl(g, offset + fbpa_0_intr_status_r());
+	if ((status & (ecc_subp0_mask | ecc_subp1_mask)) == 0U) {
+		nvgpu_err(g, "Unknown interrupt fbpa %u status %08x",
+				fbpa_id, status);
+		nvgpu_err(g, "Suspected unrecoverable EDC interrupt;"
+				" HW no longer reliable");
+		nvgpu_sw_quiesce(g);
+		return;
+	}
+
+	if ((status & ecc_subp0_mask) != 0U) {
+		tu104_fbpa_handle_ecc_intr(g, fbpa_id, 0u);
+	}
+	if ((status & ecc_subp1_mask) != 0U) {
+		tu104_fbpa_handle_ecc_intr(g, fbpa_id, 1u);
+	}
+}
+
+int nvgpu_ecc_counter_init_per_fbpa(struct gk20a *g,
+		struct nvgpu_ecc_stat **stat, const char *name)
+{
+	u32 i;
+	u32 num_fbpa = nvgpu_get_litter_value(g, GPU_LIT_NUM_FBPAS);
+	struct nvgpu_ecc_stat *stats;
+	char fbpa_str[10] = {0};
+
+	stats = nvgpu_kzalloc(g, nvgpu_safe_mult_u64(sizeof(*stats),
+						     (size_t)num_fbpa));
+	if (stats == NULL) {
+		return -ENOMEM;
+	}
+
+	for (i = 0; i < num_fbpa; i++) {
+		/**
+		 * Store stats name as below:
+		 * fbpa<fbpa_value>_<name_string>
+		 */
+		(void)strcpy(stats[i].name, "fbpa");
+		(void)nvgpu_strnadd_u32(fbpa_str, i, sizeof(fbpa_str), 10U);
+		(void)strncat(stats[i].name, fbpa_str,
+					NVGPU_ECC_STAT_NAME_MAX_SIZE -
+					strlen(stats[i].name));
+		(void)strncat(stats[i].name, "_",
+					NVGPU_ECC_STAT_NAME_MAX_SIZE -
+					strlen(stats[i].name));
+		(void)strncat(stats[i].name, name,
+					NVGPU_ECC_STAT_NAME_MAX_SIZE -
+					strlen(stats[i].name));
+
+		nvgpu_ecc_stat_add(g, &stats[i]);
+	}
+
+	*stat = stats;
+	return 0;
+}
+
+static void free_fbpa_ecc_stat_count_array(struct gk20a *g,
+				      struct nvgpu_ecc_stat **stats_p)
+{
+	u32 num_fbpa = nvgpu_get_litter_value(g, GPU_LIT_NUM_FBPAS);
+	struct nvgpu_ecc_stat *stats;
+	u32 i;
+
+	if (*stats_p != NULL) {
+		stats = *stats_p;
+
+		for (i = 0; i < num_fbpa; i++) {
+			nvgpu_ecc_stat_del(g, &stats[i]);
+		}
+
+		nvgpu_kfree(g, stats);
+		*stats_p = NULL;
+	}
+}
+
+int tu104_fbpa_ecc_init(struct gk20a *g)
+{
+	int err;
+
+	err = NVGPU_ECC_COUNTER_INIT_PER_FBPA(fbpa_ecc_sec_err_count);
+	if (err != 0) {
+		goto done;
+	}
+	err = NVGPU_ECC_COUNTER_INIT_PER_FBPA(fbpa_ecc_ded_err_count);
+	if (err != 0) {
+		goto done;
+	}
+
+done:
+	if (err != 0) {
+		nvgpu_err(g, "ecc counter allocate failed, err=%d", err);
+		tu104_fbpa_ecc_free(g);
+	}
+
+	return err;
+}
+
+void tu104_fbpa_ecc_free(struct gk20a *g)
+{
+	struct nvgpu_ecc *ecc = &g->ecc;
+
+	free_fbpa_ecc_stat_count_array(g, &ecc->fbpa.fbpa_ecc_sec_err_count);
+	free_fbpa_ecc_stat_count_array(g, &ecc->fbpa.fbpa_ecc_ded_err_count);
+}

drivers/gpu/nvgpu/hal/fbpa/fbpa_tu104.h

@@ -0,0 +1,55 @@
+/*
+ * TU104 FBPA
+ *
+ * Copyright (c) 2018-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef NVGPU_FBPA_TU104_H
+#define NVGPU_FBPA_TU104_H
+
+struct gk20a;
+
+int tu104_fbpa_init(struct gk20a *g);
+void tu104_fbpa_handle_intr(struct gk20a *g, u32 fbpa_id);
+
+/**
+ * @brief Allocate and initialize error counters for all fbpa instances.
+ *
+ * @param g [in] The GPU driver struct.
+ * @param stat [out] Pointer to array of tpc error counters.
+ * @param name [in] Unique name for error counter.
+ *
+ * Calculates the total number of fbpa instances, allocates memory for each
+ * instance of error counter, initializes the counter with 0 and the specified
+ * string identifier. Finally the counter is added to the stats_list of
+ * struct nvgpu_ecc.
+ *
+ * @return 0 in case of success, less than 0 for failure.
+ */
+int nvgpu_ecc_counter_init_per_fbpa(struct gk20a *g,
+		struct nvgpu_ecc_stat **stat, const char *name);
+#define NVGPU_ECC_COUNTER_INIT_PER_FBPA(stat) \
+	nvgpu_ecc_counter_init_per_fbpa(g, &g->ecc.fbpa.stat, #stat)
+
+int tu104_fbpa_ecc_init(struct gk20a *g);
+void tu104_fbpa_ecc_free(struct gk20a *g);
+
+#endif /* NVGPU_FBPA_TU104_H */

drivers/gpu/nvgpu/hal/fifo/channel_gk20a.c

@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2011-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/channel.h>
+#include <nvgpu/log.h>
+#include <nvgpu/atomic.h>
+#include <nvgpu/io.h>
+#include <nvgpu/barrier.h>
+#include <nvgpu/bug.h>
+#include <nvgpu/gk20a.h>
+
+#include "channel_gk20a.h"
+
+#include <nvgpu/hw/gk20a/hw_ccsr_gk20a.h>
+
+void gk20a_channel_unbind(struct nvgpu_channel *ch)
+{
+	struct gk20a *g = ch->g;
+
+	nvgpu_log_fn(g, " ");
+
+	if (nvgpu_atomic_cmpxchg(&ch->bound, 1, 0) != 0) {
+		gk20a_writel(g, ccsr_channel_inst_r(ch->chid),
+			ccsr_channel_inst_ptr_f(0) |
+			ccsr_channel_inst_bind_false_f());
+	}
+}

drivers/gpu/nvgpu/hal/fifo/channel_gk20a.h

@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2011-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef FIFO_CHANNEL_GK20A_H
+#define FIFO_CHANNEL_GK20A_H
+
+struct gk20a;
+struct nvgpu_channel;
+struct nvgpu_channel_hw_state;
+struct nvgpu_debug_context;
+struct nvgpu_channel_dump_info;
+
+void gk20a_channel_enable(struct nvgpu_channel *ch);
+void gk20a_channel_disable(struct nvgpu_channel *ch);
+void gk20a_channel_read_state(struct gk20a *g, struct nvgpu_channel *ch,
+		struct nvgpu_channel_hw_state *state);
+
+#ifdef CONFIG_NVGPU_HAL_NON_FUSA
+void gk20a_channel_debug_dump(struct gk20a *g,
+	     struct nvgpu_debug_context *o,
+	     struct nvgpu_channel_dump_info *info);
+void gk20a_channel_unbind(struct nvgpu_channel *ch);
+#endif
+
+#endif /* FIFO_CHANNEL_GK20A_H */

drivers/gpu/nvgpu/hal/fifo/channel_gk20a_fusa.c

@@ -0,0 +1,101 @@
+/*
+ * Copyright (c) 2011-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/channel.h>
+#include <nvgpu/log.h>
+#include <nvgpu/atomic.h>
+#include <nvgpu/io.h>
+#include <nvgpu/barrier.h>
+#include <nvgpu/bug.h>
+#include <nvgpu/gk20a.h>
+
+#include "hal/fifo/pbdma_gm20b.h"
+
+#include "channel_gk20a.h"
+
+#include <nvgpu/hw/gk20a/hw_ccsr_gk20a.h>
+
+void gk20a_channel_enable(struct nvgpu_channel *ch)
+{
+	nvgpu_writel(ch->g, ccsr_channel_r(ch->chid),
+		gk20a_readl(ch->g, ccsr_channel_r(ch->chid)) |
+		ccsr_channel_enable_set_true_f());
+}
+
+void gk20a_channel_disable(struct nvgpu_channel *ch)
+{
+	nvgpu_writel(ch->g, ccsr_channel_r(ch->chid),
+		gk20a_readl(ch->g,
+			ccsr_channel_r(ch->chid)) |
+			ccsr_channel_enable_clr_true_f());
+}
+
+/* ccsr_channel_status_v is four bits long */
+static const char * const ccsr_chan_status_str[] = {
+	"idle",
+	"pending",
+	"pending_ctx_reload",
+	"pending_acquire",
+	"pending_acq_ctx_reload",
+	"on_pbdma",
+	"on_pbdma_and_eng",
+	"on_eng",
+	"on_eng_pending_acquire",
+	"on_eng_pending",
+	"on_pbdma_ctx_reload",
+	"on_pbdma_and_eng_ctx_reload",
+	"on_eng_ctx_reload",
+	"on_eng_pending_ctx_reload",
+	"on_eng_pending_acq_ctx_reload",
+	"N/A",
+};
+
+void gk20a_channel_read_state(struct gk20a *g, struct nvgpu_channel *ch,
+		struct nvgpu_channel_hw_state *state)
+{
+	u32 reg = nvgpu_readl(g, ccsr_channel_r(ch->chid));
+	u32 status_v = ccsr_channel_status_v(reg);
+
+	state->next = ccsr_channel_next_v(reg) == ccsr_channel_next_true_v();
+	state->enabled = ccsr_channel_enable_v(reg) ==
+			    ccsr_channel_enable_in_use_v();
+	state->ctx_reload =
+		(status_v ==
+			ccsr_channel_status_pending_ctx_reload_v()) ||
+		(status_v ==
+			ccsr_channel_status_pending_acq_ctx_reload_v()) ||
+		(status_v ==
+			ccsr_channel_status_on_pbdma_ctx_reload_v()) ||
+		(status_v ==
+			ccsr_channel_status_on_pbdma_and_eng_ctx_reload_v()) ||
+		(status_v ==
+			ccsr_channel_status_on_eng_ctx_reload_v()) ||
+		(status_v ==
+			ccsr_channel_status_on_eng_pending_ctx_reload_v()) ||
+		(status_v ==
+			ccsr_channel_status_on_eng_pending_acq_ctx_reload_v());
+	state->busy = ccsr_channel_busy_v(reg) == ccsr_channel_busy_true_v();
+	state->pending_acquire =
+		(status_v == ccsr_channel_status_pending_acquire_v()) ||
+		(status_v == ccsr_channel_status_on_eng_pending_acquire_v());
+	state->status_string = ccsr_chan_status_str[status_v];
+}

drivers/gpu/nvgpu/hal/fifo/channel_gm20b.c

@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2014-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/channel.h>
+#include <nvgpu/log.h>
+#include <nvgpu/atomic.h>
+#include <nvgpu/io.h>
+#include <nvgpu/barrier.h>
+#include <nvgpu/mm.h>
+#include <nvgpu/gk20a.h>
+
+#include "channel_gm20b.h"
+
+#include <nvgpu/hw/gm20b/hw_ccsr_gm20b.h>
+#include <nvgpu/hw/gm20b/hw_ram_gm20b.h>
+
+u32 gm20b_channel_count(struct gk20a *g)
+{
+	return ccsr_channel__size_1_v();
+}

drivers/gpu/nvgpu/hal/fifo/channel_gm20b.h

@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2014-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef FIFO_CHANNEL_GM20B_H
+#define FIFO_CHANNEL_GM20B_H
+
+#include <nvgpu/types.h>
+
+struct nvgpu_channel;
+struct gk20a;
+
+void gm20b_channel_bind(struct nvgpu_channel *c);
+void gm20b_channel_force_ctx_reload(struct nvgpu_channel *ch);
+
+#ifdef CONFIG_NVGPU_HAL_NON_FUSA
+u32 gm20b_channel_count(struct gk20a *g);
+#endif
+
+#endif /* FIFO_CHANNEL_GM20B_H */

drivers/gpu/nvgpu/hal/fifo/channel_gm20b_fusa.c

@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2014-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/channel.h>
+#include <nvgpu/log.h>
+#include <nvgpu/atomic.h>
+#include <nvgpu/io.h>
+#include <nvgpu/barrier.h>
+#include <nvgpu/mm.h>
+#include <nvgpu/gk20a.h>
+
+#include "channel_gm20b.h"
+
+#include <nvgpu/hw/gm20b/hw_ccsr_gm20b.h>
+#include <nvgpu/hw/gm20b/hw_ram_gm20b.h>
+
+void gm20b_channel_bind(struct nvgpu_channel *c)
+{
+	struct gk20a *g = c->g;
+
+	u32 inst_ptr = nvgpu_inst_block_ptr(g, &c->inst_block);
+
+	nvgpu_log_info(g, "bind channel %d inst ptr 0x%08x",
+		c->chid, inst_ptr);
+
+	nvgpu_writel(g, ccsr_channel_inst_r(c->chid),
+		     ccsr_channel_inst_ptr_f(inst_ptr) |
+		     nvgpu_aperture_mask(g, &c->inst_block,
+				ccsr_channel_inst_target_sys_mem_ncoh_f(),
+				ccsr_channel_inst_target_sys_mem_coh_f(),
+				ccsr_channel_inst_target_vid_mem_f()) |
+		     ccsr_channel_inst_bind_true_f());
+
+	nvgpu_writel(g, ccsr_channel_r(c->chid),
+		(nvgpu_readl(g, ccsr_channel_r(c->chid)) &
+		 ~ccsr_channel_enable_set_f(~U32(0U))) |
+		 ccsr_channel_enable_set_true_f());
+
+	nvgpu_atomic_set(&c->bound, 1);
+}
+
+void gm20b_channel_force_ctx_reload(struct nvgpu_channel *ch)
+{
+	struct gk20a *g = ch->g;
+	u32 reg = nvgpu_readl(g, ccsr_channel_r(ch->chid));
+
+	nvgpu_writel(g, ccsr_channel_r(ch->chid),
+		reg | ccsr_channel_force_ctx_reload_true_f());
+}

drivers/gpu/nvgpu/hal/fifo/channel_gv100.c

@@ -0,0 +1,30 @@
+/*
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include "channel_gv100.h"
+
+#include <nvgpu/hw/gv100/hw_ccsr_gv100.h>
+
+u32 gv100_channel_count(struct gk20a *g)
+{
+	return ccsr_channel__size_1_v();
+}

drivers/gpu/nvgpu/hal/fifo/channel_gv100.h

@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef FIFO_CHANNEL_GV100_H
+#define FIFO_CHANNEL_GV100_H
+
+#include <nvgpu/types.h>
+
+struct gk20a;
+
+u32 gv100_channel_count(struct gk20a *g);
+
+#endif /* FIFO_CHANNEL_GV100_H */

drivers/gpu/nvgpu/hal/fifo/channel_gv11b.h

@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2016-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef FIFO_CHANNEL_GV11B_H
+#define FIFO_CHANNEL_GV11B_H
+
+#include <nvgpu/types.h>
+
+struct gk20a;
+struct nvgpu_channel;
+struct nvgpu_channel_hw_state;
+struct nvgpu_debug_context;
+
+void gv11b_channel_unbind(struct nvgpu_channel *ch);
+u32 gv11b_channel_count(struct gk20a *g);
+void gv11b_channel_read_state(struct gk20a *g, struct nvgpu_channel *ch,
+		struct nvgpu_channel_hw_state *state);
+void gv11b_channel_reset_faulted(struct gk20a *g, struct nvgpu_channel *ch,
+		bool eng, bool pbdma);
+#endif /* FIFO_CHANNEL_GV11B_H */

drivers/gpu/nvgpu/hal/fifo/channel_gv11b_fusa.c

@@ -0,0 +1,82 @@
+/*
+ * Copyright (c) 2016-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+
+#include <nvgpu/channel.h>
+#include <nvgpu/log.h>
+#include <nvgpu/atomic.h>
+#include <nvgpu/io.h>
+#include <nvgpu/gk20a.h>
+
+#include "channel_gk20a.h"
+#include "channel_gv11b.h"
+
+#include <nvgpu/hw/gv11b/hw_ccsr_gv11b.h>
+
+void gv11b_channel_unbind(struct nvgpu_channel *ch)
+{
+	struct gk20a *g = ch->g;
+
+	nvgpu_log_fn(g, " ");
+
+	if (nvgpu_atomic_cmpxchg(&ch->bound, 1, 0) != 0) {
+		nvgpu_writel(g, ccsr_channel_inst_r(ch->chid),
+			ccsr_channel_inst_ptr_f(0U) |
+			ccsr_channel_inst_bind_false_f());
+
+		nvgpu_writel(g, ccsr_channel_r(ch->chid),
+			ccsr_channel_enable_clr_true_f() |
+			ccsr_channel_pbdma_faulted_reset_f() |
+			ccsr_channel_eng_faulted_reset_f());
+	}
+}
+
+u32 gv11b_channel_count(struct gk20a *g)
+{
+	return ccsr_channel__size_1_v();
+}
+
+void gv11b_channel_read_state(struct gk20a *g, struct nvgpu_channel *ch,
+		struct nvgpu_channel_hw_state *state)
+{
+	u32 reg = nvgpu_readl(g, ccsr_channel_r(ch->chid));
+
+	gk20a_channel_read_state(g, ch, state);
+
+	state->eng_faulted = ccsr_channel_eng_faulted_v(reg) ==
+		ccsr_channel_eng_faulted_true_v();
+}
+
+void gv11b_channel_reset_faulted(struct gk20a *g, struct nvgpu_channel *ch,
+		bool eng, bool pbdma)
+{
+	u32 reg = nvgpu_readl(g, ccsr_channel_r(ch->chid));
+
+	if (eng) {
+		reg |= ccsr_channel_eng_faulted_reset_f();
+	}
+	if (pbdma) {
+		reg |= ccsr_channel_pbdma_faulted_reset_f();
+	}
+
+	nvgpu_writel(g, ccsr_channel_r(ch->chid), reg);
+}

drivers/gpu/nvgpu/hal/fifo/ctxsw_timeout_gk20a.c

@@ -0,0 +1,138 @@
+/*
+ * Copyright (c) 2011-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/gk20a.h>
+#include <nvgpu/log.h>
+#include <nvgpu/io.h>
+#include <nvgpu/soc.h>
+#include <nvgpu/ptimer.h>
+#include <nvgpu/channel.h>
+#include <nvgpu/rc.h>
+#include <nvgpu/engines.h>
+
+#include <hal/fifo/ctxsw_timeout_gk20a.h>
+
+#include <nvgpu/hw/gk20a/hw_fifo_gk20a.h>
+
+void gk20a_fifo_ctxsw_timeout_enable(struct gk20a *g, bool enable)
+{
+	u32 timeout;
+
+	if (enable) {
+		timeout = g->ctxsw_timeout_period_ms * 1000U; /* in us */
+		timeout = scale_ptimer(timeout,
+			ptimer_scalingfactor10x(g->ptimer_src_freq));
+		timeout |= fifo_eng_timeout_detection_enabled_f();
+		nvgpu_writel(g, fifo_eng_timeout_r(), timeout);
+	} else {
+		timeout = nvgpu_readl(g, fifo_eng_timeout_r());
+		timeout &= ~(fifo_eng_timeout_detection_enabled_f());
+		nvgpu_writel(g, fifo_eng_timeout_r(), timeout);
+	}
+}
+
+bool gk20a_fifo_handle_ctxsw_timeout(struct gk20a *g)
+{
+	u32 sched_error;
+	u32 engine_id;
+	u32 id = U32_MAX;
+	bool is_tsg = false;
+	bool recover = false;
+	struct nvgpu_channel *ch = NULL;
+	struct nvgpu_tsg *tsg = NULL;
+	struct nvgpu_fifo *f = &g->fifo;
+	u32 ms = 0;
+	bool debug_dump = false;
+
+	/* read the scheduler error register */
+	sched_error = nvgpu_readl(g, fifo_intr_sched_error_r());
+
+	engine_id = nvgpu_engine_find_busy_doing_ctxsw(g, &id, &is_tsg);
+	/*
+	 * Could not find the engine
+	 * Possible Causes:
+	 * a)
+	 * On hitting engine reset, h/w drops the ctxsw_status to INVALID in
+	 * fifo_engine_status register. Also while the engine is held in reset
+	 * h/w passes busy/idle straight through. fifo_engine_status registers
+	 * are correct in that there is no context switch outstanding
+	 * as the CTXSW is aborted when reset is asserted.
+	 * This is just a side effect of how gv100 and earlier versions of
+	 * ctxsw_timeout behave.
+	 * With gv11b and later, h/w snaps the context at the point of error
+	 * so that s/w can see the tsg_id which caused the HW timeout.
+	 * b)
+	 * If engines are not busy and ctxsw state is valid then intr occurred
+	 * in the past and if the ctxsw state has moved on to VALID from LOAD
+	 * or SAVE, it means that whatever timed out eventually finished
+	 * anyways. The problem with this is that s/w cannot conclude which
+	 * context caused the problem as maybe more switches occurred before
+	 * intr is handled.
+	 */
+	if (engine_id == NVGPU_INVALID_ENG_ID) {
+		nvgpu_info(g, "fifo ctxsw timeout: 0x%08x, failed to find engine "
+				"that is busy doing ctxsw. "
+				"May be ctxsw already happened", sched_error);
+		return false;
+	}
+
+	if (!nvgpu_engine_check_valid_id(g, engine_id)) {
+		nvgpu_err(g, "fifo ctxsw timeout: 0x%08x, engine_id %u not valid",
+			sched_error, engine_id);
+		return false;
+	}
+
+	if (id > f->num_channels) {
+		nvgpu_err(g, "fifo ctxsw timeout error: id is invalid %u", id);
+		return false;
+	}
+
+	if (is_tsg) {
+		tsg = nvgpu_tsg_check_and_get_from_id(g, id);
+	} else {
+		ch = nvgpu_channel_from_id(g, id);
+		if (ch != NULL) {
+			tsg = nvgpu_tsg_from_ch(ch);
+			nvgpu_channel_put(ch);
+		}
+	}
+#ifdef CONFIG_NVGPU_KERNEL_MODE_SUBMIT
+	if (tsg != NULL) {
+		recover = g->ops.tsg.check_ctxsw_timeout(tsg, &debug_dump, &ms);
+		if (recover) {
+			nvgpu_err(g,
+				"fifo ctxsw timeout error: "
+				"engine=%u, %s=%d, ms=%u",
+				engine_id, is_tsg ? "tsg" : "ch", id, ms);
+
+			nvgpu_rc_ctxsw_timeout(g, BIT32(engine_id), tsg, debug_dump);
+			return recover;
+		}
+	}
+#endif
+
+	nvgpu_log_info(g,
+		"fifo is waiting for ctxsw switch for %d ms, "
+			"%s=%d", ms, is_tsg ? "tsg" : "ch", id);
+
+	return recover;
+}

drivers/gpu/nvgpu/hal/fifo/ctxsw_timeout_gk20a.h

@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2011-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef NVGPU_FIFO_CTXSW_TIMEOUT_GK20A_H
+#define NVGPU_FIFO_CTXSW_TIMEOUT_GK20A_H
+
+#include <nvgpu/types.h>
+
+struct gk20a;
+
+void gk20a_fifo_ctxsw_timeout_enable(struct gk20a *g, bool enable);
+bool gk20a_fifo_handle_ctxsw_timeout(struct gk20a *g);
+
+#endif /* NVGPU_FIFO_CTXSW_TIMEOUT_GK20A_H */

drivers/gpu/nvgpu/hal/fifo/ctxsw_timeout_gv11b.h

@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2016-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef NVGPU_CTXSW_TIMEOUT_GV11B_H
+#define NVGPU_CTXSW_TIMEOUT_GV11B_H
+
+#include <nvgpu/types.h>
+
+struct gk20a;
+
+void gv11b_fifo_ctxsw_timeout_enable(struct gk20a *g, bool enable);
+bool gv11b_fifo_handle_ctxsw_timeout(struct gk20a *g);
+
+#endif /* NVGPU_CTXSW_TIMEOUT_GV11B_H */

drivers/gpu/nvgpu/hal/fifo/ctxsw_timeout_gv11b_fusa.c

@@ -0,0 +1,253 @@
+/*
+ * Copyright (c) 2016-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/gk20a.h>
+#include <nvgpu/log.h>
+#include <nvgpu/io.h>
+#include <nvgpu/soc.h>
+#include <nvgpu/ptimer.h>
+#include <nvgpu/tsg.h>
+#include <nvgpu/rc.h>
+#include <nvgpu/nvgpu_err.h>
+#include <nvgpu/device.h>
+
+#include <hal/fifo/ctxsw_timeout_gv11b.h>
+
+#include <nvgpu/hw/gv11b/hw_fifo_gv11b.h>
+
+void gv11b_fifo_ctxsw_timeout_enable(struct gk20a *g, bool enable)
+{
+	u32 timeout;
+
+	if (enable) {
+		/* clear ctxsw timeout interrupts */
+		nvgpu_writel(g, fifo_intr_ctxsw_timeout_r(), ~U32(0U));
+
+		if (nvgpu_platform_is_silicon(g)) {
+			timeout = g->ctxsw_timeout_period_ms * 1000U;
+			timeout = scale_ptimer(timeout,
+				ptimer_scalingfactor10x(g->ptimer_src_freq));
+			timeout |= fifo_eng_ctxsw_timeout_detection_enabled_f();
+			nvgpu_writel(g, fifo_eng_ctxsw_timeout_r(), timeout);
+		} else {
+			timeout = nvgpu_readl(g, fifo_eng_ctxsw_timeout_r());
+			nvgpu_log_info(g,
+				"fifo_eng_ctxsw_timeout reg val = 0x%08x",
+				 timeout);
+			timeout = set_field(timeout,
+					fifo_eng_ctxsw_timeout_period_m(),
+					fifo_eng_ctxsw_timeout_period_max_f());
+			timeout = set_field(timeout,
+					fifo_eng_ctxsw_timeout_detection_m(),
+					fifo_eng_ctxsw_timeout_detection_disabled_f());
+			nvgpu_log_info(g,
+				"new fifo_eng_ctxsw_timeout reg val = 0x%08x",
+				 timeout);
+			nvgpu_writel(g, fifo_eng_ctxsw_timeout_r(), timeout);
+		}
+
+	} else {
+		timeout = nvgpu_readl(g, fifo_eng_ctxsw_timeout_r());
+		timeout = set_field(timeout,
+				fifo_eng_ctxsw_timeout_detection_m(),
+				fifo_eng_ctxsw_timeout_detection_disabled_f());
+		nvgpu_writel(g, fifo_eng_ctxsw_timeout_r(), timeout);
+		timeout = nvgpu_readl(g, fifo_eng_ctxsw_timeout_r());
+		nvgpu_log_info(g,
+			"fifo_eng_ctxsw_timeout disabled val = 0x%08x",
+			timeout);
+		/* clear ctxsw timeout interrupts */
+		nvgpu_writel(g, fifo_intr_ctxsw_timeout_r(), ~U32(0U));
+	}
+}
+
+static u32 gv11b_fifo_ctxsw_timeout_info(struct gk20a *g, u32 active_eng_id,
+						u32 *info_status)
+{
+	u32 tsgid = NVGPU_INVALID_TSG_ID;
+	u32 timeout_info;
+	u32 ctx_status;
+
+	timeout_info = nvgpu_readl(g,
+			 fifo_intr_ctxsw_timeout_info_r(active_eng_id));
+
+	/*
+	 * ctxsw_state and tsgid are snapped at the point of the timeout and
+	 * will not change while the corresponding INTR_CTXSW_TIMEOUT_ENGINE bit
+	 * is PENDING.
+	 */
+	ctx_status = fifo_intr_ctxsw_timeout_info_ctxsw_state_v(timeout_info);
+	if (ctx_status ==
+		fifo_intr_ctxsw_timeout_info_ctxsw_state_load_v()) {
+
+		tsgid = fifo_intr_ctxsw_timeout_info_next_tsgid_v(timeout_info);
+
+	} else if ((ctx_status ==
+			fifo_intr_ctxsw_timeout_info_ctxsw_state_switch_v()) ||
+		(ctx_status ==
+			fifo_intr_ctxsw_timeout_info_ctxsw_state_save_v())) {
+
+		tsgid = fifo_intr_ctxsw_timeout_info_prev_tsgid_v(timeout_info);
+	} else {
+		nvgpu_log_info(g, "ctxsw_timeout_info_ctxsw_state: 0x%08x",
+			ctx_status);
+	}
+	nvgpu_log_info(g, "ctxsw timeout info: tsgid = %d", tsgid);
+
+	/*
+	 * STATUS indicates whether the context request ack was eventually
+	 * received and whether a subsequent request timed out.  This field is
+	 * updated live while the corresponding INTR_CTXSW_TIMEOUT_ENGINE bit
+	 * is PENDING. STATUS starts in AWAITING_ACK, and progresses to
+	 * ACK_RECEIVED and finally ends with DROPPED_TIMEOUT.
+	 *
+	 * AWAITING_ACK - context request ack still not returned from engine.
+	 * ENG_WAS_RESET - The engine was reset via a PRI write to NV_PMC_ENABLE
+	 * or NV_PMC_ELPG_ENABLE prior to receiving the ack.  Host will not
+	 * expect ctx ack to return, but if it is already in flight, STATUS will
+	 * transition shortly to ACK_RECEIVED unless the interrupt is cleared
+	 * first.  Once the engine is reset, additional context switches can
+	 * occur; if one times out, STATUS will transition to DROPPED_TIMEOUT
+	 * if the interrupt isn't cleared first.
+	 * ACK_RECEIVED - The ack for the timed-out context request was
+	 * received between the point of the timeout and this register being
+	 * read.  Note this STATUS can be reported during the load stage of the
+	 * same context switch that timed out if the timeout occurred during the
+	 * save half of a context switch.  Additional context requests may have
+	 * completed or may be outstanding, but no further context timeout has
+	 * occurred.  This simplifies checking for spurious context switch
+	 * timeouts.
+	 * DROPPED_TIMEOUT - The originally timed-out context request acked,
+	 * but a subsequent context request then timed out.
+	 * Information about the subsequent timeout is not stored; in fact, that
+	 * context request may also have already been acked by the time SW
+	 * SW reads this register.  If not, there is a chance SW can get the
+	 * dropped information by clearing the corresponding
+	 * INTR_CTXSW_TIMEOUT_ENGINE bit and waiting for the timeout to occur
+	 * again. Note, however, that if the engine does time out again,
+	 * it may not be from the  original request that caused the
+	 * DROPPED_TIMEOUT state, as that request may
+	 * be acked in the interim.
+	 */
+	*info_status = fifo_intr_ctxsw_timeout_info_status_v(timeout_info);
+	if (*info_status ==
+		 fifo_intr_ctxsw_timeout_info_status_ack_received_v()) {
+
+		nvgpu_log_info(g, "ctxsw timeout info: ack received");
+		/* no need to recover */
+		tsgid = NVGPU_INVALID_TSG_ID;
+
+	} else if (*info_status ==
+		fifo_intr_ctxsw_timeout_info_status_dropped_timeout_v()) {
+
+		nvgpu_log_info(g, "ctxsw timeout info: dropped timeout");
+		/* no need to recover */
+		tsgid = NVGPU_INVALID_TSG_ID;
+
+	} else {
+		nvgpu_log_info(g, "ctxsw timeout info status: 0x%08x",
+			*info_status);
+	}
+
+	return tsgid;
+}
+
+bool gv11b_fifo_handle_ctxsw_timeout(struct gk20a *g)
+{
+	bool recover = false;
+	u32 tsgid = NVGPU_INVALID_TSG_ID;
+	u32 i;
+	u32 timeout_val, ctxsw_timeout_engines;
+	u32 info_status;
+	struct nvgpu_tsg *tsg = NULL;
+
+	/* get ctxsw timedout engines */
+	ctxsw_timeout_engines = nvgpu_readl(g, fifo_intr_ctxsw_timeout_r());
+	if (ctxsw_timeout_engines == 0U) {
+		nvgpu_err(g, "no eng ctxsw timeout pending");
+		return false;
+	}
+
+	timeout_val = nvgpu_readl(g, fifo_eng_ctxsw_timeout_r());
+	timeout_val = fifo_eng_ctxsw_timeout_period_v(timeout_val);
+
+	nvgpu_log_info(g, "eng ctxsw timeout period = 0x%x", timeout_val);
+
+	for (i = 0; i < g->fifo.num_engines; i++) {
+		const struct nvgpu_device *dev = g->fifo.active_engines[i];
+
+		if ((ctxsw_timeout_engines &
+			fifo_intr_ctxsw_timeout_engine_pending_f(
+				dev->engine_id)) != 0U) {
+			u32 ms = 0;
+#ifdef CONFIG_NVGPU_KERNEL_MODE_SUBMIT
+			bool debug_dump = false;
+			const char *const ctxsw_timeout_status_desc[] = {
+				"awaiting ack",
+				"eng was reset",
+				"ack received",
+				"dropped timeout"
+			};
+#endif
+			tsgid = gv11b_fifo_ctxsw_timeout_info(g, dev->engine_id,
+						&info_status);
+			tsg = nvgpu_tsg_check_and_get_from_id(g, tsgid);
+			if (tsg == NULL) {
+				continue;
+			}
+
+			nvgpu_report_host_err(g, NVGPU_ERR_MODULE_HOST,
+					0, GPU_HOST_PFIFO_CTXSW_TIMEOUT_ERROR,
+					tsgid);
+
+#ifdef CONFIG_NVGPU_KERNEL_MODE_SUBMIT
+			recover = g->ops.tsg.check_ctxsw_timeout(tsg,
+					&debug_dump, &ms);
+			if (recover) {
+				const char *info_status_str = "invalid";
+				if (info_status <
+					ARRAY_SIZE(ctxsw_timeout_status_desc)) {
+					info_status_str =
+					ctxsw_timeout_status_desc[info_status];
+				}
+
+				nvgpu_err(g,
+					  "ctxsw timeout error: engine_id=%u"
+					  "%s=%d, info: %s ms=%u",
+					  dev->engine_id, "tsg", tsgid,
+					  info_status_str, ms);
+
+				nvgpu_rc_ctxsw_timeout(g, BIT32(dev->engine_id),
+					tsg, debug_dump);
+				continue;
+			}
+#endif
+			nvgpu_log_info(g,
+				"fifo is waiting for ctxsw switch: "
+				"for %d ms, %s=%d", ms, "tsg", tsgid);
+		}
+	}
+	/* clear interrupt */
+	nvgpu_writel(g, fifo_intr_ctxsw_timeout_r(), ctxsw_timeout_engines);
+
+	return recover;
+}

drivers/gpu/nvgpu/hal/fifo/engine_status_gm20b.c

@@ -0,0 +1,75 @@
+/*
+ * Copyright (c) 2019-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/io.h>
+#include <nvgpu/debug.h>
+#include <nvgpu/gk20a.h>
+#include <nvgpu/engine_status.h>
+#include <nvgpu/engines.h>
+#include <nvgpu/fifo.h>
+
+#include <nvgpu/hw/gm20b/hw_fifo_gm20b.h>
+
+#include "engine_status_gm20b.h"
+
+void gm20b_dump_engine_status(struct gk20a *g, struct nvgpu_debug_context *o)
+{
+	u32 i, host_num_engines;
+	struct nvgpu_engine_status_info engine_status;
+
+	host_num_engines = nvgpu_get_litter_value(g, GPU_LIT_HOST_NUM_ENGINES);
+
+	gk20a_debug_output(o, "Engine status - chip %-5s", g->name);
+	gk20a_debug_output(o, "--------------------------");
+
+	for (i = 0; i < host_num_engines; i++) {
+		if (!nvgpu_engine_check_valid_id(g, i)) {
+			/* Skip invalid engines */
+			continue;
+		}
+
+		g->ops.engine_status.read_engine_status_info(g, i, &engine_status);
+
+		gk20a_debug_output(o,
+			"Engine %d | "
+			"ID: %d - %-9s next_id: %d %-9s | status: %s",
+			i,
+			engine_status.ctx_id,
+			nvgpu_engine_status_is_ctx_type_tsg(
+				&engine_status) ?
+				"[tsg]" : "[channel]",
+			engine_status.ctx_next_id,
+			nvgpu_engine_status_is_next_ctx_type_tsg(
+				&engine_status) ?
+				"[tsg]" : "[channel]",
+			nvgpu_fifo_decode_pbdma_ch_eng_status(
+				engine_status.ctxsw_state));
+
+		if (engine_status.is_faulted) {
+			gk20a_debug_output(o, "  State: faulted");
+		}
+		if (engine_status.is_busy) {
+			gk20a_debug_output(o, "  State: busy");
+		}
+	}
+	gk20a_debug_output(o, " ");
+}

drivers/gpu/nvgpu/hal/fifo/engine_status_gm20b.h

@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef NVGPU_ENGINE_STATUS_GM20B_H
+#define NVGPU_ENGINE_STATUS_GM20B_H
+
+#include <nvgpu/types.h>
+
+struct gk20a;
+struct nvgpu_engine_status_info;
+struct nvgpu_debug_context;
+
+void gm20b_read_engine_status_info(struct gk20a *g, u32 engine_id,
+		struct nvgpu_engine_status_info *status);
+
+#ifdef CONFIG_NVGPU_HAL_NON_FUSA
+void gm20b_dump_engine_status(struct gk20a *g, struct nvgpu_debug_context *o);
+#endif
+
+#endif /* NVGPU_ENGINE_STATUS_GM20B_H */
+

drivers/gpu/nvgpu/hal/fifo/engine_status_gm20b_fusa.c

@@ -0,0 +1,174 @@
+/*
+ * Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/io.h>
+#include <nvgpu/debug.h>
+#include <nvgpu/gk20a.h>
+#include <nvgpu/engine_status.h>
+#include <nvgpu/engines.h>
+#include <nvgpu/fifo.h>
+
+#include <nvgpu/hw/gm20b/hw_fifo_gm20b.h>
+
+#include "engine_status_gm20b.h"
+
+static void populate_invalid_ctxsw_status_info(
+		struct nvgpu_engine_status_info *status_info)
+{
+	status_info->ctx_id = ENGINE_STATUS_CTX_ID_INVALID;
+	status_info->ctx_id_type = ENGINE_STATUS_CTX_NEXT_ID_TYPE_INVALID;
+	status_info->ctx_next_id =
+		ENGINE_STATUS_CTX_NEXT_ID_INVALID;
+	status_info->ctx_next_id_type = ENGINE_STATUS_CTX_NEXT_ID_TYPE_INVALID;
+	status_info->ctxsw_status = NVGPU_CTX_STATUS_INVALID;
+}
+
+static void populate_valid_ctxsw_status_info(
+		struct nvgpu_engine_status_info *status_info)
+{
+	bool id_type_tsg;
+	u32 engine_status = status_info->reg_data;
+
+	status_info->ctx_id =
+		fifo_engine_status_id_v(status_info->reg_data);
+	id_type_tsg = fifo_engine_status_id_type_v(engine_status) ==
+			fifo_engine_status_id_type_tsgid_v();
+	status_info->ctx_id_type =
+		id_type_tsg ? ENGINE_STATUS_CTX_ID_TYPE_TSGID :
+			ENGINE_STATUS_CTX_ID_TYPE_CHID;
+	status_info->ctx_next_id =
+		ENGINE_STATUS_CTX_NEXT_ID_INVALID;
+	status_info->ctx_next_id_type = ENGINE_STATUS_CTX_NEXT_ID_TYPE_INVALID;
+	status_info->ctxsw_status = NVGPU_CTX_STATUS_VALID;
+}
+
+static void populate_load_ctxsw_status_info(
+		struct nvgpu_engine_status_info *status_info)
+{
+	bool next_id_type_tsg;
+	u32 engine_status = status_info->reg_data;
+
+	status_info->ctx_id = ENGINE_STATUS_CTX_ID_INVALID;
+	status_info->ctx_id_type = ENGINE_STATUS_CTX_ID_TYPE_INVALID;
+	status_info->ctx_next_id =
+		fifo_engine_status_next_id_v(
+			status_info->reg_data);
+	next_id_type_tsg = fifo_engine_status_next_id_type_v(engine_status) ==
+			fifo_engine_status_next_id_type_tsgid_v();
+	status_info->ctx_next_id_type =
+		next_id_type_tsg ? ENGINE_STATUS_CTX_NEXT_ID_TYPE_TSGID :
+			ENGINE_STATUS_CTX_NEXT_ID_TYPE_CHID;
+	status_info->ctxsw_status = NVGPU_CTX_STATUS_CTXSW_LOAD;
+}
+
+static void populate_save_ctxsw_status_info(
+		struct nvgpu_engine_status_info *status_info)
+{
+	bool id_type_tsg;
+	u32 engine_status = status_info->reg_data;
+
+	status_info->ctx_id =
+		fifo_engine_status_id_v(status_info->reg_data);
+	id_type_tsg = fifo_engine_status_id_type_v(engine_status) ==
+			fifo_engine_status_id_type_tsgid_v();
+	status_info->ctx_id_type =
+		id_type_tsg ? ENGINE_STATUS_CTX_ID_TYPE_TSGID :
+			ENGINE_STATUS_CTX_ID_TYPE_CHID;
+	status_info->ctx_next_id =
+		ENGINE_STATUS_CTX_NEXT_ID_INVALID;
+	status_info->ctx_next_id_type = ENGINE_STATUS_CTX_NEXT_ID_TYPE_INVALID;
+	status_info->ctxsw_status = NVGPU_CTX_STATUS_CTXSW_SAVE;
+}
+
+static void populate_switch_ctxsw_status_info(
+		struct nvgpu_engine_status_info *status_info)
+{
+	bool id_type_tsg;
+	bool next_id_type_tsg;
+	u32 engine_status = status_info->reg_data;
+
+	status_info->ctx_id =
+		fifo_engine_status_id_v(status_info->reg_data);
+	id_type_tsg = fifo_engine_status_id_type_v(engine_status) ==
+			fifo_engine_status_id_type_tsgid_v();
+	status_info->ctx_id_type =
+		id_type_tsg ? ENGINE_STATUS_CTX_ID_TYPE_TSGID :
+			ENGINE_STATUS_CTX_ID_TYPE_CHID;
+	status_info->ctx_next_id =
+		fifo_engine_status_next_id_v(
+			status_info->reg_data);
+	next_id_type_tsg = fifo_engine_status_next_id_type_v(engine_status) ==
+			fifo_engine_status_next_id_type_tsgid_v();
+	status_info->ctx_next_id_type =
+		next_id_type_tsg ? ENGINE_STATUS_CTX_NEXT_ID_TYPE_TSGID :
+			ENGINE_STATUS_CTX_NEXT_ID_TYPE_CHID;
+	status_info->ctxsw_status = NVGPU_CTX_STATUS_CTXSW_SWITCH;
+}
+
+void gm20b_read_engine_status_info(struct gk20a *g, u32 engine_id,
+		struct nvgpu_engine_status_info *status)
+{
+	u32 engine_reg_data;
+	u32 ctxsw_state;
+
+	(void) memset(status, 0, sizeof(*status));
+
+	if (engine_id == NVGPU_INVALID_ENG_ID) {
+		/* just return NULL info */
+		return;
+	}
+	engine_reg_data = nvgpu_readl(g, fifo_engine_status_r(engine_id));
+
+	status->reg_data = engine_reg_data;
+
+	/* populate the engine_state enum */
+	status->is_busy = fifo_engine_status_engine_v(engine_reg_data) ==
+			fifo_engine_status_engine_busy_v();
+
+	/* populate the engine_faulted_state enum */
+	status->is_faulted = fifo_engine_status_faulted_v(engine_reg_data) ==
+			fifo_engine_status_faulted_true_v();
+
+	/* populate the ctxsw_in_progress_state */
+	status->ctxsw_in_progress = ((engine_reg_data &
+			fifo_engine_status_ctxsw_in_progress_f()) != 0U);
+
+	/* populate the ctxsw related info */
+	ctxsw_state = fifo_engine_status_ctx_status_v(engine_reg_data);
+
+	status->ctxsw_state = ctxsw_state;
+
+	if (ctxsw_state == fifo_engine_status_ctx_status_valid_v()) {
+		populate_valid_ctxsw_status_info(status);
+	} else if (ctxsw_state ==
+			fifo_engine_status_ctx_status_ctxsw_load_v()) {
+		populate_load_ctxsw_status_info(status);
+	} else if (ctxsw_state ==
+			fifo_engine_status_ctx_status_ctxsw_save_v()) {
+		populate_save_ctxsw_status_info(status);
+	} else if (ctxsw_state ==
+			fifo_engine_status_ctx_status_ctxsw_switch_v()) {
+		populate_switch_ctxsw_status_info(status);
+	} else {
+		populate_invalid_ctxsw_status_info(status);
+	}
+}