gpu: nvgpu: move pbdma HAL functions to hal/fifo/pbdma

The following HAL pointers are moved to a separate HAL unit named pbdma.

pbdma_acquire_val
get_pbdma_signature
dump_pbdma_status
handle_pbdma_intr_0
handle_pbdma_intr_1
read_pbdma_data
reset_pbdma_header

The functions corresponding to these HAL units are also moved to
pbdma_{arch} files under hal/fifo correspondinging to arch gm20b,
gp10b, gv11b and tu104. Any calls to gk20a_readl and gk20a_writel
are replaced by nvgpu_readl and nvgpu_writel respectively.

Jira NVGPU-2950

Change-Id: I9723f30ddf6582df02c03fceb1fba26a206e1230
Signed-off-by: Debarshi Dutta <ddutta@nvidia.com>
Reviewed-on: https://git-master.nvidia.com/r/2071782
Reviewed-by: mobile promotions <svcmobile_promotions@nvidia.com>
Tested-by: mobile promotions <svcmobile_promotions@nvidia.com>

drivers/gpu/nvgpu/Makefile

@@ -205,6 +205,10 @@ nvgpu-y += \
 	hal/fuse/fuse_gp106.o \
 	hal/fifo/engines_gm20b.o \
 	hal/fifo/engines_gv11b.o \
+	hal/fifo/pbdma_gm20b.o \
+	hal/fifo/pbdma_gp10b.o \
+	hal/fifo/pbdma_gv11b.o \
+	hal/fifo/pbdma_tu104.o \
 	hal/fifo/engine_status_gm20b.o \
 	hal/fifo/engine_status_gv100.o \
 	hal/fifo/pbdma_status_gm20b.o \

drivers/gpu/nvgpu/Makefile.sources

@@ -358,6 +358,10 @@ srcs += common/sim.c \
 	hal/fuse/fuse_gp106.c \
 	hal/fifo/engines_gm20b.c \
 	hal/fifo/engines_gv11b.c \
+	hal/fifo/pbdma_gm20b.c \
+	hal/fifo/pbdma_gp10b.c \
+	hal/fifo/pbdma_gv11b.c \
+	hal/fifo/pbdma_tu104.c \
 	hal/fifo/engine_status_gm20b.c \
 	hal/fifo/engine_status_gv100.c \
 	hal/fifo/pbdma_status_gm20b.c \

drivers/gpu/nvgpu/common/vgpu/gp10b/vgpu_hal_gp10b.c

@@ -23,6 +23,8 @@
 #include "hal/bus/bus_gk20a.h"
 #include "hal/bus/bus_gm20b.h"
 #include "hal/fifo/engines_gm20b.h"
+#include "hal/fifo/pbdma_gm20b.h"
+#include "hal/fifo/pbdma_gp10b.h"
 #include "hal/therm/therm_gm20b.h"
 #include "hal/therm/therm_gp10b.h"
 #include "hal/ltc/ltc_gm20b.h"
@@ -380,7 +382,6 @@ static const struct gpu_ops vgpu_gp10b_ops = {
 		.userd_gp_put = gk20a_fifo_userd_gp_put,
 		.userd_pb_get = gk20a_fifo_userd_pb_get,
 		.userd_entry_size = gk20a_fifo_userd_entry_size,
-		.pbdma_acquire_val = gk20a_fifo_pbdma_acquire_val,
 		.preempt_channel = vgpu_fifo_preempt_channel,
 		.preempt_tsg = vgpu_fifo_preempt_tsg,
 		.enable_tsg = vgpu_enable_tsg,
@@ -392,14 +393,12 @@ static const struct gpu_ops vgpu_gp10b_ops = {
 		.get_mmu_fault_desc = gp10b_fifo_get_mmu_fault_desc,
 		.get_mmu_fault_client_desc = gp10b_fifo_get_mmu_fault_client_desc,
 		.get_mmu_fault_gpc_desc = gm20b_fifo_get_mmu_fault_gpc_desc,
-		.get_pbdma_signature = gp10b_fifo_get_pbdma_signature,
 		.tsg_set_timeslice = vgpu_tsg_set_timeslice,
 		.tsg_open = vgpu_tsg_open,
 		.tsg_release = vgpu_tsg_release,
 		.force_reset_ch = vgpu_fifo_force_reset_ch,
 		.init_engine_info = vgpu_fifo_init_engine_info,
 		.get_engines_mask_on_id = NULL,
-		.dump_pbdma_status = NULL,
 		.dump_channel_status_ramfc = NULL,
 		.capture_channel_ram_dump = NULL,
 		.intr_0_error_mask = gk20a_fifo_intr_0_error_mask,
@@ -408,8 +407,6 @@ static const struct gpu_ops vgpu_gp10b_ops = {
 		.reset_enable_hw = NULL,
 		.teardown_ch_tsg = NULL,
 		.handle_sched_error = NULL,
-		.handle_pbdma_intr_0 = NULL,
-		.handle_pbdma_intr_1 = gk20a_fifo_handle_pbdma_intr_1,
 		.tsg_bind_channel = vgpu_tsg_bind_channel,
 		.tsg_unbind_channel = vgpu_tsg_unbind_channel,
 		.post_event_id = gk20a_tsg_event_id_post_event,
@@ -427,6 +424,15 @@ static const struct gpu_ops vgpu_gp10b_ops = {
 	.engine = {
 		.is_fault_engine_subid_gpc = gm20b_is_fault_engine_subid_gpc,
 	},
+	.pbdma = {
+		.pbdma_acquire_val = gm20b_pbdma_acquire_val,
+		.get_pbdma_signature = gp10b_pbdma_get_signature,
+		.dump_pbdma_status = NULL,
+		.handle_pbdma_intr_0 = NULL,
+		.handle_pbdma_intr_1 = gm20b_pbdma_handle_intr_1,
+		.read_pbdma_data = NULL,
+		.reset_pbdma_header = NULL,
+	},
 	.sync = {
 #ifdef CONFIG_TEGRA_GK20A_NVHOST
 		.alloc_syncpt_buf = gk20a_alloc_syncpt_buf,
@@ -682,6 +688,7 @@ int vgpu_gp10b_init_hal(struct gk20a *g)
 	gops->clock_gating = vgpu_gp10b_ops.clock_gating;
 	gops->fifo = vgpu_gp10b_ops.fifo;
 	gops->engine = vgpu_gp10b_ops.engine;
+	gops->pbdma = vgpu_gp10b_ops.pbdma;
 	gops->runlist = vgpu_gp10b_ops.runlist;
 	gops->channel = vgpu_gp10b_ops.channel;
 	gops->sync = vgpu_gp10b_ops.sync;

drivers/gpu/nvgpu/common/vgpu/gv11b/vgpu_hal_gv11b.c

@@ -23,6 +23,9 @@
 #include "hal/bus/bus_gk20a.h"
 #include "hal/bus/bus_gm20b.h"
 #include "hal/fifo/engines_gv11b.h"
+#include "hal/fifo/pbdma_gm20b.h"
+#include "hal/fifo/pbdma_gp10b.h"
+#include "hal/fifo/pbdma_gv11b.h"
 #include "hal/therm/therm_gm20b.h"
 #include "hal/therm/therm_gp10b.h"
 #include "hal/therm/therm_gv11b.h"
@@ -447,7 +450,6 @@ static const struct gpu_ops vgpu_gv11b_ops = {
 		.userd_gp_put = gv11b_userd_gp_put,
 		.userd_pb_get = gv11b_userd_pb_get,
 		.userd_entry_size = gk20a_fifo_userd_entry_size,
-		.pbdma_acquire_val = gk20a_fifo_pbdma_acquire_val,
 		.preempt_channel = vgpu_fifo_preempt_channel,
 		.preempt_tsg = vgpu_fifo_preempt_tsg,
 		.enable_tsg = vgpu_gv11b_enable_tsg,
@@ -461,14 +463,12 @@ static const struct gpu_ops vgpu_gv11b_ops = {
 		.get_mmu_fault_desc = NULL,
 		.get_mmu_fault_client_desc = NULL,
 		.get_mmu_fault_gpc_desc = NULL,
-		.get_pbdma_signature = gp10b_fifo_get_pbdma_signature,
 		.tsg_set_timeslice = vgpu_tsg_set_timeslice,
 		.tsg_open = vgpu_tsg_open,
 		.tsg_release = vgpu_tsg_release,
 		.force_reset_ch = vgpu_fifo_force_reset_ch,
 		.init_engine_info = vgpu_fifo_init_engine_info,
 		.get_engines_mask_on_id = NULL,
-		.dump_pbdma_status = NULL,
 		.dump_channel_status_ramfc = NULL,
 		.capture_channel_ram_dump = NULL,
 		.intr_0_error_mask = gv11b_fifo_intr_0_error_mask,
@@ -477,8 +477,6 @@ static const struct gpu_ops vgpu_gv11b_ops = {
 		.reset_enable_hw = NULL,
 		.teardown_ch_tsg = NULL,
 		.handle_sched_error = NULL,
-		.handle_pbdma_intr_0 = NULL,
-		.handle_pbdma_intr_1 = gv11b_fifo_handle_pbdma_intr_1,
 		.init_eng_method_buffers = gv11b_fifo_init_eng_method_buffers,
 		.deinit_eng_method_buffers =
 			gv11b_fifo_deinit_eng_method_buffers,
@@ -504,6 +502,15 @@ static const struct gpu_ops vgpu_gv11b_ops = {
 	.engine = {
 		.is_fault_engine_subid_gpc = gv11b_is_fault_engine_subid_gpc,
 	},
+	.pbdma = {
+		.pbdma_acquire_val = gm20b_pbdma_acquire_val,
+		.get_pbdma_signature = gp10b_pbdma_get_signature,
+		.dump_pbdma_status = NULL,
+		.handle_pbdma_intr_0 = NULL,
+		.handle_pbdma_intr_1 = gv11b_pbdma_handle_intr_1,
+		.read_pbdma_data = NULL,
+		.reset_pbdma_header = NULL,
+	},
 	.sync = {
 #ifdef CONFIG_TEGRA_GK20A_NVHOST
 		.alloc_syncpt_buf = vgpu_gv11b_fifo_alloc_syncpt_buf,
@@ -761,6 +768,7 @@ int vgpu_gv11b_init_hal(struct gk20a *g)
 	gops->clock_gating = vgpu_gv11b_ops.clock_gating;
 	gops->fifo = vgpu_gv11b_ops.fifo;
 	gops->engine = vgpu_gv11b_ops.engine;
+	gops->pbdma = vgpu_gv11b_ops.pbdma;
 	gops->runlist = vgpu_gv11b_ops.runlist;
 	gops->channel = vgpu_gv11b_ops.channel;
 	gops->sync = vgpu_gv11b_ops.sync;

drivers/gpu/nvgpu/gk20a/fifo_gk20a.c

@@ -68,20 +68,6 @@
 
 #define FECS_METHOD_WFI_RESTORE 0x80000U
 
-static const char *const pbdma_intr_fault_type_desc[] = {
-	"MEMREQ timeout", "MEMACK_TIMEOUT", "MEMACK_EXTRA acks",
-	"MEMDAT_TIMEOUT", "MEMDAT_EXTRA acks", "MEMFLUSH noack",
-	"MEMOP noack", "LBCONNECT noack", "NONE - was LBREQ",
-	"LBACK_TIMEOUT", "LBACK_EXTRA acks", "LBDAT_TIMEOUT",
-	"LBDAT_EXTRA acks", "GPFIFO won't fit", "GPPTR invalid",
-	"GPENTRY invalid", "GPCRC mismatch", "PBPTR get>put",
-	"PBENTRY invld", "PBCRC mismatch", "NONE - was XBARC",
-	"METHOD invld", "METHODCRC mismat", "DEVICE sw method",
-	"[ENGINE]", "SEMAPHORE invlid", "ACQUIRE timeout",
-	"PRI forbidden", "ILLEGAL SYNCPT", "[NO_CTXSW_SEG]",
-	"PBSEG badsplit", "SIGNATURE bad"
-};
-
 void nvgpu_report_host_error(struct gk20a *g, u32 inst,
 		u32 err_id, u32 intr_info)
 {
@@ -1573,167 +1559,6 @@ static u32 fifo_error_isr(struct gk20a *g, u32 fifo_intr)
 	return handled;
 }
 
-void gk20a_fifo_reset_pbdma_header(struct gk20a *g, u32 pbdma_id)
-{
-	gk20a_writel(g, pbdma_pb_header_r(pbdma_id),
-			pbdma_pb_header_first_true_f() |
-			pbdma_pb_header_type_non_inc_f());
-}
-
-void gk20a_fifo_reset_pbdma_method(struct gk20a *g, u32 pbdma_id,
-						u32 pbdma_method_index)
-{
-	u32 pbdma_method_stride;
-	u32 pbdma_method_reg;
-
-	pbdma_method_stride = pbdma_method1_r(pbdma_id) -
-				pbdma_method0_r(pbdma_id);
-
-	pbdma_method_reg = pbdma_method0_r(pbdma_id) +
-		(pbdma_method_index * pbdma_method_stride);
-
-	gk20a_writel(g, pbdma_method_reg,
-			pbdma_method0_valid_true_f() |
-			pbdma_method0_first_true_f() |
-			pbdma_method0_addr_f(
-			     pbdma_udma_nop_r() >> 2));
-}
-
-static bool gk20a_fifo_is_sw_method_subch(struct gk20a *g, u32 pbdma_id,
-						u32 pbdma_method_index)
-{
-	u32 pbdma_method_stride;
-	u32 pbdma_method_reg, pbdma_method_subch;
-
-	pbdma_method_stride = pbdma_method1_r(pbdma_id) -
-				pbdma_method0_r(pbdma_id);
-
-	pbdma_method_reg = pbdma_method0_r(pbdma_id) +
-			(pbdma_method_index * pbdma_method_stride);
-
-	pbdma_method_subch = pbdma_method0_subch_v(
-			gk20a_readl(g, pbdma_method_reg));
-
-	if (pbdma_method_subch == 5U ||
-	    pbdma_method_subch == 6U ||
-	    pbdma_method_subch == 7U) {
-		return true;
-	}
-
-	return false;
-}
-
-u32 gk20a_fifo_read_pbdma_data(struct gk20a *g, u32 pbdma_id)
-{
-	return nvgpu_readl(g, pbdma_hdr_shadow_r(pbdma_id));
-}
-
-unsigned int gk20a_fifo_handle_pbdma_intr_0(struct gk20a *g, u32 pbdma_id,
-			u32 pbdma_intr_0, u32 *handled, u32 *error_notifier)
-{
-	struct fifo_gk20a *f = &g->fifo;
-	unsigned int rc_type = RC_TYPE_NO_RC;
-	u32 i;
-	unsigned long pbdma_intr_err;
-	unsigned long bit;
-
-	if (((f->intr.pbdma.device_fatal_0 |
-	      f->intr.pbdma.channel_fatal_0 |
-	      f->intr.pbdma.restartable_0) & pbdma_intr_0) != 0U) {
-
-		pbdma_intr_err = (unsigned long)pbdma_intr_0;
-		for_each_set_bit(bit, &pbdma_intr_err, 32U) {
-			nvgpu_err(g, "PBDMA intr %s Error",
-				pbdma_intr_fault_type_desc[bit]);
-		}
-
-		nvgpu_err(g,
-			"pbdma_intr_0(%d):0x%08x PBH: %08x "
-			"SHADOW: %08x gp shadow0: %08x gp shadow1: %08x"
-			"M0: %08x %08x %08x %08x ",
-			pbdma_id, pbdma_intr_0,
-			gk20a_readl(g, pbdma_pb_header_r(pbdma_id)),
-			g->ops.fifo.read_pbdma_data(g, pbdma_id),
-			gk20a_readl(g, pbdma_gp_shadow_0_r(pbdma_id)),
-			gk20a_readl(g, pbdma_gp_shadow_1_r(pbdma_id)),
-			gk20a_readl(g, pbdma_method0_r(pbdma_id)),
-			gk20a_readl(g, pbdma_method1_r(pbdma_id)),
-			gk20a_readl(g, pbdma_method2_r(pbdma_id)),
-			gk20a_readl(g, pbdma_method3_r(pbdma_id))
-			);
-
-		rc_type = RC_TYPE_PBDMA_FAULT;
-		*handled |= ((f->intr.pbdma.device_fatal_0 |
-			     f->intr.pbdma.channel_fatal_0 |
-			     f->intr.pbdma.restartable_0) &
-			    pbdma_intr_0);
-	}
-
-	if ((pbdma_intr_0 & pbdma_intr_0_acquire_pending_f()) != 0U) {
-		u32 val = gk20a_readl(g, pbdma_acquire_r(pbdma_id));
-
-		val &= ~pbdma_acquire_timeout_en_enable_f();
-		gk20a_writel(g, pbdma_acquire_r(pbdma_id), val);
-		if (nvgpu_is_timeouts_enabled(g)) {
-			rc_type = RC_TYPE_PBDMA_FAULT;
-			nvgpu_err(g,
-				"semaphore acquire timeout!");
-			*error_notifier = NVGPU_ERR_NOTIFIER_GR_SEMAPHORE_TIMEOUT;
-		}
-		*handled |= pbdma_intr_0_acquire_pending_f();
-	}
-
-	if ((pbdma_intr_0 & pbdma_intr_0_pbentry_pending_f()) != 0U) {
-		g->ops.fifo.reset_pbdma_header(g, pbdma_id);
-		gk20a_fifo_reset_pbdma_method(g, pbdma_id, 0);
-		rc_type = RC_TYPE_PBDMA_FAULT;
-	}
-
-	if ((pbdma_intr_0 & pbdma_intr_0_method_pending_f()) != 0U) {
-		gk20a_fifo_reset_pbdma_method(g, pbdma_id, 0);
-		rc_type = RC_TYPE_PBDMA_FAULT;
-	}
-
-	if ((pbdma_intr_0 & pbdma_intr_0_pbcrc_pending_f()) != 0U) {
-		*error_notifier =
-			NVGPU_ERR_NOTIFIER_PBDMA_PUSHBUFFER_CRC_MISMATCH;
-		rc_type = RC_TYPE_PBDMA_FAULT;
-	}
-
-	if ((pbdma_intr_0 & pbdma_intr_0_device_pending_f()) != 0U) {
-		g->ops.fifo.reset_pbdma_header(g, pbdma_id);
-
-		for (i = 0U; i < 4U; i++) {
-			if (gk20a_fifo_is_sw_method_subch(g,
-					pbdma_id, i)) {
-				gk20a_fifo_reset_pbdma_method(g,
-						pbdma_id, i);
-			}
-		}
-		rc_type = RC_TYPE_PBDMA_FAULT;
-	}
-
-	return rc_type;
-}
-
-unsigned int gk20a_fifo_handle_pbdma_intr_1(struct gk20a *g,
-			u32 pbdma_id, u32 pbdma_intr_1,
-			u32 *handled, u32 *error_notifier)
-{
-	unsigned int rc_type = RC_TYPE_PBDMA_FAULT;
-
-	/*
-	 * all of the interrupts in _intr_1 are "host copy engine"
-	 * related, which is not supported. For now just make them
-	 * channel fatal.
-	 */
-	nvgpu_err(g, "hce err: pbdma_intr_1(%d):0x%08x",
-		pbdma_id, pbdma_intr_1);
-	*handled |= pbdma_intr_1;
-
-	return rc_type;
-}
-
 static void gk20a_fifo_pbdma_fault_rc(struct gk20a *g,
 			struct fifo_gk20a *f, u32 pbdma_id,
 			u32 error_notifier)
@@ -1791,7 +1616,7 @@ u32 gk20a_fifo_handle_pbdma_intr(struct gk20a *g, struct fifo_gk20a *f,
 			"pbdma id %d intr_0 0x%08x pending",
 			pbdma_id, pbdma_intr_0);
 
-		if (g->ops.fifo.handle_pbdma_intr_0(g, pbdma_id, pbdma_intr_0,
+		if (g->ops.pbdma.handle_pbdma_intr_0(g, pbdma_id, pbdma_intr_0,
 			&handled, &error_notifier) != RC_TYPE_NO_RC) {
 			rc_type = RC_TYPE_PBDMA_FAULT;
 		}
@@ -1803,7 +1628,7 @@ u32 gk20a_fifo_handle_pbdma_intr(struct gk20a *g, struct fifo_gk20a *f,
 			"pbdma id %d intr_1 0x%08x pending",
 			pbdma_id, pbdma_intr_1);
 
-		if (g->ops.fifo.handle_pbdma_intr_1(g, pbdma_id, pbdma_intr_1,
+		if (g->ops.pbdma.handle_pbdma_intr_1(g, pbdma_id, pbdma_intr_1,
 			&handled, &error_notifier) != RC_TYPE_NO_RC) {
 			rc_type = RC_TYPE_PBDMA_FAULT;
 		}
@@ -2153,11 +1978,6 @@ bool gk20a_fifo_mmu_fault_pending(struct gk20a *g)
 	}
 }
 
-u32 gk20a_fifo_get_pbdma_signature(struct gk20a *g)
-{
-	return pbdma_signature_hw_valid_f() | pbdma_signature_sw_zero_f();
-}
-
 static const char * const pbdma_chan_eng_ctx_status_str[] = {
 	"invalid",
 	"valid",
@@ -2359,56 +2179,6 @@ void gk20a_debug_dump_all_channel_status_ramfc(struct gk20a *g,
 	nvgpu_kfree(g, infos);
 }
 
-void gk20a_dump_pbdma_status(struct gk20a *g,
-				 struct gk20a_debug_output *o)
-{
-	u32 i, host_num_pbdma;
-	struct nvgpu_pbdma_status_info pbdma_status;
-
-	host_num_pbdma = nvgpu_get_litter_value(g, GPU_LIT_HOST_NUM_PBDMA);
-
-	gk20a_debug_output(o, "PBDMA Status - chip %-5s", g->name);
-	gk20a_debug_output(o, "-------------------------");
-
-	for (i = 0; i < host_num_pbdma; i++) {
-		g->ops.pbdma_status.read_pbdma_status_info(g, i,
-			&pbdma_status);
-
-		gk20a_debug_output(o, "pbdma %d:", i);
-		gk20a_debug_output(o,
-			"  id: %d - %-9s next_id: - %d %-9s | status: %s",
-			pbdma_status.id,
-			nvgpu_pbdma_status_is_id_type_tsg(&pbdma_status) ?
-				   "[tsg]" : "[channel]",
-			pbdma_status.next_id,
-			nvgpu_pbdma_status_is_next_id_type_tsg(
-				&pbdma_status) ?
-				   "[tsg]" : "[channel]",
-			gk20a_decode_pbdma_chan_eng_ctx_status(
-				pbdma_status.pbdma_channel_status));
-		gk20a_debug_output(o,
-			"  PBDMA_PUT %016llx PBDMA_GET %016llx",
-			(u64)gk20a_readl(g, pbdma_put_r(i)) +
-			((u64)gk20a_readl(g, pbdma_put_hi_r(i)) << 32ULL),
-			(u64)gk20a_readl(g, pbdma_get_r(i)) +
-			((u64)gk20a_readl(g, pbdma_get_hi_r(i)) << 32ULL));
-		gk20a_debug_output(o,
-			"  GP_PUT    %08x  GP_GET  %08x  "
-			"FETCH   %08x HEADER %08x",
-			gk20a_readl(g, pbdma_gp_put_r(i)),
-			gk20a_readl(g, pbdma_gp_get_r(i)),
-			gk20a_readl(g, pbdma_gp_fetch_r(i)),
-			gk20a_readl(g, pbdma_pb_header_r(i)));
-		gk20a_debug_output(o,
-			"  HDR       %08x  SHADOW0 %08x  SHADOW1 %08x",
-			gk20a_readl(g, pbdma_hdr_shadow_r(i)),
-			gk20a_readl(g, pbdma_gp_shadow_0_r(i)),
-			gk20a_readl(g, pbdma_gp_shadow_1_r(i)));
-	}
-
-	gk20a_debug_output(o, " ");
-}
-
 static int gk20a_fifo_commit_userd(struct channel_gk20a *c)
 {
 	u32 addr_lo;
@@ -2465,7 +2235,7 @@ int gk20a_fifo_setup_ramfc(struct channel_gk20a *c,
 		pbdma_gp_base_hi_limit2_f((u32)limit2_val));
 
 	nvgpu_mem_wr32(g, mem, ram_fc_signature_w(),
-		 c->g->ops.fifo.get_pbdma_signature(c->g));
+		 c->g->ops.pbdma.get_pbdma_signature(c->g));
 
 	nvgpu_mem_wr32(g, mem, ram_fc_formats_w(),
 		pbdma_formats_gp_fermi0_f() |
@@ -2488,7 +2258,7 @@ int gk20a_fifo_setup_ramfc(struct channel_gk20a *c,
 	nvgpu_mem_wr32(g, mem, ram_fc_target_w(), pbdma_target_engine_sw_f());
 
 	nvgpu_mem_wr32(g, mem, ram_fc_acquire_w(),
-		g->ops.fifo.pbdma_acquire_val(timeout));
+		g->ops.pbdma.pbdma_acquire_val(timeout));
 
 	nvgpu_mem_wr32(g, mem, ram_fc_runlist_timeslice_w(),
 		fifo_runlist_timeslice_timeout_128_f() |
@@ -2603,49 +2373,6 @@ u32 gk20a_fifo_userd_entry_size(struct gk20a *g)
 	return BIT32(ram_userd_base_shift_v());
 }
 
-u32 gk20a_fifo_pbdma_acquire_val(u64 timeout)
-{
-	u32 val, exponent, mantissa;
-	unsigned int val_len;
-	u64 tmp;
-
-	val = pbdma_acquire_retry_man_2_f() |
-		pbdma_acquire_retry_exp_2_f();
-
-	if (timeout == 0ULL) {
-		return val;
-	}
-
-	timeout *= 80UL;
-	do_div(timeout, 100U); /* set acquire timeout to 80% of channel wdt */
-	timeout *= 1000000UL; /* ms -> ns */
-	do_div(timeout, 1024U); /* in unit of 1024ns */
-	tmp = fls(timeout >> 32U);
-	BUG_ON(tmp > U64(U32_MAX));
-	val_len = (u32)tmp + 32U;
-	if (val_len == 32U) {
-		val_len = (u32)fls(timeout);
-	}
-	if (val_len > 16U + pbdma_acquire_timeout_exp_max_v()) { /* man: 16bits */
-		exponent = pbdma_acquire_timeout_exp_max_v();
-		mantissa = pbdma_acquire_timeout_man_max_v();
-	} else if (val_len > 16U) {
-		exponent = val_len - 16U;
-		BUG_ON((timeout >> exponent) > U64(U32_MAX));
-		mantissa = (u32)(timeout >> exponent);
-	} else {
-		exponent = 0;
-		BUG_ON(timeout > U64(U32_MAX));
-		mantissa = (u32)timeout;
-	}
-
-	val |= pbdma_acquire_timeout_exp_f(exponent) |
-		pbdma_acquire_timeout_man_f(mantissa) |
-		pbdma_acquire_timeout_en_enable_f();
-
-	return val;
-}
-
 bool gk20a_fifo_find_pbdma_for_runlist(struct fifo_gk20a *f, u32 runlist_id,
 								u32 *pbdma_id)
 {

drivers/gpu/nvgpu/gk20a/fifo_gk20a.h

@@ -276,7 +276,6 @@ int gk20a_fifo_tsg_unbind_channel(struct channel_gk20a *ch);
 
 void fifo_gk20a_finish_mmu_fault_handling(struct gk20a *g,
 		unsigned long fault_id);
-u32 gk20a_fifo_get_pbdma_signature(struct gk20a *g);
 u32 gk20a_fifo_get_failing_engine_data(struct gk20a *g,
 		u32 *__id, bool *__is_tsg);
 void gk20a_fifo_abort_tsg(struct gk20a *g, struct tsg_gk20a *tsg, bool preempt);
@@ -324,8 +323,6 @@ void gk20a_capture_channel_ram_dump(struct gk20a *g,
 		struct nvgpu_channel_dump_info *info);
 void gk20a_debug_dump_all_channel_status_ramfc(struct gk20a *g,
 		 struct gk20a_debug_output *o);
-void gk20a_dump_pbdma_status(struct gk20a *g,
-				 struct gk20a_debug_output *o);
 const char *gk20a_decode_pbdma_chan_eng_ctx_status(u32 index);
 
 int gk20a_fifo_tsg_unbind_channel_verify_status(struct channel_gk20a *ch);
@@ -346,7 +343,6 @@ void gk20a_fifo_setup_ramfc_for_privileged_channel(struct channel_gk20a *c);
 int gk20a_fifo_alloc_inst(struct gk20a *g, struct channel_gk20a *ch);
 void gk20a_fifo_free_inst(struct gk20a *g, struct channel_gk20a *ch);
 int gk20a_fifo_setup_userd(struct channel_gk20a *c);
-u32 gk20a_fifo_pbdma_acquire_val(u64 timeout);
 
 
 u32 gk20a_fifo_runlist_busy_engines(struct gk20a *g, u32 runlist_id);
@@ -362,12 +358,6 @@ void gk20a_fifo_teardown_mask_intr(struct gk20a *g);
 void gk20a_fifo_teardown_unmask_intr(struct gk20a *g);
 bool gk20a_fifo_handle_sched_error(struct gk20a *g);
 
-void gk20a_fifo_reset_pbdma_method(struct gk20a *g, u32 pbdma_id,
-			 u32 pbdma_method_index);
-unsigned int gk20a_fifo_handle_pbdma_intr_0(struct gk20a *g, u32 pbdma_id,
-			u32 pbdma_intr_0, u32 *handled, u32 *error_notifier);
-unsigned int gk20a_fifo_handle_pbdma_intr_1(struct gk20a *g, u32 pbdma_id,
-			u32 pbdma_intr_1, u32 *handled, u32 *error_notifier);
 u32 gk20a_fifo_handle_pbdma_intr(struct gk20a *g, struct fifo_gk20a *f,
 			u32 pbdma_id, unsigned int rc);
 
@@ -385,7 +375,5 @@ u32 gk20a_fifo_userd_entry_size(struct gk20a *g);
 
 bool gk20a_fifo_find_pbdma_for_runlist(struct fifo_gk20a *f, u32 runlist_id,
 			u32 *pbdma_id);
-u32 gk20a_fifo_read_pbdma_data(struct gk20a *g, u32 pbdma_id);
-void gk20a_fifo_reset_pbdma_header(struct gk20a *g, u32 pbdma_id);
 int gk20a_fifo_init_pbdma_info(struct fifo_gk20a *f);
 #endif /* FIFO_GK20A_H */

drivers/gpu/nvgpu/gm20b/hal_gm20b.c

@@ -46,6 +46,7 @@
 #include "hal/ltc/ltc_gm20b.h"
 #include "hal/fb/fb_gm20b.h"
 #include "hal/fuse/fuse_gm20b.h"
+#include "hal/fifo/pbdma_gm20b.h"
 #include "hal/fifo/engines_gm20b.h"
 #include "hal/fifo/engine_status_gm20b.h"
 #include "hal/fifo/pbdma_status_gm20b.h"
@@ -518,7 +519,6 @@ static const struct gpu_ops gm20b_ops = {
 		.userd_gp_put = gk20a_fifo_userd_gp_put,
 		.userd_pb_get = gk20a_fifo_userd_pb_get,
 		.userd_entry_size = gk20a_fifo_userd_entry_size,
-		.pbdma_acquire_val = gk20a_fifo_pbdma_acquire_val,
 		.preempt_channel = gk20a_fifo_preempt_channel,
 		.preempt_tsg = gk20a_fifo_preempt_tsg,
 		.enable_tsg = gk20a_enable_tsg,
@@ -530,13 +530,11 @@ static const struct gpu_ops gm20b_ops = {
 		.get_mmu_fault_desc = gk20a_fifo_get_mmu_fault_desc,
 		.get_mmu_fault_client_desc = gk20a_fifo_get_mmu_fault_client_desc,
 		.get_mmu_fault_gpc_desc = gm20b_fifo_get_mmu_fault_gpc_desc,
-		.get_pbdma_signature = gk20a_fifo_get_pbdma_signature,
 		.tsg_set_timeslice = gk20a_fifo_tsg_set_timeslice,
 		.force_reset_ch = gk20a_fifo_force_reset_ch,
 		.init_pbdma_info = gk20a_fifo_init_pbdma_info,
 		.init_engine_info = gm20b_fifo_init_engine_info,
 		.get_engines_mask_on_id = gk20a_fifo_engines_on_id,
-		.dump_pbdma_status = gk20a_dump_pbdma_status,
 		.dump_channel_status_ramfc = gk20a_dump_channel_status_ramfc,
 		.capture_channel_ram_dump = gk20a_capture_channel_ram_dump,
 		.intr_0_error_mask = gk20a_fifo_intr_0_error_mask,
@@ -547,8 +545,6 @@ static const struct gpu_ops gm20b_ops = {
 		.teardown_mask_intr = gk20a_fifo_teardown_mask_intr,
 		.teardown_unmask_intr = gk20a_fifo_teardown_unmask_intr,
 		.handle_sched_error = gk20a_fifo_handle_sched_error,
-		.handle_pbdma_intr_0 = gk20a_fifo_handle_pbdma_intr_0,
-		.handle_pbdma_intr_1 = gk20a_fifo_handle_pbdma_intr_1,
 		.tsg_bind_channel = gk20a_tsg_bind_channel,
 		.tsg_unbind_channel = gk20a_fifo_tsg_unbind_channel,
 		.post_event_id = gk20a_tsg_event_id_post_event,
@@ -564,12 +560,19 @@ static const struct gpu_ops gm20b_ops = {
 		.runlist_busy_engines = gk20a_fifo_runlist_busy_engines,
 		.find_pbdma_for_runlist = gk20a_fifo_find_pbdma_for_runlist,
 		.init_ce_engine_info = gm20b_fifo_init_ce_engine_info,
-		.read_pbdma_data = gk20a_fifo_read_pbdma_data,
-		.reset_pbdma_header = gk20a_fifo_reset_pbdma_header,
 	},
 	.engine = {
 		.is_fault_engine_subid_gpc = gm20b_is_fault_engine_subid_gpc,
 	},
+	.pbdma = {
+		.pbdma_acquire_val = gm20b_pbdma_acquire_val,
+		.get_pbdma_signature = gm20b_pbdma_get_signature,
+		.dump_pbdma_status = gm20b_pbdma_dump_status,
+		.handle_pbdma_intr_0 = gm20b_pbdma_handle_intr_0,
+		.handle_pbdma_intr_1 = gm20b_pbdma_handle_intr_1,
+		.read_pbdma_data = gm20b_pbdma_read_data,
+		.reset_pbdma_header = gm20b_pbdma_reset_header,
+	},
 	.sync = {
 #ifdef CONFIG_TEGRA_GK20A_NVHOST
 		.alloc_syncpt_buf = gk20a_alloc_syncpt_buf,
@@ -877,6 +880,7 @@ int gm20b_init_hal(struct gk20a *g)
 	gops->clock_gating = gm20b_ops.clock_gating;
 	gops->fifo = gm20b_ops.fifo;
 	gops->engine = gm20b_ops.engine;
+	gops->pbdma = gm20b_ops.pbdma;
 	gops->runlist = gm20b_ops.runlist;
 	gops->channel = gm20b_ops.channel;
 	gops->sync = gm20b_ops.sync;

drivers/gpu/nvgpu/gp10b/fifo_gp10b.c

@@ -93,7 +93,7 @@ int channel_gp10b_setup_ramfc(struct channel_gk20a *c,
 		pbdma_gp_base_hi_limit2_f((u32)ilog2(gpfifo_entries)));
 
 	nvgpu_mem_wr32(g, mem, ram_fc_signature_w(),
-		 c->g->ops.fifo.get_pbdma_signature(c->g));
+		 c->g->ops.pbdma.get_pbdma_signature(c->g));
 
 	nvgpu_mem_wr32(g, mem, ram_fc_formats_w(),
 		pbdma_formats_gp_fermi0_f() |
@@ -116,7 +116,7 @@ int channel_gp10b_setup_ramfc(struct channel_gk20a *c,
 	nvgpu_mem_wr32(g, mem, ram_fc_target_w(), pbdma_target_engine_sw_f());
 
 	nvgpu_mem_wr32(g, mem, ram_fc_acquire_w(),
-		g->ops.fifo.pbdma_acquire_val(acquire_timeout));
+		g->ops.pbdma.pbdma_acquire_val(acquire_timeout));
 
 	nvgpu_mem_wr32(g, mem, ram_fc_runlist_timeslice_w(),
 		pbdma_runlist_timeslice_timeout_128_f() |
@@ -136,12 +136,6 @@ int channel_gp10b_setup_ramfc(struct channel_gk20a *c,
 	return channel_gp10b_commit_userd(c);
 }
 
-u32 gp10b_fifo_get_pbdma_signature(struct gk20a *g)
-{
-	return g->ops.get_litter_value(g, GPU_LIT_GPFIFO_CLASS)
-		| pbdma_signature_sw_zero_f();
-}
-
 int gp10b_fifo_resetup_ramfc(struct channel_gk20a *c)
 {
 	u32 new_syncpt = 0, old_syncpt;

drivers/gpu/nvgpu/gp10b/fifo_gp10b.h

@@ -33,7 +33,6 @@ struct mmu_fault_info;
 int channel_gp10b_setup_ramfc(struct channel_gk20a *c,
 			u64 gpfifo_base, u32 gpfifo_entries,
 			unsigned long acquire_timeout, u32 flags);
-u32 gp10b_fifo_get_pbdma_signature(struct gk20a *g);
 int gp10b_fifo_resetup_ramfc(struct channel_gk20a *c);
 void gp10b_device_info_data_parse(struct gk20a *g, u32 table_entry,
 				u32 *inst_id, u32 *pri_base, u32 *fault_id);
@@ -44,4 +43,5 @@ void gp10b_fifo_get_mmu_fault_desc(struct mmu_fault_info *mmfault);
 void gp10b_fifo_get_mmu_fault_client_desc(struct mmu_fault_info *mmfault);
 int channel_gp10b_commit_userd(struct channel_gk20a *c);
 int gp10b_fifo_init_ce_engine_info(struct fifo_gk20a *f);
+
 #endif

drivers/gpu/nvgpu/gp10b/hal_gp10b.c

@@ -54,6 +54,8 @@
 #include "hal/fb/fb_gp10b.h"
 #include "hal/fuse/fuse_gm20b.h"
 #include "hal/fuse/fuse_gp10b.h"
+#include "hal/fifo/pbdma_gm20b.h"
+#include "hal/fifo/pbdma_gp10b.h"
 #include "hal/fifo/engines_gm20b.h"
 #include "hal/fifo/engine_status_gm20b.h"
 #include "hal/fifo/pbdma_status_gm20b.h"
@@ -591,7 +593,6 @@ static const struct gpu_ops gp10b_ops = {
 		.userd_gp_put = gk20a_fifo_userd_gp_put,
 		.userd_pb_get = gk20a_fifo_userd_pb_get,
 		.userd_entry_size = gk20a_fifo_userd_entry_size,
-		.pbdma_acquire_val = gk20a_fifo_pbdma_acquire_val,
 		.preempt_channel = gk20a_fifo_preempt_channel,
 		.preempt_tsg = gk20a_fifo_preempt_tsg,
 		.enable_tsg = gk20a_enable_tsg,
@@ -603,13 +604,11 @@ static const struct gpu_ops gp10b_ops = {
 		.get_mmu_fault_desc = gp10b_fifo_get_mmu_fault_desc,
 		.get_mmu_fault_client_desc = gp10b_fifo_get_mmu_fault_client_desc,
 		.get_mmu_fault_gpc_desc = gm20b_fifo_get_mmu_fault_gpc_desc,
-		.get_pbdma_signature = gp10b_fifo_get_pbdma_signature,
 		.tsg_set_timeslice = gk20a_fifo_tsg_set_timeslice,
 		.force_reset_ch = gk20a_fifo_force_reset_ch,
 		.init_pbdma_info = gk20a_fifo_init_pbdma_info,
 		.init_engine_info = gm20b_fifo_init_engine_info,
 		.get_engines_mask_on_id = gk20a_fifo_engines_on_id,
-		.dump_pbdma_status = gk20a_dump_pbdma_status,
 		.dump_channel_status_ramfc = gk20a_dump_channel_status_ramfc,
 		.capture_channel_ram_dump = gk20a_capture_channel_ram_dump,
 		.intr_0_error_mask = gk20a_fifo_intr_0_error_mask,
@@ -620,8 +619,6 @@ static const struct gpu_ops gp10b_ops = {
 		.teardown_mask_intr = gk20a_fifo_teardown_mask_intr,
 		.teardown_unmask_intr = gk20a_fifo_teardown_unmask_intr,
 		.handle_sched_error = gk20a_fifo_handle_sched_error,
-		.handle_pbdma_intr_0 = gk20a_fifo_handle_pbdma_intr_0,
-		.handle_pbdma_intr_1 = gk20a_fifo_handle_pbdma_intr_1,
 		.tsg_bind_channel = gk20a_tsg_bind_channel,
 		.tsg_unbind_channel = gk20a_fifo_tsg_unbind_channel,
 		.post_event_id = gk20a_tsg_event_id_post_event,
@@ -638,12 +635,19 @@ static const struct gpu_ops gp10b_ops = {
 		.runlist_busy_engines = gk20a_fifo_runlist_busy_engines,
 		.find_pbdma_for_runlist = gk20a_fifo_find_pbdma_for_runlist,
 		.init_ce_engine_info = gp10b_fifo_init_ce_engine_info,
-		.read_pbdma_data = gk20a_fifo_read_pbdma_data,
-		.reset_pbdma_header = gk20a_fifo_reset_pbdma_header,
 	},
 	.engine = {
 		.is_fault_engine_subid_gpc = gm20b_is_fault_engine_subid_gpc,
 	},
+	.pbdma = {
+		.pbdma_acquire_val = gm20b_pbdma_acquire_val,
+		.get_pbdma_signature = gp10b_pbdma_get_signature,
+		.dump_pbdma_status = gm20b_pbdma_dump_status,
+		.handle_pbdma_intr_0 = gm20b_pbdma_handle_intr_0,
+		.handle_pbdma_intr_1 = gm20b_pbdma_handle_intr_1,
+		.read_pbdma_data = gm20b_pbdma_read_data,
+		.reset_pbdma_header = gm20b_pbdma_reset_header,
+	},
 	.sync = {
 #ifdef CONFIG_TEGRA_GK20A_NVHOST
 		.alloc_syncpt_buf = gk20a_alloc_syncpt_buf,
@@ -959,6 +963,7 @@ int gp10b_init_hal(struct gk20a *g)
 	gops->clock_gating = gp10b_ops.clock_gating;
 	gops->fifo = gp10b_ops.fifo;
 	gops->engine = gp10b_ops.engine;
+	gops->pbdma = gp10b_ops.pbdma;
 	gops->runlist = gp10b_ops.runlist;
 	gops->channel = gp10b_ops.channel;
 	gops->sync = gp10b_ops.sync;

drivers/gpu/nvgpu/gv100/hal_gv100.c

@@ -45,6 +45,9 @@
 #include "hal/fuse/fuse_gm20b.h"
 #include "hal/fuse/fuse_gp10b.h"
 #include "hal/fuse/fuse_gp106.h"
+#include "hal/fifo/pbdma_gm20b.h"
+#include "hal/fifo/pbdma_gp10b.h"
+#include "hal/fifo/pbdma_gv11b.h"
 #include "hal/fifo/engines_gv11b.h"
 #include "hal/fifo/engine_status_gv100.h"
 #include "hal/fifo/pbdma_status_gm20b.h"
@@ -759,7 +762,6 @@ static const struct gpu_ops gv100_ops = {
 		.userd_gp_put = gv11b_userd_gp_put,
 		.userd_pb_get = gv11b_userd_pb_get,
 		.userd_entry_size = gk20a_fifo_userd_entry_size,
-		.pbdma_acquire_val = gk20a_fifo_pbdma_acquire_val,
 		.preempt_channel = gv11b_fifo_preempt_channel,
 		.preempt_tsg = gv11b_fifo_preempt_tsg,
 		.enable_tsg = gv11b_fifo_enable_tsg,
@@ -772,13 +774,11 @@ static const struct gpu_ops gv100_ops = {
 		.get_mmu_fault_desc = NULL,
 		.get_mmu_fault_client_desc = NULL,
 		.get_mmu_fault_gpc_desc = NULL,
-		.get_pbdma_signature = gp10b_fifo_get_pbdma_signature,
 		.tsg_set_timeslice = gk20a_fifo_tsg_set_timeslice,
 		.force_reset_ch = gk20a_fifo_force_reset_ch,
 		.init_engine_info = gm20b_fifo_init_engine_info,
 		.init_pbdma_info = gk20a_fifo_init_pbdma_info,
 		.get_engines_mask_on_id = gk20a_fifo_engines_on_id,
-		.dump_pbdma_status = gk20a_dump_pbdma_status,
 		.dump_channel_status_ramfc = gv11b_dump_channel_status_ramfc,
 		.capture_channel_ram_dump = gv11b_capture_channel_ram_dump,
 		.intr_0_error_mask = gv11b_fifo_intr_0_error_mask,
@@ -789,8 +789,6 @@ static const struct gpu_ops gv100_ops = {
 		.teardown_mask_intr = gv100_fifo_teardown_mask_intr,
 		.teardown_unmask_intr = gv100_fifo_teardown_unmask_intr,
 		.handle_sched_error = gk20a_fifo_handle_sched_error,
-		.handle_pbdma_intr_0 = gv11b_fifo_handle_pbdma_intr_0,
-		.handle_pbdma_intr_1 = gv11b_fifo_handle_pbdma_intr_1,
 		.init_eng_method_buffers = gv11b_fifo_init_eng_method_buffers,
 		.deinit_eng_method_buffers =
 			gv11b_fifo_deinit_eng_method_buffers,
@@ -814,12 +812,19 @@ static const struct gpu_ops gv100_ops = {
 		.runlist_busy_engines = gk20a_fifo_runlist_busy_engines,
 		.find_pbdma_for_runlist = gk20a_fifo_find_pbdma_for_runlist,
 		.init_ce_engine_info = gp10b_fifo_init_ce_engine_info,
-		.read_pbdma_data = gk20a_fifo_read_pbdma_data,
-		.reset_pbdma_header = gk20a_fifo_reset_pbdma_header,
 	},
 	.engine = {
 		.is_fault_engine_subid_gpc = gv11b_is_fault_engine_subid_gpc,
 	},
+	.pbdma = {
+		.pbdma_acquire_val = gm20b_pbdma_acquire_val,
+		.get_pbdma_signature = gp10b_pbdma_get_signature,
+		.dump_pbdma_status = gm20b_pbdma_dump_status,
+		.handle_pbdma_intr_0 = gv11b_pbdma_handle_intr_0,
+		.handle_pbdma_intr_1 = gv11b_pbdma_handle_intr_1,
+		.read_pbdma_data = gm20b_pbdma_read_data,
+		.reset_pbdma_header = gm20b_pbdma_reset_header,
+	},
 	.sync = {
 #ifdef CONFIG_TEGRA_GK20A_NVHOST
 		.alloc_syncpt_buf = gv11b_alloc_syncpt_buf,
@@ -1245,6 +1250,7 @@ int gv100_init_hal(struct gk20a *g)
 	gops->clock_gating = gv100_ops.clock_gating;
 	gops->fifo = gv100_ops.fifo;
 	gops->engine = gv100_ops.engine;
+	gops->pbdma = gv100_ops.pbdma;
 	gops->runlist = gv100_ops.runlist;
 	gops->channel = gv100_ops.channel;
 	gops->sync = gv100_ops.sync;

drivers/gpu/nvgpu/gv11b/fifo_gv11b.c

@@ -103,7 +103,7 @@ int channel_gv11b_setup_ramfc(struct channel_gk20a *c,
 		pbdma_gp_base_hi_limit2_f(ilog2(gpfifo_entries)));
 
 	nvgpu_mem_wr32(g, mem, ram_fc_signature_w(),
-		c->g->ops.fifo.get_pbdma_signature(c->g));
+		c->g->ops.pbdma.get_pbdma_signature(c->g));
 
 	nvgpu_mem_wr32(g, mem, ram_fc_pb_header_w(),
 		pbdma_pb_header_method_zero_f() |
@@ -123,7 +123,7 @@ int channel_gv11b_setup_ramfc(struct channel_gk20a *c,
 		pbdma_target_engine_sw_f());
 
 	nvgpu_mem_wr32(g, mem, ram_fc_acquire_w(),
-		g->ops.fifo.pbdma_acquire_val(acquire_timeout));
+		g->ops.pbdma.pbdma_acquire_val(acquire_timeout));
 
 	nvgpu_mem_wr32(g, mem, ram_fc_runlist_timeslice_w(),
 		pbdma_runlist_timeslice_timeout_128_f() |
@@ -1533,155 +1533,6 @@ bool gv11b_fifo_handle_ctxsw_timeout(struct gk20a *g, u32 fifo_intr)
 	return ret;
 }
 
-static void report_pbdma_error(struct gk20a *g, u32 pbdma_id,
-		u32 pbdma_intr_0)
-{
-	u32 err_type = GPU_HOST_INVALID_ERROR;
-
-	/*
-	 * Multiple errors have been grouped as part of a single
-	 * top-level error.
-	 */
-	if ((pbdma_intr_0 & (
-		pbdma_intr_0_memreq_pending_f() |
-		pbdma_intr_0_memack_timeout_pending_f() |
-		pbdma_intr_0_memdat_timeout_pending_f() |
-		pbdma_intr_0_memflush_pending_f() |
-		pbdma_intr_0_memop_pending_f() |
-		pbdma_intr_0_lbconnect_pending_f() |
-		pbdma_intr_0_lback_timeout_pending_f() |
-		pbdma_intr_0_lbdat_timeout_pending_f())) != 0U) {
-			err_type = GPU_HOST_PBDMA_TIMEOUT_ERROR;
-	}
-	if ((pbdma_intr_0 & (
-		pbdma_intr_0_memack_extra_pending_f() |
-		pbdma_intr_0_memdat_extra_pending_f() |
-		pbdma_intr_0_lback_extra_pending_f() |
-		pbdma_intr_0_lbdat_extra_pending_f())) != 0U) {
-			err_type = GPU_HOST_PBDMA_EXTRA_ERROR;
-	}
-	if ((pbdma_intr_0 & (
-		pbdma_intr_0_gpfifo_pending_f() |
-		pbdma_intr_0_gpptr_pending_f() |
-		pbdma_intr_0_gpentry_pending_f() |
-		pbdma_intr_0_gpcrc_pending_f() |
-		pbdma_intr_0_pbptr_pending_f() |
-		pbdma_intr_0_pbentry_pending_f() |
-		pbdma_intr_0_pbcrc_pending_f())) != 0U) {
-			err_type = GPU_HOST_PBDMA_GPFIFO_PB_ERROR;
-	}
-	if ((pbdma_intr_0 & (
-		pbdma_intr_0_clear_faulted_error_pending_f() |
-		pbdma_intr_0_method_pending_f() |
-		pbdma_intr_0_methodcrc_pending_f() |
-		pbdma_intr_0_device_pending_f() |
-		pbdma_intr_0_eng_reset_pending_f() |
-		pbdma_intr_0_semaphore_pending_f() |
-		pbdma_intr_0_acquire_pending_f() |
-		pbdma_intr_0_pri_pending_f() |
-		pbdma_intr_0_pbseg_pending_f())) != 0U) {
-			err_type = GPU_HOST_PBDMA_METHOD_ERROR;
-	}
-	if ((pbdma_intr_0 &
-		pbdma_intr_0_signature_pending_f()) != 0U) {
-			err_type = GPU_HOST_PBDMA_SIGNATURE_ERROR;
-	}
-	if (err_type != GPU_HOST_INVALID_ERROR) {
-		nvgpu_report_host_error(g, pbdma_id,
-				err_type, pbdma_intr_0);
-	}
-	return;
-}
-
-unsigned int gv11b_fifo_handle_pbdma_intr_0(struct gk20a *g,
-			u32 pbdma_id, u32 pbdma_intr_0,
-			u32 *handled, u32 *error_notifier)
-{
-	unsigned int rc_type = RC_TYPE_NO_RC;
-
-	rc_type = gk20a_fifo_handle_pbdma_intr_0(g, pbdma_id,
-			 pbdma_intr_0, handled, error_notifier);
-
-	if ((pbdma_intr_0 & pbdma_intr_0_clear_faulted_error_pending_f()) != 0U) {
-		nvgpu_log(g, gpu_dbg_intr, "clear faulted error on pbdma id %d",
-				 pbdma_id);
-		gk20a_fifo_reset_pbdma_method(g, pbdma_id, 0);
-		*handled |= pbdma_intr_0_clear_faulted_error_pending_f();
-		rc_type = RC_TYPE_PBDMA_FAULT;
-	}
-
-	if ((pbdma_intr_0 & pbdma_intr_0_eng_reset_pending_f()) != 0U) {
-		nvgpu_log(g, gpu_dbg_intr, "eng reset intr on pbdma id %d",
-				 pbdma_id);
-		*handled |= pbdma_intr_0_eng_reset_pending_f();
-		rc_type = RC_TYPE_PBDMA_FAULT;
-	}
-	report_pbdma_error(g, pbdma_id, pbdma_intr_0);
-	return rc_type;
-}
-
-/*
- * Pbdma which encountered the ctxnotvalid interrupt will stall and
- * prevent the channel which was loaded at the time the interrupt fired
- * from being swapped out until the interrupt is cleared.
- * CTXNOTVALID pbdma interrupt indicates error conditions related
- * to the *_CTX_VALID fields for a channel.  The following
- * conditions trigger the interrupt:
- * * CTX_VALID bit for the targeted engine is FALSE
- * * At channel start/resume, all preemptible eng have CTX_VALID FALSE but:
- *       - CTX_RELOAD is set in CCSR_CHANNEL_STATUS,
- *       - PBDMA_TARGET_SHOULD_SEND_HOST_TSG_EVENT is TRUE, or
- *       - PBDMA_TARGET_NEEDS_HOST_TSG_EVENT is TRUE
- * The field is left NOT_PENDING and the interrupt is not raised if the PBDMA is
- * currently halted.  This allows SW to unblock the PBDMA and recover.
- * SW may read METHOD0, CHANNEL_STATUS and TARGET to determine whether the
- * interrupt was due to an engine method, CTX_RELOAD, SHOULD_SEND_HOST_TSG_EVENT
- * or NEEDS_HOST_TSG_EVENT.  If METHOD0 VALID is TRUE, lazy context creation
- * can be used or the TSG may be destroyed.
- * If METHOD0 VALID is FALSE, the error is likely a bug in SW, and the TSG
- * will have to be destroyed.
- */
-
-unsigned int gv11b_fifo_handle_pbdma_intr_1(struct gk20a *g,
-			u32 pbdma_id, u32 pbdma_intr_1,
-			u32 *handled, u32 *error_notifier)
-{
-	unsigned int rc_type = RC_TYPE_PBDMA_FAULT;
-	u32 pbdma_intr_1_current = gk20a_readl(g, pbdma_intr_1_r(pbdma_id));
-
-	/* minimize race with the gpu clearing the pending interrupt */
-	if ((pbdma_intr_1_current &
-	     pbdma_intr_1_ctxnotvalid_pending_f()) == 0U) {
-		pbdma_intr_1 &= ~pbdma_intr_1_ctxnotvalid_pending_f();
-	}
-
-	if (pbdma_intr_1 == 0U) {
-		return RC_TYPE_NO_RC;
-	}
-
-	nvgpu_report_host_error(g, pbdma_id,
-			GPU_HOST_PBDMA_HCE_ERROR, pbdma_intr_1);
-
-	if ((pbdma_intr_1 & pbdma_intr_1_ctxnotvalid_pending_f()) != 0U) {
-		nvgpu_log(g, gpu_dbg_intr, "ctxnotvalid intr on pbdma id %d",
-				 pbdma_id);
-		nvgpu_err(g, "pbdma_intr_1(%d)= 0x%08x ",
-				pbdma_id, pbdma_intr_1);
-		*handled |= pbdma_intr_1_ctxnotvalid_pending_f();
-	} else{
-		/*
-		 * rest of the interrupts in _intr_1 are "host copy engine"
-		 * related, which is not supported. For now just make them
-		 * channel fatal.
-		 */
-		nvgpu_err(g, "hce err: pbdma_intr_1(%d):0x%08x",
-			pbdma_id, pbdma_intr_1);
-		*handled |= pbdma_intr_1;
-	}
-
-	return rc_type;
-}
-
 void gv11b_fifo_init_ramfc_eng_method_buffer(struct gk20a *g,
 			struct channel_gk20a *ch, struct nvgpu_mem *mem)
 {

drivers/gpu/nvgpu/gv11b/fifo_gv11b.h

@@ -87,12 +87,6 @@ void gv11b_fifo_init_pbdma_intr_descs(struct fifo_gk20a *f);
 int gv11b_init_fifo_reset_enable_hw(struct gk20a *g);
 bool gv11b_fifo_handle_sched_error(struct gk20a *g);
 bool gv11b_fifo_handle_ctxsw_timeout(struct gk20a *g, u32 fifo_intr);
-unsigned int gv11b_fifo_handle_pbdma_intr_0(struct gk20a *g,
-			u32 pbdma_id, u32 pbdma_intr_0,
-			u32 *handled, u32 *error_notifier);
-unsigned int gv11b_fifo_handle_pbdma_intr_1(struct gk20a *g,
-			u32 pbdma_id, u32 pbdma_intr_1,
-			u32 *handled, u32 *error_notifier);
 void gv11b_fifo_init_eng_method_buffers(struct gk20a *g,
 					struct tsg_gk20a *tsg);
 void gv11b_fifo_deinit_eng_method_buffers(struct gk20a *g,

drivers/gpu/nvgpu/gv11b/hal_gv11b.c

@@ -45,6 +45,9 @@
 #include "hal/fb/fb_gv11b.h"
 #include "hal/fuse/fuse_gm20b.h"
 #include "hal/fuse/fuse_gp10b.h"
+#include "hal/fifo/pbdma_gm20b.h"
+#include "hal/fifo/pbdma_gp10b.h"
+#include "hal/fifo/pbdma_gv11b.h"
 #include "hal/fifo/engine_status_gv100.h"
 #include "hal/fifo/pbdma_status_gm20b.h"
 #include "hal/fifo/engines_gv11b.h"
@@ -715,7 +718,6 @@ static const struct gpu_ops gv11b_ops = {
 		.userd_gp_put = gv11b_userd_gp_put,
 		.userd_pb_get = gv11b_userd_pb_get,
 		.userd_entry_size = gk20a_fifo_userd_entry_size,
-		.pbdma_acquire_val = gk20a_fifo_pbdma_acquire_val,
 		.preempt_channel = gv11b_fifo_preempt_channel,
 		.preempt_tsg = gv11b_fifo_preempt_tsg,
 		.enable_tsg = gv11b_fifo_enable_tsg,
@@ -728,13 +730,11 @@ static const struct gpu_ops gv11b_ops = {
 		.get_mmu_fault_desc = NULL,
 		.get_mmu_fault_client_desc = NULL,
 		.get_mmu_fault_gpc_desc = NULL,
-		.get_pbdma_signature = gp10b_fifo_get_pbdma_signature,
 		.tsg_set_timeslice = gk20a_fifo_tsg_set_timeslice,
 		.force_reset_ch = gk20a_fifo_force_reset_ch,
 		.init_engine_info = gm20b_fifo_init_engine_info,
 		.init_pbdma_info = gk20a_fifo_init_pbdma_info,
 		.get_engines_mask_on_id = gk20a_fifo_engines_on_id,
-		.dump_pbdma_status = gk20a_dump_pbdma_status,
 		.dump_channel_status_ramfc = gv11b_dump_channel_status_ramfc,
 		.capture_channel_ram_dump = gv11b_capture_channel_ram_dump,
 		.intr_0_error_mask = gv11b_fifo_intr_0_error_mask,
@@ -745,8 +745,6 @@ static const struct gpu_ops gv11b_ops = {
 		.teardown_mask_intr = gv11b_fifo_teardown_mask_intr,
 		.teardown_unmask_intr = gv11b_fifo_teardown_unmask_intr,
 		.handle_sched_error = gv11b_fifo_handle_sched_error,
-		.handle_pbdma_intr_0 = gv11b_fifo_handle_pbdma_intr_0,
-		.handle_pbdma_intr_1 = gv11b_fifo_handle_pbdma_intr_1,
 		.init_eng_method_buffers = gv11b_fifo_init_eng_method_buffers,
 		.deinit_eng_method_buffers =
 			gv11b_fifo_deinit_eng_method_buffers,
@@ -771,12 +769,19 @@ static const struct gpu_ops gv11b_ops = {
 		.runlist_busy_engines = gk20a_fifo_runlist_busy_engines,
 		.find_pbdma_for_runlist = gk20a_fifo_find_pbdma_for_runlist,
 		.init_ce_engine_info = gp10b_fifo_init_ce_engine_info,
-		.read_pbdma_data = gk20a_fifo_read_pbdma_data,
-		.reset_pbdma_header = gk20a_fifo_reset_pbdma_header,
 	},
 	.engine = {
 		.is_fault_engine_subid_gpc = gv11b_is_fault_engine_subid_gpc,
 	},
+	.pbdma = {
+		.pbdma_acquire_val = gm20b_pbdma_acquire_val,
+		.get_pbdma_signature = gp10b_pbdma_get_signature,
+		.dump_pbdma_status = gm20b_pbdma_dump_status,
+		.handle_pbdma_intr_0 = gv11b_pbdma_handle_intr_0,
+		.handle_pbdma_intr_1 = gv11b_pbdma_handle_intr_1,
+		.read_pbdma_data = gm20b_pbdma_read_data,
+		.reset_pbdma_header = gm20b_pbdma_reset_header,
+	},
 	.sync = {
 #ifdef CONFIG_TEGRA_GK20A_NVHOST
 		.alloc_syncpt_buf = gv11b_alloc_syncpt_buf,
@@ -1118,6 +1123,7 @@ int gv11b_init_hal(struct gk20a *g)
 	gops->clock_gating = gv11b_ops.clock_gating;
 	gops->fifo = gv11b_ops.fifo;
 	gops->engine = gv11b_ops.engine;
+	gops->pbdma = gv11b_ops.pbdma;
 	gops->runlist = gv11b_ops.runlist;
 	gops->channel = gv11b_ops.channel;
 	gops->sync = gv11b_ops.sync;

drivers/gpu/nvgpu/hal/fifo/pbdma_gm20b.c

@@ -0,0 +1,307 @@
+/*
+ * Copyright (c) 2014-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/log.h>
+#include <nvgpu/io.h>
+#include <nvgpu/bitops.h>
+#include <nvgpu/bug.h>
+#include <nvgpu/debug.h>
+#include <nvgpu/error_notifier.h>
+#include <nvgpu/fifo.h>
+#include <nvgpu/gk20a.h>
+#include <nvgpu/pbdma_status.h>
+
+#include <nvgpu/hw/gm20b/hw_pbdma_gm20b.h>
+
+#include "pbdma_gm20b.h"
+
+static const char *const pbdma_intr_fault_type_desc[] = {
+	"MEMREQ timeout", "MEMACK_TIMEOUT", "MEMACK_EXTRA acks",
+	"MEMDAT_TIMEOUT", "MEMDAT_EXTRA acks", "MEMFLUSH noack",
+	"MEMOP noack", "LBCONNECT noack", "NONE - was LBREQ",
+	"LBACK_TIMEOUT", "LBACK_EXTRA acks", "LBDAT_TIMEOUT",
+	"LBDAT_EXTRA acks", "GPFIFO won't fit", "GPPTR invalid",
+	"GPENTRY invalid", "GPCRC mismatch", "PBPTR get>put",
+	"PBENTRY invld", "PBCRC mismatch", "NONE - was XBARC",
+	"METHOD invld", "METHODCRC mismat", "DEVICE sw method",
+	"[ENGINE]", "SEMAPHORE invlid", "ACQUIRE timeout",
+	"PRI forbidden", "ILLEGAL SYNCPT", "[NO_CTXSW_SEG]",
+	"PBSEG badsplit", "SIGNATURE bad"
+};
+
+static bool gm20b_pbdma_is_sw_method_subch(struct gk20a *g, u32 pbdma_id,
+						u32 pbdma_method_index)
+{
+	u32 pbdma_method_stride;
+	u32 pbdma_method_reg, pbdma_method_subch;
+
+	pbdma_method_stride = pbdma_method1_r(pbdma_id) -
+				pbdma_method0_r(pbdma_id);
+
+	pbdma_method_reg = pbdma_method0_r(pbdma_id) +
+			(pbdma_method_index * pbdma_method_stride);
+
+	pbdma_method_subch = pbdma_method0_subch_v(
+			nvgpu_readl(g, pbdma_method_reg));
+
+	if (pbdma_method_subch == 5U ||
+	    pbdma_method_subch == 6U ||
+	    pbdma_method_subch == 7U) {
+		return true;
+	}
+
+	return false;
+}
+
+unsigned int gm20b_pbdma_handle_intr_0(struct gk20a *g, u32 pbdma_id,
+			u32 pbdma_intr_0, u32 *handled, u32 *error_notifier)
+{
+	struct fifo_gk20a *f = &g->fifo;
+	unsigned int rc_type = RC_TYPE_NO_RC;
+	u32 i;
+	unsigned long pbdma_intr_err;
+	unsigned long bit;
+
+	if (((f->intr.pbdma.device_fatal_0 |
+	      f->intr.pbdma.channel_fatal_0 |
+	      f->intr.pbdma.restartable_0) & pbdma_intr_0) != 0U) {
+
+		pbdma_intr_err = (unsigned long)pbdma_intr_0;
+		for_each_set_bit(bit, &pbdma_intr_err, 32U) {
+			nvgpu_err(g, "PBDMA intr %s Error",
+				pbdma_intr_fault_type_desc[bit]);
+		}
+
+		nvgpu_err(g,
+			"pbdma_intr_0(%d):0x%08x PBH: %08x "
+			"SHADOW: %08x gp shadow0: %08x gp shadow1: %08x"
+			"M0: %08x %08x %08x %08x ",
+			pbdma_id, pbdma_intr_0,
+			nvgpu_readl(g, pbdma_pb_header_r(pbdma_id)),
+			g->ops.pbdma.read_pbdma_data(g, pbdma_id),
+			nvgpu_readl(g, pbdma_gp_shadow_0_r(pbdma_id)),
+			nvgpu_readl(g, pbdma_gp_shadow_1_r(pbdma_id)),
+			nvgpu_readl(g, pbdma_method0_r(pbdma_id)),
+			nvgpu_readl(g, pbdma_method1_r(pbdma_id)),
+			nvgpu_readl(g, pbdma_method2_r(pbdma_id)),
+			nvgpu_readl(g, pbdma_method3_r(pbdma_id))
+			);
+
+		rc_type = RC_TYPE_PBDMA_FAULT;
+		*handled |= ((f->intr.pbdma.device_fatal_0 |
+			     f->intr.pbdma.channel_fatal_0 |
+			     f->intr.pbdma.restartable_0) &
+			    pbdma_intr_0);
+	}
+
+	if ((pbdma_intr_0 & pbdma_intr_0_acquire_pending_f()) != 0U) {
+		u32 val = nvgpu_readl(g, pbdma_acquire_r(pbdma_id));
+
+		val &= ~pbdma_acquire_timeout_en_enable_f();
+		nvgpu_writel(g, pbdma_acquire_r(pbdma_id), val);
+		if (nvgpu_is_timeouts_enabled(g)) {
+			rc_type = RC_TYPE_PBDMA_FAULT;
+			nvgpu_err(g,
+				"semaphore acquire timeout!");
+			*error_notifier = NVGPU_ERR_NOTIFIER_GR_SEMAPHORE_TIMEOUT;
+		}
+		*handled |= pbdma_intr_0_acquire_pending_f();
+	}
+
+	if ((pbdma_intr_0 & pbdma_intr_0_pbentry_pending_f()) != 0U) {
+		g->ops.pbdma.reset_pbdma_header(g, pbdma_id);
+		gm20b_pbdma_reset_method(g, pbdma_id, 0);
+		rc_type = RC_TYPE_PBDMA_FAULT;
+	}
+
+	if ((pbdma_intr_0 & pbdma_intr_0_method_pending_f()) != 0U) {
+		gm20b_pbdma_reset_method(g, pbdma_id, 0);
+		rc_type = RC_TYPE_PBDMA_FAULT;
+	}
+
+	if ((pbdma_intr_0 & pbdma_intr_0_pbcrc_pending_f()) != 0U) {
+		*error_notifier =
+			NVGPU_ERR_NOTIFIER_PBDMA_PUSHBUFFER_CRC_MISMATCH;
+		rc_type = RC_TYPE_PBDMA_FAULT;
+	}
+
+	if ((pbdma_intr_0 & pbdma_intr_0_device_pending_f()) != 0U) {
+		g->ops.pbdma.reset_pbdma_header(g, pbdma_id);
+
+		for (i = 0U; i < 4U; i++) {
+			if (gm20b_pbdma_is_sw_method_subch(g,
+					pbdma_id, i)) {
+				gm20b_pbdma_reset_method(g,
+						pbdma_id, i);
+			}
+		}
+		rc_type = RC_TYPE_PBDMA_FAULT;
+	}
+
+	return rc_type;
+}
+
+unsigned int gm20b_pbdma_handle_intr_1(struct gk20a *g,
+			u32 pbdma_id, u32 pbdma_intr_1,
+			u32 *handled, u32 *error_notifier)
+{
+	unsigned int rc_type = RC_TYPE_PBDMA_FAULT;
+
+	/*
+	 * all of the interrupts in _intr_1 are "host copy engine"
+	 * related, which is not supported. For now just make them
+	 * channel fatal.
+	 */
+	nvgpu_err(g, "hce err: pbdma_intr_1(%d):0x%08x",
+		pbdma_id, pbdma_intr_1);
+	*handled |= pbdma_intr_1;
+
+	return rc_type;
+}
+
+void gm20b_pbdma_reset_header(struct gk20a *g, u32 pbdma_id)
+{
+	nvgpu_writel(g, pbdma_pb_header_r(pbdma_id),
+			pbdma_pb_header_first_true_f() |
+			pbdma_pb_header_type_non_inc_f());
+}
+
+void gm20b_pbdma_reset_method(struct gk20a *g, u32 pbdma_id,
+			u32 pbdma_method_index)
+{
+	u32 pbdma_method_stride;
+	u32 pbdma_method_reg;
+
+	pbdma_method_stride = pbdma_method1_r(pbdma_id) -
+				pbdma_method0_r(pbdma_id);
+
+	pbdma_method_reg = pbdma_method0_r(pbdma_id) +
+		(pbdma_method_index * pbdma_method_stride);
+
+	nvgpu_writel(g, pbdma_method_reg,
+			pbdma_method0_valid_true_f() |
+			pbdma_method0_first_true_f() |
+			pbdma_method0_addr_f(
+			     pbdma_udma_nop_r() >> 2));
+}
+
+u32 gm20b_pbdma_get_signature(struct gk20a *g)
+{
+	return pbdma_signature_hw_valid_f() | pbdma_signature_sw_zero_f();
+}
+
+u32 gm20b_pbdma_acquire_val(u64 timeout)
+{
+	u32 val, exponent, mantissa;
+	unsigned int val_len;
+	u64 tmp;
+
+	val = pbdma_acquire_retry_man_2_f() |
+		pbdma_acquire_retry_exp_2_f();
+
+	if (timeout == 0ULL) {
+		return val;
+	}
+
+	timeout *= 80UL;
+	do_div(timeout, 100U); /* set acquire timeout to 80% of channel wdt */
+	timeout *= 1000000UL; /* ms -> ns */
+	do_div(timeout, 1024U); /* in unit of 1024ns */
+	tmp = fls(timeout >> 32U);
+	BUG_ON(tmp > U64(U32_MAX));
+	val_len = (u32)tmp + 32U;
+	if (val_len == 32U) {
+		val_len = (u32)fls(timeout);
+	}
+	if (val_len > 16U + pbdma_acquire_timeout_exp_max_v()) { /* man: 16bits */
+		exponent = pbdma_acquire_timeout_exp_max_v();
+		mantissa = pbdma_acquire_timeout_man_max_v();
+	} else if (val_len > 16U) {
+		exponent = val_len - 16U;
+		BUG_ON((timeout >> exponent) > U64(U32_MAX));
+		mantissa = (u32)(timeout >> exponent);
+	} else {
+		exponent = 0;
+		BUG_ON(timeout > U64(U32_MAX));
+		mantissa = (u32)timeout;
+	}
+
+	val |= pbdma_acquire_timeout_exp_f(exponent) |
+		pbdma_acquire_timeout_man_f(mantissa) |
+		pbdma_acquire_timeout_en_enable_f();
+
+	return val;
+}
+
+void gm20b_pbdma_dump_status(struct gk20a *g, struct gk20a_debug_output *o)
+{
+	u32 i, host_num_pbdma;
+	struct nvgpu_pbdma_status_info pbdma_status;
+
+	host_num_pbdma = nvgpu_get_litter_value(g, GPU_LIT_HOST_NUM_PBDMA);
+
+	gk20a_debug_output(o, "PBDMA Status - chip %-5s", g->name);
+	gk20a_debug_output(o, "-------------------------");
+
+	for (i = 0; i < host_num_pbdma; i++) {
+		g->ops.pbdma_status.read_pbdma_status_info(g, i,
+			&pbdma_status);
+
+		gk20a_debug_output(o, "pbdma %d:", i);
+		gk20a_debug_output(o,
+			"  id: %d - %-9s next_id: - %d %-9s | status: %s",
+			pbdma_status.id,
+			nvgpu_pbdma_status_is_id_type_tsg(&pbdma_status) ?
+				   "[tsg]" : "[channel]",
+			pbdma_status.next_id,
+			nvgpu_pbdma_status_is_next_id_type_tsg(
+				&pbdma_status) ?
+				   "[tsg]" : "[channel]",
+			gk20a_decode_pbdma_chan_eng_ctx_status(
+				pbdma_status.pbdma_channel_status));
+		gk20a_debug_output(o,
+			"  PBDMA_PUT %016llx PBDMA_GET %016llx",
+			(u64)nvgpu_readl(g, pbdma_put_r(i)) +
+			((u64)nvgpu_readl(g, pbdma_put_hi_r(i)) << 32ULL),
+			(u64)nvgpu_readl(g, pbdma_get_r(i)) +
+			((u64)nvgpu_readl(g, pbdma_get_hi_r(i)) << 32ULL));
+		gk20a_debug_output(o,
+			"  GP_PUT    %08x  GP_GET  %08x  "
+			"FETCH   %08x HEADER %08x",
+			nvgpu_readl(g, pbdma_gp_put_r(i)),
+			nvgpu_readl(g, pbdma_gp_get_r(i)),
+			nvgpu_readl(g, pbdma_gp_fetch_r(i)),
+			nvgpu_readl(g, pbdma_pb_header_r(i)));
+		gk20a_debug_output(o,
+			"  HDR       %08x  SHADOW0 %08x  SHADOW1 %08x",
+			nvgpu_readl(g, pbdma_hdr_shadow_r(i)),
+			nvgpu_readl(g, pbdma_gp_shadow_0_r(i)),
+			nvgpu_readl(g, pbdma_gp_shadow_1_r(i)));
+	}
+
+	gk20a_debug_output(o, " ");
+}
+
+u32 gm20b_pbdma_read_data(struct gk20a *g, u32 pbdma_id)
+{
+	return nvgpu_readl(g, pbdma_hdr_shadow_r(pbdma_id));
+}

drivers/gpu/nvgpu/hal/fifo/pbdma_gm20b.h

@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2014-2018, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef NVGPU_PBDMA_GM20B_H
+#define NVGPU_PBDMA_GM20B_H
+
+#include <nvgpu/types.h>
+
+struct gk20a;
+struct gk20a_debug_output;
+
+unsigned int gm20b_pbdma_handle_intr_0(struct gk20a *g, u32 pbdma_id,
+			u32 pbdma_intr_0, u32 *handled, u32 *error_notifier);
+unsigned int gm20b_pbdma_handle_intr_1(struct gk20a *g, u32 pbdma_id,
+			u32 pbdma_intr_1, u32 *handled, u32 *error_notifier);
+u32 gm20b_pbdma_get_signature(struct gk20a *g);
+u32 gm20b_pbdma_read_data(struct gk20a *g, u32 pbdma_id);
+void gm20b_pbdma_reset_header(struct gk20a *g, u32 pbdma_id);
+void gm20b_pbdma_reset_method(struct gk20a *g, u32 pbdma_id,
+			u32 pbdma_method_index);
+u32 gm20b_pbdma_acquire_val(u64 timeout);
+void gm20b_pbdma_dump_status(struct gk20a *g, struct gk20a_debug_output *o);
+
+#endif /* NVGPU_PBDMA_GM20B_H */

drivers/gpu/nvgpu/hal/fifo/pbdma_gp10b.c

@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2016-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/types.h>
+#include <nvgpu/gk20a.h>
+
+#include <nvgpu/hw/gp10b/hw_pbdma_gp10b.h>
+
+#include "pbdma_gp10b.h"
+
+u32 gp10b_pbdma_get_signature(struct gk20a *g)
+{
+	return g->ops.get_litter_value(g, GPU_LIT_GPFIFO_CLASS)
+		| pbdma_signature_sw_zero_f();
+}

drivers/gpu/nvgpu/hal/fifo/pbdma_gp10b.h

@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2016-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef NVGPU_PBDMA_GP10B_H
+#define NVGPU_PBDMA_GP10B_H
+
+#include <nvgpu/types.h>
+
+struct gk20a;
+
+u32 gp10b_pbdma_get_signature(struct gk20a *g);
+
+#endif /* NVGPU_PBDMA_GP10B_H */

drivers/gpu/nvgpu/hal/fifo/pbdma_gv11b.c

@@ -0,0 +1,180 @@
+/*
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/log.h>
+#include <nvgpu/io.h>
+#include <nvgpu/nvgpu_err.h>
+#include <nvgpu/fifo.h>
+
+#include <nvgpu/hw/gv11b/hw_pbdma_gv11b.h>
+
+#include "pbdma_gm20b.h"
+#include "pbdma_gv11b.h"
+
+static void report_pbdma_error(struct gk20a *g, u32 pbdma_id,
+		u32 pbdma_intr_0)
+{
+	u32 err_type = GPU_HOST_INVALID_ERROR;
+
+	/*
+	 * Multiple errors have been grouped as part of a single
+	 * top-level error.
+	 */
+	if ((pbdma_intr_0 & (
+		pbdma_intr_0_memreq_pending_f() |
+		pbdma_intr_0_memack_timeout_pending_f() |
+		pbdma_intr_0_memdat_timeout_pending_f() |
+		pbdma_intr_0_memflush_pending_f() |
+		pbdma_intr_0_memop_pending_f() |
+		pbdma_intr_0_lbconnect_pending_f() |
+		pbdma_intr_0_lback_timeout_pending_f() |
+		pbdma_intr_0_lbdat_timeout_pending_f())) != 0U) {
+			err_type = GPU_HOST_PBDMA_TIMEOUT_ERROR;
+	}
+	if ((pbdma_intr_0 & (
+		pbdma_intr_0_memack_extra_pending_f() |
+		pbdma_intr_0_memdat_extra_pending_f() |
+		pbdma_intr_0_lback_extra_pending_f() |
+		pbdma_intr_0_lbdat_extra_pending_f())) != 0U) {
+			err_type = GPU_HOST_PBDMA_EXTRA_ERROR;
+	}
+	if ((pbdma_intr_0 & (
+		pbdma_intr_0_gpfifo_pending_f() |
+		pbdma_intr_0_gpptr_pending_f() |
+		pbdma_intr_0_gpentry_pending_f() |
+		pbdma_intr_0_gpcrc_pending_f() |
+		pbdma_intr_0_pbptr_pending_f() |
+		pbdma_intr_0_pbentry_pending_f() |
+		pbdma_intr_0_pbcrc_pending_f())) != 0U) {
+			err_type = GPU_HOST_PBDMA_GPFIFO_PB_ERROR;
+	}
+	if ((pbdma_intr_0 & (
+		pbdma_intr_0_clear_faulted_error_pending_f() |
+		pbdma_intr_0_method_pending_f() |
+		pbdma_intr_0_methodcrc_pending_f() |
+		pbdma_intr_0_device_pending_f() |
+		pbdma_intr_0_eng_reset_pending_f() |
+		pbdma_intr_0_semaphore_pending_f() |
+		pbdma_intr_0_acquire_pending_f() |
+		pbdma_intr_0_pri_pending_f() |
+		pbdma_intr_0_pbseg_pending_f())) != 0U) {
+			err_type = GPU_HOST_PBDMA_METHOD_ERROR;
+	}
+	if ((pbdma_intr_0 &
+		pbdma_intr_0_signature_pending_f()) != 0U) {
+			err_type = GPU_HOST_PBDMA_SIGNATURE_ERROR;
+	}
+	if (err_type != GPU_HOST_INVALID_ERROR) {
+		nvgpu_report_host_error(g, pbdma_id,
+				err_type, pbdma_intr_0);
+	}
+	return;
+}
+
+unsigned int gv11b_pbdma_handle_intr_0(struct gk20a *g,
+			u32 pbdma_id, u32 pbdma_intr_0,
+			u32 *handled, u32 *error_notifier)
+{
+	unsigned int rc_type = RC_TYPE_NO_RC;
+
+	rc_type = gm20b_pbdma_handle_intr_0(g, pbdma_id,
+			 pbdma_intr_0, handled, error_notifier);
+
+	if ((pbdma_intr_0 & pbdma_intr_0_clear_faulted_error_pending_f()) != 0U) {
+		nvgpu_log(g, gpu_dbg_intr, "clear faulted error on pbdma id %d",
+				 pbdma_id);
+		gm20b_pbdma_reset_method(g, pbdma_id, 0);
+		*handled |= pbdma_intr_0_clear_faulted_error_pending_f();
+		rc_type = RC_TYPE_PBDMA_FAULT;
+	}
+
+	if ((pbdma_intr_0 & pbdma_intr_0_eng_reset_pending_f()) != 0U) {
+		nvgpu_log(g, gpu_dbg_intr, "eng reset intr on pbdma id %d",
+				 pbdma_id);
+		*handled |= pbdma_intr_0_eng_reset_pending_f();
+		rc_type = RC_TYPE_PBDMA_FAULT;
+	}
+	report_pbdma_error(g, pbdma_id, pbdma_intr_0);
+	return rc_type;
+}
+
+/*
+ * Pbdma which encountered the ctxnotvalid interrupt will stall and
+ * prevent the channel which was loaded at the time the interrupt fired
+ * from being swapped out until the interrupt is cleared.
+ * CTXNOTVALID pbdma interrupt indicates error conditions related
+ * to the *_CTX_VALID fields for a channel.  The following
+ * conditions trigger the interrupt:
+ * * CTX_VALID bit for the targeted engine is FALSE
+ * * At channel start/resume, all preemptible eng have CTX_VALID FALSE but:
+ *       - CTX_RELOAD is set in CCSR_CHANNEL_STATUS,
+ *       - PBDMA_TARGET_SHOULD_SEND_HOST_TSG_EVENT is TRUE, or
+ *       - PBDMA_TARGET_NEEDS_HOST_TSG_EVENT is TRUE
+ * The field is left NOT_PENDING and the interrupt is not raised if the PBDMA is
+ * currently halted.  This allows SW to unblock the PBDMA and recover.
+ * SW may read METHOD0, CHANNEL_STATUS and TARGET to determine whether the
+ * interrupt was due to an engine method, CTX_RELOAD, SHOULD_SEND_HOST_TSG_EVENT
+ * or NEEDS_HOST_TSG_EVENT.  If METHOD0 VALID is TRUE, lazy context creation
+ * can be used or the TSG may be destroyed.
+ * If METHOD0 VALID is FALSE, the error is likely a bug in SW, and the TSG
+ * will have to be destroyed.
+ */
+
+unsigned int gv11b_pbdma_handle_intr_1(struct gk20a *g,
+			u32 pbdma_id, u32 pbdma_intr_1,
+			u32 *handled, u32 *error_notifier)
+{
+	unsigned int rc_type = RC_TYPE_PBDMA_FAULT;
+	u32 pbdma_intr_1_current = gk20a_readl(g, pbdma_intr_1_r(pbdma_id));
+
+	/* minimize race with the gpu clearing the pending interrupt */
+	if ((pbdma_intr_1_current &
+	     pbdma_intr_1_ctxnotvalid_pending_f()) == 0U) {
+		pbdma_intr_1 &= ~pbdma_intr_1_ctxnotvalid_pending_f();
+	}
+
+	if (pbdma_intr_1 == 0U) {
+		return RC_TYPE_NO_RC;
+	}
+
+	nvgpu_report_host_error(g, pbdma_id,
+			GPU_HOST_PBDMA_HCE_ERROR, pbdma_intr_1);
+
+	if ((pbdma_intr_1 & pbdma_intr_1_ctxnotvalid_pending_f()) != 0U) {
+		nvgpu_log(g, gpu_dbg_intr, "ctxnotvalid intr on pbdma id %d",
+				 pbdma_id);
+		nvgpu_err(g, "pbdma_intr_1(%d)= 0x%08x ",
+				pbdma_id, pbdma_intr_1);
+		*handled |= pbdma_intr_1_ctxnotvalid_pending_f();
+	} else{
+		/*
+		 * rest of the interrupts in _intr_1 are "host copy engine"
+		 * related, which is not supported. For now just make them
+		 * channel fatal.
+		 */
+		nvgpu_err(g, "hce err: pbdma_intr_1(%d):0x%08x",
+			pbdma_id, pbdma_intr_1);
+		*handled |= pbdma_intr_1;
+	}
+
+	return rc_type;
+}

drivers/gpu/nvgpu/hal/fifo/pbdma_gv11b.h

@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2017-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef NVGPU_PBDMA_GV11B_H
+#define NVGPU_PBDMA_GV11B_H
+
+#include <nvgpu/types.h>
+
+struct gk20a;
+
+unsigned int gv11b_pbdma_handle_intr_0(struct gk20a *g,
+			u32 pbdma_id, u32 pbdma_intr_0,
+			u32 *handled, u32 *error_notifier);
+unsigned int gv11b_pbdma_handle_intr_1(struct gk20a *g,
+			u32 pbdma_id, u32 pbdma_intr_1,
+			u32 *handled, u32 *error_notifier);
+
+#endif /* NVGPU_PBDMA_GV11B_H */

drivers/gpu/nvgpu/hal/fifo/pbdma_tu104.c

@@ -0,0 +1,66 @@
+/*
+ * Copyright (c) 2018-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <nvgpu/io.h>
+#include <nvgpu/gk20a.h>
+
+#include <nvgpu/hw/tu104/hw_pbdma_tu104.h>
+
+#include "pbdma_gm20b.h"
+#include "pbdma_tu104.h"
+
+void tu104_pbdma_reset_header(struct gk20a *g, u32 pbdma_id)
+{
+	gm20b_pbdma_reset_header(g, pbdma_id);
+	nvgpu_writel(g, pbdma_data0_r(pbdma_id), 0);
+}
+
+u32 tu104_pbdma_read_data(struct gk20a *g, u32 pbdma_id)
+{
+	u32 pb_inst;
+	u32 pb_header, pb_header_type;
+	u32 pb_count;
+
+	/*
+	 * In order to determine the location of the PB entry that cause the
+	 * interrupt, NV_PPBDMA_PB_HEADER and NV_PPBDMA_PB_COUNT need to be
+	 * checked. If the TYPE field of the NV_PPBDMA_PB_HEADER is IMMD or the
+	 * VALUE field of the NV_PPBDMA_PB_COUNT is zero, then the raw PB
+	 * instruction stored in NV_PPBDMA_PB_DATA0 is the one that triggered
+	 * the interrupt. Otherwise, the raw PB instruction that triggered the
+	 * interrupt is stored in NV_PPBDMA_HDR_SHADOW and NV_PPBDMA_PB_HEADER
+	 * stores the decoded version.
+	 */
+
+	pb_header = nvgpu_readl(g, pbdma_pb_header_r(pbdma_id));
+	pb_count = nvgpu_readl(g, pbdma_pb_count_r(pbdma_id));
+
+	pb_header_type = pb_header & pbdma_pb_header_type_m();
+	if ((pbdma_pb_count_value_v(pb_count) == pbdma_pb_count_value_zero_f())
+	    || (pb_header_type == pbdma_pb_header_type_immd_f())) {
+		pb_inst = nvgpu_readl(g, pbdma_data0_r(pbdma_id));
+	} else {
+		pb_inst = nvgpu_readl(g, pbdma_hdr_shadow_r(pbdma_id));
+	}
+
+	return pb_inst;
+}

drivers/gpu/nvgpu/hal/fifo/pbdma_tu104.h

@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2018-2019, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef NVGPU_PBDMA_TU104_H
+#define NVGPU_PBDMA_TU104_H
+
+#include <nvgpu/types.h>
+
+struct gk20a;
+
+void tu104_pbdma_reset_header(struct gk20a *g, u32 pbdma_id);
+u32 tu104_pbdma_read_data(struct gk20a *g, u32 pbdma_id);
+
+#endif /* NVGPU_PBDMA_TU104_H */

drivers/gpu/nvgpu/include/nvgpu/gk20a.h

@@ -850,7 +850,6 @@ struct gpu_ops {
 		void (*get_mmu_fault_gpc_desc)(struct mmu_fault_info *mmfault);
 		void (*apply_pb_timeout)(struct gk20a *g);
 		void (*apply_ctxsw_timeout_intr)(struct gk20a *g);
-		u32 (*get_pbdma_signature)(struct gk20a *g);
 		int (*tsg_set_timeslice)(struct tsg_gk20a *tsg, u32 timeslice);
 		u32 (*default_timeslice_us)(struct gk20a *g);
 		int (*force_reset_ch)(struct channel_gk20a *ch,
@@ -869,8 +868,6 @@ struct gpu_ops {
 		u64 (*userd_pb_get)(struct gk20a *g, struct channel_gk20a *ch);
 		u32 (*userd_entry_size)(struct gk20a  *g);
 		void (*free_channel_ctx_header)(struct channel_gk20a *ch);
-		void (*dump_pbdma_status)(struct gk20a *g,
-				struct gk20a_debug_output *o);
 		void (*dump_channel_status_ramfc)(struct gk20a *g,
 				struct gk20a_debug_output *o,
 				struct nvgpu_channel_dump_info *info);
@@ -883,7 +880,6 @@ struct gpu_ops {
 		void (*init_pbdma_intr_descs)(struct fifo_gk20a *f);
 		int (*reset_enable_hw)(struct gk20a *g);
 		int (*setup_userd)(struct channel_gk20a *c);
-		u32 (*pbdma_acquire_val)(u64 timeout);
 		void (*teardown_ch_tsg)(struct gk20a *g, u32 act_eng_bitmask,
 			u32 id, unsigned int id_type, unsigned int rc_type,
 			 struct mmu_fault_info *mmfault);
@@ -891,12 +887,6 @@ struct gpu_ops {
 		void (*teardown_unmask_intr)(struct gk20a *g);
 		bool (*handle_sched_error)(struct gk20a *g);
 		bool (*handle_ctxsw_timeout)(struct gk20a *g, u32 fifo_intr);
-		unsigned int (*handle_pbdma_intr_0)(struct gk20a *g,
-					u32 pbdma_id, u32 pbdma_intr_0,
-					u32 *handled, u32 *error_notifier);
-		unsigned int (*handle_pbdma_intr_1)(struct gk20a *g,
-					u32 pbdma_id, u32 pbdma_intr_1,
-					u32 *handled, u32 *error_notifier);
 		void (*init_eng_method_buffers)(struct gk20a *g,
 						struct tsg_gk20a *tsg);
 		void (*deinit_eng_method_buffers)(struct gk20a *g,
@@ -922,8 +912,6 @@ struct gpu_ops {
 		bool (*find_pbdma_for_runlist)(struct fifo_gk20a *f,
 				u32 runlist_id, u32 *pbdma_id);
 		int (*init_ce_engine_info)(struct fifo_gk20a *f);
-		u32 (*read_pbdma_data)(struct gk20a *g, u32 pbdma_id);
-		void (*reset_pbdma_header)(struct gk20a *g, u32 pbdma_id);
 		struct {
 			int (*report_host_err)(struct gk20a *g,
 					u32 hw_id, u32 inst, u32 err_id,
@@ -958,6 +946,22 @@ struct gpu_ops {
 					 u32 engine_subid);
 	} engine;
 
+	struct {
+		unsigned int (*handle_pbdma_intr_0)(struct gk20a *g,
+					u32 pbdma_id, u32 pbdma_intr_0,
+					u32 *handled, u32 *error_notifier);
+		unsigned int (*handle_pbdma_intr_1)(struct gk20a *g,
+					u32 pbdma_id, u32 pbdma_intr_1,
+					u32 *handled, u32 *error_notifier);
+		u32 (*get_pbdma_signature)(struct gk20a *g);
+		void (*dump_pbdma_status)(struct gk20a *g,
+				struct gk20a_debug_output *o);
+		u32 (*pbdma_acquire_val)(u64 timeout);
+		u32 (*read_pbdma_data)(struct gk20a *g, u32 pbdma_id);
+		void (*reset_pbdma_header)(struct gk20a *g, u32 pbdma_id);
+
+	} pbdma;
+
 	struct {
 #ifdef CONFIG_TEGRA_GK20A_NVHOST
 		int (*alloc_syncpt_buf)(struct channel_gk20a *c,

drivers/gpu/nvgpu/os/linux/debug.c

@@ -66,7 +66,7 @@ void gk20a_debug_output(struct gk20a_debug_output *o, const char *fmt, ...)
 void gk20a_debug_show_dump(struct gk20a *g, struct gk20a_debug_output *o)
 {
 	gk20a_debug_dump_all_channel_status_ramfc(g, o);
-	g->ops.fifo.dump_pbdma_status(g, o);
+	g->ops.pbdma.dump_pbdma_status(g, o);
 	g->ops.engine_status.dump_engine_status(g, o);
 }
 

drivers/gpu/nvgpu/tu104/fifo_tu104.c

@@ -65,7 +65,7 @@ int channel_tu104_setup_ramfc(struct channel_gk20a *c,
 		pbdma_gp_base_hi_limit2_f(ilog2(gpfifo_entries)));
 
 	nvgpu_mem_wr32(g, mem, ram_fc_signature_w(),
-		c->g->ops.fifo.get_pbdma_signature(c->g));
+		c->g->ops.pbdma.get_pbdma_signature(c->g));
 
 	nvgpu_mem_wr32(g, mem, ram_fc_pb_header_w(),
 		pbdma_pb_header_method_zero_f() |
@@ -85,7 +85,7 @@ int channel_tu104_setup_ramfc(struct channel_gk20a *c,
 		pbdma_target_engine_sw_f());
 
 	nvgpu_mem_wr32(g, mem, ram_fc_acquire_w(),
-		g->ops.fifo.pbdma_acquire_val(acquire_timeout));
+		g->ops.pbdma.pbdma_acquire_val(acquire_timeout));
 
 	nvgpu_mem_wr32(g, mem, ram_fc_set_channel_info_w(),
 		pbdma_set_channel_info_veid_f(c->subctx_id));
@@ -239,40 +239,3 @@ void tu104_deinit_pdb_cache_war(struct gk20a *g)
 		nvgpu_dma_free(g, &g->pdb_cache_war_mem);
 	}
 }
-
-u32 tu104_fifo_read_pbdma_data(struct gk20a *g, u32 pbdma_id)
-{
-	u32 pb_inst;
-	u32 pb_header, pb_header_type;
-	u32 pb_count;
-
-	/*
-	 * In order to determine the location of the PB entry that cause the
-	 * interrupt, NV_PPBDMA_PB_HEADER and NV_PPBDMA_PB_COUNT need to be
-	 * checked. If the TYPE field of the NV_PPBDMA_PB_HEADER is IMMD or the
-	 * VALUE field of the NV_PPBDMA_PB_COUNT is zero, then the raw PB
-	 * instruction stored in NV_PPBDMA_PB_DATA0 is the one that triggered
-	 * the interrupt. Otherwise, the raw PB instruction that triggered the
-	 * interrupt is stored in NV_PPBDMA_HDR_SHADOW and NV_PPBDMA_PB_HEADER
-	 * stores the decoded version.
-	 */
-
-	pb_header = nvgpu_readl(g, pbdma_pb_header_r(pbdma_id));
-	pb_count = nvgpu_readl(g, pbdma_pb_count_r(pbdma_id));
-
-	pb_header_type = pb_header & pbdma_pb_header_type_m();
-	if ((pbdma_pb_count_value_v(pb_count) == pbdma_pb_count_value_zero_f())
-	    || (pb_header_type == pbdma_pb_header_type_immd_f())) {
-		pb_inst = nvgpu_readl(g, pbdma_data0_r(pbdma_id));
-	} else {
-		pb_inst = nvgpu_readl(g, pbdma_hdr_shadow_r(pbdma_id));
-	}
-
-	return pb_inst;
-}
-
-void tu104_fifo_reset_pbdma_header(struct gk20a *g, u32 pbdma_id)
-{
-	gk20a_fifo_reset_pbdma_header(g, pbdma_id);
-	nvgpu_writel(g, pbdma_data0_r(pbdma_id), 0);
-}

drivers/gpu/nvgpu/tu104/fifo_tu104.h

@@ -38,7 +38,5 @@ u32 tu104_fifo_doorbell_token(struct channel_gk20a *c);
 
 int tu104_init_pdb_cache_war(struct gk20a *g);
 void tu104_deinit_pdb_cache_war(struct gk20a *g);
-u32 tu104_fifo_read_pbdma_data(struct gk20a *g, u32 pbdma_id);
-void tu104_fifo_reset_pbdma_header(struct gk20a *g, u32 pbdma_id);
 
 #endif /* NVGPU_FIFO_TU104_H */

drivers/gpu/nvgpu/tu104/hal_tu104.c

@@ -49,6 +49,10 @@
 #include "hal/fuse/fuse_gm20b.h"
 #include "hal/fuse/fuse_gp10b.h"
 #include "hal/fuse/fuse_gp106.h"
+#include "hal/fifo/pbdma_gm20b.h"
+#include "hal/fifo/pbdma_gp10b.h"
+#include "hal/fifo/pbdma_gv11b.h"
+#include "hal/fifo/pbdma_tu104.h"
 #include "hal/fifo/engines_gv11b.h"
 #include "hal/gr/fecs_trace/fecs_trace_gm20b.h"
 #include "hal/gr/fecs_trace/fecs_trace_gv11b.h"
@@ -794,7 +798,6 @@ static const struct gpu_ops tu104_ops = {
 		.userd_gp_put = gv11b_userd_gp_put,
 		.userd_pb_get = gv11b_userd_pb_get,
 		.userd_entry_size = gk20a_fifo_userd_entry_size,
-		.pbdma_acquire_val = gk20a_fifo_pbdma_acquire_val,
 		.preempt_channel = gv11b_fifo_preempt_channel,
 		.preempt_tsg = gv11b_fifo_preempt_tsg,
 		.enable_tsg = gv11b_fifo_enable_tsg,
@@ -807,13 +810,11 @@ static const struct gpu_ops tu104_ops = {
 		.get_mmu_fault_desc = NULL,
 		.get_mmu_fault_client_desc = NULL,
 		.get_mmu_fault_gpc_desc = NULL,
-		.get_pbdma_signature = gp10b_fifo_get_pbdma_signature,
 		.tsg_set_timeslice = gk20a_fifo_tsg_set_timeslice,
 		.force_reset_ch = gk20a_fifo_force_reset_ch,
 		.init_engine_info = gm20b_fifo_init_engine_info,
 		.init_pbdma_info = gk20a_fifo_init_pbdma_info,
 		.get_engines_mask_on_id = gk20a_fifo_engines_on_id,
-		.dump_pbdma_status = gk20a_dump_pbdma_status,
 		.dump_channel_status_ramfc = gv11b_dump_channel_status_ramfc,
 		.capture_channel_ram_dump = gv11b_capture_channel_ram_dump,
 		.intr_0_error_mask = gv11b_fifo_intr_0_error_mask,
@@ -824,8 +825,6 @@ static const struct gpu_ops tu104_ops = {
 		.teardown_mask_intr = gv11b_fifo_teardown_mask_intr,
 		.teardown_unmask_intr = gv11b_fifo_teardown_unmask_intr,
 		.handle_sched_error = gv11b_fifo_handle_sched_error,
-		.handle_pbdma_intr_0 = gv11b_fifo_handle_pbdma_intr_0,
-		.handle_pbdma_intr_1 = gv11b_fifo_handle_pbdma_intr_1,
 		.init_eng_method_buffers = gv11b_fifo_init_eng_method_buffers,
 		.deinit_eng_method_buffers =
 			gv11b_fifo_deinit_eng_method_buffers,
@@ -852,12 +851,19 @@ static const struct gpu_ops tu104_ops = {
 		.runlist_busy_engines = gk20a_fifo_runlist_busy_engines,
 		.find_pbdma_for_runlist = gk20a_fifo_find_pbdma_for_runlist,
 		.init_ce_engine_info = gp10b_fifo_init_ce_engine_info,
-		.read_pbdma_data = tu104_fifo_read_pbdma_data,
-		.reset_pbdma_header = tu104_fifo_reset_pbdma_header,
 	},
 	.engine = {
 		.is_fault_engine_subid_gpc = gv11b_is_fault_engine_subid_gpc,
 	},
+	.pbdma = {
+		.pbdma_acquire_val = gm20b_pbdma_acquire_val,
+		.get_pbdma_signature = gp10b_pbdma_get_signature,
+		.dump_pbdma_status = gm20b_pbdma_dump_status,
+		.handle_pbdma_intr_0 = gv11b_pbdma_handle_intr_0,
+		.handle_pbdma_intr_1 = gv11b_pbdma_handle_intr_1,
+		.read_pbdma_data = tu104_pbdma_read_data,
+		.reset_pbdma_header = tu104_pbdma_reset_header,
+	},
 	.sync = {
 #ifdef CONFIG_TEGRA_GK20A_NVHOST
 		.alloc_syncpt_buf = gv11b_alloc_syncpt_buf,
@@ -1289,6 +1295,7 @@ int tu104_init_hal(struct gk20a *g)
 	gops->clock_gating = tu104_ops.clock_gating;
 	gops->fifo = tu104_ops.fifo;
 	gops->engine = tu104_ops.engine;
+	gops->pbdma = tu104_ops.pbdma;
 	gops->runlist = tu104_ops.runlist;
 	gops->channel = tu104_ops.channel;
 	gops->sync = tu104_ops.sync;