diff --git a/drivers/misc/mods/Makefile b/drivers/misc/mods/Makefile
index 94a7d142..4d9132a0 100644
--- a/drivers/misc/mods/Makefile
+++ b/drivers/misc/mods/Makefile
@@ -1,9 +1,40 @@
 # SPDX-License-Identifier: GPL-2.0
-# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
 
-# NOTE: Do not change or add anything in this makefile.
-# The source code and makefile rules are copied from the
-# kernel/nvidia/drivers/misc/mods. This file is
-# just place-holder for empty makefile to avoid any build
-# issue when copy is not done from command line and building
-# the tree independent of source copy.
+ccflags-y += -I$(srctree.nvidia)/include
+ccflags-y += -DMODS_HAS_TEGRA
+
+ifeq ($(CONFIG_TEGRA_OOT_MODULE),m)
+CONFIG_MODS := m
+endif
+
+obj-$(CONFIG_MODS)                 := mods.o
+
+mods-y                             := mods_irq.o
+mods-y                             += mods_krnl.o
+mods-y                             += mods_mem.o
+
+mods-$(CONFIG_ACPI)                += mods_acpi.o
+mods-$(CONFIG_TEGRA_NVADSP)        += mods_adsp.o
+mods-$(CONFIG_ARM_FFA_TRANSPORT)   += mods_arm_ffa.o
+mods-$(CONFIG_TEGRA_IVC)           += mods_bpmpipc.o
+mods-$(CONFIG_COMMON_CLK)          += mods_clock.o
+mods-$(CONFIG_DEBUG_FS)            += mods_debugfs.o
+mods-$(CONFIG_DMA_ENGINE)          += mods_dma.o
+mods-$(CONFIG_ARCH_TEGRA)          += mods_ipi.o
+mods-$(CONFIG_NET)                 += mods_netdevice.o
+mods-$(CONFIG_ARCH_TEGRA)          += mods_oist.o
+mods-$(CONFIG_OPTEE)               += mods_optee.o
+mods-$(CONFIG_PCI)                 += mods_pci.o
+mods-$(CONFIG_ARCH_TEGRA_19x_SOC)  += mods_ras.o
+mods-$(CONFIG_ARCH_TEGRA)          += mods_smmu_drv.o
+mods-$(CONFIG_TEGRA_DC)            += mods_tegradc.o
+mods-$(CONFIG_TRUSTY)              += mods_tz.o
+
+mods-objs                          := mods.dtb.o
+
+ifneq ($(CONFIG_TEGRA_OOT_MODULE),m)
+ccflags-y                          += -DMODS_HAS_DMABUF
+ccflags-y                          += -DMODS_HAS_PROD
+mods-$(CONFIG_DMA_SHARED_BUFFER)   += mods_dmabuf.o
+mods-$(CONFIG_ARCH_TEGRA)          += mods_tegraprod.o
+endif
diff --git a/drivers/misc/mods/mods.dts b/drivers/misc/mods/mods.dts
new file mode 100644
index 00000000..673eeaf1
--- /dev/null
+++ b/drivers/misc/mods/mods.dts
@@ -0,0 +1,28 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * This file is part of NVIDIA MODS kernel driver.
+ *
+ * Copyright (c) 2015-2022, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * NVIDIA MODS kernel driver is free software: you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * NVIDIA MODS kernel driver is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NVIDIA MODS kernel driver.
+ * If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/dts-v1/;
+
+/ {
+	mods {
+		compatible = "nvidia,tegra-mods";
+		status = "okay";
+	};
+};
diff --git a/drivers/misc/mods/mods.h b/drivers/misc/mods/mods.h
new file mode 100644
index 00000000..adb43beb
--- /dev/null
+++ b/drivers/misc/mods/mods.h
@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * This file is part of NVIDIA MODS kernel driver.
+ *
+ * Copyright (c) 2020-2022, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * NVIDIA MODS kernel driver is free software: you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * NVIDIA MODS kernel driver is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NVIDIA MODS kernel driver.
+ * If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <uapi/misc/mods.h>
diff --git a/drivers/misc/mods/mods_acpi.c b/drivers/misc/mods/mods_acpi.c
new file mode 100644
index 00000000..8bb1b664
--- /dev/null
+++ b/drivers/misc/mods/mods_acpi.c
@@ -0,0 +1,726 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * This file is part of NVIDIA MODS kernel driver.
+ *
+ * Copyright (c) 2008-2022, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * NVIDIA MODS kernel driver is free software: you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * NVIDIA MODS kernel driver is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NVIDIA MODS kernel driver.
+ * If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "mods_internal.h"
+
+#include <linux/acpi.h>
+#include <linux/device.h>
+#include <acpi/acpi.h>
+#include <acpi/acpi_bus.h>
+
+static acpi_status mods_acpi_find_acpi_handler(acpi_handle,
+					       u32,
+					       void *,
+					       void **);
+
+/*********************
+ * PRIVATE FUNCTIONS *
+ *********************/
+
+struct acpi_dev_children_in_data {
+	u32         expected_acpi_ids_buf[4];
+	u32         num_expected_acpi_ids;
+	u32         acpi_dev_mask;
+	acpi_handle dev_handle;
+};
+
+typedef int (*dev_children_fptr)(struct mods_client *,
+				 u32,
+				 acpi_handle,
+				 void *);
+
+static int acpi_store_dev_children(struct mods_client *client,
+				   u32                 curr_acpi_id,
+				   acpi_handle         dev_handle,
+				   void               *out_data)
+{
+	struct MODS_GET_ACPI_DEV_CHILDREN *acpi_out_data
+			= (struct MODS_GET_ACPI_DEV_CHILDREN *)out_data;
+
+	if (acpi_out_data->num_children >= ACPI_MAX_DEV_CHILDREN) {
+		cl_error("ACPI: output buffer too small to store all children\n");
+		return -ENOBUFS;
+	}
+
+	acpi_out_data->children[acpi_out_data->num_children] = curr_acpi_id;
+	++acpi_out_data->num_children;
+
+	return OK;
+}
+
+static int acpi_compare_dev_id(struct mods_client *client,
+			       u32                 curr_acpi_id,
+			       acpi_handle         dev_handle,
+			       void               *out_data)
+{
+	u32                               i;
+	struct acpi_dev_children_in_data *dev_child_data
+			= (struct acpi_dev_children_in_data *)out_data;
+
+	for (i = 0; i < dev_child_data->num_expected_acpi_ids; ++i) {
+		if ((dev_child_data->expected_acpi_ids_buf[i] & dev_child_data->acpi_dev_mask)
+			== (curr_acpi_id & dev_child_data->acpi_dev_mask)) {
+			dev_child_data->dev_handle = dev_handle;
+			/*
+			 * returning unique error code to signal caller that value is found
+			 */
+			return -EALREADY;
+		}
+	}
+	return -ENODEV;
+}
+
+struct acpi_dev_check_context {
+	struct mods_client *client;
+	dev_children_fptr   fptr;
+	void               *out_data;
+};
+
+static int acpi_dev_check_one(struct acpi_device *adev, void *data)
+{
+	int                 err       = OK;
+	unsigned long long  device_id = 0;
+	acpi_status         status;
+	struct acpi_dev_check_context *adwc = data;
+
+	status = acpi_evaluate_integer(adev->handle,
+			"_ADR",
+			NULL,
+			&device_id);
+	if (ACPI_FAILURE(status))
+		/* Couldnt query device_id for this device */
+		return OK;
+
+	err = adwc->fptr(adwc->client, device_id, adev->handle, adwc->out_data);
+	return ((err == -EALREADY) ? OK : err);
+}
+
+#if KERNEL_VERSION(6, 0, 0) > MODS_KERNEL_VERSION
+static int acpi_dev_each_child_node(struct acpi_device *adev,
+				    int (*fptr)(struct acpi_device *, void *),
+				    void *data)
+{
+	struct list_head   *node    = NULL;
+	struct list_head   *next    = NULL;
+
+	list_for_each_safe(node, next, &adev->children) {
+		struct acpi_device *dev =
+			list_entry(node, struct acpi_device, node);
+
+		fptr(dev, data);
+	}
+	return OK;
+}
+#else
+#define acpi_dev_each_child_node acpi_dev_for_each_child
+#endif
+
+static int acpi_get_dev_children(struct mods_client *client,
+				 dev_children_fptr   fptr,
+				 acpi_handle         dev_handle,
+				 void               *out_data)
+{
+	int                           err    = OK;
+	struct acpi_device           *device = NULL;
+	struct acpi_dev_check_context adcc   = {
+		.client		= client,
+		.fptr		= fptr,
+		.out_data	= out_data,
+	};
+
+	LOG_ENT();
+
+#ifdef MODS_HAS_ACPI_FETCH
+	device = acpi_fetch_acpi_dev(dev_handle);
+	err = device ? 0 : -EINVAL;
+#else
+	err = acpi_bus_get_device(dev_handle, &device);
+#endif
+	if (unlikely(err))
+		cl_error("ACPI: device for fetching device children not found\n");
+	else
+		err = acpi_dev_each_child_node(device, acpi_dev_check_one, &adcc);
+
+	LOG_EXT();
+	return err;
+}
+
+
+/* store handle if found. */
+static void mods_acpi_handle_init(struct mods_client *client,
+				  char               *method_name,
+				  acpi_handle        *handler)
+{
+	MODS_ACPI_WALK_NAMESPACE(ACPI_TYPE_ANY,
+				 ACPI_ROOT_OBJECT,
+				 ACPI_UINT32_MAX,
+				 mods_acpi_find_acpi_handler,
+				 method_name,
+				 handler);
+
+	if (!(*handler)) {
+		cl_debug(DEBUG_ACPI,
+			 "ACPI method %s not found\n",
+			 method_name);
+	}
+}
+
+static acpi_status mods_acpi_find_acpi_handler(
+	acpi_handle handle,
+	u32 nest_level,
+	void *dummy1,
+	void **dummy2
+)
+{
+	acpi_handle acpi_method_handler_temp;
+
+	if (!acpi_get_handle(handle, dummy1, &acpi_method_handler_temp))
+		*dummy2 = acpi_method_handler_temp;
+
+	return OK;
+}
+
+static int mods_extract_acpi_object(struct mods_client *client,
+				    char               *method,
+				    union acpi_object  *obj,
+				    u8                **buf,
+				    u8                 *buf_end)
+{
+	int err = OK;
+
+	switch (obj->type) {
+
+	case ACPI_TYPE_BUFFER:
+		if (obj->buffer.length == 0) {
+			cl_error(
+				"empty ACPI output buffer from ACPI method %s\n",
+				method);
+			err = -EINVAL;
+		} else if (obj->buffer.length <= buf_end-*buf) {
+			u32 size = obj->buffer.length;
+
+			memcpy(*buf, obj->buffer.pointer, size);
+			*buf += size;
+		} else {
+			cl_error("output buffer too small for ACPI method %s\n",
+				 method);
+			err = -EINVAL;
+		}
+		break;
+
+	case ACPI_TYPE_INTEGER:
+		if (buf_end - *buf >= 4) {
+			if (obj->integer.value > 0xFFFFFFFFU) {
+				cl_error(
+					"integer value from ACPI method %s out of range\n",
+					method);
+				err = -EINVAL;
+			} else {
+				memcpy(*buf, &obj->integer.value, 4);
+				*buf += 4;
+			}
+		} else {
+			cl_error("output buffer too small for ACPI method %s\n",
+				 method);
+			err = -EINVAL;
+		}
+		break;
+
+	case ACPI_TYPE_PACKAGE:
+		if (obj->package.count == 0) {
+			cl_error(
+				"empty ACPI output package from ACPI method %s\n",
+				method);
+			err = -EINVAL;
+		} else {
+			union acpi_object *elements = obj->package.elements;
+			u32                size     = 0;
+			u32                i;
+
+			for (i = 0; i < obj->package.count; i++) {
+				u8 *old_buf = *buf;
+				u32 new_size;
+
+				err = mods_extract_acpi_object(client,
+							       method,
+							       &elements[i],
+							       buf,
+							       buf_end);
+				if (err)
+					break;
+
+				new_size = *buf - old_buf;
+
+				if (size == 0) {
+					size = new_size;
+				} else if (size != new_size) {
+					cl_error(
+						"ambiguous package element size from ACPI method %s\n",
+						method);
+					err = -EINVAL;
+				}
+			}
+		}
+		break;
+
+	case ACPI_TYPE_LOCAL_REFERENCE:
+		if (obj->reference.actual_type == ACPI_TYPE_POWER) {
+			memcpy(*buf, &obj->reference.handle,
+					sizeof(obj->reference.handle));
+			*buf += sizeof(obj->reference.handle);
+		} else {
+			cl_error("Unsupported ACPI reference type\n");
+			err = -EINVAL;
+		}
+		break;
+
+	default:
+		cl_error("unsupported ACPI output type 0x%02x from method %s\n",
+			 (unsigned int)obj->type,
+			 method);
+		err = -EINVAL;
+		break;
+
+	}
+	return err;
+}
+
+static int mods_eval_acpi_method(struct mods_client           *client,
+				 struct MODS_EVAL_ACPI_METHOD *p,
+				 struct mods_pci_dev_2        *pdevice,
+				 u32                           acpi_id)
+{
+	int err = OK;
+	int i;
+	acpi_status status;
+	struct acpi_object_list input;
+	struct acpi_buffer output = { ACPI_ALLOCATE_BUFFER, NULL };
+	union acpi_object *acpi_method = NULL;
+	union acpi_object acpi_params[ACPI_MAX_ARGUMENT_NUMBER];
+	acpi_handle acpi_method_handler = NULL;
+	struct pci_dev *dev = NULL;
+
+	LOG_ENT();
+
+	if (p->argument_count >= ACPI_MAX_ARGUMENT_NUMBER) {
+		cl_error("invalid argument count for ACPI call\n");
+		LOG_EXT();
+		return -EINVAL;
+	}
+
+	if (pdevice) {
+		cl_debug(DEBUG_ACPI,
+			 "ACPI %s for dev %04x:%02x:%02x.%x\n",
+			 p->method_name,
+			 pdevice->domain,
+			 pdevice->bus,
+			 pdevice->device,
+			 pdevice->function);
+
+		err = mods_find_pci_dev(client, pdevice, &dev);
+		if (unlikely(err)) {
+			if (err == -ENODEV)
+				cl_error(
+					"ACPI: dev %04x:%02x:%02x.%x not found\n",
+					pdevice->domain,
+					pdevice->bus,
+					pdevice->device,
+					pdevice->function);
+			LOG_EXT();
+			return err;
+		}
+		acpi_method_handler = MODS_ACPI_HANDLE(&dev->dev);
+	} else {
+		cl_debug(DEBUG_ACPI, "ACPI %s\n", p->method_name);
+		mods_acpi_handle_init(client,
+				      p->method_name,
+				      &acpi_method_handler);
+	}
+
+	if (acpi_id != ACPI_MODS_IGNORE_ACPI_ID) {
+		struct acpi_dev_children_in_data in_data = { {}, 0, 0, NULL};
+
+		in_data.expected_acpi_ids_buf[0] = acpi_id;
+		in_data.num_expected_acpi_ids = 1;
+		in_data.acpi_dev_mask = 0xffffffff;
+
+		err = acpi_get_dev_children(client, acpi_compare_dev_id,
+				acpi_method_handler, (void *)&in_data);
+		if (err) {
+			pci_dev_put(dev);
+			LOG_EXT();
+			return -EINVAL;
+		}
+		acpi_method_handler = in_data.dev_handle;
+	}
+
+	if (!acpi_method_handler) {
+		cl_debug(DEBUG_ACPI,
+			 "ACPI: handle for %s not found\n",
+			 p->method_name);
+		pci_dev_put(dev);
+		LOG_EXT();
+		return -EINVAL;
+	}
+	cl_debug(DEBUG_ACPI,
+		 "ACPI: found %s (id = 0x%x) on dev %04x:%02x:%02x.%x\n",
+		 p->method_name,
+		 (unsigned int)acpi_id,
+		 pdevice ? pdevice->domain : 0U,
+		 pdevice ? pdevice->bus : 0U,
+		 pdevice ? pdevice->device : 0U,
+		 pdevice ? pdevice->function : 0U);
+
+	input.count = p->argument_count;
+	input.pointer = acpi_params;
+
+	for (i = 0; i < p->argument_count; i++) {
+		switch (p->argument[i].type) {
+		case ACPI_MODS_TYPE_INTEGER: {
+			acpi_params[i].integer.type = ACPI_TYPE_INTEGER;
+			acpi_params[i].integer.value
+				= p->argument[i].integer.value;
+			break;
+		}
+		case ACPI_MODS_TYPE_BUFFER: {
+			acpi_params[i].buffer.type = ACPI_TYPE_BUFFER;
+			acpi_params[i].buffer.length
+				= p->argument[i].buffer.length;
+			acpi_params[i].buffer.pointer
+				= p->in_buffer + p->argument[i].buffer.offset;
+			break;
+		}
+		case ACPI_MODS_TYPE_METHOD: {
+			memcpy(&acpi_method_handler,
+					&p->argument[i].method.handle,
+					sizeof(acpi_method_handler));
+
+			if (!acpi_method_handler) {
+				cl_error("ACPI: Invalid reference handle 0\n");
+				pci_dev_put(dev);
+				LOG_EXT();
+				return -EINVAL;
+			}
+
+			if (i != p->argument_count - 1) {
+				cl_error("ACPI: Invalid argument count\n");
+				pci_dev_put(dev);
+				LOG_EXT();
+				return -EINVAL;
+			}
+
+			--input.count;
+			break;
+		}
+		default: {
+			cl_error("unsupported ACPI argument type\n");
+			pci_dev_put(dev);
+			LOG_EXT();
+			return -EINVAL;
+		}
+		}
+	}
+
+	status = acpi_evaluate_object(acpi_method_handler,
+				      pdevice ? p->method_name : NULL,
+				      &input,
+				      &output);
+
+	if (ACPI_FAILURE(status)) {
+		cl_info("ACPI method %s failed\n", p->method_name);
+		pci_dev_put(dev);
+		LOG_EXT();
+		return -EINVAL;
+	}
+
+	acpi_method = output.pointer;
+	if (!acpi_method) {
+		cl_error("missing output from ACPI method %s\n",
+			 p->method_name);
+		err = -EINVAL;
+	} else {
+		u8 *buf = p->out_buffer;
+
+		err = mods_extract_acpi_object(client,
+					       p->method_name,
+					       acpi_method,
+					       &buf,
+					       buf+sizeof(p->out_buffer));
+		p->out_data_size = err ? 0 : (buf - p->out_buffer);
+	}
+
+	kfree(output.pointer);
+	pci_dev_put(dev);
+	LOG_EXT();
+	return err;
+}
+
+static int mods_acpi_get_ddc(struct mods_client         *client,
+			     struct MODS_ACPI_GET_DDC_2 *p,
+			     struct mods_pci_dev_2      *pci_device)
+{
+	int                               err;
+	acpi_status                       status;
+	union acpi_object                *ddc;
+	union acpi_object                 ddc_arg0       = { ACPI_TYPE_INTEGER };
+	struct acpi_object_list           input          = { 1, &ddc_arg0 };
+	u32                               i;
+	acpi_handle                       dev_handle     = NULL;
+	acpi_handle                       lcd_dev_handle = NULL;
+	struct pci_dev                   *dev            = NULL;
+	bool                              data_found     = false;
+	struct acpi_dev_children_in_data  in_data        = { {}, 0, 0, NULL };
+
+	LOG_ENT();
+
+	cl_debug(DEBUG_ACPI,
+		 "ACPI _DDC (EDID) for dev %04x:%02x:%02x.%x\n",
+		 pci_device->domain,
+		 pci_device->bus,
+		 pci_device->device,
+		 pci_device->function);
+
+	err = mods_find_pci_dev(client, pci_device, &dev);
+	if (unlikely(err)) {
+		if (err == -ENODEV)
+			cl_error("ACPI: dev %04x:%02x:%02x.%x not found\n",
+				 pci_device->domain,
+				 pci_device->bus,
+				 pci_device->device,
+				 pci_device->function);
+		LOG_EXT();
+		return err;
+	}
+
+	dev_handle = MODS_ACPI_HANDLE(&dev->dev);
+	if (!dev_handle) {
+		cl_debug(DEBUG_ACPI, "ACPI: handle for _DDC not found\n");
+		pci_dev_put(dev);
+		LOG_EXT();
+		return -EINVAL;
+	}
+
+	/*
+	 * List of supported display's (panels)
+	 * Reference: https://uefi.org/sites/default/files/resources/ACPI_6_1.pdf
+	 * Section: Table B-390 Video Output Device Attributes
+	 */
+	in_data.expected_acpi_ids_buf[0] = 0x0110;
+	in_data.expected_acpi_ids_buf[1] = 0x0118;
+	in_data.expected_acpi_ids_buf[2] = 0x0400;
+	in_data.expected_acpi_ids_buf[3] = 0xa450;
+	in_data.num_expected_acpi_ids = 4;
+	in_data.acpi_dev_mask = 0xffff;
+
+	err = acpi_get_dev_children(client, acpi_compare_dev_id,
+			dev_handle, (void *)&in_data);
+	if (err) {
+		pci_dev_put(dev);
+		LOG_EXT();
+		return -EINVAL;
+	}
+	lcd_dev_handle = in_data.dev_handle;
+
+	if (lcd_dev_handle == NULL) {
+		cl_error("ACPI: LCD not found for dev %04x:%02x:%02x.%x\n",
+			 p->device.domain,
+			 p->device.bus,
+			 p->device.device,
+			 p->device.function);
+		pci_dev_put(dev);
+		LOG_EXT();
+		return -EINVAL;
+	}
+
+	cl_debug(DEBUG_ACPI,
+		 "ACPI: Found LCD on dev %04x:%02x:%02x.%x\n",
+		 p->device.domain,
+		 p->device.bus,
+		 p->device.device,
+		 p->device.function);
+
+	/*
+	 * As per ACPI Spec 3.0:
+	 * ARG0 is size of EDID buffer in 128-byte blocks.
+	 */
+	err = -EINVAL;
+	for (i = 4; i >= 1; --i) {
+		struct acpi_buffer output = { ACPI_ALLOCATE_BUFFER, NULL };
+
+		ddc_arg0.integer.value = i;
+		status = acpi_evaluate_object(lcd_dev_handle,
+					      "_DDC",
+					      &input,
+					      &output);
+		if (ACPI_SUCCESS(status)) {
+			ddc = output.pointer;
+			if (ddc && (ddc->type != ACPI_TYPE_BUFFER))
+				continue;
+
+			if (!ddc || ddc->buffer.length == 0) {
+				cl_error("unsupported ACPI output type\n");
+			} else if (ddc->buffer.length > sizeof(p->out_buffer)) {
+				cl_error(
+					"output buffer too small for ACPI method _DDC (EDID)\n");
+			} else {
+				p->out_data_size = ddc->buffer.length;
+				memcpy(p->out_buffer,
+					   ddc->buffer.pointer,
+					   p->out_data_size);
+				err = OK;
+				data_found = true;
+			}
+
+			kfree(ddc);
+
+			break;
+		}
+	}
+
+	if (!data_found)
+		cl_error("ACPI method _DDC (EDID) failed\n");
+
+	pci_dev_put(dev);
+	LOG_EXT();
+	return err;
+}
+
+/*************************
+ * ESCAPE CALL FUNCTIONS *
+ *************************/
+
+int esc_mods_eval_acpi_method(struct mods_client           *client,
+			      struct MODS_EVAL_ACPI_METHOD *p)
+{
+	return mods_eval_acpi_method(client, p, NULL, ACPI_MODS_IGNORE_ACPI_ID);
+}
+
+int esc_mods_eval_dev_acpi_method_3(struct mods_client                 *client,
+				    struct MODS_EVAL_DEV_ACPI_METHOD_3 *p)
+{
+	return mods_eval_acpi_method(client,
+				     &p->method,
+				     &p->device,
+				     p->acpi_id);
+}
+
+int esc_mods_eval_dev_acpi_method_2(struct mods_client                 *client,
+				    struct MODS_EVAL_DEV_ACPI_METHOD_2 *p)
+{
+	return mods_eval_acpi_method(client,
+				     &p->method,
+				     &p->device,
+				     ACPI_MODS_IGNORE_ACPI_ID);
+}
+
+int esc_mods_eval_dev_acpi_method(struct mods_client               *client,
+				  struct MODS_EVAL_DEV_ACPI_METHOD *p)
+{
+	struct mods_pci_dev_2 device = {0};
+
+	device.domain		= 0;
+	device.bus		= p->device.bus;
+	device.device		= p->device.device;
+	device.function		= p->device.function;
+	return mods_eval_acpi_method(client, &p->method, &device,
+				     ACPI_MODS_IGNORE_ACPI_ID);
+}
+
+int esc_mods_acpi_get_ddc_2(struct mods_client         *client,
+			    struct MODS_ACPI_GET_DDC_2 *p)
+{
+	return mods_acpi_get_ddc(client, p, &p->device);
+}
+
+int esc_mods_acpi_get_ddc(struct mods_client       *client,
+			  struct MODS_ACPI_GET_DDC *p)
+{
+	struct MODS_ACPI_GET_DDC_2 *pp     = (struct MODS_ACPI_GET_DDC_2 *) p;
+	struct mods_pci_dev_2       device = {0};
+
+	device.domain	= 0;
+	device.bus	= p->device.bus;
+	device.device	= p->device.device;
+	device.function	= p->device.function;
+
+	return mods_acpi_get_ddc(client, pp, &device);
+}
+
+int esc_mods_get_acpi_dev_children(struct mods_client                *client,
+				   struct MODS_GET_ACPI_DEV_CHILDREN *p)
+{
+	int             err;
+	struct pci_dev *dev        = NULL;
+	acpi_handle     dev_handle = NULL;
+
+	LOG_ENT();
+
+	cl_debug(DEBUG_ACPI,
+		 "ACPI: failed to get children for dev %04x:%02x:%02x.%x\n",
+		 p->device.domain,
+		 p->device.bus,
+		 p->device.device,
+		 p->device.function);
+
+	err = mods_find_pci_dev(client, &p->device, &dev);
+	if (unlikely(err)) {
+		if (err == -ENODEV)
+			cl_error("ACPI: dev %04x:%02x:%02x.%x not found\n",
+				 p->device.domain,
+				 p->device.bus,
+				 p->device.device,
+				 p->device.function);
+		LOG_EXT();
+		return err;
+	}
+
+	dev_handle = MODS_ACPI_HANDLE(&dev->dev);
+	if (!dev_handle) {
+		cl_error("ACPI: handle for fetching device children not found\n");
+		pci_dev_put(dev);
+		LOG_EXT();
+		return -EINVAL;
+	}
+
+	p->num_children = 0;
+	err = acpi_get_dev_children(client, acpi_store_dev_children,
+			dev_handle, (void *)p);
+
+	if (err) {
+		cl_error("ACPI: failed to get children for dev %04x:%02x:%02x.%x\n",
+			 p->device.domain,
+			 p->device.bus,
+			 p->device.device,
+			 p->device.function);
+	}
+
+	pci_dev_put(dev);
+	LOG_EXT();
+	return err;
+}
+
+#ifdef MODS_HAS_PXM_TO_NODE
+int esc_mods_proximity_to_numa_node(struct mods_client                 *client,
+				    struct MODS_PROXIMITY_TO_NUMA_NODE *p)
+{
+	p->numa_node = acpi_map_pxm_to_node(p->proximity);
+	return OK;
+}
+#endif
diff --git a/drivers/misc/mods/mods_adsp.c b/drivers/misc/mods/mods_adsp.c
new file mode 100644
index 00000000..2e78d504
--- /dev/null
+++ b/drivers/misc/mods/mods_adsp.c
@@ -0,0 +1,96 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * This file is part of NVIDIA MODS kernel driver.
+ *
+ * Copyright (c) 2014-2022, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * NVIDIA MODS kernel driver is free software: you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * NVIDIA MODS kernel driver is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NVIDIA MODS kernel driver.
+ * If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/uaccess.h>
+#include "mods_internal.h"
+#include <linux/tegra_nvadsp.h>
+
+int esc_mods_adsp_load(struct mods_client *client)
+{
+	return nvadsp_os_load();
+}
+
+int esc_mods_adsp_start(struct mods_client *client)
+{
+	return nvadsp_os_start();
+}
+
+int esc_mods_adsp_stop(struct mods_client *client)
+{
+	return nvadsp_os_suspend();
+}
+
+int esc_mods_adsp_run_app(struct mods_client *client,
+			  struct MODS_ADSP_RUN_APP_INFO *p)
+{
+	int rc = -1;
+	int max_retry = 3;
+	int rcount = 0;
+	nvadsp_app_handle_t handle;
+	nvadsp_app_info_t *p_app_info;
+	nvadsp_app_args_t app_args;
+
+	handle = nvadsp_app_load(p->app_name,  p->app_file_name);
+	if (!handle) {
+		cl_error("load adsp app fail");
+		return -1;
+	}
+
+	if (p->argc > 0 && p->argc <= MODS_ADSP_APP_MAX_PARAM) {
+		app_args.argc = p->argc;
+		memcpy(app_args.argv, p->argv, p->argc * sizeof(__u32));
+		p_app_info = nvadsp_app_init(handle, &app_args);
+	} else
+		p_app_info = nvadsp_app_init(handle, NULL);
+
+	if (!p_app_info) {
+		cl_error("init adsp app fail");
+		nvadsp_app_unload(handle);
+		return -1;
+	}
+
+	rc = nvadsp_app_start(p_app_info);
+	if (rc) {
+		cl_error("start adsp app fail");
+		goto failed;
+	}
+
+	while (rcount++ < max_retry) {
+		rc = wait_for_nvadsp_app_complete_timeout(p_app_info,
+						msecs_to_jiffies(p->timeout));
+		if (rc == -ERESTARTSYS)
+			continue;
+		else if (rc == 0) {
+			cl_error("app timeout(%d)", p->timeout);
+			rc = -1;
+		} else if (rc < 0) {
+			cl_error("run app failed, err=%d\n", rc);
+			rc = -1;
+		} else
+			rc = 0;
+		break;
+	}
+
+failed:
+	nvadsp_app_deinit(p_app_info);
+	nvadsp_app_unload(handle);
+
+	return rc;
+}
diff --git a/drivers/misc/mods/mods_arm_ffa.c b/drivers/misc/mods/mods_arm_ffa.c
new file mode 100644
index 00000000..64178c6b
--- /dev/null
+++ b/drivers/misc/mods/mods_arm_ffa.c
@@ -0,0 +1,189 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2022-2023, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#include "mods_internal.h"
+#include <linux/uuid.h>
+#include <linux/arm_ffa.h>
+#include <linux/errno.h>
+#include <linux/scatterlist.h>
+#include <linux/sched.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+#include <linux/types.h>
+
+static const struct ffa_device_id mods_ffa_device_id[] = {
+	{ UUID_INIT(0x1f4bfeb9, 0x0f48, 0xdd1e,
+		    0x11, 0x9c, 0x2c, 0x86, 0xc9, 0x14, 0x03, 0x22) },
+	{}
+};
+
+struct mods_ffa_ctx {
+	struct ffa_device        *ffa_dev;
+#if KERNEL_VERSION(6, 2, 0) <= MODS_KERNEL_VERSION
+	const struct ffa_msg_ops *ffa_ops;
+#else
+	const struct ffa_dev_ops *ffa_ops;
+#endif
+};
+
+static struct mods_ffa_ctx mods_ffa_info;
+
+static int ffa_probe(struct ffa_device *ffa_dev)
+{
+	int ret = 0;
+
+#if KERNEL_VERSION(6, 2, 0) <= MODS_KERNEL_VERSION
+	const struct ffa_msg_ops *ffa_ops = NULL;
+
+	if (ffa_dev->ops)
+		ffa_ops = ffa_dev->ops->msg_ops;
+#else
+	const struct ffa_dev_ops *ffa_ops;
+
+	ffa_ops = ffa_dev_ops_get(ffa_dev);
+#endif
+	if (!ffa_ops) {
+		mods_error_printk("failed \"method\" init: ffa\n");
+		return -ENOENT;
+	}
+	mods_ffa_info.ffa_dev = ffa_dev;
+	mods_ffa_info.ffa_ops = ffa_ops;
+
+	mods_debug_printk(DEBUG_TEGRADMA, "mods ffa driver registered\n");
+
+	return ret;
+}
+
+static void ffa_remove(struct ffa_device *ffa_dev)
+{
+	mods_ffa_info.ffa_dev = NULL;
+	mods_ffa_info.ffa_ops = NULL;
+}
+
+static struct ffa_driver mods_ffa_driver = {
+	.name     = "mods_arm_ffa",
+	.probe    = ffa_probe,
+	.remove   = ffa_remove,
+	.id_table = mods_ffa_device_id,
+};
+
+int mods_ffa_abi_register(void)
+{
+	mods_debug_printk(DEBUG_TEGRADMA, "registering MODS FFA driver\n");
+	return ffa_register(&mods_ffa_driver);
+}
+
+void mods_ffa_abi_unregister(void)
+{
+	ffa_unregister(&mods_ffa_driver);
+}
+
+int esc_mods_arm_ffa_cmd(struct mods_client *client,
+			 struct MODS_FFA_PARAMS *p)
+{
+	int err = -EINVAL;
+	struct ffa_send_direct_data data = { 0 };
+
+	// Fill the reg TX command parameters
+	data.data0 = p->cmd;
+	// 64 bit of the physical address
+	data.data1 = p->indata[0];
+	// 32 bit of the reg value
+	data.data2 = p->indata[1];
+
+	if (!mods_ffa_info.ffa_ops) {
+		cl_error("mods ffa cmd error, device not found\n");
+		return -ENODEV;
+	}
+
+	switch (p->cmd) {
+	case MODS_FFA_CMD_READ_REG:
+		// Read command
+		cl_debug(DEBUG_TEGRADMA, "sending data to SP :read cmd 0x%llx, addr:0x%llx\n",
+					  (unsigned long long)data.data0,
+					  (unsigned long long)data.data1);
+		break;
+	case MODS_FFA_CMD_WRITE_REG:
+		// Write command
+		cl_debug(DEBUG_TEGRADMA, "sending data to SP :write cmd 0x%llx,addr:0x%llx,write_val:0x%llx\n",
+					  (unsigned long long)data.data0,
+					  (unsigned long long)data.data1,
+					  (unsigned long long)data.data2);
+		break;
+	case MODS_FFA_CMD_READ_VER:
+		cl_debug(DEBUG_TEGRADMA, "sending cmd MODS_FFA_CMD_READ_VER to SP\n");
+		break;
+	case MODS_FFA_CMD_SE_TESTS:
+		cl_debug(DEBUG_TEGRADMA, "sending SE_TESTS data to SP :read cmd 0x%llx, alg|engineId:0x%llx\n",
+					  (unsigned long long)data.data0,
+					  (unsigned long long)data.data1);
+		break;
+	case MODS_FFA_CMD_SE_KEY_MOVER:
+		cl_debug(DEBUG_TEGRADMA, "sending SE_KEY_MOVER data to SP :read cmd 0x%llx, data:0x%llx\n",
+					  (unsigned long long)data.data0,
+					  (unsigned long long)data.data1);
+		break;
+	case MODS_FFA_CMD_HSS_TEST:
+		cl_debug(DEBUG_TEGRADMA, "sending cmd MODS_FFA_CMD_HSS_TEST to SP\n");
+		break;
+	case MODS_FFA_CMD_C2C_TEST:
+		cl_debug(DEBUG_TEGRADMA, "sending cmd MODS_FFA_CMD_C2C_TEST to SP\n");
+		break;
+	default:
+		cl_error("Unexpected command from SP 0x%llx\n", (unsigned long long)p->cmd);
+		return err;
+	}
+
+	err = mods_ffa_info.ffa_ops->sync_send_receive(mods_ffa_info.ffa_dev, &data);
+
+	switch (p->cmd) {
+	case MODS_FFA_CMD_READ_REG:
+		// Read command
+		cl_debug(DEBUG_TEGRADMA, "received read reg status from SP status:%d,read_val:0x%llx\n",
+					  err, (unsigned long long)data.data1);
+		p->outdata[0] = data.data1;
+		break;
+	case MODS_FFA_CMD_WRITE_REG:
+		// write command
+		cl_debug(DEBUG_TEGRADMA, "received write reg status from SP status: %d\n",
+					  err);
+		break;
+	case MODS_FFA_CMD_READ_VER:
+		cl_debug(DEBUG_TEGRADMA, "received version from SP : 0x%llx\n",
+					  (unsigned long long)data.data1);
+		p->outdata[0] = data.data1;
+		break;
+	case MODS_FFA_CMD_HSS_TEST:
+		cl_debug(DEBUG_TEGRADMA, "received response from SP for CMD_HSS_TEST: 0x%llx\n",
+					  (unsigned long long)data.data1);
+		p->outdata[0] = data.data1;
+		break;
+	case MODS_FFA_CMD_C2C_TEST:
+		cl_debug(DEBUG_TEGRADMA, "received response from SP for CMD_C2C_TEST: 0x%llx\n",
+					  (unsigned long long)data.data1);
+		p->outdata[0] = data.data1;
+		break;
+	}
+
+	if (err) {
+		cl_error("unexpected error from SP: %d\n", err);
+		return err;
+	}
+	// data.data0 always holds the error code of the ffa cmd
+	if (data.data0) {
+		cl_error("error response from SP: %ld\n", (long)data.data0);
+		return -EFAULT;
+	}
+	return OK;
+}
diff --git a/drivers/misc/mods/mods_bpmpipc.c b/drivers/misc/mods/mods_bpmpipc.c
new file mode 100644
index 00000000..f4a3b7a0
--- /dev/null
+++ b/drivers/misc/mods/mods_bpmpipc.c
@@ -0,0 +1,397 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * This file is part of NVIDIA MODS kernel driver.
+ *
+ * Copyright (c) 2022-2023, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * NVIDIA MODS kernel driver is free software: you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * NVIDIA MODS kernel driver is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NVIDIA MODS kernel driver.
+ * If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "mods_internal.h"
+
+#include <linux/delay.h>
+#include <linux/io.h>
+#include <linux/types.h>
+
+#include <soc/tegra/ivc.h>
+#include <soc/tegra/bpmp.h>
+
+#if (KERNEL_VERSION(6, 2, 0) <= MODS_KERNEL_VERSION)
+#include <linux/iosys-map.h>
+#endif
+
+#define IVC_CHANNEL_SIZE  256
+#define MRQ_MSG_SIZE      128
+#define BPMP_MAIL_DO_ACK  (1U << 0U)
+#define BPMP_IVC_TIMEOUT  120000 /* really large timeout to support simulation platforms */
+
+static DEFINE_MUTEX(mods_bpmpipc_lock);
+
+static const u32 MODS_CMD_UPHY_LANE_EOM_SCAN = 9;
+
+struct mods_cmd_uphy_lane_eom_scan_request {
+	u32 brick;
+	u32 lane;
+	u32 pcie_gen5;
+};
+
+struct mods_cmd_uphy_lane_eom_scan_response {
+	u32 data;
+};
+
+struct mods_mrq_uphy_request {
+	u16 lane;
+	u16 cmd;
+	struct mods_cmd_uphy_lane_eom_scan_request lane_eom_scan;
+};
+
+struct mods_mrq_uphy_response {
+	struct mods_cmd_uphy_lane_eom_scan_response eom_status;
+};
+
+struct bpmp_ipc_ch {
+	bool is_init;
+	struct tegra_ivc ivc;
+	void __iomem *db_base;
+	void __iomem *req_base;
+	void __iomem *resp_base;
+	phys_addr_t db_phys_addr;
+	phys_addr_t req_phys_addr;
+	phys_addr_t resp_phys_addr;
+};
+
+static struct bpmp_ipc_ch mods_bpmp_ch = {.is_init = false};
+
+static void bpmp_ivc_notify(struct tegra_ivc *ivc, void *data)
+{
+	struct bpmp_ipc_ch *bpmp_ipc_ch = (struct bpmp_ipc_ch *)data;
+
+	__raw_writel(1, bpmp_ipc_ch->db_base);
+}
+
+static int bpmp_ipc_send(struct mods_client *client,
+			 struct tegra_ivc *ivc,
+			 const void *data,
+			 size_t sz)
+{
+#if (KERNEL_VERSION(6, 2, 0) <= MODS_KERNEL_VERSION)
+	int err;
+	struct iosys_map ob;
+
+	err = tegra_ivc_write_get_next_frame(ivc, &ob);
+	if (err) {
+		cl_error("failed to get next tegra-ivc output frame!\n");
+		iosys_map_clear(&ob);
+		return err;
+	}
+	iosys_map_memcpy_to(&ob, 0, data, sz);
+#else
+	void *frame;
+
+	frame = tegra_ivc_write_get_next_frame(ivc);
+	if (IS_ERR(frame)) {
+		cl_error("failed to get next tegra-ivc output frame!\n");
+		return PTR_ERR(frame);
+	}
+
+	memcpy_toio(frame, data, sz);
+#endif
+
+	return tegra_ivc_write_advance(ivc);
+}
+
+static int bpmp_ipc_recv(struct mods_client *client,
+			 struct tegra_ivc *ivc,
+			 void *data,
+			 size_t sz,
+			 u32 timeout_ms)
+{
+	int err;
+#if (KERNEL_VERSION(6, 2, 0) <= MODS_KERNEL_VERSION)
+	struct iosys_map ib;
+#else
+	const void *frame;
+#endif
+	ktime_t end;
+
+	end = ktime_add_ms(ktime_get(), timeout_ms);
+
+#if (KERNEL_VERSION(6, 2, 0) <= MODS_KERNEL_VERSION)
+	do {
+		err = tegra_ivc_read_get_next_frame(ivc, &ib);
+		if (!err)
+			break;
+	} while (ktime_before(ktime_get(), end));
+	if (err) {
+		iosys_map_clear(&ib);
+		err = tegra_ivc_read_get_next_frame(ivc, &ib);
+		if (err) {
+			cl_error("get next tegra-ivc input frame timeout\n");
+			iosys_map_clear(&ib);
+			return err;
+		}
+	}
+	iosys_map_memcpy_from(data, &ib, 0, sz);
+#else
+	do {
+		frame = tegra_ivc_read_get_next_frame(ivc);
+		if (!IS_ERR(frame))
+			break;
+	} while (ktime_before(ktime_get(), end));
+
+	if (IS_ERR(frame)) {
+		frame = tegra_ivc_read_get_next_frame(ivc);
+
+		if (IS_ERR(frame)) {
+			cl_error("get next tegra-ivc input frame timeout\n");
+			return -ETIMEDOUT;
+		}
+	}
+	memcpy_fromio(data, frame, sz);
+#endif
+
+	err = tegra_ivc_read_advance(ivc);
+	if (err < 0)
+		cl_error("tegra_ivc read failed: %d\n", err);
+
+	return err;
+}
+
+static int bpmp_transfer(struct mods_client *client,
+			 struct tegra_bpmp_message *msg)
+{
+	int err;
+	struct tegra_bpmp_mb_data req;
+	struct tegra_bpmp_mb_data resp;
+
+	req.code = msg->mrq;
+	req.flags = BPMP_MAIL_DO_ACK;
+	memcpy(req.data, msg->tx.data, msg->tx.size);
+	err = bpmp_ipc_send(client, &mods_bpmp_ch.ivc, &req, sizeof(req));
+
+	if (err == 0) {
+		err = bpmp_ipc_recv(client, &mods_bpmp_ch.ivc,
+				&resp,
+				sizeof(resp),
+				BPMP_IVC_TIMEOUT);
+	}
+
+	if (err == 0) {
+		memcpy(msg->rx.data, resp.data, msg->rx.size);
+		msg->rx.ret = resp.code;
+	}
+
+	return err;
+}
+
+static int mrq_uphy_lane_eom_scan(struct mods_client *client,
+				  u32  brick,
+				  u32  lane,
+				  u32  pcie_gen5,
+				  u32  *data)
+{
+	int err;
+	struct mods_mrq_uphy_request req = {
+		.cmd = cpu_to_le32(MODS_CMD_UPHY_LANE_EOM_SCAN)
+	};
+	struct mods_mrq_uphy_response resp;
+	struct tegra_bpmp_message msg = {
+		.mrq = MRQ_UPHY,
+		.tx = {
+			.data = &req,
+			.size = sizeof(req),
+		},
+		.rx = {
+			.data = &resp,
+			.size = sizeof(resp),
+		},
+	};
+
+	req.lane_eom_scan.brick     = brick;
+	req.lane_eom_scan.lane      = lane;
+	req.lane_eom_scan.pcie_gen5 = pcie_gen5;
+
+	err = bpmp_transfer(client, &msg);
+
+	if (err < 0) {
+		return err;
+	} else if (msg.rx.ret < 0) {
+		err = -EINVAL;
+		return err;
+	}
+
+	*data = resp.eom_status.data;
+	return err;
+}
+
+static int bpmp_ioremap(struct mods_client *client,
+			struct bpmp_ipc_ch *bpmp_ipc_ch,
+			u64 db_phys_addr,
+			u64 req_phys_addr,
+			u64 resp_phys_addr)
+{
+	bpmp_ipc_ch->db_phys_addr   = db_phys_addr;
+	bpmp_ipc_ch->req_phys_addr  = req_phys_addr;
+	bpmp_ipc_ch->resp_phys_addr = resp_phys_addr;
+
+	bpmp_ipc_ch->db_base = ioremap(bpmp_ipc_ch->db_phys_addr,   64);
+	if (!bpmp_ipc_ch->db_base) {
+		cl_error("failed to remap aperture: 0x%llx\n",
+			 (unsigned long long)bpmp_ipc_ch->db_phys_addr);
+		return -ENOMEM;
+	}
+	bpmp_ipc_ch->req_base  = ioremap(bpmp_ipc_ch->req_phys_addr,  IVC_CHANNEL_SIZE);
+	if (!bpmp_ipc_ch->req_base) {
+		iounmap(bpmp_ipc_ch->db_base);
+		cl_error("failed to remap aperture: 0x%llx\n",
+			 (unsigned long long)bpmp_ipc_ch->req_phys_addr);
+		return -ENOMEM;
+	}
+	bpmp_ipc_ch->resp_base = ioremap(bpmp_ipc_ch->resp_phys_addr, IVC_CHANNEL_SIZE);
+	if (!bpmp_ipc_ch->resp_base) {
+		iounmap(bpmp_ipc_ch->db_base);
+		iounmap(bpmp_ipc_ch->req_base);
+		cl_error("failed to remap aperture: 0x%llx\n",
+			 (unsigned long long)bpmp_ipc_ch->resp_phys_addr);
+		return -ENOMEM;
+	}
+
+	return OK;
+}
+
+static void bpmp_iounmap(struct bpmp_ipc_ch *bpmp_ipc_ch)
+{
+	iounmap(bpmp_ipc_ch->db_base);
+	iounmap(bpmp_ipc_ch->req_base);
+	iounmap(bpmp_ipc_ch->resp_base);
+
+	bpmp_ipc_ch->db_phys_addr = 0;
+	bpmp_ipc_ch->req_phys_addr = 0;
+	bpmp_ipc_ch->resp_phys_addr = 0;
+}
+
+static int bpmp_ipc_channel_init(struct mods_client *client,
+				 struct bpmp_ipc_ch *bpmp_ipc_ch)
+{
+	int err;
+	ktime_t end;
+
+	err = tegra_ivc_init(&bpmp_ipc_ch->ivc, NULL,
+			 bpmp_ipc_ch->resp_base, 0,
+			 bpmp_ipc_ch->req_base, 0,
+			 1, MRQ_MSG_SIZE,
+			 bpmp_ivc_notify, bpmp_ipc_ch);
+
+
+	if (err != 0) {
+		cl_error("tegra-ivc init failed: %d\n", err);
+		return err;
+	}
+
+	tegra_ivc_reset(&bpmp_ipc_ch->ivc);
+
+	end = ktime_add_us(ktime_get(), 2000 * 1000);
+
+	while (tegra_ivc_notified(&bpmp_ipc_ch->ivc) != 0) {
+		usleep_range(100, 200);
+		if (ktime_after(ktime_get(), end)) {
+			cl_error("initialize IVC connection timeout\n");
+			err = -ETIMEDOUT;
+			break;
+		}
+	}
+
+	bpmp_ipc_ch->is_init = true;
+
+	return err;
+}
+
+static void bpmp_ipc_channel_uninit(struct bpmp_ipc_ch *bpmp_ipc_ch)
+{
+	tegra_ivc_cleanup(&bpmp_ipc_ch->ivc);
+}
+
+int mods_bpmpipc_init(struct mods_client *client,
+		      u64 db_phys_addr,
+		      u64 req_phys_addr,
+		      u64 resp_phys_addr)
+{
+	int err = OK;
+
+	if (mods_bpmp_ch.is_init) {
+		if (mods_bpmp_ch.db_phys_addr == db_phys_addr &&
+		    mods_bpmp_ch.req_phys_addr == req_phys_addr &&
+		    mods_bpmp_ch.resp_phys_addr == resp_phys_addr)
+			return OK;
+		mods_bpmpipc_cleanup();
+	}
+
+	err = bpmp_ioremap(client,
+			   &mods_bpmp_ch,
+			   db_phys_addr,
+			   req_phys_addr,
+			   resp_phys_addr);
+	if (err != OK)
+		return err;
+
+	err = bpmp_ipc_channel_init(client, &mods_bpmp_ch);
+	if (err != OK) {
+		bpmp_iounmap(&mods_bpmp_ch);
+		return err;
+	}
+
+	mods_bpmp_ch.is_init = true;
+	mods_debug_printk(DEBUG_TEGRADMA, "bpmp ipc init done\n");
+
+	return err;
+}
+
+void mods_bpmpipc_cleanup(void)
+{
+	if (!mods_bpmp_ch.is_init)
+		return;
+
+	bpmp_ipc_channel_uninit(&mods_bpmp_ch);
+	bpmp_iounmap(&mods_bpmp_ch);
+	mods_bpmp_ch.is_init = false;
+}
+
+int esc_mods_bpmp_uphy_lane_eom_scan(struct mods_client *client,
+				     struct MODS_BPMP_UPHY_LANE_EOM_SCAN_PARAMS *p)
+{
+	int err = OK;
+
+	mutex_lock(&mods_bpmpipc_lock);
+
+	err = mods_bpmpipc_init(client,
+				p->db_phys_addr,
+				p->req_phys_addr,
+				p->resp_phys_addr);
+	if (err != OK)
+		goto error;
+
+	err = mrq_uphy_lane_eom_scan(client,
+				     p->brick,
+				     p->lane,
+				     p->pcie_gen5,
+				     &p->data);
+
+	if (err != OK)
+		cl_error("mrq uphy lane eom scan failed with brick(%u), lane(%u), pcie_gen5(%u)\n",
+			 p->brick, p->lane, p->pcie_gen5);
+
+error:
+	mutex_unlock(&mods_bpmpipc_lock);
+	return err;
+}
diff --git a/drivers/misc/mods/mods_clock.c b/drivers/misc/mods/mods_clock.c
new file mode 100644
index 00000000..86477184
--- /dev/null
+++ b/drivers/misc/mods/mods_clock.c
@@ -0,0 +1,737 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * This file is part of NVIDIA MODS kernel driver.
+ *
+ * Copyright (c) 2011-2022, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * NVIDIA MODS kernel driver is free software: you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * NVIDIA MODS kernel driver is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NVIDIA MODS kernel driver.
+ * If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "mods_internal.h"
+#include <linux/clk.h>
+#include <linux/clk-provider.h>
+#include <linux/of_device.h>
+#include <linux/of.h>
+#include <linux/of_fdt.h>
+#include <linux/reset.h>
+
+#define ARBITRARY_MAX_CLK_FREQ  3500000000
+
+static struct list_head mods_clock_handles;
+static spinlock_t mods_clock_lock;
+static u32 last_handle;
+
+struct clock_entry {
+	struct clk *pclk;
+	u32 handle;
+	struct list_head list;
+};
+
+static LIST_HEAD(reset_handles);
+
+struct reset_data {
+	char name[MAX_DT_SIZE];
+	struct reset_control *rst;
+};
+
+struct reset_entry {
+	struct list_head list;
+	struct reset_data rst_data;
+	u32 handle;
+};
+
+static struct device_node *find_clocks_node(const char *name)
+{
+	const char *node_name = "mods-simple-bus";
+	struct device_node *pp = NULL, *np = NULL;
+
+	pp = of_find_node_by_name(NULL, node_name);
+
+	if (!pp) {
+		mods_warning_printk("'mods-simple-bus' node not found in device tree\n");
+		return pp;
+	}
+
+	np = of_get_child_by_name(pp, name);
+	return np;
+}
+
+void mods_init_clock_api(void)
+{
+	const char *okay_value = "okay";
+	struct device_node *mods_np = NULL;
+	struct property *pp = NULL;
+	int size_value = 0;
+
+	mods_np = find_clocks_node("mods-clocks");
+	if (!mods_np) {
+		mods_warning_printk("'mods-clocks' node not found in device tree\n");
+		goto err;
+	}
+
+	pp = of_find_property(mods_np, "status", NULL);
+	if (IS_ERR(pp)) {
+		mods_error_printk("'status' prop not found in 'mods-clocks' node.");
+		goto err;
+	}
+
+	/* if status is 'okay', then skip updating property */
+	if (of_device_is_available(mods_np))
+		goto err;
+
+	size_value = strlen(okay_value) + 1;
+	pp->value = kmalloc(size_value, GFP_KERNEL);
+	if (unlikely(!pp->value)) {
+		pp->length = 0;
+		goto err;
+	}
+	strncpy(pp->value, okay_value, size_value);
+	pp->length = size_value;
+
+err:
+	of_node_put(mods_np);
+
+	spin_lock_init(&mods_clock_lock);
+	INIT_LIST_HEAD(&mods_clock_handles);
+	last_handle = 0;
+}
+
+void mods_shutdown_clock_api(void)
+{
+	struct list_head   *head       = &mods_clock_handles;
+	struct list_head   *reset_head = &reset_handles;
+	struct reset_entry *entry      = NULL;
+	struct list_head   *iter       = NULL;
+	struct list_head   *tmp        = NULL;
+
+	spin_lock(&mods_clock_lock);
+
+	list_for_each_safe(iter, tmp, head) {
+		struct clock_entry *entry
+			= list_entry(iter, struct clock_entry, list);
+		list_del(iter);
+		kfree(entry);
+	}
+
+	list_for_each_safe(iter, tmp, reset_head) {
+		entry = list_entry(iter, struct reset_entry, list);
+		list_del(iter);
+		kfree(entry);
+	}
+
+	spin_unlock(&mods_clock_lock);
+}
+
+static u32 mods_get_clock_handle(struct clk *pclk)
+{
+	struct list_head *head = &mods_clock_handles;
+	struct list_head *iter;
+	struct clock_entry *entry = NULL;
+	u32 handle = 0;
+
+	spin_lock(&mods_clock_lock);
+
+	list_for_each(iter, head) {
+		struct clock_entry *cur
+			= list_entry(iter, struct clock_entry, list);
+		if (cur->pclk == pclk) {
+			entry = cur;
+			handle = cur->handle;
+			break;
+		}
+	}
+
+	if (!entry) {
+		entry = kmalloc(sizeof(*entry), GFP_ATOMIC);
+		if (!unlikely(!entry)) {
+			entry->pclk = pclk;
+			entry->handle = ++last_handle;
+			handle = entry->handle;
+			list_add(&entry->list, &mods_clock_handles);
+		}
+	}
+
+	spin_unlock(&mods_clock_lock);
+
+	return handle;
+}
+
+static struct clk *mods_get_clock(u32 handle)
+{
+	struct list_head *head = &mods_clock_handles;
+	struct list_head *iter = NULL;
+	struct clk *pclk = NULL;
+
+	spin_lock(&mods_clock_lock);
+
+	list_for_each(iter, head) {
+		struct clock_entry *entry
+			= list_entry(iter, struct clock_entry, list);
+		if (entry->handle == handle) {
+			pclk = entry->pclk;
+			break;
+		}
+	}
+
+	spin_unlock(&mods_clock_lock);
+
+	return pclk;
+}
+
+static struct reset_data find_reset_data(u32 handle)
+{
+	struct list_head   *entry     = NULL;
+	struct reset_entry *rst_entry = NULL;
+	struct reset_data  reset_data = {"", NULL};
+
+	spin_lock(&mods_clock_lock);
+
+	list_for_each(entry, &reset_handles) {
+		rst_entry = list_entry(entry, struct reset_entry, list);
+		if (handle == rst_entry->handle) {
+			reset_data = rst_entry->rst_data;
+			break;
+		}
+	}
+
+	spin_unlock(&mods_clock_lock);
+
+	return reset_data;
+}
+
+static int get_reset_handle(struct reset_data reset_data)
+{
+	int    handle                 = -1;
+	struct list_head *entry       = NULL;
+	struct reset_entry *rst_entry = NULL;
+
+	spin_lock(&mods_clock_lock);
+
+	/* If entry has no rst structure, we are past last cached entry */
+	list_for_each(entry, &reset_handles) {
+		rst_entry = list_entry(entry, struct reset_entry, list);
+		handle = rst_entry->handle;
+		if (strcmp(rst_entry->rst_data.name,
+			   reset_data.name) == 0) {
+			goto failed;
+		}
+	}
+
+	/* If reset not already in array, then we must add it */
+	rst_entry = kzalloc(sizeof(struct reset_entry), GFP_ATOMIC);
+	if (unlikely(!rst_entry)) {
+		handle = -1;
+		goto failed;
+	}
+	rst_entry->handle = ++handle;
+	rst_entry->rst_data = reset_data;
+	INIT_LIST_HEAD(&rst_entry->list);
+	list_add_tail(&rst_entry->list, &reset_handles);
+
+failed:
+	spin_unlock(&mods_clock_lock);
+	return handle;
+}
+
+int esc_mods_get_clock_handle(struct mods_client *client,
+			      struct MODS_GET_CLOCK_HANDLE *p)
+{
+	struct clk *pclk = NULL;
+	int ret = -EINVAL;
+
+	struct device_node *mods_np = NULL;
+	struct property *pp = NULL;
+
+	LOG_ENT();
+
+	mods_np = find_clocks_node("mods-clocks");
+	if (!mods_np || !of_device_is_available(mods_np)) {
+		cl_warn("'mods-clocks' node not found in device tree\n");
+		goto err;
+	}
+	pp = of_find_property(mods_np, "clock-names", NULL);
+	if (IS_ERR(pp)) {
+		cl_error(
+		    "No 'clock-names' prop in 'mods-clocks' node for dev %s\n",
+				  p->controller_name);
+		goto err;
+	}
+
+	pclk = of_clk_get_by_name(mods_np, p->controller_name);
+
+	if (IS_ERR(pclk))
+		cl_error("clk (%s) not found\n", p->controller_name);
+	else {
+		p->clock_handle = mods_get_clock_handle(pclk);
+		ret = OK;
+	}
+err:
+	of_node_put(mods_np);
+	LOG_EXT();
+	return ret;
+}
+
+int esc_mods_get_rst_handle(struct mods_client *client,
+			    struct MODS_GET_RESET_HANDLE *p)
+{
+	struct reset_data reset_data = {{0}, NULL};
+	struct reset_control *p_reset_ctrl = NULL;
+	int ret = -EINVAL;
+
+	struct device_node *mods_np = NULL;
+	struct property *pp = NULL;
+
+	LOG_ENT();
+
+	mods_np = find_clocks_node("mods-clocks");
+	if (!mods_np || !of_device_is_available(mods_np)) {
+		cl_warn("'mods-clocks' node not found in device tree\n");
+		goto err;
+	}
+	pp = of_find_property(mods_np, "reset-names", NULL);
+	if (IS_ERR(pp)) {
+		cl_error(
+		"No 'reset-names' prop in 'mods-clocks' node for dev %s\n",
+			p->reset_name);
+		goto err;
+	}
+
+	p_reset_ctrl = of_reset_control_get(mods_np, p->reset_name);
+
+	if (IS_ERR(p_reset_ctrl))
+		cl_error("reset (%s) not found\n", p->reset_name);
+	else {
+		strncpy(reset_data.name, p->reset_name,
+			sizeof(reset_data.name) - 1);
+		if (reset_data.name[sizeof(reset_data.name) - 1] != '\0') {
+			cl_error(
+			"reset name %sis too large to store in reset array\n",
+			reset_data.name);
+			goto err;
+		}
+		reset_data.rst = p_reset_ctrl;
+		p->reset_handle = get_reset_handle(reset_data);
+		if (p->reset_handle == -1) {
+			cl_error("no valid reset handle was acquired");
+			goto err;
+		}
+		ret = OK;
+	}
+err:
+	of_node_put(mods_np);
+	LOG_EXT();
+	return ret;
+}
+
+int esc_mods_set_clock_rate(struct mods_client *client,
+			    struct MODS_CLOCK_RATE *p)
+{
+	struct clk *pclk = NULL;
+	int ret = -EINVAL;
+
+	LOG_ENT();
+
+	pclk = mods_get_clock(p->clock_handle);
+
+	if (!pclk) {
+		cl_error("unrecognized clock handle: 0x%x\n",
+				  p->clock_handle);
+	} else {
+		ret = clk_set_rate(pclk, p->clock_rate_hz);
+		if (ret) {
+			cl_error(
+				"unable to set rate %lluHz on clock 0x%x\n",
+				p->clock_rate_hz, p->clock_handle);
+		} else {
+			cl_debug(DEBUG_CLOCK,
+				 "successfuly set rate %lluHz on clock 0x%x\n",
+				 p->clock_rate_hz, p->clock_handle);
+		}
+	}
+
+	LOG_EXT();
+	return ret;
+}
+
+int esc_mods_get_clock_rate(struct mods_client *client,
+			    struct MODS_CLOCK_RATE *p)
+{
+	struct clk *pclk = NULL;
+	int ret = -EINVAL;
+
+	LOG_ENT();
+
+	pclk = mods_get_clock(p->clock_handle);
+
+	if (!pclk) {
+		cl_error("unrecognized clock handle: 0x%x\n",
+				  p->clock_handle);
+	} else {
+		p->clock_rate_hz = clk_get_rate(pclk);
+		cl_debug(DEBUG_CLOCK, "clock 0x%x has rate %lluHz\n",
+			 p->clock_handle, p->clock_rate_hz);
+		ret = OK;
+	}
+
+	LOG_EXT();
+	return ret;
+}
+
+int esc_mods_get_clock_max_rate(struct mods_client *client,
+				struct MODS_CLOCK_RATE *p)
+{
+	struct clk *pclk = NULL;
+	int ret = -EINVAL;
+
+	LOG_ENT();
+	pclk = mods_get_clock(p->clock_handle);
+
+	if (!pclk) {
+		cl_error("unrecognized clock handle: 0x%x\n", p->clock_handle);
+	} else {
+		long rate = clk_round_rate(pclk, ARBITRARY_MAX_CLK_FREQ);
+
+		p->clock_rate_hz = rate < 0 ? ARBITRARY_MAX_CLK_FREQ
+			: (unsigned long)rate;
+		cl_debug(DEBUG_CLOCK,
+			 "clock 0x%x has max rate %lluHz\n",
+			 p->clock_handle, p->clock_rate_hz);
+		ret = OK;
+	}
+
+	LOG_EXT();
+	return ret;
+}
+
+int esc_mods_set_clock_max_rate(struct mods_client *client,
+				struct MODS_CLOCK_RATE *p)
+{
+	struct clk *pclk = NULL;
+	int ret = -EINVAL;
+
+	LOG_ENT();
+
+	pclk = mods_get_clock(p->clock_handle);
+
+	if (!pclk) {
+		cl_error("unrecognized clock handle: 0x%x\n", p->clock_handle);
+	} else {
+#if defined(CONFIG_TEGRA_CLOCK_DEBUG_FUNC)
+		ret = tegra_clk_set_max(pclk, p->clock_rate_hz);
+		if (ret) {
+			cl_error(
+				"unable to override max clock rate %lluHz on clock 0x%x\n",
+				p->clock_rate_hz, p->clock_handle);
+		} else {
+			cl_debug(DEBUG_CLOCK,
+				 "successfuly set max rate %lluHz on clock 0x%x\n",
+				 p->clock_rate_hz, p->clock_handle);
+		}
+#else
+		cl_error("unable to override max clock rate\n");
+		cl_error(
+			"reconfigure kernel with CONFIG_TEGRA_CLOCK_DEBUG_FUNC=y\n");
+		ret = -EINVAL;
+#endif
+	}
+
+	LOG_EXT();
+	return ret;
+}
+
+int esc_mods_set_clock_parent(struct mods_client *client,
+			      struct MODS_CLOCK_PARENT *p)
+{
+	struct clk *pclk = NULL;
+	struct clk *pparent = NULL;
+	int ret = -EINVAL;
+
+	LOG_ENT();
+
+	pclk = mods_get_clock(p->clock_handle);
+	pparent = mods_get_clock(p->clock_parent_handle);
+
+	if (!pclk) {
+		cl_error("unrecognized clock handle: 0x%x\n", p->clock_handle);
+	} else if (!pparent) {
+		cl_error("unrecognized parent clock handle: 0x%x\n",
+			 p->clock_parent_handle);
+	} else {
+		ret = clk_set_parent(pclk, pparent);
+		if (ret) {
+			cl_error(
+				"unable to make clock 0x%x parent of clock 0x%x\n",
+				p->clock_parent_handle, p->clock_handle);
+		} else {
+			cl_debug(DEBUG_CLOCK,
+				 "successfuly made clock 0x%x parent of clock 0x%x\n",
+				 p->clock_parent_handle, p->clock_handle);
+		}
+	}
+
+	LOG_EXT();
+	return ret;
+}
+
+int esc_mods_get_clock_parent(struct mods_client *client,
+			      struct MODS_CLOCK_PARENT *p)
+{
+	struct clk *pclk = NULL;
+	int ret = -EINVAL;
+
+	LOG_ENT();
+
+	pclk = mods_get_clock(p->clock_handle);
+
+	if (!pclk) {
+		cl_error("unrecognized clock handle: 0x%x\n", p->clock_handle);
+	} else {
+		struct clk *pparent = clk_get_parent(pclk);
+
+		p->clock_parent_handle = mods_get_clock_handle(pparent);
+		cl_debug(DEBUG_CLOCK,
+			 "clock 0x%x is parent of clock 0x%x\n",
+			 p->clock_parent_handle, p->clock_handle);
+		ret = OK;
+	}
+
+	LOG_EXT();
+	return ret;
+}
+
+int esc_mods_enable_clock(struct mods_client *client,
+			  struct MODS_CLOCK_HANDLE *p)
+{
+	struct clk *pclk = NULL;
+	int ret = -EINVAL;
+
+	LOG_ENT();
+
+	pclk = mods_get_clock(p->clock_handle);
+
+	if (!pclk) {
+		cl_error("unrecognized clock handle: 0x%x\n", p->clock_handle);
+	} else {
+		ret = clk_prepare(pclk);
+		if (ret) {
+			cl_error(
+				"unable to prepare clock 0x%x before enabling\n",
+				p->clock_handle);
+		}
+		ret = clk_enable(pclk);
+		if (ret) {
+			cl_error("failed to enable clock 0x%x\n",
+				 p->clock_handle);
+		} else {
+			cl_debug(DEBUG_CLOCK, "clock 0x%x enabled\n",
+				 p->clock_handle);
+		}
+	}
+
+	LOG_EXT();
+	return ret;
+}
+
+int esc_mods_disable_clock(struct mods_client *client,
+			   struct MODS_CLOCK_HANDLE *p)
+{
+	struct clk *pclk = NULL;
+	int ret = -EINVAL;
+
+	LOG_ENT();
+
+	pclk = mods_get_clock(p->clock_handle);
+
+	if (!pclk) {
+		cl_error("unrecognized clock handle: 0x%x\n", p->clock_handle);
+	} else {
+		clk_disable(pclk);
+		clk_unprepare(pclk);
+		cl_debug(DEBUG_CLOCK, "clock 0x%x disabled\n",
+			 p->clock_handle);
+		ret = OK;
+	}
+
+	LOG_EXT();
+	return ret;
+}
+
+int esc_mods_is_clock_enabled(struct mods_client *client,
+			      struct MODS_CLOCK_ENABLED *p)
+{
+	struct clk *pclk = NULL;
+	int ret = -EINVAL;
+
+	LOG_ENT();
+
+	pclk = mods_get_clock(p->clock_handle);
+
+	if (!pclk) {
+		cl_error("unrecognized clock handle: 0x%x\n", p->clock_handle);
+	} else {
+		p->enable_count = (u32)__clk_is_enabled(pclk);
+		cl_debug(DEBUG_CLOCK, "clock 0x%x enable count is %u\n",
+			 p->clock_handle, p->enable_count);
+		ret = OK;
+	}
+
+	LOG_EXT();
+	return ret;
+}
+
+int esc_mods_reset_assert(struct mods_client *client,
+			  struct MODS_RESET_HANDLE *p)
+{
+	int err = -EINVAL;
+	const struct reset_data reset_data = find_reset_data(p->handle);
+	struct device_node *mods_np = NULL;
+
+	LOG_ENT();
+	mods_np = find_clocks_node("mods-clocks");
+	if (!mods_np || !of_device_is_available(mods_np)) {
+		cl_warn("'mods-clocks' node not found in DTB\n");
+		goto error;
+	}
+
+	if (!reset_data.rst) {
+		cl_error("No reset corresponding to requested handle!\n");
+		goto error;
+	}
+
+	if (p->assert)
+		err = reset_control_assert(reset_data.rst);
+	else
+		err = reset_control_deassert(reset_data.rst);
+	if (err) {
+		cl_error("failed to %s reset on '%s'\n",
+			 (p->assert ? "asserted" : "deasserted"),
+			 reset_data.name);
+	} else {
+		cl_debug(DEBUG_CLOCK, "%s reset on '%s'",
+			 (p->assert ? "asserted" : "desasserted"),
+			 reset_data.name);
+	}
+error:
+	of_node_put(mods_np);
+
+	LOG_EXT();
+	return err;
+}
+
+int esc_mods_clock_reset_assert(struct mods_client *client,
+				struct MODS_CLOCK_HANDLE *p)
+{
+	struct clk *pclk = NULL;
+	int ret = -EINVAL;
+
+	LOG_ENT();
+
+	pclk = mods_get_clock(p->clock_handle);
+
+	if (!pclk) {
+		cl_error("unrecognized clock handle: 0x%x\n", p->clock_handle);
+	} else {
+		const char *clk_name = NULL;
+		struct reset_control *prst = NULL;
+		struct device_node *mods_np = NULL;
+		struct property *pp = NULL;
+
+		mods_np = find_clocks_node("mods-clocks");
+		if (!mods_np || !of_device_is_available(mods_np)) {
+			cl_warn("'mods-clocks' node not found in DTB\n");
+			goto err;
+		}
+		pp = of_find_property(mods_np, "reset-names", NULL);
+		if (IS_ERR(pp)) {
+			cl_error(
+				"No 'reset-names' prop in 'mods-clocks' node for dev %s\n",
+				__clk_get_name(pclk));
+			goto err;
+		}
+
+		clk_name = __clk_get_name(pclk);
+
+		prst = of_reset_control_get(mods_np, clk_name);
+		if (IS_ERR(prst)) {
+			cl_error("reset device %s not found\n", clk_name);
+			goto err;
+		}
+		ret = reset_control_assert(prst);
+		if (ret) {
+			cl_error("failed to assert reset on '%s'\n", clk_name);
+		} else {
+			cl_debug(DEBUG_CLOCK, "asserted reset on '%s'",
+				 clk_name);
+		}
+
+err:
+		of_node_put(mods_np);
+	}
+	LOG_EXT();
+	return ret;
+}
+
+int esc_mods_clock_reset_deassert(struct mods_client *client,
+				  struct MODS_CLOCK_HANDLE *p)
+{
+	struct clk *pclk = NULL;
+	int ret = -EINVAL;
+
+	LOG_ENT();
+
+	pclk = mods_get_clock(p->clock_handle);
+
+	if (!pclk) {
+		cl_error("unrecognized clock handle: 0x%x\n", p->clock_handle);
+	} else {
+		const char *clk_name = NULL;
+		struct reset_control *prst = NULL;
+		struct device_node *mods_np = NULL;
+		struct property *pp = NULL;
+
+		mods_np = find_clocks_node("mods-clocks");
+		if (!mods_np || !of_device_is_available(mods_np)) {
+			cl_warn("'mods-clocks' node not found in DTB\n");
+			goto err;
+		}
+		pp = of_find_property(mods_np, "reset-names", NULL);
+		if (IS_ERR(pp)) {
+			cl_error(
+				"No 'reset-names' prop in 'mods-clocks' node for dev %s\n",
+				__clk_get_name(pclk));
+			goto err;
+		}
+
+		clk_name = __clk_get_name(pclk);
+
+		prst = of_reset_control_get(mods_np, clk_name);
+		if (IS_ERR(prst)) {
+			cl_error("reset device %s not found\n", clk_name);
+			goto err;
+		}
+		ret = reset_control_deassert(prst);
+		if (ret) {
+			cl_error("failed to assert reset on '%s'\n", clk_name);
+		} else {
+			cl_debug(DEBUG_CLOCK, "deasserted reset on '%s'",
+				 clk_name);
+		}
+
+err:
+		of_node_put(mods_np);
+	}
+
+	LOG_EXT();
+	return ret;
+}
diff --git a/drivers/misc/mods/mods_config.h b/drivers/misc/mods/mods_config.h
new file mode 100644
index 00000000..b4d9615c
--- /dev/null
+++ b/drivers/misc/mods/mods_config.h
@@ -0,0 +1,118 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * This file is part of NVIDIA MODS kernel driver.
+ *
+ * Copyright (c) 2008-2023, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * NVIDIA MODS kernel driver is free software: you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * NVIDIA MODS kernel driver is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NVIDIA MODS kernel driver.
+ * If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _MODS_CONFIG_H_
+#define _MODS_CONFIG_H_
+
+#define MODS_KERNEL_VERSION LINUX_VERSION_CODE
+
+#if KERNEL_VERSION(2, 6, 30) <= MODS_KERNEL_VERSION && \
+	KERNEL_VERSION(4, 16, 0) > MODS_KERNEL_VERSION && \
+	defined(CONFIG_X86)
+#       define MODS_HAS_DMA_OPS 1
+#endif
+
+#if KERNEL_VERSION(2, 6, 30) > MODS_KERNEL_VERSION
+#       define MODS_HASNT_PCI_RESCAN_BUS 1
+#endif
+
+#if KERNEL_VERSION(2, 6, 31) <= MODS_KERNEL_VERSION
+#       define MODS_HAS_IORESOURCE_MEM_64 1
+#endif
+
+#if KERNEL_VERSION(2, 6, 33) <= MODS_KERNEL_VERSION
+#       define MODS_HAS_NEW_ACPI_WALK 1
+#else
+#       define MODS_HASNT_NUMA_NO_NODE 1
+#endif
+
+#if KERNEL_VERSION(2, 6, 34) <= MODS_KERNEL_VERSION
+#       define MODS_HAS_SET_COHERENT_MASK 1
+#endif
+
+#if KERNEL_VERSION(2, 6, 38) <= MODS_KERNEL_VERSION
+#       if defined(CONFIG_X86)
+#               define MODS_HAS_CONSOLE_LOCK 1
+#       endif
+#endif
+
+#if KERNEL_VERSION(3, 4, 0) > MODS_KERNEL_VERSION
+#       define MODS_HASNT_PCI_BUS_REMOVE_DEV 1
+#endif
+
+#if KERNEL_VERSION(3, 8, 0) <= MODS_KERNEL_VERSION
+#       define MODS_HAS_NEW_ACPI_HANDLE 1
+#       define MODS_HAS_SRIOV 1
+#endif
+
+#if KERNEL_VERSION(3, 14, 0) <= MODS_KERNEL_VERSION
+#       define MODS_HAS_MSIX_RANGE 1
+#else
+#       define MODS_HASNT_PCI_LOCK_RESCAN_REMOVE 1
+#endif
+
+#if KERNEL_VERSION(3, 16, 0) <= MODS_KERNEL_VERSION && \
+	defined(CONFIG_VT_HW_CONSOLE_BINDING)
+#       define MODS_HAS_CONSOLE_BINDING 1
+#endif
+
+#if KERNEL_VERSION(3, 19, 0) <= MODS_KERNEL_VERSION
+#       define MODS_HAS_DEV_PROPS 1
+#endif
+
+#if defined(CONFIG_PPC64) && KERNEL_VERSION(4, 5, 0) <= MODS_KERNEL_VERSION
+#       define MODS_HAS_PNV_PCI_GET_NPU_DEV 1
+#endif
+
+#if KERNEL_VERSION(4, 12, 0) <= MODS_KERNEL_VERSION && \
+	KERNEL_VERSION(4, 13, 0) > MODS_KERNEL_VERSION && \
+	defined(CONFIG_X86)
+#       define MODS_HAS_ASM_SET_MEMORY_HEADER 1
+#endif
+
+#if KERNEL_VERSION(4, 13, 0) <= MODS_KERNEL_VERSION
+#       define MODS_HAS_SET_MEMORY_HEADER 1
+#endif
+
+#if KERNEL_VERSION(4, 14, 0) <= MODS_KERNEL_VERSION
+#       define MODS_HAS_KERNEL_WRITE
+#endif
+
+#if KERNEL_VERSION(4, 16, 0) <= MODS_KERNEL_VERSION
+#       define MODS_HAS_POLL_T 1
+#endif
+
+#if defined(CONFIG_ACPI_NUMA) && KERNEL_VERSION(5, 1, 0) <= MODS_KERNEL_VERSION
+#       define MODS_HAS_PXM_TO_NODE 1
+#endif
+
+#if KERNEL_VERSION(5, 17, 0) <= MODS_KERNEL_VERSION
+#       define MODS_HAS_ACPI_FETCH 1
+#endif
+
+#if defined(MODS_HAS_TEGRA) && KERNEL_VERSION(5, 1, 0) <= MODS_KERNEL_VERSION
+#       define MODS_ENABLE_BPMP_MRQ_API 1
+#endif
+
+#if KERNEL_VERSION(6, 1, 0) > MODS_KERNEL_VERSION
+#       define MODS_HAS_FB_SET_SUSPEND 1
+#endif
+
+#endif /* _MODS_CONFIG_H_  */
diff --git a/drivers/misc/mods/mods_debugfs.c b/drivers/misc/mods/mods_debugfs.c
new file mode 100644
index 00000000..27a9fffe
--- /dev/null
+++ b/drivers/misc/mods/mods_debugfs.c
@@ -0,0 +1,769 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * This file is part of NVIDIA MODS kernel driver.
+ *
+ * Copyright (c) 2014-2022, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * NVIDIA MODS kernel driver is free software: you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * NVIDIA MODS kernel driver is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NVIDIA MODS kernel driver.
+ * If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "mods_internal.h"
+
+#include <linux/module.h>
+#include <linux/debugfs.h>
+#include <linux/seq_file.h>
+#include <linux/miscdevice.h>
+#include <linux/device.h>
+#include <linux/uaccess.h>
+
+static struct dentry *mods_debugfs_dir;
+
+#ifdef CONFIG_ARCH_TEGRA_19x_SOC
+#include "mods_ras.h"
+#endif
+
+#if defined(MODS_HAS_TEGRA) && defined(CONFIG_TEGRA_KFUSE)
+#include <soc/tegra/kfuse.h>
+#endif
+
+#if defined(MODS_HAS_TEGRA) && defined(CONFIG_TEGRA_DC)
+#include <video/tegra_dc_ext_kernel.h>
+#include <../drivers/video/tegra/dc/dc_config.h>
+#include <../drivers/video/tegra/dc/dsi.h>
+
+static int mods_dc_color_formats_show(struct seq_file *s, void *unused)
+{
+	struct tegra_dc *dc = s->private;
+	u32 i, j;
+
+	for (i = 0; i < tegra_dc_get_numof_dispwindows(); i++) {
+		struct tegra_dc_win *win;
+		u32 *fmt_masks;
+
+		win = tegra_dc_get_window(dc, i);
+		if (!win)
+			continue;
+		fmt_masks = tegra_dc_parse_feature(dc, i, GET_WIN_FORMATS);
+		if (!fmt_masks)
+			continue;
+		seq_printf(s, "window_%u:", i);
+		for (j = 0; j < WIN_FEATURE_ENTRY_SIZE; j++)
+			seq_printf(s, " 0x%08x", fmt_masks[j]);
+		seq_puts(s, "\n");
+	}
+	return 0;
+}
+
+static int mods_dc_color_formats_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, mods_dc_color_formats_show, inode->i_private);
+}
+
+static const struct file_operations mods_dc_color_formats_fops = {
+	.open		= mods_dc_color_formats_open,
+	.read		= seq_read,
+	.llseek		= seq_lseek,
+	.release	= single_release,
+};
+
+static int mods_dc_blend_gen_show(struct seq_file *s, void *unused)
+{
+	struct tegra_dc *dc = s->private;
+	u32 i;
+
+	for (i = 0; i < tegra_dc_get_numof_dispwindows(); i++) {
+		struct tegra_dc_win *win;
+		u32 *blend_gen;
+
+		win = tegra_dc_get_window(dc, i);
+		if (!win)
+			continue;
+		blend_gen = tegra_dc_parse_feature(dc, i, HAS_GEN2_BLEND);
+		if (!blend_gen)
+			continue;
+		seq_printf(s, "window_%u: %u\n", i,
+			blend_gen[BLEND_GENERATION]);
+	}
+	return 0;
+}
+
+static int mods_dc_blend_gen_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, mods_dc_blend_gen_show, inode->i_private);
+}
+
+static const struct file_operations mods_dc_blend_gen_fops = {
+	.open		= mods_dc_blend_gen_open,
+	.read		= seq_read,
+	.llseek		= seq_lseek,
+	.release	= single_release,
+};
+
+static int mods_dc_layout_show(struct seq_file *s, void *unused)
+{
+	struct tegra_dc *dc = s->private;
+	u32 i;
+
+	seq_puts(s, "          Pitch Tiled Block\n");
+	for (i = 0; i < tegra_dc_get_numof_dispwindows(); i++) {
+		struct tegra_dc_win *win;
+		u32 *layouts;
+
+		win = tegra_dc_get_window(dc, i);
+		if (!win)
+			continue;
+		layouts = tegra_dc_parse_feature(dc, i, HAS_TILED);
+		if (!layouts)
+			continue;
+		seq_printf(s, "window_%u: %5u %5u %5u\n", i,
+			layouts[PITCHED_LAYOUT],
+			layouts[TILED_LAYOUT],
+			layouts[BLOCK_LINEAR]);
+	}
+	return 0;
+}
+
+static int mods_dc_layout_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, mods_dc_layout_show, inode->i_private);
+}
+
+static const struct file_operations mods_dc_layout_fops = {
+	.open		= mods_dc_layout_open,
+	.read		= seq_read,
+	.llseek		= seq_lseek,
+	.release	= single_release,
+};
+
+static int mods_dc_invert_show(struct seq_file *s, void *unused)
+{
+	struct tegra_dc *dc = s->private;
+	u32 i;
+
+	seq_puts(s, "          FlipH FlipV ScanColumn\n");
+	for (i = 0; i < tegra_dc_get_numof_dispwindows(); i++) {
+		struct tegra_dc_win *win;
+		u32 *invert_data;
+
+		win = tegra_dc_get_window(dc, i);
+		if (!win)
+			continue;
+		invert_data = tegra_dc_parse_feature(dc, i, GET_INVERT);
+		if (!invert_data)
+			continue;
+		seq_printf(s, "window_%u: %5u %5u %10u\n", i,
+			invert_data[H_INVERT],
+			invert_data[V_INVERT],
+			invert_data[SCAN_COLUMN]);
+	}
+	return 0;
+}
+
+static int mods_dc_invert_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, mods_dc_invert_show, inode->i_private);
+}
+
+static const struct file_operations mods_dc_invert_fops = {
+	.open		= mods_dc_invert_open,
+	.read		= seq_read,
+	.llseek		= seq_lseek,
+	.release	= single_release,
+};
+
+static int mods_dc_interlaced_show(struct seq_file *s, void *unused)
+{
+	struct tegra_dc *dc = s->private;
+	u32 i;
+	const unsigned int head_interlaced = 1;
+
+	seq_printf(s, "head: %u\n", head_interlaced);
+	for (i = 0; i < tegra_dc_get_numof_dispwindows(); i++) {
+		struct tegra_dc_win *win;
+		u32 *interlaced;
+
+		win = tegra_dc_get_window(dc, i);
+
+		if (!win)
+			continue;
+		interlaced = tegra_dc_parse_feature(dc, i, HAS_INTERLACE);
+		if (!interlaced)
+			continue;
+		seq_printf(s, "window_%u: %u\n", i, interlaced[0]);
+	}
+	return 0;
+}
+
+static int mods_dc_interlaced_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, mods_dc_interlaced_show, inode->i_private);
+}
+
+static const struct file_operations mods_dc_interlaced_fops = {
+	.open		= mods_dc_interlaced_open,
+	.read		= seq_read,
+	.llseek		= seq_lseek,
+	.release	= single_release,
+};
+
+static int mods_dc_window_mask_show(struct seq_file *s, void *unused)
+{
+	struct tegra_dc *dc = s->private;
+
+	seq_printf(s, "0x%02lx\n", dc->valid_windows);
+	return 0;
+}
+
+static int mods_dc_window_mask_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, mods_dc_window_mask_show, inode->i_private);
+}
+
+static const struct file_operations mods_dc_window_mask_fops = {
+	.open		= mods_dc_window_mask_open,
+	.read		= seq_read,
+	.llseek		= seq_lseek,
+	.release	= single_release,
+};
+
+static int mods_dc_scaling_show(struct seq_file *s, void *unused)
+{
+	struct tegra_dc *dc = s->private;
+	u32 i;
+
+	seq_puts(s, "          UpH UpV DownH DownV\n");
+	for (i = 0; i < tegra_dc_get_numof_dispwindows(); i++) {
+		struct tegra_dc_win *win;
+		u32 *scaling;
+
+		win = tegra_dc_get_window(dc, i);
+		if (!win)
+			continue;
+		scaling = tegra_dc_parse_feature(dc, i, HAS_SCALE);
+		if (!scaling)
+			continue;
+		seq_printf(s, "window_%u: %3u %3u %5u %5u\n", i,
+			scaling[H_SCALE_UP],
+			scaling[V_SCALE_UP],
+			scaling[H_FILTER_DOWN],
+			scaling[V_FILTER_DOWN]);
+	}
+	return 0;
+}
+
+static int mods_dc_scaling_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, mods_dc_scaling_show, inode->i_private);
+}
+
+static const struct file_operations mods_dc_scaling_fops = {
+	.open		= mods_dc_scaling_open,
+	.read		= seq_read,
+	.llseek		= seq_lseek,
+	.release	= single_release,
+};
+
+static int mods_dsi_ganged_get(void *data, u64 *val)
+{
+	struct tegra_dc_dsi_data *dsi = data;
+
+	*val = (u64)dsi->info.ganged_type;
+	return 0;
+}
+DEFINE_SIMPLE_ATTRIBUTE(mods_dsi_ganged_fops, mods_dsi_ganged_get, NULL,
+	"%llu\n");
+
+static int mods_dsi_inst_get(void *data, u64 *val)
+{
+	struct tegra_dc_dsi_data *dsi = data;
+
+	*val = (u64)dsi->info.dsi_instance;
+	return 0;
+}
+DEFINE_SIMPLE_ATTRIBUTE(mods_dsi_inst_fops, mods_dsi_inst_get, NULL, "%llu\n");
+
+static int mods_dc_border_get(void *data, u64 *val)
+{
+	struct tegra_dc *dc = data;
+	u32 blender_reg = DC_DISP_BLEND_BACKGROUND_COLOR;
+
+	if (!dc->enabled)
+		*val = 0ULL;
+	else
+		*val = (u64)tegra_dc_readl_exported(dc, blender_reg);
+	return 0;
+}
+static int mods_dc_border_set(void *data, u64 val)
+{
+	struct tegra_dc *dc = data;
+	u32 blender_reg = DC_DISP_BLEND_BACKGROUND_COLOR;
+
+	if (!dc->enabled)
+		return 0;
+	mutex_lock(&dc->lock);
+	tegra_dc_get(dc);
+	tegra_dc_writel_exported(dc, val, blender_reg);
+	tegra_dc_writel_exported(dc, GENERAL_ACT_REQ, DC_CMD_STATE_CONTROL);
+	tegra_dc_put(dc);
+	mutex_unlock(&dc->lock);
+	return 0;
+}
+DEFINE_SIMPLE_ATTRIBUTE(mods_dc_border_fops, mods_dc_border_get,
+	mods_dc_border_set, "0x%llx\n");
+
+static int mods_dc_ocp_show(struct seq_file *s, void *unused)
+{
+	seq_puts(s, "rgb\n");
+	return 0;
+}
+
+static int mods_dc_ocp_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, mods_dc_ocp_show, inode->i_private);
+}
+
+static const struct file_operations mods_dc_ocp_fops = {
+	.open		= mods_dc_ocp_open,
+	.read		= seq_read,
+	.llseek		= seq_lseek,
+	.release	= single_release,
+};
+
+
+#define DC_DISP_CSC2_CONTROL		0x4ef
+#define  CSC2_OUTPUT_COLOR_RGB		(0 << 0)
+#define  CSC2_OUTPUT_COLOR_709		(1 << 0)
+#define  CSC2_OUTPUT_COLOR_601		(2 << 0)
+#define  CSC2_OUTPUT_COLOR(val)		(val & 0x3)
+#define  CSC2_LIMIT_RGB_DISABLE		(0 << 2)
+#define  CSC2_LIMIT_RGB_ENABLE		(1 << 2)
+
+static int mods_dc_oc_show(struct seq_file *s, void *unused)
+{
+	seq_puts(s, "rgb\n");
+	return 0;
+}
+
+static int mods_dc_oc_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, mods_dc_oc_show, inode->i_private);
+}
+
+static ssize_t mods_dc_oc_write(struct file *file, const char __user *user_buf,
+			      size_t size, loff_t *ppos)
+{
+	char buf[8];
+	int buf_size;
+
+	buf_size = min(size, (sizeof(buf) - 1));
+	if (unlikely(buf_size < 0))
+		return -EFAULT;
+	if (strncpy_from_user(buf, user_buf, buf_size) < 0)
+		return -EFAULT;
+	buf[buf_size] = 0;
+
+	if (strncmp(buf, "rgb", 3) != 0)
+		return -EINVAL;
+	*ppos += size;
+	return size;
+}
+
+static const struct file_operations mods_dc_oc_fops = {
+	.open		= mods_dc_oc_open,
+	.read		= seq_read,
+	.llseek		= seq_lseek,
+	.release	= single_release,
+	.write		= mods_dc_oc_write,
+};
+
+static int mods_dc_ddc_bus_get(void *data, u64 *val)
+{
+	struct tegra_dc *dc = data;
+	*val = dc->out->ddc_bus;
+	return 0;
+}
+DEFINE_SIMPLE_ATTRIBUTE(mods_dc_ddc_bus_fops, mods_dc_ddc_bus_get,
+	NULL, "%lld\n");
+
+
+static int mods_dc_crc_latched_show(struct seq_file *s, void *unused)
+{
+	struct tegra_dc *dc = s->private;
+	u32 crc = tegra_dc_sysfs_read_checksum_latched(dc);
+	u32 field = 0;
+	u32 val;
+
+	val = tegra_dc_readl_exported(dc, DC_DISP_INTERLACE_CONTROL);
+	if (val & INTERLACE_MODE_ENABLE)
+		field = (val & INTERLACE_STATUS_FIELD_2) ? 1 : 0;
+	seq_printf(s, "0x%08x %u\n", crc, field);
+
+	return 0;
+}
+
+static int mods_dc_crc_latched_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, mods_dc_crc_latched_show, inode->i_private);
+}
+
+static const struct file_operations mods_dc_crc_latched_fops = {
+	.open		= mods_dc_crc_latched_open,
+	.read		= seq_read,
+	.llseek		= seq_lseek,
+	.release	= single_release,
+};
+#endif /* CONFIG_TEGRA_DC */
+
+#if defined(MODS_HAS_TEGRA) && defined(CONFIG_TEGRA_KFUSE)
+static int mods_kfuse_show(struct seq_file *s, void *unused)
+{
+	unsigned int buf[KFUSE_DATA_SZ / 4];
+
+	/* copy load kfuse into buffer - only needed for early Tegra parts */
+	int ret = tegra_kfuse_read(buf, sizeof(buf));
+	int i;
+
+	if (ret)
+		return ret;
+
+	for (i = 0; i < KFUSE_DATA_SZ / 4; i += 4)
+		seq_printf(s, "0x%08x 0x%08x 0x%08x 0x%08x\n",
+			buf[i], buf[i+1], buf[i+2], buf[i+3]);
+
+	return 0;
+}
+
+static int mods_kfuse_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, mods_kfuse_show, inode->i_private);
+}
+
+static const struct file_operations mods_kfuse_fops = {
+	.open		= mods_kfuse_open,
+	.read		= seq_read,
+	.llseek		= seq_lseek,
+	.release	= single_release,
+};
+#endif /* MODS_HAS_TEGRA */
+
+static int mods_debug_get(void *data, u64 *val)
+{
+	*val = (u64)(mods_get_debug_level() & DEBUG_ALL);
+	return 0;
+}
+static int mods_debug_set(void *data, u64 val)
+{
+	mods_set_debug_level((int)val & DEBUG_ALL);
+	return 0;
+}
+DEFINE_SIMPLE_ATTRIBUTE(mods_debug_fops, mods_debug_get, mods_debug_set,
+						"0x%08llx\n");
+
+static int mods_mi_get(void *data, u64 *val)
+{
+	*val = (u64)mods_get_multi_instance();
+	return 0;
+}
+static int mods_mi_set(void *data, u64 val)
+{
+	mods_set_multi_instance((int)val);
+	return 0;
+}
+
+#ifdef CONFIG_ARCH_TEGRA_19x_SOC
+static int mods_set_err_sel(void *data, u64 val)
+{
+	set_err_sel(val);
+	return 0;
+}
+DEFINE_SIMPLE_ATTRIBUTE(mods_err_sel_fops, 0, mods_set_err_sel, "%llu\n");
+
+static int mods_set_err_ctrl(void *data, u64 val)
+{
+	set_err_ctrl(val);
+	return 0;
+}
+
+static int mods_get_err_ctrl(void *data, u64 *val)
+{
+	*val = get_err_ctrl();
+	return 0;
+}
+
+DEFINE_SIMPLE_ATTRIBUTE(mods_err_ctrl_fops, mods_get_err_ctrl,
+				mods_set_err_ctrl, "%llu\n");
+
+static int mods_enable_cpu_core_reporting(void *data, u64 val)
+{
+	enable_cpu_core_reporting(val);
+	return 0;
+}
+DEFINE_SIMPLE_ATTRIBUTE(mods_enable_cpu_fops, 0, mods_enable_cpu_core_reporting,
+								"%llu\n");
+#endif
+
+DEFINE_SIMPLE_ATTRIBUTE(mods_mi_fops, mods_mi_get, mods_mi_set, "%llu\n");
+
+void mods_remove_debugfs(void)
+{
+	debugfs_remove_recursive(mods_debugfs_dir);
+	mods_debugfs_dir = NULL;
+}
+
+int mods_create_debugfs(struct miscdevice *modsdev)
+{
+#ifdef CONFIG_ARCH_TEGRA_19x_SOC
+	struct dentry *ras_debugfs_entry;
+#endif
+	struct dentry *retval;
+	int err = 0;
+#if defined(MODS_HAS_TEGRA) && defined(CONFIG_TEGRA_DC)
+	unsigned int dc_idx;
+	int nheads = tegra_dc_get_numof_dispheads();
+	int client_reg_rc = 0;
+
+	// Check if tegra dc client registration succeeds. This will fail
+	// with ENODEV if given architecture no longer uses tegra_dc
+	// driver. Skip creating display related debugfs nodes
+	bool skip_tegradc_debug_nodes = false;
+	struct tegra_dc_client client;
+
+	memset(&client, 0, sizeof(client));
+
+	client_reg_rc = tegra_dc_register_client(&client);
+	if (client_reg_rc != 0) {
+		if (client_reg_rc == -ENODEV) {
+			skip_tegradc_debug_nodes = true;
+		} else {
+			pr_err("%s: tegra dc client registration failed\n",
+				__func__);
+			return client_reg_rc;
+		}
+	} else {
+		client_reg_rc = tegra_dc_unregister_client(&client);
+		if (client_reg_rc != 0) {
+			pr_err("%s: tegra dc client unregistration failed\n",
+				__func__);
+			return client_reg_rc;
+		}
+	}
+
+	if (nheads <= 0 && !skip_tegradc_debug_nodes) {
+		pr_err("%s: max heads:%d cannot be negative or zero\n",
+			__func__, nheads);
+		return -EINVAL;
+	}
+#endif
+
+	mods_debugfs_dir = debugfs_create_dir(dev_name(modsdev->this_device),
+		NULL);
+	if (IS_ERR(mods_debugfs_dir)) {
+		err = -EIO;
+		goto remove_out;
+	}
+
+#ifdef CONFIG_ARCH_TEGRA_19x_SOC
+	if (of_find_node_by_name(NULL, "carmel_ras")) {
+		ras_debugfs_entry = debugfs_create_dir("ras", mods_debugfs_dir);
+		if (IS_ERR(ras_debugfs_entry)) {
+			err = -EIO;
+			goto remove_out;
+		}
+
+		retval = debugfs_create_file("err_sel", 0644,
+			ras_debugfs_entry, 0, &mods_err_sel_fops);
+		if (IS_ERR(retval)) {
+			err = -EIO;
+			goto remove_out;
+		}
+
+		retval = debugfs_create_file("err_ctrl", 0644,
+			ras_debugfs_entry, 0, &mods_err_ctrl_fops);
+		if (IS_ERR(retval)) {
+			err = -EIO;
+			goto remove_out;
+		}
+		retval = debugfs_create_file("ccplex_config", 0644,
+			ras_debugfs_entry, 0, &mods_enable_cpu_fops);
+		if (IS_ERR(retval)) {
+			err = -EIO;
+			goto remove_out;
+		}
+	}
+#endif
+
+	retval = debugfs_create_file("debug", 0644,
+		mods_debugfs_dir, NULL, &mods_debug_fops);
+	if (IS_ERR(retval)) {
+		err = -EIO;
+		goto remove_out;
+	}
+
+	retval = debugfs_create_file("multi_instance", 0644,
+		mods_debugfs_dir, NULL, &mods_mi_fops);
+	if (IS_ERR(retval)) {
+		err = -EIO;
+		goto remove_out;
+	}
+
+#if defined(MODS_HAS_TEGRA) && defined(CONFIG_TEGRA_KFUSE)
+	retval = debugfs_create_file("kfuse_data", 0444,
+		mods_debugfs_dir, NULL, &mods_kfuse_fops);
+	if (IS_ERR(retval)) {
+		err = -EIO;
+		goto remove_out;
+	}
+#endif
+
+#if defined(MODS_HAS_TEGRA) && defined(CONFIG_TEGRA_DC)
+	if (!skip_tegradc_debug_nodes) {
+		for (dc_idx = 0; dc_idx < nheads; dc_idx++) {
+			struct dentry *dc_debugfs_dir;
+			char devname[16];
+			struct tegra_dc *dc = tegra_dc_get_dc(dc_idx);
+			int devname_size;
+
+			if (!dc)
+				continue;
+
+			devname_size = snprintf(devname, sizeof(devname), "tegradc.%d",
+					dc->ctrl_num);
+			if (unlikely(devname_size < 0))
+				continue;
+			dc_debugfs_dir = debugfs_create_dir(devname,
+					mods_debugfs_dir);
+
+			if (IS_ERR(dc_debugfs_dir)) {
+				err = -EIO;
+				goto remove_out;
+			}
+
+			retval = debugfs_create_file("window_mask", 0444,
+				dc_debugfs_dir, dc,
+				&mods_dc_window_mask_fops);
+			if (IS_ERR(retval)) {
+				err = -EIO;
+				goto remove_out;
+			}
+			retval = debugfs_create_file("color_formats", 0444,
+				dc_debugfs_dir, dc,
+				&mods_dc_color_formats_fops);
+			if (IS_ERR(retval)) {
+				err = -EIO;
+				goto remove_out;
+			}
+			retval = debugfs_create_file("blend_gen", 0444,
+				dc_debugfs_dir, dc, &mods_dc_blend_gen_fops);
+			if (IS_ERR(retval)) {
+				err = -EIO;
+				goto remove_out;
+			}
+			retval = debugfs_create_file("layout", 0444,
+				 dc_debugfs_dir, dc, &mods_dc_layout_fops);
+			if (IS_ERR(retval)) {
+				err = -EIO;
+				goto remove_out;
+			}
+			retval = debugfs_create_file("invert", 0444,
+				dc_debugfs_dir, dc, &mods_dc_invert_fops);
+			if (IS_ERR(retval)) {
+				err = -EIO;
+				goto remove_out;
+			}
+			retval = debugfs_create_file("interlaced", 0444,
+				dc_debugfs_dir, dc, &mods_dc_interlaced_fops);
+			if (IS_ERR(retval)) {
+				err = -EIO;
+				goto remove_out;
+			}
+			retval = debugfs_create_file("scaling", 0444,
+				dc_debugfs_dir, dc, &mods_dc_scaling_fops);
+			if (IS_ERR(retval)) {
+				err = -EIO;
+				goto remove_out;
+			}
+			retval = debugfs_create_file("border_color", 0644,
+				dc_debugfs_dir, dc, &mods_dc_border_fops);
+			if (IS_ERR(retval)) {
+				err = -EIO;
+				goto remove_out;
+			}
+
+			retval = debugfs_create_file("output_color_possible",
+				0444, dc_debugfs_dir, NULL, &mods_dc_ocp_fops);
+			if (IS_ERR(retval)) {
+				err = -EIO;
+				goto remove_out;
+			}
+			retval = debugfs_create_file("output_color", 0644,
+				dc_debugfs_dir, dc, &mods_dc_oc_fops);
+			if (IS_ERR(retval)) {
+				err = -EIO;
+				goto remove_out;
+			}
+
+			retval = debugfs_create_file("crc_checksum_latched",
+				0444, dc_debugfs_dir, dc,
+				&mods_dc_crc_latched_fops);
+			if (IS_ERR(retval)) {
+				err = -EIO;
+				goto remove_out;
+			}
+
+			if (dc->out && dc->out->type == TEGRA_DC_OUT_DSI) {
+				struct dentry *dsi_debugfs_dir;
+
+				dsi_debugfs_dir = debugfs_create_dir("dsi",
+					dc_debugfs_dir);
+				if (IS_ERR(dsi_debugfs_dir)) {
+					err = -EIO;
+					goto remove_out;
+				}
+				retval = debugfs_create_file("ganged", 0444,
+					dsi_debugfs_dir,
+					tegra_dc_get_outdata(dc),
+					&mods_dsi_ganged_fops);
+				if (IS_ERR(retval)) {
+					err = -EIO;
+					goto remove_out;
+				}
+				retval = debugfs_create_file("instance", 0444,
+					dsi_debugfs_dir,
+					tegra_dc_get_outdata(dc),
+					&mods_dsi_inst_fops);
+				if (IS_ERR(retval)) {
+					err = -EIO;
+					goto remove_out;
+				}
+			}
+
+			if (dc->out && dc->out->type == TEGRA_DC_OUT_HDMI) {
+				retval = debugfs_create_file("ddc_bus", 0444,
+					dc_debugfs_dir, dc,
+					&mods_dc_ddc_bus_fops);
+				if (IS_ERR(retval)) {
+					err = -EIO;
+					goto remove_out;
+				}
+			}
+		}
+	}
+#endif
+
+	return 0;
+remove_out:
+	dev_err(modsdev->this_device, "could not create debugfs\n");
+	mods_remove_debugfs();
+	return err;
+}
diff --git a/drivers/misc/mods/mods_dma.c b/drivers/misc/mods/mods_dma.c
new file mode 100644
index 00000000..6f33cd18
--- /dev/null
+++ b/drivers/misc/mods/mods_dma.c
@@ -0,0 +1,587 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * This file is part of NVIDIA MODS kernel driver.
+ *
+ * Copyright (c) 2017-2022, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * NVIDIA MODS kernel driver is free software: you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * NVIDIA MODS kernel driver is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NVIDIA MODS kernel driver.
+ * If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/device.h>
+#include <linux/dmaengine.h>
+#include <linux/kernel.h>
+#include <linux/platform_device.h>
+#include <linux/uaccess.h>
+#include "mods_internal.h"
+
+static struct device *mods_tegra_dma_dev;
+
+#define MODS_DMA_MAX_CHANNEL 32
+struct mods_dma_chan_info {
+	rwlock_t lock;
+	bool    in_use;
+	u32     id;
+	struct  dma_chan *pch;
+	bool    should_notify;
+};
+
+static DECLARE_BITMAP(dma_info_mask, MODS_DMA_MAX_CHANNEL);
+static struct mods_dma_chan_info dma_info_chan_list[MODS_DMA_MAX_CHANNEL];
+static DEFINE_SPINLOCK(dma_info_lock);
+
+static int mods_get_dma_id(u32 *p_id)
+{
+	u32 id;
+
+	spin_lock(&dma_info_lock);
+	id = find_first_zero_bit(dma_info_mask, MODS_DMA_MAX_CHANNEL);
+	if (id >= MODS_DMA_MAX_CHANNEL) {
+		spin_unlock(&dma_info_lock);
+		return -ENOSPC;
+	}
+	set_bit(id, dma_info_mask);
+	spin_unlock(&dma_info_lock);
+
+	*p_id = id;
+	return OK;
+}
+
+static void mods_release_dma_id(u32 id)
+{
+	spin_lock(&dma_info_lock);
+	clear_bit(id, dma_info_mask);
+	spin_unlock(&dma_info_lock);
+}
+
+static int mods_get_chan_by_id(u32 id, struct mods_dma_chan_info **p_dma_chan)
+{
+	if (id >= MODS_DMA_MAX_CHANNEL)
+		return -ERANGE;
+
+	*p_dma_chan = &dma_info_chan_list[id];
+
+	return OK;
+}
+
+static void mods_release_channel(u32 id)
+{
+	struct mods_dma_chan_info *p_mods_chan;
+	struct dma_chan *pch = NULL;
+
+	if (mods_get_chan_by_id(id, &p_mods_chan) != OK) {
+		mods_error_printk("get dma channel failed, id %d\n", id);
+		return;
+	}
+
+	write_lock(&(p_mods_chan->lock));
+	if (p_mods_chan->in_use) {
+		pch = p_mods_chan->pch;
+		p_mods_chan->pch = NULL;
+		mods_release_dma_id(id);
+		p_mods_chan->in_use = false;
+	}
+	write_unlock(&(p_mods_chan->lock));
+
+	/* do not call this when hold spin lock */
+	if (pch) {
+		dmaengine_terminate_sync(pch);
+		dma_release_channel(pch);
+	}
+}
+
+static bool mods_chan_is_inuse(struct mods_dma_chan_info *p_mods_chan)
+{
+	bool in_use = false;
+
+	read_lock(&(p_mods_chan->lock));
+	if (p_mods_chan->in_use)
+		in_use = true;
+	read_unlock(&(p_mods_chan->lock));
+
+	return in_use;
+}
+
+static int mods_get_inuse_chan_by_handle(struct MODS_DMA_HANDLE *p_handle,
+					struct mods_dma_chan_info **p_mods_chan)
+{
+	int err;
+	bool in_use;
+	struct mods_dma_chan_info *p_mods_ch;
+
+	err = mods_get_chan_by_id(p_handle->dma_id, &p_mods_ch);
+	if (err != OK) {
+		mods_error_printk("get dma channel failed, id %d\n",
+				p_handle->dma_id);
+		return -ENODEV;
+	}
+
+	in_use = mods_chan_is_inuse(p_mods_ch);
+	if (!in_use) {
+		mods_error_printk("invalid dma channel: %d, not in use\n",
+				p_handle->dma_id);
+		return -EINVAL;
+	}
+	*p_mods_chan = p_mods_ch;
+	return OK;
+}
+
+static int mods_dma_sync_wait(struct MODS_DMA_HANDLE *p_handle,
+					mods_dma_cookie_t cookie)
+{
+	int err = OK;
+	struct mods_dma_chan_info *p_mods_chan;
+
+	err = mods_get_inuse_chan_by_handle(p_handle, &p_mods_chan);
+	if (err != OK)
+		return err;
+
+	mods_debug_printk(DEBUG_TEGRADMA,
+			"Wait on chan: %p\n", p_mods_chan->pch);
+	read_lock(&(p_mods_chan->lock));
+	if (dma_sync_wait(p_mods_chan->pch, cookie) != DMA_COMPLETE)
+		err = -1;
+	read_unlock(&(p_mods_chan->lock));
+
+	return err;
+}
+
+static int mods_dma_async_is_tx_complete(struct MODS_DMA_HANDLE *p_handle,
+					mods_dma_cookie_t cookie,
+					__u32 *p_is_complete)
+{
+	int err = OK;
+	struct mods_dma_chan_info *p_mods_chan;
+	enum dma_status status;
+
+	err = mods_get_inuse_chan_by_handle(p_handle, &p_mods_chan);
+	if (err != OK)
+		return err;
+
+	read_lock(&(p_mods_chan->lock));
+	status = dma_async_is_tx_complete(p_mods_chan->pch, cookie, NULL, NULL);
+	read_unlock(&(p_mods_chan->lock));
+
+	if (status == DMA_COMPLETE)
+		*p_is_complete = true;
+	else if (status == DMA_IN_PROGRESS)
+		*p_is_complete = false;
+	else
+		err = -EINVAL;
+
+	return err;
+}
+
+int esc_mods_dma_request_channel_2(struct mods_client *client,
+				struct MODS_DMA_HANDLE_2 *p_handle)
+{
+	struct dma_chan *chan = NULL;
+	struct mods_dma_chan_info *p_mods_chan = NULL;
+	u32 id = MODS_DMA_MAX_CHANNEL;
+	int err = -EINVAL;
+
+	LOG_ENT();
+
+	if (p_handle->dma_type >= MODS_DMA_TX_TYPE_END) {
+		cl_error("dma type %d not available\n", p_handle->dma_type);
+		goto failed;
+	}
+
+	err = mods_get_dma_id(&id);
+	if (err != OK) {
+		cl_error("no dma handle available\n");
+		goto failed;
+	}
+
+	err = mods_get_chan_by_id(id, &p_mods_chan);
+	if (err != OK) {
+		cl_error("get dma channel failed\n");
+		goto failed;
+	}
+
+	read_lock(&(p_mods_chan->lock));
+	if (p_mods_chan->in_use) {
+		cl_error("mods dma channel in use\n");
+		read_unlock(&(p_mods_chan->lock));
+		err = -EBUSY;
+		goto failed;
+	}
+	read_unlock(&(p_mods_chan->lock));
+
+	// dma channel is requested in the old-fashion way
+	if (p_handle->ctrl_dir[0] == '\0') {
+		dma_cap_mask_t mask;
+
+		dma_cap_zero(mask);
+		dma_cap_set(p_handle->dma_type, mask);
+		chan = dma_request_channel(mask, NULL, NULL);
+	} else if (mods_tegra_dma_dev) {
+		// get dma chan from dt node
+		cl_debug(DEBUG_TEGRADMA, "dmach is asked for %s\n", p_handle->ctrl_dir);
+		chan = dma_request_chan(mods_tegra_dma_dev, p_handle->ctrl_dir);
+	}
+	if (!chan) {
+		cl_error("dma channel is not available\n");
+		mods_release_dma_id(id);
+		err = -EBUSY;
+		goto failed;
+	}
+
+	write_lock(&(p_mods_chan->lock));
+	p_mods_chan->pch = chan;
+	p_mods_chan->in_use = true;
+	write_unlock(&(p_mods_chan->lock));
+
+	p_handle->dma_id = id;
+	cl_debug(DEBUG_TEGRADMA, "request get dma id: %d\n", id);
+
+failed:
+	LOG_EXT();
+	return err;
+}
+
+int esc_mods_dma_request_channel(struct mods_client *client,
+				struct MODS_DMA_HANDLE *p_handle)
+{
+	int err;
+	struct MODS_DMA_HANDLE_2 handle = {0};
+
+	handle.ctrl_dir[0] = '\0';
+	handle.dma_type = p_handle->dma_type;
+	err = esc_mods_dma_request_channel_2(client, &handle);
+	if (err == 0)
+		p_handle->dma_id = handle.dma_id;
+
+	return err;
+}
+
+int esc_mods_dma_release_channel(struct mods_client *client,
+				struct MODS_DMA_HANDLE *p_handle)
+{
+	mods_release_channel(p_handle->dma_id);
+	return OK;
+}
+
+int esc_mods_dma_set_config(struct mods_client *client,
+				struct MODS_DMA_CHANNEL_CONFIG *p_config)
+
+{
+	struct dma_slave_config config;
+	struct mods_dma_chan_info *p_mods_chan;
+	int err;
+
+	LOG_ENT();
+
+	err = mods_get_inuse_chan_by_handle(&p_config->handle, &p_mods_chan);
+	if (err != OK) {
+		LOG_EXT();
+		return err;
+	}
+
+	config.direction = p_config->direction;
+	config.src_addr = p_config->src_addr;
+	config.dst_addr = p_config->dst_addr;
+	config.src_addr_width = p_config->src_addr_width;
+	config.dst_addr_width = p_config->dst_addr_width;
+	config.src_maxburst = p_config->src_maxburst;
+	config.dst_maxburst = p_config->dst_maxburst;
+	config.device_fc = (p_config->device_fc == 0) ? false : true;
+#if KERNEL_VERSION(5, 17, 0) > MODS_KERNEL_VERSION
+	config.slave_id = p_config->slave_id;
+#endif
+
+	cl_debug(DEBUG_TEGRADMA,
+		"ch: %d dir [%d], addr[%p -> %p], burst [%d %d] width [%d %d]\n",
+		p_config->handle.dma_id,
+		config.direction,
+		(void *)config.src_addr, (void *)config.dst_addr,
+		config.src_maxburst, config.dst_maxburst,
+		config.src_addr_width, config.dst_addr_width);
+
+#if KERNEL_VERSION(5, 17, 0) > MODS_KERNEL_VERSION
+	cl_debug(DEBUG_TEGRADMA,
+		"slave id %d\n",
+		config.slave_id);
+#endif
+
+	write_lock(&(p_mods_chan->lock));
+	err = dmaengine_slave_config(p_mods_chan->pch, &config);
+	write_unlock(&(p_mods_chan->lock));
+
+	LOG_EXT();
+
+	return err;
+}
+
+
+int esc_mods_dma_submit_request(struct mods_client *client,
+				struct MODS_DMA_TX_DESC *p_mods_desc)
+{
+	int err = OK;
+	struct mods_dma_chan_info *p_mods_chan;
+	struct dma_async_tx_descriptor *desc;
+	struct dma_device       *dev;
+	enum dma_ctrl_flags     flags;
+	mods_dma_cookie_t cookie = 0;
+
+	LOG_ENT();
+	flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT;
+
+	err = mods_get_inuse_chan_by_handle(&p_mods_desc->handle, &p_mods_chan);
+	if (err != OK) {
+		LOG_EXT();
+		return err;
+	}
+
+	if (p_mods_desc->mode != MODS_DMA_SINGLE) {
+		cl_error("unsupported mode: %d\n", p_mods_desc->mode);
+		LOG_EXT();
+		return -EINVAL;
+	}
+
+	cl_debug(DEBUG_TEGRADMA, "submit on chan %p\n", p_mods_chan->pch);
+
+	write_lock(&(p_mods_chan->lock));
+	if (p_mods_desc->data_dir == MODS_DMA_MEM_TO_MEM) {
+		dev = p_mods_chan->pch->device;
+		desc = dev->device_prep_dma_memcpy(
+						p_mods_chan->pch,
+						p_mods_desc->phys,
+						p_mods_desc->phys_2,
+						p_mods_desc->length,
+						flags);
+	} else {
+		cl_debug(DEBUG_TEGRADMA,
+			"Phys Addr [%p], len [%d], dir [%d]\n",
+			(void *)p_mods_desc->phys,
+			p_mods_desc->length,
+			p_mods_desc->data_dir);
+		desc = dmaengine_prep_slave_single(p_mods_chan->pch,
+							p_mods_desc->phys,
+							p_mods_desc->length,
+							p_mods_desc->data_dir,
+							flags);
+	}
+
+	if (desc == NULL) {
+		cl_error("unable to get desc for Tx\n");
+		err = -EIO;
+		goto failed;
+	}
+
+	desc->callback = NULL;
+	desc->callback_param = NULL;
+	cookie = dmaengine_submit(desc);
+
+failed:
+	write_unlock(&(p_mods_chan->lock));
+	if (dma_submit_error(cookie)) {
+		cl_error("submit cookie: %x\n", cookie);
+		LOG_EXT();
+		return -EIO;
+	}
+
+	p_mods_desc->cookie = cookie;
+
+	LOG_EXT();
+	return err;
+}
+
+int esc_mods_dma_async_issue_pending(struct mods_client *client,
+					struct MODS_DMA_HANDLE *p_handle)
+{
+	int err = OK;
+	struct mods_dma_chan_info *p_mods_chan;
+
+	LOG_ENT();
+	err = mods_get_inuse_chan_by_handle(p_handle, &p_mods_chan);
+	if (err != OK) {
+		LOG_EXT();
+		return err;
+	}
+
+	cl_debug(DEBUG_TEGRADMA, "issue pending on chan: %p\n",
+										p_mods_chan->pch);
+	read_lock(&(p_mods_chan->lock));
+	dma_async_issue_pending(p_mods_chan->pch);
+	read_unlock(&(p_mods_chan->lock));
+	LOG_EXT();
+
+	return err;
+}
+
+int esc_mods_dma_wait(struct mods_client *client,
+				struct MODS_DMA_WAIT_DESC *p_wait_desc)
+{
+	int err;
+
+	LOG_ENT();
+	if (p_wait_desc->type == MODS_DMA_SYNC_WAIT)
+		err = mods_dma_sync_wait(&p_wait_desc->handle,
+							p_wait_desc->cookie);
+	else if (p_wait_desc->type == MODS_DMA_ASYNC_WAIT)
+		err = mods_dma_async_is_tx_complete(&p_wait_desc->handle,
+							p_wait_desc->cookie,
+							&p_wait_desc->tx_complete);
+	else
+		err = -EINVAL;
+
+	LOG_EXT();
+	return err;
+}
+
+int esc_mods_dma_alloc_coherent(struct mods_client *client,
+				struct MODS_DMA_COHERENT_MEM_HANDLE *p)
+{
+	dma_addr_t    p_phys_addr;
+	void          *p_cpu_addr;
+
+	LOG_ENT();
+
+	p_cpu_addr = dma_alloc_coherent(NULL,
+					p->num_bytes,
+					&p_phys_addr,
+					GFP_KERNEL);
+
+	cl_debug(DEBUG_MEM,
+		"num_bytes=%d, p_cpu_addr=%p, p_phys_addr=%p\n",
+		p->num_bytes,
+		(void *)p_cpu_addr,
+		(void *)p_phys_addr);
+
+	if (!p_cpu_addr) {
+		cl_error(
+			"FAILED!!!num_bytes=%d, p_cpu_addr=%p, p_phys_addr=%p\n",
+			p->num_bytes,
+			(void *)p_cpu_addr,
+			(void *)p_phys_addr);
+		LOG_EXT();
+		return -1;
+	}
+
+	memset(p_cpu_addr, 0x00, p->num_bytes);
+
+	p->memory_handle_phys = (u64)p_phys_addr;
+	p->memory_handle_virt = (u64)p_cpu_addr;
+
+	LOG_EXT();
+	return 0;
+}
+
+int esc_mods_dma_free_coherent(struct mods_client *client,
+					struct MODS_DMA_COHERENT_MEM_HANDLE *p)
+{
+	LOG_ENT();
+
+	cl_debug(DEBUG_MEM,
+		"num_bytes = %d, p_cpu_addr=%p, p_phys_addr=%p\n",
+		p->num_bytes,
+		(void *)(p->memory_handle_virt),
+		(void *)(p->memory_handle_phys));
+
+	dma_free_coherent(NULL,
+		p->num_bytes,
+		(void *)(p->memory_handle_virt),
+		(dma_addr_t)(p->memory_handle_phys));
+
+	p->memory_handle_phys = (u64)0;
+	p->memory_handle_virt = (u64)0;
+
+	LOG_EXT();
+	return 0;
+}
+
+int esc_mods_dma_copy_to_user(struct mods_client *client,
+				struct MODS_DMA_COPY_TO_USER *p)
+{
+	int retval;
+
+	LOG_ENT();
+
+	cl_debug(DEBUG_MEM,
+		"memory_handle_dst=%p, memory_handle_src=%p, num_bytes=%d\n",
+		(void *)(p->memory_handle_dst),
+		(void *)(p->memory_handle_src),
+		p->num_bytes);
+
+	retval = copy_to_user((void __user *)p->memory_handle_dst,
+					(void *)p->memory_handle_src,
+					p->num_bytes);
+
+	LOG_EXT();
+
+	return retval;
+}
+
+static int tegra_dma_driver_probe(struct platform_device *pdev)
+{
+	LOG_ENT();
+
+	mods_debug_printk(DEBUG_TEGRADMA, "mods_tegra_dma probe\n");
+	mods_tegra_dma_dev = get_device(&pdev->dev);
+
+	LOG_EXT();
+	return 0;
+}
+
+static int tegra_dma_driver_remove(struct platform_device *pdev)
+{
+	put_device(&pdev->dev);
+	mods_tegra_dma_dev = NULL;
+	return 0;
+}
+
+static const struct of_device_id of_ids[] = {
+	{ .compatible = "nvidia,mods_tegra_dma" },
+	{ }
+};
+
+static struct platform_driver mods_tegra_dma_driver = {
+	.probe  = tegra_dma_driver_probe,
+	.remove = tegra_dma_driver_remove,
+	.driver = {
+		.name   = "mods_tegra_dma",
+		.owner  = THIS_MODULE,
+		.of_match_table = of_ids,
+	},
+};
+
+int mods_init_dma(void)
+{
+	int i;
+	int err;
+
+	for (i = 0; i < MODS_DMA_MAX_CHANNEL; i++) {
+		struct mods_dma_chan_info *p_chan_info;
+
+		p_chan_info = &dma_info_chan_list[i];
+		rwlock_init(&(p_chan_info->lock));
+		p_chan_info->in_use = false;
+	}
+
+	err = platform_driver_register(&mods_tegra_dma_driver);
+	if (err < 0)
+		mods_error_printk("register mods dma driver failed\n");
+
+	return err;
+}
+
+void mods_exit_dma(void)
+{
+	int i;
+
+	for (i = 0; i < MODS_DMA_MAX_CHANNEL; i++)
+		mods_release_channel(i);
+
+	platform_driver_unregister(&mods_tegra_dma_driver);
+}
diff --git a/drivers/misc/mods/mods_dmabuf.c b/drivers/misc/mods/mods_dmabuf.c
new file mode 100644
index 00000000..b55ab093
--- /dev/null
+++ b/drivers/misc/mods/mods_dmabuf.c
@@ -0,0 +1,160 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * This file is part of NVIDIA MODS kernel driver.
+ *
+ * Copyright (c) 2014-2022, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * NVIDIA MODS kernel driver is free software: you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * NVIDIA MODS kernel driver is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NVIDIA MODS kernel driver.
+ * If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/dma-buf.h>
+#include <linux/module.h>
+#include <linux/dma-mapping.h>
+#include <linux/platform_device.h>
+
+#include "mods_internal.h"
+
+static struct device *dummy_device;
+
+static bool dummy_device_registered;
+
+int esc_mods_dmabuf_get_phys_addr(struct mods_client *client,
+				  struct MODS_DMABUF_GET_PHYSICAL_ADDRESS *op)
+{
+	int                        err = 0;
+	struct dma_buf            *dmabuf = NULL;
+	struct dma_buf_attachment *attachment = NULL;
+	struct sg_table           *sgt = NULL;
+	u64                        remaining_offset = op->offset;
+	u64                        total_size = 0;
+	u32                        total_segments = 0;
+	phys_addr_t                physical_address = 0;
+	u64                        segment_size = 0;
+	struct scatterlist        *sg;
+	unsigned int               sg_index;
+
+	LOG_ENT();
+
+	cl_debug(DEBUG_MEM_DETAILED,
+		 "dmabuf get phys addr fd=%d offs=0x%llx\n",
+		 op->buf_fd, (unsigned long long)op->offset);
+
+	dmabuf = dma_buf_get(op->buf_fd);
+	if (IS_ERR_OR_NULL(dmabuf)) {
+		cl_error("failed to get dma buf from fd %d\n", op->buf_fd);
+		LOG_EXT();
+		return IS_ERR(dmabuf) ? PTR_ERR(dmabuf) : -EINVAL;
+	}
+
+	WARN_ON(!dummy_device_registered);
+	attachment = dma_buf_attach(dmabuf, dummy_device);
+	if (IS_ERR_OR_NULL(attachment)) {
+		cl_error("failed to attach dma buf fd %d\n", op->buf_fd);
+		err = IS_ERR(attachment) ? PTR_ERR(attachment) : -EFAULT;
+		goto buf_attach_fail;
+	}
+
+	sgt = dma_buf_map_attachment(attachment, DMA_BIDIRECTIONAL);
+	if (IS_ERR_OR_NULL(sgt)) {
+		cl_error("failed to map dma buf fd %d\n", op->buf_fd);
+		err = IS_ERR(sgt) ? PTR_ERR(sgt) : -EFAULT;
+		goto buf_map_fail;
+	}
+
+	for_each_sg(sgt->sgl, sg, sgt->nents, sg_index) {
+		total_size += sg->length;
+		++total_segments;
+		if (remaining_offset >= sg->length) {
+			/* haven't reached segment yet, or empty sg */
+			remaining_offset -= sg->length;
+		} else if (segment_size == 0) {
+			/* first sg in segment */
+			physical_address = sg_phys(sg) + remaining_offset;
+			segment_size = sg->length - remaining_offset;
+			remaining_offset = 0;
+		} else if (sg_phys(sg) == physical_address + segment_size) {
+			/* contiguous sg; append to segment */
+			segment_size += sg->length;
+		} else {
+			/* discontiguous sg; end segment */
+			break;
+		}
+	}
+
+	cl_debug(DEBUG_MEM_DETAILED, "traversed %u segments, 0x%llx size\n",
+		 total_segments,
+		 (unsigned long long)total_size);
+
+	if (segment_size == 0) {
+		cl_error(
+			"offset 0x%llx exceeds allocation size 0x%llx, fd %d\n",
+			(unsigned long long)op->offset,
+			(unsigned long long)total_size,
+			op->buf_fd);
+		err = -EINVAL;
+	} else {
+		op->physical_address = physical_address;
+		op->segment_size = segment_size;
+	}
+
+	dma_buf_unmap_attachment(attachment, sgt, DMA_BIDIRECTIONAL);
+
+buf_map_fail:
+	dma_buf_detach(dmabuf, attachment);
+
+buf_attach_fail:
+	dma_buf_put(dmabuf);
+
+	LOG_EXT();
+	return err;
+}
+
+static int mods_dmabuf_probe(struct platform_device *pdev)
+{
+	dma_set_mask(&pdev->dev, DMA_BIT_MASK(39));
+	dummy_device = &pdev->dev;
+
+	dummy_device_registered = true;
+	return 0;
+}
+
+static int mods_dmabuf_remove(struct platform_device *pdev)
+{
+	return 0;
+}
+
+static const struct of_device_id of_ids[] = {
+	{ .compatible = "nvidia,mods_test" },
+	{ }
+};
+
+static struct platform_driver mods_dummy_driver = {
+	.probe  = mods_dmabuf_probe,
+	.remove = mods_dmabuf_remove,
+	.driver = {
+		.name   = "nvidia_mods_dummy_driver",
+		.owner  = THIS_MODULE,
+		.of_match_table = of_ids,
+	},
+};
+
+int mods_init_dmabuf(void)
+{
+	return platform_driver_register(&mods_dummy_driver);
+}
+
+void mods_exit_dmabuf(void)
+{
+	platform_driver_unregister(&mods_dummy_driver);
+}
diff --git a/drivers/misc/mods/mods_internal.h b/drivers/misc/mods/mods_internal.h
new file mode 100644
index 00000000..a25c8cf7
--- /dev/null
+++ b/drivers/misc/mods/mods_internal.h
@@ -0,0 +1,772 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * This file is part of NVIDIA MODS kernel driver.
+ *
+ * Copyright (c) 2008-2023, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * NVIDIA MODS kernel driver is free software: you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * NVIDIA MODS kernel driver is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NVIDIA MODS kernel driver.
+ * If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _MODS_INTERNAL_H_
+#define _MODS_INTERNAL_H_
+
+#include <linux/fb.h>
+#include <linux/list.h>
+#include <linux/miscdevice.h>
+#include <linux/mutex.h>
+#include <linux/pci.h>
+#include <linux/scatterlist.h>
+#include <linux/slab.h>
+#include <linux/version.h>
+
+#include "mods_config.h"
+#include "mods.h"
+
+#ifdef MODS_HAS_ASM_SET_MEMORY_HEADER
+#include <asm/set_memory.h>
+#elif defined(MODS_HAS_SET_MEMORY_HEADER)
+#include <linux/set_memory.h>
+#endif
+
+#ifdef MODS_ENABLE_BPMP_MRQ_API
+#include <soc/tegra/bpmp.h>
+#endif
+
+#ifndef true
+#define true	1
+#define false	0
+#endif
+
+/* function return code */
+#define OK		 0
+
+#define IRQ_FOUND	 1
+#define IRQ_NOT_FOUND	 0
+
+#define DEV_FOUND	 1
+#define DEV_NOT_FOUND	 0
+
+#define MSI_DEV_FOUND	  1
+#define MSI_DEV_NOT_FOUND 0
+
+struct en_dev_entry {
+	struct pci_dev	    *dev;
+	struct en_dev_entry *next;
+	struct msix_entry   *msix_entries;
+	u32                  irq_flags;
+	u32                  nvecs;
+#ifdef MODS_HAS_SRIOV
+	u32                  num_vfs;
+#endif
+	u8                   client_id;
+};
+
+struct mem_type {
+	u64 phys_addr;
+	u64 size;
+	u8  type;
+};
+
+struct irq_q_data {
+	u32		time;
+	struct pci_dev *dev;
+	u32		irq;
+	u32		irq_index;
+};
+
+struct irq_q_info {
+	struct irq_q_data data[MODS_MAX_IRQS];
+	u32		  head;
+	u32		  tail;
+};
+
+/* The driver can be opened simultaneously multiple times, from the same or from
+ * different processes.  This structure tracks data specific to each open fd.
+ */
+struct mods_client {
+	struct list_head         irq_list;
+	struct list_head         mem_alloc_list;
+	struct list_head         mem_map_list;
+	struct list_head         free_mem_list; /* unused UC/WC chunks */
+#if defined(CONFIG_PPC64)
+	struct list_head         ppc_tce_bypass_list;
+	struct list_head         nvlink_sysmem_trained_list;
+#endif
+	wait_queue_head_t        interrupt_event;
+	struct irq_q_info        irq_queue;
+	spinlock_t               irq_lock;
+	struct en_dev_entry     *enabled_devices;
+	struct workqueue_struct *work_queue;
+	struct mem_type          mem_type;
+#if defined(CONFIG_PCI)
+	struct pci_dev          *cached_dev;
+#endif
+	struct mutex             mtx;
+	int                      mods_fb_suspended[FB_MAX];
+	u32                      access_token;
+	atomic_t                 num_allocs;
+	atomic_t                 num_pages;
+#if defined(MODS_HAS_CONSOLE_LOCK)
+	atomic_t                 console_is_locked;
+#endif
+	atomic_t                 last_bad_dbdf;
+	u8                       client_id;
+};
+
+/* VM private data */
+struct mods_vm_private_data {
+	struct mods_client *client;
+	atomic_t            usage_count;
+};
+
+/* Free WC or UC chunk, which can be reused */
+struct MODS_FREE_PHYS_CHUNK {
+	struct list_head list;
+	struct page     *p_page;
+	int              numa_node;
+	u8               order;
+	u8               cache_type : 2;
+	u8               dma32      : 1;
+};
+
+/* DMA mapping into a PCI or non-PCI device */
+struct MODS_DMA_MAP {
+	struct list_head   list;
+	struct pci_dev    *pcidev;     /* PCI device these mappings are for.
+					* Can be NULL if the device the memory
+					* was mapped to is not a PCI device.
+					*/
+	struct device     *dev;        /* device these mappings are for */
+	struct scatterlist sg[1];      /* each entry corresponds to phys chunk
+					* in sg array in MODS_MEM_INFO at the
+					* same index
+					*/
+};
+
+/* system memory allocation tracking */
+struct MODS_MEM_INFO {
+	struct list_head list;
+
+	/* List of DMA mappings for devices other than the default
+	 * PCI device specified by the dev field.
+	 */
+	struct list_head dma_map_list;
+
+	u32 num_pages;      /* total number of allocated pages */
+	u32 num_chunks;     /* number of allocated contig chunks */
+	int numa_node;      /* numa node for the allocation */
+	u8  cache_type : 2; /* MODS_ALLOC_* */
+	u8  dma32      : 1; /* true/false */
+	u8  force_numa : 1; /* true/false */
+
+	struct pci_dev     *dev;         /* (optional) pci_dev this allocation
+					  * is for.
+					  */
+	unsigned long      *wc_bitmap;   /* marks which chunks use WC/UC */
+	struct scatterlist *sg;          /* current list of chunks */
+	struct scatterlist  contig_sg;   /* contiguous merged chunk */
+	struct scatterlist  alloc_sg[1]; /* allocated memory chunks, each chunk
+					  * consists of 2^n contiguous pages
+					  */
+};
+
+static inline u32 get_num_chunks(const struct MODS_MEM_INFO *p_mem_info)
+{
+	if (unlikely(p_mem_info->sg == &p_mem_info->contig_sg))
+		return 1;
+
+	return p_mem_info->num_chunks;
+}
+
+/* map memory tracking */
+struct SYS_MAP_MEMORY {
+	struct list_head list;
+
+	/* used for offset lookup, NULL for device memory */
+	struct MODS_MEM_INFO *p_mem_info;
+
+	phys_addr_t   phys_addr;
+	unsigned long virtual_addr;
+	unsigned long mapping_offs;   /* mapped offset from the beginning of the allocation */
+	unsigned long mapping_length; /* how many bytes were mapped */
+};
+
+struct mods_smmu_dev {
+	struct device *dev;
+#ifdef MODS_ENABLE_BPMP_MRQ_API
+	struct tegra_bpmp *bpmp;   //bpmp node for mrq
+	int cid;                   //pcie ctrl id
+#endif
+	char dev_name[MAX_DT_SIZE];
+};
+
+/* functions used to avoid global debug variables */
+void mods_set_debug_level(int mask);
+int mods_get_debug_level(void);
+int mods_check_debug_level(int mask);
+int mods_get_multi_instance(void);
+void mods_set_multi_instance(int mi);
+u32 mods_get_access_token(void);
+
+#if defined(CONFIG_PPC64)
+void mods_set_ppc_tce_bypass(int bypass);
+int mods_get_ppc_tce_bypass(void);
+
+/* PPC TCE bypass tracking */
+struct PPC_TCE_BYPASS {
+	struct pci_dev *dev;
+	u64 dma_mask;
+	struct list_head   list;
+};
+
+int has_npu_dev(struct pci_dev *dev, int index);
+
+int mods_is_nvlink_sysmem_trained(struct mods_client *client,
+				  struct pci_dev     *dev);
+
+/* NvLink Trained tracking */
+struct NVL_TRAINED {
+	struct pci_dev *dev;
+	u8 trained;
+	struct list_head list;
+};
+#endif
+
+#define IRQ_MAX			(256+PCI_IRQ_MAX)
+#define PCI_IRQ_MAX		15
+#define MODS_MAX_CLIENTS	32
+
+#define IRQ_VAL_POISON		0xfafbfcfdU
+
+/* debug print masks */
+#define DEBUG_IOCTL		0x2
+#define DEBUG_PCI		0x4
+#define DEBUG_ACPI		0x8
+#define DEBUG_ISR		0x10
+#define DEBUG_MEM		0x20
+#define DEBUG_FUNC		0x40
+#define DEBUG_CLOCK		0x80
+#define DEBUG_DETAILED		0x100
+#define DEBUG_TEGRADC		0x200
+#define DEBUG_TEGRADMA		0x400
+#define DEBUG_ISR_DETAILED	(DEBUG_ISR | DEBUG_DETAILED)
+#define DEBUG_MEM_DETAILED	(DEBUG_MEM | DEBUG_DETAILED)
+#define DEBUG_ALL	        (DEBUG_IOCTL | DEBUG_PCI | DEBUG_ACPI | \
+	DEBUG_ISR | DEBUG_MEM | DEBUG_FUNC | DEBUG_CLOCK | DEBUG_DETAILED | \
+	DEBUG_TEGRADC | DEBUG_TEGRADMA)
+
+#define LOG_ENT() mods_debug_printk(DEBUG_FUNC, "> %s\n", __func__)
+#define LOG_EXT() mods_debug_printk(DEBUG_FUNC, "< %s\n", __func__)
+
+#define mods_debug_printk(level, fmt, args...)\
+	({ \
+		if (mods_check_debug_level(level)) \
+			pr_info("mods debug: " fmt, ##args); \
+	})
+
+#define cl_debug(level, fmt, args...)\
+	({ \
+		if (mods_check_debug_level(level)) \
+			pr_info("mods [%u] debug: " fmt, client->client_id, \
+				##args); \
+	})
+
+#define mods_info_printk(fmt, args...)\
+	pr_info("mods: " fmt, ##args)
+
+#define cl_info(fmt, args...)\
+	pr_info("mods [%u]: " fmt, client->client_id, ##args)
+
+#define mods_error_printk(fmt, args...)\
+	pr_err("mods error: " fmt, ##args)
+
+#define cl_error(fmt, args...)\
+	pr_err("mods [%u] error: " fmt, client->client_id, ##args)
+
+#define mods_warning_printk(fmt, args...)\
+	pr_notice("mods warning: " fmt, ##args)
+
+#define cl_warn(fmt, args...)\
+	pr_notice("mods [%u] warning: " fmt, client->client_id, ##args)
+
+struct irq_mask_info {
+	void __iomem *dev_irq_mask_reg;    /*IRQ mask register, read-only reg*/
+	void __iomem *dev_irq_state;       /* IRQ status register*/
+	void __iomem *dev_irq_disable_reg; /* potentionally a write-only reg*/
+	u64           irq_and_mask;
+	u64           irq_or_mask;
+	u8            mask_type;
+};
+
+struct dev_irq_map {
+	u8          __iomem *dev_irq_aperture;
+	u32                  apic_irq;
+	u32                  entry;
+	u8                   type;
+	u8                   client_id;
+	u8                   mask_info_cnt;
+	struct irq_mask_info mask_info[MODS_IRQ_MAX_MASKS];
+	struct pci_dev      *dev;
+	struct list_head     list;
+};
+
+struct mods_priv {
+	/* Bitmap for each allocated client id. */
+	unsigned long      client_flags;
+
+	/* Client structures */
+	struct mods_client clients[MODS_MAX_CLIENTS];
+};
+
+#ifdef MODS_HAS_POLL_T
+#	define POLL_TYPE __poll_t
+#else
+#	define POLL_TYPE unsigned int
+#endif
+
+#if ((defined(CONFIG_ARM) || defined(CONFIG_ARM64)) && \
+	  !defined(CONFIG_CPA)) || defined(CONFIG_PPC64)
+#	define MODS_SET_MEMORY_UC(addr, pages) 0
+#	define MODS_SET_MEMORY_WC(addr, pages) 0
+#	define MODS_SET_MEMORY_WB(addr, pages) 0
+#else
+#	define MODS_SET_MEMORY_UC(addr, pages) set_memory_uc(addr, pages)
+#	define MODS_SET_MEMORY_WC(addr, pages) set_memory_wc(addr, pages)
+#	define MODS_SET_MEMORY_WB(addr, pages) set_memory_wb(addr, pages)
+#endif
+
+#define MODS_PGPROT_UC pgprot_noncached
+#define MODS_PGPROT_WC pgprot_writecombine
+
+/* ACPI */
+#ifdef MODS_HAS_NEW_ACPI_WALK
+#define MODS_ACPI_WALK_NAMESPACE(type, start_object, max_depth, user_function, \
+				 context, return_value)\
+	acpi_walk_namespace(type, start_object, max_depth, user_function, NULL,\
+			    context, return_value)
+#else
+#define MODS_ACPI_WALK_NAMESPACE acpi_walk_namespace
+#endif
+#ifdef MODS_HAS_NEW_ACPI_HANDLE
+#define MODS_ACPI_HANDLE(dev) ACPI_HANDLE(dev)
+#else
+#define MODS_ACPI_HANDLE(dev) DEVICE_ACPI_HANDLE(dev)
+#endif
+
+#if KERNEL_VERSION(3, 10, 0) <= MODS_KERNEL_VERSION
+#	define MODS_SG_UNMARK_END(sg) sg_unmark_end(sg)
+#else
+#	define MODS_SG_UNMARK_END(sg) ({(sg)->page_link &= ~2; })
+#endif
+
+#if KERNEL_VERSION(5, 11, 0) <= MODS_KERNEL_VERSION
+#	define MODS_KMAP kmap_local_page
+#else
+#	define MODS_KMAP kmap
+#endif
+
+/* ************************************************************************* */
+/* ** MODULE WIDE FUNCTIONS						     */
+/* ************************************************************************* */
+
+/* client */
+struct mods_client *mods_client_from_id(u8 client_id);
+int mods_is_client_enabled(u8 client_id);
+
+/* irq */
+void mods_init_irq(void);
+struct mutex *mods_get_irq_mutex(void);
+void mods_free_client_interrupts(struct mods_client *client);
+POLL_TYPE mods_irq_event_check(u8 client_id);
+
+/* mem */
+const char *mods_get_prot_str(u8 mem_type);
+int mods_unregister_all_alloc(struct mods_client *client);
+struct MODS_MEM_INFO *mods_find_alloc(struct mods_client *client,
+				      u64                 phys_addr);
+
+#if defined(CONFIG_PPC64)
+/* ppc64 */
+int mods_unregister_all_ppc_tce_bypass(struct mods_client *client);
+
+int mods_unregister_all_nvlink_sysmem_trained(struct mods_client *client);
+#endif
+
+/* pci */
+#ifdef CONFIG_PCI
+int mods_enable_device(struct mods_client   *client,
+		       struct pci_dev       *dev,
+		       struct en_dev_entry **dev_entry);
+void mods_disable_device(struct mods_client *client,
+			 struct pci_dev     *pdev);
+#endif
+
+#ifdef CONFIG_PCI
+int mods_is_pci_dev(struct pci_dev        *dev,
+		    struct mods_pci_dev_2 *pcidev);
+int mods_find_pci_dev(struct mods_client    *client,
+		      struct mods_pci_dev_2 *pcidev,
+		      struct pci_dev       **retdev);
+#else
+#define mods_is_pci_dev(a, b) 0
+#define mods_find_pci_dev(a, b, c) (-ENODEV)
+#endif
+
+/* clock */
+#if defined(MODS_HAS_TEGRA) && defined(CONFIG_COMMON_CLK) && \
+		defined(CONFIG_OF_RESOLVE) && defined(CONFIG_OF_DYNAMIC)
+void mods_init_clock_api(void);
+void mods_shutdown_clock_api(void);
+#endif
+
+/* ioctl hanndlers */
+
+/* mem */
+int esc_mods_alloc_pages(struct mods_client      *client,
+			 struct MODS_ALLOC_PAGES *p);
+int esc_mods_device_alloc_pages(struct mods_client             *client,
+				struct MODS_DEVICE_ALLOC_PAGES *p);
+int esc_mods_device_alloc_pages_2(struct mods_client               *client,
+				  struct MODS_DEVICE_ALLOC_PAGES_2 *p);
+int esc_mods_alloc_pages_2(struct mods_client        *client,
+			   struct MODS_ALLOC_PAGES_2 *p);
+int esc_mods_free_pages(struct mods_client     *client,
+			struct MODS_FREE_PAGES *p);
+int esc_mods_merge_pages(struct mods_client      *client,
+			 struct MODS_MERGE_PAGES *p);
+int esc_mods_set_mem_type(struct mods_client      *client,
+			  struct MODS_MEMORY_TYPE *p);
+int esc_mods_get_phys_addr(struct mods_client               *client,
+			   struct MODS_GET_PHYSICAL_ADDRESS *p);
+int esc_mods_get_phys_addr_2(struct mods_client *client,
+			     struct MODS_GET_PHYSICAL_ADDRESS_3 *p);
+int esc_mods_get_mapped_phys_addr(struct mods_client       *client,
+			  struct MODS_GET_PHYSICAL_ADDRESS *p);
+int esc_mods_get_mapped_phys_addr_2(struct mods_client                 *client,
+				    struct MODS_GET_PHYSICAL_ADDRESS_2 *p);
+int esc_mods_get_mapped_phys_addr_3(struct mods_client                 *client,
+				    struct MODS_GET_PHYSICAL_ADDRESS_3 *p);
+int esc_mods_virtual_to_phys(struct mods_client              *client,
+			     struct MODS_VIRTUAL_TO_PHYSICAL *p);
+int esc_mods_phys_to_virtual(struct mods_client              *client,
+			     struct MODS_PHYSICAL_TO_VIRTUAL *p);
+int esc_mods_dma_map_memory(struct mods_client         *client,
+			    struct MODS_DMA_MAP_MEMORY *p);
+int esc_mods_dma_unmap_memory(struct mods_client         *client,
+			      struct MODS_DMA_MAP_MEMORY *p);
+int esc_mods_iommu_dma_map_memory(struct mods_client               *client,
+				  struct MODS_IOMMU_DMA_MAP_MEMORY *p);
+int esc_mods_iommu_dma_unmap_memory(struct mods_client               *client,
+				    struct MODS_IOMMU_DMA_MAP_MEMORY *p);
+
+#ifdef CONFIG_ARM
+int esc_mods_memory_barrier(struct mods_client *client);
+#endif
+
+#ifdef CONFIG_ARM64
+int esc_mods_flush_cpu_cache_range(struct mods_client                *client,
+				   struct MODS_FLUSH_CPU_CACHE_RANGE *p);
+#endif
+
+#if defined(CONFIG_PPC64)
+/* ppc64 */
+int esc_mods_set_ppc_tce_bypass(struct mods_client             *client,
+				struct MODS_SET_PPC_TCE_BYPASS *p);
+int esc_mods_get_ats_address_range(struct mods_client                *client,
+				   struct MODS_GET_ATS_ADDRESS_RANGE *p);
+int esc_mods_set_nvlink_sysmem_trained(struct mods_client             *client,
+				struct MODS_SET_NVLINK_SYSMEM_TRAINED *p);
+int esc_mods_get_nvlink_line_rate(struct mods_client               *client,
+				  struct MODS_GET_NVLINK_LINE_RATE *p);
+#endif
+
+/* acpi */
+#ifdef CONFIG_ACPI
+int esc_mods_eval_acpi_method(struct mods_client           *client,
+			      struct MODS_EVAL_ACPI_METHOD *p);
+int esc_mods_eval_dev_acpi_method(struct mods_client               *client,
+				  struct MODS_EVAL_DEV_ACPI_METHOD *p);
+int esc_mods_eval_dev_acpi_method_2(struct mods_client                 *client,
+				    struct MODS_EVAL_DEV_ACPI_METHOD_2 *p);
+int esc_mods_eval_dev_acpi_method_3(struct mods_client                 *client,
+				    struct MODS_EVAL_DEV_ACPI_METHOD_3 *p);
+int esc_mods_acpi_get_ddc(struct mods_client       *client,
+			  struct MODS_ACPI_GET_DDC *p);
+int esc_mods_acpi_get_ddc_2(struct mods_client         *client,
+			    struct MODS_ACPI_GET_DDC_2 *p);
+int esc_mods_get_acpi_dev_children(struct mods_client    *client,
+				   struct MODS_GET_ACPI_DEV_CHILDREN *p);
+#ifdef MODS_HAS_PXM_TO_NODE
+int esc_mods_proximity_to_numa_node(struct mods_client                 *client,
+				    struct MODS_PROXIMITY_TO_NUMA_NODE *p);
+#endif
+#endif
+/* pci */
+#ifdef CONFIG_PCI
+int esc_mods_find_pci_dev(struct mods_client          *client,
+			  struct MODS_FIND_PCI_DEVICE *p);
+int esc_mods_find_pci_dev_2(struct mods_client            *client,
+			    struct MODS_FIND_PCI_DEVICE_2 *p);
+int esc_mods_find_pci_class_code(struct mods_client              *client,
+				 struct MODS_FIND_PCI_CLASS_CODE *p);
+int esc_mods_find_pci_class_code_2(struct mods_client                *client,
+				   struct MODS_FIND_PCI_CLASS_CODE_2 *p);
+int esc_mods_pci_get_bar_info(struct mods_client           *client,
+			      struct MODS_PCI_GET_BAR_INFO *p);
+int esc_mods_pci_get_bar_info_2(struct mods_client             *client,
+				struct MODS_PCI_GET_BAR_INFO_2 *p);
+int esc_mods_pci_get_irq(struct mods_client      *client,
+			 struct MODS_PCI_GET_IRQ *p);
+int esc_mods_pci_get_irq_2(struct mods_client        *client,
+			   struct MODS_PCI_GET_IRQ_2 *p);
+int esc_mods_pci_read(struct mods_client *client, struct MODS_PCI_READ *p);
+int esc_mods_pci_read_2(struct mods_client *client, struct MODS_PCI_READ_2 *p);
+int esc_mods_pci_write(struct mods_client *client, struct MODS_PCI_WRITE *p);
+int esc_mods_pci_write_2(struct mods_client      *client,
+			 struct MODS_PCI_WRITE_2 *p);
+int esc_mods_pci_bus_rescan(struct mods_client         *client,
+			    struct MODS_PCI_BUS_RESCAN *p);
+int esc_mods_pci_bus_add_dev(struct mods_client              *client,
+			     struct MODS_PCI_BUS_ADD_DEVICES *p);
+int esc_mods_pci_bus_remove_dev(struct mods_client        *client,
+			   struct MODS_PCI_BUS_REMOVE_DEV *p);
+int esc_mods_pci_hot_reset(struct mods_client        *client,
+			   struct MODS_PCI_HOT_RESET *p);
+int esc_mods_pio_read(struct mods_client *client, struct MODS_PIO_READ *p);
+int esc_mods_pio_write(struct mods_client *client, struct MODS_PIO_WRITE *p);
+int esc_mods_device_numa_info(struct mods_client           *client,
+			      struct MODS_DEVICE_NUMA_INFO *p);
+int esc_mods_device_numa_info_2(struct mods_client             *client,
+				struct MODS_DEVICE_NUMA_INFO_2 *p);
+int esc_mods_device_numa_info_3(struct mods_client             *client,
+				struct MODS_DEVICE_NUMA_INFO_3 *p);
+int esc_mods_get_iommu_state(struct mods_client          *client,
+			     struct MODS_GET_IOMMU_STATE *state);
+int esc_mods_get_iommu_state_2(struct mods_client          *client,
+			       struct MODS_GET_IOMMU_STATE *state);
+int esc_mods_pci_set_dma_mask(struct mods_client      *client,
+			     struct MODS_PCI_DMA_MASK *dma_mask);
+int esc_mods_pci_reset_function(struct mods_client    *client,
+				struct mods_pci_dev_2 *pcidev);
+#ifdef MODS_HAS_DEV_PROPS
+int esc_mods_read_dev_property(struct mods_client            *client,
+			       struct MODS_READ_DEV_PROPERTY *p);
+#endif
+#endif
+/* irq */
+#if defined(MODS_HAS_TEGRA) && defined(CONFIG_OF) && defined(CONFIG_OF_IRQ)
+int esc_mods_map_irq(struct mods_client *client, struct MODS_DT_INFO *p);
+int esc_mods_map_irq_to_gpio(struct mods_client *client,
+						struct MODS_GPIO_INFO *p);
+#endif
+int esc_mods_register_irq(struct mods_client       *client,
+			  struct MODS_REGISTER_IRQ *p);
+int esc_mods_register_irq_2(struct mods_client         *client,
+			    struct MODS_REGISTER_IRQ_2 *p);
+int esc_mods_register_irq_3(struct mods_client         *client,
+			    struct MODS_REGISTER_IRQ_3 *p);
+int esc_mods_unregister_irq(struct mods_client       *client,
+			    struct MODS_REGISTER_IRQ *p);
+int esc_mods_unregister_irq_2(struct mods_client         *client,
+			      struct MODS_REGISTER_IRQ_2 *p);
+int esc_mods_query_irq(struct mods_client *client, struct MODS_QUERY_IRQ *p);
+int esc_mods_query_irq_2(struct mods_client      *client,
+			 struct MODS_QUERY_IRQ_2 *p);
+int esc_mods_irq_handled(struct mods_client       *client,
+			 struct MODS_REGISTER_IRQ *p);
+int esc_mods_irq_handled_2(struct mods_client         *client,
+			   struct MODS_REGISTER_IRQ_2 *p);
+
+int esc_mods_register_irq_4(struct mods_client         *client,
+			    struct MODS_REGISTER_IRQ_4 *p);
+int esc_mods_query_irq_3(struct mods_client      *client,
+			 struct MODS_QUERY_IRQ_3 *p);
+
+#ifdef MODS_HAS_PROD
+/* bpmp uphy */
+int esc_mods_bpmp_set_pcie_state(struct mods_client *client,
+				  struct MODS_SET_PCIE_STATE *p);
+int esc_mods_bpmp_init_pcie_ep_pll(struct mods_client *client,
+				  struct MODS_INIT_PCIE_EP_PLL *p);
+#endif
+
+#ifdef MODS_HAS_TEGRA
+
+/* clock */
+int esc_mods_get_clock_handle(struct mods_client           *client,
+			      struct MODS_GET_CLOCK_HANDLE *p);
+int esc_mods_set_clock_rate(struct mods_client     *client,
+			    struct MODS_CLOCK_RATE *p);
+int esc_mods_get_clock_rate(struct mods_client     *client,
+			    struct MODS_CLOCK_RATE *p);
+int esc_mods_get_clock_max_rate(struct mods_client     *client,
+				struct MODS_CLOCK_RATE *p);
+int esc_mods_set_clock_max_rate(struct mods_client     *client,
+				struct MODS_CLOCK_RATE *p);
+int esc_mods_set_clock_parent(struct mods_client       *client,
+			      struct MODS_CLOCK_PARENT *p);
+int esc_mods_get_clock_parent(struct mods_client       *client,
+			      struct MODS_CLOCK_PARENT *p);
+int esc_mods_enable_clock(struct mods_client       *client,
+			  struct MODS_CLOCK_HANDLE *p);
+int esc_mods_disable_clock(struct mods_client       *client,
+			   struct MODS_CLOCK_HANDLE *p);
+int esc_mods_is_clock_enabled(struct mods_client        *client,
+			      struct MODS_CLOCK_ENABLED *p);
+int esc_mods_clock_reset_assert(struct mods_client       *client,
+				struct MODS_CLOCK_HANDLE *p);
+int esc_mods_clock_reset_deassert(struct mods_client       *client,
+				  struct MODS_CLOCK_HANDLE *p);
+int esc_mods_reset_assert(struct mods_client       *client,
+			  struct MODS_RESET_HANDLE *p);
+int esc_mods_get_rst_handle(struct mods_client          *client,
+			    struct MODS_GET_RESET_HANDLE *p);
+int esc_mods_dma_alloc_coherent(struct mods_client                  *client,
+				struct MODS_DMA_COHERENT_MEM_HANDLE *p);
+int esc_mods_dma_free_coherent(struct mods_client                  *client,
+			       struct MODS_DMA_COHERENT_MEM_HANDLE *p);
+int esc_mods_dma_copy_to_user(struct mods_client           *client,
+			      struct MODS_DMA_COPY_TO_USER *p);
+
+/* oist */
+int esc_mods_oist_status(struct mods_client             *client,
+			     struct MODS_TEGRA_OIST_STATUS  *p);
+
+#ifdef CONFIG_DMA_ENGINE
+int mods_init_dma(void);
+void mods_exit_dma(void);
+int esc_mods_dma_request_channel(struct mods_client     *client,
+				 struct MODS_DMA_HANDLE *p);
+int esc_mods_dma_request_channel_2(struct mods_client       *client,
+				   struct MODS_DMA_HANDLE_2 *p_handle_2);
+int esc_mods_dma_release_channel(struct mods_client     *client,
+				 struct MODS_DMA_HANDLE *p);
+int esc_mods_dma_set_config(struct mods_client             *client,
+			    struct MODS_DMA_CHANNEL_CONFIG *p);
+int esc_mods_dma_wait(struct mods_client *client, struct MODS_DMA_WAIT_DESC *p);
+int esc_mods_dma_submit_request(struct mods_client      *client,
+				struct MODS_DMA_TX_DESC *p);
+int esc_mods_dma_async_issue_pending(struct mods_client     *client,
+				     struct MODS_DMA_HANDLE *p);
+#endif
+
+#ifdef CONFIG_TEGRA_DC
+int esc_mods_tegra_dc_config_possible(struct mods_client *client,
+				      struct MODS_TEGRA_DC_CONFIG_POSSIBLE *p);
+#endif
+
+#if defined(MODS_HAS_TEGRA) && defined(CONFIG_NET)
+int esc_mods_net_force_link(struct mods_client          *client,
+			    struct MODS_NET_DEVICE_NAME *p);
+#endif
+
+#if defined(MODS_HAS_DMABUF) && defined(CONFIG_DMA_SHARED_BUFFER)
+int esc_mods_dmabuf_get_phys_addr(struct mods_client *client,
+				  struct MODS_DMABUF_GET_PHYSICAL_ADDRESS *p);
+#else
+static inline int esc_mods_dmabuf_get_phys_addr(struct mods_client *client,
+				  struct MODS_DMABUF_GET_PHYSICAL_ADDRESS *p)
+				  { return -EINVAL; }
+#endif
+
+#ifdef CONFIG_TEGRA_NVADSP
+int esc_mods_adsp_load(struct mods_client *client);
+int esc_mods_adsp_start(struct mods_client *client);
+int esc_mods_adsp_stop(struct mods_client *client);
+int esc_mods_adsp_run_app(struct mods_client            *client,
+			  struct MODS_ADSP_RUN_APP_INFO *p);
+#endif
+
+#ifdef MODS_HAS_PROD
+/* prod */
+int mods_tegra_prod_init(const struct miscdevice *misc_dev);
+int esc_mods_tegra_prod_iterate_dt(struct mods_client *client,
+	struct MODS_TEGRA_PROD_ITERATOR *iterator);
+int esc_mods_tegra_prod_is_supported(struct mods_client *client,
+	struct MODS_TEGRA_PROD_IS_SUPPORTED *tuple);
+int esc_mods_tegra_prod_set_prod_all(struct mods_client *client,
+	struct MODS_TEGRA_PROD_SET_TUPLE *tuple);
+int esc_mods_tegra_prod_set_prod_boot(struct mods_client *client,
+	struct MODS_TEGRA_PROD_SET_TUPLE *tuple);
+int esc_mods_tegra_prod_set_prod_by_name(struct mods_client *client,
+	struct MODS_TEGRA_PROD_SET_TUPLE *tuple);
+int esc_mods_tegra_prod_set_prod_exact(struct mods_client *client,
+	struct MODS_TEGRA_PROD_SET_TUPLE *tuple);
+#endif
+
+#ifdef CONFIG_TRUSTY
+/* trustzone app call */
+int esc_mods_send_trustzone_msg(struct mods_client         *client,
+	struct MODS_TZ_PARAMS      *p);
+#endif
+
+#ifdef CONFIG_OPTEE
+/* OP-TEE TA call */
+int esc_mods_invoke_optee_ta(struct mods_client *client,
+	struct MODS_OPTEE_PARAMS *p);
+#endif
+#endif
+
+/* MODS SP call */
+#if defined(CONFIG_ARM_FFA_TRANSPORT)
+int esc_mods_arm_ffa_cmd(struct mods_client *client,
+			 struct MODS_FFA_PARAMS *p);
+#endif
+
+#ifdef CONFIG_DEBUG_FS
+int mods_create_debugfs(struct miscdevice *modsdev);
+void mods_remove_debugfs(void);
+#else
+static inline int mods_create_debugfs(struct miscdevice *modsdev)
+{
+	return 0;
+}
+static inline void mods_remove_debugfs(void) {}
+#endif /* CONFIG_DEBUG_FS */
+
+#if defined(MODS_HAS_DMABUF) && defined(CONFIG_DMA_SHARED_BUFFER)
+int mods_init_dmabuf(void);
+void mods_exit_dmabuf(void);
+#else
+static inline int mods_init_dmabuf(void) { return 0; }
+static inline void mods_exit_dmabuf(void) {}
+#endif
+
+#if defined(MODS_HAS_TEGRA)
+int get_mods_smmu_device_index(const char *name);
+struct mods_smmu_dev *get_mods_smmu_device(u32 index);
+int smmu_driver_init(void);
+void smmu_driver_exit(void);
+#endif
+
+#if defined(MODS_HAS_TEGRA)
+int esc_mods_send_ipi(struct mods_client *client, struct MODS_SEND_IPI *p);
+#endif
+
+#if defined(CONFIG_TEGRA_IVC)
+int esc_mods_bpmp_uphy_lane_eom_scan(struct mods_client *client,
+				     struct MODS_BPMP_UPHY_LANE_EOM_SCAN_PARAMS *p);
+int mods_bpmpipc_init(struct mods_client *client,
+		      u64 db_phys_addr,
+		      u64 req_phys_addr,
+		      u64 resp_phys_addr);
+void mods_bpmpipc_cleanup(void);
+#endif
+
+#if defined(CONFIG_ARM_FFA_TRANSPORT)
+int mods_ffa_abi_register(void);
+void mods_ffa_abi_unregister(void);
+#endif
+
+#endif	/* _MODS_INTERNAL_H_  */
diff --git a/drivers/misc/mods/mods_ipi.c b/drivers/misc/mods/mods_ipi.c
new file mode 100644
index 00000000..b4c89f54
--- /dev/null
+++ b/drivers/misc/mods/mods_ipi.c
@@ -0,0 +1,94 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * This file is part of NVIDIA MODS kernel driver.
+ *
+ * Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * NVIDIA MODS kernel driver is free software: you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * NVIDIA MODS kernel driver is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NVIDIA MODS kernel driver.
+ * If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "mods_internal.h"
+
+#include <asm/barrier.h>
+#include <linux/cpu.h>
+#include <linux/cpumask.h>
+#include <linux/slab.h>
+#include <linux/smp.h>
+
+#define MAX_IPI_WFI_LOOPS	5
+#define MAX_IPI_WFE_LOOPS	10
+
+static void ipi_wfi_cpu(void *p)
+{
+	struct MODS_SEND_IPI *p_ipi = (struct MODS_SEND_IPI *)p;
+	u32 i;
+
+	for (i = 0; i < p_ipi->num_loops; i++)
+		wfi();
+}
+
+static void ipi_wfe_cpu(void *p)
+{
+	struct MODS_SEND_IPI *p_ipi = (struct MODS_SEND_IPI *)p;
+	u32 i;
+
+	for (i = 0; i < p_ipi->num_loops; i++)
+		wfe();
+}
+
+int esc_mods_send_ipi(struct mods_client *client, struct MODS_SEND_IPI *p)
+{
+	LOG_ENT();
+
+	switch (p->ipi_type) {
+	case MODS_IPI_KICK: {
+		kick_all_cpus_sync();
+		break;
+	}
+	case MODS_IPI_WFI: {
+		if (p->num_loops > MAX_IPI_WFI_LOOPS) {
+			cl_error("num_loops %u exceed maximum limit %d for MODS_IPI_WFI\n",
+				p->num_loops,
+				MAX_IPI_WFI_LOOPS);
+			LOG_EXT();
+			return -EINVAL;
+		}
+		preempt_disable();
+		smp_call_function_many(cpu_online_mask, ipi_wfi_cpu, (void *)p, 1);
+		preempt_enable();
+		break;
+	}
+	case MODS_IPI_WFE: {
+		if (p->num_loops > MAX_IPI_WFE_LOOPS) {
+			cl_error("num_loops %u exceed maximum limit %d for MODS_IPI_WFE\n",
+				p->num_loops,
+				MAX_IPI_WFE_LOOPS);
+			LOG_EXT();
+			return -EINVAL;
+		}
+		preempt_disable();
+		smp_call_function_many(cpu_online_mask, ipi_wfe_cpu, (void *)p, 1);
+		preempt_enable();
+		break;
+	}
+	default: {
+		cl_error("unsupported MODS_IPI_TYPE\n");
+		LOG_EXT();
+		return -EINVAL;
+	}
+	}
+
+	LOG_EXT();
+	return 0;
+}
diff --git a/drivers/misc/mods/mods_irq.c b/drivers/misc/mods/mods_irq.c
new file mode 100644
index 00000000..cfeb2e28
--- /dev/null
+++ b/drivers/misc/mods/mods_irq.c
@@ -0,0 +1,1636 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * This file is part of NVIDIA MODS kernel driver.
+ *
+ * Copyright (c) 2008-2023, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * NVIDIA MODS kernel driver is free software: you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * NVIDIA MODS kernel driver is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NVIDIA MODS kernel driver.
+ * If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "mods_internal.h"
+
+#include <linux/sched.h>
+#include <linux/poll.h>
+#include <linux/interrupt.h>
+#include <linux/pci_regs.h>
+#if defined(MODS_HAS_TEGRA) && defined(CONFIG_OF) && defined(CONFIG_OF_IRQ)
+#include <linux/of.h>
+#include <linux/of_irq.h>
+#include <linux/io.h>
+#include <linux/platform_device.h>
+#include <linux/of_device.h>
+#include <linux/gpio.h>
+#include <linux/of_gpio.h>
+#endif
+
+#define PCI_VENDOR_ID_NVIDIA 0x10de
+#define INDEX_IRQSTAT(irq)	(irq / BITS_NUM)
+#define POS_IRQSTAT(irq)	 (irq & (BITS_NUM - 1))
+
+/* MSI */
+#define PCI_MSI_MASK_BIT	16
+#define MSI_CONTROL_REG(base)		(base + PCI_MSI_FLAGS)
+#define IS_64BIT_ADDRESS(control)	(!!(control & PCI_MSI_FLAGS_64BIT))
+#define MSI_DATA_REG(base, is64bit) \
+	((is64bit == 1) ? base + PCI_MSI_DATA_64 : base + PCI_MSI_DATA_32)
+#define TOP_TKE_TKEIE_WDT_MASK(i)	(1 << (16 + 4 * (i)))
+#define TOP_TKE_TKEIE(i)		(0x100 + 4 * (i))
+
+/*********************
+ * PRIVATE FUNCTIONS *
+ *********************/
+
+/* Mutex for guarding interrupt logic and PCI device enablement */
+static struct mutex irq_mtx;
+
+struct mutex *mods_get_irq_mutex(void)
+{
+	return &irq_mtx;
+}
+
+#ifdef CONFIG_PCI
+int mods_enable_device(struct mods_client   *client,
+		       struct pci_dev       *dev,
+		       struct en_dev_entry **dev_entry)
+{
+	int                  err   = OK;
+	struct en_dev_entry *dpriv = client->enabled_devices;
+
+	WARN_ON(!mutex_is_locked(&irq_mtx));
+
+	dpriv = pci_get_drvdata(dev);
+	if (dpriv) {
+		if (dpriv->client_id == client->client_id) {
+			if (dev_entry)
+				*dev_entry = dpriv;
+			return OK;
+		}
+
+		cl_error(
+			"invalid client for dev %04x:%02x:%02x.%x, expected %u\n",
+			pci_domain_nr(dev->bus),
+			dev->bus->number,
+			PCI_SLOT(dev->devfn),
+			PCI_FUNC(dev->devfn),
+			dpriv->client_id);
+		return -EBUSY;
+	}
+
+	dpriv = kzalloc(sizeof(*dpriv), GFP_KERNEL | __GFP_NORETRY);
+	if (unlikely(!dpriv))
+		return -ENOMEM;
+	atomic_inc(&client->num_allocs);
+
+	err = pci_enable_device(dev);
+	if (unlikely(err)) {
+		cl_error("failed to enable dev %04x:%02x:%02x.%x\n",
+			 pci_domain_nr(dev->bus),
+			 dev->bus->number,
+			 PCI_SLOT(dev->devfn),
+			 PCI_FUNC(dev->devfn));
+		kfree(dpriv);
+		atomic_dec(&client->num_allocs);
+		return err;
+	}
+
+	cl_info("enabled dev %04x:%02x:%02x.%x\n",
+		pci_domain_nr(dev->bus),
+		dev->bus->number,
+		PCI_SLOT(dev->devfn),
+		PCI_FUNC(dev->devfn));
+
+	dpriv->client_id        = client->client_id;
+	dpriv->dev              = pci_dev_get(dev);
+	dpriv->next             = client->enabled_devices;
+	client->enabled_devices = dpriv;
+	pci_set_drvdata(dev, dpriv);
+
+	if (dev_entry)
+		*dev_entry = dpriv;
+	return OK;
+}
+
+void mods_disable_device(struct mods_client *client,
+			 struct pci_dev     *dev)
+{
+	struct en_dev_entry *dpriv = pci_get_drvdata(dev);
+
+	WARN_ON(!mutex_is_locked(&irq_mtx));
+
+#ifdef MODS_HAS_SRIOV
+	if (dpriv && dpriv->num_vfs)
+		pci_disable_sriov(dev);
+#endif
+
+	if (dpriv) {
+		pci_set_drvdata(dev, NULL);
+		pci_dev_put(dev);
+	}
+
+	pci_disable_device(dev);
+
+	cl_info("disabled dev %04x:%02x:%02x.%x\n",
+		pci_domain_nr(dev->bus),
+		dev->bus->number,
+		PCI_SLOT(dev->devfn),
+		PCI_FUNC(dev->devfn));
+}
+#endif
+
+static unsigned int get_cur_time(void)
+{
+	/* This is not very precise, sched_clock() would be better */
+	return jiffies_to_usecs(jiffies);
+}
+
+static u64 irq_reg_read(const struct irq_mask_info *m, void __iomem *reg)
+{
+	if (m->mask_type == MODS_MASK_TYPE_IRQ_DISABLE64)
+		return readq(reg);
+	else
+		return readl(reg);
+}
+
+static void irq_reg_write(const struct irq_mask_info *m,
+			  u64                         value,
+			  void               __iomem *reg)
+{
+	if (m->mask_type == MODS_MASK_TYPE_IRQ_DISABLE64)
+		writeq(value, reg);
+	else {
+		const u32 val32 = (value > ~0U) ? 0 : value;
+
+		writel(val32, reg);
+	}
+}
+
+static u64 read_irq_state(const struct irq_mask_info *m)
+{
+	return irq_reg_read(m, m->dev_irq_state);
+}
+
+static u64 read_irq_mask(const struct irq_mask_info *m)
+{
+	return irq_reg_read(m, m->dev_irq_mask_reg);
+}
+
+static void write_irq_disable(u64 value, const struct irq_mask_info *m)
+{
+	irq_reg_write(m, value, m->dev_irq_disable_reg);
+}
+
+static int mods_check_interrupt(struct dev_irq_map *t)
+{
+	int ii    = 0;
+	int valid = 0;
+
+	/* For MSI - we always treat it as pending (must rearm later). */
+	/* For non-GPU devices - we can't tell. */
+	if (t->mask_info_cnt == 0)
+		return true;
+
+	for (ii = 0; ii < t->mask_info_cnt; ii++) {
+		const struct irq_mask_info *const m = &t->mask_info[ii];
+
+		if (!m->dev_irq_state || !m->dev_irq_mask_reg)
+			continue;
+
+		/* GPU device */
+		valid |= (read_irq_state(m) && read_irq_mask(m)) != 0;
+	}
+
+	return valid;
+}
+
+static void mods_disable_interrupts(struct dev_irq_map *t)
+{
+	u32 ii = 0;
+
+	for (ii = 0; ii < t->mask_info_cnt; ii++) {
+		const struct irq_mask_info *const m = &t->mask_info[ii];
+		u64 cur_mask = 0;
+
+		if (!m->dev_irq_disable_reg)
+			continue;
+
+		if (m->irq_and_mask == 0) {
+			write_irq_disable(m->irq_or_mask, m);
+			continue;
+		}
+
+		cur_mask = read_irq_mask(m);
+		cur_mask &= m->irq_and_mask;
+		cur_mask |= m->irq_or_mask;
+		write_irq_disable(cur_mask, m);
+	}
+
+	if ((ii == 0) && t->type == MODS_IRQ_TYPE_CPU) {
+		mods_debug_printk(DEBUG_ISR,
+				  "disable_irq_nosync %u",
+				  t->apic_irq);
+		disable_irq_nosync(t->apic_irq);
+	}
+}
+
+#ifdef CONFIG_PCI
+static const char *mods_irq_type_name(u8 irq_type)
+{
+	switch (irq_type) {
+	case MODS_IRQ_TYPE_INT:
+		return "INTx";
+	case MODS_IRQ_TYPE_MSI:
+		return "MSI";
+	case MODS_IRQ_TYPE_CPU:
+		return "CPU";
+	case MODS_IRQ_TYPE_MSIX:
+		return "MSI-X";
+	default:
+		return "unknown";
+	}
+}
+#endif
+
+static void wake_up_client(struct dev_irq_map *t)
+{
+	struct mods_client *client = mods_client_from_id(t->client_id);
+
+	if (client)
+		wake_up_interruptible(&client->interrupt_event);
+}
+
+static int rec_irq_done(struct mods_client *client,
+			struct dev_irq_map *t,
+			unsigned int        irq_time)
+{
+	/* Get interrupt queue */
+	struct irq_q_info *q = &client->irq_queue;
+
+	/* Don't do anything if the IRQ has already been recorded */
+	if (q->head != q->tail) {
+		unsigned int i;
+
+		for (i = q->head; i != q->tail; i++) {
+			struct irq_q_data *pd
+					= q->data+(i & (MODS_MAX_IRQS - 1));
+
+			if ((pd->irq == t->apic_irq) &&
+			    (!t->dev || (pd->dev == t->dev)))
+				return false;
+		}
+	}
+
+	/* Print an error if the queue is full */
+	/* This is deadly! */
+	if (q->tail - q->head == MODS_MAX_IRQS) {
+		mods_error_printk("IRQ queue is full\n");
+		return false;
+	}
+
+	/* Record the device which generated the IRQ in the queue */
+	q->data[q->tail & (MODS_MAX_IRQS - 1)].dev       = t->dev;
+	q->data[q->tail & (MODS_MAX_IRQS - 1)].irq       = t->apic_irq;
+	q->data[q->tail & (MODS_MAX_IRQS - 1)].irq_index = t->entry;
+	q->data[q->tail & (MODS_MAX_IRQS - 1)].time      = irq_time;
+	q->tail++;
+
+#ifdef CONFIG_PCI
+	if (t->dev) {
+		mods_debug_printk(DEBUG_ISR_DETAILED,
+				  "dev %04x:%02x:%02x.%x %s IRQ 0x%x time=%uus\n",
+				  pci_domain_nr(t->dev->bus),
+				  t->dev->bus->number,
+				  PCI_SLOT(t->dev->devfn),
+				  PCI_FUNC(t->dev->devfn),
+				  mods_irq_type_name(t->type),
+				  t->apic_irq,
+				  irq_time);
+	} else
+#endif
+		mods_debug_printk(DEBUG_ISR_DETAILED,
+				  "CPU IRQ 0x%x, time=%uus\n",
+				  t->apic_irq,
+				  irq_time);
+
+	return true;
+}
+
+/* mods_irq_handle - interrupt function */
+static irqreturn_t mods_irq_handle(int irq, void *data)
+{
+	struct dev_irq_map *t        = (struct dev_irq_map *)data;
+	int                 serviced = false;
+
+	if (irq < 0)
+		return IRQ_RETVAL(false);
+
+	if (unlikely(!t))
+		mods_error_printk("received irq %d, but no context for it\n",
+			irq);
+	else if (unlikely(t->apic_irq != irq))
+		mods_error_printk("received irq %d which doesn't match registered irq %d\n",
+			irq, t->apic_irq);
+	else {
+		unsigned long       flags    = 0;
+		int                 recorded = false;
+		unsigned int        irq_time = get_cur_time();
+		struct mods_client *client   = mods_client_from_id(t->client_id);
+
+		spin_lock_irqsave(&client->irq_lock, flags);
+
+		/* Check if the interrupt is still pending (shared INTA) */
+		if (mods_check_interrupt(t)) {
+
+			/* Disable interrupts on this device to avoid interrupt
+			 * storm
+			 */
+			mods_disable_interrupts(t);
+
+			/* Record IRQ for MODS and wake MODS up */
+			recorded = rec_irq_done(client, t, irq_time);
+
+			serviced = true;
+		}
+
+		spin_unlock_irqrestore(&client->irq_lock, flags);
+
+		if (recorded)
+			wake_up_client(t);
+	}
+
+	return IRQ_RETVAL(serviced);
+}
+
+static int mods_lookup_cpu_irq(u8 client_id, unsigned int irq)
+{
+	u8  client_idx;
+	int ret = IRQ_NOT_FOUND;
+
+	LOG_ENT();
+
+	for (client_idx = 1; client_idx <= MODS_MAX_CLIENTS; client_idx++) {
+		struct dev_irq_map *t    = NULL;
+		struct dev_irq_map *next = NULL;
+
+		if (!mods_is_client_enabled(client_idx))
+			continue;
+
+		list_for_each_entry_safe(t,
+					 next,
+					 &mods_client_from_id(client_idx)->irq_list,
+					 list) {
+
+			if (t->apic_irq == irq) {
+				ret = (!client_id || client_id == client_idx)
+				      ? IRQ_FOUND : IRQ_NOT_FOUND;
+
+				/* Break out of the outer loop */
+				client_idx = MODS_MAX_CLIENTS;
+				break;
+			}
+		}
+	}
+
+	LOG_EXT();
+	return ret;
+}
+
+#ifdef CONFIG_PCI
+static int is_nvidia_gpu(struct pci_dev *dev)
+{
+	unsigned short class_code, vendor_id;
+
+	pci_read_config_word(dev, PCI_CLASS_DEVICE, &class_code);
+	pci_read_config_word(dev, PCI_VENDOR_ID, &vendor_id);
+	if (((class_code == PCI_CLASS_DISPLAY_VGA) ||
+	    (class_code == PCI_CLASS_DISPLAY_3D)) && (vendor_id == 0x10DE)) {
+		return true;
+	}
+	return false;
+}
+
+static void setup_mask_info(struct dev_irq_map *newmap,
+			    struct MODS_REGISTER_IRQ_4 *p,
+			    struct pci_dev *dev)
+{
+	/* account for legacy adapters */
+	u8 __iomem *bar = newmap->dev_irq_aperture;
+	u32         ii  = 0;
+
+	if ((p->mask_info_cnt == 0) && is_nvidia_gpu(dev)) {
+		struct irq_mask_info *const m = &newmap->mask_info[0];
+
+		newmap->mask_info_cnt = 1;
+		m->dev_irq_mask_reg    = (void __iomem *)(bar + 0x140);
+		m->dev_irq_disable_reg = (void __iomem *)(bar + 0x140);
+		m->dev_irq_state       = (void __iomem *)(bar + 0x100);
+		m->irq_and_mask        = 0;
+		m->irq_or_mask         = 0;
+		return;
+	}
+
+	/* setup for new adapters */
+	newmap->mask_info_cnt = p->mask_info_cnt;
+	for (ii = 0; ii < p->mask_info_cnt; ii++) {
+		struct irq_mask_info         *const m = &newmap->mask_info[ii];
+		const struct mods_mask_info2 *const in_m = &p->mask_info[ii];
+
+		const u32 pend_offs = in_m->irq_pending_offset;
+		const u32 stat_offs = in_m->irq_enabled_offset;
+		const u32 dis_offs  = in_m->irq_disable_offset;
+
+		m->dev_irq_state       = (void __iomem *)(bar + pend_offs);
+		m->dev_irq_mask_reg    = (void __iomem *)(bar + stat_offs);
+		m->dev_irq_disable_reg = (void __iomem *)(bar + dis_offs);
+		m->irq_and_mask        = in_m->and_mask;
+		m->irq_or_mask         = in_m->or_mask;
+		m->mask_type           = in_m->mask_type;
+	}
+}
+#endif
+
+static int add_irq_map(struct mods_client         *client,
+		       struct pci_dev             *dev,
+		       struct MODS_REGISTER_IRQ_4 *p,
+		       u32                         irq,
+		       u32                         entry)
+{
+	struct dev_irq_map *newmap     = NULL;
+	u64                 valid_mask = IRQF_TRIGGER_NONE;
+	u64                 irq_flags  = MODS_IRQ_FLAG_FROM_FLAGS(p->irq_flags);
+	u32                 irq_type   = MODS_IRQ_TYPE_FROM_FLAGS(p->irq_flags);
+
+	LOG_ENT();
+
+	/* Get the flags based on the interrupt type*/
+	switch (irq_type) {
+	case MODS_IRQ_TYPE_INT:
+		irq_flags = IRQF_SHARED;
+		break;
+
+	case MODS_IRQ_TYPE_CPU:
+		valid_mask = IRQF_TRIGGER_RISING |
+			     IRQF_TRIGGER_FALLING |
+			     IRQF_SHARED;
+
+		/* Either use a valid flag bit or no flags */
+		if (irq_flags & ~valid_mask) {
+			cl_error("invalid device interrupt flag %llx\n",
+				 (long long)irq_flags);
+			return -EINVAL;
+		}
+		break;
+
+	default:
+		irq_flags = IRQF_TRIGGER_NONE;
+		break;
+	}
+
+	/* Allocate memory for the new entry */
+	newmap = kzalloc(sizeof(*newmap), GFP_KERNEL | __GFP_NORETRY);
+	if (unlikely(!newmap)) {
+		LOG_EXT();
+		return -ENOMEM;
+	}
+	atomic_inc(&client->num_allocs);
+
+	/* Fill out the new entry */
+	newmap->apic_irq         = irq;
+	newmap->dev              = dev;
+	newmap->client_id        = client->client_id;
+	newmap->dev_irq_aperture = NULL;
+	newmap->mask_info_cnt    = 0;
+	newmap->type             = irq_type;
+	newmap->entry            = entry;
+
+	/* Enable IRQ for this device in the kernel */
+	if (request_irq(irq,
+			&mods_irq_handle,
+			irq_flags,
+			"nvidia mods",
+			newmap)) {
+		cl_error("unable to enable IRQ 0x%x with flags %llx\n",
+			 irq,
+			 (long long)irq_flags);
+		kfree(newmap);
+		atomic_dec(&client->num_allocs);
+		LOG_EXT();
+		return -EPERM;
+	}
+
+	/* Add the new entry to the list of all registered interrupts */
+	list_add(&newmap->list, &client->irq_list);
+
+#ifdef CONFIG_PCI
+	/* Map BAR0 to be able to disable interrupts */
+	if ((irq_type == MODS_IRQ_TYPE_INT) &&
+	    (p->aperture_addr != 0) &&
+	    (p->aperture_size != 0)) {
+		u8 __iomem *bar = ioremap(p->aperture_addr, p->aperture_size);
+
+		if (!bar) {
+			cl_debug(DEBUG_ISR,
+				 "failed to remap aperture: 0x%llx size=0x%x\n",
+				 p->aperture_addr,
+				 p->aperture_size);
+			LOG_EXT();
+			return -EPERM;
+		}
+
+		newmap->dev_irq_aperture = bar;
+		setup_mask_info(newmap, p, dev);
+	}
+
+	if (dev)
+		pci_dev_get(dev);
+#endif
+
+	/* Print out successful registration string */
+	if (irq_type == MODS_IRQ_TYPE_CPU)
+		cl_debug(DEBUG_ISR,
+			 "registered CPU IRQ 0x%x with flags %llx\n",
+			 irq,
+			 (long long)irq_flags);
+#ifdef CONFIG_PCI
+	else if ((irq_type == MODS_IRQ_TYPE_INT) ||
+		 (irq_type == MODS_IRQ_TYPE_MSI) ||
+		 (irq_type == MODS_IRQ_TYPE_MSIX)) {
+		int dom = dev ? pci_domain_nr(dev->bus) : 0;
+
+		if (dom < 0)
+			dom = 0;
+
+		cl_debug(DEBUG_ISR,
+			 "dev %04x:%02x:%02x.%x registered %s IRQ 0x%x\n",
+			 dom,
+			 dev ? dev->bus->number : 0U,
+			 dev ? PCI_SLOT(dev->devfn) : 0U,
+			 dev ? PCI_FUNC(dev->devfn) : 0U,
+			 mods_irq_type_name(irq_type),
+			 irq);
+	}
+#endif
+#ifdef CONFIG_PCI_MSI
+	else if (irq_type == MODS_IRQ_TYPE_MSI) {
+		u16 control;
+		u16 data;
+		int cap_pos = pci_find_capability(dev, PCI_CAP_ID_MSI);
+
+		pci_read_config_word(dev, MSI_CONTROL_REG(cap_pos), &control);
+		if (IS_64BIT_ADDRESS(control))
+			pci_read_config_word(dev,
+						 MSI_DATA_REG(cap_pos, 1),
+						 &data);
+		else
+			pci_read_config_word(dev,
+						 MSI_DATA_REG(cap_pos, 0),
+						 &data);
+		cl_debug(DEBUG_ISR,
+			 "dev %04x:%02x:%02x.%x registered MSI IRQ 0x%x data:0x%02x\n",
+			 pci_domain_nr(dev->bus),
+			 dev->bus->number,
+			 PCI_SLOT(dev->devfn),
+			 PCI_FUNC(dev->devfn),
+			 irq,
+			 data);
+	} else if (irq_type == MODS_IRQ_TYPE_MSIX) {
+		cl_debug(DEBUG_ISR,
+			 "dev %04x:%02x:%02x.%x registered MSI-X IRQ 0x%x\n",
+			 pci_domain_nr(dev->bus),
+			 dev->bus->number,
+			 PCI_SLOT(dev->devfn),
+			 PCI_FUNC(dev->devfn),
+			 irq);
+	}
+#endif
+
+	LOG_EXT();
+	return OK;
+}
+
+static void mods_free_map(struct mods_client *client,
+			  struct dev_irq_map *del)
+{
+	unsigned long       flags  = 0;
+
+	LOG_ENT();
+
+	WARN_ON(client->client_id != del->client_id);
+
+	/* Disable interrupts on the device */
+	spin_lock_irqsave(&client->irq_lock, flags);
+	mods_disable_interrupts(del);
+	spin_unlock_irqrestore(&client->irq_lock, flags);
+
+	/* Unhook interrupts in the kernel */
+	free_irq(del->apic_irq, del);
+
+	/* Unmap aperture used for masking irqs */
+	if (del->dev_irq_aperture)
+		iounmap(del->dev_irq_aperture);
+
+#ifdef CONFIG_PCI
+	pci_dev_put(del->dev);
+#endif
+
+	/* Free memory */
+	kfree(del);
+	atomic_dec(&client->num_allocs);
+
+	LOG_EXT();
+}
+
+void mods_init_irq(void)
+{
+	LOG_ENT();
+
+	mutex_init(&irq_mtx);
+
+	LOG_EXT();
+}
+
+POLL_TYPE mods_irq_event_check(u8 client_id)
+{
+	struct irq_q_info *q;
+
+	if (!mods_is_client_enabled(client_id))
+		return POLLERR; /* client has quit */
+
+	q = &mods_client_from_id(client_id)->irq_queue;
+
+	if (q->head != q->tail)
+		return POLLIN; /* irq generated */
+
+	return 0;
+}
+
+static int mods_free_irqs(struct mods_client *client,
+			  struct pci_dev     *dev)
+{
+#ifdef CONFIG_PCI
+	struct dev_irq_map  *del    = NULL;
+	struct dev_irq_map  *next;
+	struct en_dev_entry *dpriv;
+	unsigned int         irq_type;
+
+	LOG_ENT();
+
+	if (unlikely(mutex_lock_interruptible(&irq_mtx))) {
+		LOG_EXT();
+		return -EINTR;
+	}
+
+	dpriv = pci_get_drvdata(dev);
+
+	if (!dpriv) {
+		mutex_unlock(&irq_mtx);
+		LOG_EXT();
+		return OK;
+	}
+
+	if (dpriv->client_id != client->client_id) {
+		cl_error(
+			"invalid client for dev %04x:%02x:%02x.%x, expected %u\n",
+			pci_domain_nr(dev->bus),
+			dev->bus->number,
+			PCI_SLOT(dev->devfn),
+			PCI_FUNC(dev->devfn),
+			dpriv->client_id);
+		mutex_unlock(&irq_mtx);
+		LOG_EXT();
+		return -EINVAL;
+	}
+
+	cl_debug(DEBUG_ISR_DETAILED,
+		 "free IRQ for dev %04x:%02x:%02x.%x irq_flags=0x%x nvecs=%d\n",
+		 pci_domain_nr(dev->bus),
+		 dev->bus->number,
+		 PCI_SLOT(dev->devfn),
+		 PCI_FUNC(dev->devfn),
+		 dpriv->irq_flags,
+		 dpriv->nvecs);
+
+	/* Delete device interrupts from the list */
+	list_for_each_entry_safe(del, next, &client->irq_list, list) {
+		if (dev == del->dev) {
+			u8 type = del->type;
+
+			list_del(&del->list);
+			cl_debug(DEBUG_ISR,
+				 "unregistered %s IRQ 0x%x dev %04x:%02x:%02x.%x\n",
+				 mods_irq_type_name(type),
+				 del->apic_irq,
+				 pci_domain_nr(dev->bus),
+				 dev->bus->number,
+				 PCI_SLOT(dev->devfn),
+				 PCI_FUNC(dev->devfn));
+			mods_free_map(client, del);
+
+			WARN_ON(MODS_IRQ_TYPE_FROM_FLAGS(dpriv->irq_flags)
+				!= type);
+			if (type != MODS_IRQ_TYPE_MSIX)
+				break;
+		}
+	}
+
+	cl_debug(DEBUG_ISR_DETAILED, "before disable\n");
+#ifdef CONFIG_PCI_MSI
+	irq_type = MODS_IRQ_TYPE_FROM_FLAGS(dpriv->irq_flags);
+
+	if (irq_type == MODS_IRQ_TYPE_MSIX) {
+		pci_disable_msix(dev);
+		kfree(dpriv->msix_entries);
+		if (dpriv->msix_entries)
+			atomic_dec(&client->num_allocs);
+		dpriv->msix_entries = NULL;
+	} else if (irq_type == MODS_IRQ_TYPE_MSI) {
+		pci_disable_msi(dev);
+	}
+#endif
+
+	dpriv->nvecs = 0;
+	cl_debug(DEBUG_ISR_DETAILED, "irqs freed\n");
+#endif
+
+	mutex_unlock(&irq_mtx);
+	LOG_EXT();
+	return 0;
+}
+
+void mods_free_client_interrupts(struct mods_client *client)
+{
+	struct en_dev_entry *dpriv = client->enabled_devices;
+
+	LOG_ENT();
+
+	/* Release all interrupts */
+	while (dpriv) {
+		mods_free_irqs(client, dpriv->dev);
+		dpriv = dpriv->next;
+	}
+
+	LOG_EXT();
+}
+
+#ifdef CONFIG_PCI
+static int mods_allocate_irqs(struct mods_client *client,
+			      struct pci_dev     *dev,
+			      u32                 nvecs,
+			      u32                 flags)
+{
+	struct en_dev_entry *dpriv;
+	unsigned int         irq_type = MODS_IRQ_TYPE_FROM_FLAGS(flags);
+	int                  err      = OK;
+
+	LOG_ENT();
+
+	cl_debug(DEBUG_ISR_DETAILED,
+		 "allocate %u IRQs on dev %04x:%02x:%02x.%x, flags=0x%x\n",
+		 nvecs,
+		 pci_domain_nr(dev->bus),
+		 dev->bus->number,
+		 PCI_SLOT(dev->devfn),
+		 PCI_FUNC(dev->devfn),
+		 flags);
+
+	/* Determine if the device supports requested interrupt type */
+	if (irq_type == MODS_IRQ_TYPE_MSI) {
+#ifdef CONFIG_PCI_MSI
+		if (pci_find_capability(dev, PCI_CAP_ID_MSI) == 0) {
+			cl_error("dev %04x:%02x:%02x.%x does not support MSI\n",
+				 pci_domain_nr(dev->bus),
+				 dev->bus->number,
+				 PCI_SLOT(dev->devfn),
+				 PCI_FUNC(dev->devfn));
+			LOG_EXT();
+			return -ENOENT;
+		}
+#else
+		cl_error("the kernel does not support MSI\n");
+		LOG_EXT();
+		return -EINVAL;
+#endif
+	} else if (irq_type == MODS_IRQ_TYPE_MSIX) {
+#ifdef CONFIG_PCI_MSI
+		if (pci_find_capability(dev, PCI_CAP_ID_MSIX) == 0) {
+			cl_error(
+				"dev %04x:%02x:%02x.%x does not support MSI-X\n",
+				pci_domain_nr(dev->bus),
+				dev->bus->number,
+				PCI_SLOT(dev->devfn),
+				PCI_FUNC(dev->devfn));
+			LOG_EXT();
+			return -ENOENT;
+		}
+#else
+		cl_error("the kernel does not support MSI-X\n");
+		LOG_EXT();
+		return -EINVAL;
+#endif
+	}
+
+	/* Enable device on the PCI bus */
+	err = mods_enable_device(client, dev, &dpriv);
+	if (err) {
+		LOG_EXT();
+		return err;
+	}
+
+	if (irq_type == MODS_IRQ_TYPE_INT) {
+		/* use legacy irq */
+		if (nvecs != 1) {
+			cl_error(
+				"INTA: only 1 INTA vector supported, requested %u\n",
+				nvecs);
+			LOG_EXT();
+			return -EINVAL;
+		}
+		dpriv->nvecs = 1;
+	}
+	/* Enable MSI */
+#ifdef CONFIG_PCI_MSI
+	else if (irq_type == MODS_IRQ_TYPE_MSI) {
+		if (nvecs != 1) {
+			cl_error(
+				"MSI: only 1 MSI vector supported, requested %u\n",
+				nvecs);
+			LOG_EXT();
+			return -EINVAL;
+		}
+		err = pci_enable_msi(dev);
+		if (err) {
+			cl_error(
+				"unable to enable MSI on dev %04x:%02x:%02x.%x\n",
+				pci_domain_nr(dev->bus),
+				dev->bus->number,
+				PCI_SLOT(dev->devfn),
+				PCI_FUNC(dev->devfn));
+			LOG_EXT();
+			return err;
+		}
+		dpriv->nvecs = 1;
+	} else if (irq_type == MODS_IRQ_TYPE_MSIX) {
+		struct msix_entry *entries;
+		int i = 0, cnt = 1;
+
+		entries = kcalloc(nvecs, sizeof(struct msix_entry),
+				GFP_KERNEL | __GFP_NORETRY);
+
+		if (!entries) {
+			cl_error("could not allocate %d MSI-X entries\n",
+				 nvecs);
+			LOG_EXT();
+			return -ENOMEM;
+		}
+		atomic_inc(&client->num_allocs);
+
+		for (i = 0; i < nvecs; i++)
+			entries[i].entry = (uint16_t)i;
+
+#ifdef MODS_HAS_MSIX_RANGE
+		cnt = pci_enable_msix_range(dev, entries, nvecs, nvecs);
+
+		if (cnt < 0) {
+			/* returns number of interrupts allocated
+			 * < 0 indicates a failure.
+			 */
+			cl_error(
+				"could not allocate the requested number of MSI-X vectors=%d return=%d!\n",
+				nvecs, cnt);
+			kfree(entries);
+			atomic_dec(&client->num_allocs);
+			LOG_EXT();
+			return cnt;
+		}
+#else
+		cnt = pci_enable_msix(dev, entries, nvecs);
+
+		if (cnt) {
+			/*  A return of < 0 indicates a failure.
+			 *  A return of > 0 indicates that driver request is
+			 *  exceeding the number of irqs or MSI-X
+			 *  vectors available
+			 */
+			cl_error(
+				"could not allocate the requested number of MSI-X vectors=%d return=%d!\n",
+				nvecs, cnt);
+			kfree(entries);
+			atomic_dec(&client->num_allocs);
+			LOG_EXT();
+			if (cnt > 0)
+				cnt = -ENOSPC;
+			return cnt;
+		}
+#endif
+
+		cl_debug(DEBUG_ISR,
+			 "allocated %d irq's of type %s(%d)\n",
+			 nvecs,
+			 mods_irq_type_name(irq_type),
+			 irq_type);
+
+		for (i = 0; i < nvecs; i++)
+			cl_debug(DEBUG_ISR,
+				 "vec %d %x\n",
+				 entries[i].entry,
+				 entries[i].vector);
+
+		dpriv->nvecs = nvecs;
+		dpriv->msix_entries = entries;
+	}
+#endif
+	else {
+		cl_error("unsupported irq_type %u dev %04x:%02x:%02x.%x\n",
+			 irq_type,
+			 pci_domain_nr(dev->bus),
+			 dev->bus->number,
+			 PCI_SLOT(dev->devfn),
+			 PCI_FUNC(dev->devfn));
+		LOG_EXT();
+		return -EINVAL;
+	}
+
+	WARN_ON(dpriv->client_id != client->client_id);
+	dpriv->irq_flags = flags;
+	LOG_EXT();
+	return OK;
+}
+
+static int mods_register_pci_irq(struct mods_client         *client,
+				 struct MODS_REGISTER_IRQ_4 *p)
+{
+	int                  err = OK;
+	unsigned int         irq_type = MODS_IRQ_TYPE_FROM_FLAGS(p->irq_flags);
+	struct pci_dev      *dev;
+	struct en_dev_entry *dpriv;
+	int                  i;
+
+	LOG_ENT();
+
+	if (unlikely(!p->irq_count)) {
+		cl_error("no irq's requested\n");
+		LOG_EXT();
+		return -EINVAL;
+	}
+
+	/* Get the PCI device structure for the specified device from kernel */
+	err = mods_find_pci_dev(client, &p->dev, &dev);
+	if (unlikely(err)) {
+		if (err == -ENODEV)
+			cl_error("dev %04x:%02x:%02x.%x not found\n",
+				 p->dev.domain,
+				 p->dev.bus,
+				 p->dev.device,
+				 p->dev.function);
+		LOG_EXT();
+		return err;
+	}
+
+	if (unlikely(mutex_lock_interruptible(&irq_mtx))) {
+		pci_dev_put(dev);
+		LOG_EXT();
+		return -EINTR;
+	}
+
+	dpriv = pci_get_drvdata(dev);
+	if (dpriv) {
+		if (dpriv->client_id != client->client_id) {
+			cl_error(
+				"dev %04x:%02x:%02x.%x already owned by client %u\n",
+				p->dev.domain,
+				p->dev.bus,
+				p->dev.device,
+				p->dev.function,
+				dpriv->client_id);
+			mutex_unlock(&irq_mtx);
+			pci_dev_put(dev);
+			LOG_EXT();
+			return -EBUSY;
+		}
+		if (dpriv->nvecs) {
+			cl_error(
+				"interrupt for dev %04x:%02x:%02x.%x already registered\n",
+				p->dev.domain,
+				p->dev.bus,
+				p->dev.device,
+				p->dev.function);
+			mutex_unlock(&irq_mtx);
+			pci_dev_put(dev);
+			LOG_EXT();
+			return -EINVAL;
+		}
+	}
+
+	err = mods_allocate_irqs(client, dev, p->irq_count,
+				 p->irq_flags);
+	if (err) {
+		cl_error("could not allocate irqs for irq_type %d\n",
+			 irq_type);
+		mutex_unlock(&irq_mtx);
+		pci_dev_put(dev);
+		LOG_EXT();
+		return err;
+	}
+
+	dpriv = pci_get_drvdata(dev);
+
+	for (i = 0; i < p->irq_count; i++) {
+		u32 irq = ((irq_type == MODS_IRQ_TYPE_INT) ||
+				(irq_type == MODS_IRQ_TYPE_MSI)) ? dev->irq :
+			   dpriv->msix_entries[i].vector;
+
+		err = add_irq_map(client, dev, p, irq, i);
+		if (unlikely(err)) {
+#ifdef CONFIG_PCI_MSI
+			if (irq_type == MODS_IRQ_TYPE_MSI)
+				pci_disable_msi(dev);
+			else if (irq_type == MODS_IRQ_TYPE_MSIX)
+				pci_disable_msix(dev);
+#endif
+			break;
+		}
+	}
+
+	mutex_unlock(&irq_mtx);
+	pci_dev_put(dev);
+	LOG_EXT();
+	return err;
+}
+#endif /* CONFIG_PCI */
+
+static int mods_register_cpu_irq(struct mods_client         *client,
+				 struct MODS_REGISTER_IRQ_4 *p)
+{
+	u32 irq = p->dev.bus;
+	int err;
+
+	LOG_ENT();
+
+	if (unlikely(mutex_lock_interruptible(&irq_mtx))) {
+		LOG_EXT();
+		return -EINTR;
+	}
+
+	/* Determine if the interrupt is already hooked */
+	if (mods_lookup_cpu_irq(0, irq) == IRQ_FOUND) {
+		cl_error("CPU IRQ 0x%x has already been registered\n", irq);
+		mutex_unlock(&irq_mtx);
+		LOG_EXT();
+		return -EINVAL;
+	}
+
+	/* Register interrupt */
+	err = add_irq_map(client, NULL, p, irq, 0);
+
+	mutex_unlock(&irq_mtx);
+	LOG_EXT();
+	return err;
+}
+
+#ifdef CONFIG_PCI
+static int mods_unregister_pci_irq(struct mods_client         *client,
+				   struct MODS_REGISTER_IRQ_2 *p)
+{
+	struct pci_dev *dev;
+	int             err = OK;
+
+	LOG_ENT();
+
+	/* Get the PCI device structure for the specified device from kernel */
+	err = mods_find_pci_dev(client, &p->dev, &dev);
+	if (unlikely(err)) {
+		LOG_EXT();
+		return err;
+	}
+
+	err = mods_free_irqs(client, dev);
+
+	pci_dev_put(dev);
+	LOG_EXT();
+	return err;
+}
+#endif
+
+static int mods_unregister_cpu_irq(struct mods_client *client,
+				   struct MODS_REGISTER_IRQ_2 *p)
+{
+	struct dev_irq_map *del = NULL;
+	struct dev_irq_map *next;
+	unsigned int        irq;
+
+	LOG_ENT();
+
+	irq = p->dev.bus;
+
+	if (unlikely(mutex_lock_interruptible(&irq_mtx))) {
+		LOG_EXT();
+		return -EINTR;
+	}
+
+	/* Determine if the interrupt is already hooked by this client */
+	if (mods_lookup_cpu_irq(client->client_id, irq) == IRQ_NOT_FOUND) {
+		cl_error("IRQ 0x%x not hooked, can't unhook\n", irq);
+		mutex_unlock(&irq_mtx);
+		LOG_EXT();
+		return -EINVAL;
+	}
+
+	/* Delete device interrupt from the list */
+	list_for_each_entry_safe(del, next, &client->irq_list, list) {
+		if ((irq == del->apic_irq) && (del->dev == NULL)) {
+			if (del->type != p->type) {
+				cl_error("wrong IRQ type passed\n");
+				mutex_unlock(&irq_mtx);
+				LOG_EXT();
+				return -EINVAL;
+			}
+			list_del(&del->list);
+			cl_debug(DEBUG_ISR,
+				 "unregistered CPU IRQ 0x%x\n",
+				 irq);
+			mods_free_map(client, del);
+			break;
+		}
+	}
+
+	mutex_unlock(&irq_mtx);
+	LOG_EXT();
+	return OK;
+}
+
+/*************************
+ * ESCAPE CALL FUNCTIONS *
+ *************************/
+
+int esc_mods_register_irq_4(struct mods_client         *client,
+			    struct MODS_REGISTER_IRQ_4 *p)
+{
+	u32 irq_type = MODS_IRQ_TYPE_FROM_FLAGS(p->irq_flags);
+
+	if (irq_type == MODS_IRQ_TYPE_CPU)
+		return mods_register_cpu_irq(client, p);
+#ifdef CONFIG_PCI
+	return mods_register_pci_irq(client, p);
+#else
+	cl_error("PCI not available\n");
+	return -EINVAL;
+#endif
+}
+
+int esc_mods_register_irq_3(struct mods_client         *client,
+			    struct MODS_REGISTER_IRQ_3 *p)
+{
+	struct MODS_REGISTER_IRQ_4 irq_data = { {0} };
+	u32 ii = 0;
+
+	irq_data.dev = p->dev;
+	irq_data.aperture_addr = p->aperture_addr;
+	irq_data.aperture_size = p->aperture_size;
+	irq_data.mask_info_cnt = p->mask_info_cnt;
+	for (ii = 0; ii < p->mask_info_cnt; ii++) {
+		irq_data.mask_info[ii].mask_type = p->mask_info[ii].mask_type;
+		irq_data.mask_info[ii].irq_pending_offset =
+			p->mask_info[ii].irq_pending_offset;
+		irq_data.mask_info[ii].irq_enabled_offset =
+			p->mask_info[ii].irq_enabled_offset;
+		irq_data.mask_info[ii].irq_enable_offset =
+			p->mask_info[ii].irq_enable_offset;
+		irq_data.mask_info[ii].irq_disable_offset =
+			p->mask_info[ii].irq_disable_offset;
+		irq_data.mask_info[ii].and_mask =
+			p->mask_info[ii].and_mask;
+		irq_data.mask_info[ii].or_mask =
+			p->mask_info[ii].or_mask;
+	}
+	irq_data.irq_count = 1;
+	irq_data.irq_flags = p->irq_type;
+
+	return esc_mods_register_irq_4(client, &irq_data);
+}
+
+int esc_mods_register_irq_2(struct mods_client         *client,
+			    struct MODS_REGISTER_IRQ_2 *p)
+{
+	struct MODS_REGISTER_IRQ_4 irq_data = { {0} };
+#ifdef CONFIG_PCI
+	struct pci_dev *dev;
+	int             err;
+	resource_size_t size;
+#endif
+
+	irq_data.dev = p->dev;
+	irq_data.irq_count = 1;
+	irq_data.irq_flags = p->type;
+
+#ifdef CONFIG_PCI
+	/* Get the PCI device structure */
+	err = mods_find_pci_dev(client, &p->dev, &dev);
+	if (unlikely(err))
+		return err;
+
+	irq_data.aperture_addr = pci_resource_start(dev, 0);
+
+	size = pci_resource_len(dev, 0);
+
+	pci_dev_put(dev);
+
+	if (size > ~0U) {
+		cl_error("BAR0 size of device %04x:%02x:%02x.%x is %llx and exceeds 32 bits\n",
+			 p->dev.domain,
+			 p->dev.bus,
+			 p->dev.device,
+			 p->dev.function,
+			 (unsigned long long)size);
+		return -EINVAL;
+	}
+
+	irq_data.aperture_size = (unsigned int)size;
+#endif
+
+	return esc_mods_register_irq_4(client, &irq_data);
+}
+
+int esc_mods_register_irq(struct mods_client       *client,
+			  struct MODS_REGISTER_IRQ *p)
+{
+	struct MODS_REGISTER_IRQ_2 register_irq = { {0} };
+
+	register_irq.dev.domain		= 0;
+	register_irq.dev.bus		= p->dev.bus;
+	register_irq.dev.device		= p->dev.device;
+	register_irq.dev.function	= p->dev.function;
+	register_irq.type		= p->type;
+
+	return esc_mods_register_irq_2(client, &register_irq);
+}
+
+int esc_mods_unregister_irq_2(struct mods_client         *client,
+			      struct MODS_REGISTER_IRQ_2 *p)
+{
+	if (p->type == MODS_IRQ_TYPE_CPU)
+		return mods_unregister_cpu_irq(client, p);
+#ifdef CONFIG_PCI
+	return mods_unregister_pci_irq(client, p);
+#else
+	return -EINVAL;
+#endif
+}
+
+int esc_mods_unregister_irq(struct mods_client       *client,
+			    struct MODS_REGISTER_IRQ *p)
+{
+	struct MODS_REGISTER_IRQ_2 register_irq = { {0} };
+
+	register_irq.dev.domain   = 0;
+	register_irq.dev.bus      = p->dev.bus;
+	register_irq.dev.device   = p->dev.device;
+	register_irq.dev.function = p->dev.function;
+	register_irq.type         = p->type;
+
+	return esc_mods_unregister_irq_2(client, &register_irq);
+}
+
+int esc_mods_query_irq_3(struct mods_client      *client,
+			 struct MODS_QUERY_IRQ_3 *p)
+{
+	struct irq_q_info  *q        = NULL;
+	int                 err      = OK;
+	unsigned int        i        = 0;
+	unsigned long       flags    = 0;
+	unsigned int        cur_time = get_cur_time();
+
+	LOG_ENT();
+
+	/* Clear return array */
+	memset(p->irq_list, 0xFF, sizeof(p->irq_list));
+
+	/* Lock IRQ queue */
+	spin_lock_irqsave(&client->irq_lock, flags);
+
+	/* Fill in return array with IRQ information */
+	q = &client->irq_queue;
+	for (i = 0;
+		 (q->head != q->tail) && (i < MODS_MAX_IRQS);
+		 q->head++, i++) {
+
+		unsigned int    index = q->head & (MODS_MAX_IRQS - 1);
+		struct pci_dev *dev   = q->data[index].dev;
+
+		if (dev) {
+			const int dom = pci_domain_nr(dev->bus);
+
+			if (dom < 0 || dom > 0xFFFF) {
+				cl_error("unsupported domain %d\n", dom);
+				err = -EINVAL;
+				goto error;
+			}
+			p->irq_list[i].dev.domain = dom;
+			p->irq_list[i].dev.bus = dev->bus->number;
+			p->irq_list[i].dev.device = PCI_SLOT(dev->devfn);
+			p->irq_list[i].dev.function = PCI_FUNC(dev->devfn);
+		} else {
+			if (q->data[index].irq > 0xFFFFU) {
+				cl_error("unsupported IRQ %u\n",
+					 q->data[index].irq);
+				err = -EINVAL;
+				goto error;
+			}
+			p->irq_list[i].dev.domain = 0;
+			p->irq_list[i].dev.bus = q->data[index].irq;
+			p->irq_list[i].dev.device = 0xFFU;
+			p->irq_list[i].dev.function = 0xFFU;
+		}
+		p->irq_list[i].irq_index = q->data[index].irq_index;
+		p->irq_list[i].delay = cur_time - q->data[index].time;
+
+		/* Print info about IRQ status returned */
+		if (dev) {
+			cl_debug(DEBUG_ISR_DETAILED,
+				 "retrieved IRQ index=%d dev %04x:%02x:%02x.%x, time=%uus, delay=%uus\n",
+				 p->irq_list[i].irq_index,
+				 p->irq_list[i].dev.domain,
+				 p->irq_list[i].dev.bus,
+				 p->irq_list[i].dev.device,
+				 p->irq_list[i].dev.function,
+				 q->data[index].time,
+				 p->irq_list[i].delay);
+		} else {
+			cl_debug(DEBUG_ISR_DETAILED,
+				 "retrieved IRQ 0x%x, time=%uus, delay=%uus\n",
+				 (unsigned int)p->irq_list[i].dev.bus,
+				 q->data[index].time,
+				 p->irq_list[i].delay);
+		}
+	}
+
+	/* Indicate if there are more IRQs pending */
+	if (q->head != q->tail)
+		p->more = 1;
+
+error:
+	/* Unlock IRQ queue */
+	spin_unlock_irqrestore(&client->irq_lock, flags);
+
+	LOG_EXT();
+	return err;
+}
+
+int esc_mods_query_irq_2(struct mods_client      *client,
+			 struct MODS_QUERY_IRQ_2 *p)
+{
+	int retval, i;
+	struct MODS_QUERY_IRQ_3 query_irq = { { { { 0 } } } };
+
+	retval = esc_mods_query_irq_3(client, &query_irq);
+	if (retval)
+		return retval;
+
+	for (i = 0; i < MODS_MAX_IRQS; i++) {
+		p->irq_list[i].dev   = query_irq.irq_list[i].dev;
+		p->irq_list[i].delay = query_irq.irq_list[i].delay;
+	}
+	p->more = query_irq.more;
+	return OK;
+}
+
+int esc_mods_query_irq(struct mods_client    *client,
+		       struct MODS_QUERY_IRQ *p)
+{
+	int retval, i;
+	struct MODS_QUERY_IRQ_3 query_irq = { { { { 0 } } } };
+
+	retval = esc_mods_query_irq_3(client, &query_irq);
+	if (retval)
+		return retval;
+
+	for (i = 0; i < MODS_MAX_IRQS; i++) {
+		struct mods_irq_3 *entry = &query_irq.irq_list[i];
+
+		if (entry->dev.device > 0xFFU) {
+			cl_error("unsupported device %02x\n",
+				 entry->dev.device);
+			return -EINVAL;
+		}
+		if (entry->dev.function > 0xFFU) {
+			cl_error("unsupported function %02x\n",
+				 entry->dev.function);
+			return -EINVAL;
+		}
+		p->irq_list[i].dev.bus      = entry->dev.bus;
+		p->irq_list[i].dev.device   = entry->dev.device;
+		p->irq_list[i].dev.function = entry->dev.function;
+		p->irq_list[i].delay	    = entry->delay;
+	}
+	p->more = query_irq.more;
+	return OK;
+}
+
+int esc_mods_irq_handled_2(struct mods_client         *client,
+			   struct MODS_REGISTER_IRQ_2 *p)
+{
+	unsigned long       flags = 0;
+	u32                 irq = p->dev.bus;
+	struct dev_irq_map *t = NULL;
+	struct dev_irq_map *next = NULL;
+	int                 err = -EINVAL;
+
+	if (p->type != MODS_IRQ_TYPE_CPU)
+		return -EINVAL;
+
+	LOG_ENT();
+
+	/* Print info */
+	cl_debug(DEBUG_ISR_DETAILED, "mark CPU IRQ 0x%x handled\n", irq);
+
+	/* Lock IRQ queue */
+	spin_lock_irqsave(&client->irq_lock, flags);
+
+	list_for_each_entry_safe(t, next, &client->irq_list, list) {
+		if (t->apic_irq == irq) {
+			if (t->type != p->type) {
+				cl_error(
+					"IRQ type doesn't match registered IRQ\n");
+			} else {
+				enable_irq(irq);
+				err = OK;
+			}
+			break;
+		}
+	}
+
+	/* Unlock IRQ queue */
+	spin_unlock_irqrestore(&client->irq_lock, flags);
+
+	LOG_EXT();
+	return err;
+}
+
+int esc_mods_irq_handled(struct mods_client       *client,
+			 struct MODS_REGISTER_IRQ *p)
+{
+	struct MODS_REGISTER_IRQ_2 register_irq = { {0} };
+
+	register_irq.dev.domain   = 0;
+	register_irq.dev.bus      = p->dev.bus;
+	register_irq.dev.device   = p->dev.device;
+	register_irq.dev.function = p->dev.function;
+	register_irq.type         = p->type;
+
+	return esc_mods_irq_handled_2(client, &register_irq);
+}
+
+#if defined(MODS_HAS_TEGRA) && defined(CONFIG_OF_IRQ) && defined(CONFIG_OF)
+int esc_mods_map_irq(struct mods_client  *client,
+		     struct MODS_DT_INFO *p)
+{
+	/* irq parameters */
+	struct of_phandle_args oirq;
+	/* platform device handle */
+	struct platform_device *pdev = NULL;
+	struct device_node *np;
+	int err = 0;
+	/* the physical irq */
+	int hwirq;
+
+	/* Make sure strings from userspace are null-terminated */
+	p->dt_name[sizeof(p->dt_name) - 1] = 0;
+	p->full_name[sizeof(p->full_name) - 1] = 0;
+
+	/* Search for the node by device tree name */
+	np = of_find_node_by_name(NULL, p->dt_name);
+
+	if (!np) {
+		cl_error("node %s is not valid\n", p->full_name);
+		err = -EINVAL;
+		goto error;
+	}
+
+	/* Can be multiple nodes that share the same dt name, */
+	/* make sure you get the correct node matched by the device's full */
+	/* name in device tree (i.e. watchdog@30c0000 as opposed */
+	/* to watchdog) */
+	while (of_node_cmp(np->full_name, p->full_name)) {
+		np = of_find_node_by_name(np, p->dt_name);
+		if (!np) {
+			cl_error("node %s is not valid\n", p->full_name);
+			err = -EINVAL;
+			goto error;
+		}
+	}
+
+	if (p->index > 0x7FFFFFFFU) {
+		cl_error("unsupported index %u\n", p->index);
+		err = -EINVAL;
+		goto error;
+	}
+	p->irq = irq_of_parse_and_map(np, p->index);
+	err = of_irq_parse_one(np, p->index, &oirq);
+	if (err) {
+		cl_error("could not parse IRQ\n");
+		goto error;
+	}
+	if ((int)oirq.args[1] < 0) {
+		cl_error("unsupported IRQ %d\n", (int)oirq.args[1]);
+		err = -EINVAL;
+		goto error;
+	}
+
+	hwirq = (int)oirq.args[1];
+
+	/* Get the platform device handle */
+	pdev = of_find_device_by_node(np);
+
+	if (of_node_cmp(p->dt_name, "watchdog") == 0) {
+		/* Enable and unmask interrupt for watchdog */
+		struct resource *res_src =
+			platform_get_resource(pdev, IORESOURCE_MEM, 0);
+		struct resource *res_tke =
+			platform_get_resource(pdev, IORESOURCE_MEM, 2);
+		void __iomem    *wdt_tke = NULL;
+		unsigned int     wdt_index = 0;
+
+		if (res_tke && res_src) {
+			wdt_tke = devm_ioremap(&pdev->dev, res_tke->start,
+					       resource_size(res_tke));
+			wdt_index = (unsigned int)
+				(((res_src->start >> 16) & 0xFU) - 0xCU);
+		}
+
+		if (wdt_tke) {
+			writel(TOP_TKE_TKEIE_WDT_MASK(wdt_index),
+			       wdt_tke + TOP_TKE_TKEIE(hwirq));
+		}
+	}
+
+error:
+	of_node_put(np);
+	/* enable the interrupt */
+	return err;
+}
+
+int esc_mods_map_irq_to_gpio(struct mods_client    *client,
+			     struct MODS_GPIO_INFO *p)
+{
+	//TODO: Make sure you are allocating gpio properly
+	struct device_node *np;
+	int gpio_handle;
+	int irq;
+	int err = 0;
+
+	/* Make sure strings from userspace are null-terminated */
+	p->dt_name[sizeof(p->dt_name) - 1] = 0;
+	p->full_name[sizeof(p->full_name) - 1] = 0;
+
+	np = of_find_node_by_name(NULL, p->dt_name);
+
+	if (!np) {
+		cl_error("node %s is not valid\n", p->full_name);
+		err = -EINVAL;
+		goto error;
+	}
+
+	while (of_node_cmp(np->full_name, p->full_name)) {
+		np = of_find_node_by_name(np, p->dt_name);
+		if (!np) {
+			cl_error("node %s is not valid\n", p->full_name);
+			err = -EINVAL;
+			goto error;
+		}
+	}
+
+	gpio_handle = of_get_named_gpio(np, p->name, 0);
+	if (!gpio_is_valid(gpio_handle)) {
+		cl_error("gpio %s is missing\n", p->name);
+		err = gpio_handle;
+		goto error;
+	}
+
+	err = gpio_direction_input(gpio_handle);
+	if (err < 0) {
+		cl_error("pex_rst_gpio input direction change failed\n");
+		goto error;
+	}
+
+	irq = gpio_to_irq(gpio_handle);
+	if (irq < 0) {
+		cl_error("Unable to get irq for pex_rst_gpio\n");
+		err = -EINVAL;
+		goto error;
+	}
+	p->irq = irq;
+
+error:
+	of_node_put(np);
+	return err;
+}
+#endif
diff --git a/drivers/misc/mods/mods_krnl.c b/drivers/misc/mods/mods_krnl.c
new file mode 100644
index 00000000..135d7545
--- /dev/null
+++ b/drivers/misc/mods/mods_krnl.c
@@ -0,0 +1,2887 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * This file is part of NVIDIA MODS kernel driver.
+ *
+ * Copyright (c) 2008-2023, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * NVIDIA MODS kernel driver is free software: you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * NVIDIA MODS kernel driver is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NVIDIA MODS kernel driver.
+ * If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "mods_internal.h"
+
+#include <linux/fs.h>
+#include <linux/init.h>
+#include <linux/miscdevice.h>
+#include <linux/module.h>
+#include <linux/pagemap.h>
+#include <linux/poll.h>
+#include <linux/random.h>
+#include <linux/sched.h>
+#include <linux/uaccess.h>
+#ifdef MODS_HAS_CONSOLE_LOCK
+#   include <linux/console.h>
+#   include <linux/kd.h>
+#   include <linux/tty.h>
+#   include <linux/console_struct.h>
+#   include <linux/vt_kern.h>
+#endif
+#ifdef CONFIG_X86
+#   include <linux/screen_info.h>
+#   include <asm/msr.h>
+#endif
+
+#ifndef IS_BUILTIN
+#   define IS_BUILTIN(c) 0
+#endif
+
+/***********************************************************************
+ * mods_krnl_* functions, driver interfaces called by the Linux kernel *
+ ***********************************************************************/
+static int mods_krnl_open(struct inode *, struct file *);
+static int mods_krnl_close(struct inode *, struct file *);
+static POLL_TYPE mods_krnl_poll(struct file *, poll_table *);
+static int mods_krnl_mmap(struct file *, struct vm_area_struct *);
+static long mods_krnl_ioctl(struct file *, unsigned int, unsigned long);
+
+/* character driver entry points */
+static const struct file_operations mods_fops = {
+	.owner          = THIS_MODULE,
+	.open           = mods_krnl_open,
+	.release        = mods_krnl_close,
+	.poll           = mods_krnl_poll,
+	.mmap           = mods_krnl_mmap,
+	.unlocked_ioctl = mods_krnl_ioctl,
+#if defined(HAVE_COMPAT_IOCTL)
+	.compat_ioctl   = mods_krnl_ioctl,
+#endif
+};
+
+#define DEVICE_NAME "mods"
+
+static struct miscdevice mods_dev = {
+	.minor = MISC_DYNAMIC_MINOR,
+	.name  = DEVICE_NAME,
+	.fops  = &mods_fops
+};
+
+#if defined(CONFIG_PCI)
+static pci_ers_result_t mods_pci_error_detected(struct pci_dev *,
+						pci_channel_state_t);
+static pci_ers_result_t mods_pci_mmio_enabled(struct pci_dev *);
+static void mods_pci_resume(struct pci_dev *);
+
+static struct pci_error_handlers mods_pci_error_handlers = {
+	.error_detected	= mods_pci_error_detected,
+	.mmio_enabled	= mods_pci_mmio_enabled,
+	.resume		= mods_pci_resume,
+};
+
+static const struct pci_device_id mods_pci_table[] = {
+	{
+		.vendor		= PCI_VENDOR_ID_NVIDIA,
+		.device		= PCI_ANY_ID,
+		.subvendor	= PCI_ANY_ID,
+		.subdevice	= PCI_ANY_ID,
+		.class		= (PCI_CLASS_DISPLAY_VGA << 8),
+		.class_mask	= ~0
+	},
+	{
+		.vendor		= PCI_VENDOR_ID_NVIDIA,
+		.device		= PCI_ANY_ID,
+		.subvendor	= PCI_ANY_ID,
+		.subdevice	= PCI_ANY_ID,
+		.class		= (PCI_CLASS_DISPLAY_3D << 8),
+		.class_mask	= ~0
+	},
+	{
+		.vendor		= PCI_VENDOR_ID_NVIDIA,
+		.device		= PCI_ANY_ID,
+		.subvendor	= PCI_ANY_ID,
+		.subdevice	= PCI_ANY_ID,
+		.class		= (PCI_CLASS_BRIDGE_OTHER << 8),
+		.class_mask	= ~0
+	},
+	{ 0 }
+};
+
+static int mods_pci_probe(struct pci_dev *dev, const struct pci_device_id *id)
+{
+	mods_debug_printk(DEBUG_PCI,
+			  "probed dev %04x:%02x:%02x.%x vendor %04x device %04x\n",
+			  pci_domain_nr(dev->bus),
+			  dev->bus->number,
+			  PCI_SLOT(dev->devfn),
+			  PCI_FUNC(dev->devfn),
+			  dev->vendor, dev->device);
+
+	return 0;
+}
+
+#if defined(CONFIG_PCI) && defined(MODS_HAS_SRIOV)
+static int mods_pci_sriov_configure(struct pci_dev *dev, int numvfs);
+#endif
+
+static struct pci_driver mods_pci_driver = {
+	.name            = DEVICE_NAME,
+	.id_table        = mods_pci_table,
+	.probe           = mods_pci_probe,
+	.err_handler     = &mods_pci_error_handlers,
+#ifdef MODS_HAS_SRIOV
+	.sriov_configure = mods_pci_sriov_configure,
+#endif
+};
+#endif
+
+/***********************************************
+ * module wide parameters and access functions *
+ * used to avoid globalization of variables    *
+ ***********************************************/
+
+#ifdef MODS_HAS_TEGRA
+#       define MODS_MULTI_INSTANCE_DEFAULT_VALUE 1
+#else
+#       define MODS_MULTI_INSTANCE_DEFAULT_VALUE 0
+#endif
+
+static int debug;
+static int multi_instance = MODS_MULTI_INSTANCE_DEFAULT_VALUE;
+static u32 access_token   = MODS_ACCESS_TOKEN_NONE;
+
+#if defined(CONFIG_PCI) && defined(MODS_HAS_SRIOV)
+static int mods_pci_sriov_configure(struct pci_dev *dev, int numvfs)
+{
+	int totalvfs;
+	int err = 0;
+
+	LOG_ENT();
+
+	totalvfs = pci_sriov_get_totalvfs(dev);
+
+	if (numvfs > 0) {
+		err = pci_enable_sriov(dev, numvfs);
+
+		if (unlikely(err)) {
+			mods_error_printk(
+				"failed to enable sriov on dev %04x:%02x:%02x.%x %s numvfs=%d (totalvfs=%d), err=%d\n",
+				pci_domain_nr(dev->bus),
+				dev->bus->number,
+				PCI_SLOT(dev->devfn),
+				PCI_FUNC(dev->devfn),
+				dev->is_physfn ? "physfn" : "virtfn",
+				numvfs,
+				totalvfs,
+				err);
+			numvfs = err;
+		} else {
+			mods_info_printk(
+				"enabled sriov on dev %04x:%02x:%02x.%x %s numvfs=%d (totalvfs=%d)\n",
+				pci_domain_nr(dev->bus),
+				dev->bus->number,
+				PCI_SLOT(dev->devfn),
+				PCI_FUNC(dev->devfn),
+				dev->is_physfn ? "physfn" : "virtfn",
+				numvfs,
+				totalvfs);
+		}
+
+	} else {
+		pci_disable_sriov(dev);
+
+		numvfs = 0;
+
+		mods_info_printk(
+			"disabled sriov on dev %04x:%02x:%02x.%x %s (totalvfs=%d)\n",
+			pci_domain_nr(dev->bus),
+			dev->bus->number,
+			PCI_SLOT(dev->devfn),
+			PCI_FUNC(dev->devfn),
+			dev->is_physfn ? "physfn" : "virtfn",
+			totalvfs);
+	}
+
+	/* If this function has been invoked via an ioctl, remember numvfs */
+	if (!err) {
+		struct en_dev_entry *dpriv = pci_get_drvdata(dev);
+
+		if (dpriv)
+			dpriv->num_vfs = numvfs;
+	}
+
+	LOG_EXT();
+	return numvfs;
+}
+
+static int esc_mods_set_num_vf(struct mods_client     *client,
+			       struct MODS_SET_NUM_VF *p)
+{
+	int                  err;
+	struct pci_dev      *dev = NULL;
+	struct en_dev_entry *dpriv;
+
+	LOG_ENT();
+
+	if (p->numvfs > 0xFFFFU) {
+		cl_error("invalid input numfs %u\n", p->numvfs);
+		err = -EINVAL;
+		goto error;
+	}
+
+	/* Get the PCI device structure for the specified device from kernel */
+	err = mods_find_pci_dev(client, &p->dev, &dev);
+	if (unlikely(err)) {
+		if (err == -ENODEV)
+			cl_error("dev %04x:%02x:%02x.%x not found\n",
+				 p->dev.domain,
+				 p->dev.bus,
+				 p->dev.device,
+				 p->dev.function);
+		goto error;
+	}
+
+	dpriv = pci_get_drvdata(dev);
+	if (!dpriv) {
+		cl_error(
+			"failed to enable sriov, dev %04x:%02x:%02x.%x was not enabled\n",
+			pci_domain_nr(dev->bus),
+			dev->bus->number,
+			PCI_SLOT(dev->devfn),
+			PCI_FUNC(dev->devfn));
+		err = -EBUSY;
+		goto error;
+	}
+	if (dpriv->client_id != client->client_id) {
+		cl_error(
+			"invalid client for dev %04x:%02x:%02x.%x, expected %u\n",
+			pci_domain_nr(dev->bus),
+			dev->bus->number,
+			PCI_SLOT(dev->devfn),
+			PCI_FUNC(dev->devfn),
+			dpriv->client_id);
+		err = -EBUSY;
+		goto error;
+	}
+
+	err = mods_pci_sriov_configure(dev, (u16)p->numvfs);
+
+error:
+	pci_dev_put(dev);
+	LOG_EXT();
+	return err;
+}
+
+static int esc_mods_set_total_vf(struct mods_client     *client,
+				 struct MODS_SET_NUM_VF *p)
+{
+	int                  err;
+	struct pci_dev      *dev = NULL;
+	struct en_dev_entry *dpriv;
+
+	LOG_ENT();
+
+	if (p->numvfs > 0xFFFFU) {
+		cl_error("invalid input numfs %u\n", p->numvfs);
+		err = -EINVAL;
+		goto error;
+	}
+
+	/* Get the PCI device structure for the specified device from kernel */
+	err = mods_find_pci_dev(client, &p->dev, &dev);
+	if (unlikely(err)) {
+		if (err == -ENODEV)
+			cl_error("dev %04x:%02x:%02x.%x not found\n",
+				 p->dev.domain,
+				 p->dev.bus,
+				 p->dev.device,
+				 p->dev.function);
+		goto error;
+	}
+
+	dpriv = pci_get_drvdata(dev);
+	if (!dpriv) {
+		cl_error(
+			"failed to enable sriov, dev %04x:%02x:%02x.%x was not enabled\n",
+			pci_domain_nr(dev->bus),
+			dev->bus->number,
+			PCI_SLOT(dev->devfn),
+			PCI_FUNC(dev->devfn));
+		err = -EBUSY;
+		goto error;
+	}
+	if (dpriv->client_id != client->client_id) {
+		cl_error(
+			"invalid client for dev %04x:%02x:%02x.%x, expected %u\n",
+			pci_domain_nr(dev->bus),
+			dev->bus->number,
+			PCI_SLOT(dev->devfn),
+			PCI_FUNC(dev->devfn),
+			dpriv->client_id);
+		err = -EBUSY;
+		goto error;
+	}
+
+	err = pci_sriov_set_totalvfs(dev, (u16)p->numvfs);
+
+	if (unlikely(err)) {
+		cl_error(
+			"failed to set totalvfs=%d on dev %04x:%02x:%02x.%x, err=%d\n",
+			p->numvfs,
+			p->dev.domain,
+			p->dev.bus,
+			p->dev.device,
+			p->dev.function,
+			err);
+	} else
+		cl_info("set totalvfs %d on dev %04x:%02x:%02x.%x\n",
+			p->numvfs,
+			p->dev.domain,
+			p->dev.bus,
+			p->dev.device,
+			p->dev.function);
+
+error:
+	pci_dev_put(dev);
+	LOG_EXT();
+	return err;
+}
+#endif
+
+#if defined(CONFIG_PPC64)
+static int ppc_tce_bypass = MODS_PPC_TCE_BYPASS_ON;
+
+void mods_set_ppc_tce_bypass(int bypass)
+{
+	ppc_tce_bypass = bypass;
+}
+
+int mods_get_ppc_tce_bypass(void)
+{
+	return ppc_tce_bypass;
+}
+#endif
+
+void mods_set_debug_level(int mask)
+{
+	debug = mask;
+}
+
+int mods_get_debug_level(void)
+{
+	return debug;
+}
+
+int mods_check_debug_level(int mask)
+{
+	return ((debug & mask) == mask) ? 1 : 0;
+}
+
+void mods_set_multi_instance(int mi)
+{
+	multi_instance = (mi > 0) ? 1 : -1;
+}
+
+int mods_get_multi_instance(void)
+{
+	return multi_instance > 0;
+}
+
+/*********************
+ * CLIENT MANAGEMENT *
+ *********************/
+static struct mods_priv mp;
+
+static struct mods_client *alloc_client(void)
+{
+	unsigned int idx;
+	unsigned int max_clients = 1;
+
+	LOG_ENT();
+
+	if (mods_get_multi_instance() ||
+	    (mods_get_access_token() != MODS_ACCESS_TOKEN_NONE))
+		max_clients = MODS_MAX_CLIENTS;
+
+	for (idx = 1; idx <= max_clients; idx++) {
+		if (!test_and_set_bit(idx - 1U, &mp.client_flags)) {
+			struct mods_client *client = mods_client_from_id(idx);
+
+			memset(client, 0, sizeof(*client));
+			client->client_id    = idx;
+			client->access_token = MODS_ACCESS_TOKEN_NONE;
+			atomic_set(&client->last_bad_dbdf, -1);
+
+			cl_debug(DEBUG_IOCTL,
+				 "open client (bit mask 0x%lx)\n",
+				 mp.client_flags);
+
+			mutex_init(&client->mtx);
+			spin_lock_init(&client->irq_lock);
+			init_waitqueue_head(&client->interrupt_event);
+			INIT_LIST_HEAD(&client->irq_list);
+			INIT_LIST_HEAD(&client->mem_alloc_list);
+			INIT_LIST_HEAD(&client->mem_map_list);
+			INIT_LIST_HEAD(&client->free_mem_list);
+#if defined(CONFIG_PPC64)
+			INIT_LIST_HEAD(&client->ppc_tce_bypass_list);
+			INIT_LIST_HEAD(&client->nvlink_sysmem_trained_list);
+#endif
+
+			LOG_EXT();
+			return client;
+		}
+	}
+
+	LOG_EXT();
+	return NULL;
+}
+
+static void free_client(u8 client_id)
+{
+	struct mods_client *client = mods_client_from_id(client_id);
+
+	LOG_ENT();
+
+	memset(client, 0, sizeof(*client));
+
+	/* Indicate the client_id is free */
+	clear_bit((unsigned int)client_id - 1U, &mp.client_flags);
+
+	cl_debug(DEBUG_IOCTL, "closed client\n");
+	LOG_EXT();
+}
+
+struct mods_client *mods_client_from_id(u8 client_id)
+{
+	return &mp.clients[client_id - 1];
+}
+
+int mods_is_client_enabled(u8 client_id)
+{
+	return test_bit(client_id - 1, &mp.client_flags);
+}
+
+u32 mods_get_access_token(void)
+{
+	return access_token;
+}
+
+static int validate_client(struct mods_client *client)
+{
+	if (!client) {
+		mods_error_printk("invalid client\n");
+		return false;
+	}
+
+	if (client->client_id < 1 ||
+	    client->client_id > MODS_MAX_CLIENTS ||
+	    !mods_is_client_enabled(client->client_id)) {
+		cl_error("invalid client id\n");
+		return false;
+	}
+
+	return true;
+}
+
+static int mods_set_access_token(u32 tok)
+{
+	/* When setting a null token, the existing token must match the
+	 * provided token, when setting a non-null token the existing token
+	 * must be null, use atomic compare/exchange to set it
+	 */
+	u32 req_old_token =
+	    (tok == MODS_ACCESS_TOKEN_NONE) ?
+		access_token : MODS_ACCESS_TOKEN_NONE;
+
+	if (cmpxchg(&access_token, req_old_token, tok) != req_old_token)
+		return -EFAULT;
+	return OK;
+}
+
+static int mods_check_access_token(struct mods_client *client)
+{
+	if (client->access_token != mods_get_access_token()) {
+		cl_error("invalid access token %u\n", client->access_token);
+		return -EFAULT;
+	}
+
+	return OK;
+}
+
+/******************************
+ * INIT/EXIT MODULE FUNCTIONS *
+ ******************************/
+static int __init mods_init_module(void)
+{
+	int rc;
+
+	LOG_ENT();
+
+	memset(&mp, 0, sizeof(mp));
+
+	mods_init_irq();
+
+	rc = misc_register(&mods_dev);
+	if (rc < 0)
+		return -EBUSY;
+
+#if defined(CONFIG_PCI)
+	rc = pci_register_driver(&mods_pci_driver);
+	if (rc < 0)
+		return -EBUSY;
+#endif
+
+#if defined(MODS_HAS_TEGRA) && defined(CONFIG_COMMON_CLK) && \
+	defined(CONFIG_OF_RESOLVE) && defined(CONFIG_OF_DYNAMIC)
+	mods_init_clock_api();
+#endif
+
+	rc = mods_create_debugfs(&mods_dev);
+	if (rc < 0)
+		return rc;
+
+	rc = mods_init_dmabuf();
+	if (rc < 0)
+		return rc;
+
+#if defined(MODS_HAS_TEGRA)
+	rc = smmu_driver_init();
+	if (rc < 0)
+		return rc;
+
+#if defined(MODS_HAS_PROD)
+	/* tegra prod */
+	mods_tegra_prod_init(&mods_dev);
+#endif
+
+#if defined(CONFIG_DMA_ENGINE)
+	rc = mods_init_dma();
+	if (rc < 0)
+		return rc;
+#endif
+#endif
+
+#if IS_BUILTIN(CONFIG_ARM_FFA_TRANSPORT)
+	rc = mods_ffa_abi_register();
+	if (rc < 0)
+		mods_warning_printk("error on mods_ffa_abi_register returned %d\n", rc);
+#endif
+
+	mods_info_printk("*** WARNING: DIAGNOSTIC DRIVER LOADED ***\n");
+	mods_info_printk("driver loaded, version %x.%02x\n",
+			 (MODS_DRIVER_VERSION >> 8),
+			 (MODS_DRIVER_VERSION & 0xFF));
+
+	if (debug)
+		mods_info_printk("debug level 0x%x\n", debug);
+
+	LOG_EXT();
+	return OK;
+}
+
+static void __exit mods_exit_module(void)
+{
+	int i;
+
+	LOG_ENT();
+
+	mods_exit_dmabuf();
+
+	mods_remove_debugfs();
+
+	for (i = 0; i < MODS_MAX_CLIENTS; i++) {
+		if (mp.client_flags & (1U << i))
+			free_client(i + 1);
+	}
+
+#if defined(MODS_HAS_TEGRA)
+#if defined(CONFIG_DMA_ENGINE)
+	mods_exit_dma();
+#endif
+	smmu_driver_exit();
+#endif
+
+#if defined(CONFIG_PCI)
+	pci_unregister_driver(&mods_pci_driver);
+#endif
+
+	misc_deregister(&mods_dev);
+
+#if defined(MODS_HAS_TEGRA) && defined(CONFIG_COMMON_CLK) && \
+	defined(CONFIG_OF_RESOLVE) && defined(CONFIG_OF_DYNAMIC)
+	mods_shutdown_clock_api();
+#endif
+
+#if IS_BUILTIN(CONFIG_ARM_FFA_TRANSPORT)
+	mods_ffa_abi_unregister();
+#endif
+	mods_info_printk("driver unloaded\n");
+	LOG_EXT();
+}
+
+/***************************
+ * KERNEL INTERFACE SET UP *
+ ***************************/
+module_init(mods_init_module);
+module_exit(mods_exit_module);
+
+MODULE_LICENSE("GPL");
+
+#define STRING_VALUE(x) #x
+#define MAKE_MODULE_VERSION(x, y) STRING_VALUE(x) "." STRING_VALUE(y)
+MODULE_VERSION(MAKE_MODULE_VERSION(MODS_DRIVER_VERSION_MAJOR,
+				   MODS_DRIVER_VERSION_MINOR));
+
+module_param(debug, int, 0644);
+MODULE_PARM_DESC(debug,
+	"debug bitflags (2=ioctl 4=pci 8=acpi 16=irq 32=mem 64=fun +256=detailed)");
+
+module_param(multi_instance, int, 0644);
+MODULE_PARM_DESC(multi_instance,
+	"allows more than one client to simultaneously open the driver");
+
+#if defined(CONFIG_PPC64)
+module_param(ppc_tce_bypass, int, 0644);
+MODULE_PARM_DESC(ppc_tce_bypass,
+	"PPC TCE bypass (0=sys default, 1=force bypass, 2=force non bypass)");
+#endif
+
+/********************
+ * HELPER FUNCTIONS *
+ ********************/
+static void mods_disable_all_devices(struct mods_client *client)
+{
+#ifdef CONFIG_PCI
+	if (unlikely(mutex_lock_interruptible(mods_get_irq_mutex())))
+		return;
+
+	while (client->enabled_devices != NULL) {
+		struct en_dev_entry *old = client->enabled_devices;
+
+		mods_disable_device(client, old->dev);
+		client->enabled_devices = old->next;
+		kfree(old);
+		atomic_dec(&client->num_allocs);
+	}
+
+	mutex_unlock(mods_get_irq_mutex());
+
+	if (client->cached_dev) {
+		pci_dev_put(client->cached_dev);
+		client->cached_dev = NULL;
+	}
+#else
+	WARN_ON(client->enabled_devices != NULL);
+#endif
+}
+
+#if defined(MODS_HAS_CONSOLE_LOCK)
+static int mods_resume_console(struct mods_client *client);
+#else
+static inline int mods_resume_console(struct mods_client *client) { return 0; }
+#endif
+
+/*********************
+ * MAPPING FUNCTIONS *
+ *********************/
+static int register_mapping(struct mods_client   *client,
+			    struct MODS_MEM_INFO *p_mem_info,
+			    phys_addr_t           phys_addr,
+			    unsigned long         virtual_address,
+			    unsigned long         mapping_offs,
+			    unsigned long         mapping_length)
+{
+	struct SYS_MAP_MEMORY *p_map_mem;
+
+	LOG_ENT();
+
+	p_map_mem = kzalloc(sizeof(*p_map_mem), GFP_KERNEL | __GFP_NORETRY);
+	if (unlikely(!p_map_mem)) {
+		LOG_EXT();
+		return -ENOMEM;
+	}
+	atomic_inc(&client->num_allocs);
+
+	p_map_mem->phys_addr      = phys_addr;
+	p_map_mem->virtual_addr   = virtual_address;
+	p_map_mem->mapping_offs   = mapping_offs;
+	p_map_mem->mapping_length = mapping_length;
+	p_map_mem->p_mem_info     = p_mem_info;
+
+	list_add(&p_map_mem->list, &client->mem_map_list);
+
+	cl_debug(DEBUG_MEM_DETAILED,
+		 "map alloc %p as %p: phys 0x%llx, virt 0x%lx, size 0x%lx\n",
+		 p_mem_info,
+		 p_map_mem,
+		 (unsigned long long)phys_addr,
+		 virtual_address,
+		 mapping_length);
+
+	LOG_EXT();
+	return OK;
+}
+
+static void unregister_mapping(struct mods_client    *client,
+			       struct SYS_MAP_MEMORY *p_map_mem)
+{
+	list_del(&p_map_mem->list);
+
+	kfree(p_map_mem);
+	atomic_dec(&client->num_allocs);
+}
+
+static struct SYS_MAP_MEMORY *find_mapping(struct mods_client *client,
+					   unsigned long       virtual_address)
+{
+	struct SYS_MAP_MEMORY *p_map_mem = NULL;
+	struct list_head      *head      = &client->mem_map_list;
+	struct list_head      *iter;
+
+	LOG_ENT();
+
+	list_for_each(iter, head) {
+		p_map_mem = list_entry(iter, struct SYS_MAP_MEMORY, list);
+
+		if (p_map_mem->virtual_addr == virtual_address)
+			break;
+
+		p_map_mem = NULL;
+	}
+
+	LOG_EXT();
+
+	return p_map_mem;
+}
+
+static void unregister_all_mappings(struct mods_client *client)
+{
+	struct SYS_MAP_MEMORY *p_map_mem;
+	struct list_head      *head = &client->mem_map_list;
+	struct list_head      *iter;
+	struct list_head      *tmp;
+
+	LOG_ENT();
+
+	list_for_each_safe(iter, tmp, head) {
+		p_map_mem = list_entry(iter, struct SYS_MAP_MEMORY, list);
+
+		unregister_mapping(client, p_map_mem);
+	}
+
+	LOG_EXT();
+}
+
+static pgprot_t get_prot(struct mods_client *client,
+			 u8                  mem_type,
+			 pgprot_t            prot)
+{
+	switch (mem_type) {
+	case MODS_ALLOC_CACHED:
+		return prot;
+
+	case MODS_ALLOC_UNCACHED:
+		return MODS_PGPROT_UC(prot);
+
+	case MODS_ALLOC_WRITECOMBINE:
+		return MODS_PGPROT_WC(prot);
+
+	default:
+		cl_warn("unsupported memory type: %u\n", mem_type);
+		return prot;
+	}
+}
+
+static int get_prot_for_range(struct mods_client *client,
+			      phys_addr_t         phys_addr,
+			      unsigned long       size,
+			      pgprot_t           *prot)
+{
+	const phys_addr_t req_end     = phys_addr + size;
+	const phys_addr_t range_begin = client->mem_type.phys_addr;
+	const phys_addr_t range_end   = range_begin + client->mem_type.size;
+
+	/* Check overlap with set memory type range */
+	if (phys_addr < range_end && req_end > range_begin) {
+
+		/* Check if requested range is completely inside */
+		if (likely(phys_addr >= range_begin && req_end <= range_end)) {
+			*prot = get_prot(client, client->mem_type.type, *prot);
+			return 0;
+		}
+
+		cl_error("mmap range [0x%llx, 0x%llx] does not match set memory type range [0x%llx, 0x%llx]\n",
+			 (unsigned long long)phys_addr,
+			 (unsigned long long)req_end,
+			 (unsigned long long)range_begin,
+			 (unsigned long long)range_end);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+const char *mods_get_prot_str(u8 mem_type)
+{
+	switch (mem_type) {
+	case MODS_ALLOC_CACHED:
+		return "WB";
+
+	case MODS_ALLOC_UNCACHED:
+		return "UC";
+
+	case MODS_ALLOC_WRITECOMBINE:
+		return "WC";
+
+	default:
+		return "unknown";
+	}
+}
+
+static const char *get_prot_str_for_range(struct mods_client *client,
+					  phys_addr_t         phys_addr,
+					  unsigned long       size)
+{
+	const phys_addr_t req_end     = phys_addr + size;
+	const phys_addr_t range_begin = client->mem_type.phys_addr;
+	const phys_addr_t range_end   = range_begin + client->mem_type.size;
+
+	/* Check overlap with set memory type range */
+	if (phys_addr < range_end && req_end > range_begin)
+		return mods_get_prot_str(client->mem_type.type);
+
+	return "default";
+}
+
+/********************
+ * PCI ERROR FUNCTIONS *
+ ********************/
+#if defined(CONFIG_PCI)
+static pci_ers_result_t mods_pci_error_detected(struct pci_dev *dev,
+						pci_channel_state_t state)
+{
+	mods_debug_printk(DEBUG_PCI,
+			  "pci_error_detected dev %04x:%02x:%02x.%x\n",
+			  pci_domain_nr(dev->bus),
+			  dev->bus->number,
+			  PCI_SLOT(dev->devfn),
+			  PCI_FUNC(dev->devfn));
+
+	return PCI_ERS_RESULT_CAN_RECOVER;
+}
+
+static pci_ers_result_t mods_pci_mmio_enabled(struct pci_dev *dev)
+{
+	mods_debug_printk(DEBUG_PCI,
+			  "pci_mmio_enabled dev %04x:%02x:%02x.%x\n",
+			  pci_domain_nr(dev->bus),
+			  dev->bus->number,
+			  PCI_SLOT(dev->devfn),
+			  PCI_FUNC(dev->devfn));
+
+	return PCI_ERS_RESULT_NEED_RESET;
+}
+
+static void mods_pci_resume(struct pci_dev *dev)
+{
+	mods_debug_printk(DEBUG_PCI,
+			  "pci_resume dev %04x:%02x:%02x.%x\n",
+			  pci_domain_nr(dev->bus),
+			  dev->bus->number,
+			  PCI_SLOT(dev->devfn),
+			  PCI_FUNC(dev->devfn));
+}
+#endif
+
+/********************
+ * KERNEL FUNCTIONS *
+ ********************/
+static void mods_krnl_vma_open(struct vm_area_struct *vma)
+{
+	struct mods_vm_private_data *priv;
+
+	LOG_ENT();
+	mods_debug_printk(DEBUG_MEM_DETAILED,
+			  "open vma, virt 0x%lx, size 0x%lx, phys 0x%llx\n",
+			  vma->vm_start,
+			  vma->vm_end - vma->vm_start,
+			  (unsigned long long)vma->vm_pgoff << PAGE_SHIFT);
+
+	priv = vma->vm_private_data;
+	if (priv)
+		atomic_inc(&priv->usage_count);
+
+	LOG_EXT();
+}
+
+static void mods_krnl_vma_close(struct vm_area_struct *vma)
+{
+	struct mods_vm_private_data *priv;
+
+	LOG_ENT();
+
+	priv = vma->vm_private_data;
+	if (priv && atomic_dec_and_test(&priv->usage_count)) {
+		struct mods_client    *client = priv->client;
+		struct SYS_MAP_MEMORY *p_map_mem;
+
+		mutex_lock(&client->mtx);
+
+		/* we need to unregister the mapping */
+		p_map_mem = find_mapping(client, vma->vm_start);
+		if (p_map_mem)
+			unregister_mapping(client, p_map_mem);
+
+		mutex_unlock(&client->mtx);
+
+		mods_debug_printk(DEBUG_MEM_DETAILED,
+				  "closed vma, virt 0x%lx\n",
+				  vma->vm_start);
+
+		vma->vm_private_data = NULL;
+
+		kfree(priv);
+		atomic_dec(&client->num_allocs);
+	}
+
+	LOG_EXT();
+}
+
+#ifdef CONFIG_HAVE_IOREMAP_PROT
+static int mods_krnl_vma_access(struct vm_area_struct *vma,
+				unsigned long          addr,
+				void                  *buf,
+				int                    len,
+				int                    write)
+{
+	struct mods_vm_private_data *priv = vma->vm_private_data;
+	struct mods_client          *client;
+	struct SYS_MAP_MEMORY       *p_map_mem;
+	unsigned long                map_offs;
+	int                          err  = OK;
+
+	LOG_ENT();
+
+	if (!priv) {
+		LOG_EXT();
+		return -EINVAL;
+	}
+
+	client = priv->client;
+
+	cl_debug(DEBUG_MEM_DETAILED,
+		 "access vma [virt 0x%lx, size 0x%lx, phys 0x%llx] at virt 0x%lx, len 0x%x\n",
+		 vma->vm_start,
+		 vma->vm_end - vma->vm_start,
+		 (unsigned long long)vma->vm_pgoff << PAGE_SHIFT,
+		 addr,
+		 len);
+
+	if (unlikely(mutex_lock_interruptible(&client->mtx))) {
+		LOG_EXT();
+		return -EINTR;
+	}
+
+	p_map_mem = find_mapping(client, vma->vm_start);
+
+	if (unlikely(!p_map_mem || addr < p_map_mem->virtual_addr ||
+		     addr + len > p_map_mem->virtual_addr +
+				  p_map_mem->mapping_length)) {
+		cl_error("mapped range [virt 0x%lx, size 0x%x] does not match vma [virt 0x%lx, size 0x%lx]\n",
+			 addr, len, vma->vm_start, vma->vm_end - vma->vm_start);
+		mutex_unlock(&client->mtx);
+		LOG_EXT();
+		return -ENOMEM;
+	}
+
+	map_offs = addr - vma->vm_start + p_map_mem->mapping_offs;
+
+	if (p_map_mem->p_mem_info) {
+		struct MODS_MEM_INFO *p_mem_info = p_map_mem->p_mem_info;
+		struct scatterlist   *sg         = NULL;
+		const u32             num_chunks = get_num_chunks(p_mem_info);
+		u32                   i;
+
+		for_each_sg(p_mem_info->sg, sg, num_chunks, i) {
+
+			if (map_offs < sg->length)
+				break;
+
+			map_offs -= sg->length;
+		}
+
+		if (unlikely(!sg))
+			err = -ENOMEM;
+		else {
+			void        *ptr;
+			struct page *p_page = sg_page(sg) +
+					      (map_offs >> PAGE_SHIFT);
+
+			map_offs &= ~PAGE_MASK;
+
+			if (map_offs + len > PAGE_SIZE)
+				len = (int)(PAGE_SIZE - map_offs);
+
+			ptr = MODS_KMAP(p_page);
+			if (ptr) {
+				char *bptr = (char *)ptr + map_offs;
+
+				if (write)
+					memcpy(bptr, buf, len);
+				else
+					memcpy(buf, bptr, len);
+
+				kunmap(ptr);
+
+				err = len;
+			} else
+				err = -ENOMEM;
+		}
+	} else if (!write) {
+		char __iomem *ptr;
+		phys_addr_t   pa;
+
+		map_offs += vma->vm_pgoff << PAGE_SHIFT;
+		pa       =  map_offs & PAGE_MASK;
+		map_offs &= ~PAGE_MASK;
+
+		if (map_offs + len > PAGE_SIZE)
+			len = (int)(PAGE_SIZE - map_offs);
+
+		ptr = ioremap(pa, PAGE_SIZE);
+
+		if (ptr) {
+			memcpy_fromio(buf, ptr + map_offs, len);
+
+			iounmap(ptr);
+
+			err = len;
+		} else
+			err = -ENOMEM;
+	} else
+		/* Writing to device memory from gdb is not supported */
+		err = -ENOMEM;
+
+	mutex_unlock(&client->mtx);
+
+	LOG_EXT();
+	return err;
+}
+#endif
+
+static const struct vm_operations_struct mods_krnl_vm_ops = {
+#ifdef CONFIG_HAVE_IOREMAP_PROT
+	.access = mods_krnl_vma_access,
+#endif
+	.open	= mods_krnl_vma_open,
+	.close	= mods_krnl_vma_close
+};
+
+static int mods_krnl_open(struct inode *ip, struct file *fp)
+{
+	struct mods_client *client;
+
+	LOG_ENT();
+
+	client = alloc_client();
+	if (client == NULL) {
+		mods_error_printk("too many clients\n");
+		LOG_EXT();
+		return -EBUSY;
+	}
+
+	fp->private_data = client;
+
+	cl_info("driver opened, pid=%d\n", current->pid);
+	LOG_EXT();
+	return OK;
+}
+
+static int mods_krnl_close(struct inode *ip, struct file *fp)
+{
+	struct mods_client *client    = fp->private_data;
+	int                 final_err = OK;
+	int                 err       = OK;
+	u8                  client_id;
+
+	LOG_ENT();
+
+	if (!validate_client(client)) {
+		LOG_EXT();
+		return -EINVAL;
+	}
+
+	client_id = client->client_id;
+
+	mods_free_client_interrupts(client);
+
+	mods_resume_console(client);
+
+	unregister_all_mappings(client);
+	err = mods_unregister_all_alloc(client);
+	if (err)
+		cl_error("failed to free all memory\n");
+	final_err = err;
+
+#if defined(CONFIG_PPC64)
+	err = mods_unregister_all_ppc_tce_bypass(client);
+	if (err)
+		cl_error("failed to restore dma bypass\n");
+	if (!final_err)
+		final_err = err;
+
+	err = mods_unregister_all_nvlink_sysmem_trained(client);
+	if (err)
+		cl_error("failed to free nvlink trained\n");
+	if (!final_err)
+		final_err = err;
+#endif
+
+#if defined(CONFIG_TEGRA_IVC)
+	mods_bpmpipc_cleanup();
+#endif
+
+	mods_disable_all_devices(client);
+
+	{
+		const int num_allocs = atomic_read(&client->num_allocs);
+		const int num_pages  = atomic_read(&client->num_pages);
+
+		if (num_allocs || num_pages) {
+			cl_error(
+				"not all allocations have been freed, allocs=%d, pages=%d\n",
+				num_allocs, num_pages);
+			if (!final_err)
+				final_err = -ENOMEM;
+		}
+	}
+
+	if (client->work_queue) {
+		destroy_workqueue(client->work_queue);
+		client->work_queue = NULL;
+	}
+
+	free_client(client_id);
+
+	pr_info("mods [%d]: driver closed\n", client_id);
+
+	LOG_EXT();
+	return final_err;
+}
+
+static POLL_TYPE mods_krnl_poll(struct file *fp, poll_table *wait)
+{
+	struct mods_client *client = fp->private_data;
+	POLL_TYPE           mask   = 0;
+	int                 err;
+
+	if (!validate_client(client))
+		return POLLERR;
+
+	err = mods_check_access_token(client);
+	if (err < 0)
+		return POLLERR;
+
+	if (!(fp->f_flags & O_NONBLOCK)) {
+		cl_debug(DEBUG_ISR_DETAILED, "poll wait\n");
+		poll_wait(fp, &client->interrupt_event, wait);
+	}
+
+	/* if any interrupts pending then check intr, POLLIN on irq */
+	mask |= mods_irq_event_check(client->client_id);
+
+	cl_debug(DEBUG_ISR_DETAILED, "poll mask 0x%x\n", mask);
+
+	return mask;
+}
+
+static int mods_krnl_map_inner(struct mods_client    *client,
+			       struct vm_area_struct *vma);
+
+static int mods_krnl_mmap(struct file *fp, struct vm_area_struct *vma)
+{
+	struct mods_vm_private_data *vma_private_data;
+	struct mods_client          *client = fp->private_data;
+	int err;
+
+	LOG_ENT();
+
+	if (!validate_client(client)) {
+		LOG_EXT();
+		return -EINVAL;
+	}
+
+	err = mods_check_access_token(client);
+	if (err < 0) {
+		LOG_EXT();
+		return err;
+	}
+
+	vma->vm_ops = &mods_krnl_vm_ops;
+
+	vma_private_data = kzalloc(sizeof(*vma_private_data),
+				   GFP_KERNEL | __GFP_NORETRY);
+	if (unlikely(!vma_private_data)) {
+		LOG_EXT();
+		return -ENOMEM;
+	}
+	atomic_inc(&client->num_allocs);
+
+	/* set private data for vm_area_struct */
+	atomic_set(&vma_private_data->usage_count, 0);
+	vma_private_data->client = client;
+	vma->vm_private_data = vma_private_data;
+
+	mods_krnl_vma_open(vma);
+
+	err = mutex_lock_interruptible(&client->mtx);
+	if (likely(!err)) {
+		err = mods_krnl_map_inner(client, vma);
+		mutex_unlock(&client->mtx);
+	}
+
+	if (unlikely(err))
+		mods_krnl_vma_close(vma);
+
+	LOG_EXT();
+	return err;
+}
+
+static int map_system_mem(struct mods_client    *client,
+			  struct vm_area_struct *vma,
+			  struct MODS_MEM_INFO  *p_mem_info)
+{
+	struct scatterlist *sg          = NULL;
+	const char         *cache_str   = mods_get_prot_str(p_mem_info->cache_type);
+	const phys_addr_t   req_pa = (phys_addr_t)vma->vm_pgoff << PAGE_SHIFT;
+	phys_addr_t         reg_pa      = 0;
+	const unsigned long vma_size    = vma->vm_end - vma->vm_start;
+	unsigned long       size_to_map = vma_size;
+	unsigned long       skip_size   = 0;
+	unsigned long       map_va      = 0;
+	const u32           num_chunks  = get_num_chunks(p_mem_info);
+	u32                 map_chunks;
+	u32                 i           = 0;
+	const pgprot_t      prot        = get_prot(client,
+						   p_mem_info->cache_type,
+						   vma->vm_page_prot);
+
+	/* Find the beginning of the requested range */
+	for_each_sg(p_mem_info->sg, sg, num_chunks, i) {
+		const phys_addr_t  phys_addr = sg_phys(sg);
+		const unsigned int size      = sg->length;
+
+		if ((req_pa >= phys_addr) &&
+		    (req_pa <  phys_addr + size)) {
+			break;
+		}
+
+		skip_size += size;
+	}
+
+	if (i == num_chunks) {
+		cl_error("can't satisfy requested mapping\n");
+		return -EINVAL;
+	}
+
+	if (((skip_size + vma_size) >> PAGE_SHIFT) > p_mem_info->num_pages) {
+		cl_error("requested mapping exceeds bounds\n");
+		return -EINVAL;
+	}
+
+	/* Map pages into VA space */
+	map_va     = vma->vm_start;
+	map_chunks = num_chunks - i;
+	for_each_sg(sg, sg, map_chunks, i) {
+
+		const phys_addr_t chunk_pa = sg_phys(sg);
+		phys_addr_t       map_pa   = chunk_pa;
+		unsigned int      map_size = sg->length;
+
+		if (i == 0) {
+			const phys_addr_t aoffs = req_pa - chunk_pa;
+
+			map_pa    += aoffs;
+			map_size  -= aoffs;
+			skip_size += aoffs;
+			reg_pa     = chunk_pa;
+		}
+
+		if (map_size > size_to_map)
+			map_size = (unsigned int)size_to_map;
+
+		cl_debug(DEBUG_MEM_DETAILED,
+			 "remap va 0x%lx pfn 0x%lx size 0x%x pages %u %s\n",
+			 map_va,
+			 (unsigned long)(map_pa >> PAGE_SHIFT),
+			 map_size,
+			 map_size >> PAGE_SHIFT,
+			 cache_str);
+
+		if (remap_pfn_range(vma,
+				    map_va,
+				    (unsigned long)(map_pa >> PAGE_SHIFT),
+				    map_size,
+				    prot)) {
+			cl_error("failed to map memory\n");
+			return -EAGAIN;
+		}
+
+		map_va      += map_size;
+		size_to_map -= map_size;
+		if (!size_to_map)
+			break;
+	}
+
+	register_mapping(client,
+			 p_mem_info,
+			 reg_pa,
+			 vma->vm_start,
+			 skip_size,
+			 vma_size);
+
+	return OK;
+}
+
+static int map_device_mem(struct mods_client    *client,
+			  struct vm_area_struct *vma)
+{
+	/* device memory */
+	const phys_addr_t   req_pa   = (phys_addr_t)vma->vm_pgoff << PAGE_SHIFT;
+	const unsigned long vma_size = vma->vm_end - vma->vm_start;
+	pgprot_t            prot     = vma->vm_page_prot;
+	int                 err;
+
+	cl_debug(DEBUG_MEM,
+		 "map dev: phys 0x%llx, virt 0x%lx, size 0x%lx, %s\n",
+		 (unsigned long long)req_pa,
+		 vma->vm_start,
+		 vma_size,
+		 get_prot_str_for_range(client, req_pa, vma_size));
+
+	err = get_prot_for_range(client, req_pa, vma_size, &prot);
+	if (unlikely(err))
+		return err;
+
+	if (unlikely(io_remap_pfn_range(vma,
+					vma->vm_start,
+					vma->vm_pgoff,
+					vma_size,
+					prot))) {
+		cl_error("failed to map device memory\n");
+		return -EAGAIN;
+	}
+
+	register_mapping(client,
+			 NULL,
+			 req_pa,
+			 vma->vm_start,
+			 0,
+			 vma_size);
+
+	return OK;
+}
+
+static int mods_krnl_map_inner(struct mods_client    *client,
+			       struct vm_area_struct *vma)
+{
+	const phys_addr_t     req_pa = (phys_addr_t)vma->vm_pgoff << PAGE_SHIFT;
+	struct MODS_MEM_INFO *p_mem_info = mods_find_alloc(client, req_pa);
+	const unsigned long   vma_size   = vma->vm_end - vma->vm_start;
+
+	if (unlikely((vma_size & ~PAGE_MASK) != 0)) {
+		cl_error("requested mapping is not page-aligned\n");
+		return -EINVAL;
+	}
+
+	if (p_mem_info)
+		return map_system_mem(client, vma, p_mem_info);
+	else
+		return map_device_mem(client, vma);
+}
+
+#if defined(CONFIG_X86)
+static void mods_get_screen_info(struct MODS_SCREEN_INFO *p)
+{
+	p->orig_video_mode = screen_info.orig_video_mode;
+	p->orig_video_is_vga = screen_info.orig_video_isVGA;
+	p->lfb_width = screen_info.lfb_width;
+	p->lfb_height = screen_info.lfb_height;
+	p->lfb_depth = screen_info.lfb_depth;
+	p->lfb_base = screen_info.lfb_base;
+	p->lfb_size = screen_info.lfb_size;
+	p->lfb_linelength = screen_info.lfb_linelength;
+}
+#endif
+
+/*************************
+ * ESCAPE CALL FUNCTIONS *
+ *************************/
+
+static int esc_mods_get_api_version(struct mods_client      *client,
+				    struct MODS_GET_VERSION *p)
+{
+	p->version = MODS_DRIVER_VERSION;
+	return OK;
+}
+
+static int esc_mods_get_kernel_version(struct mods_client      *client,
+				       struct MODS_GET_VERSION *p)
+{
+	p->version = MODS_KERNEL_VERSION;
+	return OK;
+}
+
+#if defined(CONFIG_X86)
+static int esc_mods_get_screen_info(struct mods_client      *client,
+				    struct MODS_SCREEN_INFO *p)
+{
+	mods_get_screen_info(p);
+
+#if defined(VIDEO_CAPABILITY_64BIT_BASE)
+	if (screen_info.ext_lfb_base)
+		return -EOVERFLOW;
+#endif
+
+	return OK;
+}
+
+static int esc_mods_get_screen_info_2(struct mods_client        *client,
+				      struct MODS_SCREEN_INFO_2 *p)
+{
+#if defined(CONFIG_FB) && defined(MODS_HAS_FB_SET_SUSPEND)
+	unsigned int i;
+	bool         found = false;
+#endif
+
+	mods_get_screen_info(&p->info);
+
+#if defined(VIDEO_CAPABILITY_64BIT_BASE)
+	p->ext_lfb_base = screen_info.ext_lfb_base;
+#else
+	p->ext_lfb_base = 0;
+#endif
+
+#if defined(CONFIG_FB) && defined(MODS_HAS_FB_SET_SUSPEND)
+	if (screen_info.orig_video_isVGA != VIDEO_TYPE_EFI)
+		return OK;
+
+	/* With pci=realloc on the kernel command line, GPU BAR1 can be
+	 * reassigned after the OS console is allocated.  When this
+	 * occurs the lfb_base variable is *not* updated for an EFI
+	 * console.  The incorrect lfb_base variable will prevent other
+	 * drivers or user space applications from identifying memory
+	 * in use by the console and potentially using it themselves.
+
+	 * For an EFI console, pull the FB base address from the FB
+	 * driver registered_fb data instead of screen_info.
+	 * Note: on kernel 6.1 and up registered_fb is not available.
+	 */
+	for (i = 0; i < ARRAY_SIZE(registered_fb); i++) {
+		bool skipped = true;
+
+		if (!registered_fb[i])
+			continue;
+
+		if (!strcmp(registered_fb[i]->fix.id, "EFI VGA") && !found) {
+			p->info.lfb_base =
+			 registered_fb[i]->fix.smem_start & 0xFFFFFFFF;
+			p->ext_lfb_base =
+			 registered_fb[i]->fix.smem_start >> 32;
+			found = true;
+			skipped = false;
+		}
+
+		cl_info("%s fb%d '%s' @0x%llx\n",
+			skipped ? "skip" : "found",
+			i, registered_fb[i]->fix.id,
+			(unsigned long long)registered_fb[i]->fix.smem_start);
+	}
+#endif
+
+	return OK;
+}
+#endif
+
+#if defined(MODS_HAS_CONSOLE_LOCK)
+
+#if defined(CONFIG_FB) && defined(MODS_HAS_FB_SET_SUSPEND)
+static int suspend_fb(struct mods_client *client)
+{
+	unsigned int i;
+	int          err = -EINVAL;
+
+	/* tell the os to block fb accesses */
+	for (i = 0; i < ARRAY_SIZE(registered_fb); i++) {
+		bool suspended = false;
+
+		if (!registered_fb[i])
+			continue;
+
+		console_lock();
+		if (registered_fb[i]->state != FBINFO_STATE_SUSPENDED) {
+			fb_set_suspend(registered_fb[i], 1);
+			client->mods_fb_suspended[i] = 1;
+			suspended = true;
+		}
+		console_unlock();
+		err = OK;
+
+		if (suspended)
+			cl_info("suspended fb%u '%s'\n", i,
+				registered_fb[i]->fix.id);
+	}
+
+	return err;
+}
+
+static int resume_fb(struct mods_client *client)
+{
+	unsigned int i;
+	int          err = -EINVAL;
+
+	for (i = 0; i < ARRAY_SIZE(registered_fb); i++) {
+		bool resumed = false;
+
+		if (!registered_fb[i])
+			continue;
+
+		console_lock();
+		if (client->mods_fb_suspended[i]) {
+			fb_set_suspend(registered_fb[i], 0);
+			client->mods_fb_suspended[i] = 0;
+			resumed = true;
+		}
+		console_unlock();
+		err = OK;
+
+		if (resumed)
+			cl_info("resumed fb%u\n", i);
+	}
+
+	return err;
+}
+#else
+#define suspend_fb(client) (-EINVAL)
+#define resume_fb(client) (-EINVAL)
+#endif
+
+static atomic_t console_is_locked;
+
+static int esc_mods_lock_console(struct mods_client *client)
+{
+	if (atomic_cmpxchg(&console_is_locked, 0, 1)) {
+		cl_error("console is already locked\n");
+		return -EINVAL;
+	}
+
+	atomic_set(&client->console_is_locked, 1);
+	console_lock();
+	return OK;
+}
+
+static int esc_mods_unlock_console(struct mods_client *client)
+{
+	if (!atomic_cmpxchg(&client->console_is_locked, 1, 0)) {
+		cl_error("console is not locked by this client\n");
+		return -EINVAL;
+	}
+
+	console_unlock();
+	atomic_set(&console_is_locked, 0);
+	return OK;
+}
+
+static int esc_mods_suspend_console(struct mods_client *client)
+{
+	int err = -EINVAL;
+
+	LOG_ENT();
+
+	if (atomic_cmpxchg(&console_is_locked, 0, 1)) {
+		cl_error("cannot suspend console, console is locked\n");
+		LOG_EXT();
+		return -EINVAL;
+	}
+
+	err = suspend_fb(client);
+
+#if defined(MODS_HAS_CONSOLE_BINDING)
+	if (&vga_con == vc_cons[fg_console].d->vc_sw) {
+		/* if the current console is the vga console driver,
+		 * have the dummy driver take over.
+		 */
+		console_lock();
+		do_take_over_console(&dummy_con, 0, 0, 0);
+		console_unlock();
+		err = OK;
+
+		cl_info("switched console to dummy\n");
+	}
+#endif
+
+	if (err)
+		cl_warn("no methods to suspend console available\n");
+
+	atomic_set(&console_is_locked, 0);
+
+	LOG_EXT();
+
+	return err;
+}
+
+static int esc_mods_resume_console(struct mods_client *client)
+{
+	return mods_resume_console(client);
+}
+
+static int mods_resume_console(struct mods_client *client)
+{
+	int err = -EINVAL;
+
+	LOG_ENT();
+
+	if (atomic_cmpxchg(&client->console_is_locked, 1, 0)) {
+		cl_warn("console was not properly unlocked\n");
+		console_unlock();
+	} else if (atomic_cmpxchg(&console_is_locked, 0, 1)) {
+		cl_error("cannot resume console, console is locked\n");
+		LOG_EXT();
+		return -EINVAL;
+	}
+
+	err = resume_fb(client);
+
+#if defined(MODS_HAS_CONSOLE_BINDING)
+	if (&dummy_con == vc_cons[fg_console].d->vc_sw) {
+		/* try to unbind the dummy driver,
+		 * the system driver should take over.
+		 */
+		console_lock();
+		do_unbind_con_driver(vc_cons[fg_console].d->vc_sw, 0, 0, 0);
+		console_unlock();
+		err = OK;
+
+		cl_info("restored vga console\n");
+	}
+#endif
+	atomic_set(&console_is_locked, 0);
+
+	LOG_EXT();
+
+	return err;
+}
+#endif
+
+static int esc_mods_acquire_access_token(struct mods_client       *client,
+					 struct MODS_ACCESS_TOKEN *ptoken)
+{
+	int err = -EINVAL;
+
+	LOG_ENT();
+
+	if (mods_get_multi_instance()) {
+		cl_error(
+			"access token ops not supported with multi_instance=1\n");
+		LOG_EXT();
+		return err;
+	}
+
+	get_random_bytes(&ptoken->token, sizeof(ptoken->token));
+	err = mods_set_access_token(ptoken->token);
+	if (err)
+		cl_error("unable to set access token\n");
+	else {
+		cl_info("set access token %u\n", ptoken->token);
+		client->access_token = ptoken->token;
+	}
+
+	LOG_EXT();
+
+	return err;
+}
+
+static int esc_mods_release_access_token(struct mods_client       *client,
+					 struct MODS_ACCESS_TOKEN *ptoken)
+{
+	int err = -EINVAL;
+
+	LOG_ENT();
+
+	if (mods_get_multi_instance()) {
+		cl_error(
+			"access token ops not supported with multi_instance=1\n");
+		LOG_EXT();
+		return err;
+	}
+
+	err = mods_set_access_token(MODS_ACCESS_TOKEN_NONE);
+	if (err)
+		cl_error("unable to clear access token\n");
+	else {
+		cl_info("released access token %u\n", client->access_token);
+		client->access_token = MODS_ACCESS_TOKEN_NONE;
+	}
+
+	LOG_EXT();
+
+	return err;
+}
+
+static int esc_mods_verify_access_token(struct mods_client       *client,
+					struct MODS_ACCESS_TOKEN *ptoken)
+{
+	int err = -EINVAL;
+
+	LOG_ENT();
+
+	if (ptoken->token == mods_get_access_token()) {
+		client->access_token = ptoken->token;
+		err = OK;
+	} else
+		cl_error("invalid access token %u\n", client->access_token);
+
+	LOG_EXT();
+
+	return err;
+}
+
+struct mods_file_work {
+	struct work_struct work;
+	const char        *path;
+	const char        *data;
+	u32                data_size;
+	ssize_t            err;
+};
+
+static void sysfs_write_task(struct work_struct *w)
+{
+	struct mods_file_work *task = container_of(w,
+						   struct mods_file_work,
+						   work);
+	struct file *f;
+
+	LOG_ENT();
+
+	task->err = -EINVAL;
+
+	f = filp_open(task->path, O_WRONLY, 0);
+	if (IS_ERR(f))
+		task->err = PTR_ERR(f);
+	else {
+#ifndef MODS_HAS_KERNEL_WRITE
+		mm_segment_t old_fs = get_fs();
+#endif
+
+		f->f_pos = 0;
+#ifdef MODS_HAS_KERNEL_WRITE
+		task->err = kernel_write(f,
+					 task->data,
+					 task->data_size,
+					 &f->f_pos);
+#else
+		set_fs(KERNEL_DS);
+
+		task->err = vfs_write(f,
+				      (__force const char __user *)task->data,
+				      task->data_size,
+				      &f->f_pos);
+
+		set_fs(old_fs);
+#endif
+		filp_close(f, NULL);
+	}
+
+	LOG_EXT();
+}
+
+static int create_work_queue(struct mods_client *client)
+{
+	int err = 0;
+
+	if (unlikely(mutex_lock_interruptible(&client->mtx)))
+		return -EINTR;
+
+	if (!client->work_queue) {
+		client->work_queue = create_singlethread_workqueue("mods_wq");
+		if (!client->work_queue) {
+			cl_error("failed to create work queue\n");
+			err = -ENOMEM;
+		}
+	}
+
+	mutex_unlock(&client->mtx);
+
+	return err;
+}
+
+static int run_write_task(struct mods_client    *client,
+			  struct mods_file_work *task)
+{
+	int err = create_work_queue(client);
+
+	if (err)
+		return err;
+
+	cl_info("write %.*s to %s\n", task->data_size, task->data, task->path);
+
+	INIT_WORK(&task->work, sysfs_write_task);
+	queue_work(client->work_queue, &task->work);
+	flush_workqueue(client->work_queue);
+
+	if (task->err < 0)
+		cl_error("failed to write %.*s to %s\n",
+			 task->data_size, task->data, task->path);
+
+	return (task->err > 0) ? 0 : (int)task->err;
+}
+
+static int esc_mods_write_sysfs_node(struct mods_client     *client,
+				     struct MODS_SYSFS_NODE *pdata)
+{
+	int                   err;
+	struct mods_file_work task;
+
+	LOG_ENT();
+
+	if (pdata->size > MODS_MAX_SYSFS_FILE_SIZE) {
+		cl_error("invalid data size %u, max allowed is %u\n",
+			 pdata->size, MODS_MAX_SYSFS_FILE_SIZE);
+		LOG_EXT();
+		return -EINVAL;
+	}
+
+	memmove(&pdata->path[5], pdata->path, sizeof(pdata->path) - 5);
+	memcpy(pdata->path, "/sys/", 5);
+	pdata->path[sizeof(pdata->path) - 1] = 0;
+
+	memset(&task, 0, sizeof(task));
+	task.path      = pdata->path;
+	task.data      = pdata->contents;
+	task.data_size = pdata->size;
+
+	err = run_write_task(client, &task);
+
+	LOG_EXT();
+	return err;
+}
+
+static int esc_mods_sysctl_write_int(struct mods_client     *client,
+				     struct MODS_SYSCTL_INT *pdata)
+{
+	int                   err;
+	struct mods_file_work task;
+	char                  data[21];
+	int                   data_size;
+
+	LOG_ENT();
+
+	memmove(&pdata->path[10], pdata->path, sizeof(pdata->path)  - 10);
+	memcpy(pdata->path, "/proc/sys/", 10);
+	pdata->path[sizeof(pdata->path) - 1] = 0;
+
+	data_size = snprintf(data, sizeof(data),
+			     "%lld", (long long)pdata->value);
+
+	if (unlikely(data_size < 0)) {
+		err = data_size;
+		goto error;
+	}
+
+	memset(&task, 0, sizeof(task));
+	task.path      = pdata->path;
+	task.data      = data;
+	task.data_size = data_size;
+
+	err = run_write_task(client, &task);
+
+error:
+	LOG_EXT();
+	return err;
+}
+
+#ifdef CONFIG_X86
+static int esc_mods_read_msr(struct mods_client *client, struct MODS_MSR *p)
+{
+	int err = -EINVAL;
+
+	LOG_ENT();
+
+	err = rdmsr_safe_on_cpu(p->cpu_num, p->reg, &p->low, &p->high);
+	if (err)
+		cl_error("could not read MSR %u\n", p->reg);
+
+	LOG_EXT();
+	return err;
+}
+
+static int esc_mods_write_msr(struct mods_client *client, struct MODS_MSR *p)
+{
+	int err = -EINVAL;
+
+	LOG_ENT();
+
+	err = wrmsr_safe_on_cpu(p->cpu_num, p->reg, p->low, p->high);
+	if (err)
+		cl_error("could not write MSR %u\n", p->reg);
+
+	LOG_EXT();
+	return err;
+}
+#endif
+
+static int esc_mods_get_driver_stats(struct mods_client *client,
+				     struct MODS_GET_DRIVER_STATS *p)
+{
+	int num_allocs;
+	int num_pages;
+
+	LOG_ENT();
+
+	num_allocs = atomic_read(&client->num_allocs);
+	num_pages  = atomic_read(&client->num_pages);
+
+	memset(p, 0, sizeof(*p));
+	p->version    = MODS_DRIVER_STATS_VERSION;
+	p->num_allocs = (num_allocs < 0) ? ~0U : num_allocs;
+	p->num_pages  = (num_pages  < 0) ? ~0U : num_pages;
+
+	LOG_EXT();
+	return 0;
+}
+
+/**************
+ * IO control *
+ **************/
+
+static long mods_krnl_ioctl(struct file  *fp,
+			    unsigned int  cmd,
+			    unsigned long i_arg)
+{
+	int                 err      = 0;
+	void               *arg_copy = NULL;
+	void        __user *arg      = (void __user *)i_arg;
+	struct mods_client *client   = fp->private_data;
+	int                 arg_size;
+	char                buf[64];
+
+	LOG_ENT();
+
+	if (!validate_client(client)) {
+		LOG_EXT();
+		return -EINVAL;
+	}
+
+	if ((cmd != MODS_ESC_VERIFY_ACCESS_TOKEN) &&
+	    (cmd != MODS_ESC_GET_API_VERSION)) {
+		err = mods_check_access_token(client);
+		if (err) {
+			LOG_EXT();
+			return err;
+		}
+	}
+
+	arg_size = _IOC_SIZE(cmd);
+
+	if (arg_size > (int)sizeof(buf)) {
+		arg_copy = kzalloc(arg_size, GFP_KERNEL | __GFP_NORETRY);
+		if (unlikely(!arg_copy)) {
+			LOG_EXT();
+			return -ENOMEM;
+		}
+		atomic_inc(&client->num_allocs);
+	} else if (arg_size > 0)
+		arg_copy = buf;
+
+	if ((arg_size > 0) && copy_from_user(arg_copy, arg, arg_size)) {
+		cl_error("failed to copy ioctl data\n");
+		if (arg_size > (int)sizeof(buf)) {
+			kfree(arg_copy);
+			atomic_dec(&client->num_allocs);
+		}
+		LOG_EXT();
+		return -EFAULT;
+	}
+
+#define MODS_IOCTL(code, function, argtype)\
+	({\
+	do {\
+		cl_debug(DEBUG_IOCTL, "ioctl(" #code ")\n");\
+		if (arg_size != sizeof(struct argtype)) {\
+			err = -EINVAL;\
+			cl_error("invalid parameter passed to ioctl " #code\
+				 "\n");\
+		} else {\
+			err = function(client, (struct argtype *)arg_copy);\
+			if ((err == OK) && \
+			    copy_to_user(arg, arg_copy, arg_size)) {\
+				err = -EFAULT;\
+				cl_error("copying return value for ioctl " \
+					 #code " to user space failed\n");\
+			} \
+		} \
+	} while (0);\
+	})
+
+#define MODS_IOCTL_NORETVAL(code, function, argtype)\
+	({\
+	do {\
+		cl_debug(DEBUG_IOCTL, "ioctl(" #code ")\n");\
+		if (arg_size != sizeof(struct argtype)) {\
+			err = -EINVAL;\
+			cl_error("invalid parameter passed to ioctl " #code\
+				 "\n");\
+		} else {\
+			err = function(client, (struct argtype *)arg_copy);\
+		} \
+	} while (0);\
+	})
+
+#define MODS_IOCTL_VOID(code, function)\
+	({\
+	do {\
+		cl_debug(DEBUG_IOCTL, "ioctl(" #code ")\n");\
+		if (arg_size != 0) {\
+			err = -EINVAL;\
+			cl_error("invalid parameter passed to ioctl " #code\
+				 "\n");\
+		} else {\
+			err = function(client);\
+		} \
+	} while (0);\
+	})
+
+	switch (cmd) {
+
+#ifdef CONFIG_PCI
+	case MODS_ESC_FIND_PCI_DEVICE:
+		MODS_IOCTL(MODS_ESC_FIND_PCI_DEVICE,
+			   esc_mods_find_pci_dev, MODS_FIND_PCI_DEVICE);
+		break;
+
+	case MODS_ESC_FIND_PCI_DEVICE_2:
+		MODS_IOCTL(MODS_ESC_FIND_PCI_DEVICE_2,
+			   esc_mods_find_pci_dev_2,
+			   MODS_FIND_PCI_DEVICE_2);
+		break;
+
+	case MODS_ESC_FIND_PCI_CLASS_CODE:
+		MODS_IOCTL(MODS_ESC_FIND_PCI_CLASS_CODE,
+			   esc_mods_find_pci_class_code,
+			   MODS_FIND_PCI_CLASS_CODE);
+		break;
+
+	case MODS_ESC_FIND_PCI_CLASS_CODE_2:
+		MODS_IOCTL(MODS_ESC_FIND_PCI_CLASS_CODE_2,
+			   esc_mods_find_pci_class_code_2,
+			   MODS_FIND_PCI_CLASS_CODE_2);
+		break;
+
+	case MODS_ESC_PCI_GET_BAR_INFO:
+		MODS_IOCTL(MODS_ESC_PCI_GET_BAR_INFO,
+			   esc_mods_pci_get_bar_info,
+			   MODS_PCI_GET_BAR_INFO);
+		break;
+
+	case MODS_ESC_PCI_GET_BAR_INFO_2:
+		MODS_IOCTL(MODS_ESC_PCI_GET_BAR_INFO_2,
+			   esc_mods_pci_get_bar_info_2,
+			   MODS_PCI_GET_BAR_INFO_2);
+		break;
+
+	case MODS_ESC_PCI_GET_IRQ:
+		MODS_IOCTL(MODS_ESC_PCI_GET_IRQ,
+			   esc_mods_pci_get_irq,
+			   MODS_PCI_GET_IRQ);
+		break;
+
+	case MODS_ESC_PCI_GET_IRQ_2:
+		MODS_IOCTL(MODS_ESC_PCI_GET_IRQ_2,
+			   esc_mods_pci_get_irq_2,
+			   MODS_PCI_GET_IRQ_2);
+		break;
+
+	case MODS_ESC_PCI_READ:
+		MODS_IOCTL(MODS_ESC_PCI_READ, esc_mods_pci_read, MODS_PCI_READ);
+		break;
+
+	case MODS_ESC_PCI_READ_2:
+		MODS_IOCTL(MODS_ESC_PCI_READ_2,
+			   esc_mods_pci_read_2, MODS_PCI_READ_2);
+		break;
+
+	case MODS_ESC_PCI_WRITE:
+		MODS_IOCTL_NORETVAL(MODS_ESC_PCI_WRITE,
+				    esc_mods_pci_write, MODS_PCI_WRITE);
+		break;
+
+	case MODS_ESC_PCI_WRITE_2:
+		MODS_IOCTL_NORETVAL(MODS_ESC_PCI_WRITE_2,
+				    esc_mods_pci_write_2,
+				    MODS_PCI_WRITE_2);
+		break;
+
+	case MODS_ESC_PCI_BUS_RESCAN:
+		MODS_IOCTL_NORETVAL(MODS_ESC_PCI_BUS_RESCAN,
+				    esc_mods_pci_bus_rescan,
+				    MODS_PCI_BUS_RESCAN);
+		break;
+
+	case MODS_ESC_PCI_BUS_ADD_DEVICES:
+		MODS_IOCTL_NORETVAL(MODS_ESC_PCI_BUS_ADD_DEVICES,
+				    esc_mods_pci_bus_add_dev,
+				    MODS_PCI_BUS_ADD_DEVICES);
+		break;
+
+	case MODS_ESC_PCI_BUS_REMOVE_DEV:
+		MODS_IOCTL_NORETVAL(MODS_ESC_PCI_BUS_REMOVE_DEV,
+			   esc_mods_pci_bus_remove_dev,
+			   MODS_PCI_BUS_REMOVE_DEV);
+		break;
+
+	case MODS_ESC_PIO_READ:
+		MODS_IOCTL(MODS_ESC_PIO_READ,
+			   esc_mods_pio_read, MODS_PIO_READ);
+		break;
+
+	case MODS_ESC_PIO_WRITE:
+		MODS_IOCTL_NORETVAL(MODS_ESC_PIO_WRITE,
+				    esc_mods_pio_write, MODS_PIO_WRITE);
+		break;
+
+	case MODS_ESC_DEVICE_NUMA_INFO:
+		MODS_IOCTL(MODS_ESC_DEVICE_NUMA_INFO,
+			   esc_mods_device_numa_info,
+			   MODS_DEVICE_NUMA_INFO);
+		break;
+
+	case MODS_ESC_DEVICE_NUMA_INFO_2:
+		MODS_IOCTL(MODS_ESC_DEVICE_NUMA_INFO_2,
+			   esc_mods_device_numa_info_2,
+			   MODS_DEVICE_NUMA_INFO_2);
+		break;
+
+	case MODS_ESC_DEVICE_NUMA_INFO_3:
+		MODS_IOCTL(MODS_ESC_DEVICE_NUMA_INFO_3,
+			   esc_mods_device_numa_info_3,
+			   MODS_DEVICE_NUMA_INFO_3);
+		break;
+
+	case MODS_ESC_GET_IOMMU_STATE:
+		MODS_IOCTL(MODS_ESC_GET_IOMMU_STATE,
+			   esc_mods_get_iommu_state,
+			   MODS_GET_IOMMU_STATE);
+		break;
+
+	case MODS_ESC_GET_IOMMU_STATE_2:
+		MODS_IOCTL(MODS_ESC_GET_IOMMU_STATE_2,
+			   esc_mods_get_iommu_state_2,
+			   MODS_GET_IOMMU_STATE);
+		break;
+
+	case MODS_ESC_PCI_SET_DMA_MASK:
+		MODS_IOCTL(MODS_ESC_PCI_SET_DMA_MASK,
+			   esc_mods_pci_set_dma_mask,
+			   MODS_PCI_DMA_MASK);
+		break;
+
+	case MODS_ESC_PCI_RESET_FUNCTION:
+		MODS_IOCTL(MODS_ESC_PCI_RESET_FUNCTION,
+			   esc_mods_pci_reset_function,
+			   mods_pci_dev_2);
+		break;
+
+#ifdef MODS_HAS_DEV_PROPS
+	case MODS_ESC_READ_DEV_PROPERTY:
+		MODS_IOCTL(MODS_ESC_READ_DEV_PROPERTY,
+			   esc_mods_read_dev_property,
+			   MODS_READ_DEV_PROPERTY);
+		break;
+#endif
+#endif
+
+	case MODS_ESC_ALLOC_PAGES:
+		MODS_IOCTL(MODS_ESC_ALLOC_PAGES,
+			   esc_mods_alloc_pages, MODS_ALLOC_PAGES);
+		break;
+
+	case MODS_ESC_DEVICE_ALLOC_PAGES:
+		MODS_IOCTL(MODS_ESC_DEVICE_ALLOC_PAGES,
+			   esc_mods_device_alloc_pages,
+			   MODS_DEVICE_ALLOC_PAGES);
+		break;
+
+	case MODS_ESC_DEVICE_ALLOC_PAGES_2:
+		MODS_IOCTL(MODS_ESC_DEVICE_ALLOC_PAGES_2,
+			   esc_mods_device_alloc_pages_2,
+			   MODS_DEVICE_ALLOC_PAGES_2);
+		break;
+
+	case MODS_ESC_ALLOC_PAGES_2:
+		MODS_IOCTL(MODS_ESC_ALLOC_PAGES_2,
+			   esc_mods_alloc_pages_2,
+			   MODS_ALLOC_PAGES_2);
+		break;
+
+	case MODS_ESC_FREE_PAGES:
+		MODS_IOCTL(MODS_ESC_FREE_PAGES,
+			   esc_mods_free_pages, MODS_FREE_PAGES);
+		break;
+
+	case MODS_ESC_MERGE_PAGES:
+		MODS_IOCTL(MODS_ESC_MERGE_PAGES,
+			   esc_mods_merge_pages, MODS_MERGE_PAGES);
+		break;
+
+	case MODS_ESC_GET_PHYSICAL_ADDRESS:
+		MODS_IOCTL(MODS_ESC_GET_PHYSICAL_ADDRESS,
+			   esc_mods_get_phys_addr,
+			   MODS_GET_PHYSICAL_ADDRESS);
+		break;
+
+	case MODS_ESC_GET_PHYSICAL_ADDRESS_2:
+		MODS_IOCTL(MODS_ESC_GET_PHYSICAL_ADDRESS_2,
+			   esc_mods_get_phys_addr_2,
+			   MODS_GET_PHYSICAL_ADDRESS_3);
+		break;
+
+	case MODS_ESC_GET_MAPPED_PHYSICAL_ADDRESS:
+		MODS_IOCTL(MODS_ESC_GET_MAPPED_PHYSICAL_ADDRESS,
+			   esc_mods_get_mapped_phys_addr,
+			   MODS_GET_PHYSICAL_ADDRESS);
+		break;
+
+	case MODS_ESC_GET_MAPPED_PHYSICAL_ADDRESS_2:
+		MODS_IOCTL(MODS_ESC_GET_MAPPED_PHYSICAL_ADDRESS_2,
+			   esc_mods_get_mapped_phys_addr_2,
+			   MODS_GET_PHYSICAL_ADDRESS_2);
+		break;
+
+	case MODS_ESC_GET_MAPPED_PHYSICAL_ADDRESS_3:
+		MODS_IOCTL(MODS_ESC_GET_MAPPED_PHYSICAL_ADDRESS_3,
+			   esc_mods_get_mapped_phys_addr_3,
+			   MODS_GET_PHYSICAL_ADDRESS_3);
+		break;
+
+	case MODS_ESC_SET_MEMORY_TYPE:
+		MODS_IOCTL_NORETVAL(MODS_ESC_SET_MEMORY_TYPE,
+				    esc_mods_set_mem_type,
+				    MODS_MEMORY_TYPE);
+		break;
+
+	case MODS_ESC_VIRTUAL_TO_PHYSICAL:
+		MODS_IOCTL(MODS_ESC_VIRTUAL_TO_PHYSICAL,
+			   esc_mods_virtual_to_phys,
+			   MODS_VIRTUAL_TO_PHYSICAL);
+		break;
+
+	case MODS_ESC_PHYSICAL_TO_VIRTUAL:
+		MODS_IOCTL(MODS_ESC_PHYSICAL_TO_VIRTUAL,
+			   esc_mods_phys_to_virtual, MODS_PHYSICAL_TO_VIRTUAL);
+		break;
+
+#if defined(CONFIG_PPC64)
+	case MODS_ESC_PCI_HOT_RESET:
+		MODS_IOCTL_NORETVAL(MODS_ESC_PCI_HOT_RESET,
+				    esc_mods_pci_hot_reset,
+				    MODS_PCI_HOT_RESET);
+		break;
+
+	case MODS_ESC_SET_PPC_TCE_BYPASS:
+		MODS_IOCTL(MODS_ESC_SET_PPC_TCE_BYPASS,
+			   esc_mods_set_ppc_tce_bypass,
+			   MODS_SET_PPC_TCE_BYPASS);
+		break;
+
+	case MODS_ESC_GET_ATS_ADDRESS_RANGE:
+		MODS_IOCTL(MODS_ESC_GET_ATS_ADDRESS_RANGE,
+			   esc_mods_get_ats_address_range,
+			   MODS_GET_ATS_ADDRESS_RANGE);
+		break;
+
+	case MODS_ESC_SET_NVLINK_SYSMEM_TRAINED:
+		MODS_IOCTL(MODS_ESC_SET_NVLINK_SYSMEM_TRAINED,
+			   esc_mods_set_nvlink_sysmem_trained,
+			   MODS_SET_NVLINK_SYSMEM_TRAINED);
+		break;
+
+	case MODS_ESC_GET_NVLINK_LINE_RATE:
+		MODS_IOCTL(MODS_ESC_GET_NVLINK_LINE_RATE,
+			   esc_mods_get_nvlink_line_rate,
+			   MODS_GET_NVLINK_LINE_RATE);
+		break;
+#endif
+
+#ifdef CONFIG_PCI
+	case MODS_ESC_DMA_MAP_MEMORY:
+		MODS_IOCTL(MODS_ESC_DMA_MAP_MEMORY,
+			   esc_mods_dma_map_memory,
+			   MODS_DMA_MAP_MEMORY);
+		break;
+
+	case MODS_ESC_DMA_UNMAP_MEMORY:
+		MODS_IOCTL(MODS_ESC_DMA_UNMAP_MEMORY,
+			   esc_mods_dma_unmap_memory,
+			   MODS_DMA_MAP_MEMORY);
+		break;
+#endif
+
+	case MODS_ESC_IRQ_REGISTER:
+	case MODS_ESC_MSI_REGISTER:
+		err = -EINVAL;
+		break;
+
+#if defined(MODS_HAS_TEGRA) && defined(CONFIG_OF) && defined(CONFIG_OF_IRQ)
+	case MODS_ESC_MAP_INTERRUPT:
+		MODS_IOCTL(MODS_ESC_MAP_INTERRUPT,
+				esc_mods_map_irq, MODS_DT_INFO);
+		break;
+
+	case MODS_ESC_MAP_GPIO:
+		MODS_IOCTL(MODS_ESC_MAP_GPIO,
+		esc_mods_map_irq_to_gpio, MODS_GPIO_INFO);
+		break;
+#endif
+
+	case MODS_ESC_REGISTER_IRQ:
+		MODS_IOCTL_NORETVAL(MODS_ESC_REGISTER_IRQ,
+				esc_mods_register_irq, MODS_REGISTER_IRQ);
+		break;
+
+	case MODS_ESC_REGISTER_IRQ_2:
+		MODS_IOCTL_NORETVAL(MODS_ESC_REGISTER_IRQ_2,
+				esc_mods_register_irq_2, MODS_REGISTER_IRQ_2);
+		break;
+
+	case MODS_ESC_REGISTER_IRQ_3:
+		MODS_IOCTL_NORETVAL(MODS_ESC_REGISTER_IRQ_3,
+				esc_mods_register_irq_3, MODS_REGISTER_IRQ_3);
+		break;
+
+	case MODS_ESC_UNREGISTER_IRQ:
+		MODS_IOCTL_NORETVAL(MODS_ESC_UNREGISTER_IRQ,
+				    esc_mods_unregister_irq, MODS_REGISTER_IRQ);
+		break;
+
+	case MODS_ESC_UNREGISTER_IRQ_2:
+		MODS_IOCTL_NORETVAL(MODS_ESC_UNREGISTER_IRQ_2,
+				    esc_mods_unregister_irq_2,
+				    MODS_REGISTER_IRQ_2);
+		break;
+
+	case MODS_ESC_QUERY_IRQ:
+		MODS_IOCTL(MODS_ESC_QUERY_IRQ,
+			   esc_mods_query_irq, MODS_QUERY_IRQ);
+		break;
+
+	case MODS_ESC_QUERY_IRQ_2:
+		MODS_IOCTL(MODS_ESC_QUERY_IRQ_2,
+			   esc_mods_query_irq_2, MODS_QUERY_IRQ_2);
+		break;
+
+	case MODS_ESC_IRQ_HANDLED:
+		MODS_IOCTL_NORETVAL(MODS_ESC_IRQ_HANDLED,
+				    esc_mods_irq_handled, MODS_REGISTER_IRQ);
+		break;
+
+	case MODS_ESC_IRQ_HANDLED_2:
+		MODS_IOCTL_NORETVAL(MODS_ESC_IRQ_HANDLED_2,
+				    esc_mods_irq_handled_2,
+				    MODS_REGISTER_IRQ_2);
+		break;
+
+#ifdef CONFIG_ACPI
+	case MODS_ESC_EVAL_ACPI_METHOD:
+		MODS_IOCTL(MODS_ESC_EVAL_ACPI_METHOD,
+			   esc_mods_eval_acpi_method, MODS_EVAL_ACPI_METHOD);
+		break;
+
+	case MODS_ESC_EVAL_DEV_ACPI_METHOD:
+		MODS_IOCTL(MODS_ESC_EVAL_DEV_ACPI_METHOD,
+			   esc_mods_eval_dev_acpi_method,
+			   MODS_EVAL_DEV_ACPI_METHOD);
+		break;
+
+	case MODS_ESC_EVAL_DEV_ACPI_METHOD_2:
+		MODS_IOCTL(MODS_ESC_EVAL_DEV_ACPI_METHOD_2,
+			   esc_mods_eval_dev_acpi_method_2,
+			   MODS_EVAL_DEV_ACPI_METHOD_2);
+		break;
+
+	case MODS_ESC_EVAL_DEV_ACPI_METHOD_3:
+		MODS_IOCTL(MODS_ESC_EVAL_DEV_ACPI_METHOD_3,
+			   esc_mods_eval_dev_acpi_method_3,
+			   MODS_EVAL_DEV_ACPI_METHOD_3);
+		break;
+
+	case MODS_ESC_ACPI_GET_DDC:
+		MODS_IOCTL(MODS_ESC_ACPI_GET_DDC,
+			   esc_mods_acpi_get_ddc, MODS_ACPI_GET_DDC);
+		break;
+
+	case MODS_ESC_ACPI_GET_DDC_2:
+		MODS_IOCTL(MODS_ESC_ACPI_GET_DDC_2,
+			   esc_mods_acpi_get_ddc_2, MODS_ACPI_GET_DDC_2);
+		break;
+
+	case MODS_ESC_GET_ACPI_DEV_CHILDREN:
+		MODS_IOCTL(MODS_ESC_GET_ACPI_DEV_CHILDREN,
+			   esc_mods_get_acpi_dev_children,
+			   MODS_GET_ACPI_DEV_CHILDREN);
+		break;
+
+#ifdef MODS_HAS_PXM_TO_NODE
+	case MODS_ESC_PROXIMITY_TO_NUMA_NODE:
+		MODS_IOCTL(MODS_ESC_PROXIMITY_TO_NUMA_NODE,
+			   esc_mods_proximity_to_numa_node,
+			   MODS_PROXIMITY_TO_NUMA_NODE);
+		break;
+#endif
+#else
+	case MODS_ESC_EVAL_ACPI_METHOD:
+		/* fallthrough */
+	case MODS_ESC_EVAL_DEV_ACPI_METHOD:
+		/* fallthrough */
+	case MODS_ESC_EVAL_DEV_ACPI_METHOD_2:
+		/* fallthrough */
+	case MODS_ESC_EVAL_DEV_ACPI_METHOD_3:
+		/* fallthrough */
+	case MODS_ESC_ACPI_GET_DDC:
+		/* fallthrough */
+	case MODS_ESC_ACPI_GET_DDC_2:
+		/* fallthrough */
+	case MODS_ESC_GET_ACPI_DEV_CHILDREN:
+		/* Silent failure to avoid clogging kernel log */
+		err = -EINVAL;
+		break;
+#endif
+	case MODS_ESC_GET_API_VERSION:
+		MODS_IOCTL(MODS_ESC_GET_API_VERSION,
+			   esc_mods_get_api_version, MODS_GET_VERSION);
+		break;
+
+	case MODS_ESC_GET_KERNEL_VERSION:
+		MODS_IOCTL(MODS_ESC_GET_KERNEL_VERSION,
+			   esc_mods_get_kernel_version, MODS_GET_VERSION);
+		break;
+
+#if defined(MODS_HAS_TEGRA) && defined(CONFIG_COMMON_CLK) && \
+	defined(CONFIG_OF_RESOLVE) && defined(CONFIG_OF_DYNAMIC)
+	case MODS_ESC_GET_CLOCK_HANDLE:
+		MODS_IOCTL(MODS_ESC_GET_CLOCK_HANDLE,
+			   esc_mods_get_clock_handle, MODS_GET_CLOCK_HANDLE);
+		break;
+
+	case MODS_ESC_SET_CLOCK_RATE:
+		MODS_IOCTL_NORETVAL(MODS_ESC_SET_CLOCK_RATE,
+				    esc_mods_set_clock_rate, MODS_CLOCK_RATE);
+		break;
+
+	case MODS_ESC_GET_CLOCK_RATE:
+		MODS_IOCTL(MODS_ESC_GET_CLOCK_RATE,
+			   esc_mods_get_clock_rate, MODS_CLOCK_RATE);
+		break;
+
+	case MODS_ESC_GET_CLOCK_MAX_RATE:
+		MODS_IOCTL(MODS_ESC_GET_CLOCK_MAX_RATE,
+			   esc_mods_get_clock_max_rate, MODS_CLOCK_RATE);
+		break;
+
+	case MODS_ESC_SET_CLOCK_MAX_RATE:
+		MODS_IOCTL_NORETVAL(MODS_ESC_SET_CLOCK_MAX_RATE,
+				    esc_mods_set_clock_max_rate,
+				    MODS_CLOCK_RATE);
+		break;
+
+	case MODS_ESC_SET_CLOCK_PARENT:
+		MODS_IOCTL_NORETVAL(MODS_ESC_SET_CLOCK_PARENT,
+				    esc_mods_set_clock_parent,
+				    MODS_CLOCK_PARENT);
+		break;
+
+	case MODS_ESC_GET_CLOCK_PARENT:
+		MODS_IOCTL(MODS_ESC_GET_CLOCK_PARENT,
+			   esc_mods_get_clock_parent, MODS_CLOCK_PARENT);
+		break;
+
+	case MODS_ESC_ENABLE_CLOCK:
+		MODS_IOCTL_NORETVAL(MODS_ESC_ENABLE_CLOCK,
+				    esc_mods_enable_clock, MODS_CLOCK_HANDLE);
+		break;
+
+	case MODS_ESC_DISABLE_CLOCK:
+		MODS_IOCTL_NORETVAL(MODS_ESC_DISABLE_CLOCK,
+				    esc_mods_disable_clock, MODS_CLOCK_HANDLE);
+		break;
+
+	case MODS_ESC_IS_CLOCK_ENABLED:
+		MODS_IOCTL(MODS_ESC_IS_CLOCK_ENABLED,
+			   esc_mods_is_clock_enabled, MODS_CLOCK_ENABLED);
+		break;
+
+	case MODS_ESC_CLOCK_RESET_ASSERT:
+		MODS_IOCTL_NORETVAL(MODS_ESC_CLOCK_RESET_ASSERT,
+				    esc_mods_clock_reset_assert,
+				    MODS_CLOCK_HANDLE);
+		break;
+
+	case MODS_ESC_CLOCK_RESET_DEASSERT:
+		MODS_IOCTL_NORETVAL(MODS_ESC_CLOCK_RESET_DEASSERT,
+				    esc_mods_clock_reset_deassert,
+				    MODS_CLOCK_HANDLE);
+		break;
+
+	case MODS_ESC_RESET_ASSERT:
+		MODS_IOCTL_NORETVAL(MODS_ESC_RESET_ASSERT,
+				    esc_mods_reset_assert,
+				    MODS_RESET_HANDLE);
+		break;
+
+	case MODS_ESC_GET_RESET_HANDLE:
+		MODS_IOCTL(MODS_ESC_GET_RESET_HANDLE,
+			   esc_mods_get_rst_handle,
+			   MODS_GET_RESET_HANDLE);
+		break;
+#endif
+#if defined(MODS_HAS_PROD)
+	case MODS_ESC_BPMP_SET_PCIE_STATE:
+		MODS_IOCTL(MODS_ESC_BPMP_SET_PCIE_STATE,
+			   esc_mods_bpmp_set_pcie_state,
+			   MODS_SET_PCIE_STATE);
+		break;
+
+	case MODS_ESC_BPMP_INIT_PCIE_EP_PLL:
+		MODS_IOCTL(MODS_ESC_BPMP_INIT_PCIE_EP_PLL,
+			   esc_mods_bpmp_init_pcie_ep_pll,
+			   MODS_INIT_PCIE_EP_PLL);
+		break;
+#endif
+#if defined(MODS_HAS_TEGRA)
+	case MODS_ESC_DMA_ALLOC_COHERENT:
+		MODS_IOCTL(MODS_ESC_DMA_ALLOC_COHERENT,
+			   esc_mods_dma_alloc_coherent,
+			   MODS_DMA_COHERENT_MEM_HANDLE);
+		break;
+	case MODS_ESC_DMA_FREE_COHERENT:
+		MODS_IOCTL(MODS_ESC_DMA_FREE_COHERENT,
+			   esc_mods_dma_free_coherent,
+			   MODS_DMA_COHERENT_MEM_HANDLE);
+		break;
+	case MODS_ESC_DMA_COPY_TO_USER:
+		MODS_IOCTL(MODS_ESC_DMA_COPY_TO_USER,
+			   esc_mods_dma_copy_to_user,
+			   MODS_DMA_COPY_TO_USER);
+		break;
+	case MODS_ESC_IOMMU_DMA_MAP_MEMORY:
+		MODS_IOCTL(MODS_ESC_IOMMU_DMA_MAP_MEMORY,
+			   esc_mods_iommu_dma_map_memory,
+			   MODS_IOMMU_DMA_MAP_MEMORY);
+		break;
+
+	case MODS_ESC_IOMMU_DMA_UNMAP_MEMORY:
+		MODS_IOCTL(MODS_ESC_IOMMU_DMA_UNMAP_MEMORY,
+			   esc_mods_iommu_dma_unmap_memory,
+			   MODS_IOMMU_DMA_MAP_MEMORY);
+		break;
+#if defined(CONFIG_DMA_ENGINE)
+	case MODS_ESC_DMA_REQUEST_HANDLE:
+		MODS_IOCTL(MODS_ESC_DMA_REQUEST_HANDLE,
+			   esc_mods_dma_request_channel,
+			   MODS_DMA_HANDLE);
+		break;
+	case MODS_ESC_DMA_REQUEST_HANDLE_2:
+		MODS_IOCTL(MODS_ESC_DMA_REQUEST_HANDLE_2,
+			   esc_mods_dma_request_channel_2,
+			   MODS_DMA_HANDLE_2);
+		break;
+	case MODS_ESC_DMA_RELEASE_HANDLE:
+		MODS_IOCTL_NORETVAL(MODS_ESC_DMA_RELEASE_HANDLE,
+			   esc_mods_dma_release_channel,
+			   MODS_DMA_HANDLE);
+		break;
+	case MODS_ESC_DMA_ISSUE_PENDING:
+		MODS_IOCTL_NORETVAL(MODS_ESC_DMA_ISSUE_PENDING,
+				    esc_mods_dma_async_issue_pending,
+				    MODS_DMA_HANDLE);
+		break;
+	case MODS_ESC_DMA_SET_CONFIG:
+		MODS_IOCTL_NORETVAL(MODS_ESC_DMA_SET_CONFIG,
+				    esc_mods_dma_set_config,
+				    MODS_DMA_CHANNEL_CONFIG);
+		break;
+	case MODS_ESC_DMA_TX_SUBMIT:
+		MODS_IOCTL(MODS_ESC_DMA_TX_SUBMIT,
+			   esc_mods_dma_submit_request,
+			   MODS_DMA_TX_DESC);
+		break;
+	case MODS_ESC_DMA_TX_WAIT:
+		MODS_IOCTL(MODS_MODS_ESC_DMA_TX_WAIT,
+			   esc_mods_dma_wait,
+			   MODS_DMA_WAIT_DESC);
+		break;
+#endif
+#ifdef CONFIG_TEGRA_DC
+	case MODS_ESC_TEGRA_DC_CONFIG_POSSIBLE:
+		MODS_IOCTL(MODS_ESC_TEGRA_DC_CONFIG_POSSIBLE,
+				   esc_mods_tegra_dc_config_possible,
+				   MODS_TEGRA_DC_CONFIG_POSSIBLE);
+		break;
+#endif
+#if defined(MODS_HAS_TEGRA) && defined(CONFIG_NET)
+	case MODS_ESC_NET_FORCE_LINK:
+		MODS_IOCTL(MODS_ESC_NET_FORCE_LINK,
+			   esc_mods_net_force_link, MODS_NET_DEVICE_NAME);
+		break;
+#endif
+#endif
+#ifdef CONFIG_ARM
+	case MODS_ESC_MEMORY_BARRIER:
+		MODS_IOCTL_VOID(MODS_ESC_MEMORY_BARRIER,
+				esc_mods_memory_barrier);
+		break;
+#endif
+#ifdef CONFIG_ARM64
+	case MODS_ESC_FLUSH_CPU_CACHE_RANGE:
+		MODS_IOCTL_NORETVAL(MODS_ESC_FLUSH_CPU_CACHE_RANGE,
+				    esc_mods_flush_cpu_cache_range,
+				    MODS_FLUSH_CPU_CACHE_RANGE);
+		break;
+#endif
+#if defined(MODS_HAS_DMABUF)
+	case MODS_ESC_DMABUF_GET_PHYSICAL_ADDRESS:
+		MODS_IOCTL(MODS_ESC_DMABUF_GET_PHYSICAL_ADDRESS,
+			   esc_mods_dmabuf_get_phys_addr,
+			   MODS_DMABUF_GET_PHYSICAL_ADDRESS);
+		break;
+#endif
+#ifdef CONFIG_TEGRA_NVADSP
+	case MODS_ESC_ADSP_LOAD:
+		MODS_IOCTL_VOID(MODS_ESC_ADSP_LOAD,
+				esc_mods_adsp_load);
+		break;
+
+	case MODS_ESC_ADSP_START:
+		MODS_IOCTL_VOID(MODS_ESC_ADSP_START,
+				esc_mods_adsp_start);
+		break;
+
+	case MODS_ESC_ADSP_STOP:
+		MODS_IOCTL_VOID(MODS_ESC_ADSP_STOP,
+				esc_mods_adsp_stop);
+		break;
+
+	case MODS_ESC_ADSP_RUN_APP:
+		MODS_IOCTL_NORETVAL(MODS_ESC_ADSP_RUN_APP,
+				    esc_mods_adsp_run_app,
+				    MODS_ADSP_RUN_APP_INFO);
+		break;
+#endif
+
+#ifdef CONFIG_X86
+	case MODS_ESC_GET_SCREEN_INFO:
+		MODS_IOCTL(MODS_ESC_GET_SCREEN_INFO,
+			   esc_mods_get_screen_info, MODS_SCREEN_INFO);
+		break;
+	case MODS_ESC_GET_SCREEN_INFO_2:
+		MODS_IOCTL(MODS_ESC_GET_SCREEN_INFO_2,
+			   esc_mods_get_screen_info_2, MODS_SCREEN_INFO_2);
+		break;
+#endif
+
+#if defined(MODS_HAS_CONSOLE_LOCK)
+	case MODS_ESC_LOCK_CONSOLE:
+		MODS_IOCTL_VOID(MODS_ESC_LOCK_CONSOLE,
+			   esc_mods_lock_console);
+		break;
+	case MODS_ESC_UNLOCK_CONSOLE:
+		MODS_IOCTL_VOID(MODS_ESC_UNLOCK_CONSOLE,
+			   esc_mods_unlock_console);
+		break;
+	case MODS_ESC_SUSPEND_CONSOLE:
+		MODS_IOCTL_VOID(MODS_ESC_SUSPEND_CONSOLE,
+			   esc_mods_suspend_console);
+		break;
+	case MODS_ESC_RESUME_CONSOLE:
+		MODS_IOCTL_VOID(MODS_ESC_RESUME_CONSOLE,
+			   esc_mods_resume_console);
+		break;
+#endif
+
+#if defined(MODS_HAS_PROD)
+	case MODS_ESC_TEGRA_PROD_IS_SUPPORTED:
+		MODS_IOCTL(MODS_ESC_TEGRA_PROD_IS_SUPPORTED,
+			   esc_mods_tegra_prod_is_supported,
+			   MODS_TEGRA_PROD_IS_SUPPORTED);
+		break;
+
+	case MODS_ESC_TEGRA_PROD_SET_PROD_ALL:
+		MODS_IOCTL_NORETVAL(MODS_ESC_TEGRA_PROD_SET_PROD_ALL,
+				    esc_mods_tegra_prod_set_prod_all,
+				    MODS_TEGRA_PROD_SET_TUPLE);
+		break;
+
+	case MODS_ESC_TEGRA_PROD_SET_PROD_BOOT:
+		MODS_IOCTL_NORETVAL(MODS_ESC_TEGRA_PROD_SET_PROD_BOOT,
+				    esc_mods_tegra_prod_set_prod_boot,
+				    MODS_TEGRA_PROD_SET_TUPLE);
+		break;
+
+	case MODS_ESC_TEGRA_PROD_SET_PROD_BY_NAME:
+		MODS_IOCTL_NORETVAL(MODS_ESC_TEGRA_PROD_SET_PROD_BY_NAME,
+				    esc_mods_tegra_prod_set_prod_by_name,
+				    MODS_TEGRA_PROD_SET_TUPLE);
+		break;
+
+	case MODS_ESC_TEGRA_PROD_SET_PROD_EXACT:
+		MODS_IOCTL_NORETVAL(MODS_ESC_TEGRA_PROD_SET_PROD_EXACT,
+				    esc_mods_tegra_prod_set_prod_exact,
+				    MODS_TEGRA_PROD_SET_TUPLE);
+		break;
+
+	case MODS_ESC_TEGRA_PROD_ITERATE_DT:
+		MODS_IOCTL(MODS_ESC_TEGRA_PROD_ITERATE_DT,
+			   esc_mods_tegra_prod_iterate_dt,
+			   MODS_TEGRA_PROD_ITERATOR);
+		break;
+#endif
+
+#if defined(MODS_HAS_TEGRA)
+
+#ifdef CONFIG_TRUSTY
+	case MODS_ESC_SEND_TZ_MSG:
+		MODS_IOCTL(MODS_ESC_SEND_TZ_MSG,
+			esc_mods_send_trustzone_msg, MODS_TZ_PARAMS);
+		break;
+#endif
+
+#ifdef CONFIG_OPTEE
+	case MODS_ESC_INVOKE_OPTEE_TA:
+		MODS_IOCTL(MODS_ESC_INVOKE_OPTEE_TA,
+			esc_mods_invoke_optee_ta, MODS_OPTEE_PARAMS);
+		break;
+#endif
+
+	case MODS_ESC_OIST_STATUS:
+		MODS_IOCTL(MODS_ESC_OIST_STATUS,
+			   esc_mods_oist_status, MODS_TEGRA_OIST_STATUS);
+		break;
+
+	case MODS_ESC_MODS_SEND_IPI:
+		MODS_IOCTL(MODS_ESC_MODS_SEND_IPI,
+			   esc_mods_send_ipi, MODS_SEND_IPI);
+		break;
+
+#endif
+
+#if IS_BUILTIN(CONFIG_ARM_FFA_TRANSPORT)
+	case MODS_ESC_FFA_CMD:
+		MODS_IOCTL(MODS_ESC_FFA_CMD, esc_mods_arm_ffa_cmd, MODS_FFA_PARAMS);
+		break;
+#endif
+
+	case MODS_ESC_ACQUIRE_ACCESS_TOKEN:
+		MODS_IOCTL(MODS_ESC_ACQUIRE_ACCESS_TOKEN,
+			   esc_mods_acquire_access_token,
+			   MODS_ACCESS_TOKEN);
+		break;
+
+	case MODS_ESC_RELEASE_ACCESS_TOKEN:
+		MODS_IOCTL_NORETVAL(MODS_ESC_RELEASE_ACCESS_TOKEN,
+				    esc_mods_release_access_token,
+				    MODS_ACCESS_TOKEN);
+		break;
+
+	case MODS_ESC_VERIFY_ACCESS_TOKEN:
+		MODS_IOCTL_NORETVAL(MODS_ESC_VERIFY_ACCESS_TOKEN,
+				    esc_mods_verify_access_token,
+				    MODS_ACCESS_TOKEN);
+		break;
+
+	case MODS_ESC_WRITE_SYSFS_NODE:
+		MODS_IOCTL_NORETVAL(MODS_ESC_WRITE_SYSFS_NODE,
+				    esc_mods_write_sysfs_node,
+				    MODS_SYSFS_NODE);
+		break;
+
+	case MODS_ESC_SYSCTL_WRITE_INT:
+		MODS_IOCTL_NORETVAL(MODS_ESC_SYSCTL_WRITE_INT,
+				    esc_mods_sysctl_write_int,
+				    MODS_SYSCTL_INT);
+		break;
+
+	case MODS_ESC_REGISTER_IRQ_4:
+		MODS_IOCTL_NORETVAL(MODS_ESC_REGISTER_IRQ_4,
+				esc_mods_register_irq_4, MODS_REGISTER_IRQ_4);
+		break;
+
+	case MODS_ESC_QUERY_IRQ_3:
+		MODS_IOCTL(MODS_ESC_QUERY_IRQ_3,
+			   esc_mods_query_irq_3, MODS_QUERY_IRQ_3);
+		break;
+
+#if defined(CONFIG_PCI) && defined(MODS_HAS_SRIOV)
+	case MODS_ESC_SET_NUM_VF:
+		MODS_IOCTL_NORETVAL(MODS_ESC_SET_NUM_VF,
+			   esc_mods_set_num_vf, MODS_SET_NUM_VF);
+		break;
+
+	case MODS_ESC_SET_TOTAL_VF:
+		MODS_IOCTL_NORETVAL(MODS_ESC_SET_TOTAL_VF,
+			   esc_mods_set_total_vf, MODS_SET_NUM_VF);
+		break;
+#endif
+
+#ifdef CONFIG_X86
+	case MODS_ESC_READ_MSR:
+		MODS_IOCTL(MODS_ESC_READ_MSR,
+			   esc_mods_read_msr, MODS_MSR);
+		break;
+
+	case MODS_ESC_WRITE_MSR:
+		MODS_IOCTL_NORETVAL(MODS_ESC_WRITE_MSR,
+			   esc_mods_write_msr, MODS_MSR);
+		break;
+#endif
+
+	case MODS_ESC_MODS_GET_DRIVER_STATS:
+		MODS_IOCTL(MODS_ESC_MODS_GET_DRIVER_STATS,
+			   esc_mods_get_driver_stats, MODS_GET_DRIVER_STATS);
+		break;
+
+#ifdef CONFIG_TEGRA_IVC
+	case MODS_ESC_BPMP_UPHY_LANE_EOM_SCAN:
+		MODS_IOCTL(MODS_ESC_BPMP_UPHY_LANE_EOM_SCAN,
+			   esc_mods_bpmp_uphy_lane_eom_scan,
+			   MODS_BPMP_UPHY_LANE_EOM_SCAN_PARAMS);
+		break;
+#endif
+
+	default:
+		cl_error(
+			"unrecognized ioctl 0x%x, dir %u, type 0x%x, nr %u, size 0x%x\n",
+			cmd,
+			_IOC_DIR(cmd),
+			_IOC_TYPE(cmd),
+			_IOC_NR(cmd),
+			_IOC_SIZE(cmd));
+		err = -EINVAL;
+		break;
+	}
+
+	if (arg_size > (int)sizeof(buf)) {
+		kfree(arg_copy);
+		atomic_dec(&client->num_allocs);
+	}
+
+	LOG_EXT();
+	return err;
+}
diff --git a/drivers/misc/mods/mods_mem.c b/drivers/misc/mods/mods_mem.c
new file mode 100644
index 00000000..65530f20
--- /dev/null
+++ b/drivers/misc/mods/mods_mem.c
@@ -0,0 +1,2568 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * This file is part of NVIDIA MODS kernel driver.
+ *
+ * Copyright (c) 2008-2023, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * NVIDIA MODS kernel driver is free software: you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * NVIDIA MODS kernel driver is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NVIDIA MODS kernel driver.
+ * If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "mods_internal.h"
+
+#include <linux/bitops.h>
+#include <linux/pagemap.h>
+#include <linux/sched.h>
+
+#if defined(MODS_HAS_SET_DMA_MASK)
+#include <linux/dma-mapping.h>
+#include <linux/of.h>
+#endif
+
+#ifdef CONFIG_ARM64
+#include <linux/cache.h>
+#endif
+
+static struct MODS_MEM_INFO *get_mem_handle(struct mods_client *client,
+					    u64                 handle)
+{
+	/* For now just check if we hit first or last page, i.e. if
+	 * we have a valid pointer.  In the future, add proper handle
+	 * accounting.
+	 */
+	if (unlikely((handle + PAGE_SIZE) < (2 * PAGE_SIZE))) {
+		cl_error("invalid memory handle 0x%llx\n",
+			 (unsigned long long)handle);
+		return NULL;
+	}
+
+	return (struct MODS_MEM_INFO *)(size_t)handle;
+}
+
+static bool validate_mem_handle(struct mods_client   *client,
+				struct MODS_MEM_INFO *p_mem_info)
+{
+	struct list_head *head = &client->mem_alloc_list;
+	struct list_head *iter;
+
+	if (unlikely(!p_mem_info))
+		return false;
+
+	list_for_each(iter, head) {
+		struct MODS_MEM_INFO *p_mem =
+			list_entry(iter, struct MODS_MEM_INFO, list);
+
+		if (p_mem == p_mem_info)
+			return true;
+	}
+
+	return false;
+}
+
+/****************************
+ * DMA MAP HELPER FUNCTIONS *
+ ****************************/
+
+/*
+ * Starting on Power9 systems, DMA addresses for NVLink are no longer
+ * the same as used over PCIE.
+ *
+ * Power9 supports a 56-bit Real Address. This address range is compressed
+ * when accessed over NvLink to allow the GPU to access all of memory using
+ * its 47-bit Physical address.
+ *
+ * If there is an NPU device present on the system, it implies that NvLink
+ * sysmem links are present and we need to apply the required address
+ * conversion for NvLink within the driver. This is intended to be temporary
+ * to ease the transition to kernel APIs to handle NvLink DMA mappings
+ * via the NPU device.
+ *
+ * Note, a deviation from the documented compression scheme is that the
+ * upper address bits (i.e. bit 56-63) instead of being set to zero are
+ * preserved during NvLink address compression so the orignal PCIE DMA
+ * address can be reconstructed on expansion. These bits can be safely
+ * ignored on NvLink since they are truncated by the GPU.
+ */
+#if defined(CONFIG_PPC64) && defined(CONFIG_PCI)
+static dma_addr_t compress_nvlink_addr(struct pci_dev *dev, dma_addr_t addr)
+{
+	dma_addr_t addr47 = addr;
+
+	/* Note, one key difference from the documented compression scheme
+	 * is that BIT59 used for TCE bypass mode on PCIe is preserved during
+	 * NVLink address compression to allow for the resulting DMA address to
+	 * be used transparently on PCIe.
+	 */
+	if (dev && has_npu_dev(dev, 0)) {
+		addr47 = addr & (1LLU << 59);
+		addr47 |= ((addr >> 45) & 0x3) << 43;
+		addr47 |= ((addr >> 49) & 0x3) << 45;
+		addr47 |= addr & ((1LLU << 43) - 1);
+	}
+
+	return addr47;
+}
+#else
+#define compress_nvlink_addr(dev, addr) (addr)
+#endif
+
+static void copy_wc_bitmap(struct MODS_MEM_INFO *p_dest_mem_info,
+			   unsigned long         first_dst_chunk,
+			   struct MODS_MEM_INFO *p_src_mem_info,
+			   unsigned long         num_chunks)
+{
+	unsigned long src_pos = 0;
+
+	WARN_ON(p_dest_mem_info->cache_type != p_src_mem_info->cache_type);
+
+	if (p_src_mem_info->cache_type == MODS_ALLOC_CACHED)
+		return;
+
+	WARN_ON(!p_dest_mem_info->wc_bitmap);
+	WARN_ON(!p_src_mem_info->wc_bitmap);
+
+	for (;;) {
+		src_pos = find_next_bit(p_src_mem_info->wc_bitmap,
+					num_chunks,
+					src_pos);
+
+		if (src_pos >= num_chunks)
+			break;
+
+		set_bit(src_pos + first_dst_chunk, p_dest_mem_info->wc_bitmap);
+
+		++src_pos;
+	}
+}
+
+static inline bool is_chunk_wc(struct MODS_MEM_INFO *p_mem_info, u32 ichunk)
+{
+	return p_mem_info->wc_bitmap && test_bit(ichunk, p_mem_info->wc_bitmap);
+}
+
+static void mark_chunk_wc(struct MODS_MEM_INFO *p_mem_info, u32 ichunk)
+{
+	WARN_ON(p_mem_info->cache_type == MODS_ALLOC_CACHED);
+	WARN_ON(!p_mem_info->wc_bitmap);
+	set_bit(ichunk, p_mem_info->wc_bitmap);
+}
+
+static void print_map_info(struct mods_client *client,
+			   const char         *action,
+			   struct scatterlist *sg,
+			   u32                 nents,
+			   struct device      *dev)
+{
+	u32 i;
+
+	for_each_sg(sg, sg, nents, i) {
+		cl_debug(DEBUG_MEM_DETAILED,
+			 "dma %s iova=0x%llx dma_len=0x%x phys=0x%llx size=0x%x on dev %s\n",
+			 action,
+			 (unsigned long long)sg_dma_address(sg),
+			 sg_dma_len(sg),
+			 (unsigned long long)sg_phys(sg),
+			 sg->length,
+			 dev_name(dev));
+	}
+}
+
+static int map_sg(struct mods_client *client,
+		  struct device      *dev,
+		  struct scatterlist *sg,
+		  u32                 num_chunks,
+		  u32                 num_pages)
+{
+	const u32 max_pages = (u32)(0x100000000ULL >> PAGE_SHIFT);
+
+	if (num_pages >= max_pages)
+		cl_warn("requested to map %u pages in %u chunks\n",
+			num_pages, num_chunks);
+
+	do {
+		u32 chunks_to_map = num_chunks;
+		u32 pages_to_map  = num_pages;
+		int mapped;
+
+		/* Some HW IOMMU drivers can coalesce multiple chunks into
+		 * a single contiguous VA mapping, which is exposed via the
+		 * first chunk.  However, dma_length field is unsigned int
+		 * and not able to represent mappings which exceed 4GB.
+		 * To alleviate it, split large allocations into multiple
+		 * mappings.
+		 */
+		if (num_pages >= max_pages) {
+
+			struct scatterlist *cur_sg;
+
+			pages_to_map = 0;
+
+			for_each_sg(sg, cur_sg, num_chunks, chunks_to_map) {
+
+				const unsigned int len = cur_sg->length;
+				const u32 cur_pages = len >> PAGE_SHIFT;
+
+				if ((u64)pages_to_map + cur_pages >= max_pages)
+					break;
+
+				pages_to_map += cur_pages;
+			}
+		}
+
+		mapped = dma_map_sg(dev, sg, (int)chunks_to_map,
+				    DMA_BIDIRECTIONAL);
+
+		if (mapped == 0) {
+			cl_error(
+				"failed to dma map %u chunks at 0x%llx to dev %s with dma mask 0x%llx\n",
+				num_chunks,
+				(unsigned long long)sg_phys(sg),
+				dev_name(dev),
+				(unsigned long long)dma_get_mask(dev));
+
+			return -EIO;
+		}
+
+		sg         += chunks_to_map;
+		num_chunks -= chunks_to_map;
+		num_pages  -= pages_to_map;
+
+	} while (num_chunks);
+
+	return OK;
+}
+
+static void unmap_sg(struct device      *dev,
+		     struct scatterlist *sg,
+		     u32                 num_chunks)
+{
+	do {
+		struct scatterlist *cur_sg;
+		u32                 chunks_to_unmap = 0;
+
+		for_each_sg(sg, cur_sg, num_chunks, chunks_to_unmap)
+			if (!sg_dma_len(cur_sg))
+				break;
+
+		dma_unmap_sg(dev, sg, (int)chunks_to_unmap, DMA_BIDIRECTIONAL);
+
+		sg         += chunks_to_unmap;
+		num_chunks -= chunks_to_unmap;
+
+		/* Skip chunks which don't maintain any DMA mappings.
+		 * This can happen for large allocations with the workaround
+		 * in map_sg().
+		 */
+		if (num_chunks) {
+			for_each_sg(sg, sg, num_chunks, chunks_to_unmap)
+				if (sg_dma_len(sg))
+					break;
+			num_chunks -= chunks_to_unmap;
+		}
+
+	} while (num_chunks);
+}
+
+/* Unmap and delete the specified DMA mapping */
+static void dma_unmap_and_free(struct mods_client   *client,
+			       struct MODS_MEM_INFO *p_mem_info,
+			       struct MODS_DMA_MAP  *p_del_map)
+
+{
+	const u32 nents = get_num_chunks(p_mem_info);
+
+	print_map_info(client, "unmap", p_del_map->sg, nents, p_del_map->dev);
+
+	unmap_sg(p_del_map->dev, p_del_map->sg, nents);
+
+	pci_dev_put(p_del_map->pcidev);
+
+	list_del(&p_del_map->list);
+
+	kfree(p_del_map);
+	atomic_dec(&client->num_allocs);
+}
+
+/* Unmap and delete all DMA mappings for the specified allocation */
+static int dma_unmap_all(struct mods_client   *client,
+			 struct MODS_MEM_INFO *p_mem_info,
+			 struct device        *dev)
+{
+	int               err  = OK;
+	struct list_head *head = &p_mem_info->dma_map_list;
+	struct list_head *iter;
+	struct list_head *tmp;
+
+#ifdef CONFIG_PCI
+	if (sg_dma_address(p_mem_info->sg) &&
+	    (dev == &p_mem_info->dev->dev || !dev)) {
+
+		unmap_sg(&p_mem_info->dev->dev,
+			 p_mem_info->sg,
+			 get_num_chunks(p_mem_info));
+
+		sg_dma_address(p_mem_info->sg) = 0;
+	}
+#endif
+
+	list_for_each_safe(iter, tmp, head) {
+		struct MODS_DMA_MAP *p_dma_map;
+
+		p_dma_map = list_entry(iter, struct MODS_DMA_MAP, list);
+
+		if (!dev || (p_dma_map->dev == dev)) {
+			dma_unmap_and_free(client, p_mem_info, p_dma_map);
+			if (dev)
+				break;
+		}
+	}
+
+	return err;
+}
+
+/* Create a DMA map on the specified allocation for the pci device.
+ * Lazy-initialize the map list structure if one does not yet exist.
+ */
+static int create_dma_map(struct mods_client   *client,
+			  struct MODS_MEM_INFO *p_mem_info,
+			  struct pci_dev       *pcidev,
+			  struct device        *dev)
+{
+	struct MODS_DMA_MAP *p_dma_map;
+	struct scatterlist  *sg;
+	const u32            num_chunks = get_num_chunks(p_mem_info);
+	size_t               alloc_size;
+	u32                  i;
+	int                  err;
+
+	alloc_size = sizeof(struct MODS_DMA_MAP) +
+		     (num_chunks - 1) * sizeof(struct scatterlist);
+
+	p_dma_map = kzalloc(alloc_size, GFP_KERNEL | __GFP_NORETRY);
+
+	if (unlikely(!p_dma_map)) {
+		cl_error("failed to allocate device map data\n");
+		return -ENOMEM;
+	}
+	atomic_inc(&client->num_allocs);
+
+#ifdef CONFIG_PCI
+	p_dma_map->pcidev = pcidev ? pci_dev_get(pcidev) : NULL;
+#endif
+	p_dma_map->dev    = dev;
+
+	sg_init_table(p_dma_map->sg, num_chunks);
+
+	for_each_sg(p_mem_info->sg, sg, num_chunks, i)
+		sg_set_page(&p_dma_map->sg[i], sg_page(sg), sg->length, 0);
+
+	err = map_sg(client, dev, p_dma_map->sg, num_chunks,
+		     p_mem_info->num_pages);
+
+	print_map_info(client, "map", p_dma_map->sg, num_chunks, dev);
+
+	if (unlikely(err)) {
+		pci_dev_put(pcidev);
+		kfree(p_dma_map);
+		atomic_dec(&client->num_allocs);
+	} else {
+		list_add(&p_dma_map->list, &p_mem_info->dma_map_list);
+	}
+
+	return err;
+}
+
+#ifdef CONFIG_PCI
+/* DMA-map memory to the device for which it has been allocated, if it hasn't
+ * been mapped already.
+ */
+static int dma_map_to_default_dev(struct mods_client   *client,
+				  struct MODS_MEM_INFO *p_mem_info)
+{
+	struct device *const dev        = &p_mem_info->dev->dev;
+	const u32            num_chunks = get_num_chunks(p_mem_info);
+	int                  err;
+
+	if (sg_dma_address(p_mem_info->sg)) {
+		cl_debug(DEBUG_MEM_DETAILED,
+			 "memory %p already mapped to dev %s\n",
+			 p_mem_info,
+			 dev_name(dev));
+		return OK;
+	}
+
+	err = map_sg(client, dev, p_mem_info->sg, num_chunks,
+		     p_mem_info->num_pages);
+
+	print_map_info(client, "map default", p_mem_info->sg, num_chunks, dev);
+
+	return err;
+}
+#endif /* CONFIG_PCI */
+
+#ifdef CONFIG_ARM64
+static void clear_contiguous_cache(struct mods_client *client,
+				   u64                 virt_start,
+				   u32                 size);
+#endif
+
+static int setup_cache_attr(struct mods_client   *client,
+			    struct MODS_MEM_INFO *p_mem_info,
+			    u32                   ichunk)
+{
+	const bool need_wc = p_mem_info->cache_type != MODS_ALLOC_CACHED;
+	int        err = 0;
+
+	if (need_wc && !is_chunk_wc(p_mem_info, ichunk)) {
+		struct scatterlist *sg = &p_mem_info->alloc_sg[ichunk];
+		unsigned int        offs;
+
+		for (offs = 0; offs < sg->length; offs += PAGE_SIZE) {
+			void *ptr;
+
+			ptr = MODS_KMAP(sg_page(sg) + (offs >> PAGE_SHIFT));
+			if (unlikely(!ptr)) {
+				cl_error("kmap failed\n");
+				return -ENOMEM;
+			}
+#ifdef CONFIG_ARM64
+			clear_contiguous_cache(client,
+					       (u64)(size_t)ptr,
+					       PAGE_SIZE);
+#else
+			if (p_mem_info->cache_type == MODS_ALLOC_WRITECOMBINE)
+				err = MODS_SET_MEMORY_WC((unsigned long)ptr, 1);
+			else
+				err = MODS_SET_MEMORY_UC((unsigned long)ptr, 1);
+#endif
+			kunmap(ptr);
+			if (unlikely(err)) {
+				cl_error("set cache type failed\n");
+				return err;
+			}
+
+			/* Set this flag early, so that when an error occurs,
+			 * release_chunks() will restore cache attributes
+			 * for all pages.  It's OK to restore cache attributes
+			 * even for chunks where we haven't change them.
+			 */
+			mark_chunk_wc(p_mem_info, ichunk);
+
+			/* Avoid superficial lockups */
+			cond_resched();
+		}
+	}
+
+	return err;
+}
+
+/* Find the dma mapping chunk for the specified memory. */
+static struct MODS_DMA_MAP *find_dma_map(struct MODS_MEM_INFO  *p_mem_info,
+					 struct mods_pci_dev_2 *pcidev)
+{
+	struct MODS_DMA_MAP *p_dma_map = NULL;
+	struct list_head    *head      = &p_mem_info->dma_map_list;
+	struct list_head    *iter;
+
+	if (!head)
+		return NULL;
+
+	list_for_each(iter, head) {
+		p_dma_map = list_entry(iter, struct MODS_DMA_MAP, list);
+
+		if (mods_is_pci_dev(p_dma_map->pcidev, pcidev))
+			return p_dma_map;
+	}
+
+	return NULL;
+}
+
+/* In order to map pages as UC or WC to the CPU, we need to change their
+ * attributes by calling set_memory_uc()/set_memory_wc(), respectively.
+ * On some CPUs this operation is extremely slow.  In order to incur
+ * this penalty only once, we save pages mapped as UC or WC so that
+ * we can reuse them later.
+ */
+static void save_non_wb_chunks(struct mods_client   *client,
+			       struct MODS_MEM_INFO *p_mem_info)
+{
+	struct scatterlist *sg  = NULL;
+	u32                 ichunk;
+
+	if (p_mem_info->cache_type == MODS_ALLOC_CACHED)
+		return;
+
+	if (unlikely(mutex_lock_interruptible(&client->mtx)))
+		return;
+
+	/* Steal the chunks from MODS_MEM_INFO and put them on free list. */
+
+	for_each_sg(p_mem_info->alloc_sg, sg, p_mem_info->num_chunks, ichunk) {
+
+		struct MODS_FREE_PHYS_CHUNK *free_chunk;
+		u32                          order;
+
+		if (!sg)
+			break;
+
+		WARN_ON(!sg_page(sg));
+
+		if (!is_chunk_wc(p_mem_info, ichunk))
+			continue;
+
+		free_chunk = kzalloc(sizeof(struct MODS_FREE_PHYS_CHUNK),
+				     GFP_KERNEL | __GFP_NORETRY);
+
+		if (unlikely(!free_chunk))
+			break;
+		atomic_inc(&client->num_allocs);
+
+		order = get_order(sg->length);
+		WARN_ON((PAGE_SIZE << order) != sg->length);
+
+		free_chunk->numa_node  = p_mem_info->numa_node;
+		free_chunk->order      = order;
+		free_chunk->cache_type = p_mem_info->cache_type;
+		free_chunk->dma32      = p_mem_info->dma32;
+		free_chunk->p_page     = sg_page(sg);
+
+		sg_set_page(sg, NULL, 0, 0);
+
+		cl_debug(DEBUG_MEM_DETAILED,
+			 "save %p 2^%u pages %s\n",
+			 free_chunk->p_page,
+			 order,
+			 p_mem_info->cache_type == MODS_ALLOC_WRITECOMBINE
+			     ? "WC" : "UC");
+
+		list_add(&free_chunk->list, &client->free_mem_list);
+	}
+
+	mutex_unlock(&client->mtx);
+}
+
+static int restore_cache_one_chunk(struct page *p_page, u8 order)
+{
+	int final_err = 0;
+	u32 num_pages = 1U << order;
+	u32 i;
+
+	for (i = 0; i < num_pages; i++) {
+		void *ptr = MODS_KMAP(p_page + i);
+		int   err = -ENOMEM;
+
+		if (likely(ptr))
+			err = MODS_SET_MEMORY_WB((unsigned long)ptr, 1);
+
+		kunmap(ptr);
+
+		if (likely(!final_err))
+			final_err = err;
+	}
+
+	return final_err;
+}
+
+static int release_free_chunks(struct mods_client *client)
+{
+	struct list_head *head;
+	struct list_head *iter;
+	struct list_head *next;
+	unsigned long     num_restored = 0;
+	unsigned long     num_failed   = 0;
+	unsigned long     pages_freed  = 0;
+	int               final_err    = 0;
+
+	mutex_lock(&client->mtx);
+
+	head = &client->free_mem_list;
+
+	list_for_each_prev_safe(iter, next, head) {
+
+		struct MODS_FREE_PHYS_CHUNK *free_chunk;
+		int                          err;
+
+		free_chunk = list_entry(iter,
+					struct MODS_FREE_PHYS_CHUNK,
+					list);
+
+		list_del(iter);
+
+		err = restore_cache_one_chunk(free_chunk->p_page,
+					      free_chunk->order);
+		if (likely(!final_err))
+			final_err = err;
+
+		if (unlikely(err))
+			++num_failed;
+		else
+			++num_restored;
+
+		pages_freed += 1u << free_chunk->order;
+
+		__free_pages(free_chunk->p_page, free_chunk->order);
+		atomic_sub(1 << free_chunk->order, &client->num_pages);
+
+		kfree(free_chunk);
+		atomic_dec(&client->num_allocs);
+	}
+
+	mutex_unlock(&client->mtx);
+
+	if (pages_freed) {
+		cl_debug(DEBUG_MEM, "released %lu free WC/UC pages, restored cache on %lu free chunks\n",
+			 pages_freed, num_restored);
+		if (unlikely(num_failed))
+			cl_error("failed to restore cache on %lu (out of %lu) free chunks\n",
+				 num_failed, num_failed + num_restored);
+	}
+
+	return final_err;
+}
+
+static int restore_cache(struct mods_client   *client,
+			 struct MODS_MEM_INFO *p_mem_info)
+{
+	struct scatterlist *sg;
+	unsigned int        i;
+	int                 final_err = 0;
+
+	if (p_mem_info->cache_type == MODS_ALLOC_CACHED)
+		return 0;
+
+	for_each_sg(p_mem_info->alloc_sg, sg, p_mem_info->num_chunks, i) {
+
+		const u32 order = get_order(sg->length);
+		int       err;
+
+		WARN_ON((PAGE_SIZE << order) != sg->length);
+
+		if (!sg_page(sg) || !is_chunk_wc(p_mem_info, i))
+			continue;
+
+		err = restore_cache_one_chunk(sg_page(sg), order);
+		if (likely(!final_err))
+			final_err = err;
+	}
+
+	if (unlikely(final_err))
+		cl_error("failed to restore cache attributes\n");
+
+	return final_err;
+}
+
+static void release_chunks(struct mods_client   *client,
+			   struct MODS_MEM_INFO *p_mem_info)
+{
+	u32 i;
+
+	WARN_ON(sg_dma_address(p_mem_info->sg));
+	WARN_ON(!list_empty(&p_mem_info->dma_map_list));
+
+	restore_cache(client, p_mem_info);
+
+	/* release in reverse order */
+	for (i = p_mem_info->num_chunks; i > 0; ) {
+		struct scatterlist *sg;
+		u32                 order;
+
+		--i;
+		sg = &p_mem_info->alloc_sg[i];
+		if (!sg_page(sg))
+			continue;
+
+		order = get_order(sg->length);
+		WARN_ON((PAGE_SIZE << order) != sg->length);
+
+		__free_pages(sg_page(sg), order);
+		atomic_sub(1u << order, &client->num_pages);
+
+		sg_set_page(sg, NULL, 0, 0);
+	}
+}
+
+static gfp_t get_alloc_flags(struct MODS_MEM_INFO *p_mem_info, u32 order)
+{
+	gfp_t flags = GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN;
+
+	if (p_mem_info->force_numa)
+		flags |= __GFP_THISNODE;
+
+	if (order)
+		flags |= __GFP_COMP;
+
+	if (p_mem_info->dma32)
+#ifdef CONFIG_ZONE_DMA32
+		flags |= __GFP_DMA32;
+#else
+		flags |= __GFP_DMA;
+#endif
+	else
+		flags |= __GFP_HIGHMEM;
+
+	return flags;
+}
+
+static struct page *alloc_chunk(struct mods_client   *client,
+				struct MODS_MEM_INFO *p_mem_info,
+				u32                   order,
+				int                  *need_cup)
+{
+	struct page *p_page     = NULL;
+	u8           cache_type = p_mem_info->cache_type;
+	u8           dma32      = p_mem_info->dma32;
+	int          numa_node  = p_mem_info->numa_node;
+
+	if ((cache_type != MODS_MEMORY_CACHED) &&
+	    likely(!mutex_lock_interruptible(&client->mtx))) {
+
+		struct list_head            *iter;
+		struct list_head            *head = &client->free_mem_list;
+		struct MODS_FREE_PHYS_CHUNK *free_chunk = NULL;
+
+		list_for_each(iter, head) {
+			free_chunk = list_entry(iter,
+						struct MODS_FREE_PHYS_CHUNK,
+						list);
+
+			if (free_chunk->cache_type == cache_type &&
+			    free_chunk->dma32      == dma32 &&
+			    free_chunk->numa_node  == numa_node &&
+			    free_chunk->order      == order) {
+
+				list_del(iter);
+				break;
+			}
+
+			free_chunk = NULL;
+		}
+
+		mutex_unlock(&client->mtx);
+
+		if (free_chunk) {
+			p_page = free_chunk->p_page;
+			kfree(free_chunk);
+			atomic_dec(&client->num_allocs);
+
+			cl_debug(DEBUG_MEM_DETAILED, "reuse %p 2^%u pages %s\n",
+				 p_page, order,
+				 cache_type == MODS_ALLOC_WRITECOMBINE
+				     ? "WC" : "UC");
+
+			*need_cup = 0;
+			return p_page;
+		}
+	}
+
+	p_page = alloc_pages_node(p_mem_info->numa_node,
+				  get_alloc_flags(p_mem_info, order),
+				  order);
+
+	*need_cup = 1;
+
+	if (likely(p_page))
+		atomic_add(1 << order, &client->num_pages);
+
+	return p_page;
+}
+
+static int alloc_contig_sys_pages(struct mods_client   *client,
+				  struct MODS_MEM_INFO *p_mem_info)
+{
+	const unsigned long req_bytes = (unsigned long)p_mem_info->num_pages
+					<< PAGE_SHIFT;
+	struct page *p_page;
+	u64          phys_addr;
+	u64          end_addr = 0;
+	u32          order    = 0;
+	int          is_wb    = 1;
+	int          err      = -ENOMEM;
+
+	LOG_ENT();
+
+	while ((1U << order) < p_mem_info->num_pages)
+		order++;
+
+	p_page = alloc_chunk(client, p_mem_info, order, &is_wb);
+
+	if (unlikely(!p_page))
+		goto failed;
+
+	p_mem_info->num_pages = 1U << order;
+
+	sg_set_page(p_mem_info->alloc_sg, p_page, PAGE_SIZE << order, 0);
+
+	if (!is_wb)
+		mark_chunk_wc(p_mem_info, 0);
+
+	phys_addr = sg_phys(p_mem_info->alloc_sg);
+	if (unlikely(phys_addr == 0)) {
+		cl_error("failed to determine physical address\n");
+		goto failed;
+	}
+
+	cl_debug(DEBUG_MEM,
+		 "alloc contig 0x%lx bytes, 2^%u pages, %s, node %d,%s phys 0x%llx\n",
+		 req_bytes,
+		 order,
+		 mods_get_prot_str(p_mem_info->cache_type),
+		 p_mem_info->numa_node,
+		 p_mem_info->dma32 ? " dma32," : "",
+		 (unsigned long long)phys_addr);
+
+	end_addr = phys_addr +
+		   ((unsigned long)p_mem_info->num_pages << PAGE_SHIFT);
+	if (unlikely(p_mem_info->dma32 && (end_addr > 0x100000000ULL))) {
+		cl_error("allocation exceeds 32-bit addressing\n");
+		goto failed;
+	}
+
+	err = setup_cache_attr(client, p_mem_info, 0);
+
+failed:
+	LOG_EXT();
+	return err;
+}
+
+static u32 get_max_order_needed(u32 num_pages)
+{
+	const u32 order = min(10, get_order(num_pages << PAGE_SHIFT));
+
+	return ((1u << order) <= num_pages) ? order : (order >> 1u);
+}
+
+static int alloc_noncontig_sys_pages(struct mods_client   *client,
+				     struct MODS_MEM_INFO *p_mem_info)
+{
+	const unsigned long req_bytes = (unsigned long)p_mem_info->num_pages
+					<< PAGE_SHIFT;
+	u32 pages_needed = p_mem_info->num_pages;
+	u32 num_chunks   = 0;
+	int err;
+
+	LOG_ENT();
+
+	p_mem_info->num_pages = 0;
+
+	for (; pages_needed > 0; ++num_chunks) {
+		struct scatterlist *sg = &p_mem_info->alloc_sg[num_chunks];
+		u64 phys_addr       = 0;
+		u32 order           = get_max_order_needed(pages_needed);
+		u32 allocated_pages = 0;
+		int is_wb           = 1;
+
+		/* Fail if memory fragmentation is very high */
+		if (unlikely(num_chunks >= p_mem_info->num_chunks)) {
+			cl_error("detected high memory fragmentation\n");
+			err = -ENOMEM;
+			goto failed;
+		}
+
+		for (;;) {
+			struct page *p_page = alloc_chunk(client,
+							  p_mem_info,
+							  order,
+							  &is_wb);
+			if (p_page) {
+				sg_set_page(sg, p_page, PAGE_SIZE << order, 0);
+				allocated_pages = 1u << order;
+				break;
+			}
+			if (order == 0)
+				break;
+			--order;
+		}
+
+		if (unlikely(!allocated_pages)) {
+			cl_error("out of memory\n");
+			err = -ENOMEM;
+			goto failed;
+		}
+
+		if (!is_wb)
+			mark_chunk_wc(p_mem_info, num_chunks);
+
+		pages_needed -= min(allocated_pages, pages_needed);
+		p_mem_info->num_pages += allocated_pages;
+
+		phys_addr = sg_phys(sg);
+		if (unlikely(phys_addr == 0)) {
+			cl_error("phys addr lookup failed\n");
+			err = -ENOMEM;
+			goto failed;
+		}
+
+		cl_debug(DEBUG_MEM,
+			 "alloc 0x%lx bytes [%u], 2^%u pages, %s, node %d,%s phys 0x%llx\n",
+			 req_bytes,
+			 (unsigned int)num_chunks,
+			 order,
+			 mods_get_prot_str(p_mem_info->cache_type),
+			 p_mem_info->numa_node,
+			 p_mem_info->dma32 ? " dma32," : "",
+			 (unsigned long long)phys_addr);
+
+		err = setup_cache_attr(client, p_mem_info, num_chunks);
+		if (unlikely(err))
+			goto failed;
+	}
+
+	err = 0;
+
+failed:
+	if (num_chunks)
+		sg_mark_end(&p_mem_info->alloc_sg[num_chunks - 1]);
+
+	LOG_EXT();
+	return err;
+}
+
+static int register_alloc(struct mods_client   *client,
+			  struct MODS_MEM_INFO *p_mem_info)
+{
+	const int err = mutex_lock_interruptible(&client->mtx);
+
+	if (likely(!err)) {
+
+		list_add(&p_mem_info->list, &client->mem_alloc_list);
+
+		mutex_unlock(&client->mtx);
+	}
+
+	return err;
+}
+
+static int unregister_and_free_alloc(struct mods_client   *client,
+				     struct MODS_MEM_INFO *p_del_mem)
+{
+	struct MODS_MEM_INFO *p_mem_info = NULL;
+	struct list_head     *head;
+	struct list_head     *iter;
+	int                   err;
+
+	cl_debug(DEBUG_MEM_DETAILED, "free %p\n", p_del_mem);
+
+	mutex_lock(&client->mtx);
+
+	head = &client->mem_alloc_list;
+
+	list_for_each(iter, head) {
+		p_mem_info = list_entry(iter, struct MODS_MEM_INFO, list);
+
+		if (p_del_mem == p_mem_info) {
+			list_del(iter);
+			break;
+		}
+
+		p_mem_info = NULL;
+	}
+
+	mutex_unlock(&client->mtx);
+
+	if (likely(p_mem_info)) {
+		dma_unmap_all(client, p_mem_info, NULL);
+		save_non_wb_chunks(client, p_mem_info);
+		release_chunks(client, p_mem_info);
+
+		pci_dev_put(p_mem_info->dev);
+
+		kfree(p_mem_info);
+		atomic_dec(&client->num_allocs);
+
+		err = OK;
+	} else {
+		cl_error("failed to unregister allocation %p\n", p_del_mem);
+		err = -EINVAL;
+	}
+
+	return err;
+}
+
+int mods_unregister_all_alloc(struct mods_client *client)
+{
+	int               final_err = OK;
+	int               err;
+	struct list_head *head      = &client->mem_alloc_list;
+	struct list_head *iter;
+	struct list_head *tmp;
+
+	list_for_each_safe(iter, tmp, head) {
+
+		struct MODS_MEM_INFO *p_mem_info;
+
+		p_mem_info = list_entry(iter, struct MODS_MEM_INFO, list);
+		err = unregister_and_free_alloc(client, p_mem_info);
+		if (likely(!final_err))
+			final_err = err;
+	}
+
+	err = release_free_chunks(client);
+	if (likely(!final_err))
+		final_err = err;
+
+	return final_err;
+}
+
+static int get_addr_range(struct mods_client                 *client,
+			  struct MODS_GET_PHYSICAL_ADDRESS_3 *p,
+			  struct mods_pci_dev_2              *pcidev)
+{
+	struct scatterlist   *sg;
+	struct MODS_MEM_INFO *p_mem_info;
+	struct MODS_DMA_MAP  *p_dma_map = NULL;
+	u64                   offs;
+	u32                   num_chunks;
+	u32                   ichunk;
+	int                   err       = OK;
+
+	LOG_ENT();
+
+	p->physical_address = 0;
+
+	p_mem_info = get_mem_handle(client, p->memory_handle);
+	if (unlikely(!p_mem_info)) {
+		LOG_EXT();
+		return -EINVAL;
+	}
+
+	if (unlikely(pcidev && (pcidev->bus > 0xFFU ||
+				pcidev->device > 0xFFU))) {
+		cl_error("dev %04x:%02x:%02x.%x not found\n",
+			 pcidev->domain,
+			 pcidev->bus,
+			 pcidev->device,
+			 pcidev->function);
+		LOG_EXT();
+		return -EINVAL;
+	}
+
+	sg         = p_mem_info->sg;
+	num_chunks = get_num_chunks(p_mem_info);
+
+	err = mutex_lock_interruptible(&client->mtx);
+	if (err) {
+		LOG_EXT();
+		return err;
+	}
+
+	/* If pcidev was specified, retrieve IOVA,
+	 * otherwise retrieve physical address.
+	 */
+	if (pcidev) {
+		if (mods_is_pci_dev(p_mem_info->dev, pcidev)) {
+			if (!sg_dma_address(sg))
+				err = -EINVAL;
+		} else {
+			p_dma_map = find_dma_map(p_mem_info, pcidev);
+			if (!p_dma_map)
+				err = -EINVAL;
+			else
+				sg = p_dma_map->sg;
+		}
+
+		if (err) {
+			mutex_unlock(&client->mtx);
+
+			cl_error(
+				"allocation %p is not mapped to dev %04x:%02x:%02x.%x\n",
+				p_mem_info,
+				pcidev->domain,
+				pcidev->bus,
+				pcidev->device,
+				pcidev->function);
+
+			LOG_EXT();
+			return err;
+		}
+	}
+
+	offs = p->offset;
+	err  = -EINVAL;
+
+	for_each_sg(sg, sg, num_chunks, ichunk) {
+		unsigned int size;
+
+		if (!sg)
+			break;
+
+		size = pcidev ? sg_dma_len(sg) : sg->length;
+		if (size <= offs) {
+			offs -= size;
+			continue;
+		}
+
+		if (pcidev) {
+			dma_addr_t addr = sg_dma_address(sg) + offs;
+
+			addr = compress_nvlink_addr(p_mem_info->dev, addr);
+
+			p->physical_address = (u64)addr;
+		} else {
+			p->physical_address = (u64)sg_phys(sg) + offs;
+		}
+
+		err = OK;
+		break;
+	}
+
+	mutex_unlock(&client->mtx);
+
+	if (err && pcidev) {
+		cl_error(
+			"invalid offset 0x%llx requested for va on dev %04x:%02x:%02x.%x in allocation %p of size 0x%llx\n",
+			(unsigned long long)p->offset,
+			pcidev->domain,
+			pcidev->bus,
+			pcidev->device,
+			pcidev->function,
+			p_mem_info,
+			(unsigned long long)p_mem_info->num_pages << PAGE_SHIFT);
+	} else if (err && !pcidev) {
+		cl_error(
+			"invalid offset 0x%llx requested for pa in allocation %p of size 0x%llx\n",
+			(unsigned long long)p->offset,
+			p_mem_info,
+			(unsigned long long)p_mem_info->num_pages << PAGE_SHIFT);
+	}
+
+	LOG_EXT();
+	return err;
+}
+
+/* Returns an offset within an allocation deduced from physical address.
+ * If physical address doesn't belong to the allocation, returns non-zero.
+ */
+static int get_alloc_offset(struct MODS_MEM_INFO *p_mem_info,
+			    u64                   phys_addr,
+			    u64                  *ret_offs)
+{
+	struct scatterlist *sg;
+	u64                 offset     = 0;
+	const u32           num_chunks = get_num_chunks(p_mem_info);
+	u32                 ichunk;
+
+	for_each_sg(p_mem_info->sg, sg, num_chunks, ichunk) {
+		dma_addr_t   addr;
+		unsigned int size;
+
+		addr = sg_phys(sg);
+		size = sg->length;
+
+		if (phys_addr >= addr && phys_addr < addr + size) {
+			*ret_offs = phys_addr - addr + offset;
+			return 0;
+		}
+
+		offset += size;
+	}
+
+	/* The physical address doesn't belong to the allocation */
+	return -EINVAL;
+}
+
+struct MODS_MEM_INFO *mods_find_alloc(struct mods_client *client, u64 phys_addr)
+{
+	struct list_head     *plist_head = &client->mem_alloc_list;
+	struct list_head     *plist_iter;
+	struct MODS_MEM_INFO *p_mem_info;
+	u64                   offset;
+
+	list_for_each(plist_iter, plist_head) {
+		p_mem_info = list_entry(plist_iter,
+					struct MODS_MEM_INFO,
+					list);
+
+		if (!get_alloc_offset(p_mem_info, phys_addr, &offset))
+			return p_mem_info;
+	}
+
+	/* The physical address doesn't belong to any allocation */
+	return NULL;
+}
+
+/* Estimate the initial number of chunks supported, assuming medium memory
+ * fragmentation.
+ */
+static u32 estimate_num_chunks(u32 num_pages)
+{
+	u32 num_chunks = 0;
+	u32 bit_scan;
+
+	/* Count each contiguous block <=256KB */
+	for (bit_scan = num_pages; bit_scan && num_chunks < 6; bit_scan >>= 1)
+		++num_chunks;
+
+	/* Count remaining contiguous blocks >256KB */
+	num_chunks += bit_scan;
+
+	/* 4x slack for medium memory fragmentation, except huge allocs */
+	if (num_chunks < 32 * 1024)
+		num_chunks <<= 2;
+	else if (num_chunks < 64 * 1024)
+		num_chunks <<= 1;
+
+	/* No sense to allocate more chunks than pages */
+	if (num_chunks > num_pages)
+		num_chunks = num_pages;
+
+	return num_chunks;
+}
+
+static inline size_t calc_mem_info_size_no_bitmap(u32 num_chunks)
+{
+	return sizeof(struct MODS_MEM_INFO) +
+		(num_chunks - 1) * sizeof(struct scatterlist);
+}
+
+static inline u32 calc_mem_info_size(u32 num_chunks, u8 cache_type)
+{
+	size_t size = calc_mem_info_size_no_bitmap(num_chunks);
+
+	if (cache_type != MODS_ALLOC_CACHED)
+		size += sizeof(long) * BITS_TO_LONGS(num_chunks);
+
+	return (u32)size;
+}
+
+static void init_mem_info(struct MODS_MEM_INFO *p_mem_info,
+			  u32                   num_chunks,
+			  u8                    cache_type)
+{
+	p_mem_info->sg         = p_mem_info->alloc_sg;
+	p_mem_info->num_chunks = num_chunks;
+	p_mem_info->cache_type = cache_type;
+
+	if (cache_type != MODS_ALLOC_CACHED)
+		p_mem_info->wc_bitmap = (unsigned long *)
+			&p_mem_info->alloc_sg[num_chunks];
+
+	INIT_LIST_HEAD(&p_mem_info->dma_map_list);
+}
+
+static struct MODS_MEM_INFO *alloc_mem_info(struct mods_client *client,
+					    u32                 num_chunks,
+					    u8                  cache_type,
+					    u32                *alloc_size)
+{
+	struct MODS_MEM_INFO *p_mem_info = NULL;
+
+	const u32 calc_size = calc_mem_info_size(num_chunks, cache_type);
+
+	*alloc_size = calc_size;
+
+	p_mem_info = kzalloc(calc_size, GFP_KERNEL | __GFP_NORETRY);
+
+	if (likely(p_mem_info)) {
+		atomic_inc(&client->num_allocs);
+
+		sg_init_table(&p_mem_info->contig_sg, 1);
+		sg_init_table(p_mem_info->alloc_sg,   num_chunks);
+	}
+
+	return p_mem_info;
+}
+
+/* For large non-contiguous allocations, we typically use significantly less
+ * chunks than originally estimated.  This function reallocates the
+ * MODS_MEM_INFO struct so that it uses only as much memory as it needs.
+ */
+static struct MODS_MEM_INFO *optimize_chunks(struct mods_client   *client,
+					     struct MODS_MEM_INFO *p_mem_info)
+{
+	struct scatterlist   *sg;
+	struct MODS_MEM_INFO *p_new_mem_info = NULL;
+	u32                   num_chunks     = 0;
+	u32                   alloc_size     = 0;
+
+	for_each_sg(p_mem_info->alloc_sg, sg,
+		    p_mem_info->num_chunks, num_chunks) {
+		if (!sg || !sg_page(sg))
+			break;
+	}
+
+	if (num_chunks < p_mem_info->num_chunks)
+		p_new_mem_info = alloc_mem_info(client, num_chunks,
+						p_mem_info->cache_type,
+						&alloc_size);
+
+	if (p_new_mem_info) {
+		const size_t copy_size =
+			calc_mem_info_size_no_bitmap(num_chunks);
+
+		memcpy(p_new_mem_info, p_mem_info, copy_size);
+		init_mem_info(p_new_mem_info, num_chunks,
+			      p_mem_info->cache_type);
+		copy_wc_bitmap(p_new_mem_info, 0, p_mem_info, num_chunks);
+
+		kfree(p_mem_info);
+		atomic_dec(&client->num_allocs);
+
+		p_mem_info = p_new_mem_info;
+	}
+
+	return p_mem_info;
+}
+
+/************************
+ * ESCAPE CALL FUNCTONS *
+ ************************/
+
+int esc_mods_alloc_pages_2(struct mods_client        *client,
+			   struct MODS_ALLOC_PAGES_2 *p)
+{
+	struct MODS_MEM_INFO *p_mem_info = NULL;
+	u32                   num_pages;
+	u32                   alloc_size;
+	u32                   num_chunks;
+	int                   err        = -EINVAL;
+	u8                    cache_type;
+
+	LOG_ENT();
+
+	p->memory_handle = 0;
+
+	cl_debug(DEBUG_MEM_DETAILED,
+		 "alloc 0x%llx bytes flags=0x%x (%s %s%s%s%s%s) node=%d on dev %04x:%02x:%02x.%x\n",
+		 (unsigned long long)p->num_bytes,
+		 p->flags,
+		 mods_get_prot_str(p->flags & MODS_ALLOC_CACHE_MASK),
+		 (p->flags & MODS_ALLOC_CONTIGUOUS) ? "contiguous" :
+						      "noncontiguous",
+		 (p->flags & MODS_ALLOC_DMA32) ? " dma32" : "",
+		 (p->flags & MODS_ALLOC_USE_NUMA) ? " usenuma" : "",
+		 (p->flags & MODS_ALLOC_FORCE_NUMA) ? " forcenuma" : "",
+		 (p->flags & MODS_ALLOC_MAP_DEV) ? " dmamap" : "",
+		 p->numa_node,
+		 p->pci_device.domain,
+		 p->pci_device.bus,
+		 p->pci_device.device,
+		 p->pci_device.function);
+
+	if (unlikely(!p->num_bytes)) {
+		cl_error("zero bytes requested\n");
+		goto failed;
+	}
+
+	num_pages = (u32)((p->num_bytes + PAGE_SIZE - 1) >> PAGE_SHIFT);
+	if (p->flags & MODS_ALLOC_CONTIGUOUS)
+		num_chunks = 1;
+	else
+		num_chunks = estimate_num_chunks(num_pages);
+
+	if (unlikely(((u64)num_pages << PAGE_SHIFT) < p->num_bytes)) {
+		cl_error("invalid allocation size requested: 0x%llx\n",
+			 (unsigned long long)p->num_bytes);
+		goto failed;
+	}
+
+	if (unlikely((p->flags & MODS_ALLOC_USE_NUMA) &&
+		     (p->numa_node != MODS_ANY_NUMA_NODE) &&
+		      ((unsigned int)p->numa_node >=
+		       (unsigned int)num_possible_nodes()))) {
+
+		cl_error("invalid NUMA node: %d\n", p->numa_node);
+		goto failed;
+	}
+
+#ifdef CONFIG_PPC64
+	if (unlikely((p->flags & MODS_ALLOC_CACHE_MASK) != MODS_ALLOC_CACHED)) {
+		cl_error("unsupported cache attr %u (%s)\n",
+			 p->flags & MODS_ALLOC_CACHE_MASK,
+			 mods_get_prot_str(p->flags & MODS_ALLOC_CACHE_MASK));
+		err = -ENOMEM;
+		goto failed;
+	}
+#endif
+
+	cache_type = (u8)(p->flags & MODS_ALLOC_CACHE_MASK);
+
+	p_mem_info = alloc_mem_info(client, num_chunks, cache_type,
+				    &alloc_size);
+
+	if (unlikely(!p_mem_info)) {
+		cl_error("failed to allocate auxiliary 0x%x bytes for %u chunks to hold %u pages\n",
+			 alloc_size, num_chunks, num_pages);
+		err = -ENOMEM;
+		goto failed;
+	}
+
+	init_mem_info(p_mem_info, num_chunks, cache_type);
+
+	p_mem_info->num_pages  = num_pages;
+	p_mem_info->dma32      = (p->flags & MODS_ALLOC_DMA32) ? true : false;
+	p_mem_info->force_numa = (p->flags & MODS_ALLOC_FORCE_NUMA)
+				 ? true : false;
+#ifdef MODS_HASNT_NUMA_NO_NODE
+	p_mem_info->numa_node  = numa_node_id();
+#else
+	p_mem_info->numa_node  = NUMA_NO_NODE;
+#endif
+	p_mem_info->dev        = NULL;
+
+	if ((p->flags & MODS_ALLOC_USE_NUMA) &&
+	    p->numa_node != MODS_ANY_NUMA_NODE)
+		p_mem_info->numa_node = p->numa_node;
+
+#ifdef CONFIG_PCI
+	if (!(p->flags & MODS_ALLOC_USE_NUMA) ||
+	    (p->flags & MODS_ALLOC_MAP_DEV)) {
+
+		struct pci_dev *dev = NULL;
+
+		err = mods_find_pci_dev(client, &p->pci_device, &dev);
+		if (unlikely(err)) {
+			cl_error("dev %04x:%02x:%02x.%x not found\n",
+				 p->pci_device.domain,
+				 p->pci_device.bus,
+				 p->pci_device.device,
+				 p->pci_device.function);
+			goto failed;
+		}
+
+		p_mem_info->dev = dev;
+		if (!(p->flags & MODS_ALLOC_USE_NUMA))
+			p_mem_info->numa_node = dev_to_node(&dev->dev);
+
+#ifdef CONFIG_PPC64
+		if (!mods_is_nvlink_sysmem_trained(client, dev)) {
+			/* Until NvLink is trained, we must use memory
+			 * on node 0.
+			 */
+			if (has_npu_dev(dev, 0))
+				p_mem_info->numa_node = 0;
+		}
+#endif
+		cl_debug(DEBUG_MEM_DETAILED,
+			 "affinity dev %04x:%02x:%02x.%x node %d\n",
+			 p->pci_device.domain,
+			 p->pci_device.bus,
+			 p->pci_device.device,
+			 p->pci_device.function,
+			 p_mem_info->numa_node);
+	}
+#endif
+
+	if (p->flags & MODS_ALLOC_CONTIGUOUS)
+		err = alloc_contig_sys_pages(client, p_mem_info);
+	else {
+		err = alloc_noncontig_sys_pages(client, p_mem_info);
+
+		if (likely(!err))
+			p_mem_info = optimize_chunks(client, p_mem_info);
+	}
+
+	if (unlikely(err)) {
+		cl_error("failed to alloc 0x%lx %s bytes, %s, node %d%s\n",
+			 ((unsigned long)num_pages) << PAGE_SHIFT,
+			 (p->flags & MODS_ALLOC_CONTIGUOUS) ? "contiguous" :
+							      "non-contiguous",
+			 mods_get_prot_str(p_mem_info->cache_type),
+			 p_mem_info->numa_node,
+			 p_mem_info->dma32 ? ", dma32" : "");
+		goto failed;
+	}
+
+	err = register_alloc(client, p_mem_info);
+	if (unlikely(err))
+		goto failed;
+
+	p->memory_handle = (u64)(size_t)p_mem_info;
+
+	cl_debug(DEBUG_MEM_DETAILED, "alloc %p: %u chunks, %u pages\n",
+		 p_mem_info, p_mem_info->num_chunks, p_mem_info->num_pages);
+
+failed:
+	if (unlikely(err && p_mem_info)) {
+		dma_unmap_all(client, p_mem_info, NULL);
+		release_chunks(client, p_mem_info);
+		pci_dev_put(p_mem_info->dev);
+
+		kfree(p_mem_info);
+		atomic_dec(&client->num_allocs);
+	}
+
+	LOG_EXT();
+	return err;
+}
+
+int esc_mods_device_alloc_pages_2(struct mods_client               *client,
+				  struct MODS_DEVICE_ALLOC_PAGES_2 *p)
+{
+	int err;
+	u32 flags = 0;
+	struct MODS_ALLOC_PAGES_2 dev_alloc_pages = {0};
+
+	LOG_ENT();
+
+	if (p->contiguous)
+		flags |= MODS_ALLOC_CONTIGUOUS;
+
+	if (p->address_bits == 32)
+		flags |= MODS_ALLOC_DMA32;
+
+	if (p->attrib == MODS_MEMORY_UNCACHED)
+		flags |= MODS_ALLOC_UNCACHED;
+	else if (p->attrib == MODS_MEMORY_WRITECOMBINE)
+		flags |= MODS_ALLOC_WRITECOMBINE;
+	else if (unlikely(p->attrib != MODS_MEMORY_CACHED)) {
+		cl_error("invalid cache attrib: %u\n", p->attrib);
+		LOG_EXT();
+		return -ENOMEM;
+	}
+
+	if (p->pci_device.bus > 0xFFU || p->pci_device.device > 0xFFU)
+		flags |= MODS_ALLOC_USE_NUMA;
+	else
+		flags |= MODS_ALLOC_MAP_DEV | MODS_ALLOC_FORCE_NUMA;
+
+	dev_alloc_pages.num_bytes  = p->num_bytes;
+	dev_alloc_pages.flags      = flags;
+	dev_alloc_pages.numa_node  = MODS_ANY_NUMA_NODE;
+	dev_alloc_pages.pci_device = p->pci_device;
+
+	err = esc_mods_alloc_pages_2(client, &dev_alloc_pages);
+	if (likely(!err))
+		p->memory_handle = dev_alloc_pages.memory_handle;
+
+	LOG_EXT();
+	return err;
+}
+
+int esc_mods_device_alloc_pages(struct mods_client             *client,
+				struct MODS_DEVICE_ALLOC_PAGES *p)
+{
+	int err;
+	u32 flags = 0;
+	struct MODS_ALLOC_PAGES_2 dev_alloc_pages = {0};
+
+	LOG_ENT();
+
+	if (p->contiguous)
+		flags |= MODS_ALLOC_CONTIGUOUS;
+
+	if (p->address_bits == 32)
+		flags |= MODS_ALLOC_DMA32;
+
+	if (p->attrib == MODS_MEMORY_UNCACHED)
+		flags |= MODS_ALLOC_UNCACHED;
+	else if (p->attrib == MODS_MEMORY_WRITECOMBINE)
+		flags |= MODS_ALLOC_WRITECOMBINE;
+	else if (unlikely(p->attrib != MODS_MEMORY_CACHED)) {
+		cl_error("invalid cache attrib: %u\n", p->attrib);
+		LOG_EXT();
+		return -ENOMEM;
+	}
+
+	if (p->pci_device.bus > 0xFFU || p->pci_device.device > 0xFFU)
+		flags |= MODS_ALLOC_USE_NUMA;
+	else
+		flags |= MODS_ALLOC_MAP_DEV | MODS_ALLOC_FORCE_NUMA;
+
+	dev_alloc_pages.num_bytes           = p->num_bytes;
+	dev_alloc_pages.flags               = flags;
+	dev_alloc_pages.numa_node           = MODS_ANY_NUMA_NODE;
+	dev_alloc_pages.pci_device.domain   = 0;
+	dev_alloc_pages.pci_device.bus      = p->pci_device.bus;
+	dev_alloc_pages.pci_device.device   = p->pci_device.device;
+	dev_alloc_pages.pci_device.function = p->pci_device.function;
+
+	err = esc_mods_alloc_pages_2(client, &dev_alloc_pages);
+	if (likely(!err))
+		p->memory_handle = dev_alloc_pages.memory_handle;
+
+	LOG_EXT();
+	return err;
+}
+
+int esc_mods_alloc_pages(struct mods_client *client, struct MODS_ALLOC_PAGES *p)
+{
+	int err;
+	u32 flags = MODS_ALLOC_USE_NUMA;
+	struct MODS_ALLOC_PAGES_2 dev_alloc_pages = {0};
+
+	LOG_ENT();
+
+	if (p->contiguous)
+		flags |= MODS_ALLOC_CONTIGUOUS;
+
+	if (p->address_bits == 32)
+		flags |= MODS_ALLOC_DMA32;
+
+	if (p->attrib == MODS_MEMORY_UNCACHED)
+		flags |= MODS_ALLOC_UNCACHED;
+	else if (p->attrib == MODS_MEMORY_WRITECOMBINE)
+		flags |= MODS_ALLOC_WRITECOMBINE;
+	else if (unlikely(p->attrib != MODS_MEMORY_CACHED)) {
+		cl_error("invalid cache attrib: %u\n", p->attrib);
+		LOG_EXT();
+		return -ENOMEM;
+	}
+
+	dev_alloc_pages.num_bytes           = p->num_bytes;
+	dev_alloc_pages.flags               = flags;
+	dev_alloc_pages.numa_node           = MODS_ANY_NUMA_NODE;
+	dev_alloc_pages.pci_device.domain   = 0xFFFFU;
+	dev_alloc_pages.pci_device.bus      = 0xFFFFU;
+	dev_alloc_pages.pci_device.device   = 0xFFFFU;
+	dev_alloc_pages.pci_device.function = 0xFFFFU;
+
+	err = esc_mods_alloc_pages_2(client, &dev_alloc_pages);
+	if (likely(!err))
+		p->memory_handle = dev_alloc_pages.memory_handle;
+
+	LOG_EXT();
+	return err;
+}
+
+int esc_mods_free_pages(struct mods_client *client, struct MODS_FREE_PAGES *p)
+{
+	struct MODS_MEM_INFO *p_mem_info;
+	int                   err = -EINVAL;
+
+	LOG_ENT();
+
+	p_mem_info = get_mem_handle(client, p->memory_handle);
+
+	if (likely(p_mem_info))
+		err = unregister_and_free_alloc(client, p_mem_info);
+
+	LOG_EXT();
+
+	return err;
+}
+
+static phys_addr_t get_contig_pa(struct mods_client   *client,
+				 struct MODS_MEM_INFO *p_mem_info)
+{
+	struct scatterlist *sg;
+	struct scatterlist *prev_sg = NULL;
+	u32                 i;
+	bool                contig  = true;
+
+	for_each_sg(p_mem_info->alloc_sg, sg, p_mem_info->num_chunks, i) {
+		if ((i > 0) &&
+			(sg_phys(prev_sg) + prev_sg->length != sg_phys(sg))) {
+
+			cl_debug(DEBUG_MEM_DETAILED,
+				 "merge is non-contiguous because alloc %p chunk %u pa 0x%llx size 0x%x and chunk %u pa 0x%llx\n",
+				 p_mem_info,
+				 i - 1,
+				 (unsigned long long)sg_phys(prev_sg),
+				 prev_sg->length,
+				 i,
+				 (unsigned long long)sg_phys(sg));
+			contig = false;
+			break;
+		}
+
+		prev_sg = sg;
+	}
+
+	return contig ? sg_phys(p_mem_info->alloc_sg) : 0;
+}
+
+int esc_mods_merge_pages(struct mods_client      *client,
+			 struct MODS_MERGE_PAGES *p)
+{
+	struct MODS_MEM_INFO *p_mem_info;
+	int          err        = OK;
+	u32          num_chunks = 0;
+	u32          alloc_size = 0;
+	unsigned int i;
+	bool         contig     = true;
+	u32          cache_type;
+
+	LOG_ENT();
+
+	if (unlikely(p->num_in_handles < 2 ||
+		     p->num_in_handles > MODS_MAX_MERGE_HANDLES)) {
+		cl_error("invalid number of input handles: %u\n",
+			 p->num_in_handles);
+		LOG_EXT();
+		return -EINVAL;
+	}
+
+	err = mutex_lock_interruptible(&client->mtx);
+	if (unlikely(err)) {
+		LOG_EXT();
+		return err;
+	}
+
+	{
+		const char   *err_msg = NULL;
+		phys_addr_t   prev_pa;
+		unsigned long prev_size;
+
+		p_mem_info = get_mem_handle(client, p->in_memory_handles[0]);
+
+		if (unlikely(!validate_mem_handle(client, p_mem_info))) {
+			cl_error("handle 0: invalid handle %p\n", p_mem_info);
+			err = -EINVAL;
+			goto failed;
+		}
+
+		WARN_ON(p_mem_info->num_pages == 0);
+		if (unlikely(!list_empty(&p_mem_info->dma_map_list) ||
+			     sg_dma_address(p_mem_info->sg))) {
+			cl_error("handle 0: found dma mappings\n");
+			err = -EINVAL;
+			goto failed;
+		}
+
+		cache_type = p_mem_info->cache_type;
+		num_chunks = p_mem_info->num_chunks;
+		prev_pa    = get_contig_pa(client, p_mem_info);
+		prev_size  = p_mem_info->num_pages << PAGE_SHIFT;
+
+		for (i = 1; i < p->num_in_handles; i++) {
+			struct MODS_MEM_INFO *const p_other =
+				get_mem_handle(client, p->in_memory_handles[i]);
+			phys_addr_t  next_pa;
+			unsigned int j;
+
+			if (!validate_mem_handle(client, p_other)) {
+				cl_error("handle %u: invalid handle %p\n",
+					 i, p);
+				err = -EINVAL;
+				goto failed;
+			}
+
+			for (j = 0; j < i; j++) {
+				if (unlikely(p->in_memory_handles[i] ==
+					     p->in_memory_handles[j])) {
+					err_msg = "duplicate handle";
+					break;
+				}
+			}
+			if (err_msg)
+				break;
+
+			if (unlikely(p_mem_info->cache_type !=
+				     p_other->cache_type)) {
+				err_msg = "cache attr mismatch";
+				break;
+			}
+
+			if (unlikely(p_mem_info->force_numa &&
+			    p_mem_info->numa_node != p_other->numa_node)) {
+				err_msg = "numa node mismatch";
+				break;
+			}
+
+			if (unlikely(p_mem_info->dma32 != p_other->dma32)) {
+				err_msg = "dma32 mismatch";
+				break;
+			}
+
+			if (p_mem_info->dev) {
+				if (unlikely(p_mem_info->dev !=
+					     p_other->dev)) {
+					err_msg = "device mismatch";
+					break;
+				}
+			}
+
+			WARN_ON(p_other->num_pages == 0);
+			if (unlikely(!list_empty(&p_other->dma_map_list) ||
+				     sg_dma_address(p_other->sg))) {
+				err_msg = "found dma mappings";
+				break;
+			}
+
+			num_chunks += p_other->num_chunks;
+			next_pa    =  get_contig_pa(client, p_other);
+
+			if (contig && ((prev_pa + prev_size) != next_pa)) {
+				contig = false;
+				cl_debug(DEBUG_MEM_DETAILED,
+					 "merge is non-contiguous because alloc %u %p pa 0x%llx size 0x%lx and alloc %u %p pa 0x%llx\n",
+					 i - 1,
+					 get_mem_handle(client,
+					     p->in_memory_handles[i - 1]),
+					 (unsigned long long)prev_pa,
+					 prev_size,
+					 i,
+					 p_other,
+					 (unsigned long long)next_pa);
+			}
+
+			prev_pa   = next_pa;
+			prev_size = p_other->num_pages << PAGE_SHIFT;
+		}
+
+		if (unlikely(err_msg)) {
+			cl_error("merging handle %u: %s\n", i, err_msg);
+			err = -EINVAL;
+			goto failed;
+		}
+	}
+
+	p_mem_info = alloc_mem_info(client, num_chunks, cache_type,
+				    &alloc_size);
+
+	if (unlikely(!p_mem_info)) {
+		err = -ENOMEM;
+		goto failed;
+	}
+
+	for (i = 0; i < p->num_in_handles; i++) {
+		struct MODS_MEM_INFO *p_other =
+			get_mem_handle(client, p->in_memory_handles[i]);
+		const u32 other_chunks = p_other->num_chunks;
+
+		cl_debug(DEBUG_MEM_DETAILED, "merge %p (%u) into %p[%u..%u], phys 0x%llx\n",
+			 p_other, i, p_mem_info, p_mem_info->num_chunks,
+			 p_mem_info->num_chunks + other_chunks - 1,
+			 (unsigned long long)sg_phys(p_other->sg));
+
+		list_del(&p_other->list);
+
+		if (i == 0) {
+			const size_t copy_size =
+				calc_mem_info_size_no_bitmap(other_chunks);
+
+			memcpy(p_mem_info, p_other, copy_size);
+			init_mem_info(p_mem_info, num_chunks,
+				      p_other->cache_type);
+			p_mem_info->num_chunks = other_chunks;
+			copy_wc_bitmap(p_mem_info, 0, p_other, other_chunks);
+
+			list_add(&p_mem_info->list, &client->mem_alloc_list);
+		} else {
+			const u32 num_chunks = p_mem_info->num_chunks;
+
+			memcpy(&p_mem_info->alloc_sg[num_chunks],
+			       p_other->alloc_sg,
+			       other_chunks * sizeof(struct scatterlist));
+			copy_wc_bitmap(p_mem_info, num_chunks,
+				       p_other, other_chunks);
+
+			MODS_SG_UNMARK_END(&p_mem_info->alloc_sg[num_chunks - 1]);
+
+			p_mem_info->num_chunks += other_chunks;
+			p_mem_info->num_pages  += p_other->num_pages;
+		}
+
+		kfree(p_other);
+		atomic_dec(&client->num_allocs);
+	}
+
+	cl_debug(DEBUG_MEM, "merge alloc %p: %u chunks, %u pages\n",
+		 p_mem_info, p_mem_info->num_chunks, p_mem_info->num_pages);
+
+	WARN_ON(num_chunks != p_mem_info->num_chunks);
+
+	if (contig) {
+		p_mem_info->sg = &p_mem_info->contig_sg;
+
+		sg_set_page(&p_mem_info->contig_sg,
+			    sg_page(p_mem_info->alloc_sg),
+			    p_mem_info->num_pages << PAGE_SHIFT,
+			    0);
+	}
+
+	p->memory_handle = (u64)(size_t)p_mem_info;
+
+failed:
+	mutex_unlock(&client->mtx);
+
+	LOG_EXT();
+
+	return err;
+}
+
+int esc_mods_set_mem_type(struct mods_client      *client,
+			  struct MODS_MEMORY_TYPE *p)
+{
+	struct MODS_MEM_INFO *p_mem_info;
+	u8                    type = MODS_ALLOC_CACHED;
+	int                   err;
+
+	LOG_ENT();
+
+	switch (p->type) {
+	case MODS_MEMORY_CACHED:
+		break;
+
+	case MODS_MEMORY_UNCACHED:
+		type = MODS_ALLOC_UNCACHED;
+		break;
+
+	case MODS_MEMORY_WRITECOMBINE:
+		type = MODS_ALLOC_WRITECOMBINE;
+		break;
+
+	default:
+		cl_error("unsupported memory type: %u\n", p->type);
+		LOG_EXT();
+		return -EINVAL;
+	}
+
+	err = mutex_lock_interruptible(&client->mtx);
+	if (unlikely(err)) {
+		LOG_EXT();
+		return err;
+	}
+
+	p_mem_info = mods_find_alloc(client, p->physical_address);
+	if (unlikely(p_mem_info)) {
+		cl_error("cannot set mem type on phys addr 0x%llx\n",
+			 p->physical_address);
+		err = -EINVAL;
+	} else {
+		client->mem_type.phys_addr = p->physical_address;
+		client->mem_type.size      = p->size;
+		client->mem_type.type      = type;
+	}
+
+	mutex_unlock(&client->mtx);
+
+	LOG_EXT();
+	return err;
+}
+
+int esc_mods_get_phys_addr(struct mods_client               *client,
+			   struct MODS_GET_PHYSICAL_ADDRESS *p)
+{
+	struct MODS_GET_PHYSICAL_ADDRESS_3 range;
+	int err;
+
+	LOG_ENT();
+
+	range.memory_handle = p->memory_handle;
+	range.offset        = p->offset;
+	memset(&range.pci_device, 0, sizeof(range.pci_device));
+
+	err = get_addr_range(client, &range, NULL);
+
+	if (!err)
+		p->physical_address = range.physical_address;
+
+	LOG_EXT();
+	return err;
+}
+
+int esc_mods_get_phys_addr_2(struct mods_client                 *client,
+			     struct MODS_GET_PHYSICAL_ADDRESS_3 *p)
+{
+	struct MODS_GET_PHYSICAL_ADDRESS_3 range;
+	int err;
+
+	LOG_ENT();
+
+	range.memory_handle = p->memory_handle;
+	range.offset        = p->offset;
+	memset(&range.pci_device, 0, sizeof(range.pci_device));
+
+	err = get_addr_range(client, &range, NULL);
+
+	if (!err)
+		p->physical_address = range.physical_address;
+
+	LOG_EXT();
+	return err;
+}
+
+int esc_mods_get_mapped_phys_addr(struct mods_client               *client,
+				  struct MODS_GET_PHYSICAL_ADDRESS *p)
+{
+	struct MODS_GET_PHYSICAL_ADDRESS_3 range;
+	struct MODS_MEM_INFO *p_mem_info;
+	int err;
+
+	LOG_ENT();
+
+	p_mem_info = get_mem_handle(client, p->memory_handle);
+	if (unlikely(!p_mem_info)) {
+		LOG_EXT();
+		return -EINVAL;
+	}
+
+	range.memory_handle = p->memory_handle;
+	range.offset        = p->offset;
+
+	if (p_mem_info->dev) {
+		range.pci_device.domain   =
+			pci_domain_nr(p_mem_info->dev->bus);
+		range.pci_device.bus	   =
+			p_mem_info->dev->bus->number;
+		range.pci_device.device   =
+			PCI_SLOT(p_mem_info->dev->devfn);
+		range.pci_device.function =
+			PCI_FUNC(p_mem_info->dev->devfn);
+
+		err = get_addr_range(client, &range, &range.pci_device);
+	} else {
+		memset(&range.pci_device, 0, sizeof(range.pci_device));
+		err = get_addr_range(client, &range, NULL);
+	}
+
+	if (!err)
+		p->physical_address = range.physical_address;
+
+	LOG_EXT();
+	return err;
+}
+
+int esc_mods_get_mapped_phys_addr_2(struct mods_client                 *client,
+				    struct MODS_GET_PHYSICAL_ADDRESS_2 *p)
+{
+	struct MODS_GET_PHYSICAL_ADDRESS_3 range;
+	int err;
+
+	LOG_ENT();
+
+	range.memory_handle = p->memory_handle;
+	range.offset        = p->offset;
+	range.pci_device    = p->pci_device;
+
+	err = get_addr_range(client, &range, &range.pci_device);
+
+	if (!err)
+		p->physical_address = range.physical_address;
+
+	LOG_EXT();
+	return err;
+}
+
+int esc_mods_get_mapped_phys_addr_3(struct mods_client                 *client,
+				    struct MODS_GET_PHYSICAL_ADDRESS_3 *p)
+{
+	struct MODS_GET_PHYSICAL_ADDRESS_3 range;
+	int err;
+
+	LOG_ENT();
+
+	range.memory_handle = p->memory_handle;
+	range.offset        = p->offset;
+	range.pci_device    = p->pci_device;
+
+	err = get_addr_range(client, &range, &range.pci_device);
+
+	if (!err)
+		p->physical_address = range.physical_address;
+
+	LOG_EXT();
+	return err;
+}
+
+int esc_mods_virtual_to_phys(struct mods_client              *client,
+			     struct MODS_VIRTUAL_TO_PHYSICAL *p)
+{
+	struct MODS_GET_PHYSICAL_ADDRESS_3 range;
+	struct list_head                  *head;
+	struct list_head                  *iter;
+	int                                err;
+
+	LOG_ENT();
+
+	memset(&range, 0, sizeof(range));
+
+	err = mutex_lock_interruptible(&client->mtx);
+	if (unlikely(err)) {
+		LOG_EXT();
+		return err;
+	}
+
+	head = &client->mem_map_list;
+
+	list_for_each(iter, head) {
+		struct SYS_MAP_MEMORY *p_map_mem;
+		u64                    begin, end;
+
+		p_map_mem = list_entry(iter, struct SYS_MAP_MEMORY, list);
+
+		begin = p_map_mem->virtual_addr;
+		end   = p_map_mem->virtual_addr + p_map_mem->mapping_length;
+
+		if (p->virtual_address >= begin && p->virtual_address < end) {
+
+			u64 virt_offs = p->virtual_address - begin;
+
+			/* device memory mapping */
+			if (!p_map_mem->p_mem_info) {
+				p->physical_address = p_map_mem->phys_addr
+						      + virt_offs;
+				mutex_unlock(&client->mtx);
+
+				cl_debug(DEBUG_MEM_DETAILED,
+					 "get phys: map %p virt 0x%llx -> 0x%llx\n",
+					 p_map_mem,
+					 p->virtual_address,
+					 p->physical_address);
+
+				LOG_EXT();
+				return OK;
+			}
+
+			range.memory_handle =
+				(u64)(size_t)p_map_mem->p_mem_info;
+			range.offset = virt_offs + p_map_mem->mapping_offs;
+
+			mutex_unlock(&client->mtx);
+
+			err = get_addr_range(client, &range, NULL);
+			if (err) {
+				LOG_EXT();
+				return err;
+			}
+
+			p->physical_address = range.physical_address;
+
+			cl_debug(DEBUG_MEM_DETAILED,
+				 "get phys: map %p virt 0x%llx -> 0x%llx\n",
+				 p_map_mem,
+				 p->virtual_address,
+				 p->physical_address);
+
+			LOG_EXT();
+			return OK;
+		}
+	}
+
+	mutex_unlock(&client->mtx);
+
+	cl_error("invalid virtual address 0x%llx\n", p->virtual_address);
+	LOG_EXT();
+	return -EINVAL;
+}
+
+int esc_mods_phys_to_virtual(struct mods_client              *client,
+			     struct MODS_PHYSICAL_TO_VIRTUAL *p)
+{
+	struct SYS_MAP_MEMORY *p_map_mem;
+	struct list_head      *head;
+	struct list_head      *iter;
+	u64                    offset;
+	u64                    map_offset;
+	int                    err;
+
+	LOG_ENT();
+
+	err = mutex_lock_interruptible(&client->mtx);
+	if (unlikely(err)) {
+		LOG_EXT();
+		return err;
+	}
+
+	head = &client->mem_map_list;
+
+	list_for_each(iter, head) {
+		p_map_mem = list_entry(iter, struct SYS_MAP_MEMORY, list);
+
+		/* device memory mapping */
+		if (!p_map_mem->p_mem_info) {
+			u64 end = p_map_mem->phys_addr
+				+ p_map_mem->mapping_length;
+			if (p->physical_address <  p_map_mem->phys_addr ||
+			    p->physical_address >= end)
+				continue;
+
+			offset = p->physical_address - p_map_mem->phys_addr;
+			p->virtual_address = p_map_mem->virtual_addr
+					     + offset;
+			mutex_unlock(&client->mtx);
+
+			cl_debug(DEBUG_MEM_DETAILED,
+				 "get virt: map %p phys 0x%llx -> 0x%llx\n",
+				 p_map_mem,
+				 p->physical_address,
+				 p->virtual_address);
+
+			LOG_EXT();
+			return OK;
+		}
+
+		/* offset from the beginning of the allocation */
+		if (get_alloc_offset(p_map_mem->p_mem_info,
+				     p->physical_address,
+				     &offset))
+			continue;
+
+		/* offset from the beginning of the mapping */
+		map_offset = p_map_mem->mapping_offs;
+
+		if ((offset >= map_offset) &&
+		    (offset <  map_offset + p_map_mem->mapping_length)) {
+			p->virtual_address = p_map_mem->virtual_addr
+					   + offset - map_offset;
+
+			mutex_unlock(&client->mtx);
+			cl_debug(DEBUG_MEM_DETAILED,
+				 "get virt: map %p phys 0x%llx -> 0x%llx\n",
+				 p_map_mem,
+				 p->physical_address,
+				 p->virtual_address);
+
+			LOG_EXT();
+			return OK;
+		}
+	}
+
+	mutex_unlock(&client->mtx);
+
+	cl_error("phys addr 0x%llx is not mapped\n", p->physical_address);
+	LOG_EXT();
+	return -EINVAL;
+}
+
+#if defined(CONFIG_ARM)
+int esc_mods_memory_barrier(struct mods_client *client)
+{
+	/* Full memory barrier on ARMv7 */
+	wmb();
+	return OK;
+}
+#endif
+
+#ifdef CONFIG_PCI
+int esc_mods_dma_map_memory(struct mods_client         *client,
+			    struct MODS_DMA_MAP_MEMORY *p)
+{
+	struct MODS_MEM_INFO *p_mem_info;
+	struct MODS_DMA_MAP  *p_dma_map;
+	struct pci_dev       *dev    = NULL;
+	int                   err    = -EINVAL;
+	bool                  locked = false;
+
+	LOG_ENT();
+
+	p_mem_info = get_mem_handle(client, p->memory_handle);
+	if (unlikely(!p_mem_info))
+		goto failed;
+
+	err = mutex_lock_interruptible(&client->mtx);
+	if (unlikely(err))
+		goto failed;
+	locked = true;
+
+	if (mods_is_pci_dev(p_mem_info->dev, &p->pci_device)) {
+		err = dma_map_to_default_dev(client, p_mem_info);
+		goto failed;
+	}
+
+	if (mods_is_pci_dev(client->cached_dev, &p->pci_device))
+		dev = pci_dev_get(client->cached_dev);
+	else {
+		mutex_unlock(&client->mtx);
+		locked = false;
+
+		err = mods_find_pci_dev(client, &p->pci_device, &dev);
+		if (unlikely(err)) {
+			if (err == -ENODEV)
+				cl_error("dev %04x:%02x:%02x.%x not found\n",
+					 p->pci_device.domain,
+					 p->pci_device.bus,
+					 p->pci_device.device,
+					 p->pci_device.function);
+			goto failed;
+		}
+
+		err = mutex_lock_interruptible(&client->mtx);
+		if (unlikely(err))
+			goto failed;
+		locked = true;
+	}
+
+	p_dma_map = find_dma_map(p_mem_info, &p->pci_device);
+	if (unlikely(p_dma_map)) {
+		cl_debug(DEBUG_MEM_DETAILED,
+			 "memory %p already mapped to dev %04x:%02x:%02x.%x\n",
+			 p_mem_info,
+			 p->pci_device.domain,
+			 p->pci_device.bus,
+			 p->pci_device.device,
+			 p->pci_device.function);
+		goto failed;
+	}
+
+	err = create_dma_map(client, p_mem_info, dev, &dev->dev);
+
+failed:
+	if (locked)
+		mutex_unlock(&client->mtx);
+
+	pci_dev_put(dev);
+
+	LOG_EXT();
+	return err;
+}
+
+int esc_mods_dma_unmap_memory(struct mods_client         *client,
+			      struct MODS_DMA_MAP_MEMORY *p)
+{
+	struct MODS_MEM_INFO *p_mem_info;
+	struct pci_dev       *dev = NULL;
+	int                   err = -EINVAL;
+
+	LOG_ENT();
+
+	p_mem_info = get_mem_handle(client, p->memory_handle);
+	if (unlikely(!p_mem_info))
+		goto failed;
+
+	err = mods_find_pci_dev(client, &p->pci_device, &dev);
+	if (unlikely(err)) {
+		if (err == -ENODEV)
+			cl_error("dev %04x:%02x:%02x.%x not found\n",
+				 p->pci_device.domain,
+				 p->pci_device.bus,
+				 p->pci_device.device,
+				 p->pci_device.function);
+		goto failed;
+	}
+
+	err = mutex_lock_interruptible(&client->mtx);
+	if (unlikely(err))
+		goto failed;
+
+	err = dma_unmap_all(client, p_mem_info, &dev->dev);
+
+	mutex_unlock(&client->mtx);
+
+failed:
+	pci_dev_put(dev);
+
+	LOG_EXT();
+	return err;
+}
+#endif /* CONFIG_PCI */
+
+#ifdef MODS_HAS_TEGRA
+/* map dma buffer by iommu */
+int esc_mods_iommu_dma_map_memory(struct mods_client               *client,
+				  struct MODS_IOMMU_DMA_MAP_MEMORY *p)
+{
+	struct scatterlist   *sg;
+	struct MODS_MEM_INFO *p_mem_info;
+	char                 *dev_name  = p->dev_name;
+	struct mods_smmu_dev *smmu_pdev = NULL;
+	struct MODS_DMA_MAP  *p_dma_map;
+	dma_addr_t            next_iova = 0;
+	u32                   num_chunks;
+	u32                   i;
+	int                   smmudev_idx;
+	int                   err       = -EINVAL;
+	bool                  locked    = false;
+
+	LOG_ENT();
+
+	if (!(p->flags & MODS_IOMMU_MAP_CONTIGUOUS))
+		cl_error("contiguous flag not set\n");
+
+	p_mem_info = get_mem_handle(client, p->memory_handle);
+	if (unlikely(!p_mem_info))
+		goto failed;
+
+	if (!list_empty(&p_mem_info->dma_map_list)) {
+		cl_error("smmu is already mapped\n");
+		goto failed;
+	}
+
+	smmudev_idx = get_mods_smmu_device_index(dev_name);
+	if (smmudev_idx >= 0)
+		smmu_pdev = get_mods_smmu_device(smmudev_idx);
+	if (!smmu_pdev || smmudev_idx < 0) {
+		cl_error("smmu device %s not found\n", dev_name);
+		err = -ENODEV;
+		goto failed;
+	}
+
+	err = mutex_lock_interruptible(&client->mtx);
+	if (unlikely(err))
+		goto failed;
+	locked = true;
+
+	/* do smmu mapping */
+	err = create_dma_map(client, p_mem_info, NULL, smmu_pdev->dev);
+	if (err)
+		goto failed;
+
+	/* Check if IOVAs are contiguous */
+	p_dma_map = list_first_entry(&p_mem_info->dma_map_list,
+				     struct MODS_DMA_MAP, list);
+	num_chunks = get_num_chunks(p_mem_info);
+	for_each_sg(p_dma_map->sg, sg, num_chunks, i) {
+
+		const dma_addr_t iova     = sg_dma_address(sg);
+		const dma_addr_t iova_end = iova + sg_dma_len(sg);
+
+		/* Skip checking if IOMMU driver merged it into a single
+		 * contiguous chunk.
+		 */
+		if (iova_end == iova)
+			continue;
+
+		if ((i > 0) && (iova != next_iova)) {
+			cl_error("sg not contiguous: dma 0x%llx, expected 0x%llx\n",
+				 (unsigned long long)sg_dma_address(sg),
+				 (unsigned long long)next_iova);
+
+			dma_unmap_and_free(client, p_mem_info, p_dma_map);
+			err = -EINVAL;
+			goto failed;
+		}
+
+		next_iova = iova_end;
+	}
+
+	p->physical_address = sg_dma_address(p_dma_map->sg);
+
+failed:
+	if (locked)
+		mutex_unlock(&client->mtx);
+
+	LOG_EXT();
+	return err;
+}
+
+/* unmap dma buffer by iommu */
+int esc_mods_iommu_dma_unmap_memory(struct mods_client               *client,
+				    struct MODS_IOMMU_DMA_MAP_MEMORY *p)
+{
+	struct MODS_MEM_INFO *p_mem_info;
+	struct MODS_DMA_MAP  *p_dma_map;
+	int                   err = -EINVAL;
+
+	LOG_ENT();
+
+	p_mem_info = get_mem_handle(client, p->memory_handle);
+	if (unlikely(!p_mem_info))
+		goto failed;
+
+	if (!list_is_singular(&p_mem_info->dma_map_list)) {
+		cl_error("smmu buffer is not mapped, handle=0x%llx\n",
+			 (unsigned long long)p_mem_info);
+		goto failed;
+	}
+
+	err = mutex_lock_interruptible(&client->mtx);
+	if (unlikely(err))
+		goto failed;
+
+	p_dma_map = list_first_entry(&p_mem_info->dma_map_list,
+				     struct MODS_DMA_MAP,
+				     list);
+
+	dma_unmap_and_free(client, p_mem_info, p_dma_map);
+
+	mutex_unlock(&client->mtx);
+
+failed:
+	LOG_EXT();
+	return err;
+}
+#endif /* MODS_HAS_TEGRA */
+
+#ifdef CONFIG_ARM64
+static void clear_contiguous_cache(struct mods_client *client,
+				   u64                 virt_start,
+				   u32                 size)
+{
+	u64 end = virt_start + size;
+	u64 cur;
+	u64 d_size;
+	static u32 d_line_shift;
+
+	if (!d_line_shift) {
+#if KERNEL_VERSION(6, 0, 0) <= MODS_KERNEL_VERSION
+		const u64 ctr_el0 = read_sanitised_ftr_reg(SYS_CTR_EL0);
+
+		d_line_shift =
+			cpuid_feature_extract_unsigned_field(ctr_el0, CTR_EL0_DminLine_SHIFT);
+#elif KERNEL_VERSION(5, 10, 0) <= MODS_KERNEL_VERSION
+		const u64 ctr_el0 = read_sanitised_ftr_reg(SYS_CTR_EL0);
+
+		d_line_shift =
+			cpuid_feature_extract_unsigned_field(ctr_el0, CTR_DMINLINE_SHIFT);
+#else
+		d_line_shift = 4; /* Fallback for kernel 5.9 or older */
+#endif
+	}
+
+	d_size = (u64)4 << d_line_shift;
+	cur = virt_start & ~(d_size - 1);
+	do {
+		asm volatile("dc civac, %0" : : "r" (cur) : "memory");
+
+		/* Avoid superficial lockups */
+		cond_resched();
+	} while (cur += d_size, cur < end);
+	asm volatile("dsb sy" : : : "memory");
+
+	cl_debug(DEBUG_MEM_DETAILED,
+		 "flush cache virt 0x%llx size 0x%x\n",
+		 virt_start, size);
+}
+
+int esc_mods_flush_cpu_cache_range(struct mods_client                *client,
+				   struct MODS_FLUSH_CPU_CACHE_RANGE *p)
+{
+	struct list_head *head;
+	struct list_head *iter;
+	int               err;
+
+	LOG_ENT();
+
+	if (irqs_disabled() || in_interrupt() ||
+	    p->virt_addr_start > p->virt_addr_end) {
+
+		cl_debug(DEBUG_MEM_DETAILED, "cannot flush cache\n");
+		LOG_EXT();
+		return -EINVAL;
+	}
+
+	if (p->flags == MODS_INVALIDATE_CPU_CACHE) {
+		cl_debug(DEBUG_MEM_DETAILED, "cannot invalidate cache\n");
+		LOG_EXT();
+		return -EINVAL;
+	}
+
+	err = mutex_lock_interruptible(&client->mtx);
+	if (unlikely(err)) {
+		LOG_EXT();
+		return err;
+	}
+
+	head = &client->mem_map_list;
+	err  = -EINVAL;
+
+	list_for_each(iter, head) {
+		struct SYS_MAP_MEMORY *p_map_mem
+			= list_entry(iter, struct SYS_MAP_MEMORY, list);
+
+		const u64 mapped_va   = p_map_mem->virtual_addr;
+		const u64 mapped_end  = mapped_va + p_map_mem->mapping_length;
+		const u64 flush_start = p->virt_addr_start < mapped_va ? mapped_va
+								       : p->virt_addr_start;
+		const u64 flush_end   = p->virt_addr_end > mapped_end ? mapped_end
+								      : p->virt_addr_end;
+
+		if (flush_start >= flush_end)
+			continue;
+
+		clear_contiguous_cache(client, flush_start, flush_end - flush_start);
+		err = OK;
+	}
+
+	mutex_unlock(&client->mtx);
+
+	if (err)
+		cl_error("va range 0x%lx..0x%lx not flushed\n",
+			 (unsigned long)p->virt_addr_start,
+			 (unsigned long)p->virt_addr_end);
+
+	LOG_EXT();
+	return err;
+}
+#endif /* CONFIG_ARM64 */
diff --git a/drivers/misc/mods/mods_netdevice.c b/drivers/misc/mods/mods_netdevice.c
new file mode 100644
index 00000000..65c6e602
--- /dev/null
+++ b/drivers/misc/mods/mods_netdevice.c
@@ -0,0 +1,45 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * This file is part of NVIDIA MODS kernel driver.
+ *
+ * Copyright (c) 2015-2022, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * NVIDIA MODS kernel driver is free software: you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * NVIDIA MODS kernel driver is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NVIDIA MODS kernel driver.
+ * If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "mods_internal.h"
+#include <linux/netdevice.h>
+
+int esc_mods_net_force_link(struct mods_client *client,
+			    struct MODS_NET_DEVICE_NAME *p)
+{
+	struct net_device *ndev;
+
+	if (!p ||
+	    (strnlen(p->device_name, MAX_NET_DEVICE_NAME_LENGTH) == 0) ||
+	    (!memchr(p->device_name, '\0', MAX_NET_DEVICE_NAME_LENGTH))) {
+		cl_error("invalid device name\n");
+		return -EINVAL;
+	}
+
+	for_each_netdev(&init_net, ndev)
+		if (!strcmp(ndev->name, p->device_name)) {
+			netif_carrier_on(ndev);
+			cl_info("carrier forced on: %s\n", p->device_name);
+			return OK;
+		}
+
+	cl_error("failed to find network device %s\n", p->device_name);
+	return -EINVAL;
+}
diff --git a/drivers/misc/mods/mods_oist.c b/drivers/misc/mods/mods_oist.c
new file mode 100644
index 00000000..db104947
--- /dev/null
+++ b/drivers/misc/mods/mods_oist.c
@@ -0,0 +1,36 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#include "mods_internal.h"
+#include <linux/arm-smccc.h>
+
+#define SMCCC_VERSION 0x80000000
+
+int esc_mods_oist_status(struct mods_client             *client,
+		struct MODS_TEGRA_OIST_STATUS  *p)
+{
+	int ret = 0;
+	struct arm_smccc_res res = { 0 };
+
+	if (p->smc_func_id == SMCCC_VERSION) {
+		// For SMC version, We are only reading res.a0 value, not a1,a2,a3
+		arm_smccc_1_1_smc(p->smc_func_id, res.a0, &res);
+		p->smc_status = res.a0;
+	} else {
+		arm_smccc_1_1_smc(p->smc_func_id, p->a1, p->a2, &res);
+		p->smc_status = res.a1;
+	}
+
+	return ret;
+}
diff --git a/drivers/misc/mods/mods_optee.c b/drivers/misc/mods/mods_optee.c
new file mode 100644
index 00000000..753c69fe
--- /dev/null
+++ b/drivers/misc/mods/mods_optee.c
@@ -0,0 +1,125 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#include "mods_internal.h"
+
+#include <linux/limits.h>
+#include <linux/tee_drv.h>
+#include <linux/uuid.h>
+
+static const uuid_t mods_ta_uuid =
+	UUID_INIT(0xf14e9858, 0x1526, 0x4935,
+		  0xb1, 0x92, 0x4d, 0xf3, 0x86, 0x4f, 0x1f, 0xf9);
+
+static int optee_ctx_match(struct tee_ioctl_version_data *ver, const void *data)
+{
+	if (ver->impl_id == TEE_IMPL_ID_OPTEE)
+		return 1;
+	else
+		return 0;
+}
+
+int esc_mods_invoke_optee_ta(struct mods_client *client,
+			     struct MODS_OPTEE_PARAMS *p)
+{
+	int ret;
+	u32 session_id;
+	u8 *temp_buf;
+	struct tee_context *ctx;
+	struct tee_ioctl_open_session_arg sess_arg;
+	struct tee_ioctl_invoke_arg invoke_arg;
+	struct tee_shm *shm;
+	struct tee_param params[4];
+
+	/* Open context with TEE driver */
+	ctx = tee_client_open_context(NULL, optee_ctx_match, NULL, NULL);
+	if (IS_ERR(ctx)) {
+		ret = -ENODEV;
+		goto fail;
+	}
+
+	/* Open session with TA */
+	memset(&sess_arg, 0, sizeof(sess_arg));
+	export_uuid(sess_arg.uuid, &mods_ta_uuid);
+	sess_arg.clnt_login = TEE_IOCTL_LOGIN_PUBLIC;
+	sess_arg.num_params = 0;
+
+	ret = tee_client_open_session(ctx, &sess_arg, NULL);
+	if ((ret < 0) || (sess_arg.ret != 0)) {
+		cl_info("tee_client_open_session failed, err: %x\n", sess_arg.ret);
+		ret = -EINVAL;
+		goto out_ctx;
+	}
+	session_id = sess_arg.session;
+
+	/* Allocate dynamic shared memory with TA */
+	shm = tee_shm_alloc_kernel_buf(ctx, p->buf_size);
+	if (IS_ERR(shm)) {
+		cl_info("tee_shm_alloc_kernel_buf failed\n");
+		ret = -ENOMEM;
+		goto out_session;
+	}
+
+	/* Invoke comannd of TA */
+	memset(&invoke_arg, 0, sizeof(invoke_arg));
+	memset(&params, 0, sizeof(params));
+
+	invoke_arg.func = p->command_id;
+	invoke_arg.session = session_id;
+	invoke_arg.num_params = 4;
+
+	params[0].attr = TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INOUT;
+	params[0].u.memref.shm = shm;
+	params[0].u.memref.size = p->buf_size;
+	params[0].u.memref.shm_offs = 0;
+	params[1].attr = TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_OUTPUT;
+
+	temp_buf = tee_shm_get_va(shm, 0);
+	if (IS_ERR(temp_buf)) {
+		cl_info("tee_shm_get_va failed\n");
+		ret = PTR_ERR(temp_buf);
+		goto out_shm;
+	}
+	memmove(temp_buf, p->buf, p->buf_size);
+
+	ret = tee_client_invoke_func(ctx, &invoke_arg, params);
+	if (ret < 0) {
+		cl_info("tee_client_invoke_func failed.\n");
+		goto out_shm;
+	}
+
+	/*
+	 * OP-TEE complies with GlobalPlatform API specification.
+	 * Its output value(TEEC_VALUE) should be in the u32 range.
+	 */
+	if (params[1].u.value.a > U32_MAX || params[1].u.value.b > U32_MAX) {
+		ret = EOVERFLOW;
+		goto out_shm;
+	} else {
+		p->out_a = (__u32)params[1].u.value.a;
+		p->out_b = (__u32)params[1].u.value.b;
+	}
+
+	memmove(p->buf, temp_buf, p->buf_size);
+	p->tee_ret = invoke_arg.ret;
+
+out_shm:
+	tee_shm_free(shm);
+out_session:
+	tee_client_close_session(ctx, session_id);
+out_ctx:
+	tee_client_close_context(ctx);
+fail:
+	return ret;
+}
diff --git a/drivers/misc/mods/mods_pci.c b/drivers/misc/mods/mods_pci.c
new file mode 100644
index 00000000..fb3c90a0
--- /dev/null
+++ b/drivers/misc/mods/mods_pci.c
@@ -0,0 +1,1148 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * This file is part of NVIDIA MODS kernel driver.
+ *
+ * Copyright (c) 2008-2023, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * NVIDIA MODS kernel driver is free software: you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * NVIDIA MODS kernel driver is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NVIDIA MODS kernel driver.
+ * If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "mods_internal.h"
+
+#include <linux/device.h>
+#if defined(MODS_HAS_DMA_OPS)
+#include <linux/dma-mapping.h>
+#endif
+#include <linux/io.h>
+#include <linux/fs.h>
+#include <linux/pci.h>
+#if KERNEL_VERSION(2, 32, 0) <= MODS_KERNEL_VERSION
+#include <linux/pm_runtime.h>
+#endif
+#if KERNEL_VERSION(3, 19, 0) <= MODS_KERNEL_VERSION
+#include <linux/property.h>
+#endif
+
+/* Address in config space is 8bit for base caps and 12bit for extended caps */
+#define MODS_MAX_PCI_CFG_ADDR 0xFFFU
+
+int mods_is_pci_dev(struct pci_dev        *dev,
+		    struct mods_pci_dev_2 *pcidev)
+{
+	unsigned int devfn = PCI_DEVFN(pcidev->device, pcidev->function);
+
+	return dev &&
+	       pci_domain_nr(dev->bus) == pcidev->domain &&
+	       dev->bus->number == pcidev->bus &&
+	       dev->devfn == devfn;
+}
+
+int mods_find_pci_dev(struct mods_client    *client,
+		      struct mods_pci_dev_2 *pcidev,
+		      struct pci_dev	   **retdev)
+{
+	struct pci_dev *dev;
+	int             err;
+
+	if (unlikely(mutex_lock_interruptible(&client->mtx)))
+		return -EINTR;
+
+	dev = client->cached_dev;
+
+	if (mods_is_pci_dev(dev, pcidev)) {
+		*retdev = pci_dev_get(dev);
+		mutex_unlock(&client->mtx);
+		return OK;
+	}
+
+	mutex_unlock(&client->mtx);
+
+	dev = NULL;
+
+#ifdef MODS_HAS_NEW_ACPI_WALK
+	dev = pci_get_domain_bus_and_slot(pcidev->domain,
+					  pcidev->bus,
+					  PCI_DEVFN(pcidev->device,
+						    pcidev->function));
+#else
+	while ((dev = pci_get_device(PCI_ANY_ID, PCI_ANY_ID, dev)))
+		if (mods_is_pci_dev(dev, pcidev))
+			break;
+#endif
+
+	if (dev) {
+		if (unlikely(mutex_lock_interruptible(&client->mtx))) {
+			pci_dev_put(dev);
+			return -EINTR;
+		}
+
+		if (dev != client->cached_dev) {
+			pci_dev_put(client->cached_dev);
+			client->cached_dev = pci_dev_get(dev);
+		}
+
+		mutex_unlock(&client->mtx);
+
+		err = OK;
+	} else
+		err = -ENODEV;
+
+	*retdev = dev;
+	return err;
+}
+
+static int find_pci_dev_impl(struct mods_client            *client,
+			     struct MODS_FIND_PCI_DEVICE_2 *p,
+			     int                            enum_non_zero_dom)
+{
+	struct pci_dev *dev   = NULL;
+	int             index = -1;
+
+	LOG_ENT();
+
+	if (p->index > 0xFFFFU) {
+		cl_error("invalid device index %u\n", p->index);
+		LOG_EXT();
+		return -EINVAL;
+	}
+
+	cl_debug(DEBUG_PCI,
+		 "find pci dev %04x:%04x, index %u\n",
+		 p->vendor_id,
+		 p->device_id,
+		 p->index);
+
+	do {
+		dev = pci_get_device(p->vendor_id, p->device_id, dev);
+		if (!dev) {
+			LOG_EXT();
+			return -EINVAL;
+		}
+
+		if (enum_non_zero_dom || !pci_domain_nr(dev->bus))
+			++index;
+	} while (index < (int)(p->index));
+
+	p->pci_device.domain   = pci_domain_nr(dev->bus);
+	p->pci_device.bus      = dev->bus->number;
+	p->pci_device.device   = PCI_SLOT(dev->devfn);
+	p->pci_device.function = PCI_FUNC(dev->devfn);
+
+	pci_dev_put(dev);
+	LOG_EXT();
+	return OK;
+}
+
+int esc_mods_find_pci_dev_2(struct mods_client            *client,
+			    struct MODS_FIND_PCI_DEVICE_2 *p)
+{
+	return find_pci_dev_impl(client, p, 1);
+}
+
+int esc_mods_find_pci_dev(struct mods_client          *client,
+			  struct MODS_FIND_PCI_DEVICE *p)
+{
+	struct MODS_FIND_PCI_DEVICE_2 p2;
+	int                           err;
+
+	p2.device_id = p->device_id;
+	p2.vendor_id = p->vendor_id;
+	p2.index     = p->index;
+
+	err = find_pci_dev_impl(client, &p2, 0);
+
+	if (!err) {
+		p->bus_number      = p2.pci_device.bus;
+		p->device_number   = p2.pci_device.device;
+		p->function_number = p2.pci_device.function;
+	}
+
+	return err;
+}
+
+static int mods_find_pci_class_code(struct mods_client                *client,
+				    struct MODS_FIND_PCI_CLASS_CODE_2 *p,
+				    int enum_non_zero_dom)
+{
+	struct pci_dev *dev   = NULL;
+	int             index = -1;
+
+	LOG_ENT();
+
+	if (p->index > 0xFFFFU) {
+		cl_error("invalid device index %u\n", p->index);
+		LOG_EXT();
+		return -EINVAL;
+	}
+
+	cl_debug(DEBUG_PCI,
+		 "find pci class code %04x, index %u\n",
+		 p->class_code,
+		 p->index);
+
+	do {
+		dev = pci_get_class(p->class_code, dev);
+		if (!dev) {
+			LOG_EXT();
+			return -EINVAL;
+		}
+
+		if (enum_non_zero_dom || !pci_domain_nr(dev->bus))
+			++index;
+	} while (index < (int)(p->index));
+
+	p->pci_device.domain   = pci_domain_nr(dev->bus);
+	p->pci_device.bus      = dev->bus->number;
+	p->pci_device.device   = PCI_SLOT(dev->devfn);
+	p->pci_device.function = PCI_FUNC(dev->devfn);
+
+	pci_dev_put(dev);
+	LOG_EXT();
+	return OK;
+}
+
+int esc_mods_find_pci_class_code_2(struct mods_client                *client,
+				   struct MODS_FIND_PCI_CLASS_CODE_2 *p)
+{
+	return mods_find_pci_class_code(client, p, 1);
+}
+
+int esc_mods_find_pci_class_code(struct mods_client              *client,
+				 struct MODS_FIND_PCI_CLASS_CODE *p)
+{
+	struct MODS_FIND_PCI_CLASS_CODE_2 p2;
+	int                               err;
+
+	p2.class_code = p->class_code;
+	p2.index      = p->index;
+
+	err = mods_find_pci_class_code(client, &p2, 0);
+
+	if (!err) {
+		p->bus_number      = p2.pci_device.bus;
+		p->device_number   = p2.pci_device.device;
+		p->function_number = p2.pci_device.function;
+	}
+
+	return err;
+}
+
+int esc_mods_pci_get_bar_info_2(struct mods_client             *client,
+				struct MODS_PCI_GET_BAR_INFO_2 *p)
+{
+	struct pci_dev *dev;
+	unsigned int bar_resource_offset;
+	unsigned int i;
+	int err;
+#if !defined(MODS_HAS_IORESOURCE_MEM_64)
+	__u32 temp;
+#endif
+
+	LOG_ENT();
+
+	err = mods_find_pci_dev(client, &p->pci_device, &dev);
+	if (unlikely(err)) {
+		LOG_EXT();
+		return err;
+	}
+
+	cl_debug(DEBUG_PCI,
+		 "pci get bar info dev %04x:%02x:%02x:%x, bar index %u\n",
+		 p->pci_device.domain,
+		 p->pci_device.bus,
+		 p->pci_device.device,
+		 p->pci_device.function,
+		 p->bar_index);
+
+#if defined(CONFIG_PPC64)
+	if (unlikely(mutex_lock_interruptible(mods_get_irq_mutex()))) {
+		pci_dev_put(dev);
+		LOG_EXT();
+		return -EINTR;
+	}
+
+	/* Enable device on the PCI bus */
+	err = mods_enable_device(client, dev, NULL);
+	if (err) {
+		cl_error("unable to enable dev %04x:%02x:%02x.%x\n",
+			 p->pci_device.domain,
+			 p->pci_device.bus,
+			 p->pci_device.device,
+			 p->pci_device.function);
+		mutex_unlock(mods_get_irq_mutex());
+		pci_dev_put(dev);
+		LOG_EXT();
+		return err;
+	}
+
+	mutex_unlock(mods_get_irq_mutex());
+#endif
+
+	bar_resource_offset = 0;
+	for (i = 0; i < p->bar_index; i++) {
+#if defined(MODS_HAS_IORESOURCE_MEM_64)
+		if (pci_resource_flags(dev, bar_resource_offset)
+		    & IORESOURCE_MEM_64) {
+#else
+		pci_read_config_dword(dev,
+				      (PCI_BASE_ADDRESS_0
+				       + (bar_resource_offset * 4)),
+				      &temp);
+		if (temp & PCI_BASE_ADDRESS_MEM_TYPE_64) {
+#endif
+			bar_resource_offset += 2;
+		} else {
+			bar_resource_offset += 1;
+		}
+	}
+	p->base_address = pci_resource_start(dev, bar_resource_offset);
+	p->bar_size	= pci_resource_len(dev, bar_resource_offset);
+
+	pci_dev_put(dev);
+	LOG_EXT();
+	return OK;
+}
+
+int esc_mods_pci_get_bar_info(struct mods_client           *client,
+			      struct MODS_PCI_GET_BAR_INFO *p)
+{
+	int err;
+	struct MODS_PCI_GET_BAR_INFO_2 get_bar_info = { {0} };
+
+	get_bar_info.pci_device.domain	 = 0;
+	get_bar_info.pci_device.bus	 = p->pci_device.bus;
+	get_bar_info.pci_device.device	 = p->pci_device.device;
+	get_bar_info.pci_device.function = p->pci_device.function;
+	get_bar_info.bar_index		 = p->bar_index;
+
+	err = esc_mods_pci_get_bar_info_2(client, &get_bar_info);
+
+	if (likely(!err)) {
+		p->base_address	= get_bar_info.base_address;
+		p->bar_size	= get_bar_info.bar_size;
+	}
+
+	return err;
+}
+
+int esc_mods_pci_get_irq_2(struct mods_client        *client,
+			   struct MODS_PCI_GET_IRQ_2 *p)
+{
+	struct pci_dev *dev;
+	int err;
+
+	LOG_ENT();
+
+	err = mods_find_pci_dev(client, &p->pci_device, &dev);
+	if (unlikely(err)) {
+		LOG_EXT();
+		return err;
+	}
+
+	cl_debug(DEBUG_PCI,
+		 "pci get irq dev %04x:%02x:%02x:%x irq=%u\n",
+		 p->pci_device.domain,
+		 p->pci_device.bus,
+		 p->pci_device.device,
+		 p->pci_device.function,
+		 dev->irq);
+
+	p->irq = dev->irq;
+
+	pci_dev_put(dev);
+	LOG_EXT();
+	return OK;
+}
+
+int esc_mods_pci_get_irq(struct mods_client      *client,
+			 struct MODS_PCI_GET_IRQ *p)
+{
+	int err;
+	struct MODS_PCI_GET_IRQ_2 get_irq = { {0} };
+
+	get_irq.pci_device.domain   = 0;
+	get_irq.pci_device.bus	    = p->pci_device.bus;
+	get_irq.pci_device.device   = p->pci_device.device;
+	get_irq.pci_device.function = p->pci_device.function;
+
+	err = esc_mods_pci_get_irq_2(client, &get_irq);
+
+	if (likely(!err))
+		p->irq = get_irq.irq;
+
+	return err;
+}
+
+int esc_mods_pci_read_2(struct mods_client *client, struct MODS_PCI_READ_2 *p)
+{
+	struct pci_dev *dev;
+	int err;
+	int dbdf;
+
+	LOG_ENT();
+
+	err = mods_find_pci_dev(client, &p->pci_device, &dev);
+	if (unlikely(err)) {
+		LOG_EXT();
+		return err;
+	}
+
+	if (p->address > MODS_MAX_PCI_CFG_ADDR) {
+		cl_error("invalid pci config space address 0x%x\n", p->address);
+		LOG_EXT();
+		return -EINVAL;
+	}
+
+	p->data = 0;
+	switch (p->data_size) {
+	case 1: {
+			u8 value;
+
+			pci_read_config_byte(dev, p->address, &value);
+			p->data = value;
+		}
+		break;
+	case 2: {
+			u16 value;
+
+			pci_read_config_word(dev, p->address, &value);
+			p->data = value;
+		}
+		break;
+	case 4:
+		pci_read_config_dword(dev, p->address, (u32 *) &p->data);
+		break;
+	default:
+		err = -EINVAL;
+		break;
+	}
+
+	cl_debug(DEBUG_PCI | DEBUG_DETAILED,
+		 "pci read dev %04x:%02x:%02x.%x, addr 0x%04x, size %u, data 0x%x\n",
+		 p->pci_device.domain,
+		 p->pci_device.bus,
+		 p->pci_device.device,
+		 p->pci_device.function,
+		 p->address,
+		 p->data_size,
+		 p->data);
+
+	dbdf = (int)(((u32)p->pci_device.domain << 16) |
+		     ((u32)(p->pci_device.bus & 0xFFU) << 8) |
+		     (u32)(p->pci_device.device & 0xFFU));
+
+	/* Usually one of the first reads from PCI config space occurs
+	 * at address 0 and or 2 to read PCI device vendor/id.
+	 * If this reads all Fs, the device probably fell off the bus.
+	 */
+	if (p->address <= 4 && (p->data == ~0U || p->data == 0xFFFFU)) {
+		if (dbdf != atomic_read(&client->last_bad_dbdf))
+			cl_warn("pci read dev %04x:%02x:%02x.%x, addr 0x%04x, size %u, data 0x%x\n",
+				p->pci_device.domain,
+				p->pci_device.bus,
+				p->pci_device.device,
+				p->pci_device.function,
+				p->address,
+				p->data_size,
+				p->data);
+		atomic_set(&client->last_bad_dbdf, dbdf);
+	} else if (dbdf == atomic_read(&client->last_bad_dbdf))
+		atomic_set(&client->last_bad_dbdf, -1);
+
+	pci_dev_put(dev);
+	LOG_EXT();
+	return err;
+}
+
+int esc_mods_pci_read(struct mods_client *client, struct MODS_PCI_READ *p)
+{
+	int err;
+	struct MODS_PCI_READ_2 pci_read = { {0} };
+
+	if (p->bus_number > 0xFFU) {
+		cl_error("invalid bus number 0x%x\n", p->bus_number);
+		return -EINVAL;
+	}
+	if (p->device_number > 0xFFU) {
+		cl_error("invalid device number 0x%x\n", p->device_number);
+		return -EINVAL;
+	}
+	if (p->function_number > 0xFU) {
+		cl_error("invalid function number 0x%x\n", p->function_number);
+		return -EINVAL;
+	}
+
+	pci_read.pci_device.domain	= 0;
+	pci_read.pci_device.bus		= p->bus_number;
+	pci_read.pci_device.device	= p->device_number;
+	pci_read.pci_device.function	= p->function_number;
+	pci_read.address		= p->address;
+	pci_read.data_size		= p->data_size;
+
+	err = esc_mods_pci_read_2(client, &pci_read);
+
+	if (likely(!err))
+		p->data = pci_read.data;
+
+	return err;
+}
+
+int esc_mods_pci_write_2(struct mods_client *client, struct MODS_PCI_WRITE_2 *p)
+{
+	struct pci_dev *dev;
+	int err;
+
+	LOG_ENT();
+
+	cl_debug(DEBUG_PCI | DEBUG_DETAILED,
+		 "pci write dev %04x:%02x:%02x.%x, addr 0x%04x, size %u, data 0x%x\n",
+		 p->pci_device.domain,
+		 p->pci_device.bus,
+		 p->pci_device.device,
+		 p->pci_device.function,
+		 p->address,
+		 p->data_size,
+		 p->data);
+
+	err = mods_find_pci_dev(client, &p->pci_device, &dev);
+	if (unlikely(err)) {
+		if (err == -ENODEV)
+			cl_error("dev %04x:%02x:%02x.%x not found\n",
+				 p->pci_device.domain,
+				 p->pci_device.bus,
+				 p->pci_device.device,
+				 p->pci_device.function);
+		LOG_EXT();
+		return err;
+	}
+
+	if (p->address > MODS_MAX_PCI_CFG_ADDR) {
+		cl_error("invalid pci config space address 0x%x\n", p->address);
+		LOG_EXT();
+		return -EINVAL;
+	}
+
+	switch (p->data_size) {
+	case 1:
+		if (p->data > 0xFFU) {
+			cl_error("invalid byte data 0x%x\n", p->data);
+			err = -EINVAL;
+		} else
+			pci_write_config_byte(dev, p->address, p->data);
+		break;
+	case 2:
+		if (p->data > 0xFFFFU) {
+			cl_error("invalid word data 0x%x\n", p->data);
+			err = -EINVAL;
+		} else
+			pci_write_config_word(dev, p->address, p->data);
+		break;
+	case 4:
+		pci_write_config_dword(dev, p->address, p->data);
+		break;
+	default:
+		cl_error("invalid data size %u\n", p->data_size);
+		err = -EINVAL;
+		break;
+	}
+
+	pci_dev_put(dev);
+	LOG_EXT();
+	return err;
+}
+
+int esc_mods_pci_write(struct mods_client    *client,
+		       struct MODS_PCI_WRITE *p)
+{
+	struct MODS_PCI_WRITE_2 pci_write = { {0} };
+
+	if (p->bus_number > 0xFFU) {
+		cl_error("invalid bus number 0x%x\n", p->bus_number);
+		return -EINVAL;
+	}
+	if (p->device_number > 0xFFU) {
+		cl_error("invalid device number 0x%x\n", p->device_number);
+		return -EINVAL;
+	}
+	if (p->function_number > 0xFU) {
+		cl_error("invalid function number 0x%x\n", p->function_number);
+		return -EINVAL;
+	}
+
+	pci_write.pci_device.domain	= 0;
+	pci_write.pci_device.bus	= p->bus_number;
+	pci_write.pci_device.device	= p->device_number;
+	pci_write.pci_device.function	= p->function_number;
+	pci_write.address		= p->address;
+	pci_write.data			= p->data;
+	pci_write.data_size		= p->data_size;
+
+	return esc_mods_pci_write_2(client, &pci_write);
+}
+
+int esc_mods_pci_bus_add_dev(struct mods_client              *client,
+			     struct MODS_PCI_BUS_ADD_DEVICES *scan)
+{
+	struct MODS_PCI_BUS_RESCAN rescan = { 0, 0 };
+
+	if (scan->bus > 0xFFU) {
+		cl_error("invalid bus number 0x%x\n", scan->bus);
+		return -EINVAL;
+	}
+
+	rescan.bus = (u16)scan->bus;
+
+	return esc_mods_pci_bus_rescan(client, &rescan);
+}
+
+int esc_mods_pci_bus_rescan(struct mods_client         *client,
+			    struct MODS_PCI_BUS_RESCAN *rescan)
+{
+#ifndef MODS_HASNT_PCI_RESCAN_BUS
+	struct pci_bus *bus;
+	int    err = OK;
+
+	LOG_ENT();
+
+	cl_info("scanning pci bus %04x:%02x\n", rescan->domain, rescan->bus);
+
+	bus = pci_find_bus(rescan->domain, rescan->bus);
+
+	if (likely(bus)) {
+#ifndef MODS_HASNT_PCI_LOCK_RESCAN_REMOVE
+		pci_lock_rescan_remove();
+#endif
+		pci_rescan_bus(bus);
+#ifndef MODS_HASNT_PCI_LOCK_RESCAN_REMOVE
+		pci_unlock_rescan_remove();
+#endif
+	} else {
+		cl_error("bus %04x:%02x not found\n",
+			 rescan->domain,
+			 rescan->bus);
+		err = -EINVAL;
+	}
+
+	LOG_EXT();
+
+	return err;
+#else
+	return -EINVAL;
+#endif
+}
+
+int esc_mods_pci_bus_remove_dev(struct mods_client             *client,
+				struct MODS_PCI_BUS_REMOVE_DEV *p)
+{
+#if !defined(MODS_HASNT_PCI_BUS_REMOVE_DEV)
+	struct pci_dev *dev;
+	int err;
+
+	LOG_ENT();
+
+	err = mods_find_pci_dev(client, &p->pci_device, &dev);
+	if (unlikely(err)) {
+		if (err == -ENODEV)
+			cl_error(
+				"pci_remove cannot find dev %04x:%02x:%02x.%x\n",
+				p->pci_device.domain,
+				p->pci_device.bus,
+				p->pci_device.device,
+				p->pci_device.function);
+		LOG_EXT();
+		return err;
+	}
+
+	cl_debug(DEBUG_PCI,
+		 "pci remove on dev %04x:%02x:%02x.%x\n",
+		 p->pci_device.domain,
+		 p->pci_device.bus,
+		 p->pci_device.device,
+		 p->pci_device.function);
+
+	pci_stop_and_remove_bus_device(dev);
+	LOG_EXT();
+	return err;
+#else
+	return -EINVAL;
+#endif
+}
+
+/************************
+ * PIO ESCAPE FUNCTIONS *
+ ************************/
+
+int esc_mods_pio_read(struct mods_client *client, struct MODS_PIO_READ *p)
+{
+	LOG_ENT();
+	switch (p->data_size) {
+	case 1:
+		p->data = inb(p->port);
+		break;
+	case 2:
+		p->data = inw(p->port);
+		break;
+	case 4:
+		p->data = inl(p->port);
+		break;
+	default:
+		return -EINVAL;
+	}
+	LOG_EXT();
+	return OK;
+}
+
+int esc_mods_pio_write(struct mods_client *client, struct MODS_PIO_WRITE *p)
+{
+	int err = OK;
+
+	LOG_ENT();
+
+	switch (p->data_size) {
+	case 1:
+		if (p->data > 0xFFU) {
+			cl_error("invalid byte data 0x%x\n", p->data);
+			err = -EINVAL;
+		} else
+			outb(p->data, p->port);
+		break;
+	case 2:
+		if (p->data > 0xFFFFU) {
+			cl_error("invalid word data 0x%x\n", p->data);
+			err = -EINVAL;
+		} else
+			outw(p->data, p->port);
+		break;
+	case 4:
+		outl(p->data, p->port);
+		break;
+	default:
+		cl_error("invalid data size %u\n", p->data_size);
+		err = -EINVAL;
+	}
+
+	LOG_EXT();
+	return err;
+}
+
+int esc_mods_device_numa_info_3(struct mods_client             *client,
+				struct MODS_DEVICE_NUMA_INFO_3 *p)
+{
+	struct pci_dev *dev;
+	int err;
+
+	LOG_ENT();
+
+	err = mods_find_pci_dev(client, &p->pci_device, &dev);
+	if (unlikely(err)) {
+		if (err == -ENODEV)
+			cl_error("dev %04x:%02x:%02x.%x not found\n",
+				 p->pci_device.domain,
+				 p->pci_device.bus,
+				 p->pci_device.device,
+				 p->pci_device.function);
+		LOG_EXT();
+		return err;
+	}
+
+	p->node = dev_to_node(&dev->dev);
+	if (p->node != -1) {
+		u32                  first_offset = ~0U;
+		unsigned int         i;
+		const unsigned long *maskp;
+
+		maskp = cpumask_bits(cpumask_of_node(p->node));
+
+		memset(&p->node_cpu_mask, 0, sizeof(p->node_cpu_mask));
+
+		for (i = 0; i < nr_cpumask_bits; i += 32) {
+
+			const u32 word     = i / BITS_PER_LONG;
+			const u32 bit      = i % BITS_PER_LONG;
+			const u32 cur_mask = (u32)(maskp[word] >> bit);
+			u32       mask_idx;
+
+			if (first_offset == ~0U) {
+				if (cur_mask) {
+					first_offset             = i / 32;
+					p->first_cpu_mask_offset = first_offset;
+				} else
+					continue;
+			}
+
+			mask_idx = (i / 32) - first_offset;
+
+			if (cur_mask && mask_idx >= MAX_CPU_MASKS_3) {
+
+				cl_error("too many CPUs (%d) for mask bits\n",
+					 nr_cpumask_bits);
+				pci_dev_put(dev);
+				LOG_EXT();
+				return -EINVAL;
+			}
+
+			if (mask_idx < MAX_CPU_MASKS_3)
+				p->node_cpu_mask[mask_idx] = cur_mask;
+		}
+
+		if (first_offset == ~0U)
+			p->first_cpu_mask_offset = 0;
+	}
+	p->node_count = num_possible_nodes();
+	p->cpu_count  = num_possible_cpus();
+
+	pci_dev_put(dev);
+	LOG_EXT();
+	return OK;
+}
+
+int esc_mods_device_numa_info_2(struct mods_client             *client,
+				struct MODS_DEVICE_NUMA_INFO_2 *p)
+{
+	int err;
+	struct MODS_DEVICE_NUMA_INFO_3 numa_info = { {0} };
+
+	numa_info.pci_device = p->pci_device;
+
+	err = esc_mods_device_numa_info_3(client, &numa_info);
+
+	if (likely(!err)) {
+		int i;
+
+		p->node	      = numa_info.node;
+		p->node_count = numa_info.node_count;
+		p->cpu_count  = numa_info.cpu_count;
+
+		memset(&p->node_cpu_mask, 0, sizeof(p->node_cpu_mask));
+
+		for (i = 0; i < MAX_CPU_MASKS_3; i++) {
+
+			const u32 cur_mask = numa_info.node_cpu_mask[i];
+			const u32 dst      = i +
+					     numa_info.first_cpu_mask_offset;
+
+			if (cur_mask && dst >= MAX_CPU_MASKS) {
+				cl_error("too many CPUs (%d) for mask bits\n",
+					 nr_cpumask_bits);
+				err = -EINVAL;
+				break;
+			}
+
+			if (dst < MAX_CPU_MASKS)
+				p->node_cpu_mask[dst]
+					= numa_info.node_cpu_mask[i];
+		}
+	}
+
+	return err;
+}
+
+int esc_mods_device_numa_info(struct mods_client           *client,
+			      struct MODS_DEVICE_NUMA_INFO *p)
+{
+	int err;
+	struct MODS_DEVICE_NUMA_INFO_3 numa_info = { {0} };
+
+	numa_info.pci_device.domain    = 0;
+	numa_info.pci_device.bus       = p->pci_device.bus;
+	numa_info.pci_device.device    = p->pci_device.device;
+	numa_info.pci_device.function  = p->pci_device.function;
+
+	err = esc_mods_device_numa_info_3(client, &numa_info);
+
+	if (likely(!err)) {
+		int i;
+
+		p->node	      = numa_info.node;
+		p->node_count = numa_info.node_count;
+		p->cpu_count  = numa_info.cpu_count;
+
+		memset(&p->node_cpu_mask, 0, sizeof(p->node_cpu_mask));
+
+		for (i = 0; i < MAX_CPU_MASKS_3; i++) {
+
+			const u32 cur_mask = numa_info.node_cpu_mask[i];
+			const u32 dst      = i +
+					     numa_info.first_cpu_mask_offset;
+
+			if (cur_mask && dst >= MAX_CPU_MASKS) {
+				cl_error("too many CPUs (%d) for mask bits\n",
+					 nr_cpumask_bits);
+				err = -EINVAL;
+				break;
+			}
+
+			if (dst < MAX_CPU_MASKS)
+				p->node_cpu_mask[dst]
+					= numa_info.node_cpu_mask[i];
+		}
+	}
+
+	return err;
+}
+
+int esc_mods_get_iommu_state(struct mods_client          *client,
+			     struct MODS_GET_IOMMU_STATE *state)
+{
+	int err = esc_mods_get_iommu_state_2(client, state);
+
+	if (!err)
+		state->state = (state->state == MODS_SWIOTLB_DISABLED) ? 1 : 0;
+
+	return err;
+}
+
+int esc_mods_get_iommu_state_2(struct mods_client          *client,
+			       struct MODS_GET_IOMMU_STATE *state)
+{
+#if !defined(CONFIG_SWIOTLB)
+	state->state = MODS_SWIOTLB_DISABLED;
+#elif defined(MODS_HAS_DMA_OPS)
+
+	const struct dma_map_ops *ops;
+	struct pci_dev           *dev;
+	int                       err;
+
+	LOG_ENT();
+
+	err = mods_find_pci_dev(client, &state->pci_device, &dev);
+	if (unlikely(err)) {
+		LOG_EXT();
+		return err;
+	}
+
+	ops = get_dma_ops(&dev->dev);
+
+	state->state = ops->map_sg != swiotlb_map_sg_attrs
+		       ? MODS_SWIOTLB_DISABLED : MODS_SWIOTLB_ACTIVE;
+
+	pci_dev_put(dev);
+	LOG_EXT();
+
+#else
+	/* No way to detect it */
+	state->state = MODS_SWIOTLB_INDETERMINATE;
+#endif
+	return OK;
+}
+
+int esc_mods_pci_set_dma_mask(struct mods_client       *client,
+			      struct MODS_PCI_DMA_MASK *dma_mask)
+{
+	int             err;
+	struct pci_dev *dev;
+	u64             mask;
+
+	LOG_ENT();
+
+	if (unlikely(dma_mask->num_bits > 64)) {
+		cl_error("num_bits=%u exceeds 64\n",
+			 dma_mask->num_bits);
+		LOG_EXT();
+		return -EINVAL;
+	}
+
+	err = mods_find_pci_dev(client, &dma_mask->pci_device, &dev);
+	if (unlikely(err)) {
+		if (err == -ENODEV)
+			cl_error("dev %04x:%02x:%02x.%x not found\n",
+				 dma_mask->pci_device.domain,
+				 dma_mask->pci_device.bus,
+				 dma_mask->pci_device.device,
+				 dma_mask->pci_device.function);
+		LOG_EXT();
+		return err;
+	}
+
+	mask = dma_mask->num_bits == 64 ? ~0ULL : (1ULL<<dma_mask->num_bits)-1;
+
+	err = dma_set_mask(&dev->dev, mask);
+	if (err) {
+		cl_error(
+			"failed to set dma mask 0x%llx (%u) for dev %04x:%02x:%02x.%x\n",
+			mask,
+			dma_mask->num_bits,
+			dma_mask->pci_device.domain,
+			dma_mask->pci_device.bus,
+			dma_mask->pci_device.device,
+			dma_mask->pci_device.function);
+#if defined(CONFIG_PPC64)
+		/* Ignore error if TCE bypass is on */
+		if (dev->dma_mask == ~0ULL)
+			err = OK;
+#endif
+	} else {
+#if defined(MODS_HAS_SET_COHERENT_MASK)
+		err = dma_set_coherent_mask(&dev->dev, mask);
+#else
+		err = pci_set_consistent_dma_mask(dev, mask);
+#endif
+		if (err)
+			cl_error(
+				"failed to set consistent dma mask 0x%llx (%u) for dev %04x:%02x:%02x.%x\n",
+				mask,
+				dma_mask->num_bits,
+				dma_mask->pci_device.domain,
+				dma_mask->pci_device.bus,
+				dma_mask->pci_device.device,
+				dma_mask->pci_device.function);
+	}
+
+	if (!err)
+		cl_info("set dma mask %u for dev %04x:%02x:%02x.%x\n",
+			dma_mask->num_bits,
+			dma_mask->pci_device.domain,
+			dma_mask->pci_device.bus,
+			dma_mask->pci_device.device,
+			dma_mask->pci_device.function);
+
+	pci_dev_put(dev);
+	LOG_EXT();
+	return err;
+}
+
+static int check_flr(struct pci_dev *dev)
+{
+	int cap_pos;
+	u32 cap;
+
+#if KERNEL_VERSION(4, 12, 0) <= MODS_KERNEL_VERSION
+	if (dev->dev_flags & PCI_DEV_FLAGS_NO_FLR_RESET)
+		return -ENOTTY;
+#endif
+
+	cap_pos = pci_find_capability(dev, PCI_CAP_ID_EXP);
+	if (!cap_pos)
+		return -ENOTTY;
+
+	pci_read_config_dword(dev, cap_pos + PCI_EXP_DEVCAP, &cap);
+	if (!(cap & PCI_EXP_DEVCAP_FLR))
+		return -ENOTTY;
+
+	return 0;
+}
+
+int esc_mods_pci_reset_function(struct mods_client    *client,
+				struct mods_pci_dev_2 *pcidev)
+{
+	struct pci_dev *dev;
+	int             err;
+
+	LOG_ENT();
+
+	err = mods_find_pci_dev(client, pcidev, &dev);
+	if (unlikely(err)) {
+		if (err == -ENODEV)
+			cl_error("dev %04x:%02x:%02x.%x not found\n",
+				 pcidev->domain,
+				 pcidev->bus,
+				 pcidev->device,
+				 pcidev->function);
+		LOG_EXT();
+		return err;
+	}
+
+	err = check_flr(dev);
+	if (unlikely(err)) {
+		cl_error(
+			 "function level reset not supported on dev %04x:%02x:%02x.%x\n",
+			 pcidev->domain,
+			 pcidev->bus,
+			 pcidev->device,
+			 pcidev->function);
+		goto error;
+	}
+
+#if KERNEL_VERSION(2, 32, 0) <= MODS_KERNEL_VERSION
+	pm_runtime_get_sync(&dev->dev);
+#endif
+
+	err = pci_reset_function(dev);
+
+#if KERNEL_VERSION(2, 32, 0) <= MODS_KERNEL_VERSION
+	pm_runtime_put(&dev->dev);
+#endif
+
+	if (unlikely(err))
+		cl_error("pcie_flr failed on dev %04x:%02x:%02x.%x\n",
+			 pcidev->domain,
+			 pcidev->bus,
+			 pcidev->device,
+			 pcidev->function);
+	else
+		cl_info("pcie_flr succeeded on dev %04x:%02x:%02x.%x\n",
+			pcidev->domain,
+			pcidev->bus,
+			pcidev->device,
+			pcidev->function);
+
+error:
+	pci_dev_put(dev);
+	LOG_EXT();
+	return err;
+}
+
+#ifdef MODS_HAS_DEV_PROPS
+int esc_mods_read_dev_property(struct mods_client            *client,
+			       struct MODS_READ_DEV_PROPERTY *p)
+{
+	struct pci_dev *dev = NULL;
+	int             err = -EINVAL;
+
+	LOG_ENT();
+
+	if (unlikely(p->type != MODS_PROP_TYPE_U64)) {
+		cl_error("invalid property type %u\n", p->type);
+		goto error;
+	}
+
+	if (unlikely(sizeof(64) * p->array_size > sizeof(p->output))) {
+		cl_error("requested size %zu exceeds output array size %zu\n",
+			 sizeof(u64) * p->array_size,
+			 sizeof(p->output));
+		goto error;
+	}
+
+	if (unlikely(p->array_size == 0)) {
+		cl_error("invalid output array size 0\n");
+		goto error;
+	}
+
+	if (!memchr(p->prop_name, 0, sizeof(p->prop_name))) {
+		cl_error("invalid property name, misses terminating NUL\n");
+		goto error;
+	}
+
+	err = mods_find_pci_dev(client, &p->pci_device, &dev);
+	if (unlikely(err)) {
+		if (err == -ENODEV)
+			cl_error("dev %04x:%02x:%02x.%x not found\n",
+				 p->pci_device.domain,
+				 p->pci_device.bus,
+				 p->pci_device.device,
+				 p->pci_device.function);
+		goto error;
+	}
+
+	err = device_property_read_u64_array(&dev->dev, p->prop_name,
+					     (u64 *)p->output, p->array_size);
+	if (unlikely(err))
+		cl_error("failed to read property %s\n", p->prop_name);
+
+error:
+	pci_dev_put(dev);
+	LOG_EXT();
+	return err;
+}
+#endif
diff --git a/drivers/misc/mods/mods_ras.c b/drivers/misc/mods/mods_ras.c
new file mode 100644
index 00000000..b470eb54
--- /dev/null
+++ b/drivers/misc/mods/mods_ras.c
@@ -0,0 +1,74 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2017-2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#include "mods_ras.h"
+
+#include <linux/err.h>
+#include <linux/platform/tegra/carmel_ras.h>
+#include <linux/platform/tegra/tegra18_cpu_map.h>
+
+/* Encodes requested core/cluster to report
+ * ras for. If 0, refers to ccplex
+ */
+static u64 ras_ccplex_config;
+
+void enable_cpu_core_reporting(u64 config)
+{
+	ras_ccplex_config = config;
+}
+
+void set_err_sel(u64 sel_val)
+{
+	u8 core, cluster, ccplex;
+	u64 errx;
+
+	ccplex = 0;
+	core = ras_ccplex_config >> 1;
+	cluster = ras_ccplex_config & 1;
+	if (!ras_ccplex_config)
+		ccplex = 1;
+
+	if (ccplex) {
+		errx = sel_val;
+	} else {
+		if (!cluster) {
+			errx = (tegra18_logical_to_cluster(core) << 5) +
+				(tegra18_logical_to_cpu(core) << 4) +
+				sel_val;
+		} else {
+			errx = 512 + (tegra18_logical_to_cluster(core) << 4)
+				+ sel_val;
+		}
+	}
+
+	mods_debug_printk(DEBUG_FUNC, "ERR_SEL is %llu, Core is %u\n",
+							errx, core);
+
+	ras_write_errselr(errx);
+}
+
+void set_err_ctrl(u64 ctrl_val)
+{
+	u64 updated_val;
+
+	ras_write_error_control(ctrl_val);
+	updated_val = ras_read_error_control();
+	mods_debug_printk(DEBUG_ALL, "ERR_CTRL updated value is %llu",
+							updated_val);
+}
+
+u64 get_err_ctrl(void)
+{
+	return ras_read_error_control();
+}
diff --git a/drivers/misc/mods/mods_ras.h b/drivers/misc/mods/mods_ras.h
new file mode 100644
index 00000000..4fd9d851
--- /dev/null
+++ b/drivers/misc/mods/mods_ras.h
@@ -0,0 +1,39 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * This file is part of NVIDIA MODS kernel driver.
+ *
+ * Copyright (c) 2019-2022, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * NVIDIA MODS kernel driver is free software: you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * NVIDIA MODS kernel driver is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NVIDIA MODS kernel driver.
+ * If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "mods_internal.h"
+
+void enable_cpu_core_reporting(u64 config);
+
+/*Set the ERR_SEL register to choose the
+ *node for which to enable/disable errors for
+ */
+void set_err_sel(u64 sel_val);
+
+/*Set the ERR_CTRL register selected
+ *by ERR_SEL
+ */
+void set_err_ctrl(u64 ctrl_val);
+
+/*Get the ERR_CTRL register selected
+ *by ERR_SEL
+ */
+u64 get_err_ctrl(void);
+
diff --git a/drivers/misc/mods/mods_smmu_drv.c b/drivers/misc/mods/mods_smmu_drv.c
new file mode 100644
index 00000000..9d2134b1
--- /dev/null
+++ b/drivers/misc/mods/mods_smmu_drv.c
@@ -0,0 +1,161 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * This file is part of NVIDIA MODS kernel driver.
+ *
+ * Copyright (c) 2019-2023, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * NVIDIA MODS kernel driver is free software: you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * NVIDIA MODS kernel driver is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NVIDIA MODS kernel driver.
+ * If not, see <http://www.gnu.org/licenses/>.
+ */
+#include <linux/kernel.h>
+#include <linux/device.h>
+#include <linux/slab.h>
+#include <linux/module.h>
+#include <linux/vmalloc.h>
+#include <linux/dma-mapping.h>
+#include <linux/debugfs.h>
+#include <linux/sched.h>
+#include <linux/version.h>
+#include <linux/dma-buf.h>
+
+#include <linux/of.h>
+#include <linux/of_platform.h>
+#include <linux/acpi.h>
+#include "mods_internal.h"
+
+#define MODS_MAX_SMMU_DEVICES 16
+static struct mods_smmu_dev mods_smmu_devs[MODS_MAX_SMMU_DEVICES];
+static int mods_smmu_dev_num;
+
+struct mods_smmu_dev *get_mods_smmu_device(u32 index)
+{
+	struct mods_smmu_dev *pdev = NULL;
+
+	if (index < mods_smmu_dev_num)
+		pdev = &mods_smmu_devs[index];
+	else
+		mods_error_printk("mods smmu dev index %d error\n", index);
+	return pdev;
+}
+
+int get_mods_smmu_device_index(const char *name)
+{
+	int idx;
+	int dev_num = mods_smmu_dev_num;
+
+	for (idx = 0; idx < dev_num; idx++) {
+		if (strcmp(mods_smmu_devs[idx].dev_name, name) == 0)
+			return idx;
+	}
+	mods_error_printk("mods smmu device %s not found\n", name);
+	return -EINVAL;
+}
+
+static int mods_smmu_driver_probe(struct platform_device *pdev)
+{
+	struct device      *dev = &pdev->dev;
+	const char         *dev_name = NULL;
+	int                err = 0;
+	int                dev_idx;
+
+	LOG_ENT();
+
+	if (device_property_present(dev, "dev-names")) {
+		err = device_property_read_string(dev, "dev-names", &dev_name);
+		if (err < 0) {
+			mods_error_printk(
+				"smmu probe failed to read dev-names, ret=%d\n", err);
+			LOG_EXT();
+			return err;
+		}
+
+	} else {
+#if KERNEL_VERSION(5, 10, 0) <= MODS_KERNEL_VERSION
+		dev_name = (const char *)acpi_device_get_match_data(dev);
+#endif
+		if (!dev_name) {
+			mods_error_printk(
+				"smmu probe failed to read dev-names\n");
+			LOG_EXT();
+			return -EINVAL;
+		}
+	}
+	mods_debug_printk(DEBUG_MEM, "smmu probe: dev-names=%s, dev_idx=%d\n",
+			  dev_name,
+			  mods_smmu_dev_num);
+	if (mods_smmu_dev_num < MODS_MAX_SMMU_DEVICES) {
+		dev_idx = mods_smmu_dev_num;
+		mods_smmu_dev_num++;
+	} else {
+		mods_error_printk("Max Number of MODS Smmu Device Reached\n");
+		LOG_EXT();
+		return -ENOMEM;
+	}
+	mods_smmu_devs[dev_idx].dev = &pdev->dev;
+	strncpy(mods_smmu_devs[dev_idx].dev_name,
+		dev_name,
+		MAX_DT_SIZE - 1);
+#ifdef MODS_ENABLE_BPMP_MRQ_API
+	mods_smmu_devs[dev_idx].bpmp = tegra_bpmp_get(dev);
+#endif
+
+
+	LOG_EXT();
+	return err;
+}
+
+static int mods_smmu_driver_remove(struct platform_device *pdev)
+{
+	mods_smmu_dev_num = 0;
+	return 0;
+}
+
+static const struct of_device_id of_ids[] = {
+	{ .compatible = "nvidia,mods_smmu" },
+	{ }
+};
+
+#ifdef CONFIG_ACPI
+static const struct acpi_device_id tegra_mods_acpi_match[] = {
+	{
+		.id = "NVDA1513",
+		.driver_data = (kernel_ulong_t)"mods_qspi1_dma",
+	},
+	{}
+};
+
+MODULE_DEVICE_TABLE(acpi, tegra_mods_acpi_match);
+#endif
+
+static struct platform_driver mods_smmu_driver = {
+	.probe  = mods_smmu_driver_probe,
+	.remove = mods_smmu_driver_remove,
+	.driver = {
+		.name   = "mods_smmu",
+		.owner  = THIS_MODULE,
+		.of_match_table = of_ids,
+		.acpi_match_table = ACPI_PTR(tegra_mods_acpi_match),
+	},
+};
+
+int smmu_driver_init(void)
+{
+	mods_smmu_dev_num = 0;
+	memset(&mods_smmu_devs[0], 0, sizeof(mods_smmu_devs));
+	return platform_driver_register(&mods_smmu_driver);
+}
+
+void smmu_driver_exit(void)
+{
+	platform_driver_unregister(&mods_smmu_driver);
+}
diff --git a/drivers/misc/mods/mods_tegradc.c b/drivers/misc/mods/mods_tegradc.c
new file mode 100644
index 00000000..5223c81e
--- /dev/null
+++ b/drivers/misc/mods/mods_tegradc.c
@@ -0,0 +1,154 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * This file is part of NVIDIA MODS kernel driver.
+ *
+ * Copyright (c) 2014-2022, NVIDIA CORPORATION. All rights reserved.
+ *
+ * NVIDIA MODS kernel driver is free software: you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * NVIDIA MODS kernel driver is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NVIDIA MODS kernel driver.
+ * If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/uaccess.h>
+#include <../drivers/video/tegra/dc/dc_priv.h>
+#include <video/tegra_dc_ext_kernel.h>
+#include <linux/platform/tegra/mc.h>
+#include "mods_internal.h"
+
+static void mods_tegra_dc_set_windowattr_basic(
+	struct mods_client                *client,
+	struct tegra_dc_win               *win,
+	const struct MODS_TEGRA_DC_WINDOW *mods_win)
+{
+	win->global_alpha = 0;
+	win->z            = 0;
+	win->stride       = 0;
+	win->stride_uv    = 0;
+
+	win->flags = TEGRA_WIN_FLAG_ENABLED;
+	if (mods_win->flags & MODS_TEGRA_DC_WINDOW_FLAG_TILED)
+		win->flags |= TEGRA_WIN_FLAG_TILED;
+	if (mods_win->flags & MODS_TEGRA_DC_WINDOW_FLAG_SCAN_COL)
+		win->flags |= TEGRA_WIN_FLAG_SCAN_COLUMN;
+
+	win->fmt = mods_win->pixformat;
+	win->x.full = mods_win->x;
+	win->y.full = mods_win->y;
+	win->w.full = mods_win->w;
+	win->h.full = mods_win->h;
+	/* XXX verify that this doesn't go outside display's active region */
+	win->out_x = mods_win->out_x;
+	win->out_y = mods_win->out_y;
+	win->out_w = mods_win->out_w;
+	win->out_h = mods_win->out_h;
+
+	cl_debug(DEBUG_TEGRADC,
+		 "set_windowattr_basic window %u:\n"
+		 "\tflags : 0x%08x\n"
+		 "\tfmt   : %u\n"
+		 "\tinput : (%u, %u, %u, %u)\n"
+		 "\toutput: (%u, %u, %u, %u)\n",
+		 win->idx, win->flags, win->fmt, dfixed_trunc(win->x),
+		 dfixed_trunc(win->y), dfixed_trunc(win->w),
+		 dfixed_trunc(win->h), win->out_x, win->out_y, win->out_w,
+		 win->out_h);
+}
+
+int esc_mods_tegra_dc_config_possible(struct mods_client *client,
+				struct MODS_TEGRA_DC_CONFIG_POSSIBLE *args)
+{
+	int i;
+	struct tegra_dc *dc = tegra_dc_get_dc(args->head);
+	struct tegra_dc_win *dc_wins[DC_N_WINDOWS];
+#ifndef CONFIG_TEGRA_ISOMGR
+	struct clk *emc_clk = 0;
+	unsigned long max_bandwidth = 0;
+	unsigned long current_emc_freq = 0;
+	unsigned long max_available_bandwidth = 0;
+#else
+	int ret = -EINVAL;
+#endif
+
+	LOG_ENT();
+
+	BUG_ON(args->win_num > tegra_dc_get_numof_dispwindows());
+
+	if (!dc) {
+		LOG_EXT();
+		return -EINVAL;
+	}
+
+	for (i = 0; i < args->win_num; i++) {
+		unsigned int idx;
+
+		if (args->windows[i].index < 0) {
+			cl_debug(DEBUG_TEGRADC,
+				 "invalid index %d for win %d\n",
+				 i, args->windows[i].index);
+			return -EINVAL;
+		}
+		idx = (unsigned int)args->windows[i].index;
+
+		if (args->windows[i].flags &
+			MODS_TEGRA_DC_WINDOW_FLAG_ENABLED) {
+			mods_tegra_dc_set_windowattr_basic(client,
+							   &dc->tmp_wins[idx],
+							   &args->windows[i]);
+		} else {
+			dc->tmp_wins[idx].flags = 0;
+		}
+		dc_wins[i] = &dc->tmp_wins[idx];
+		cl_debug(DEBUG_TEGRADC,
+			 "head %u, using index %d for win %d\n",
+			 args->head, i, idx);
+	}
+
+	cl_debug(DEBUG_TEGRADC,
+		 "head %u, dc->mode.pclk %u\n",
+		 args->head, dc->mode.pclk);
+
+#ifndef CONFIG_TEGRA_ISOMGR
+	max_bandwidth = tegra_dc_get_bandwidth(dc_wins, args->win_num);
+
+	emc_clk = clk_get_sys("tegra_emc", "emc");
+	if (IS_ERR(emc_clk)) {
+		cl_debug(DEBUG_TEGRADC,
+			 "invalid clock specified when fetching EMC clock\n");
+	} else {
+		current_emc_freq = clk_get_rate(emc_clk);
+		current_emc_freq /= 1000;
+		max_available_bandwidth =
+			8 * tegra_emc_freq_req_to_bw(current_emc_freq);
+		max_available_bandwidth = (max_available_bandwidth / 100) * 50;
+	}
+
+	cl_debug(DEBUG_TEGRADC,
+		 "b/w needed %lu, b/w available %lu\n",
+		 max_bandwidth, max_available_bandwidth);
+
+	args->possible = (max_bandwidth <= max_available_bandwidth);
+#else
+	ret = tegra_dc_bandwidth_negotiate_bw(dc, dc_wins, args->win_num);
+	args->possible = (ret == 0);
+#endif
+	for (i = 0; i < args->win_num; i++) {
+		args->windows[i].bandwidth = dc_wins[i]->new_bandwidth;
+		cl_debug(DEBUG_TEGRADC,
+			 "head %u, win %d, b/w %d\n",
+			 args->head,
+			 dc_wins[i]->idx,
+			 dc_wins[i]->new_bandwidth);
+	}
+
+	LOG_EXT();
+	return 0;
+}
diff --git a/drivers/misc/mods/mods_tegraprod.c b/drivers/misc/mods/mods_tegraprod.c
new file mode 100644
index 00000000..2a2e83f1
--- /dev/null
+++ b/drivers/misc/mods/mods_tegraprod.c
@@ -0,0 +1,770 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * This file is part of NVIDIA MODS kernel driver.
+ *
+ * Copyright (c) 2017-2022, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * NVIDIA MODS kernel driver is free software: you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * NVIDIA MODS kernel driver is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NVIDIA MODS kernel driver.
+ * If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "mods_internal.h"
+#include <linux/err.h>
+#include <linux/miscdevice.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/of_device.h>
+#include <linux/tegra_prod.h>
+
+
+#define MAX_REG_INFO_ENTRY	400
+#define MAX_IO_MAP_ENTRY	200
+
+static struct device *mods_tegra_prod_dev;
+
+/**
+ * mods_tegra_prod_init - Initialize tegra prod module.
+ * @misc_dev:       pointer to mods device
+ *
+ * Returns 0 on success, others for error.
+ */
+int mods_tegra_prod_init(const struct miscdevice *misc_dev)
+{
+	if (!misc_dev) {
+		mods_error_printk("mods device pointer is NULL\n");
+		return -EINVAL;
+	}
+
+	mods_tegra_prod_dev = misc_dev->this_device;
+	if (!mods_tegra_prod_dev) {
+		mods_error_printk("this_device in mods device is NULL\n");
+		return -EINVAL;
+	}
+
+	return OK;
+}
+
+/**
+ * esc_mods_tegra_prod_iterate_dt - Find device_node by device name.
+ * @MODS_TEGRA_PROD_ITERATOR: Input information, only the following
+ *				members are used.
+ *   +device_handle:	Last node pointer for current iteration
+ *   +name:		Current node name
+ *   +next_name:	Next node name
+ *   +index:		Index of the device nodes shared the same name
+ *			(NOTE: index starts from 0)
+ *   +is_leaf:		Is the node with current name a leaf node
+ *   +next_device_handle: Result node pointer for next iteration
+ * Returns 0 on success, others for error or no matching node found.
+ */
+int esc_mods_tegra_prod_iterate_dt(
+	struct mods_client *client,
+	struct MODS_TEGRA_PROD_ITERATOR *iterator
+)
+{
+	struct device_node *dev_node;
+	struct device_node *child_node = NULL;
+	struct resource res;
+	int ret_resource = -1;
+	int i;
+
+	dev_node = (struct device_node *)iterator->device_handle;
+
+	if (!iterator->name[0]) {
+		cl_error("node name is missing for tegra prod value\n");
+		return -EINVAL;
+	}
+	if (!iterator->next_name[0] && !iterator->is_leaf) {
+		cl_error("inner node with empty next_name\n");
+		return -EINVAL;
+	}
+
+	/* Need to search several times since some nodes may share same name */
+	for (i = 0; i <= iterator->index; ++i) {
+		/* Search from Left to Right in DT */
+		dev_node = of_find_node_by_name(dev_node, iterator->name);
+		if (!dev_node) {
+			cl_error("node %s not found in device tree\n",
+				 iterator->name);
+			return -EINVAL;
+		}
+
+		/*
+		 * Search from Top to Bottom in DT
+		 * Check: does this node need to be filtered out
+		 */
+		if (iterator->is_leaf) {
+			/* Leaf node : search for "prod-settings" node */
+			child_node = of_get_child_by_name(dev_node,
+							"prod-settings");
+			ret_resource = of_address_to_resource(
+							dev_node, 0, &res);
+		} else {
+			/* Inner node : search for next device node */
+			child_node = of_get_child_by_name(dev_node,
+							iterator->next_name);
+			ret_resource = -1;
+		}
+		if (!child_node && ret_resource < 0)
+			--i;	/* Not a valid device node */
+	}
+
+	/* Return next_device_handler to ioctl caller */
+	iterator->next_device_handle = (__u64)dev_node;
+	return OK;
+}
+
+/**
+ * mods_read_reg_info - Read register information (address and size)
+ * @dev_node:		 Device node pointer
+ * @reg_info:		 Register information array
+ * @count_reg_cells:	 (OUT) Pointer to count of cells in "reg"
+ * @count_addrsize_pair: (OUT) Pointer to count of each pair in "reg"
+ * @count_address_cells: (OUT) Pointer to count of "address" cell in "reg"
+ * @count_size_cells:	 (OUT) Pointer to count of "size" cell in "reg"
+ *
+ * Return Value :
+ *    0   - OK
+ * Others - ERROR
+ *
+ */
+static int mods_read_reg_info(struct mods_client       *client,
+			      const struct device_node *dev_node,
+			      __u32                    *reg_info,
+			      __u32                    *count_reg_cells,
+			      __u32                    *count_addrsize_pair,
+			      __u32                    *count_address_cells,
+			      __u32                    *count_size_cells)
+{
+	struct device_node *parent_node;
+	int ret;
+
+	/* Get parent node to read the format of "reg" property */
+	parent_node = dev_node->parent;
+	ret = of_property_read_u32(parent_node, "#address-cells",
+				count_address_cells);
+	if (ret < 0) {
+		cl_error("read #address-cells failed\n");
+		return ret;
+
+	} else if (*count_address_cells == 0) {
+		cl_error("#address-cells cannot be 0\n");
+		return -EINVAL;
+	}
+
+	ret = of_property_read_u32(parent_node, "#size-cells",
+				count_size_cells);
+	if (ret < 0) {
+		cl_error("read #size-cells failed\n");
+		return ret;
+
+	} else if (*count_size_cells == 0) {
+		cl_error("#size-cells cannot be 0\n");
+		return -EINVAL;
+	}
+	*count_addrsize_pair = *count_address_cells + *count_size_cells;
+
+	/* Read count of cells in "reg" property */
+	ret = of_property_count_u32_elems(dev_node, "reg");
+	if (ret < 0) {
+		cl_error(
+			"unable to get count of cells in \"reg\" of node %s\n",
+			dev_node->name);
+		return ret;
+	}
+
+	*count_reg_cells = (__u32)ret;
+	if (*count_reg_cells == 0) {
+		cl_error("no \"reg\" info is available\n");
+		return -EINVAL;
+
+	} else if (*count_reg_cells % *count_addrsize_pair != 0) {
+		/*
+		 * "reg" property may have more than 1 pairs of address
+		 * and size. The length of each pair is represented by
+		 * "count_addrsize_pair"; the total length of "reg"
+		 * property is represented by "count_reg_cells". If
+		 * "count_reg_cells" is not an integral multiple of
+		 * "count_addrsize_pair", the read "reg" information is
+		 * incomplete or incorrect.
+		 */
+		cl_error("\"reg\" property has invalid length : %d\n",
+			 *count_reg_cells);
+		return -EINVAL;
+	}
+
+	/* Read register information from "reg" property */
+	ret = of_property_read_u32_array(dev_node, "reg", reg_info,
+					(size_t)(*count_reg_cells));
+	if (ret < 0) {
+		cl_error("Unable to read \"reg\" property of node %s\n",
+			 dev_node->name);
+		return ret;
+	}
+
+	return OK;
+}
+
+/**
+ * mods_batch_iounmap - Unmap all virtual memory mapped by IO address.
+ * @io_base:		IO address array
+ * @count_io_base:	Count of IO addresses are mapped
+ *
+ */
+static void mods_batch_iounmap(struct mods_client *client,
+			       void __iomem      **io_base,
+			       __u32               count_io_base)
+{
+	int i;
+
+	if (!io_base) {
+		cl_error("IO base address array is NULL\n");
+		return;
+	}
+
+	for (i = 0; i < count_io_base; ++i)
+		iounmap(io_base[i]);
+}
+
+/**
+ * mods_batch_iomap - Get all segments of IO address from a device node,
+ *                    and create a virtual memory mapping.
+ * @dev_node:		Device node pointer
+ * @io_base:		(OUT) IO address array
+ * @count_io_base:	(OUT) Pointer to the count of IO addresses are mapped
+ *
+ * Return Value :
+ *    0   - OK
+ * Others - ERROR (NOTE: partially mapped IO memory will be unmapped on error)
+ *
+ */
+static int mods_batch_iomap(struct mods_client       *client,
+			    const struct device_node *dev_node,
+			    void __iomem            **io_base,
+			    __u32                    *count_io_base)
+{
+	__u32 reg_info[MAX_REG_INFO_ENTRY];
+	__u32 count_reg_cells = 0;
+	__u32 count_addrsize_pair, count_address_cells, count_size_cells;
+	__u64 io_addr_base, io_addr_length;
+	void __iomem *io_addr_mapped;
+	int ret;
+	int i;
+
+	/* Check arguments */
+	if (!dev_node) {
+		cl_error("controller device handle is NULL\n");
+		return -EINVAL;
+	}
+	if (!io_base) {
+		cl_error("IO base address array is NULL\n");
+		return -EINVAL;
+	}
+	if (!count_io_base) {
+		cl_error("count of IO base address array is NULL\n");
+		return -EINVAL;
+	}
+
+	/* Get registers information */
+	ret = mods_read_reg_info(client,
+				 dev_node,
+				 reg_info,
+				 &count_reg_cells,
+				 &count_addrsize_pair,
+				 &count_address_cells,
+				 &count_size_cells);
+	if (ret < 0)
+		return ret;
+
+	/* IO mapping - using 64-bit physical address */
+	*count_io_base = 0;
+	for (i = 0; i < count_reg_cells;
+		i += count_addrsize_pair, *count_io_base += 1) {
+
+		/* Compute IO physical address */
+		/* Lower 32-bit IO physical address in DTB */
+		io_addr_base = reg_info[i + count_address_cells - 1];
+		if (count_address_cells > 1) {
+			/* Higher 32-bit IO physical address in DTB */
+			io_addr_base += ((__u64)reg_info
+					[i + count_address_cells - 2]) << 32;
+		}
+
+		/* Compute IO address space length */
+		/* Lower 32-bit IO address space length in DTB */
+		io_addr_length = reg_info[i + count_addrsize_pair - 1];
+		if (count_size_cells > 1) {
+			/* Higher 32-bit IO address space length in DTB */
+			io_addr_length += ((__u64)reg_info
+					[i + count_addrsize_pair - 2]) << 32;
+		}
+
+		/* Map physical address to virtual address space */
+		io_addr_mapped = ioremap((resource_size_t)io_addr_base,
+					(resource_size_t)io_addr_length);
+		if (io_addr_mapped == NULL) {
+			cl_error(
+				"Unable to map io address 0x%llx, length 0x%llx\n",
+				io_addr_base,
+				io_addr_length);
+			/* Clean :
+			 * Unmap the IO memory that have already been mapped
+			 */
+			mods_batch_iounmap(client, io_base, *count_io_base);
+			return -ENXIO;
+		}
+		io_base[*count_io_base] = io_addr_mapped;
+	}
+
+	return OK;
+}
+
+
+/**
+ * mods_tegra_get_prod_list - Get tegra_prod list by valid device node.
+ * @dev_node:       Device node.
+ *
+ * Returns non-NULL on success, NULL on error.
+ */
+static struct tegra_prod *mods_tegra_get_prod_list(struct mods_client *client,
+						   struct device_node *dev_node)
+{
+	struct tegra_prod *prod_list;
+
+	if (!mods_tegra_prod_dev) {
+		cl_error("tegra prod is not initialized\n");
+		return NULL;
+	}
+
+	if (!dev_node) {
+		cl_error("device node is NULL\n");
+		return NULL;
+	}
+
+	prod_list = devm_tegra_prod_get_from_node(mods_tegra_prod_dev,
+						dev_node);
+	if (IS_ERR(prod_list)) {
+		cl_error("failed to get prod_list : %s\n",
+				dev_node->name);
+		return NULL;
+	}
+
+	return prod_list;
+}
+
+/**
+ * mods_tegra_get_prod_info - Get tegra_prod_list & mapped IO addresses
+ * @MODS_TEGRA_PROD_SET_TUPLE:  Input information, only the following
+ *				members are used.
+ *   +prod_node_handle:	Handle of device node contained prod values.
+ *   +ctrl_node_handle:	Handle of controller device node.
+ * @tegra_prod_list:	(OUT) Pointer to tegra_prod_list.
+ * @ctrl_base:		(OUT) Array of mapped IO address.
+ * @count_ctrl_base:	(OUT) Pointer to count of mapped IO address.
+ *
+ * Return Value :
+ *    0   - OK
+ * Others - ERROR
+ *
+ */
+static int mods_tegra_get_prod_info(
+	struct mods_client                     *client,
+	const struct MODS_TEGRA_PROD_SET_TUPLE *tuple,
+	struct tegra_prod                     **tegra_prod_list,
+	void __iomem                          **ctrl_base,
+	__u32                                  *count_ctrl_base)
+{
+	struct device_node *prod_node, *ctrl_node;
+
+	if (!tegra_prod_list || !ctrl_base || !count_ctrl_base) {
+		cl_error("detected NULL pointer for out value.");
+		return -EINVAL;
+	}
+
+	prod_node = (struct device_node *)tuple->prod_dev_handle;
+	if (!prod_node) {
+		cl_error("prod device handle is NULL\n");
+		return -EINVAL;
+	}
+
+	*tegra_prod_list = mods_tegra_get_prod_list(client, prod_node);
+	if (!(*tegra_prod_list)) {
+		cl_error("failed to get prod_list with prod handle 0x%llx\n",
+			 tuple->prod_dev_handle);
+		return -EINVAL;
+	}
+
+	ctrl_node = (struct device_node *)tuple->ctrl_dev_handle;
+	if (!ctrl_node) {
+		cl_error("controller device handle is NULL\n");
+		return -EINVAL;
+	}
+
+	return mods_batch_iomap(client, ctrl_node, ctrl_base, count_ctrl_base);
+}
+
+/**
+ * esc_mods_tegra_prod_is_supported - Test the prod is supported by
+ *					given prod name.
+ * @MODS_TEGRA_PROD_IS_SUPPORTED:  Input information, only the following
+				   members are used.
+ *   +prod_node_handle:     Handle of device node contained prod values.
+ *   +prod_name:            Prod name.
+ *   +is_supported:         Result of supporting given prod configuration.
+ *
+ * Return Value :
+ *    0   - OK
+ * Others - ERROR
+ *
+ */
+int esc_mods_tegra_prod_is_supported(
+	struct mods_client *client,
+	struct MODS_TEGRA_PROD_IS_SUPPORTED *tuple
+)
+{
+	struct device_node *prod_node;
+	struct tegra_prod *tegra_prod;
+	bool is_supported;
+
+	prod_node = (struct device_node *)tuple->prod_dev_handle;
+	if (!prod_node) {
+		cl_error("prod device handle is NULL\n");
+		return -EINVAL;
+	}
+
+	tegra_prod = mods_tegra_get_prod_list(client, prod_node);
+	if (!tegra_prod) {
+		cl_error("failed to get prod_list with prod handle 0x%llx\n",
+			 tuple->prod_dev_handle);
+		return -EINVAL;
+	}
+
+	is_supported = tegra_prod_by_name_supported(tegra_prod,
+						tuple->prod_name);
+	tuple->is_supported = is_supported ? 1 : 0;
+
+	return OK;
+}
+
+/**
+ * esc_mods_tegra_prod_set_prod_all - Read prod values from prod device node,
+ *                                    and set prod values to controller device
+ *				      node.
+ * @MODS_TEGRA_PROD_SET_TUPLE:  Input information, only the following
+ *				members are used.
+ *   +prod_node_handle:	Handle of device node contained prod values.
+ *   +ctrl_node_handle:	Handle of controller device node.
+ *
+ * Return Value :
+ *    0   - OK
+ * Others - ERROR
+ *
+ */
+int esc_mods_tegra_prod_set_prod_all(
+	struct mods_client *client,
+	struct MODS_TEGRA_PROD_SET_TUPLE *tuple
+)
+{
+	int ret;
+	struct tegra_prod *tegra_prod_list;
+	void __iomem *ctrl_base[MAX_IO_MAP_ENTRY];
+	__u32 count_ctrl_base;
+
+	ret = mods_tegra_get_prod_info(client,
+				       tuple,
+				       &tegra_prod_list,
+				       ctrl_base,
+				       &count_ctrl_base);
+	if (ret < 0)
+		return ret;
+
+	ret = tegra_prod_set_list(ctrl_base, tegra_prod_list);
+	if (ret < 0)
+		cl_error("set prod failed\n");
+
+	mods_batch_iounmap(client, ctrl_base, count_ctrl_base);
+
+	return ret;
+}
+
+/**
+ * esc_mods_tegra_prod_set_prod_boot - Read prod values from prod device
+ *				       node, and set prod values to
+ *				       controller device node, which are
+ *				       required for boot initialization.
+ * @MODS_TEGRA_PROD_SET_TUPLE:  Input information, only the following
+ *				members are used.
+ *   +prod_node_handle:	Handle of device node contained prod values.
+ *   +ctrl_node_handle:	Handle of controller device node.
+ *
+ * Return Value :
+ *    0   - OK
+ * Others - ERROR
+ *
+ */
+int esc_mods_tegra_prod_set_prod_boot(
+	struct mods_client *client,
+	struct MODS_TEGRA_PROD_SET_TUPLE *tuple
+)
+{
+	int ret;
+	struct tegra_prod *tegra_prod_list;
+	void __iomem *ctrl_base[MAX_IO_MAP_ENTRY];
+	__u32 count_ctrl_base;
+
+	ret = mods_tegra_get_prod_info(client,
+				       tuple,
+				       &tegra_prod_list,
+				       ctrl_base,
+				       &count_ctrl_base);
+	if (ret < 0)
+		return ret;
+
+	ret = tegra_prod_set_boot_init(ctrl_base, tegra_prod_list);
+	if (ret < 0)
+		cl_error("set boot init prod failed\n");
+
+	mods_batch_iounmap(client, ctrl_base, count_ctrl_base);
+
+	return ret;
+}
+
+/**
+ * esc_mods_tegra_prod_set_prod_by_name - Read prod values from prod
+ *					  device node, and set prod values
+ *					  to controller device node based
+ *					  on prod name.
+ * @MODS_TEGRA_PROD_SET_TUPLE:  Input information, only the following
+ *				members are used.
+ *   +prod_node_handle:	Handle of device node contained prod values.
+ *   +ctrl_node_handle:	Handle of controller device node.
+ *   +prod_name:	Prod name.
+ *
+ * Return Value :
+ *    0   - OK
+ * Others - ERROR
+ *
+ */
+int esc_mods_tegra_prod_set_prod_by_name(
+	struct mods_client *client,
+	struct MODS_TEGRA_PROD_SET_TUPLE *tuple
+)
+{
+	int ret;
+	struct tegra_prod *tegra_prod_list;
+	void __iomem *ctrl_base[MAX_IO_MAP_ENTRY];
+	__u32 count_ctrl_base;
+
+	ret = mods_tegra_get_prod_info(client,
+				       tuple,
+				       &tegra_prod_list,
+				       ctrl_base,
+				       &count_ctrl_base);
+	if (ret < 0)
+		return ret;
+
+	ret = tegra_prod_set_by_name(ctrl_base, tuple->prod_name,
+				tegra_prod_list);
+	if (ret < 0) {
+		cl_error("set prod by name \"%s\" failed\n",
+				tuple->prod_name);
+	}
+
+	mods_batch_iounmap(client, ctrl_base, count_ctrl_base);
+
+	return ret;
+}
+
+/**
+ * esc_mods_tegra_prod_set_prod_exact - Read prod values from prod device
+ *					node, and set prod values to
+ *					controller device node based
+ *					on prod name, index, offset, and mask.
+ * @MODS_TEGRA_PROD_SET_TUPLE:	Input information, only the following
+				members are used.
+ *   +prod_node_handle:	Handle of device node contained prod values.
+ *   +ctrl_node_handle:	Handle of controller device node.
+ *   +prod_name:	Prod name.
+ *   +index:		Index of base address.
+ *   +offset:		Offset of the register.
+ *   +mask:		Mask field on given register.
+ *
+ * Return Value :
+ *    0   - OK
+ * Others - ERROR
+ *
+ */
+int esc_mods_tegra_prod_set_prod_exact(
+	struct mods_client *client,
+	struct MODS_TEGRA_PROD_SET_TUPLE *tuple
+)
+{
+	int ret;
+	struct tegra_prod *tegra_prod_list;
+	void __iomem *ctrl_base[MAX_IO_MAP_ENTRY];
+	__u32 count_ctrl_base;
+
+	ret = mods_tegra_get_prod_info(client,
+				       tuple,
+				       &tegra_prod_list,
+				       ctrl_base,
+				       &count_ctrl_base);
+	if (ret < 0)
+		return ret;
+
+	ret = tegra_prod_set_by_name_partially(ctrl_base, tuple->prod_name,
+				tegra_prod_list, tuple->index, tuple->offset,
+				tuple->mask);
+	if (ret < 0) {
+		cl_error("set prod exact by name \"%s\" failed\n",
+			 tuple->prod_name);
+		cl_error("index [%x]; offset [%x]; mask [%x]\n",
+			 tuple->index, tuple->offset, tuple->mask);
+	}
+
+	mods_batch_iounmap(client, ctrl_base, count_ctrl_base);
+
+	return ret;
+}
+
+
+#ifdef MODS_ENABLE_BPMP_MRQ_API
+static int tegra_pcie_bpmp_set_ctrl_state(struct mods_smmu_dev *pcie_dev,
+					  bool enable)
+{
+	struct mrq_uphy_response resp;
+	struct tegra_bpmp_message msg;
+	struct mrq_uphy_request req;
+
+	memset(&req, 0, sizeof(req));
+	memset(&resp, 0, sizeof(resp));
+
+	req.cmd = CMD_UPHY_PCIE_CONTROLLER_STATE;
+	req.controller_state.pcie_controller = pcie_dev->cid;
+	req.controller_state.enable = enable;
+
+	memset(&msg, 0, sizeof(msg));
+	msg.mrq = MRQ_UPHY;
+	msg.tx.data = &req;
+	msg.tx.size = sizeof(req);
+	msg.rx.data = &resp;
+	msg.rx.size = sizeof(resp);
+
+	return tegra_bpmp_transfer(pcie_dev->bpmp, &msg);
+}
+
+static int uphy_bpmp_pcie_controller_state_set(int controller, int enable)
+{
+	#define MAX_DEV_NAME_LEN 32
+	char dev_name[MAX_DEV_NAME_LEN];
+	struct mods_smmu_dev *smmu_pdev = NULL;
+	int smmudev_idx, n;
+
+	memset(dev_name, 0, MAX_DEV_NAME_LEN);
+	n = snprintf(dev_name, MAX_DEV_NAME_LEN, "mods_pcie%d", controller);
+	if (n < 0 || n >= MAX_DEV_NAME_LEN)
+		return -EINVAL;
+	smmudev_idx = get_mods_smmu_device_index(dev_name);
+	if (smmudev_idx >= 0)
+		smmu_pdev = get_mods_smmu_device(smmudev_idx);
+	if (!smmu_pdev || smmudev_idx < 0) {
+		mods_error_printk("smmu device %s is not found\n", dev_name);
+		return -ENODEV;
+	}
+	smmu_pdev->cid = controller;
+	return tegra_pcie_bpmp_set_ctrl_state(smmu_pdev, enable);
+}
+#else
+
+static int uphy_bpmp_pcie_controller_state_set(int controller, int enable)
+{
+	mods_error_printk("bpmp mrq api is not supported\n");
+	return -ENODEV;
+}
+#endif
+
+int esc_mods_bpmp_set_pcie_state(
+	struct mods_client *client,
+	struct MODS_SET_PCIE_STATE *p
+)
+{
+
+	return uphy_bpmp_pcie_controller_state_set(p->controller, p->enable);
+}
+
+#ifdef MODS_ENABLE_BPMP_MRQ_API
+static int tegra_pcie_bpmp_set_pll_state(struct mods_smmu_dev *pcie_dev,
+					 bool enable)
+{
+	struct mrq_uphy_response resp;
+	struct tegra_bpmp_message msg;
+	struct mrq_uphy_request req;
+
+	memset(&req, 0, sizeof(req));
+	memset(&resp, 0, sizeof(resp));
+
+	if (enable) {
+		req.cmd = CMD_UPHY_PCIE_EP_CONTROLLER_PLL_INIT;
+		req.ep_ctrlr_pll_init.ep_controller = pcie_dev->cid;
+	} else {
+		req.cmd = CMD_UPHY_PCIE_EP_CONTROLLER_PLL_OFF;
+		req.ep_ctrlr_pll_off.ep_controller = pcie_dev->cid;
+	}
+
+	memset(&msg, 0, sizeof(msg));
+	msg.mrq = MRQ_UPHY;
+	msg.tx.data = &req;
+	msg.tx.size = sizeof(req);
+	msg.rx.data = &resp;
+	msg.rx.size = sizeof(resp);
+
+	return tegra_bpmp_transfer(pcie_dev->bpmp, &msg);
+}
+
+static int uphy_bpmp_pcie_set_pll_state(int controller, int enable)
+{
+	#define MAX_DEV_NAME_LEN 32
+	char dev_name[MAX_DEV_NAME_LEN];
+	struct mods_smmu_dev *smmu_pdev = NULL;
+	int smmudev_idx, n;
+
+	memset(dev_name, 0, MAX_DEV_NAME_LEN);
+	n = snprintf(dev_name, MAX_DEV_NAME_LEN, "mods_pcie%d", controller);
+	if (n < 0 || n >= MAX_DEV_NAME_LEN)
+		return -EINVAL;
+	smmudev_idx = get_mods_smmu_device_index(dev_name);
+	if (smmudev_idx >= 0)
+		smmu_pdev = get_mods_smmu_device(smmudev_idx);
+	if (!smmu_pdev || smmudev_idx < 0) {
+		mods_error_printk("smmu device %s is not found\n", dev_name);
+		return -ENODEV;
+	}
+	smmu_pdev->cid = controller;
+	return tegra_pcie_bpmp_set_pll_state(smmu_pdev, enable);
+}
+#else
+
+static int uphy_bpmp_pcie_set_pll_state(int controller, int enable)
+{
+	mods_error_printk("bpmp mrq api is not supported\n");
+	return -ENODEV;
+}
+#endif
+
+int esc_mods_bpmp_init_pcie_ep_pll(
+	struct mods_client *client,
+	struct MODS_INIT_PCIE_EP_PLL *p
+)
+{
+	return uphy_bpmp_pcie_set_pll_state(p->ep_id, 1);
+}
diff --git a/drivers/misc/mods/mods_tz.c b/drivers/misc/mods/mods_tz.c
new file mode 100644
index 00000000..f61a04c3
--- /dev/null
+++ b/drivers/misc/mods/mods_tz.c
@@ -0,0 +1,45 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#include "mods_internal.h"
+
+#include <linux/trusty/trusty_ipc.h>
+
+#define MODS_PORT "com.nvidia.srv.mods"
+
+int esc_mods_send_trustzone_msg(struct mods_client         *client,
+				struct MODS_TZ_PARAMS      *p)
+{
+	int ret;
+	void *chan_ctx = NULL;
+
+	ret = te_open_trusted_session(MODS_PORT, &chan_ctx);
+	if (ret < 0) {
+		cl_error("Couldn't open connection mods service\n");
+		goto error;
+	}
+
+	ret = te_launch_trusted_oper(p->buf, p->buf_size, p->cmd, chan_ctx);
+	if (ret < 0) {
+		cl_error("Trusted operation failed\n");
+		goto error;
+	}
+
+error:
+	p->status = ret;
+	if (chan_ctx)
+		te_close_trusted_session(chan_ctx);
+	return ret;
+}
+
diff --git a/include/uapi/misc/mods.h b/include/uapi/misc/mods.h
new file mode 100644
index 00000000..d15ce23a
--- /dev/null
+++ b/include/uapi/misc/mods.h
@@ -0,0 +1,2163 @@
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
+/*
+ * This file is part of NVIDIA MODS kernel driver.
+ *
+ * Copyright (c) 2008-2023, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * NVIDIA MODS kernel driver is free software: you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * NVIDIA MODS kernel driver is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with NVIDIA MODS kernel driver.
+ * If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _UAPI_MODS_H_
+#define _UAPI_MODS_H_
+
+#include <linux/types.h>
+
+/* Driver version */
+#define MODS_DRIVER_VERSION_MAJOR 4
+#define MODS_DRIVER_VERSION_MINOR 15
+#define MODS_DRIVER_VERSION ((MODS_DRIVER_VERSION_MAJOR << 8) | \
+			     ((MODS_DRIVER_VERSION_MINOR / 10) << 4) | \
+			     (MODS_DRIVER_VERSION_MINOR % 10))
+
+/* Structures defined below are used with ioctls.  We force 1-byte packing so
+ * that the structures are compatible between architectures (e.g. 32-bit and
+ * 64-bit userland) and compilers.
+ *
+ * The structure members are manually ordered in such a way that all members are
+ * naturally aligned, e.g. 32-bit data members are aligned on 4 bytes and
+ * so on.
+ */
+#pragma pack(push, 1)
+
+/* Used by ioctls:
+ * - MODS_ESC_ACQUIRE_ACCESS_TOKEN,
+ * - MODS_ESC_RELEASE_ACCESS_TOKEN
+ * - MODS_ESC_VERIFY_ACCESS_TOKEN
+ *
+ * These ioctls are used to establish a session for known clients of the
+ * driver.  This is to prevent another random client to connect to the driver
+ * while it is in use.  This also allows multiple clients to connect to the
+ * driver and use it.  Without this, only one client can use the driver at
+ * any time.  (Unless the driver has been loaded with multi_instance=1
+ * option, which is deprecated.)
+ *
+ * The first client has to make a MODS_ESC_ACQUIRE_ACCESS_TOKEN request.
+ * This returns a new token generated randomly.
+ *
+ * Further clients can open the driver and communicate with it, but the first
+ * request of any such client after opening the device node must be
+ * MODS_ESC_VERIFY_ACCESS_TOKEN and the client needs to provide a token which
+ * has already been established via MODS_ESC_ACQUIRE_ACCESS_TOKEN, otherwise
+ * all requests to the driver will fail.
+ *
+ * After the last client is done, it has to issue a
+ * MODS_ESC_RELEASE_ACCESS_TOKEN ioctl with the abovementioned token value
+ * in order to release the token.  Then other clients will be able to connect
+ * and start again.
+ */
+struct MODS_ACCESS_TOKEN {
+	/* IN/OUT */
+	__u32 token;
+};
+
+/* Special value used to indicate that access token has not been set */
+#define MODS_ACCESS_TOKEN_NONE ~0U
+
+/* PCI device location */
+struct mods_pci_dev_2 {
+	__u16 domain;
+	__u16 bus;
+	__u16 device;
+	__u16 function;
+};
+
+/* PCI device location (legacy) */
+struct mods_pci_dev {
+	__u16 bus;
+	__u8  device;
+	__u8  function;
+};
+
+/* Used by MODS_ESC_ALLOC_PAGES_2 ioctl.
+ *
+ * It is used to allocate system memory as physical CPU pages.
+ *
+ * The memory can be allocated as contiguous or non-contiguous, albeit
+ * the kernel limits how much contiguous memory can be allocated in
+ * one shot (typically max is around 2MB).
+ *
+ * The memory can be allocated from any NUMA node, from the selected
+ * NUMA node or from the NUMA node corresponding to the specified
+ * PCI device.
+ *
+ * If specified using one of the flags defined below, the memory can
+ * be automatically mapped to the PCI device using IOMMU, so that no
+ * additional ioctls are needed.
+ *
+ * The caching flags passed to this ioctl apply only to the CPU pages
+ * and are in effect when the memory is mapped with mmap().
+ *
+ * Here are common operations on memory allocated with this ioctl:
+ * - MODS_ESC_DMA_MAP_MEMORY: maps memory to device via IOMMU.
+ * - MODS_ESC_GET_PHYSICAL_ADDRESS_2: retrieves physical addresses.
+ * - mmap(): maps memory to userspace.
+ * - MODS_ESC_FREE_PAGES: frees the memory.
+ */
+struct MODS_ALLOC_PAGES_2 {
+	/* IN */
+	__u64                 num_bytes;
+	__u32                 flags; /* MODS_ALLOC_* flags */
+	__s32                 numa_node;
+	struct mods_pci_dev_2 pci_device;
+
+	/* OUT */
+	__u64                 memory_handle;
+};
+
+/* Special value for numa_node member above to indicate that memory can be
+ * allocated on any NUMA node.
+ */
+#define MODS_ANY_NUMA_NODE (-1)
+
+/* Bit flags for the flags member above */
+#define MODS_ALLOC_CACHED        0   /* Default WB cache attr */
+#define MODS_ALLOC_UNCACHED      1   /* UC cache attr */
+#define MODS_ALLOC_WRITECOMBINE  2   /* WC cache attr */
+#define MODS_ALLOC_CACHE_MASK    7U  /* The first three bits are cache attr */
+
+#define MODS_ALLOC_DMA32         8   /* Force 32-bit PA, else any PA */
+#define MODS_ALLOC_CONTIGUOUS    16  /* Force contiguous, else non-contig */
+#define MODS_ALLOC_USE_NUMA      32  /* Use numa_node member instead of PCI dev
+				      * for NUMA node hint
+				      */
+#define MODS_ALLOC_FORCE_NUMA    64  /* Force memory to be from a given NUMA
+				      * node (specified by PCI dev or
+				      * numa_node).  Otherwise use PCI dev or
+				      * numa_node as a hint only.
+				      */
+#define MODS_ALLOC_MAP_DEV       128 /* DMA map to PCI device */
+
+/* Used by MODS_ESC_ALLOC_PAGES ioctl */
+struct MODS_ALLOC_PAGES {
+	/* IN */
+	__u32 num_bytes;
+	__u32 contiguous;
+	__u32 address_bits;
+	__u32 attrib;
+
+	/* OUT */
+	__u64 memory_handle;
+};
+
+/* Used by legacy MODS_ESC_DEVICE_ALLOC_PAGES_2 ioctl */
+struct MODS_DEVICE_ALLOC_PAGES_2 {
+	/* IN */
+	__u32                 num_bytes;
+	__u32                 contiguous;
+	__u32                 address_bits;
+	__u32                 attrib;
+	struct mods_pci_dev_2 pci_device;
+
+	/* OUT */
+	__u64                 memory_handle;
+};
+
+/* Used by legacy MODS_ESC_DEVICE_ALLOC_PAGES ioctl */
+struct MODS_DEVICE_ALLOC_PAGES {
+	/* IN */
+	__u32		    num_bytes;
+	__u32		    contiguous;
+	__u32		    address_bits;
+	__u32		    attrib;
+	struct mods_pci_dev pci_device;
+
+	/* OUT */
+	__u64		    memory_handle;
+};
+
+/* Used by MODS_ESC_FREE_PAGES ioctl.
+ *
+ * Frees pages allocated using any of the calls described above.
+ *
+ * Note that if the memory allocated earlier has been mmapped() to
+ * userland process, it will still be usable, even if it is freed
+ * with this ioctl or even if the driver is closed.
+ *
+ * Otherwise all allocated memory is automatically freed when the
+ * driver is closed for whatever reason (including a crash).
+ */
+struct MODS_FREE_PAGES {
+	/* IN */
+	__u64	memory_handle;
+};
+
+#define MODS_MAX_MERGE_HANDLES 64
+
+/* Used by MODS_ESC_MERGE_PAGES ioctl.
+ *
+ * This can be used to merge multiple, adjacent allocations into one.
+ *
+ * The memory regions described by the input handles must be physically
+ * adjacent (i.e. adjacent in physical address space).
+ */
+struct MODS_MERGE_PAGES {
+	/* IN */
+	__u64 in_memory_handles[MODS_MAX_MERGE_HANDLES];
+	__u32 num_in_handles;
+	__u32 dummy_align;
+
+	/* OUT */
+	__u64 memory_handle;
+};
+
+/* Used by legacy ioctls:
+ * - MODS_ESC_GET_PHYSICAL_ADDRESS
+ * - MODS_ESC_GET_MAPPED_PHYSICAL_ADDRESS
+ */
+struct MODS_GET_PHYSICAL_ADDRESS {
+	/* IN */
+	__u64	memory_handle;
+	__u32	offset;
+
+	/* OUT */
+	__u64	physical_address;
+};
+
+/* Used by legacy MODS_ESC_GET_MAPPED_PHYSICAL_ADDRESS_2 ioctl */
+struct MODS_GET_PHYSICAL_ADDRESS_2 {
+	/* IN */
+	__u64                 memory_handle;
+	__u32	              offset;
+	struct mods_pci_dev_2 pci_device;
+
+	/* OUT */
+	__u64                 physical_address;
+};
+
+/* Used by ioctls:
+ * - MODS_ESC_GET_PHYSICAL_ADDRESS_2
+ * - MODS_ESC_GET_MAPPED_PHYSICAL_ADDRESS_3
+ *
+ * MODS_ESC_GET_PHYSICAL_ADDRESS_2 retrieves physical address for a given
+ * memory allocation.
+ *
+ * MODS_ESC_GET_MAPPED_PHYSICAL_ADDRESS_3 retrieves IO virtual address
+ * (DMA address) usable with the specified PCI device for a given memory
+ * allocation.
+ */
+struct MODS_GET_PHYSICAL_ADDRESS_3 {
+	/* IN */
+	__u64                 memory_handle;
+		/* Page offset from the beginning of the allocation */
+	__u64                 offset;
+		/* PCI device only for MODS_ESC_GET_MAPPED_PHYSICAL_ADDRESS_3 */
+	struct mods_pci_dev_2 pci_device;
+
+	/* OUT */
+	__u64                 physical_address;
+};
+
+/* Used by MODS_ESC_DMABUF_GET_PHYSICAL_ADDRESS ioctl.
+ *
+ * Retrieves physical address for a given dmabuf surface at a given offset.
+ *
+ * Available only on Tegra.
+ */
+struct MODS_DMABUF_GET_PHYSICAL_ADDRESS {
+	/* IN */
+	__s32 buf_fd;
+	__u32 padding;
+	__u64 offset;
+
+	/* OUT */
+	__u64 physical_address;
+	__u64 segment_size;
+};
+
+/* Used by MODS_ESC_VIRTUAL_TO_PHYSICAL ioctl.
+ *
+ * Converts a given userspace virtual address obtained with mmap() on /dev/mods
+ * to a system physical address.
+ *
+ * This call is discouraged, because it is expensive and when IOMMU is
+ * enabled, it is not very useful.  Use MODS_ESC_GET_MAPPED_PHYSICAL_ADDRESS_3
+ * instead.
+ */
+struct MODS_VIRTUAL_TO_PHYSICAL {
+	/* IN */
+	__u64	virtual_address;
+
+	/* OUT */
+	__u64	physical_address;
+};
+
+/* Used by MODS_ESC_PHYSICAL_TO_VIRTUAL ioctl.
+ *
+ * Converts a given system physical address to a userspace virtual address
+ * obtained with mmap() on /dev/mods.
+ *
+ * This call is discouraged, because it is expensive and when IOMMU is
+ * enabled, it is not very useful.
+ */
+struct MODS_PHYSICAL_TO_VIRTUAL {
+	/* IN */
+	__u64	physical_address;
+
+	/* OUT */
+	__u64	virtual_address;
+};
+
+/* Used by MODS_ESC_FLUSH_CPU_CACHE_RANGE ioctl.
+ *
+ * This call is available only on ARM architecture and can be used to flush
+ * or invalidate cache for memory ranges allocated with WB cache attr.
+ *
+ * This call is useful only for devices which do not support I/O coherency.
+ */
+struct MODS_FLUSH_CPU_CACHE_RANGE {
+	/* IN */
+	__u64 virt_addr_start;
+	__u64 virt_addr_end;
+	__u32 flags;
+};
+
+/* Flags for the flags member above */
+#define MODS_FLUSH_CPU_CACHE	  1
+#define MODS_INVALIDATE_CPU_CACHE 2
+
+/* Used by ioctls:
+ * - MODS_ESC_DMA_MAP_MEMORY
+ * - MODS_ESC_DMA_UNMAP_MEMORY
+ *
+ * MODS_ESC_DMA_MAP_MEMORY maps the specified memory to IOMMU for the specified
+ * PCI device.
+ *
+ * - This is a no-op if the memory has already been mapped.
+ *
+ * MODS_ESC_DMA_UNMAP_MEMORY unmaps memory from IOMMU for the specified PCI
+ * device if it has already been mapped.
+ *
+ * - This applies only to non-default mappings done with
+ *   MODS_ESC_DMA_MAP_MEMORY.
+ *
+ * - If the memory was mapped to a PCI device during allocation with
+ *   MODS_ALLOC_MAP_DEV flag, then this call will not unmap it.
+ *
+ * - If multiple calls to MODS_ESC_DMA_MAP_MEMORY have been made to map the
+ *   same memory region to the same PCI device, the first call to
+ *   MODS_ESC_DMA_UNMAP_MEMORY will unmap it, there is no reference counting
+ *   or any other protection in place.
+ */
+struct MODS_DMA_MAP_MEMORY {
+	/* IN */
+	__u64                 memory_handle;
+	struct mods_pci_dev_2 pci_device;
+};
+
+/* Used by MODS_ESC_PCI_SET_DMA_MASK ioctl.
+ *
+ * Tells the kernel that the PCI device supports a DMA address of the given
+ * bit width.  By default the kernel assumes that every device supports
+ * 32-bit addresses.  If the device is intended to be used with physical
+ * addresses or IOMMU virtual addresses exceeding 32 bits, this call must
+ * be made to set that up.
+ */
+struct MODS_PCI_DMA_MASK {
+	/* IN */
+	struct mods_pci_dev_2 pci_device;
+	__u32                 num_bits;
+};
+
+/* Used by ioctls:
+ * - MODS_ESC_GET_IOMMU_STATE_2
+ * - MODS_ESC_GET_IOMMU_STATE (deprecated)
+ *
+ * MODS_ESC_GET_IOMMU_STATE_2 attempts to determine whether IOMMU
+ * has been enabled for the given PCI device.
+ *
+ * This call returns useful information only on certain kernel versions and
+ * certain architectures.
+ */
+struct MODS_GET_IOMMU_STATE {
+	/* IN */
+	struct mods_pci_dev_2 pci_device;
+	/* OUT */
+	__u32                 state;
+};
+
+/* Flags for the state member defined above */
+#define MODS_SWIOTLB_DISABLED      0 /* Hardware IOMMU may be enabled */
+#define MODS_SWIOTLB_ACTIVE        1 /* Hardware IOMMU is disabled */
+#define MODS_SWIOTLB_INDETERMINATE 2 /* Unable to tell the state of SWIOTLB */
+
+/* Used by MODS_ESC_FIND_PCI_DEVICE_2 ioctl.
+ *
+ * Searches for PCI devices with the specified vendor id and device id.
+ *
+ * The first call to this ioctl is performed with index=0 and then for
+ * every successful call index is incremented until this call fails.
+ */
+struct MODS_FIND_PCI_DEVICE_2 {
+	/* IN */
+	__u32 device_id;
+	__u32 vendor_id;
+	__u32 index;
+
+	/* OUT */
+	struct mods_pci_dev_2 pci_device;
+};
+
+/* Used by MODS_ESC_BPMP_SET_PCIE_STATE ioctl.
+ *
+ * Set PCIE state through BPMP Uphy driver
+ */
+struct MODS_SET_PCIE_STATE {
+	/* IN */
+	__u32 controller;
+	__u32 enable;
+};
+
+/* Used by MODS_ESC_BPMP_INIT_PCIE_EP_PLL ioctl.
+ *
+ * Initialize PCIE EP PLL through BPMP Uphy driver
+ */
+struct MODS_INIT_PCIE_EP_PLL {
+	/* IN */
+	__u32 ep_id;
+};
+
+/* Used by legacy MODS_ESC_FIND_PCI_DEVICE ioctl */
+struct MODS_FIND_PCI_DEVICE {
+	/* IN */
+	__u32 device_id;
+	__u32 vendor_id;
+	__u32 index;
+
+	/* OUT */
+	__u32 bus_number;
+	__u32 device_number;
+	__u32 function_number;
+};
+
+/* Used by MODS_ESC_FIND_PCI_CLASS_CODE_2 ioctl.
+ *
+ * Searches for PCI devices with the specified PCI class code.
+ *
+ * The first call to this ioctl is performed with index=0 and then for
+ * every successful call index is incremented until this call fails.
+ */
+struct MODS_FIND_PCI_CLASS_CODE_2 {
+	/* IN */
+	__u32 class_code;
+	__u32 index;
+
+	/* OUT */
+	struct mods_pci_dev_2 pci_device;
+};
+
+/* Used by legacy MODS_ESC_FIND_PCI_CLASS_CODE ioctl */
+struct MODS_FIND_PCI_CLASS_CODE {
+	/* IN */
+	__u32 class_code;
+	__u32 index;
+
+	/* OUT */
+	__u32 bus_number;
+	__u32 device_number;
+	__u32 function_number;
+};
+
+/* Used by MODS_ESC_PCI_GET_BAR_INFO_2 ioctl.
+ *
+ * Returns physical address and size of the queried PCI BAR.
+ */
+struct MODS_PCI_GET_BAR_INFO_2 {
+	/* IN */
+	struct mods_pci_dev_2 pci_device;
+	__u32                 bar_index;
+
+	/* OUT */
+	__u64 base_address;
+	__u64 bar_size;
+};
+
+/* Used by legacy MODS_ESC_PCI_GET_BAR_INFO ioctl */
+struct MODS_PCI_GET_BAR_INFO {
+	/* IN */
+	struct mods_pci_dev pci_device;
+	__u32               bar_index;
+
+	/* OUT */
+	__u64 base_address;
+	__u64 bar_size;
+};
+
+/* Used by MODS_ESC_PCI_GET_IRQ_2 ioctl.
+ *
+ * Returns IRQ number assigned to the PCI device.  This is typically an APIC
+ * IRQ number.  The value returned may or may not be relevant.  This value
+ * matters only for legacy interrupts and is irrelevant for MSI.  This value
+ * is for information purposes only.
+ */
+struct MODS_PCI_GET_IRQ_2 {
+	/* IN */
+	struct mods_pci_dev_2 pci_device;
+
+	/* OUT */
+	__u32 irq;
+};
+
+/* Used by legacy MODS_ESC_PCI_GET_IRQ ioctl */
+struct MODS_PCI_GET_IRQ {
+	/* IN */
+	struct mods_pci_dev pci_device;
+
+	/* OUT */
+	__u32 irq;
+};
+
+/* Used by MODS_ESC_PCI_READ_2 ioctl.
+ *
+ * Reads data from PCI config space of a PCI device.
+ * data_size specifies how many bytes to read at the specified address.
+ * It can be 1, 2 or 4.
+ */
+struct MODS_PCI_READ_2 {
+	/* IN */
+	struct mods_pci_dev_2 pci_device;
+	__u32                 address;
+	__u32                 data_size;
+
+	/* OUT */
+	__u32                 data;
+};
+
+/* Used by legacy MODS_ESC_PCI_READ ioctl */
+struct MODS_PCI_READ {
+	/* IN */
+	__u32 bus_number;
+	__u32 device_number;
+	__u32 function_number;
+	__u32 address;
+	__u32 data_size;
+
+	/* OUT */
+	__u32 data;
+};
+
+/* Used by MODS_ESC_PCI_WRITE_2 ioctl.
+ *
+ * Writes data into PCI config space of a PCI device.
+ * data_size specifies how many bytes to write at the specified address.
+ * It can be 1, 2 or 4.
+ */
+struct MODS_PCI_WRITE_2 {
+	/* IN */
+	struct mods_pci_dev_2 pci_device;
+	__u32                 address;
+	__u32                 data;
+	__u32                 data_size;
+};
+
+/* Used by legacy MODS_ESC_PCI_WRITE ioctl */
+struct MODS_PCI_WRITE {
+	/* IN */
+	__u32 bus_number;
+	__u32 device_number;
+	__u32 function_number;
+	__u32 address;
+	__u32 data;
+	__u32 data_size;
+};
+
+/* Used by MODS_ESC_PCI_HOT_RESET ioctl.
+ *
+ * This call is implemented only for ppc64le architecture and it is currently
+ * unused.
+ */
+struct MODS_PCI_HOT_RESET {
+	/* IN */
+	struct mods_pci_dev_2 pci_device;
+};
+
+/* Used by MODS_ESC_PCI_BUS_REMOVE_DEV ioctl.
+ *
+ * Stops and removes a PCI device from the bus.
+ */
+struct MODS_PCI_BUS_REMOVE_DEV {
+	/* IN */
+	struct mods_pci_dev_2 pci_device;
+};
+
+/* Used by MODS_ESC_PCI_BUS_RESCAN ioctl.
+ *
+ * Initiates a PCI bus scan for the specified PCI bus.
+ * This is necessary e.g. to discover any new PCI devices which may have been
+ * hot-plugged.
+ */
+struct MODS_PCI_BUS_RESCAN {
+	/* IN */
+	__u16 domain;
+	__u16 bus;
+};
+
+/* Used by legacy MODS_ESC_PCI_BUS_ADD_DEVICES ioctl */
+struct MODS_PCI_BUS_ADD_DEVICES {
+	/* IN */
+	__u32 bus;
+};
+
+/* Used by ioctls:
+ * - MODS_ESC_SET_NUM_VF
+ * - MODS_ESC_SET_TOTAL_VF
+ *
+ * MODS_ESC_SET_NUM_VF is used to enable or disable SR-IOV on the specified
+ * PCI device.
+ *
+ * To disable SR-IOV, MODS_ESC_SET_NUM_VF is called with numvfs set to 0.
+ *
+ * MODS_ESC_SET_TOTAL_VF is used to set maximum number of virtual
+ * functions.
+ */
+struct MODS_SET_NUM_VF {
+	/* IN */
+	struct mods_pci_dev_2 dev;
+	__u32                 numvfs;  /* Number of virtual functions */
+};
+
+/* Used by legacy MODS_ESC_PCI_MAP_RESOURCE ioctl */
+struct MODS_PCI_MAP_RESOURCE {
+	/* IN */
+	struct mods_pci_dev_2 local_pci_device;
+	struct mods_pci_dev_2 remote_pci_device;
+	__u32                 resource_index;
+	__u64                 page_count;
+
+	/* IN/OUT */
+	__u64                 va;
+};
+
+/* Used by legacy MODS_ESC_PCI_UNMAP_RESOURCE ioctl */
+struct MODS_PCI_UNMAP_RESOURCE {
+	/* IN */
+	struct mods_pci_dev_2 pci_device;
+	__u64                 va;
+};
+
+/* Used by MODS_ESC_PIO_READ ioctl.
+ *
+ * Reads data from legacy x86 I/O port.  This ioctl is x86-specific.
+ */
+struct MODS_PIO_READ {
+	/* IN */
+	__u16 port;
+	__u32 data_size;
+
+	/* OUT */
+	__u32 data;
+};
+
+/* Used by MODS_ESC_PIO_WRITE ioctl.
+ *
+ * Writes data to legacy x86 I/O port.  This ioctl is x86-specific.
+ */
+struct MODS_PIO_WRITE {
+	/* IN */
+	__u16 port;
+	__u32 data;
+	__u32 data_size;
+};
+
+/* Used by ioctls:
+ * - MODS_ESC_READ_MSR
+ * - MODS_ESC_WRITE_MSR
+ *
+ * These ioctls are used to read and write model specific registers (MSR)
+ * and are available only on x86 architecture.
+ */
+struct MODS_MSR {
+	/* IN */
+	__u32 reg;
+	__u32 cpu_num;
+
+	/* IN/OUT */
+	__u32 low;
+	__u32 high;
+};
+
+/* Deprecated */
+struct mods_irq_data {
+	__u32 irq;
+	__u32 delay;
+};
+
+#define INQ_CNT 8
+
+/* Deprecated */
+struct mods_irq_status {
+	struct mods_irq_data data[INQ_CNT];
+	__u32 irqbits:INQ_CNT;
+	__u32 otherirq:1;
+};
+
+/* Used by several deprecated ioctls */
+struct MODS_IRQ {
+	/* IN */
+	__u32                  cmd;
+	__u32                  size;
+	__u32                  irq;
+
+	/* IN OUT */
+	__u32                  channel;
+
+	/* OUT */
+	struct mods_irq_status stat;
+	__u64                  phys;
+};
+
+/* Mask used to disable incoming legacy interrupts on x86.
+ *
+ * The driver propagates interrupts to userspace where they are serviced.
+ * For legacy interrupts, the driver uses these masks to disable interrupts
+ * in order to prevent interrupt storm, because as soon as the interrupt
+ * service routine exits, the interrupt is still enabled and will keep
+ * popping up until userspace has a chance to service it and clear it.
+ *
+ * When a legacy interrupt is triggered, the driver maps a region of physical
+ * address space described by aperture_addr and aperture_size, then reads data
+ * (32-bit or 64-bit, depending on mask_type) at offset specified by the
+ * irq_pending_offset field to determine whether the interrupt is pending.
+ * This is because legacy interrupts are shared and multiple devices may
+ * be triggering them (e.g. an interrupt line can be shared between a network
+ * adapter and a disk drive).  The driver also reads data (of the same bit
+ * width) from offset specified by the irq_enabled_offset field to determine
+ * whether the interrupt is actually enabled on the device.  If both fields
+ * read a non-zero value, the interrupt is considered as pending (i.e. if the
+ * interrupt is both pending and enabled).
+ *
+ * Once the driver determines that the interrupt is pending using the
+ * irq_pending_offset and irq_enabled_offset fields, the driver applies the
+ * mask at offset specified with the irq_disable_offset field.  This is either
+ * a 32-bit or 64-bit value, depending on mask_type.  The value read from that
+ * field is AND-ed with and_mask and then OR-ed with or-mask, then written
+ * back at irq_disable_offset.
+ *
+ * All offsets are relative to aperture_addr.
+ */
+struct mods_mask_info2 {
+	__u8  mask_type;   /* Mask size: 32-bit or 64-bit (see below) */
+	__u8  reserved[7]; /* Force 64-bit member alignment */
+	__u32 irq_pending_offset; /* Register to read IRQ pending status */
+	__u32 irq_enabled_offset; /* Register to read IRQ enabled status */
+	__u32 irq_enable_offset;  /* Unused */
+	__u32 irq_disable_offset; /* Register to write to disable IRQs */
+	__u64 and_mask;   /* AND mask for clearing bits in this register */
+	__u64 or_mask;    /* OR mask for setting bit in this register */
+};
+
+/* Values for the mask_type field above */
+#define MODS_MASK_TYPE_IRQ_DISABLE   0 /* 32-bit mask */
+#define MODS_MASK_TYPE_IRQ_DISABLE64 1 /* 64-bit mask */
+
+#define MODS_IRQ_MAX_MASKS 16
+
+/* Used by MODS_ESC_REGISTER_IRQ_4 ioctl.
+ *
+ * Hooks an interrupt in the driver for the specified device.
+ *
+ * The userspace "driver" will subsequently invoke poll() on the open device
+ * file descriptor to wait for incoming interrupts.  When poll() is released,
+ * userspace calls MODS_ESC_QUERY_IRQ_3 to retrieve pending interrupt
+ * vectors.  Userspace is responsible for properly servicing and rearming or
+ * re-enabling the interrupts.
+ *
+ * The MODS_ESC_UNREGISTER_IRQ_2 ioctl is used to unhook interrupts.
+ *
+ * The interrupts will also be unhooked automatically when the driver is
+ * closed, e.g. in the event of a crash in userspace, however this does not
+ * de-configure the interrupts on the device.  In case of legacy interrupts,
+ * the driver will mask the interrupts using the masks provided to prevent
+ * an interrupt storm, but the interrupts may remain pending on the device.
+ */
+struct MODS_REGISTER_IRQ_4 {
+	/* IN */
+
+	/* PCI device for which interrupt is being hooked, if irq_flags
+	 * has MODS_IRQ_TYPE_INT, _MSI or _MSIX flag.
+	 *
+	 * For CPU interrupts, the bus member holds the index of the
+	 * interrupt to hook.
+	 */
+	struct mods_pci_dev_2 dev;
+
+	/* Information used to mask incoming interrupts.  This is used
+	 * primarily for legacy interrupts (MODS_IRQ_TYPE_INT), although
+	 * it can be used for CPU interrupts as well.
+	 */
+
+	/* Physical address of aperture to map */
+	__u64 aperture_addr;
+	/* Size of the mapped region */
+	__u32 aperture_size;
+	/* Number of entries in mask_info[] */
+	__u32 mask_info_cnt;
+	/* Interrupt masks */
+	struct mods_mask_info2 mask_info[MODS_IRQ_MAX_MASKS];
+
+	/* Number of interrupt vectors to allocate.  This can be greater than
+	 * 1 for MODS_IRQ_TYPE_MSIX and must be 1 for all other types.
+	 */
+	__u32 irq_count;
+
+	/* Interrupt type and flags */
+	__u32 irq_flags;
+};
+
+/* Flags for the irq_flags member above */
+#define MODS_IRQ_TYPE_INT  0    /* Legacy interrupt, only x86 */
+#define MODS_IRQ_TYPE_MSI  1    /* MSI */
+#define MODS_IRQ_TYPE_CPU  2    /* CPU interrupt vector, only Tegra */
+#define MODS_IRQ_TYPE_MSIX 3    /* MSI-X */
+#define MODS_IRQ_TYPE_MASK 0xff /* Mask for the type field in irq_flags */
+
+#define MODS_IRQ_TYPE_FROM_FLAGS(flags) ((flags) & MODS_IRQ_TYPE_MASK)
+#define MODS_IRQ_FLAG_FROM_FLAGS(flags) ((flags) >> 8)
+
+/* Used by legacy MODS_ESC_REGISTER_IRQ_3 ioctl */
+struct MODS_REGISTER_IRQ_3 {
+	/* IN */
+	struct mods_pci_dev_2  dev;
+	__u64                  aperture_addr;
+	__u32                  aperture_size;
+	__u32                  mask_info_cnt;
+	struct mods_mask_info2 mask_info[MODS_IRQ_MAX_MASKS];
+	__u8                   irq_type;
+	__u8                   reserved[7];
+};
+
+/* Used by ioctls:
+ * - MODS_ESC_UNREGISTER_IRQ_2
+ * - MODS_ESC_REGISTER_IRQ_2 (deprectated)
+ * - MODS_ESC_IRQ_HANDLED_2 (only Tegra)
+ *
+ * MODS_ESC_UNREGISTER_IRQ_2 is used to unhook interrupt for the given device.
+ */
+struct MODS_REGISTER_IRQ_2 {
+	/* IN */
+	struct mods_pci_dev_2 dev;
+	__u8                  type; /* MODS_IRQ_TYPE_* */
+};
+
+/* Used by legacy ioctls:
+ * - MODS_ESC_REGISTER_IRQ
+ * - MODS_ESC_UNREGISTER_IRQ
+ * - MODS_ESC_IRQ_HANDLED
+ */
+struct MODS_REGISTER_IRQ {
+	/* IN */
+	struct mods_pci_dev dev;
+	__u8                type;
+};
+
+/* Describes a pending interrupt */
+struct mods_irq_3 {
+	struct mods_pci_dev_2 dev; /* Device which generated the interrupt */
+	__u32  irq_index;          /* Index of MSI-X interrupt, or else 0 */
+	__u32  delay;              /* Nanoseconds since the irq occurred */
+};
+
+/* Deprecated */
+struct mods_irq_2 {
+	__u32                 delay;
+	struct mods_pci_dev_2 dev;
+};
+
+/* Deprecated */
+struct mods_irq {
+	__u32               delay;
+	struct mods_pci_dev dev;
+};
+
+#define MODS_MAX_IRQS 32
+
+/* Used by MODS_ESC_QUERY_IRQ_3 ioctl.
+ *
+ * Userspace typically waits on the open device file descriptor using poll()
+ * and when poll() returns, userspace uses this ioctl to retrieve pending
+ * interrupts.
+ *
+ * For every pending interrupt, one entry in irq_list specifies which
+ * device triggered the interrupt:
+ * - For legacy interrupts and MSI, the dev entry is the PCI device which
+ *   raised the interrupt and irq_index is 0.
+ * - For MSI-X, the dev entry is the PCI device which raised the interrupt
+ *   and irq_index is the interrupt (message) index.
+ * - For CPU interrupts, dev.bus is the index of the interrupt which was
+ *   triggered and irq_index is 0.
+ *
+ * Valid fields have dev.bus different than 0xFFFF.  Entries in the structure
+ * which haven't been filled by the driver have dev.bus set to 0xFFFF.
+ *
+ * If there are more interrupts pending than the structure can handle, the
+ * more field is set to a non-zero value, in which case the userspace needs
+ * to invoke this ioctl again.  If more field is 0, then userspace can
+ * continue polling to wait for more interrupts to occur.
+ */
+struct MODS_QUERY_IRQ_3 {
+	/* OUT */
+	struct mods_irq_3 irq_list[MODS_MAX_IRQS];
+	__u8              more;
+};
+
+/* Used by legacy MODS_ESC_QUERY_IRQ_2 ioctl */
+struct MODS_QUERY_IRQ_2 {
+	/* OUT */
+	struct mods_irq_2 irq_list[MODS_MAX_IRQS];
+	__u8               more;
+};
+
+/* Used by legacy MODS_ESC_QUERY_IRQ ioctl */
+struct MODS_QUERY_IRQ {
+	/* OUT */
+	struct mods_irq irq_list[MODS_MAX_IRQS];
+	__u8            more;
+};
+
+/* Deprecated */
+struct mods_mask_info {
+	__u8  mask_type;  /*mask type 32/64 bit access*/
+	__u8  reserved[3];
+	__u32 reg_offset; /* offset of register within the bar aperture*/
+	__u64 and_mask;   /*and mask for clearing bits in this register */
+	__u64 or_mask;    /*or value for setting bit in this register */
+};
+
+/* Used by legacy MODS_ESC_SET_IRQ_MULTIMASK ioctl */
+struct MODS_SET_IRQ_MULTIMASK {
+	/* IN */
+	__u64                 aperture_addr;
+	__u32                 aperture_size;
+	struct mods_pci_dev_2 dev;
+	__u32                 mask_info_cnt;
+	struct mods_mask_info mask_info[MODS_IRQ_MAX_MASKS];
+	__u8                  irq_type;
+};
+
+/* Used by legacy MODS_ESC_SET_IRQ_MASK_2 ioctl */
+struct MODS_SET_IRQ_MASK_2 {
+	/* IN */
+	__u64                 aperture_addr;
+	__u32                 aperture_size;
+	__u32                 reg_offset;
+	__u64                 and_mask;
+	__u64                 or_mask;
+	struct mods_pci_dev_2 dev;
+	__u8                  irq_type;
+	__u8                  mask_type;
+};
+
+/* Used by legacy MODS_ESC_SET_IRQ_MASK ioctl */
+struct MODS_SET_IRQ_MASK {
+	/* IN */
+	__u64               aperture_addr;
+	__u32               aperture_size;
+	__u32               reg_offset;
+	__u32               and_mask;
+	__u32               or_mask;
+	struct mods_pci_dev dev;
+	__u8                irq_type;
+	__u8                mask_type;
+};
+
+#define MAX_DT_SIZE 64
+#define MAX_FULL_SIZE 128
+
+/* Used by MODS_ESC_MAP_INTERRUPT ioctl.
+ *
+ * Finds GIC interrupt number for device by looking it up in device tree.
+ *
+ * Available only on Tegra.
+ */
+struct MODS_DT_INFO {
+	/* OUT */
+
+	/* GIC irq number*/
+	__u32 irq;
+
+	/* IN */
+
+	/* Name of device tree node, for lookup */
+	char  dt_name[MAX_DT_SIZE];
+	/* Full name of node as in device tree */
+	char  full_name[MAX_FULL_SIZE];
+	/* Interrupt index corresponding to physical irq */
+	__u32 index;
+};
+
+#define MAX_GPIO_NAME_SIZE 256
+
+/* Used by MODS_ESC_MAP_GPIO ioctl.
+ *
+ * Finds GIC interrupt number for a GPIO owned by specific device.
+ *
+ * Available only on Tegra.
+ */
+struct MODS_GPIO_INFO {
+	/* OUTPUT */
+
+	/* GIC irq number*/
+	__u32 irq;
+
+	/* IN */
+
+	/* Name of GPIO */
+	char name[MAX_GPIO_NAME_SIZE];
+	/* Name of device tree node owning the GPIO, for lookup */
+	char dt_name[MAX_DT_SIZE];
+	/* Full name of node owning the GPIO, as in device tree */
+	char full_name[MAX_DT_SIZE];
+};
+
+/* Describes parameter to an ACPI function */
+union ACPI_ARGUMENT {
+	__u32 type;
+
+	struct {
+		__u32 type;
+		__u32 value;
+	} integer;
+
+	struct {
+		__u32 type;
+		__u32 length; /* Number of bytes */
+		__u32 offset; /* Offset in in_buffer or out_buffer */
+	} buffer;
+
+	struct {
+		__u32 type;
+		__u64 handle;
+	} method;
+};
+
+/* Argument type (for the type field above) */
+#define ACPI_MODS_TYPE_INTEGER   1
+#define ACPI_MODS_TYPE_BUFFER    2
+#define ACPI_MODS_TYPE_METHOD    3
+
+#define ACPI_MAX_BUFFER_LENGTH   4096
+#define ACPI_MAX_DEV_CHILDREN    16
+#define ACPI_MAX_METHOD_LENGTH   12
+#define ACPI_MAX_ARGUMENT_NUMBER 12
+
+/* Used by MODS_ESC_EVAL_ACPI_METHOD ioctl.
+ *
+ * Executes the specified ACPI method.  The kernel looks up the specified
+ * method in the ACPI tables and then executes it.
+ *
+ * This variant is for ACPI methods not related to any PCI devices.
+ */
+struct MODS_EVAL_ACPI_METHOD {
+	/* IN */
+	char                method_name[ACPI_MAX_METHOD_LENGTH];
+	__u32               argument_count;
+	union ACPI_ARGUMENT argument[ACPI_MAX_ARGUMENT_NUMBER];
+	__u8                in_buffer[ACPI_MAX_BUFFER_LENGTH];
+
+	/* IN OUT */
+	__u32               out_data_size;
+
+	/* OUT */
+	__u8                out_buffer[ACPI_MAX_BUFFER_LENGTH];
+	__u32               out_status;
+};
+
+/* Used by MODS_ESC_EVAL_DEV_ACPI_METHOD_3 ioctl.
+ *
+ * Executes the specified ACPI method.  The kernel looks up the specified
+ * method in the ACPI tables and then executes it.
+ *
+ * This variant is for ACPI methods related to a specific PCI device.
+ */
+struct MODS_EVAL_DEV_ACPI_METHOD_3 {
+	/* IN OUT */
+	struct MODS_EVAL_ACPI_METHOD method;
+
+	/* IN */
+	struct mods_pci_dev_2        device;
+	__u32                        acpi_id;
+};
+
+/* Used in the acpi_id member above to indicate that there is no ACPI id
+ * associated with this method.
+ */
+#define ACPI_MODS_IGNORE_ACPI_ID 0xffffffff
+
+/* Used by MODS_ESC_EVAL_DEV_ACPI_METHOD_2 ioctl.
+ *
+ * Executes the specified ACPI method.  The kernel looks up the specified
+ * method in the ACPI tables and then executes it.
+ *
+ * This variant is for ACPI methods related to a specific PCI device.
+ */
+struct MODS_EVAL_DEV_ACPI_METHOD_2 {
+	/* IN OUT */
+	struct MODS_EVAL_ACPI_METHOD method;
+
+	/* IN */
+	struct mods_pci_dev_2 device;
+};
+
+/* Used by legacy MODS_ESC_EVAL_DEV_ACPI_METHOD ioctl */
+struct MODS_EVAL_DEV_ACPI_METHOD {
+	/* IN OUT */
+	struct MODS_EVAL_ACPI_METHOD method;
+
+	/* IN */
+	struct mods_pci_dev device;
+};
+
+/* Used by MODS_ESC_ACPI_GET_DDC_2 ioctl.
+ *
+ * Finds an LCD device in the ACPI tables using _ADR ACPI method for the
+ * specified PCI device, then uses the _DDC ACPI method to retrieve
+ * EDID for that LCD device.
+ */
+struct MODS_ACPI_GET_DDC_2 {
+	/* OUT */
+	__u32                 out_data_size;
+	__u8                  out_buffer[ACPI_MAX_BUFFER_LENGTH];
+
+	/* IN */
+	struct mods_pci_dev_2 device;
+};
+
+/* Used by legacy MODS_ESC_ACPI_GET_DDC ioctl */
+struct MODS_ACPI_GET_DDC {
+	/* OUT */
+	__u32               out_data_size;
+	__u8                out_buffer[ACPI_MAX_BUFFER_LENGTH];
+
+	/* IN */
+	struct mods_pci_dev device;
+};
+
+/* Used by MODS_ESC_GET_ACPI_DEV_CHILDREN ioctl */
+struct MODS_GET_ACPI_DEV_CHILDREN {
+	/* OUT */
+	__u32                 num_children;
+	__u32                 children[ACPI_MAX_DEV_CHILDREN];
+
+	/* IN */
+	struct mods_pci_dev_2 device;
+};
+
+/* Used by ioctls:
+ * - MODS_ESC_GET_API_VERSION
+ * - MODS_ESC_GET_KERNEL_VERSION (deprecated)
+ *
+ * MODS_ESC_GET_API_VERSION returns driver version.
+ */
+struct MODS_GET_VERSION {
+	/* OUT */
+	__u64 version;
+};
+
+/* Used by legacy MODS_ESC_SET_DRIVER_PARA ioctl */
+struct MODS_SET_PARA {
+	/* IN */
+	__u64 Highmem4g;
+	__u64 debug;
+};
+
+/* Used by MODS_ESC_SET_MEMORY_TYPE ioctl.
+ *
+ * This ioctl is used with non-memory regions of physical address space,
+ * typically referred to as I/O ranges, to advertise a specific caching
+ * attribute.
+ *
+ * The caching attribute takes effect when mmap() is executed on the
+ * device's file descriptor with the physical address range for mapping
+ * which fits in the range described by this ioctl.
+ */
+struct MODS_MEMORY_TYPE {
+	/* IN */
+	__u64 physical_address; /* First address of the memory range */
+	__u64 size;             /* Size of the memory range */
+	__u32 type;             /* Caching attribute (see below) */
+};
+
+/* Caching attributes for the type member above */
+#define MODS_MEMORY_CACHED       5
+#define MODS_MEMORY_UNCACHED     1
+#define MODS_MEMORY_WRITECOMBINE 2
+
+#define MAX_CPU_MASKS_3 128 /* CPU indices can be at most 4096 apart */
+
+/* Used by MODS_ESC_DEVICE_NUMA_INFO_3 ioctl.
+ *
+ * Retrieves information about a NUMA node to which the specified
+ * PCI device is physically attached.
+ */
+struct MODS_DEVICE_NUMA_INFO_3 {
+	/* IN */
+	struct mods_pci_dev_2 pci_device;
+
+	/* OUT */
+	__s32 node;       /* Index of the NUMA node where the PCI device is.
+			   * -1 if NUMA is disabled.
+			   */
+	__u32 node_count; /* Total number of NUMA nodes */
+	__u32 cpu_count;  /* Number of CPUs in the NUMA node */
+			  /* Relative index for node_cpu_mask. */
+	__u32 first_cpu_mask_offset;
+			  /* Bitmask which specifies which CPUs are on this NUMA
+			   * node.  Bit 0 in the first element of this array
+			   * corresponds to CPU 32*first_cpu_mask_offset.  For
+			   * example, if first_cpu_mask_offset is 2, then
+			   * node_cpu_mask[0] has bit flags for CPUs 64..95.
+			   * This allows for retrieving CPU indices when there
+			   * are lots of CPUs in the system.
+			   */
+	__u32 node_cpu_mask[MAX_CPU_MASKS_3];
+};
+
+#if defined(__powerpc64__)
+#define MAX_CPU_MASKS 64  /* 32 masks of 32bits = 2048 CPUs max */
+#else
+#define MAX_CPU_MASKS 32  /* 32 masks of 32bits = 1024 CPUs max */
+#endif
+
+/* Used by MODS_ESC_DEVICE_NUMA_INFO_2 ioctl.
+ *
+ * Retrieves information about a NUMA node to which the specified
+ * PCI device is physically attached.
+ */
+struct MODS_DEVICE_NUMA_INFO_2 {
+	/* IN */
+	struct mods_pci_dev_2 pci_device;
+
+	/* OUT */
+	__s32 node;       /* Index of the NUMA node where the PCI device is.
+			   * -1 if NUMA is disabled.
+			   */
+	__u32 node_count; /* Total number of NUMA nodes */
+	__u32 node_cpu_mask[MAX_CPU_MASKS];
+			  /* Zero-based bitmask which specifies which CPUs are
+			   * on this NUMA node
+			   */
+	__u32 cpu_count;  /* Number of CPUs in the NUMA node */
+};
+
+/* Used by legacy MODS_ESC_DEVICE_NUMA_INFO ioctl */
+struct MODS_DEVICE_NUMA_INFO {
+	/* IN */
+	struct mods_pci_dev pci_device;
+
+	/* OUT */
+	__s32 node;
+	__u32 node_count;
+	__u32 node_cpu_mask[MAX_CPU_MASKS];
+	__u32 cpu_count;
+};
+
+/* Used by MODS_ESC_GET_RESET_HANDLE ioctl.
+ * Used to get an index in reset array corresponding to a reset name
+ * so that a client can reference it easily in future calls to toggle
+ * reset
+ */
+struct MODS_GET_RESET_HANDLE {
+	/* OUT */
+	__u32 reset_handle;
+
+	/* IN */
+	char  reset_name[MAX_DT_SIZE];
+};
+
+/* Used by MODS_ESC_RESET_ASSERT */
+struct MODS_RESET_HANDLE {
+	/* IN */
+	__u32 handle;
+	__u8  assert;
+};
+
+/* Used by legacy MODS_ESC_GET_SCREEN_INFO ioctl and as a member of
+ * MODS_SCREEN_INFO_2.
+ */
+struct MODS_SCREEN_INFO {
+	/* OUT */
+	__u8  orig_video_mode;
+	__u8  orig_video_is_vga;
+	__u16 lfb_width;
+	__u16 lfb_height;
+	__u16 lfb_depth;
+	__u32 lfb_base;
+	__u32 lfb_size;
+	__u16 lfb_linelength;
+};
+
+/* Used by MODS_ESC_GET_SCREEN_INFO_2 ioctl.
+ *
+ * Retrieves information about system console, especially framebuffer console
+ * when the system has booted in EFI mode.
+ *
+ * Available only on x86.
+ */
+struct MODS_SCREEN_INFO_2 {
+	/* OUT */
+	struct MODS_SCREEN_INFO info;
+	__u32                   ext_lfb_base;
+};
+
+/* Used by MODS_ESC_SET_PPC_TCE_BYPASS ioctl.
+ *
+ * This call is specific to the ppc64le architecture.
+ * It configures the TCE bypass.
+ */
+struct MODS_SET_PPC_TCE_BYPASS {
+	/* IN */
+	__u8                  mode;
+	__u8                  _dummy_align[7];
+	struct mods_pci_dev_2 pci_device;
+	__u64                 device_dma_mask;
+
+	/* OUT */
+	__u64                 dma_base_address;
+};
+
+/* Values for the mode member above */
+#define MODS_PPC_TCE_BYPASS_DEFAULT 0 /* Use default value from module param */
+#define MODS_PPC_TCE_BYPASS_ON      1 /* Enable TCE bypass */
+#define MODS_PPC_TCE_BYPASS_OFF     2 /* Disable TCE bypass */
+
+/* Used by MODS_ESC_GET_ATS_ADDRESS_RANGE ioctl.
+ *
+ * Retrieves information about NPU device location and address translation
+ * services (part of IOMMU).
+ *
+ * This call is specific to ppc64le architecture.
+ */
+struct MODS_GET_ATS_ADDRESS_RANGE {
+	/* IN */
+	struct mods_pci_dev_2 pci_device;
+	__s32                 npu_index;
+	__u8                  reserved[4]; /* Alignment */
+
+	/* OUT */
+	struct mods_pci_dev_2 npu_device;
+	__u64                 phys_addr;
+	__u64                 guest_addr;
+	__u64                 aperture_size;
+	__s32                 numa_memory_node;
+};
+
+/* Used by MODS_ESC_SET_NVLINK_SYSMEM_TRAINED ioctl.
+ *
+ * Tells the driver whether NVLINK connection to system memory has been trained.
+ *
+ * This call is specific to ppc64le architecture.
+ */
+struct MODS_SET_NVLINK_SYSMEM_TRAINED {
+	/* IN */
+	struct mods_pci_dev_2 pci_device;
+	__u8                  trained;
+};
+
+/* Used by MODS_ESC_GET_NVLINK_LINE_RATE ioctl.
+ *
+ * Retrieves NVLINK speed from device tree.
+ *
+ * This call is specific to ppc64le architecture.
+ */
+struct MODS_GET_NVLINK_LINE_RATE {
+	/* IN */
+	struct mods_pci_dev_2 pci_device;
+	__s32                 npu_index;
+
+	/* OUT */
+	__u32                 speed;
+};
+
+#define MODS_MAX_SYSFS_PATH_BUF_SIZE 512
+#define MODS_MAX_SYSFS_FILE_SIZE 4096
+
+/* Used by MODS_ESC_WRITE_SYSFS_NODE ioctl.
+ *
+ * Writes specified contents to the given sysfs node.
+ *
+ * 'path' parameter is relative to /sys/.
+ */
+struct MODS_SYSFS_NODE {
+	/* IN */
+	char path[MODS_MAX_SYSFS_PATH_BUF_SIZE];
+	char contents[MODS_MAX_SYSFS_FILE_SIZE];
+	__u32 size; /* Size of the contents buffer, in bytes */
+};
+
+/* Used by MODS_ESC_SYSCTL_WRITE_INT ioctl.
+ *
+ * Writes specified integer value into a node under /proc/sys/.
+ *
+ * 'path' parameter is relative to /proc/sys/.
+ */
+struct MODS_SYSCTL_INT {
+	/* IN */
+	char  path[MODS_MAX_SYSFS_PATH_BUF_SIZE];
+	__s64 value;
+};
+
+#define MODS_DRIVER_STATS_VERSION 1
+
+/* Used by MODS_ESC_MODS_GET_DRIVER_STATS ioctl.
+ *
+ * Returns driver operation statistics.
+ */
+struct MODS_GET_DRIVER_STATS {
+	/* OUT */
+	__u64 version;
+	__u64 num_allocs;
+	__u64 num_pages;
+	__u64 reserved[13];
+};
+
+#define MAX_CLOCK_HANDLE_NAME 64
+
+/* Used by MODS_ESC_GET_CLOCK_HANDLE ioctl.
+ *
+ * Available only on Tegra.
+ */
+struct MODS_GET_CLOCK_HANDLE {
+	/* OUT */
+	__u32 clock_handle;
+
+	/* IN */
+	char  device_name[MAX_CLOCK_HANDLE_NAME];
+	char  controller_name[MAX_CLOCK_HANDLE_NAME];
+};
+
+/* Used by ioctls:
+ * - MODS_ESC_SET_CLOCK_RATE
+ * - MODS_ESC_GET_CLOCK_RATE
+ * - MODS_ESC_GET_CLOCK_MAX_RATE
+ * - MODS_ESC_SET_CLOCK_MAX_RATE
+ *
+ * Available only on Tegra.
+ */
+struct MODS_CLOCK_RATE {
+	/* IN/OUT */
+	__u64 clock_rate_hz;
+
+	/* IN */
+	__u32 clock_handle;
+};
+
+/* Used by ioctls:
+ * - MODS_ESC_SET_CLOCK_PARENT
+ * - MODS_ESC_GET_CLOCK_PARENT
+ *
+ * Available only on Tegra.
+ */
+struct MODS_CLOCK_PARENT {
+	/* IN */
+	__u32 clock_handle;
+
+	/* IN/OUT */
+	__u32 clock_parent_handle;
+};
+
+/* Used by ioctls:
+ * - MODS_ESC_ENABLE_CLOCK
+ * - MODS_ESC_DISABLE_CLOCK
+ * - MODS_ESC_CLOCK_RESET_ASSERT
+ * - MODS_ESC_CLOCK_RESET_DEASSERT
+ *
+ * Available only on Tegra.
+ */
+struct MODS_CLOCK_HANDLE {
+	/* IN */
+	__u32 clock_handle;
+};
+
+/* Used by MODS_ESC_IS_CLOCK_ENABLED ioctl.
+ *
+ * Available only on Tegra.
+ */
+struct MODS_CLOCK_ENABLED {
+	/* IN */
+	__u32 clock_handle;
+
+	/* OUT */
+	__u32 enable_count;
+};
+
+struct MODS_TEGRA_DC_WINDOW {
+	__s32 index;
+	__u32 flags;
+	__u32 x;
+	__u32 y;
+	__u32 w;
+	__u32 h;
+	__u32 out_x;
+	__u32 out_y;
+	__u32 out_w;
+	__u32 out_h;
+	__u32 pixformat; /* NVDC pix format */
+
+	__u32 bandwidth;
+};
+
+#define MODS_TEGRA_DC_WINDOW_FLAG_ENABLED   (1 << 0)
+#define MODS_TEGRA_DC_WINDOW_FLAG_TILED     (1 << 1)
+#define MODS_TEGRA_DC_WINDOW_FLAG_SCAN_COL  (1 << 2)
+#define MODS_TEGRA_DC_MAX_WINDOWS           (6)
+
+/* Used by MODS_ESC_TEGRA_DC_CONFIG_POSSIBLE ioctl.
+ *
+ * Available only on Tegra.
+ */
+struct MODS_TEGRA_DC_CONFIG_POSSIBLE {
+	/* IN/OUT */
+	struct MODS_TEGRA_DC_WINDOW windows[MODS_TEGRA_DC_MAX_WINDOWS];
+
+	/* IN */
+	__u8 head;
+	__u8 win_num;
+
+	/* OUT */
+	__u8  possible;
+};
+
+#define MODS_TEGRA_DC_SETUP_SD_LUT_SIZE  9
+#define MODS_TEGRA_DC_SETUP_BLTF_SIZE   16
+
+/* Used by MODS_ESC_TEGRA_DC_SETUP_SD ioctl
+ *
+ * Available only on Tegra.
+ */
+struct MODS_TEGRA_DC_SETUP_SD {
+	/* IN */
+	__u8 head;
+	__u8 enable;
+
+	__u8 use_vid_luma;
+	__u8 csc_r;
+	__u8 csc_g;
+	__u8 csc_b;
+	__u8 aggressiveness;
+	__u8 bin_width_log2;
+
+	__u32 lut[MODS_TEGRA_DC_SETUP_SD_LUT_SIZE];
+	__u32 bltf[MODS_TEGRA_DC_SETUP_BLTF_SIZE];
+
+	__u32 klimit;
+	__u32 soft_clipping_threshold;
+	__u32 smooth_k_inc;
+	__u8  k_init_bias;
+
+
+	__u32 win_x;
+	__u32 win_y;
+	__u32 win_w;
+	__u32 win_h;
+};
+
+#define MODS_ADSP_APP_NAME_SIZE 64
+#define MODS_ADSP_APP_MAX_PARAM 128
+
+/* Used by MODS_ESC_ADSP_RUN_APP ioctl.
+ *
+ * Available only on Tegra.
+ */
+struct MODS_ADSP_RUN_APP_INFO {
+	char app_name[MODS_ADSP_APP_NAME_SIZE];
+	char app_file_name[MODS_ADSP_APP_NAME_SIZE];
+	__u32 argc;
+	__u32 argv[MODS_ADSP_APP_MAX_PARAM];
+	__u32 timeout;
+};
+
+enum MODS_DMA_TRANSACTION_TYPE {
+	MODS_DMA_MEMCPY,
+	MODS_DMA_XOR,
+	MODS_DMA_PQ,
+	MODS_DMA_XOR_VAL,
+	MODS_DMA_PQ_VAL,
+	MODS_DMA_MEMSET,
+	MODS_DMA_MEMSET_SG,
+	MODS_DMA_INTERRUPT,
+	MODS_DMA_SG,
+	MODS_DMA_PRIVATE,
+	MODS_DMA_ASYNC_TX,
+	MODS_DMA_SLAVE,
+	MODS_DMA_CYCLIC,
+	MODS_DMA_INTERLEAVE,
+/* last transaction type for creation of the capabilities mask */
+	MODS_DMA_TX_TYPE_END
+};
+
+/* Used by ioctls:
+ * - MODS_ESC_DMA_REQUEST_HANDLE
+ * - MODS_ESC_DMA_RELEASE_HANDLE
+ * - MODS_ESC_DMA_ISSUE_PENDING
+ *
+ * Available only on Tegra.
+ */
+struct MODS_DMA_HANDLE {
+	/* IN */
+	__u32 dma_type; /* Indicate the DMA Type*/
+	/* OUT */
+	__u32 dma_id; /* Identifier for the DMA */
+};
+
+#define MODS_DMA_HANDLE_CTRL_DIR_LEN 20
+
+/* Used by ioctls:
+ * - MODS_ESC_DMA_REQUEST_HANDLE_2
+ *
+ * Available only on Tegra.
+ */
+struct MODS_DMA_HANDLE_2 {
+	/* IN */
+	__u32 dma_type; /* Indicate the DMA Type*/
+	/* OUT */
+	__u32 dma_id; /* Identifier for the DMA */
+
+	/* ctrl#index_dir
+	 * for example, spi0_tx, i2c1_rx
+	 * to keep back-compatibility, strlen of ctrl_dir is allowed to be zero
+	 */
+	char ctrl_dir[MODS_DMA_HANDLE_CTRL_DIR_LEN];
+};
+
+enum MODS_DMA_TRANSFER_DIRECTION {
+	MODS_DMA_MEM_TO_MEM,
+	MODS_DMA_MEM_TO_DEV,
+	MODS_DMA_DEV_TO_MEM,
+	MODS_DMA_DEV_TO_DEV,
+	MODS_DMA_TRANS_NONE
+};
+
+enum MODS_DMA_BUSWIDTH {
+	MODS_DMA_BUSWIDTH_UNDEFINED = 0,
+	MODS_DMA_BUSWIDTH_1_BYTE = 1,
+	MODS_DMA_BUSWIDTH_2_BYTES = 2,
+	MODS_DMA_BUSWIDTH_4_BYTES = 4,
+	MODS_DMA_BUSWIDTH_8_BYTES = 8
+};
+
+/* Used by MODS_ESC_DMA_SET_CONFIG ioctl.
+ *
+ * Available only on Tegra.
+ */
+struct MODS_DMA_CHANNEL_CONFIG {
+	__u64 src_addr;
+	__u64 dst_addr;
+	struct MODS_DMA_HANDLE handle;
+	__u32 direction;
+	__u32 src_addr_width;
+	__u32 dst_addr_width;
+	__u32 src_maxburst;
+	__u32 dst_maxburst;
+	__u32 slave_id;
+	__u32 device_fc;
+};
+
+/* Node: Only support SINGLE MODS so far*/
+enum MODS_DMA_TX_MODE {
+	MODS_DMA_SINGLE = 0,
+	MODS_DMA_TX_CYCLIC,
+	MODS_DMA_INTERLEAVED /* Common to Slave as well as M2M clients. */
+};
+
+/* Deprecated */
+typedef __s32 mods_dma_cookie_t;
+
+/* Used by MODS_ESC_DMA_TX_SUBMIT ioctl.
+ *
+ * Available only on Tegra.
+ */
+struct MODS_DMA_TX_DESC	{
+	/* IN */
+	__u64 phys;
+	__u64 phys_2; /* only valid for MEMCPY */
+	struct MODS_DMA_HANDLE handle;
+	__u32 mode;
+	__u32 data_dir;
+	__u32 length;
+	__u32 flags;
+	/* OUT */
+	__s32 cookie;
+};
+
+enum MODS_DMA_WAIT_TYPE {
+	MODS_DMA_SYNC_WAIT,     /* wait until finished */
+	MODS_DMA_ASYNC_WAIT     /* just check tx status */
+};
+
+/* Used by MODS_ESC_DMA_TX_WAIT ioctl.
+ *
+ * Available only on Tegra.
+ */
+struct MODS_DMA_WAIT_DESC {
+	struct MODS_DMA_HANDLE handle;
+	__s32 cookie;
+	__u32 type;
+	/* OUT */
+	__u32 tx_complete;
+};
+
+#define MAX_NET_DEVICE_NAME_LENGTH 16
+
+/* Used by MODS_ESC_NET_FORCE_LINK ioctl.
+ *
+ * Available only on Tegra.
+ */
+struct MODS_NET_DEVICE_NAME {
+	/* in */
+	char device_name[MAX_NET_DEVICE_NAME_LENGTH];
+};
+
+/* Used by ioctls:
+ * - MODS_ESC_DMA_ALLOC_COHERENT
+ * - MODS_ESC_DMA_FREE_COHERENT
+ *
+ * Available only on Tegra.
+ */
+struct MODS_DMA_COHERENT_MEM_HANDLE {
+	__u32	num_bytes;
+	__u32	attrib;
+	__u64	memory_handle_phys;
+	__u64	memory_handle_virt;
+};
+
+/* Used by MODS_ESC_DMA_COPY_TO_USER ioctl.
+ *
+ * Available only on Tegra.
+ */
+struct MODS_DMA_COPY_TO_USER {
+	__u32	num_bytes;
+	__u32	attrib;
+	__u64	memory_handle_src;
+	__u64	memory_handle_dst;
+};
+
+/* Used by iocls:
+ * - MODS_ESC_TEGRA_PROD_SET_PROD_ALL
+ * - MODS_ESC_TEGRA_PROD_SET_PROD_BOOT
+ * - MODS_ESC_TEGRA_PROD_SET_PROD_BY_NAME
+ * - MODS_ESC_TEGRA_PROD_SET_PROD_EXACT
+ *
+ * Available only on Tegra.
+ */
+struct MODS_TEGRA_PROD_SET_TUPLE {
+	/* IN */
+	__u64 prod_dev_handle;
+	__u64 ctrl_dev_handle;
+	char prod_name[MAX_DT_SIZE];
+	__u32 index;
+	__u32 offset;
+	__u32 mask;
+};
+
+/* Used by MODS_ESC_TEGRA_PROD_IS_SUPPORTED ioctl.
+ *
+ * Available only on Tegra.
+ */
+struct MODS_TEGRA_PROD_IS_SUPPORTED {
+	/* IN */
+	__u64 prod_dev_handle;
+	char prod_name[MAX_DT_SIZE];
+	/* OUT */
+	__u32 is_supported;
+};
+
+/* Used by MODS_ESC_TEGRA_PROD_ITERATE_DT ioctl.
+ *
+ * Available only on Tegra.
+ */
+struct MODS_TEGRA_PROD_ITERATOR {
+	/* IN */
+	__u64 device_handle;
+	char name[MAX_DT_SIZE];
+	char next_name[MAX_DT_SIZE];
+	__u32 index;
+	__u32 is_leaf;
+	/* OUT */
+	__u64 next_device_handle;
+};
+
+/* Used by MODS_ESC_IOMMU_DMA_MAP_MEMORY ioctl.
+ *
+ * Available only on Tegra.
+ */
+struct MODS_IOMMU_DMA_MAP_MEMORY {
+	/* IN */
+	__u64 memory_handle;
+	char  dev_name[MAX_DT_SIZE];
+	__u8  flags;
+	__u8  reserved[7]; /* Force 64-bit member alignment */
+
+	/* OUT */
+	__u64 physical_address;
+};
+
+#define MAX_TZ_BUFFER_SIZE 512
+/* Used by MODS_ESC_SEND_TZ_MSG.
+ *
+ * Available only on Tegra.
+ */
+struct MODS_TZ_PARAMS {
+	/* IN */
+	__u8 buf[MAX_TZ_BUFFER_SIZE];
+	__u32 buf_size;
+	__u32 cmd;
+
+	/* OUT */
+	int status;
+};
+
+#define MAX_OPTEE_BUFFER_SIZE 512
+/* Used by MODS_ESC_INVOKE_OPTEE_TA.
+ *
+ * Available only on Tegra.
+ */
+struct MODS_OPTEE_PARAMS {
+	/* IN */
+	__u8 buf[MAX_OPTEE_BUFFER_SIZE];
+	__u32 buf_size;
+	__u32 command_id;
+
+	/* OUT */
+	__u32 tee_ret;
+	__u32 out_a;
+	__u32 out_b;
+};
+
+/* Used by MODS_ESC_OIST_STATUS ioctl.
+ *
+ * Available only on Tegra.
+ */
+struct MODS_TEGRA_OIST_STATUS {
+	/* IN */
+	__u64 smc_func_id;
+	/* IN */
+	__u64 a1;
+	/* IN */
+	__u64 a2;
+	/* OUT */
+	__u64 smc_status;
+};
+
+enum MODS_IPI_TYPE {
+	MODS_IPI_KICK,
+	MODS_IPI_WFI,
+	MODS_IPI_WFE
+};
+
+/* Used by MODS_ESC_MODS_SEND_IPI ioctl.
+ *
+ * Available only on Tegra.
+ */
+struct MODS_SEND_IPI {
+	/* IN */
+	__u32 ipi_type;
+	/* IN */
+	__u32 num_loops;
+};
+
+/* Used by MODS_ESC_FFA_CMD ioctl.
+ */
+struct MODS_FFA_PARAMS {
+	/* IN */
+	__u64 cmd;
+	__u64 indata[4];
+	/* OUT */
+	__u64 outdata[4];
+};
+
+enum MODS_SP_FFA_CMD_TYPE {
+	MODS_FFA_CMD_READ_REG,
+	MODS_FFA_CMD_WRITE_REG,
+	MODS_FFA_CMD_READ_VER,
+	MODS_FFA_CMD_SE_TESTS,
+	MODS_FFA_CMD_SE_KEY_MOVER,
+	MODS_FFA_CMD_HSS_TEST,
+	MODS_FFA_CMD_C2C_TEST,
+	MODS_FFA_CMD_TYPE_END
+};
+
+#define MODS_IOMMU_MAP_CONTIGUOUS 1
+
+#define MODS_MAX_PROP_NAME_LEN 64
+#define MODS_MAX_PROP_SIZE     8192
+
+#define MODS_PROP_TYPE_U64 8
+
+/* Used by MODS_ESC_READ_DEV_PROPERTY
+ *
+ * Reads a device property from firmware.
+ */
+struct MODS_READ_DEV_PROPERTY {
+	/* IN */
+	struct mods_pci_dev_2 pci_device;
+	char                  prop_name[MODS_MAX_PROP_NAME_LEN];
+	__u32                 array_size;
+	__u8                  type;
+	__u8                  dummy_align[3];
+
+	/* OUT */
+	__u8                  output[MODS_MAX_PROP_SIZE];
+};
+
+/* Used by MODS_ESC_PROXIMITY_TO_NUMA_NODE
+ *
+ * Converts proximity id to NUMA node id.
+ */
+struct MODS_PROXIMITY_TO_NUMA_NODE {
+	/* IN */
+	__s32 proximity;
+
+	/* OUT */
+	__s32 numa_node;
+};
+
+/* Used by MODS_ESC_BPMP_UPHY_LANE_EOM_SCAN
+ *
+ * Get EOM data of UPHY lane from bpmp
+ */
+struct MODS_BPMP_UPHY_LANE_EOM_SCAN_PARAMS {
+	/* IN */
+	/* the address of bpmp db/req/resp is various for different
+	 * chip/socket, the value of them will be passed from user.
+	 */
+	__u64 db_phys_addr;
+	__u64 req_phys_addr;
+	__u64 resp_phys_addr;
+	__u32 brick;
+	__u32 lane;
+	__u32 pcie_gen5;
+
+	/* OUT */
+	__u32 data;
+};
+
+#pragma pack(pop)
+
+#define MODS_IOC_MAGIC 'x'
+
+#define MODSIO(dir, num, type) _IO##dir(MODS_IOC_MAGIC, num, struct type)
+
+#define MODS_ESC_ALLOC_PAGES MODSIO(WR, 0, MODS_ALLOC_PAGES)
+#define MODS_ESC_FREE_PAGES MODSIO(WR, 1, MODS_FREE_PAGES)
+/* Deprecated */
+#define MODS_ESC_GET_PHYSICAL_ADDRESS MODSIO(WR, 2, MODS_GET_PHYSICAL_ADDRESS)
+#define MODS_ESC_VIRTUAL_TO_PHYSICAL MODSIO(WR, 3, MODS_VIRTUAL_TO_PHYSICAL)
+#define MODS_ESC_PHYSICAL_TO_VIRTUAL MODSIO(WR, 4, MODS_PHYSICAL_TO_VIRTUAL)
+/* Deprecated */
+#define MODS_ESC_FIND_PCI_DEVICE MODSIO(WR, 5, MODS_FIND_PCI_DEVICE)
+/* Deprecated */
+#define MODS_ESC_FIND_PCI_CLASS_CODE MODSIO(WR, 6, MODS_FIND_PCI_CLASS_CODE)
+/* Deprecated */
+#define MODS_ESC_PCI_READ MODSIO(WR, 7, MODS_PCI_READ)
+/* Deprecated */
+#define MODS_ESC_PCI_WRITE MODSIO(WR, 8, MODS_PCI_WRITE)
+#define MODS_ESC_PIO_READ MODSIO(WR, 9, MODS_PIO_READ)
+#define MODS_ESC_PIO_WRITE MODSIO(WR, 10, MODS_PIO_WRITE)
+/* Deprecated */
+#define MODS_ESC_IRQ_REGISTER MODSIO(WR, 11, MODS_IRQ)
+/* Deprecated */
+#define MODS_ESC_IRQ_FREE MODSIO(WR, 12, MODS_IRQ)
+/* Deprecated */
+#define MODS_ESC_IRQ_INQUIRY MODSIO(WR, 13, MODS_IRQ)
+#define MODS_ESC_EVAL_ACPI_METHOD MODSIO(WR_BAD, 16, MODS_EVAL_ACPI_METHOD)
+#define MODS_ESC_GET_API_VERSION MODSIO(WR, 17, MODS_GET_VERSION)
+/* Deprecated */
+#define MODS_ESC_GET_KERNEL_VERSION MODSIO(WR, 18, MODS_GET_VERSION)
+/* Deprecated */
+#define MODS_ESC_SET_DRIVER_PARA MODSIO(WR, 19, MODS_SET_PARA)
+/* Deprecated */
+#define MODS_ESC_MSI_REGISTER MODSIO(WR, 20, MODS_IRQ)
+/* Deprecated */
+#define MODS_ESC_REARM_MSI MODSIO(WR, 21, MODS_IRQ)
+#define MODS_ESC_SET_MEMORY_TYPE MODSIO(W, 22, MODS_MEMORY_TYPE)
+/* Deprecated */
+#define MODS_ESC_PCI_BUS_ADD_DEVICES MODSIO(W, 23, MODS_PCI_BUS_ADD_DEVICES)
+/* Deprecated */
+#define MODS_ESC_REGISTER_IRQ MODSIO(W, 24, MODS_REGISTER_IRQ)
+/* Deprecated */
+#define MODS_ESC_UNREGISTER_IRQ MODSIO(W, 25, MODS_REGISTER_IRQ)
+#define MODS_ESC_QUERY_IRQ MODSIO(R, 26, MODS_QUERY_IRQ)
+/* Deprecated */
+#define MODS_ESC_EVAL_DEV_ACPI_METHOD MODSIO(WR_BAD, 27, \
+					     MODS_EVAL_DEV_ACPI_METHOD)
+#define MODS_ESC_ACPI_GET_DDC MODSIO(WR, 28, MODS_ACPI_GET_DDC)
+#define MODS_ESC_GET_CLOCK_HANDLE MODSIO(WR, 29, MODS_GET_CLOCK_HANDLE)
+#define MODS_ESC_SET_CLOCK_RATE MODSIO(W, 30, MODS_CLOCK_RATE)
+#define MODS_ESC_GET_CLOCK_RATE MODSIO(WR, 31, MODS_CLOCK_RATE)
+#define MODS_ESC_SET_CLOCK_PARENT MODSIO(W, 32, MODS_CLOCK_PARENT)
+#define MODS_ESC_GET_CLOCK_PARENT MODSIO(WR, 33, MODS_CLOCK_PARENT)
+#define MODS_ESC_ENABLE_CLOCK MODSIO(W, 34, MODS_CLOCK_HANDLE)
+#define MODS_ESC_DISABLE_CLOCK MODSIO(W, 35, MODS_CLOCK_HANDLE)
+#define MODS_ESC_IS_CLOCK_ENABLED MODSIO(WR, 36, MODS_CLOCK_ENABLED)
+#define MODS_ESC_CLOCK_RESET_ASSERT MODSIO(W, 37, MODS_CLOCK_HANDLE)
+#define MODS_ESC_CLOCK_RESET_DEASSERT MODSIO(W, 38, MODS_CLOCK_HANDLE)
+/* Deprecated */
+#define MODS_ESC_SET_IRQ_MASK MODSIO(W, 39, MODS_SET_IRQ_MASK)
+/* Deprecated */
+#define MODS_ESC_MEMORY_BARRIER _IO(MODS_IOC_MAGIC, 40)
+/* Deprecated */
+#define MODS_ESC_IRQ_HANDLED MODSIO(W, 41, MODS_REGISTER_IRQ)
+#define MODS_ESC_FLUSH_CPU_CACHE_RANGE MODSIO(W, 42, MODS_FLUSH_CPU_CACHE_RANGE)
+#define MODS_ESC_GET_CLOCK_MAX_RATE MODSIO(WR, 43, MODS_CLOCK_RATE)
+#define MODS_ESC_SET_CLOCK_MAX_RATE MODSIO(W, 44, MODS_CLOCK_RATE)
+#define MODS_ESC_DEVICE_ALLOC_PAGES MODSIO(WR, 45, MODS_DEVICE_ALLOC_PAGES)
+/* Deprecated */
+#define MODS_ESC_DEVICE_NUMA_INFO MODSIO(WR, 46, MODS_DEVICE_NUMA_INFO)
+#define MODS_ESC_TEGRA_DC_CONFIG_POSSIBLE MODSIO(WR, 47, \
+						 MODS_TEGRA_DC_CONFIG_POSSIBLE)
+#define MODS_ESC_TEGRA_DC_SETUP_SD MODSIO(W, 48, MODS_TEGRA_DC_SETUP_SD)
+#define MODS_ESC_DMABUF_GET_PHYSICAL_ADDRESS MODSIO(WR, 49, \
+					MODS_DMABUF_GET_PHYSICAL_ADDRESS)
+#define MODS_ESC_ADSP_LOAD _IO(MODS_IOC_MAGIC, 50)
+#define MODS_ESC_ADSP_START _IO(MODS_IOC_MAGIC, 51)
+#define MODS_ESC_ADSP_STOP _IO(MODS_IOC_MAGIC, 52)
+#define MODS_ESC_ADSP_RUN_APP MODSIO(W, 53, MODS_ADSP_RUN_APP_INFO)
+/* Deprecated */
+#define MODS_ESC_PCI_GET_BAR_INFO MODSIO(WR, 54, MODS_PCI_GET_BAR_INFO)
+/* Deprecated */
+#define MODS_ESC_PCI_GET_IRQ MODSIO(WR, 55, MODS_PCI_GET_IRQ)
+/* Deprecated */
+#define MODS_ESC_GET_MAPPED_PHYSICAL_ADDRESS MODSIO(WR, 56, \
+						    MODS_GET_PHYSICAL_ADDRESS)
+#define MODS_ESC_DEVICE_ALLOC_PAGES_2 MODSIO(WR, 57, MODS_DEVICE_ALLOC_PAGES_2)
+#define MODS_ESC_FIND_PCI_DEVICE_2 MODSIO(WR, 58, MODS_FIND_PCI_DEVICE_2)
+#define MODS_ESC_FIND_PCI_CLASS_CODE_2 MODSIO(WR, 59, \
+					      MODS_FIND_PCI_CLASS_CODE_2)
+#define MODS_ESC_PCI_GET_BAR_INFO_2 MODSIO(WR, 60, MODS_PCI_GET_BAR_INFO_2)
+#define MODS_ESC_PCI_GET_IRQ_2 MODSIO(WR, 61, MODS_PCI_GET_IRQ_2)
+#define MODS_ESC_PCI_READ_2 MODSIO(WR, 62, MODS_PCI_READ_2)
+#define MODS_ESC_PCI_WRITE_2 MODSIO(W, 63, MODS_PCI_WRITE_2)
+/* Deprecated */
+#define MODS_ESC_REGISTER_IRQ_2 MODSIO(W, 64, MODS_REGISTER_IRQ_2)
+#define MODS_ESC_UNREGISTER_IRQ_2 MODSIO(W, 65, MODS_REGISTER_IRQ_2)
+#define MODS_ESC_IRQ_HANDLED_2 MODSIO(W, 66, MODS_REGISTER_IRQ_2)
+/* Deprecated */
+#define MODS_ESC_QUERY_IRQ_2 MODSIO(R, 67, MODS_QUERY_IRQ_2)
+/* Deprecated */
+#define MODS_ESC_SET_IRQ_MASK_2 MODSIO(W, 68, MODS_SET_IRQ_MASK_2)
+#define MODS_ESC_EVAL_DEV_ACPI_METHOD_2 MODSIO(WR_BAD, 69, \
+					       MODS_EVAL_DEV_ACPI_METHOD_2)
+#define MODS_ESC_DEVICE_NUMA_INFO_2 MODSIO(WR, 70, MODS_DEVICE_NUMA_INFO_2)
+#define MODS_ESC_ACPI_GET_DDC_2 MODSIO(WR, 71, MODS_ACPI_GET_DDC_2)
+/* Deprecated */
+#define MODS_ESC_GET_SCREEN_INFO MODSIO(R, 72, MODS_SCREEN_INFO)
+#define MODS_ESC_PCI_HOT_RESET MODSIO(W, 73, MODS_PCI_HOT_RESET)
+#define MODS_ESC_SET_PPC_TCE_BYPASS MODSIO(WR, 74, MODS_SET_PPC_TCE_BYPASS)
+#define MODS_ESC_DMA_MAP_MEMORY MODSIO(W, 75, MODS_DMA_MAP_MEMORY)
+#define MODS_ESC_DMA_UNMAP_MEMORY MODSIO(W, 76, MODS_DMA_MAP_MEMORY)
+/* Deprecated */
+#define MODS_ESC_GET_MAPPED_PHYSICAL_ADDRESS_2 MODSIO(WR, 77, \
+						MODS_GET_PHYSICAL_ADDRESS_2)
+/* Deprecated */
+#define MODS_ESC_PCI_MAP_RESOURCE MODSIO(WR, 78, MODS_PCI_MAP_RESOURCE)
+/* Deprecated */
+#define MODS_ESC_PCI_UNMAP_RESOURCE MODSIO(W, 79, MODS_PCI_UNMAP_RESOURCE)
+#define MODS_ESC_DMA_REQUEST_HANDLE MODSIO(R,  80, MODS_DMA_HANDLE)
+#define MODS_ESC_DMA_REQUEST_HANDLE_2 MODSIO(R,  80, MODS_DMA_HANDLE_2)
+#define MODS_ESC_DMA_RELEASE_HANDLE MODSIO(W,  81, MODS_DMA_HANDLE)
+#define MODS_ESC_DMA_SET_CONFIG MODSIO(W,  82, MODS_DMA_CHANNEL_CONFIG)
+#define MODS_ESC_DMA_TX_SUBMIT MODSIO(W,  83, MODS_DMA_TX_DESC)
+#define MODS_ESC_DMA_TX_WAIT MODSIO(WR, 84, MODS_DMA_WAIT_DESC)
+#define MODS_ESC_DMA_ISSUE_PENDING MODSIO(W,  85, MODS_DMA_HANDLE)
+/* Deprecated */
+#define MODS_ESC_SET_IRQ_MULTIMASK MODSIO(W, 86, MODS_SET_IRQ_MULTIMASK)
+#define MODS_ESC_NET_FORCE_LINK MODSIO(W, 87, MODS_NET_DEVICE_NAME)
+/* Deprecated */
+#define MODS_ESC_REGISTER_IRQ_3 MODSIO(W, 88, MODS_REGISTER_IRQ_3)
+#define MODS_ESC_DMA_ALLOC_COHERENT MODSIO(WR, 89, MODS_DMA_COHERENT_MEM_HANDLE)
+#define MODS_ESC_DMA_FREE_COHERENT MODSIO(WR, 90, MODS_DMA_COHERENT_MEM_HANDLE)
+#define MODS_ESC_DMA_COPY_TO_USER MODSIO(WR, 91, MODS_DMA_COPY_TO_USER)
+#define MODS_ESC_MAP_INTERRUPT MODSIO(WR, 92, MODS_DT_INFO)
+/* Locks system console */
+#define MODS_ESC_LOCK_CONSOLE _IO(MODS_IOC_MAGIC, 93)
+/* Unlocks system console */
+#define MODS_ESC_UNLOCK_CONSOLE _IO(MODS_IOC_MAGIC, 94)
+#define MODS_ESC_TEGRA_PROD_IS_SUPPORTED MODSIO(WR, 95, \
+						MODS_TEGRA_PROD_IS_SUPPORTED)
+#define MODS_ESC_TEGRA_PROD_SET_PROD_ALL MODSIO(W, 96, \
+						MODS_TEGRA_PROD_SET_TUPLE)
+#define MODS_ESC_TEGRA_PROD_SET_PROD_BOOT MODSIO(W, 97, \
+						 MODS_TEGRA_PROD_SET_TUPLE)
+#define MODS_ESC_TEGRA_PROD_SET_PROD_BY_NAME MODSIO(W, 98, \
+						    MODS_TEGRA_PROD_SET_TUPLE)
+#define MODS_ESC_TEGRA_PROD_SET_PROD_EXACT MODSIO(W, 99, \
+						  MODS_TEGRA_PROD_SET_TUPLE)
+#define MODS_ESC_TEGRA_PROD_ITERATE_DT MODSIO(WR, 100, MODS_TEGRA_PROD_ITERATOR)
+#define MODS_ESC_GET_ATS_ADDRESS_RANGE MODSIO(WR, 101, \
+					      MODS_GET_ATS_ADDRESS_RANGE)
+#define MODS_ESC_SET_NVLINK_SYSMEM_TRAINED MODSIO(W, 102, \
+						MODS_SET_NVLINK_SYSMEM_TRAINED)
+#define MODS_ESC_GET_NVLINK_LINE_RATE MODSIO(WR, 103, MODS_GET_NVLINK_LINE_RATE)
+/* Suspends system console */
+#define MODS_ESC_SUSPEND_CONSOLE _IO(MODS_IOC_MAGIC, 104)
+/* Resumes system console */
+#define MODS_ESC_RESUME_CONSOLE _IO(MODS_IOC_MAGIC, 105)
+#define MODS_ESC_GET_SCREEN_INFO_2 MODSIO(R, 106, MODS_SCREEN_INFO_2)
+#define MODS_ESC_ACQUIRE_ACCESS_TOKEN MODSIO(R, 107, MODS_ACCESS_TOKEN)
+#define MODS_ESC_RELEASE_ACCESS_TOKEN MODSIO(W, 108, MODS_ACCESS_TOKEN)
+#define MODS_ESC_VERIFY_ACCESS_TOKEN MODSIO(W, 109, MODS_ACCESS_TOKEN)
+/* Deprecated */
+#define MODS_ESC_GET_IOMMU_STATE MODSIO(WR, 110, MODS_GET_IOMMU_STATE)
+#define MODS_ESC_WRITE_SYSFS_NODE MODSIO(W, 111, MODS_SYSFS_NODE)
+#define MODS_ESC_GET_PHYSICAL_ADDRESS_2 MODSIO(WR, 112, \
+					       MODS_GET_PHYSICAL_ADDRESS_3)
+#define MODS_ESC_GET_MAPPED_PHYSICAL_ADDRESS_3 MODSIO(WR, 113, \
+						MODS_GET_PHYSICAL_ADDRESS_3)
+#define MODS_ESC_REGISTER_IRQ_4 MODSIO(W, 114, MODS_REGISTER_IRQ_4)
+#define MODS_ESC_QUERY_IRQ_3 MODSIO(R, 115, MODS_QUERY_IRQ_3)
+#define MODS_ESC_SET_NUM_VF MODSIO(W, 116, MODS_SET_NUM_VF)
+#define MODS_ESC_SET_TOTAL_VF MODSIO(W, 117, MODS_SET_NUM_VF)
+#define MODS_ESC_PCI_SET_DMA_MASK MODSIO(W, 118, MODS_PCI_DMA_MASK)
+#define MODS_ESC_GET_IOMMU_STATE_2 MODSIO(WR, 119, MODS_GET_IOMMU_STATE)
+#define MODS_ESC_READ_MSR MODSIO(WR, 120, MODS_MSR)
+#define MODS_ESC_WRITE_MSR MODSIO(W, 121, MODS_MSR)
+#define MODS_ESC_EVAL_DEV_ACPI_METHOD_3 MODSIO(WR_BAD, 122, \
+					       MODS_EVAL_DEV_ACPI_METHOD_3)
+#define MODS_ESC_PCI_BUS_REMOVE_DEV MODSIO(W, 123, MODS_PCI_BUS_REMOVE_DEV)
+#define MODS_ESC_ALLOC_PAGES_2 MODSIO(WR, 124, MODS_ALLOC_PAGES_2)
+#define MODS_ESC_MERGE_PAGES MODSIO(WR, 125, MODS_MERGE_PAGES)
+#define MODS_ESC_DEVICE_NUMA_INFO_3 MODSIO(WR, 126, MODS_DEVICE_NUMA_INFO_3)
+#define MODS_ESC_PCI_BUS_RESCAN MODSIO(W, 127, MODS_PCI_BUS_RESCAN)
+#define MODS_ESC_MAP_GPIO MODSIO(WR, 128, MODS_GPIO_INFO)
+#define MODS_ESC_IOMMU_DMA_MAP_MEMORY MODSIO(W, 129, MODS_IOMMU_DMA_MAP_MEMORY)
+#define MODS_ESC_IOMMU_DMA_UNMAP_MEMORY MODSIO(W, 130, \
+					       MODS_IOMMU_DMA_MAP_MEMORY)
+#define MODS_ESC_RESET_ASSERT MODSIO(W, 131, MODS_RESET_HANDLE)
+#define MODS_ESC_GET_RESET_HANDLE MODSIO(WR, 132, MODS_GET_RESET_HANDLE)
+#define MODS_ESC_SYSCTL_WRITE_INT MODSIO(W, 133, MODS_SYSCTL_INT)
+#define MODS_ESC_PCI_RESET_FUNCTION MODSIO(W, 134, mods_pci_dev_2)
+#define MODS_ESC_MODS_GET_DRIVER_STATS MODSIO(R, 135, MODS_GET_DRIVER_STATS)
+#define MODS_ESC_BPMP_SET_PCIE_STATE MODSIO(W, 136, MODS_SET_PCIE_STATE)
+#define MODS_ESC_BPMP_INIT_PCIE_EP_PLL MODSIO(W, 137, MODS_INIT_PCIE_EP_PLL)
+#define MODS_ESC_GET_ACPI_DEV_CHILDREN MODSIO(WR, 138, MODS_GET_ACPI_DEV_CHILDREN)
+#define MODS_ESC_SEND_TZ_MSG MODSIO(WR, 139, MODS_TZ_PARAMS)
+#define MODS_ESC_OIST_STATUS MODSIO(WR, 140, MODS_TEGRA_OIST_STATUS)
+#define MODS_ESC_INVOKE_OPTEE_TA MODSIO(WR, 141, MODS_OPTEE_PARAMS)
+#define MODS_ESC_READ_DEV_PROPERTY MODSIO(WR, 142, MODS_READ_DEV_PROPERTY)
+#define MODS_ESC_PROXIMITY_TO_NUMA_NODE MODSIO(WR, 143, MODS_PROXIMITY_TO_NUMA_NODE)
+#define MODS_ESC_MODS_SEND_IPI MODSIO(W, 144, MODS_SEND_IPI)
+#define MODS_ESC_FFA_CMD MODSIO(WR, 145, MODS_FFA_PARAMS)
+#define MODS_ESC_BPMP_UPHY_LANE_EOM_SCAN MODSIO(WR, 146, \
+						MODS_BPMP_UPHY_LANE_EOM_SCAN_PARAMS)
+
+#endif /* _UAPI_MODS_H_  */
diff --git a/kernel-src-files-copy-list.txt b/kernel-src-files-copy-list.txt
index 267b319f..69c5c948 100644
--- a/kernel-src-files-copy-list.txt
+++ b/kernel-src-files-copy-list.txt
@@ -53,10 +53,6 @@ nvidia/include/uapi/linux/nvpva_ioctl.h include/uapi/linux/nvpva_ioctl.h
 nvidia/drivers/platform/tegra/hwpm drivers/platform/tegra
 nvidia/include/uapi/linux/tegra-soc-hwpm-uapi.h include/uapi/linux/tegra-soc-hwpm-uapi.h
 
-# Files/directories for MODS
-nvidia/drivers/misc/mods drivers/misc
-nvidia/include/uapi/misc/mods.h include/uapi/misc/mods.h
-
 # Files/directory for DCE-KMD
 nvidia/drivers/platform/tegra/dce drivers/platform/tegra
 nvidia/include/linux/platform/tegra/dce/dce-client-ipc.h include/linux/platform/tegra/dce/dce-client-ipc.h