drivers: misc: add nvscic2c-pcie in OOT

nvscic2c enables xfer between two SoC connected over PCIe.
nvscic2c already available with K5.10

Major changes in nvscic2c-pcie for K5.15:
 Integrate with upstream PCIe endpoint function changes.

 Allocate new IOVA domain to be used for
 empty IOVA allocation.
 This IOVA range would be used in iommu_map() to map
 physical backing with IOMMU domain attached in PCIe device.

 Migrate from nvhost to host1x and tegra-drm interfaces.

Bug 3739487
Jira C2C-826
Jira C2C-830

Change-Id: Ic4d8ac680921807fb17247855ca7037623681cb7
Signed-off-by: dbadgaiyan <dbadgaiyan@nvidia.com>
Reviewed-on: https://git-master.nvidia.com/r/c/linux-nv-oot/+/2810806
Reviewed-by: Nagarjuna Kristam <nkristam@nvidia.com>
Reviewed-by: Arihant Jejani <ajejani@nvidia.com>
Reviewed-by: Laxman Dewangan <ldewangan@nvidia.com>
GVS: Gerrit_Virtual_Submit <buildbot_gerritrpt@nvidia.com>

drivers/misc/Makefile

@@ -5,3 +5,4 @@ obj-m += mods/
 obj-m += nvsciipc/
 obj-m += tegra-pcie-dma-test.o
 obj-m += bluedroid_pm.o
+obj-m += nvscic2c-pcie/

drivers/misc/nvscic2c-pcie/Kconfig

@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0
+config NVSCIC2C_PCIE
+	tristate "NVIDIA Chip-to-Chip between PCIe-RP and PCIe-EP"
+	depends on PCI && PCI_ENDPOINT
+	default n
+	help
+	 This enables SoftwareCommunicationInterface for Host-to-Host
+	 communication between PCIe Rootport and PCIe Endpoint.
+	 If unsure, Please say N.

drivers/misc/nvscic2c-pcie/Makefile

@@ -0,0 +1,14 @@
+# SPDX-License-Identifier: GPL-2.0
+
+ccflags-y += -Werror
+ccflags-y += -DDYNAMIC_DEBUG_MODULE
+ccflags-y += -I$(srctree.nvidia-oot)/drivers/misc/nvscic2c-pcie
+#host1x-next and Tegra DRM are not moved to nvidia-oot include path yet.
+ccflags-y += -I$(srctree.nvidia)/drivers/gpu/host1x/include
+ccflags-y += -I$(srctree.nvidia)/drivers/gpu/drm/tegra/include
+
+ifdef CONFIG_PCIE_TEGRA194_EP
+obj-m := nvscic2c-pcie-epc.o nvscic2c-pcie-epf.o
+nvscic2c-pcie-epc-y := comm-channel.o dt.o endpoint.o epc/module.o iova-alloc.o iova-mngr.o pci-client.o stream-extensions.o vmap.o vmap-pin.o
+nvscic2c-pcie-epf-y := comm-channel.o dt.o endpoint.o epf/module.o iova-alloc.o iova-mngr.o pci-client.o stream-extensions.o vmap.o vmap-pin.o
+endif

drivers/misc/nvscic2c-pcie/comm-channel.c

@@ -0,0 +1,889 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#define pr_fmt(fmt)	"nvscic2c-pcie: comm-channel: " fmt
+
+#include <linux/atomic.h>
+#include <linux/dma-fence.h>
+#include <linux/dma-iommu.h>
+#include <linux/errno.h>
+#include <linux/host1x-next.h>
+#include <linux/kernel.h>
+#include <linux/kthread.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/types.h>
+#include <linux/wait.h>
+
+#include <uapi/misc/nvscic2c-pcie-ioctl.h>
+
+#include "comm-channel.h"
+#include "common.h"
+#include "module.h"
+#include "pci-client.h"
+
+#define CACHE_ALIGN		(64)
+
+/* Fifo size */
+/*
+ * This is wrong, but to have private communication channel functional at the
+ * earliest, we allocate large set of frames assuming all the available
+ * endpoints can share all possible export descriptors without having to block
+ * and wait for channel to become writeable.
+ *
+ * Despite this huge fifo size, if msg cannot be send, it either means remote
+ * is busy processing them quite slow (unlikely) or ill. In such a case, we
+ * shall return -EAGAIN for application to retry and application can bail-out
+ * after few retries.
+ */
+#define COMM_CHANNEL_NFRAMES	(1024)
+#define COMM_CHANNEL_FRAME_SZ	(64)
+
+/* fifo header.*/
+struct header {
+	u32 wr_count;
+	u32 rd_count;
+	u8 pad[CACHE_ALIGN - sizeof(u32) - sizeof(u32)];
+} __packed;
+
+/* kthread. */
+struct task_t {
+	struct task_struct *task;
+	wait_queue_head_t waitq;
+	struct completion shutdown_compl;
+	bool shutdown;
+	bool created;
+};
+
+/* Notification handling. */
+struct syncpt_t {
+	u32 id;
+	u32 threshold;
+	struct host1x_syncpt *sp;
+
+	/* PCIe aperture for writes to peer syncpoint for same comm-channel. */
+	struct pci_aper_t peer_mem;
+
+	/* syncpoint physical address for stitching to PCIe BAR backing.*/
+	size_t size;
+	phys_addr_t phy_addr;
+
+	/* iova mapping of client choice.*/
+	void *iova_block_h;
+	u64 iova;
+	bool mapped_iova;
+
+	/* Lock to protect fences between callback and deinit. */
+	struct mutex lock;
+	/* Fence to specific Threshold. */
+	struct dma_fence *fence;
+	struct dma_fence_cb fence_cb;
+	/* Work to notify and allocate new fence. */
+	struct work_struct work;
+	void (*notifier)(void *data);
+	void *notifier_data;
+	bool fence_release;
+};
+
+struct fifo_t {
+	/* slot/frames for the comm-channel.*/
+	u32 nframes;
+	u32 frame_sz;
+
+	/* fifo operations.*/
+	struct header *send_hdr;
+	struct header *recv_hdr;
+	struct header *local_hdr;
+	u8 *send;
+	u8 *recv;
+	u32 wr_pos;
+	u32 rd_pos;
+	/* serialize send operations.*/
+	struct mutex send_lock;
+
+	/* fifo physical pages and stitched to iova of client choice(recv).*/
+	struct cpu_buff_t self_mem;
+	void *iova_block_h;
+	u64 iova;
+	bool mapped_iova;
+
+	/* PCIe aperture for writes to peer comm fifo. */
+	struct pci_aper_t peer_mem;
+};
+
+struct comm_channel_ctx_t {
+	/* data. */
+	struct fifo_t fifo;
+
+	/* Notification. */
+	struct syncpt_t syncpt;
+
+	/* receive message task.*/
+	struct task_t r_task;
+	atomic_t recv_count;
+
+	/* Callbacks registered for recv messages. */
+	struct mutex cb_ops_lock;
+	struct callback_ops cb_ops[COMM_MSG_TYPE_MAXIMUM];
+
+	/* pci client handle.*/
+	void *pci_client_h;
+
+	/* nvscic2c-pcie DT node reference, used in getting syncpoint shim. */
+	struct device_node *of_node;
+	struct platform_device *host1x_pdev;
+};
+
+static inline bool
+can_send(struct fifo_t *fifo, int *ret)
+{
+	bool send = false;
+	u32 peer_toread =
+		(fifo->local_hdr->wr_count - fifo->recv_hdr->rd_count);
+
+	if (peer_toread < fifo->nframes) {
+		/* space available - can send.*/
+		send = true;
+		*ret = 0;
+	} else if (peer_toread == fifo->nframes) {
+		/* full. client can try again (at the moment.)*/
+		send = false;
+		*ret = -EAGAIN;	// -ENOMEM;
+	} else if (peer_toread > fifo->nframes) {
+		/* erroneous.*/
+		send = false;
+		*ret = -EOVERFLOW;
+	}
+
+	return send;
+}
+
+static inline bool
+can_recv(struct fifo_t *fifo, int *ret)
+{
+	bool recv = false;
+	u32 toread = (fifo->recv_hdr->wr_count - fifo->local_hdr->rd_count);
+
+	if (toread == 0) {
+		/* no frame available to read.*/
+		recv = false;
+		*ret = -ENODATA;
+	} else if (toread <= fifo->nframes) {
+		/* frames available - can read.*/
+		recv = true;
+		*ret = 0;
+	} else if (toread > fifo->nframes) {
+		/* erroneous.*/
+		recv = false;
+		*ret = -EOVERFLOW;
+	}
+
+	return recv;
+}
+
+static int
+send_msg(struct comm_channel_ctx_t *comm_ctx, struct comm_msg *msg)
+{
+	int ret = 0;
+	size_t size = 0;
+	void *from = NULL;
+	void __iomem *to = NULL;
+	struct fifo_t *fifo = NULL;
+	struct syncpt_t *syncpt = NULL;
+	enum peer_cpu_t peer_cpu = NVCPU_ORIN;
+
+	fifo = &comm_ctx->fifo;
+	syncpt = &comm_ctx->syncpt;
+
+	peer_cpu = pci_client_get_peer_cpu(comm_ctx->pci_client_h);
+	mutex_lock(&fifo->send_lock);
+
+	/* if no space available, at the moment, client can try again. */
+	if (!can_send(fifo, &ret)) {
+		mutex_unlock(&fifo->send_lock);
+		return ret;
+	}
+
+	to = (void __iomem *)(fifo->send + (fifo->wr_pos * fifo->frame_sz));
+	from = (void *)(msg);
+	size = sizeof(*msg);
+	memcpy_toio(to, from, size);
+
+	fifo->local_hdr->wr_count++;
+	writel(fifo->local_hdr->wr_count,
+	       (void __iomem *)(&fifo->send_hdr->wr_count));
+
+	if (peer_cpu == NVCPU_X86_64) {
+	/* comm-channel irq verctor always take from index 0 */
+		ret = pci_client_raise_irq(comm_ctx->pci_client_h, PCI_EPC_IRQ_MSI, 0);
+	} else {
+	/* notify peer for each write.*/
+		writel(0x1, syncpt->peer_mem.pva);
+	}
+
+	fifo->wr_pos = fifo->wr_pos + 1;
+	if (fifo->wr_pos >= fifo->nframes)
+		fifo->wr_pos = 0;
+
+	mutex_unlock(&fifo->send_lock);
+
+	return ret;
+}
+
+int
+comm_channel_ctrl_msg_send(void *comm_channel_h, struct comm_msg *msg)
+{
+	struct comm_channel_ctx_t *comm_ctx =
+				(struct comm_channel_ctx_t *)comm_channel_h;
+
+	if (WARN_ON(!comm_ctx || !msg))
+		return -EINVAL;
+
+	if (WARN_ON(msg->type != COMM_MSG_TYPE_BOOTSTRAP &&
+		    msg->type != COMM_MSG_TYPE_EDMA_RX_DESC_IOVA_RETURN &&
+		    msg->type != COMM_MSG_TYPE_SHUTDOWN &&
+		    msg->type != COMM_MSG_TYPE_LINK))
+		return -EINVAL;
+
+	/*
+	 * this is a special ctrl message where the sender: @DRV_MODE_EPC
+	 * or @DRV_MODE_EPF shares control information with peer. We do not
+	 * check for nvscic2c-pcie link status as these messages can flow
+	 * even when nvscic2c-pcie link status is down but possibly with
+	 * PCIE physical link between PCIe RP SoC and PCIe EP SoC alive.
+	 */
+
+	return send_msg(comm_ctx, msg);
+}
+
+int
+comm_channel_msg_send(void *comm_channel_h, struct comm_msg *msg)
+{
+	enum nvscic2c_pcie_link link = NVSCIC2C_PCIE_LINK_DOWN;
+	struct comm_channel_ctx_t *comm_ctx =
+				(struct comm_channel_ctx_t *)comm_channel_h;
+
+	if (WARN_ON(!comm_ctx || !msg))
+		return -EINVAL;
+
+	if (WARN_ON(msg->type != COMM_MSG_TYPE_REGISTER &&
+		    msg->type != COMM_MSG_TYPE_UNREGISTER))
+		return -EINVAL;
+
+	link = pci_client_query_link_status(comm_ctx->pci_client_h);
+
+	if (link != NVSCIC2C_PCIE_LINK_UP)
+		return -ENOLINK;
+
+	return send_msg(comm_ctx, msg);
+}
+
+static int
+recv_taskfn(void *arg)
+{
+	int ret = 0;
+	struct comm_channel_ctx_t *comm_ctx = NULL;
+	struct comm_msg *msg = NULL;
+	struct task_t *task = NULL;
+	struct fifo_t *fifo = NULL;
+	struct callback_ops *cb_ops = NULL;
+
+	comm_ctx = (struct comm_channel_ctx_t *)(arg);
+	task = &comm_ctx->r_task;
+	fifo = &comm_ctx->fifo;
+
+	while (!task->shutdown) {
+		/* wait for notification from peer or shutdown. */
+		wait_event_interruptible(task->waitq,
+					 (atomic_read(&comm_ctx->recv_count) ||
+					  task->shutdown));
+		/* task is exiting.*/
+		if (task->shutdown)
+			continue;
+
+		/* read all on single notify.*/
+		atomic_dec(&comm_ctx->recv_count);
+		while (can_recv(fifo, &ret)) {
+			msg = (struct comm_msg *)
+				(fifo->recv + (fifo->rd_pos * fifo->frame_sz));
+
+			if (msg->type > COMM_MSG_TYPE_INVALID &&
+			    msg->type < COMM_MSG_TYPE_MAXIMUM) {
+				mutex_lock(&comm_ctx->cb_ops_lock);
+				cb_ops = &comm_ctx->cb_ops[msg->type];
+
+				if (cb_ops->callback)
+					cb_ops->callback
+						((void *)msg, cb_ops->ctx);
+				mutex_unlock(&comm_ctx->cb_ops_lock);
+			}
+
+			fifo->local_hdr->rd_count++;
+
+			writel(fifo->local_hdr->rd_count,
+			       (void __iomem *)(&fifo->send_hdr->rd_count));
+
+			/* do not noifty peer for space availability. */
+
+			fifo->rd_pos = fifo->rd_pos + 1;
+			if (fifo->rd_pos >= fifo->nframes)
+				fifo->rd_pos = 0;
+		}
+
+		/* if nothing (left) to read, go back waiting. */
+		continue;
+	}
+
+	/* we do not use kthread_stop(), but wait on this.*/
+	complete(&task->shutdown_compl);
+	return 0;
+}
+
+/* Shall be invoked on expiry of syncpoint shim fence/trigger from remote. */
+static void
+syncpt_callback(void *data)
+{
+	struct comm_channel_ctx_t *comm_ctx = NULL;
+
+	if (WARN_ON(!data))
+		return;
+
+	comm_ctx = (struct comm_channel_ctx_t *)(data);
+
+	/* kick r_task for processing this notification.*/
+	atomic_inc(&comm_ctx->recv_count);
+	wake_up_interruptible_all(&comm_ctx->r_task.waitq);
+}
+
+static void
+host1x_cb_func(struct dma_fence *f, struct dma_fence_cb *cb)
+{
+	struct syncpt_t *syncpt = container_of(cb, struct syncpt_t, fence_cb);
+
+	schedule_work(&syncpt->work);
+}
+
+static int
+allocate_fence(struct syncpt_t *syncpt)
+{
+	int ret = 0;
+	struct dma_fence *fence = NULL;
+
+	fence = host1x_fence_create(syncpt->sp, ++syncpt->threshold, false);
+	if (IS_ERR_OR_NULL(fence)) {
+		ret = PTR_ERR(fence);
+		pr_err("host1x_fence_create failed with: %d\n", ret);
+		return ret;
+	}
+
+	mutex_lock(&syncpt->lock);
+	ret = dma_fence_add_callback(fence, &syncpt->fence_cb, host1x_cb_func);
+	if (ret != 0) {
+		/* If already expired. */
+		if (ret == -ENOENT) {
+			ret = 0;
+			schedule_work(&syncpt->work);
+		}
+		goto put_fence;
+	}
+	syncpt->fence = fence;
+	mutex_unlock(&syncpt->lock);
+
+	return ret;
+
+put_fence:
+	dma_fence_put(fence);
+	mutex_unlock(&syncpt->lock);
+	return ret;
+}
+
+static void
+fence_do_work(struct work_struct *work)
+{
+	int ret = 0;
+	struct syncpt_t *syncpt = container_of(work, struct syncpt_t, work);
+
+	if (syncpt->notifier)
+		syncpt->notifier(syncpt->notifier_data);
+
+	mutex_lock(&syncpt->lock);
+	/* If deinit triggered, no need to proceed. */
+	if (syncpt->fence_release)
+		return;
+
+	if (syncpt->fence) {
+		dma_fence_put(syncpt->fence);
+		syncpt->fence = NULL;
+	}
+	mutex_unlock(&syncpt->lock);
+
+	ret = allocate_fence(syncpt);
+	if (ret != 0) {
+		mutex_unlock(&syncpt->lock);
+		pr_err("allocate_fence failed with: %d\n", ret);
+		return;
+	}
+}
+
+static int
+start_msg_handling(struct comm_channel_ctx_t *comm_ctx)
+{
+	int ret = 0;
+	struct task_t *r_task = &comm_ctx->r_task;
+	struct syncpt_t *syncpt = &comm_ctx->syncpt;
+
+	/* start the recv msg processing task.*/
+	init_waitqueue_head(&r_task->waitq);
+	init_completion(&r_task->shutdown_compl);
+	r_task->shutdown = false;
+	r_task->task = kthread_run(recv_taskfn, comm_ctx,
+				   "comm-channel-recv-task");
+	if (IS_ERR_OR_NULL(r_task->task)) {
+		pr_err("Failed to create comm channel recv task\n");
+		return PTR_ERR(r_task->task);
+	}
+	r_task->created = true;
+
+	/* enable syncpt notifications handling. */
+	mutex_init(&syncpt->lock);
+	syncpt->threshold = host1x_syncpt_read(syncpt->sp);
+	syncpt->notifier = syncpt_callback;
+	syncpt->notifier_data = comm_ctx;
+	INIT_WORK(&syncpt->work, fence_do_work);
+	syncpt->fence_release = false;
+
+	ret = allocate_fence(syncpt);
+	if (ret != 0) {
+		pr_err("allocate_fence failed with: %d\n", ret);
+		return ret;
+	}
+
+	return ret;
+}
+
+static int
+stop_msg_handling(struct comm_channel_ctx_t *comm_ctx)
+{
+	int ret = 0;
+	struct task_t *r_task = NULL;
+	struct syncpt_t *syncpt = NULL;
+
+	if (!comm_ctx)
+		return ret;
+
+	syncpt = &comm_ctx->syncpt;
+	r_task = &comm_ctx->r_task;
+
+	if (r_task->created) {
+		/* Remove dma fence callback. */
+		mutex_lock(&syncpt->lock);
+		syncpt->fence_release = true;
+		if (syncpt->fence) {
+			ret = dma_fence_remove_callback(syncpt->fence,
+							&syncpt->fence_cb);
+			if (ret) {
+				/*
+				 * If dma_fence_remove_callback() returns true
+				 * means callback is removed successfully.
+				 * Cancel the fence to drop the refcount.
+				 */
+				host1x_fence_cancel(syncpt->fence);
+			}
+			dma_fence_put(syncpt->fence);
+			syncpt->fence = NULL;
+		}
+		mutex_unlock(&syncpt->lock);
+		cancel_work_sync(&syncpt->work);
+		mutex_destroy(&syncpt->lock);
+
+		/*
+		 * initiate stop.
+		 * we do not use kthread_stop(), but wait on this.
+		 */
+		r_task->shutdown = true;
+		wake_up_interruptible(&r_task->waitq);
+		ret = wait_for_completion_interruptible(&r_task->shutdown_compl);
+		if (ret)
+			pr_err("Failed to wait for completion\n");
+
+		r_task->created = false;
+	}
+
+	return ret;
+}
+
+static void
+free_syncpoint(struct comm_channel_ctx_t *comm_ctx)
+{
+	struct syncpt_t *syncpt = NULL;
+
+	if (!comm_ctx)
+		return;
+
+	syncpt = &comm_ctx->syncpt;
+
+	if (syncpt->peer_mem.pva) {
+		iounmap(syncpt->peer_mem.pva);
+		syncpt->peer_mem.pva = NULL;
+	}
+
+	if (syncpt->mapped_iova) {
+		pci_client_unmap_addr(comm_ctx->pci_client_h,
+				      syncpt->iova, syncpt->size);
+		syncpt->mapped_iova = false;
+	}
+
+	if (syncpt->iova_block_h) {
+		pci_client_free_iova(comm_ctx->pci_client_h,
+				     &syncpt->iova_block_h);
+		syncpt->iova_block_h = NULL;
+	}
+
+	if (syncpt->sp) {
+		host1x_syncpt_put(syncpt->sp);
+		syncpt->sp = NULL;
+	}
+}
+
+static int
+allocate_syncpoint(struct comm_channel_ctx_t *comm_ctx)
+{
+	int ret = 0;
+	int prot = 0;
+	struct host1x *host1x = NULL;
+	struct syncpt_t *syncpt = NULL;
+	size_t offsetof = 0x0;
+
+	syncpt = &comm_ctx->syncpt;
+
+	host1x = platform_get_drvdata(comm_ctx->host1x_pdev);
+	if (!host1x) {
+		pr_err("Host1x handle is null.");
+		return -EINVAL;
+	}
+	syncpt->sp = host1x_syncpt_alloc(host1x, HOST1X_SYNCPT_CLIENT_MANAGED,
+					 "nvscic2c-pcie-comm-ch");
+	if (IS_ERR_OR_NULL(syncpt->sp)) {
+		ret = -ENOMEM;
+		pr_err("Failed to reserve comm notify syncpt\n");
+		goto err;
+	}
+
+	syncpt->id = host1x_syncpt_id(syncpt->sp);
+	/* physical address of syncpoint shim. */
+	syncpt->phy_addr = get_syncpt_shim_offset(syncpt->id);
+	syncpt->size = SP_MAP_SIZE;
+
+	/* reserve iova with the iova manager.*/
+	ret = pci_client_alloc_iova(comm_ctx->pci_client_h, syncpt->size,
+				    &syncpt->iova, &offsetof,
+				    &syncpt->iova_block_h);
+	if (ret) {
+		pr_err("Err reserving comm syncpt iova region of size: 0x%lx\n",
+		       syncpt->size);
+		goto err;
+	}
+
+	/* map the pages to the reserved iova. */
+	prot = (IOMMU_CACHE | IOMMU_READ | IOMMU_WRITE);
+	ret = pci_client_map_addr(comm_ctx->pci_client_h, syncpt->iova,
+				  syncpt->phy_addr, syncpt->size, prot);
+	if (ret) {
+		pr_err("Err mapping comm SP physical addr to reserved iova\n");
+		goto err;
+	}
+	syncpt->mapped_iova = true;
+
+	pr_debug("mapped phy:0x%pa[p]+0x%lx to iova:0x%llx\n",
+		 &syncpt->phy_addr, syncpt->size, syncpt->iova);
+
+	/*
+	 * get peer's aperture offset. Map tx (pcie aper for notif tx.)
+	 * for peer's access of comm-syncpt, it is assumed offsets are
+	 * same on both SoC.
+	 */
+	syncpt->peer_mem.size = syncpt->size;
+	ret = pci_client_get_peer_aper(comm_ctx->pci_client_h, offsetof,
+				       syncpt->peer_mem.size,
+				       &syncpt->peer_mem.aper);
+	if (ret) {
+		pr_err("Failed to get comm peer's syncpt aperture\n");
+		goto err;
+	}
+	syncpt->peer_mem.pva = ioremap(syncpt->peer_mem.aper,
+				       syncpt->peer_mem.size);
+	if (!syncpt->peer_mem.pva) {
+		ret = -ENOMEM;
+		pr_err("Failed to ioremap comm peer's syncpt pcie aperture\n");
+		goto err;
+	}
+
+	return ret;
+err:
+	free_syncpoint(comm_ctx);
+	return ret;
+}
+
+static void
+free_fifo_memory(struct comm_channel_ctx_t *comm_ctx)
+{
+	struct fifo_t *fifo = NULL;
+
+	if (!comm_ctx)
+		return;
+
+	fifo = &comm_ctx->fifo;
+
+	if (fifo->local_hdr) {
+		kfree((void *)fifo->local_hdr);
+		fifo->local_hdr = NULL;
+	}
+
+	if (fifo->peer_mem.pva) {
+		iounmap(fifo->peer_mem.pva);
+		fifo->peer_mem.pva = NULL;
+	}
+
+	if (fifo->mapped_iova) {
+		pci_client_unmap_addr(comm_ctx->pci_client_h,
+				      fifo->iova, fifo->self_mem.size);
+		fifo->mapped_iova = false;
+	}
+
+	if (fifo->iova_block_h) {
+		pci_client_free_iova(comm_ctx->pci_client_h,
+				     &fifo->iova_block_h);
+		fifo->iova_block_h = NULL;
+	}
+
+	if (fifo->self_mem.pva) {
+		free_pages_exact(fifo->self_mem.pva,
+				 fifo->self_mem.size);
+		fifo->self_mem.pva = NULL;
+	}
+
+	mutex_destroy(&fifo->send_lock);
+}
+
+static int
+allocate_fifo_memory(struct comm_channel_ctx_t *comm_ctx)
+{
+	int ret = 0;
+	int prot = 0;
+	size_t offsetof = 0x0;
+	struct fifo_t *fifo = &comm_ctx->fifo;
+
+	mutex_init(&fifo->send_lock);
+
+	/* memory size includes frames and header.*/
+	fifo->nframes = COMM_CHANNEL_NFRAMES;
+	fifo->frame_sz = COMM_CHANNEL_FRAME_SZ;
+	fifo->self_mem.size = (fifo->nframes * fifo->frame_sz);
+	fifo->self_mem.size += sizeof(struct header);
+	fifo->self_mem.size = ALIGN(fifo->self_mem.size, PAGE_SIZE);
+	fifo->self_mem.pva = alloc_pages_exact(fifo->self_mem.size,
+					       (GFP_KERNEL | __GFP_ZERO));
+	if (!fifo->self_mem.pva) {
+		pr_err("Error allocating fifo contiguous pages: (%lu)\n",
+		       (fifo->self_mem.size >> PAGE_SHIFT));
+		return -ENOMEM;
+	}
+	fifo->self_mem.phys_addr =
+				page_to_phys(virt_to_page(fifo->self_mem.pva));
+
+	/* reserve iova for stitching comm channel memory for peer access.*/
+	ret = pci_client_alloc_iova(comm_ctx->pci_client_h, fifo->self_mem.size,
+				    &fifo->iova, &offsetof,
+				    &fifo->iova_block_h);
+	if (ret) {
+		pr_err("Failed reserving fifo iova region of size: 0x%lx\n",
+		       fifo->self_mem.size);
+		goto err;
+	}
+
+	/* map the pages to the reserved iova.*/
+	prot = (IOMMU_CACHE | IOMMU_READ | IOMMU_WRITE);
+	ret = pci_client_map_addr(comm_ctx->pci_client_h, fifo->iova,
+				  fifo->self_mem.phys_addr, fifo->self_mem.size,
+				  prot);
+	if (ret) {
+		pr_err("Failed to map comm fifo pages to reserved iova\n");
+		goto err;
+	}
+	fifo->mapped_iova = true;
+
+	pr_debug("comm fifo mapped page:0x%pa[p]+0x%lx to iova:0x%llx\n",
+		 &fifo->self_mem.phys_addr, fifo->self_mem.size, fifo->iova);
+
+	/*
+	 * for peer's access of comm-fifo, it is assumed offsets are
+	 * same on both SoC.
+	 */
+	fifo->peer_mem.size = fifo->self_mem.size;
+	ret = pci_client_get_peer_aper(comm_ctx->pci_client_h, offsetof,
+				       fifo->peer_mem.size,
+				       &fifo->peer_mem.aper);
+	if (ret) {
+		pr_err("Failed to get comm peer's fifo aperture\n");
+		goto err;
+	}
+	fifo->peer_mem.pva = ioremap(fifo->peer_mem.aper, fifo->peer_mem.size);
+	if (!fifo->peer_mem.pva) {
+		ret = -ENOMEM;
+		pr_err("Failed to ioremap peer's comm fifo aperture\n");
+		goto err;
+	}
+
+	/* allocate local header.*/
+	fifo->local_hdr = kzalloc(sizeof(*fifo->local_hdr), GFP_KERNEL);
+	if (WARN_ON(!fifo->local_hdr)) {
+		ret = -ENOMEM;
+		goto err;
+	}
+
+	fifo->recv_hdr = (struct header *)(fifo->self_mem.pva);
+	fifo->send_hdr = (__force struct header *)(fifo->peer_mem.pva);
+	fifo->recv = ((u8 *)fifo->recv_hdr + sizeof(struct header));
+	fifo->send = ((u8 *)fifo->send_hdr + sizeof(struct header));
+
+	return ret;
+
+err:
+	free_fifo_memory(comm_ctx);
+	return ret;
+}
+
+int
+comm_channel_init(struct driver_ctx_t *drv_ctx, void **comm_channel_h)
+{
+	int ret = 0;
+	struct comm_channel_ctx_t *comm_ctx = NULL;
+
+	if (WARN_ON(sizeof(struct comm_msg) > COMM_CHANNEL_FRAME_SZ))
+		return -EINVAL;
+
+	/* should not be an already instantiated. */
+	if (WARN_ON(!drv_ctx || !comm_channel_h || *comm_channel_h))
+		return -EINVAL;
+
+	/* start by allocating the comm ctx.*/
+	comm_ctx = kzalloc(sizeof(*comm_ctx), GFP_KERNEL);
+	if (WARN_ON(!comm_ctx))
+		return -ENOMEM;
+	mutex_init(&comm_ctx->cb_ops_lock);
+	atomic_set(&comm_ctx->recv_count, 0);
+
+	comm_ctx->pci_client_h = drv_ctx->pci_client_h;
+	comm_ctx->of_node = drv_ctx->drv_param.of_node;
+	comm_ctx->host1x_pdev = drv_ctx->drv_param.host1x_pdev;
+
+	/*
+	 * allocate fifo area, make it visible to peer. Assume same aperture
+	 * for peer access too.
+	 */
+	ret = allocate_fifo_memory(comm_ctx);
+	if (ret)
+		goto err;
+
+	/*
+	 * allocate notification for comm-channel, Assume same aperture
+	 * for peer access too.
+	 */
+	ret = allocate_syncpoint(comm_ctx);
+	if (ret)
+		goto err;
+
+	/* we can now wait for notifications/messages to be received.*/
+	ret = start_msg_handling(comm_ctx);
+	if (ret)
+		goto err;
+
+	*comm_channel_h = comm_ctx;
+	return ret;
+err:
+	comm_channel_deinit((void **)&comm_ctx);
+	return ret;
+}
+
+void
+comm_channel_deinit(void **comm_channel_h)
+{
+	struct comm_channel_ctx_t *comm_ctx =
+				(struct comm_channel_ctx_t *)(*comm_channel_h);
+	if (!comm_ctx)
+		return;
+
+	stop_msg_handling(comm_ctx);
+	free_syncpoint(comm_ctx);
+	free_fifo_memory(comm_ctx);
+	mutex_destroy(&comm_ctx->cb_ops_lock);
+	kfree(comm_ctx);
+
+	*comm_channel_h = NULL;
+}
+
+int
+comm_channel_register_msg_cb(void *comm_channel_h, enum comm_msg_type type,
+			     struct callback_ops *ops)
+{
+	int ret = 0;
+	struct callback_ops *cb_ops = NULL;
+	struct comm_channel_ctx_t *comm_ctx =
+				(struct comm_channel_ctx_t *)comm_channel_h;
+
+	if (WARN_ON(!comm_ctx || !ops || !ops->callback))
+		return -EINVAL;
+
+	if (WARN_ON(type <= COMM_MSG_TYPE_INVALID ||
+		    type >= COMM_MSG_TYPE_MAXIMUM))
+		return -EINVAL;
+
+	mutex_lock(&comm_ctx->cb_ops_lock);
+
+	cb_ops = &comm_ctx->cb_ops[type];
+	if (cb_ops->callback) {
+		pr_err("Callback for msg type: (%u) is already taken\n", type);
+		ret = -EBUSY;
+	} else {
+		cb_ops->callback = ops->callback;
+		cb_ops->ctx = ops->ctx;
+	}
+
+	mutex_unlock(&comm_ctx->cb_ops_lock);
+	return ret;
+}
+
+int
+comm_channel_unregister_msg_cb(void *comm_channel_h, enum comm_msg_type type)
+{
+	int ret = 0;
+	struct callback_ops *cb_ops = NULL;
+	struct comm_channel_ctx_t *comm_ctx =
+				(struct comm_channel_ctx_t *)comm_channel_h;
+
+	if (WARN_ON(!comm_ctx))
+		return -EINVAL;
+
+	if (WARN_ON(type <= COMM_MSG_TYPE_INVALID ||
+		    type >= COMM_MSG_TYPE_MAXIMUM))
+		return -EINVAL;
+
+	mutex_lock(&comm_ctx->cb_ops_lock);
+	cb_ops = &comm_ctx->cb_ops[type];
+	cb_ops->callback = NULL;
+	cb_ops->ctx = NULL;
+	mutex_unlock(&comm_ctx->cb_ops_lock);
+
+	return ret;
+}

drivers/misc/nvscic2c-pcie/comm-channel.h

@@ -0,0 +1,138 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#ifndef __COMM_CHANNEL_H__
+#define __COMM_CHANNEL_H__
+
+#include <uapi/misc/nvscic2c-pcie-ioctl.h>
+
+#include "common.h"
+
+/* forward declaration. */
+struct driver_ctx_t;
+
+enum comm_msg_type {
+	/* invalid.*/
+	COMM_MSG_TYPE_INVALID = 0,
+
+	/*
+	 * One time message from peer @DRV_MODE_EPC (PCIe RP) towards
+	 * @DRV_MODE_EPF(PCIe EP) for boot-strap mechanism.
+	 */
+	COMM_MSG_TYPE_BOOTSTRAP,
+
+	/* Link status shared between @DRV_MODE_EPC and @DRV_MODE_EPF.*/
+	COMM_MSG_TYPE_LINK,
+
+	/* Share/Register export object with peer.*/
+	COMM_MSG_TYPE_REGISTER,
+
+	/* Unregister exported object back with peer.*/
+	COMM_MSG_TYPE_UNREGISTER,
+
+	/* return edma rx descriptor iova to peer x86 */
+	COMM_MSG_TYPE_EDMA_RX_DESC_IOVA_RETURN,
+
+	/*
+	 * One time message from peer @DRV_MODE_EPC (PCIe RP) towards
+	 * @DRV_MODE_EPF(PCIe EP) for initiating shutdown.
+	 */
+	COMM_MSG_TYPE_SHUTDOWN,
+
+	/* Maximum. */
+	COMM_MSG_TYPE_MAXIMUM,
+};
+
+/*
+ * For @DRV_MODE_EPF(PCIe EP), it doesn't know the send area towards
+ * @DRV_MODE_EPC (PCIe RP - there is not BAR with PCIe RP). This is the first
+ * message and only once sent by @DRV_MODE_EPC towards @DRV_MODE_EPF for latter
+ * to configure it's outbound translation.
+ */
+struct comm_msg_bootstrap {
+	u64 iova;
+	enum peer_cpu_t  peer_cpu;
+};
+
+/* to simply,only one channel c2c remote edma case   */
+struct comm_msg_edma_rx_desc_iova {
+	dma_addr_t iova;
+};
+
+/* Link status shared between @DRV_MODE_EPC and @DRV_MODE_EPF.
+ * Possible use: @DRV_MODE_EPC sends bootstrap message
+ * to @DRV_MODE_EPF without setting it's own PCIe link = UP, therefore,
+ * on compeleting initialization, @DRV_MODE_EPF(once bootstrap msg
+ * is received) shall send link = up message to @DRV_MODE_EPC.
+ */
+struct comm_msg_link {
+	enum nvscic2c_pcie_link status;
+};
+
+/*
+ * Private channel communication message sent by NvSciC2cPcie consumer
+ * towards producer to register the exported object at the NvSciC2cPcie
+ * producer SoC.
+ */
+struct comm_msg_register {
+	u64 export_desc;
+	u64 iova;
+	size_t size;
+	size_t offsetof;
+};
+
+/*
+ * Private channel communication message sent by NvSciC2cPcie producer
+ * towards onsumer to unregister it's exported object.
+ */
+struct comm_msg_unregister {
+	u64 export_desc;
+	u64 iova;
+	size_t size;
+	size_t offsetof;
+};
+
+struct comm_msg {
+	enum comm_msg_type type;
+	union data {
+		struct comm_msg_bootstrap bootstrap;
+		struct comm_msg_link link;
+		struct comm_msg_register reg;
+		struct comm_msg_unregister unreg;
+		struct comm_msg_edma_rx_desc_iova edma_rx_desc_iova;
+	} u;
+} __aligned(8);
+
+int
+comm_channel_init(struct driver_ctx_t *drv_ctx, void **comm_channel_h);
+
+void
+comm_channel_deinit(void **comm_channel_h);
+
+int
+comm_channel_msg_send(void *comm_channel_h, struct comm_msg *msg);
+
+int
+comm_channel_ctrl_msg_send(void *comm_channel_h, struct comm_msg *msg);
+
+int
+comm_channel_register_msg_cb(void *comm_channel_h, enum comm_msg_type type,
+			     struct callback_ops *ops);
+int
+comm_channel_unregister_msg_cb(void *comm_channel_h, enum comm_msg_type type);
+
+int
+comm_channel_edma_rx_desc_iova_send(void *comm_channel_h, struct comm_msg *msg);
+
+#endif //__COMM_CHANNEL_H__

drivers/misc/nvscic2c-pcie/common.h

@@ -0,0 +1,216 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#ifndef __COMMON_H__
+#define __COMMON_H__
+
+#include <linux/types.h>
+#include <linux/bitops.h>
+
+#define MODULE_NAME		"nvscic2c-pcie"
+#define DRIVER_NAME_EPF		"nvscic2c-pcie-epf"
+#define DRIVER_NAME_EPC		"nvscic2c-pcie-epc"
+
+/* STREAM_OBJ_TYPE. */
+#define STREAM_OBJ_TYPE_MEM	(0)
+#define STREAM_OBJ_TYPE_SYNC	(1)
+
+/*
+ * This capped number shall be used to derive export descriptor, therefore any
+ * change should be evaluated thoroughly.
+ */
+#define MAX_STREAM_MEMOBJS	(1024)
+
+/*
+ * This capped number shall be used to derive export descriptor, therefore any
+ * change should be evaluated thoroughly.
+ */
+#define MAX_STREAM_SYNCOBJS	(1024)
+
+/*
+ * In a topology of interconnected Boards + SoCs.
+ *
+ * This capped number shall be used to derive export descriptor, therefore any
+ * change should be evaluated thoroughly.
+ */
+#define MAX_BOARDS		(16)
+#define MAX_SOCS		(16)
+#define MAX_PCIE_CNTRLRS	(16)
+
+/*
+ * Maximum NvSciIpc INTER_CHHIP(NvSciC2cPcie) endpoints that can be supported
+ * for single pair of PCIe RP<>EP connection (referred just as 'connection'
+ * henceforth). We have specific customer need for a set of Eleven NvSciC2cPcie
+ * endpoints for single connection.
+ *
+ * This capped number shall be used to derive export descriptor, therefore any
+ * change should be evaluated thoroughly.
+ */
+#define MAX_ENDPOINTS		(16)
+
+/*
+ * Each NvSciIpc INTER_CHIP(NvSciC2cPcie) endpoint shall require at least one
+ * distinct notification Id (MSI/MSI-X, GIC SPI or NvRmHost1xSyncpointShim).
+ * Also, these notification mechanisms: MSI/MSI-X, GIC SPI, SyncpointShim are
+ * limited on SoC or per connection (configurable via device-tree).
+ *
+ * Also, there is a private communication channel between the two ends of a
+ * single connection that need notification Ids for message passing. Assuming
+ * this private communication channel to be a Queue-Pair (Cmd, Resp), need
+ * at least 2 distinct notification Ids for it on a single connection.
+ */
+#define MIN_NUM_NOTIFY		(MAX_ENDPOINTS + (2))
+
+/* NvRmHost1xSyncpointShim have size: 64KB on Orin.*/
+#define SP_SIZE			(0x10000)
+
+/*
+ * Represents SyncpointShimBase on all T234.
+ * Each syncpoint is offset at (syncpt_id * SP_SIZE) on SHIM_BASE.
+ */
+#define SHIM_BASE		(0x60000000)
+/*
+ * For our use-case, if only 4 bytes of NvRmHost1xSynpointShim aperture mapped
+ * to PCIe device, any writes of (SZ_4B) from remote is enough to increment
+ * the Syncpoint. Hence we only map 4KB/PAGE_SIZE instead of full 64KB.
+ */
+#define SP_MAP_SIZE		(0x1000)
+
+/* With Tegra as PCIe function we can have only one PCIe function. */
+#define PCIE_VFNO		(0)
+
+/*
+ * NvSciC2c supports three concurrent PCI RP<>EP connection.
+ * These are three PCI Function Device ID's to be configured in PCI header
+ * when Tegra acting as PCI Function to peer Tegra acting as PCI RP.
+ */
+#define PCI_DEVICE_ID_C2C_1	(0x22CB)
+#define PCI_DEVICE_ID_C2C_2	(0x22CC)
+#define PCI_DEVICE_ID_C2C_3	(0x22CD)
+
+/*
+ * For NvStreams extensions over NvSciC2cPcie, an endpoint is a producer on
+ * one SoC and a corresponding consumer on the remote SoC. The role
+ * classification cannot be deduced in KMD.
+ */
+
+/*
+ * PCIe BAR aperture for Tx to/Rx from peer.
+ */
+struct pci_aper_t {
+	/* physical Pcie aperture.*/
+	phys_addr_t aper;
+
+	/* process virtual address for CPU access.*/
+	void __iomem *pva;
+
+	/* size of the perture.*/
+	size_t size;
+};
+
+/*
+ * DMA'able memory registered/exported to peer -
+ * either allocated by dma_buf API or physical pages pinned to
+ * pcie address space(dma_handle).
+ */
+struct dma_buff_t {
+	/* process virtual address for CPU access. */
+	void *pva;
+
+	/* iova(iommu=ON) or bus address/physical address for device access. */
+	dma_addr_t dma_handle;
+
+	/* physical address.*/
+	u64 phys_addr;
+
+	/* size of the memory allocated. */
+	size_t size;
+};
+
+/*
+ * CPU-only accessible memory which is not PCIe aper or PCIe
+ * DMA'able memory. This shall contain information of memory
+ * allocated via kalloc()/likewise.
+ */
+struct cpu_buff_t {
+	/* process virtual address for CPU access. */
+	void *pva;
+
+	/* (va->pa) physical address. */
+	u64 phys_addr;
+
+	/* size of the memory allocated. */
+	size_t size;
+};
+
+/*
+ * Callback options for user to register with occurrence of an event.
+ */
+struct callback_ops {
+	/*
+	 * User callback to be invoked.
+	 * @data: Event-type or likewise data. read-only for user.
+	 * @ctx: user ctx returned as-is in the callback.
+	 */
+	void (*callback)(void *data, void *ctx);
+
+	/* user context that shall be passed with @callback.*/
+	void *ctx;
+};
+
+/*
+ * Node information. A combination of Board + SoC + PCIe controller
+ * should be unique within the PCIe controllers/SoCs/Boards interconnected
+ for NvSciC2cPcie.
+ */
+struct node_info_t {
+	u32 board_id;
+	u32 soc_id;
+	u32 cntrlr_id;
+};
+
+/*
+ * NvSciC2cPcie either works as EndpointClient module - client driver for
+ * remote PCIe EP (runs on the PCIe RP SoC) or as EndpointFunction module -
+ * PCIe EP function driver (runs on the PCIe EP SoC).
+ */
+enum drv_mode_t {
+	/* Invalid. */
+	DRV_MODE_INVALID = 0,
+
+	/* Driver module runs as EndpointClient driver.*/
+	DRV_MODE_EPC,
+
+	/* Drive module runs as EndpointFunction driver.*/
+	DRV_MODE_EPF,
+
+	/* Maximum.*/
+	DRV_MODE_MAXIMUM,
+};
+
+/*
+ * NvSciC2cPcie the cpu on peer
+ */
+enum peer_cpu_t {
+	NVCPU_ORIN = 0,
+	NVCPU_X86_64,
+	NVCPU_MAXIMUM,
+};
+
+/* Returns aperture offset of syncpoint on SHIM_BASE. */
+static inline u64 get_syncpt_shim_offset(u32 id)
+{
+	return (SHIM_BASE + (id * SP_SIZE));
+}
+#endif //__COMMON_H__

drivers/misc/nvscic2c-pcie/descriptor.h

@@ -0,0 +1,173 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#ifndef __DESCRIPTOR_H__
+#define __DESCRIPTOR_H__
+
+#include <linux/errno.h>
+
+#include "common.h"
+
+/* Magic code for descriptor.*/
+#define DESC_MAGIC_CODE_32BIT	(0x69152734)
+
+/*
+ * Format of Export Descriptor (at the moment)
+ * 0xXXXXXXXXRRRREIII
+ * 32bit(XXXXXXXX00000000): Reserved.
+ * 04bit(00000000B0000000): Peer Board Id.
+ * 04bit(000000000S000000): Peer SoC Id.
+ * 04bit(0000000000C00000): Peer PCIe Controller Id.
+ * 04bit(00000000000E0000): Endpoint Id.
+ * 04bit(000000000000X000): Reserved.
+ * 12bit(0000000000000III): Obj type(1bit) + Obj Id(11bits).
+ *                          (Bit 11 : ObjType - Mem/Sync)
+ *                          (Bit 0-10 : ObjId - Mem or Sync Obj Id)
+ *
+ * Board Id and SoC Id together can be a Node Id to allow for cases, where SoC
+ * on a single board: [0-63] and number of boards: [0-3]. Essentially uniquely
+ * identifying each SoC inter-connected within or across the boards.
+ */
+
+/*
+ * Topology can have a:
+ * A or a Set of boards
+ *   - (Assumed [0, 15]).
+ * Each Board can have a or a set of SoC(s)
+ *   - ID : [0, 15].
+ * Each SoC can have a or a set of PCIe controllers either in RP or EP mode.
+ *   - ID:  [0, 15].
+ * Each Controller can have a or a set of NvSciIpc INTER_CHIP endpoints.
+ *   - ID:  [0, 15].
+ * Each NvSciIpc INTER_CHIP can export either a Mem object or Sync object
+ *   - STREAM_OBJ_TYPE_MEM or STREAM_OBJ_TYPE_SYNC
+ *   - Type: [0, 1].
+ * Each NvSciIpc INTER_CHIP can export a set of either Mem or Sync objects.
+ *   - ID:   [0, 2047].
+ */
+struct descriptor_bit_t {
+	u64 reserved1     : 32;
+	u64 board_id      : 4;
+	u64 soc_id        : 4;
+	u64 cntrlr_id     : 4;
+	u64 endpoint_id   : 4;
+	u64 reserved2     : 4;
+	u64 handle_type   : 1;
+	u64 handle_id     : 11;
+};
+
+/* bit-field manipulation. */
+union descriptor_t {
+	u64 value;
+	struct descriptor_bit_t bit;
+};
+
+/* Generate a descriptor (auth token) */
+static inline u64
+gen_desc(u32 peer_board_id, u32 peer_soc_id, u32 peer_cntrlr_id, u32 ep_id,
+	 u32 handle_type, s32 handle_id)
+{
+	union descriptor_t desc;
+
+	desc.bit.reserved1 = DESC_MAGIC_CODE_32BIT;
+	desc.bit.board_id = peer_board_id;
+	desc.bit.soc_id = peer_soc_id;
+	desc.bit.cntrlr_id = peer_cntrlr_id;
+	desc.bit.endpoint_id = ep_id;
+	desc.bit.handle_type = handle_type;
+	desc.bit.handle_id = handle_id;
+
+	return desc.value;
+}
+
+/* Validate a descriptor (auth token) */
+static inline int
+validate_desc(u64 in_desc, u32 local_board_id, u32 local_soc_id,
+	      u32 local_cntrlr_id, u32 ep_id)
+{
+	int ret = 0;
+	union descriptor_t desc;
+
+	desc.value = in_desc;
+	if (desc.bit.reserved1 != DESC_MAGIC_CODE_32BIT ||
+	    desc.bit.board_id != local_board_id ||
+	    desc.bit.soc_id != local_soc_id ||
+	    desc.bit.cntrlr_id != local_cntrlr_id ||
+	    desc.bit.endpoint_id != ep_id) {
+		ret = -EINVAL;
+	}
+
+	return ret;
+}
+
+/* return handle type embedded in the descriptor (auth token) */
+static inline u32
+get_handle_type_from_desc(u64 in_desc)
+{
+	union descriptor_t desc;
+
+	desc.value = in_desc;
+	return (u32)desc.bit.handle_type;
+}
+
+/*
+ * Board Id, SoC Id, PCIe Controller Id should not be beyond 16 [0-15] - We have
+ * reserved 4b each for boardId to generate export descriptor.
+ */
+#if MAX_BOARDS > (0xF + 1)
+	#error MAX_BOARDS assumed to be less-than or equal to (16)
+#endif
+#if MAX_SOCS > (0xF + 1)
+	#error MAX_SOCS assumed to be less-than or equal to (16)
+#endif
+#if MAX_PCIE_CNTRLRS > (0xF + 1)
+	#error MAX_PCIE_CNTRLRS assumed to be less-than or equal to (16)
+#endif
+
+/*
+ * Endpoints should not be beyond 16 [0-15] - We have reserved 4b for
+ * endpoint Id to generate export descriptor (although we could use
+ * up the reserved2 if needed).
+ */
+#if MAX_ENDPOINTS > (0xF + 1)
+	#error MAX_ENDPOINTS to be less than or equal to (16)
+#endif
+
+/*
+ * Memory or Sync object indicator in descriptor should not be beyond 1 [0-1].
+ * The value must be less than 1 as the descriptor accounts just 1b (1bit).
+ */
+#if STREAM_OBJ_TYPE_MEM > (0x1)
+	#error STREAM_OBJ_TYPE_MEM to be less-than or equal-to (1)
+#endif
+#if STREAM_OBJ_TYPE_SYNC > (0x1)
+	#error STREAM_OBJ_TYPE_SYNC to be less-than or equal-to (1)
+#endif
+
+/*
+ * Mem objects should not be beyond 2048 [0-2047] - We have reserved 11b for
+ * Obj Id to generate export descriptor.
+ */
+#if MAX_STREAM_MEMOBJS > (0x7FF + 1)
+	#error MAX_STREAM_MEMOBJS to be less than or equal to (2048)
+#endif
+
+/*
+ * Sync objects should not be beyond 2048 [0-2047] - We have reserved 11b for
+ * Obj Id to generate export descriptor.
+ */
+#if MAX_STREAM_SYNCOBJS > (0x7FF + 1)
+	#error MAX_STREAM_SYNCOBJS to be less than or equal to (2048)
+#endif
+#endif //__DESCRIPTOR_H__

drivers/misc/nvscic2c-pcie/dt.c

@@ -0,0 +1,690 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#define pr_fmt(fmt)	"nvscic2c-pcie: dt: " fmt
+
+#include <linux/errno.h>
+#include <linux/io.h>
+#include <linux/of.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+#include <linux/printk.h>
+
+#include "common.h"
+#include "module.h"
+
+#define COMPATIBLE_EPC_PROP_VAL		("nvidia,tegra-nvscic2c-pcie-epc")
+#define COMPATIBLE_EPF_PROP_VAL		("nvidia,tegra-nvscic2c-pcie-epf")
+#define HOST1X_PHANDLE_PROP_NAME	("nvidia,host1x")
+#define EDMA_PHANDLE_PROP_NAME		("nvidia,pcie-edma")
+#define PCI_DEV_ID_PROP_NAME		("nvidia,pci-dev-id")
+#define BAR_WIN_SZ_PROP_NAME		("nvidia,bar-win-size")
+#define BOARD_ID_PROP_NAME		("nvidia,board-id")
+#define SOC_ID_PROP_NAME		("nvidia,soc-id")
+#define CNTRLR_ID_PROP_NAME		("nvidia,cntrlr-id")
+#define ENDPOINT_DB_PROP_NAME		("nvidia,endpoint-db")
+#define MAX_PROP_LEN			(1024)
+#define FRAME_SZ_ALIGN			(64)
+
+#define MAX_FRAME_SZ			(SZ_32K)
+#define MAX_NFRAMES			(64)
+#define MIN_BAR_WIN_SZ			(SZ_64M)
+
+/*
+ * Debug only.
+ */
+static void
+dt_print(struct driver_param_t *drv_param)
+{
+	u32 i = 0;
+	struct node_info_t *local_node = &drv_param->local_node;
+	struct node_info_t *peer_node = &drv_param->peer_node;
+
+	pr_debug("dt parsing leads to:\n");
+	pr_debug("\tdriver mode  = (%s)\n",
+		 ((drv_param->drv_mode == DRV_MODE_EPC) ? ("epc") : ("epf")));
+	pr_debug("\tpci dev id   = 0x%x\n", drv_param->pci_dev_id);
+	pr_debug("\tNode information\n");
+	pr_debug("\t\tlocal board id = %u\n", local_node->board_id);
+	pr_debug("\t\tpeer board id  = %u\n", peer_node->board_id);
+	pr_debug("\t\tlocal soc id   = %u\n", local_node->soc_id);
+	pr_debug("\t\tpeer soc id    = %u\n", peer_node->soc_id);
+	pr_debug("\t\tlocal pcie cntrlr id = %u\n", local_node->cntrlr_id);
+	pr_debug("\t\tpeer pcie cntrlr id  = %u\n", peer_node->cntrlr_id);
+	if (drv_param->drv_mode == DRV_MODE_EPF)
+		pr_debug("\tbar win size = 0x%x\n", drv_param->bar_win_size);
+	pr_debug("\ttotal endpoints	= (%u)\n", drv_param->nr_endpoint);
+	for (i = 0; i < drv_param->nr_endpoint; i++) {
+		struct endpoint_prop_t *prop = NULL;
+
+		prop = &drv_param->endpoint_props[i];
+		pr_debug("\t\t(%s)::\n", prop->name);
+		pr_debug("\t\t\tnframes   = (%02u) frame_size=(%08u)",
+			 prop->nframes, prop->frame_sz);
+	}
+	pr_debug("dt parsing ends\n");
+}
+
+/*
+ * helper function to tokenize the string with caller provided
+ * delimiter.
+ */
+static char *
+tokenize(char **input, const char *delim)
+{
+	/* skipping args check - internal api.*/
+
+	char *token = NULL;
+
+	token = strsep(input, delim);
+	if (!token) {
+		pr_err("Error parsing endpoint name\n");
+	} else {
+		/* remove any whitespaces. */
+		token = strim(token);
+		if (!token)
+			pr_err("Error trimming endpoint name\n");
+	}
+
+	return token;
+}
+
+/*
+ * helper function to tokenize the string with caller provided
+ * delimiter and provide the sting->uint8_t value.
+ *
+ * @param input is an in,out parameter.
+ *
+ */
+static int
+tokenize_u8(char **input, const char *delim,
+	    u32 base, u8 *value)
+{
+	int ret = 0;
+	char *token = NULL;
+
+	/* skipping args check - internal api.*/
+
+	token = tokenize(input, delim);
+	if (!token)
+		ret = -ENODATA;
+	else
+		ret = kstrtou8(token, base, value);
+
+	return ret;
+}
+
+/*
+ * helper function to tokenize the string with caller provided
+ * delimiter and provide the sting->u32 value.
+ *
+ * @param input is an in,out parameter.
+ *
+ */
+static int
+tokenize_u32(char **input, const char *delim,
+	     u32 base, u32 *value)
+{
+	int ret = 0;
+	char *token = NULL;
+
+	/* skipping args check - internal api.*/
+
+	token = tokenize(input, delim);
+	if (!token)
+		ret = -ENODATA;
+	else
+		ret = kstrtou32(token, base, value);
+
+	return ret;
+}
+
+/* find a compatible node carrying the pci_dev_id.*/
+static struct device_node*
+find_compatible_node(const char *compatible, u32 pci_dev_id)
+{
+	int ret = 0;
+	u32 ret_id = 0;
+	struct device_node *dn = NULL;
+	struct device_node *dn_found = NULL;
+
+	/* look all device nodes with matching compatible and pci-dev-id.*/
+	while ((dn = of_find_compatible_node(dn, NULL, compatible)) != NULL) {
+		if (of_device_is_available(dn) == false)
+			continue;
+
+		ret = of_property_read_u32(dn, PCI_DEV_ID_PROP_NAME, &ret_id);
+		if (ret < 0) {
+			pr_err("Failed to read: (%s) from device node: (%s)\n",
+			       PCI_DEV_ID_PROP_NAME, dn->name);
+			of_node_put(dn);
+			goto err;
+		}
+
+		if (ret_id == pci_dev_id) {
+			if (dn_found) {
+				ret = -EINVAL;
+				pr_err("pci-dev-id: (0x%x) first repeated in:(%s)\n",
+				       ret_id, dn->name);
+				of_node_put(dn);
+				goto err;
+			} else {
+				dn_found = dn;
+			}
+		}
+	}
+
+	if (!dn_found) {
+		ret = -EINVAL;
+		pr_err("Matching pci-dev-id: (0x%x) not found\n", pci_dev_id);
+		goto err;
+	}
+
+	return dn_found;
+
+err:
+	return ERR_PTR(ret);
+}
+
+/* Parse the host1x phandle and create host1x pdev.*/
+static int
+parse_host1x_phandle(struct driver_param_t *drv_param)
+{
+	int ret = 0;
+	struct device_node *np = NULL;
+
+	np = drv_param->pdev->dev.of_node;
+
+	drv_param->host1x_np =
+			of_parse_phandle(np, HOST1X_PHANDLE_PROP_NAME, 0);
+	if (!drv_param->host1x_np) {
+		ret = -EINVAL;
+		pr_err("Error parsing host1x phandle property: (%s)\n",
+		       HOST1X_PHANDLE_PROP_NAME);
+	} else {
+		drv_param->host1x_pdev =
+				of_find_device_by_node(drv_param->host1x_np);
+		if (!drv_param->host1x_pdev) {
+			ret = -ENODEV;
+			pr_err("Host1x device not available\n");
+		}
+	}
+
+	return ret;
+}
+
+/* Parse the pcie-edma phandle.*/
+static int
+parse_edma_phandle(struct driver_param_t *drv_param)
+{
+	int ret = 0;
+	struct device_node *np = NULL;
+
+	np = drv_param->pdev->dev.of_node;
+
+	drv_param->edma_np = of_parse_phandle(np, EDMA_PHANDLE_PROP_NAME, 0);
+	if (!drv_param->edma_np) {
+		ret = -EINVAL;
+		pr_err("Error parsing pcie-edma phandle property: (%s)\n",
+		       EDMA_PHANDLE_PROP_NAME);
+	}
+
+	return ret;
+}
+
+/* Parse the pci device id.*/
+static int
+parse_pci_dev_id(struct driver_param_t *drv_param)
+{
+	int ret = 0;
+	struct device_node *np = NULL;
+
+	np = drv_param->pdev->dev.of_node;
+
+	ret = of_property_read_u32(np, PCI_DEV_ID_PROP_NAME,
+				   &drv_param->pci_dev_id);
+	if (ret) {
+		pr_err("Error parsing pci dev id prop:(%s)\n",
+		       PCI_DEV_ID_PROP_NAME);
+		goto err;
+	}
+
+	/* validate.*/
+	if (drv_param->pci_dev_id != PCI_DEVICE_ID_C2C_1 &&
+	    drv_param->pci_dev_id != PCI_DEVICE_ID_C2C_2 &&
+	    drv_param->pci_dev_id != PCI_DEVICE_ID_C2C_3) {
+		pr_err("Invalid value for property: (%s)\n",
+		       PCI_DEV_ID_PROP_NAME);
+		goto err;
+	}
+
+err:
+	return ret;
+}
+
+static int
+validate_node_information(struct driver_param_t *drv_param)
+{
+	struct node_info_t *local_node = NULL;
+	struct node_info_t *peer_node = NULL;
+
+	local_node = &drv_param->local_node;
+	peer_node = &drv_param->peer_node;
+
+	if (local_node->board_id >= MAX_BOARDS ||
+	    peer_node->board_id >= MAX_BOARDS) {
+		pr_err("Board Ids must be in the range [0, %u]\n", MAX_BOARDS);
+		return -EINVAL;
+	}
+	if (local_node->soc_id >= MAX_SOCS ||
+	    peer_node->soc_id >= MAX_SOCS) {
+		pr_err("SoC Ids must be in the range [0, %u]\n", MAX_SOCS);
+		return -EINVAL;
+	}
+	if (local_node->cntrlr_id >= MAX_PCIE_CNTRLRS ||
+	    peer_node->cntrlr_id >= MAX_PCIE_CNTRLRS) {
+		pr_err("PCIe controller Ids must be in the range [0, %u]\n",
+		       MAX_PCIE_CNTRLRS);
+		return -EINVAL;
+	}
+
+	/*
+	 * From the node information, we must have either
+	 * one of the three properties different between
+	 * local and peer.
+	 * Same board, same SoC, different controller
+	 * Same board, different SoC, same controller
+	 * likewise.
+	 *
+	 * Essentially the tuple of board+soc+cntrlr shouldn't
+	 * be same for local and peer.
+	 */
+	if (local_node->board_id == peer_node->board_id &&
+	    local_node->soc_id == peer_node->soc_id &&
+	    local_node->cntrlr_id == peer_node->cntrlr_id)
+		return -EINVAL;
+
+	return 0;
+}
+
+/* Parse the node information: board, soc, controller information.*/
+static int
+parse_node_info(struct driver_param_t *drv_param)
+{
+	int ret = 0;
+	struct device_node *np = NULL;
+	struct node_info_t *local_node = NULL;
+	struct node_info_t *peer_node = NULL;
+
+	np = drv_param->pdev->dev.of_node;
+	local_node = &drv_param->local_node;
+	peer_node = &drv_param->peer_node;
+
+	/* board-id: local and peer.*/
+	ret = of_property_read_u32_index(np, BOARD_ID_PROP_NAME, 0,
+					 &local_node->board_id);
+	if (ret == 0) {
+		ret = of_property_read_u32_index(np, BOARD_ID_PROP_NAME, 1,
+						 &peer_node->board_id);
+	}
+	if (ret) {
+		pr_err("Error parsing board id prop:(%s) information\n",
+		       BOARD_ID_PROP_NAME);
+		goto err;
+	}
+
+	/* soc-id: local and peer.*/
+	ret = of_property_read_u32_index(np, SOC_ID_PROP_NAME, 0,
+					 &local_node->soc_id);
+	if (ret == 0) {
+		ret = of_property_read_u32_index(np, SOC_ID_PROP_NAME, 1,
+						 &peer_node->soc_id);
+	}
+	if (ret) {
+		pr_err("Error parsing soc id prop:(%s) information\n",
+		       SOC_ID_PROP_NAME);
+		goto err;
+	}
+
+	/* pcie controller-id: local and peer.*/
+	ret = of_property_read_u32_index(np, CNTRLR_ID_PROP_NAME, 0,
+					 &local_node->cntrlr_id);
+	if (ret == 0) {
+		ret = of_property_read_u32_index(np, CNTRLR_ID_PROP_NAME, 1,
+						 &peer_node->cntrlr_id);
+	}
+	if (ret) {
+		pr_err("Error parsing pcie controller id prop:(%s) information\n",
+		       CNTRLR_ID_PROP_NAME);
+		goto err;
+	}
+
+	ret = validate_node_information(drv_param);
+	if (ret) {
+		pr_err("Node information for board:soc:cntrlr is not sane\n");
+		goto err;
+	}
+
+err:
+	return ret;
+}
+
+/* Parse the bar-window-size.*/
+static int
+parse_bar_win_size(struct driver_param_t *drv_param)
+{
+	int ret = 0;
+	struct device_node *np = NULL;
+
+	np = drv_param->pdev->dev.of_node;
+
+	/* bar-win-size should be checked only when running as epf.*/
+	ret = of_property_read_u32(np, BAR_WIN_SZ_PROP_NAME,
+				   &drv_param->bar_win_size);
+	if (drv_param->drv_mode == DRV_MODE_EPF) {
+		if (ret) {
+			ret = -EINVAL;
+			pr_err("Error parsing bar window size prop:(%s)\n",
+			       BAR_WIN_SZ_PROP_NAME);
+		}
+	} else {
+		/* success is not expected for EPC.*/
+		if (ret == 0) {
+			ret = -EINVAL;
+			pr_err("Property (%s): must be present only with (%s)\n",
+			       BAR_WIN_SZ_PROP_NAME, COMPATIBLE_EPF_PROP_VAL);
+			goto err;
+		}
+		/* proceed, as error is expected with EPC (node absent).*/
+		ret = 0;
+		goto err;
+	}
+
+	/* validate. */
+	if (!drv_param->bar_win_size) {
+		ret = -EINVAL;
+		pr_err("Invalid BAR window size: (%u)\n",
+		       drv_param->bar_win_size);
+		goto err;
+	}
+	if (drv_param->bar_win_size & (drv_param->bar_win_size - 1)) {
+		ret = -EINVAL;
+		pr_err("BAR window size: (%u) not a power of 2\n",
+		       drv_param->bar_win_size);
+		goto err;
+	}
+	if (drv_param->bar_win_size < MIN_BAR_WIN_SZ) {
+		ret = -EINVAL;
+		pr_err("BAR window size: (%u) less than minimum: (%u)\n",
+		       drv_param->bar_win_size, MIN_BAR_WIN_SZ);
+		goto err;
+	}
+err:
+	return ret;
+}
+
+/*
+ * helper function to validate per-endpoint parameters:
+ * nframes and frame_size primarily.
+ *
+ * Add more when required (probably crypto, eDMA, etc.)
+ */
+static int
+validate_endpoint_prop(struct endpoint_prop_t *prop)
+{
+	int ret = 0;
+
+	/* skipping args check - internal api.*/
+
+	if ((prop->name[0] == '\0')) {
+		ret = -EINVAL;
+		pr_err("Endpoint must have a name\n");
+	} else if (prop->nframes == 0) {
+		ret = -EINVAL;
+		pr_err("(%s): Invalid number of frames\n", prop->name);
+	} else if (prop->frame_sz == 0) {
+		ret = -EINVAL;
+		pr_err("(%s): Invalid frame size\n", prop->name);
+	} else if ((prop->frame_sz & (FRAME_SZ_ALIGN - 1)) != 0) {
+		ret = -EINVAL;
+		pr_err("(%s): Frame size unaligned to (%u)\n",
+		       prop->name, FRAME_SZ_ALIGN);
+	} else if (prop->frame_sz > MAX_FRAME_SZ) {
+		ret = -EINVAL;
+		pr_err("(%s): Frame size greater than: (%u)\n",
+		       prop->name, (MAX_FRAME_SZ));
+	} else if (prop->nframes > MAX_NFRAMES) {
+		ret = -EINVAL;
+		pr_err("(%s): Number of frames greater than: (%u)\n",
+		       prop->name, (MAX_NFRAMES));
+	}
+
+	return ret;
+}
+
+/*
+ * Parse all the endpoint information available in DT property
+ * of nvscic2c-pcie dt node.
+ */
+static int
+parse_endpoint_db(struct driver_param_t *drv_param)
+{
+	int ret = 0;
+	u8 nr_endpoint = 0;
+	struct device_node *np = NULL;
+
+	np = drv_param->pdev->dev.of_node;
+
+	ret = of_property_count_strings(np, ENDPOINT_DB_PROP_NAME);
+	if (ret < 0) {
+		pr_err("Failed to query endpoint count from property: (%s)\n",
+		       ENDPOINT_DB_PROP_NAME);
+		return -EFAULT;
+	}
+	nr_endpoint = ret;
+
+	if (nr_endpoint == 0) {
+		ret = -EINVAL;
+		pr_err("No endpoint information in property: (%s)\n",
+		       ENDPOINT_DB_PROP_NAME);
+		goto err;
+	} else if (nr_endpoint > MAX_ENDPOINTS) {
+		ret = -EINVAL;
+		pr_err("Invalid endpoint count:(%u) from property: (%s)\n",
+		       nr_endpoint, ENDPOINT_DB_PROP_NAME);
+		goto err;
+
+	} else {
+		u8 i = 0;
+		char *inp = NULL;
+		u32 base = 10;
+		const char *entry = NULL;
+		struct property *prop = NULL;
+		char entry_dup[MAX_PROP_LEN] = {0};
+
+		/* for each endpoint entry in endpointdb.*/
+		of_property_for_each_string(np, ENDPOINT_DB_PROP_NAME,
+					    prop, entry) {
+			char *name = NULL;
+			struct endpoint_prop_t *ep_prop =
+				&drv_param->endpoint_props[i];
+
+			/*
+			 * per endpoint entry in endpointdb is longer than
+			 * expected.
+			 */
+			if (strlen(entry) > (MAX_PROP_LEN - 1)) {
+				ret = -EINVAL;
+				pr_err("Endpoint entry invalid\n");
+				break;
+			}
+			memcpy(entry_dup, entry, (strlen(entry)));
+			inp = &entry_dup[0];
+
+			/* parse endpoint name.*/
+			name = tokenize(&inp, ",");
+			if (!name) {
+				ret = -EFAULT;
+				pr_err("Error parsing endpoint name\n");
+				break;
+			}
+			if (strlen(name) > (NAME_MAX - 1)) {
+				ret = -EINVAL;
+				pr_err("Endpoint name: (%s) long, max char:(%u)\n",
+				       name, (NAME_MAX - 1));
+				break;
+			}
+			strcpy(ep_prop->name, name);
+
+			/* parse number of frames.*/
+			ret = tokenize_u8(&inp, ",", base, &ep_prop->nframes);
+			if (ret) {
+				pr_err("Error parsing token nframes\n");
+				break;
+			}
+
+			/* parse size of each frame.*/
+			ret = tokenize_u32(&inp, ",", base, &ep_prop->frame_sz);
+			if (ret) {
+				pr_err("Error parsing token frame_sz\n");
+				break;
+			}
+
+			/* validate some basic properties of endpoint.*/
+			ret = validate_endpoint_prop(ep_prop);
+			if (ret) {
+				pr_err("(%s): endpoint has invalid properties\n",
+				       ep_prop->name);
+				break;
+			}
+
+			/* all okay, assign the id.*/
+			ep_prop->id = i;
+			i++;
+		}
+	}
+
+	/* all okay.*/
+	drv_param->nr_endpoint = nr_endpoint;
+
+err:
+	return ret;
+}
+
+/*
+ * Look-up device tree node for the compatible string. Check for the
+ * pci-dev-id within the compatible node, if more than one such node found also
+ * return error.
+ */
+int
+dt_parse(u32 pci_dev_id, enum drv_mode_t drv_mode,
+	 struct driver_param_t *drv_param)
+{
+	int ret = 0;
+	char *compatible = NULL;
+	struct device_node *dn = NULL;
+
+	if (WARN_ON(!pci_dev_id))
+		return -EINVAL;
+
+	if (WARN_ON(!drv_param))
+		return -EINVAL;
+
+	if (drv_mode == DRV_MODE_EPC)
+		compatible = COMPATIBLE_EPC_PROP_VAL;
+	else if (drv_mode == DRV_MODE_EPF)
+		compatible = COMPATIBLE_EPF_PROP_VAL;
+	else
+		return -EINVAL;
+
+	dn = find_compatible_node(compatible, pci_dev_id);
+	if (IS_ERR_OR_NULL(dn)) {
+		ret = -EINVAL;
+		goto err;
+	}
+
+	memset(drv_param, 0x0, sizeof(*drv_param));
+	drv_param->drv_mode = drv_mode;
+
+	/* dn may not have refcount, by doing this we exlpicitly have one.*/
+	drv_param->pdev = of_find_device_by_node(dn);
+	if (!drv_param->pdev) {
+		pr_err("Failed to find platform device for: (0x%x)\n",
+		       pci_dev_id);
+		goto err;
+	}
+	drv_param->of_node = drv_param->pdev->dev.of_node;
+
+	ret = parse_host1x_phandle(drv_param);
+	if (ret)
+		goto err;
+
+	ret = parse_edma_phandle(drv_param);
+	if (ret)
+		goto err;
+
+	ret = parse_pci_dev_id(drv_param);
+	if (ret)
+		goto err;
+
+	ret = parse_node_info(drv_param);
+	if (ret)
+		goto err;
+
+	ret = parse_bar_win_size(drv_param);
+	if (ret)
+		goto err;
+
+	ret = parse_endpoint_db(drv_param);
+	if (ret)
+		goto err;
+
+	/* all okay.*/
+	dt_print(drv_param);
+	return ret;
+err:
+	dt_release(drv_param);
+	return ret;
+}
+
+/*
+ * to free-up any memory and decrement ref_count of device nodes
+ * accessed.
+ */
+int
+dt_release(struct driver_param_t *drv_param)
+{
+	int ret = 0;
+
+	if (!drv_param)
+		return ret;
+
+	if (drv_param->host1x_pdev) {
+		platform_device_put(drv_param->host1x_pdev);
+		drv_param->host1x_pdev = NULL;
+	}
+	if (drv_param->host1x_np) {
+		of_node_put(drv_param->host1x_np);
+		drv_param->host1x_np = NULL;
+	}
+	if (drv_param->edma_np) {
+		of_node_put(drv_param->edma_np);
+		drv_param->edma_np = NULL;
+	}
+	if (drv_param->pdev) {
+		platform_device_put(drv_param->pdev);
+		drv_param->pdev = NULL;
+	}
+	return ret;
+}

drivers/misc/nvscic2c-pcie/endpoint.h

@@ -0,0 +1,39 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#ifndef __ENDPOINT_H__
+#define __ENDPOINT_H__
+
+#include "common.h"
+
+/* forward declaration. */
+struct driver_ctx_t;
+
+/*
+ * Entry point for the endpoint(s) char device sub-module/abstraction.
+ *
+ * On successful return (0), devices would have been created and ready to
+ * accept ioctls from user-space application.
+ */
+int
+endpoints_setup(struct driver_ctx_t *drv_ctx, void **endpoints_h);
+
+/* Exit point for nvscic2c-pcie endpoints: Wait for all endpoints to close.*/
+int
+endpoints_waitfor_close(void *endpoints_h);
+
+/* exit point for nvscic2c-pcie endpoints char device sub-module/abstraction.*/
+int
+endpoints_release(void **endpoints_h);
+#endif /*__ENDPOINT_H__ */

drivers/misc/nvscic2c-pcie/epc/module.c

@@ -0,0 +1,507 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#define pr_fmt(fmt)	"nvscic2c-pcie: epc: " fmt
+
+#include <linux/dma-iommu.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/mm.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/pci.h>
+#include <linux/printk.h>
+#include <linux/slab.h>
+#include <linux/tegra-pcie-edma.h>
+#include <linux/types.h>
+
+#include "comm-channel.h"
+#include "common.h"
+#include "endpoint.h"
+#include "iova-alloc.h"
+#include "module.h"
+#include "pci-client.h"
+#include "vmap.h"
+
+static const struct pci_device_id nvscic2c_pcie_epc_tbl[] = {
+	{ PCI_DEVICE(PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_C2C_1) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_C2C_2) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_C2C_3) },
+	{},
+};
+
+/* wrapper over tegra-pcie-edma init api. */
+static int
+edma_module_init(struct driver_ctx_t *drv_ctx)
+{
+	u8 i = 0;
+	int ret = 0;
+	struct tegra_pcie_edma_init_info info = {0};
+
+	if (WARN_ON(!drv_ctx || !drv_ctx->drv_param.edma_np))
+		return -EINVAL;
+
+	memset(&info, 0x0, sizeof(info));
+	info.np = drv_ctx->drv_param.edma_np;
+	info.edma_remote = NULL;
+	for (i = 0; i < DMA_WR_CHNL_NUM; i++) {
+		info.tx[i].ch_type = EDMA_CHAN_XFER_ASYNC;
+		info.tx[i].num_descriptors = NUM_EDMA_DESC;
+	}
+	/*No use-case for RD channels.*/
+
+	drv_ctx->edma_h = tegra_pcie_edma_initialize(&info);
+	if (!drv_ctx->edma_h)
+		ret = -ENODEV;
+
+	return ret;
+}
+
+/* should stop any ongoing eDMA transfers.*/
+static void
+edma_module_stop(struct driver_ctx_t *drv_ctx)
+{
+	if (!drv_ctx || !drv_ctx->edma_h)
+		return;
+
+	tegra_pcie_edma_stop(drv_ctx->edma_h);
+}
+
+/* should not have any ongoing eDMA transfers.*/
+static void
+edma_module_deinit(struct driver_ctx_t *drv_ctx)
+{
+	if (!drv_ctx || !drv_ctx->edma_h)
+		return;
+
+	tegra_pcie_edma_deinit(drv_ctx->edma_h);
+	drv_ctx->edma_h = NULL;
+}
+
+static void
+free_inbound_area(struct pci_dev *pdev, struct dma_buff_t *self_mem)
+{
+	struct driver_ctx_t *drv_ctx = NULL;
+
+	if (!pdev || !self_mem)
+		return;
+
+	drv_ctx = pci_get_drvdata(pdev);
+	if (!drv_ctx)
+		return;
+
+	if (self_mem->dma_handle && drv_ctx->ivd_h)
+		iova_alloc_deinit(self_mem->dma_handle, self_mem->size,
+				  &drv_ctx->ivd_h);
+	self_mem->dma_handle = 0x0;
+}
+
+/*
+ * Allocate area visible to PCIe EP/DRV_MODE_EPF. To have symmetry between the
+ * two modules, even PCIe RP/DRV_MODE_EPC allocates an empty area for all writes
+ * from PCIe EP/DRV_MODE_EPF to land into. Also, all CPU access from PCIe EP/
+ * DRV_MODE_EPF need be for one continguous region.
+ */
+static int
+allocate_inbound_area(struct pci_dev *pdev, size_t win_size,
+		      struct dma_buff_t *self_mem)
+{
+	int ret = 0;
+	struct driver_ctx_t *drv_ctx = NULL;
+
+	drv_ctx = pci_get_drvdata(pdev);
+	if (!drv_ctx) {
+		pr_err("Could not fetch driver data.");
+		return -ENOMEM;
+	}
+
+	self_mem->size = win_size;
+	ret = iova_alloc_init(&pdev->dev, win_size, &self_mem->dma_handle,
+			      &drv_ctx->ivd_h);
+	if (ret) {
+		pr_err("iova_alloc_init() failed for size:(0x%lx)\n",
+		       self_mem->size);
+	}
+
+	return ret;
+}
+
+static void
+free_outbound_area(struct pci_dev *pdev, struct pci_aper_t *peer_mem)
+{
+	if (!pdev || !peer_mem)
+		return;
+
+	peer_mem->aper = 0x0;
+	peer_mem->size = 0;
+}
+
+/* Assign outbound pcie aperture for CPU/eDMA access towards PCIe EP. */
+static int
+assign_outbound_area(struct pci_dev *pdev, size_t win_size,
+		     struct pci_aper_t *peer_mem)
+{
+	int ret = 0;
+
+	peer_mem->size = win_size;
+	peer_mem->aper = pci_resource_start(pdev, 0);
+
+	return ret;
+}
+
+/* Handle link message from @DRV_MODE_EPF. */
+static void
+link_msg_cb(void *data, void *ctx)
+{
+	struct comm_msg *msg = (struct comm_msg *)data;
+	struct driver_ctx_t *drv_ctx = (struct driver_ctx_t *)ctx;
+
+	if (WARN_ON(!msg || !drv_ctx))
+		return;
+
+	if (msg->u.link.status == NVSCIC2C_PCIE_LINK_UP) {
+		complete_all(&drv_ctx->epc_ctx->epf_ready_cmpl);
+	} else if (msg->u.link.status == NVSCIC2C_PCIE_LINK_DOWN) {
+		complete_all(&drv_ctx->epc_ctx->epf_shutdown_cmpl);
+	} else {
+		pr_err("(%s): spurious link message received from EPF\n",
+		       drv_ctx->drv_name);
+		return;
+	}
+
+	/* inidicate link status to application.*/
+	pci_client_change_link_status(drv_ctx->pci_client_h,
+				      msg->u.link.status);
+}
+
+/*
+ * PCIe subsystem invokes .shutdown()/.remove() handler when the PCIe EP
+ * is hot-unplugged (gracefully) or @DRV_MODE_EPC(this) is unloaded while
+ * the PCIe link was still active or when PCIe EP goes abnormal shutdown/
+ * reboot.
+ */
+#define MAX_EPF_SHUTDOWN_TIMEOUT_MSEC	(5000)
+static void
+nvscic2c_pcie_epc_remove(struct pci_dev *pdev)
+{
+	int ret = 0;
+	long timeout = 0;
+	struct comm_msg msg = {0};
+	struct driver_ctx_t *drv_ctx = NULL;
+
+	if (!pdev)
+		return;
+
+	drv_ctx = pci_get_drvdata(pdev);
+	if (!drv_ctx)
+		return;
+
+	/*
+	 * send link down message to EPF. EPF apps can stop processing when
+	 * they see this, else the EPF apps can continue processing as EPC
+	 * waits for its apps to close before sending SHUTDOWN msg.
+	 */
+	msg.type = COMM_MSG_TYPE_LINK;
+	msg.u.link.status = NVSCIC2C_PCIE_LINK_DOWN;
+	ret = comm_channel_ctrl_msg_send(drv_ctx->comm_channel_h, &msg);
+	if (ret)
+		pr_err("(%s): Failed to send LINK(DOWN) message\n",
+		       drv_ctx->drv_name);
+
+	/* local apps can stop processing if they see this.*/
+	pci_client_change_link_status(drv_ctx->pci_client_h,
+				      NVSCIC2C_PCIE_LINK_DOWN);
+	/*
+	 * stop ongoing and pending edma xfers, this edma module shall not
+	 * accept new xfer submissions after this.
+	 */
+	edma_module_stop(drv_ctx);
+
+	/* wait for @DRV_MODE_EPC (local) endpoints to close. */
+	ret = endpoints_waitfor_close(drv_ctx->endpoints_h);
+	if (ret)
+		pr_err("(%s): Error waiting for endpoints to close\n",
+		       drv_ctx->drv_name);
+
+	/* if PCIe EP SoC went away abruptly already, jump to local deinit. */
+	if (!pci_device_is_present(pdev))
+		goto deinit;
+
+	/*
+	 * Wait for @DRV_MODE_EPF to ACK it's closure too.
+	 *
+	 * Before this @DRV_MODE_EPC(this) endpoints must be closed already
+	 * as @DRV_MODE_EPF in response to this msg shall free all it's
+	 * endpoint mappings.
+	 */
+	reinit_completion(&drv_ctx->epc_ctx->epf_shutdown_cmpl);
+	msg.type = COMM_MSG_TYPE_SHUTDOWN;
+	if (comm_channel_ctrl_msg_send(drv_ctx->comm_channel_h, &msg)) {
+		pr_err("(%s): Failed to send shutdown message\n",
+		       drv_ctx->drv_name);
+		goto deinit;
+	}
+
+	while (timeout == 0) {
+		timeout = wait_for_completion_interruptible_timeout
+				(&drv_ctx->epc_ctx->epf_shutdown_cmpl,
+				 msecs_to_jiffies(MAX_EPF_SHUTDOWN_TIMEOUT_MSEC));
+		if (timeout == -ERESTARTSYS) {
+			pr_err("(%s): Wait for nvscic2c-pcie-epf to close - interrupted\n",
+			       drv_ctx->drv_name);
+		} else if (timeout == 0) {
+			/*
+			 * continue wait only if PCIe EP SoC is still there. It can
+			 * go away abruptly waiting for it's own endpoints to close.
+			 */
+			if (pci_device_is_present(pdev)) {
+				pr_err("(%s): Still waiting for nvscic2c-pcie-epf to close\n",
+				       drv_ctx->drv_name);
+			} else {
+				pr_debug("(%s): nvscic2c-pcie-epf went away\n",
+					 drv_ctx->drv_name);
+				break;
+			}
+		} else if (timeout > 0) {
+			pr_debug("(%s): nvscic2c-pcie-epf closed\n",
+				 drv_ctx->drv_name);
+		}
+	}
+
+deinit:
+	comm_channel_unregister_msg_cb(drv_ctx->comm_channel_h,
+				       COMM_MSG_TYPE_LINK);
+	endpoints_release(&drv_ctx->endpoints_h);
+	edma_module_deinit(drv_ctx);
+	vmap_deinit(&drv_ctx->vmap_h);
+	comm_channel_deinit(&drv_ctx->comm_channel_h);
+	pci_client_deinit(&drv_ctx->pci_client_h);
+	free_outbound_area(pdev, &drv_ctx->peer_mem);
+	free_inbound_area(pdev, &drv_ctx->self_mem);
+
+	pci_release_region(pdev, 0);
+	pci_clear_master(pdev);
+	pci_disable_device(pdev);
+
+	dt_release(&drv_ctx->drv_param);
+
+	pci_set_drvdata(pdev, NULL);
+	kfree(drv_ctx->epc_ctx);
+	kfree_const(drv_ctx->drv_name);
+	kfree(drv_ctx);
+}
+
+#define MAX_EPF_SETUP_TIMEOUT_MSEC	(5000)
+static int
+nvscic2c_pcie_epc_probe(struct pci_dev *pdev,
+			const struct pci_device_id *id)
+{
+	int ret = 0;
+	char *name = NULL;
+	size_t win_size = 0;
+	struct comm_msg msg = {0};
+	unsigned long timeout = 0;
+	struct callback_ops cb_ops = {0};
+	struct driver_ctx_t *drv_ctx = NULL;
+	struct epc_context_t *epc_ctx = NULL;
+	struct pci_client_params params = {0};
+
+	/* allocate module context.*/
+	drv_ctx = kzalloc(sizeof(*drv_ctx), GFP_KERNEL);
+	if (WARN_ON(!drv_ctx))
+		return -ENOMEM;
+
+	name = kasprintf(GFP_KERNEL, "%s-%x", DRIVER_NAME_EPC, id->device);
+	if (WARN_ON(!name)) {
+		kfree(drv_ctx);
+		return -ENOMEM;
+	}
+
+	epc_ctx = kzalloc(sizeof(*epc_ctx), GFP_KERNEL);
+	if (WARN_ON(!epc_ctx)) {
+		kfree(name);
+		kfree(drv_ctx);
+		return -ENOMEM;
+	}
+	init_completion(&epc_ctx->epf_ready_cmpl);
+	init_completion(&epc_ctx->epf_shutdown_cmpl);
+
+	drv_ctx->drv_mode = DRV_MODE_EPC;
+	drv_ctx->drv_name = name;
+	drv_ctx->epc_ctx = epc_ctx;
+	pci_set_drvdata(pdev, drv_ctx);
+
+	/* check for the device tree node against this Id, must be only one.*/
+	ret = dt_parse(id->device, drv_ctx->drv_mode, &drv_ctx->drv_param);
+	if (ret)
+		goto err_dt_parse;
+
+	ret = pcim_enable_device(pdev);
+	if (ret)
+		goto err_enable_device;
+	pci_set_master(pdev);
+	ret = pci_request_region(pdev, 0, MODULE_NAME);
+	if (ret)
+		goto err_request_region;
+
+	win_size = pci_resource_len(pdev, 0);
+	ret = allocate_inbound_area(pdev, win_size, &drv_ctx->self_mem);
+	if (ret)
+		goto err_alloc_inbound;
+
+	ret = assign_outbound_area(pdev, win_size, &drv_ctx->peer_mem);
+	if (ret)
+		goto err_assign_outbound;
+
+	params.dev = &pdev->dev;
+	params.self_mem = &drv_ctx->self_mem;
+	params.peer_mem = &drv_ctx->peer_mem;
+	ret = pci_client_init(&params, &drv_ctx->pci_client_h);
+	if (ret) {
+		pr_err("(%s): pci_client_init() failed\n",
+		       drv_ctx->drv_name);
+		goto err_pci_client;
+	}
+	pci_client_save_driver_ctx(drv_ctx->pci_client_h, drv_ctx);
+	pci_client_save_peer_cpu(drv_ctx->pci_client_h, NVCPU_ORIN);
+
+	ret = comm_channel_init(drv_ctx, &drv_ctx->comm_channel_h);
+	if (ret) {
+		pr_err("(%s): Failed to initialize comm-channel\n",
+		       drv_ctx->drv_name);
+		goto err_comm_init;
+	}
+
+	ret = vmap_init(drv_ctx, &drv_ctx->vmap_h);
+	if (ret) {
+		pr_err("(%s): Failed to initialize vmap\n",
+		       drv_ctx->drv_name);
+		goto err_vmap_init;
+	}
+
+	ret = edma_module_init(drv_ctx);
+	if (ret) {
+		pr_err("(%s): Failed to initialize edma module\n",
+		       drv_ctx->drv_name);
+		goto err_edma_init;
+	}
+
+	ret = endpoints_setup(drv_ctx, &drv_ctx->endpoints_h);
+	if (ret) {
+		pr_err("(%s): Failed to initialize endpoints\n",
+		       drv_ctx->drv_name);
+		goto err_endpoints_init;
+	}
+
+	/* register for link status message from @DRV_MODE_EPF (PCIe EP).*/
+	cb_ops.callback = link_msg_cb;
+	cb_ops.ctx = (void *)drv_ctx;
+	ret = comm_channel_register_msg_cb(drv_ctx->comm_channel_h,
+					   COMM_MSG_TYPE_LINK, &cb_ops);
+	if (ret) {
+		pr_err("(%s): Failed to register for link message\n",
+		       drv_ctx->drv_name);
+		goto err_register_msg;
+	}
+
+	/*
+	 * share iova with @DRV_MODE_EPF for it's outbound translation.
+	 * This must be send only after comm-channel, endpoint memory backing
+	 * is created and mapped to self_mem. @DRV_MODE_EPF on seeing this
+	 * message shall send link-up message over comm-channel and possibly
+	 * applications can also start endpoint negotiation, therefore.
+	 */
+	reinit_completion(&drv_ctx->epc_ctx->epf_ready_cmpl);
+	msg.type = COMM_MSG_TYPE_BOOTSTRAP;
+	msg.u.bootstrap.iova = drv_ctx->self_mem.dma_handle;
+	msg.u.bootstrap.peer_cpu = NVCPU_ORIN;
+	ret = comm_channel_ctrl_msg_send(drv_ctx->comm_channel_h, &msg);
+	if (ret) {
+		pr_err("(%s): Failed to send comm bootstrap message\n",
+		       drv_ctx->drv_name);
+		goto err_msg_send;
+	}
+
+	/* wait for @DRV_MODE_EPF to acknowledge it's endpoints are created.*/
+	timeout =
+	wait_for_completion_timeout(&drv_ctx->epc_ctx->epf_ready_cmpl,
+				    msecs_to_jiffies(MAX_EPF_SETUP_TIMEOUT_MSEC));
+	if (timeout == 0) {
+		ret = -ENOLINK;
+		pr_err("(%s): Timed-out waiting for nvscic2c-pcie-epf\n",
+		       drv_ctx->drv_name);
+		goto err_epf_ready;
+	}
+
+	pci_set_drvdata(pdev, drv_ctx);
+	return ret;
+
+err_epf_ready:
+err_msg_send:
+	comm_channel_unregister_msg_cb(drv_ctx->comm_channel_h,
+				       COMM_MSG_TYPE_LINK);
+err_register_msg:
+	endpoints_release(&drv_ctx->endpoints_h);
+
+err_endpoints_init:
+	edma_module_deinit(drv_ctx);
+
+err_edma_init:
+	vmap_deinit(&drv_ctx->vmap_h);
+
+err_vmap_init:
+	comm_channel_deinit(&drv_ctx->comm_channel_h);
+
+err_comm_init:
+	pci_client_deinit(&drv_ctx->pci_client_h);
+
+err_pci_client:
+	free_outbound_area(pdev, &drv_ctx->peer_mem);
+
+err_assign_outbound:
+	free_inbound_area(pdev, &drv_ctx->self_mem);
+
+err_alloc_inbound:
+	pci_release_region(pdev, 0);
+
+err_request_region:
+	pci_clear_master(pdev);
+	pci_disable_device(pdev);
+
+err_enable_device:
+	dt_release(&drv_ctx->drv_param);
+
+err_dt_parse:
+	pci_set_drvdata(pdev, NULL);
+	kfree_const(drv_ctx->drv_name);
+	kfree(drv_ctx);
+	return ret;
+}
+
+MODULE_DEVICE_TABLE(pci, nvscic2c_pcie_epc_tbl);
+static struct pci_driver nvscic2c_pcie_epc_driver = {
+	.name		= DRIVER_NAME_EPC,
+	.id_table	= nvscic2c_pcie_epc_tbl,
+	.probe		= nvscic2c_pcie_epc_probe,
+	.remove		= nvscic2c_pcie_epc_remove,
+	.shutdown	= nvscic2c_pcie_epc_remove,
+};
+module_pci_driver(nvscic2c_pcie_epc_driver);
+
+#define DRIVER_LICENSE		"GPL v2"
+#define DRIVER_DESCRIPTION	"NVIDIA Chip-to-Chip transfer module for PCIeRP"
+#define DRIVER_AUTHOR		"Nvidia Corporation"
+MODULE_DESCRIPTION(DRIVER_DESCRIPTION);
+MODULE_LICENSE(DRIVER_LICENSE);
+MODULE_AUTHOR(DRIVER_AUTHOR);

drivers/misc/nvscic2c-pcie/epf/module.c

@@ -0,0 +1,849 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#define pr_fmt(fmt)	"nvscic2c-pcie: epf: " fmt
+
+#include <linux/dma-iommu.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/mm.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/pci-epc.h>
+#include <linux/pci-epf.h>
+#include <linux/printk.h>
+#include <linux/slab.h>
+#include <linux/tegra-pcie-edma.h>
+#include <linux/types.h>
+
+#include "comm-channel.h"
+#include "common.h"
+#include "endpoint.h"
+#include "iova-alloc.h"
+#include "module.h"
+#include "pci-client.h"
+#include "vmap.h"
+
+static const struct pci_epf_device_id nvscic2c_pcie_epf_ids[] = {
+	{
+		.name = "nvscic2c_epf_22CB",
+		.driver_data = (kernel_ulong_t)PCI_DEVICE_ID_C2C_1,
+	},
+	{
+		.name = "nvscic2c_epf_22CC",
+		.driver_data = (kernel_ulong_t)PCI_DEVICE_ID_C2C_2,
+	},
+	{
+		.name = "nvscic2c_epf_22CD",
+		.driver_data = (kernel_ulong_t)PCI_DEVICE_ID_C2C_3,
+	},
+	{},
+};
+
+/* wrapper over tegra-pcie-edma init api. */
+static int
+edma_module_init(struct driver_ctx_t *drv_ctx)
+{
+	u8 i = 0;
+	int ret = 0;
+	struct tegra_pcie_edma_init_info info = {0};
+
+	if (WARN_ON(!drv_ctx || !drv_ctx->drv_param.edma_np))
+		return -EINVAL;
+
+	memset(&info, 0x0, sizeof(info));
+	info.np = drv_ctx->drv_param.edma_np;
+	info.edma_remote = NULL;
+
+	for (i = 0; i < DMA_WR_CHNL_NUM; i++) {
+		info.tx[i].ch_type = EDMA_CHAN_XFER_ASYNC;
+		info.tx[i].num_descriptors = NUM_EDMA_DESC;
+	}
+	/*No use-case for RD channels.*/
+
+	drv_ctx->edma_h = tegra_pcie_edma_initialize(&info);
+	if (!drv_ctx->edma_h)
+		ret = -ENODEV;
+
+	return ret;
+}
+
+/* should stop any ongoing eDMA transfers.*/
+static void
+edma_module_stop(struct driver_ctx_t *drv_ctx)
+{
+	if (!drv_ctx || !drv_ctx->edma_h)
+		return;
+
+	tegra_pcie_edma_stop(drv_ctx->edma_h);
+}
+
+/* should not have any ongoing eDMA transfers.*/
+static void
+edma_module_deinit(struct driver_ctx_t *drv_ctx)
+{
+	if (!drv_ctx || !drv_ctx->edma_h)
+		return;
+
+	tegra_pcie_edma_deinit(drv_ctx->edma_h);
+	drv_ctx->edma_h = NULL;
+}
+
+static void
+free_inbound_area(struct pci_epf *epf, struct dma_buff_t *self_mem)
+{
+	struct driver_ctx_t *drv_ctx = NULL;
+
+	if (!epf || !self_mem)
+		return;
+
+	drv_ctx = epf_get_drvdata(epf);
+	if (!drv_ctx)
+		return;
+
+	if (self_mem->dma_handle && drv_ctx->ivd_h)
+		iova_alloc_deinit(self_mem->dma_handle, self_mem->size,
+				  &drv_ctx->ivd_h);
+	self_mem->dma_handle = 0x0;
+}
+
+/*
+ * Allocate BAR backing iova region. Writes from peer SoC shall
+ * land in this region for it to be forwarded to system iommu to eventually
+ * land in BAR backing physical region.
+ */
+static int
+allocate_inbound_area(struct pci_epf *epf, size_t win_size,
+		      struct dma_buff_t *self_mem)
+{
+	int ret = 0;
+	struct driver_ctx_t *drv_ctx = NULL;
+
+	drv_ctx = epf_get_drvdata(epf);
+	if (!drv_ctx)
+		return -ENOMEM;
+
+	self_mem->size = win_size;
+	ret = iova_alloc_init(epf->epc->dev.parent, win_size,
+			      &self_mem->dma_handle, &drv_ctx->ivd_h);
+	if (ret) {
+		pr_err("iova_domain_init() failed for size:(0x%lx)\n",
+		       self_mem->size);
+	}
+
+	return ret;
+}
+
+static void
+free_outbound_area(struct pci_epf *epf, struct pci_aper_t *peer_mem)
+{
+	if (!epf || !peer_mem || !peer_mem->pva)
+		return;
+
+	pci_epc_mem_free_addr(epf->epc, peer_mem->aper,
+			      peer_mem->pva,
+			      peer_mem->size);
+	peer_mem->pva = NULL;
+}
+
+/*
+ * Allocate outbound pcie aperture for CPU access towards PCIe RP.
+ * It is assumed that PCIe RP shall also allocate an equivalent size of inbound
+ * area as PCIe EP (it's BAR0 length).
+ */
+static int
+allocate_outbound_area(struct pci_epf *epf, size_t win_size,
+		       struct pci_aper_t *peer_mem)
+{
+	int ret = 0;
+
+	peer_mem->size = win_size;
+	peer_mem->pva = pci_epc_mem_alloc_addr(epf->epc,
+					       &peer_mem->aper,
+					       peer_mem->size);
+	if (!peer_mem->pva) {
+		ret = -ENOMEM;
+		pr_err("pci_epc_mem_alloc_addr() fails for size:(0x%lx)\n",
+		       peer_mem->size);
+	}
+
+	return ret;
+}
+
+static void
+clear_inbound_translation(struct pci_epf *epf)
+{
+	struct pci_epf_bar *epf_bar = &epf->bar[BAR_0];
+
+	pci_epc_clear_bar(epf->epc, epf->func_no, PCIE_VFNO, epf_bar);
+
+	/* no api to clear epf header.*/
+}
+
+static int
+set_inbound_translation(struct pci_epf *epf)
+{
+	int ret = 0;
+	struct pci_epc *epc = epf->epc;
+	struct pci_epf_bar *epf_bar = &epf->bar[BAR_0];
+
+	ret = pci_epc_write_header(epc, epf->func_no, PCIE_VFNO, epf->header);
+	if (ret < 0) {
+		pr_err("Failed to write PCIe header\n");
+		return ret;
+	}
+
+	/* BAR:0 setttings are already done in _bind().*/
+	ret = pci_epc_set_bar(epc, epf->func_no, PCIE_VFNO, epf_bar);
+	if (ret) {
+		pr_err("pci_epc_set_bar() failed\n");
+		return ret;
+	}
+
+	ret = pci_epc_set_msi(epc, epf->func_no, PCIE_VFNO,
+			      epf->msi_interrupts);
+	if (ret) {
+		pr_err("pci_epc_set_msi() failed\n");
+		return ret;
+	}
+
+	return ret;
+}
+
+static void
+clear_outbound_translation(struct pci_epf *epf, struct pci_aper_t *peer_mem)
+{
+	return pci_epc_unmap_addr(epf->epc, epf->func_no, PCIE_VFNO,
+				  peer_mem->aper);
+}
+
+static int
+set_outbound_translation(struct pci_epf *epf, struct pci_aper_t *peer_mem,
+			 u64 peer_iova)
+{
+	return pci_epc_map_addr(epf->epc, epf->func_no, PCIE_VFNO,
+				peer_mem->aper,	peer_iova, peer_mem->size);
+}
+
+static void
+edma_rx_desc_iova_send(struct driver_ctx_t *drv_ctx)
+{
+	int ret;
+	struct comm_msg msg = {0};
+
+	msg.type = COMM_MSG_TYPE_EDMA_RX_DESC_IOVA_RETURN;
+	msg.u.edma_rx_desc_iova.iova = pci_client_get_edma_rx_desc_iova(drv_ctx->pci_client_h);
+
+	ret = comm_channel_ctrl_msg_send(drv_ctx->comm_channel_h, &msg);
+	if (ret)
+		pr_err("failed sending COMM_MSG_TYPE_EDMA_CH_DESC_IOVA_RETURN  message\n");
+}
+
+/* Handle bootstrap message from @DRV_MODE_EPC. */
+static void
+bootstrap_msg_cb(void *data, void *ctx)
+{
+	int ret = 0;
+	struct driver_ctx_t *drv_ctx = NULL;
+	struct pci_epf *epf = (struct pci_epf *)ctx;
+	struct comm_msg *msg = (struct comm_msg *)data;
+
+	if (WARN_ON(!msg || !epf))
+		return;
+
+	drv_ctx = epf_get_drvdata(epf);
+	if (!drv_ctx)
+		return;
+
+	/*
+	 * setup outbound translation for CPU access from @DRV_MODE_EPF ->
+	 * @DRV_MODE_EPC using the iova received from @DRV_MODE_EPC in
+	 * bootstrap message.
+	 *
+	 * Must be done here, as return of the comm-channel message callback
+	 * shall use CPU on @DRV_MODE_EPF to indicate message read.
+	 */
+	ret = set_outbound_translation(epf, &drv_ctx->peer_mem,
+				       msg->u.bootstrap.iova);
+	if (ret) {
+		pr_err("Failed to set outbound (peer) memory translation\n");
+		return;
+	}
+
+	pci_client_save_peer_cpu(drv_ctx->pci_client_h, msg->u.bootstrap.peer_cpu);
+
+	/* send edma rx desc iova  to x86 peer(rp) */
+	if (msg->u.bootstrap.peer_cpu == NVCPU_X86_64)
+		edma_rx_desc_iova_send(drv_ctx);
+
+	/*
+	 * schedule initialization of remaining interfaces as it could not
+	 * be done in _notifier()(PCIe EP controller is still uninitialized
+	 * then). Also abstraction: vmap registers with comm-channel, such
+	 * callback registrations cannot happen while in the context of
+	 * another comm-channel callback (this function).
+	 */
+	schedule_work(&drv_ctx->epf_ctx->initialization_work);
+}
+
+/*
+ * tasklet/scheduled work for initialization of remaining interfaces
+ * (that which could not be done in _bind(), Reason: endpoint abstraction
+ *  requires:
+ *   - peer iova - not available unless bootstrap message.
+ *   - edma cookie - cannot be done during _notifier() - interrupt context).
+ * )
+ */
+static void
+init_work(struct work_struct *work)
+{
+	int ret = 0;
+	struct comm_msg msg = {0};
+	struct epf_context_t *epf_ctx =
+		container_of(work, struct epf_context_t, initialization_work);
+	struct driver_ctx_t *drv_ctx = (struct driver_ctx_t *)epf_ctx->drv_ctx;
+
+	if (atomic_read(&drv_ctx->epf_ctx->epf_initialized)) {
+		pr_err("(%s): Already initialized\n", drv_ctx->drv_name);
+		goto err;
+	}
+
+	ret = vmap_init(drv_ctx, &drv_ctx->vmap_h);
+	if (ret) {
+		pr_err("(%s): vmap_init() failed\n", drv_ctx->drv_name);
+		goto err;
+	}
+
+	ret = edma_module_init(drv_ctx);
+	if (ret) {
+		pr_err("(%s): edma_module_init() failed\n", drv_ctx->drv_name);
+		goto err_edma_init;
+	}
+
+	ret = endpoints_setup(drv_ctx, &drv_ctx->endpoints_h);
+	if (ret) {
+		pr_err("(%s): endpoints_setup() failed\n", drv_ctx->drv_name);
+		goto err_endpoint;
+	}
+
+	/*
+	 * this is an acknowledgment to @DRV_MODE_EPC in response to it's
+	 * bootstrap message to indicate @DRV_MODE_EPF endpoints are ready.
+	 */
+	msg.type = COMM_MSG_TYPE_LINK;
+	msg.u.link.status = NVSCIC2C_PCIE_LINK_UP;
+	ret = comm_channel_ctrl_msg_send(drv_ctx->comm_channel_h, &msg);
+	if (ret) {
+		pr_err("(%s): Failed to send LINK(UP) message\n",
+		       drv_ctx->drv_name);
+		goto err_msg_send;
+	}
+
+	/* inidicate link-up to applications.*/
+	atomic_set(&drv_ctx->epf_ctx->epf_initialized, 1);
+	pci_client_change_link_status(drv_ctx->pci_client_h,
+				      NVSCIC2C_PCIE_LINK_UP);
+	return;
+
+err_msg_send:
+	endpoints_release(&drv_ctx->endpoints_h);
+err_endpoint:
+	edma_module_deinit(drv_ctx);
+err_edma_init:
+	vmap_deinit(&drv_ctx->vmap_h);
+err:
+	return;
+}
+
+/*
+ * PCIe subsystem calls struct pci_epc_event_ops.core_init
+ * when PCIe hot-plug is initiated and before link trainig
+ * starts with PCIe RP SoC (before @DRV_MODE_EPC .probe() handler is invoked).
+ *
+ * Because, CORE_INIT impacts link training timeout, it shall do only minimum
+ * required for @DRV_MODE_EPF for PCIe EP initialization.
+ *
+ * This is received in interrupt context.
+ */
+static int
+nvscic2c_pcie_epf_core_init(struct pci_epf *epf)
+{
+	int ret = 0;
+	struct driver_ctx_t *drv_ctx = NULL;
+
+	drv_ctx = epf_get_drvdata(epf);
+	if (!drv_ctx)
+		return -EINVAL;
+
+	if (atomic_read(&drv_ctx->epf_ctx->core_initialized)) {
+		pr_err("(%s): Received CORE_INIT callback again\n",
+		       drv_ctx->drv_name);
+		return -EINVAL;
+	}
+
+	ret = set_inbound_translation(epf);
+	if (ret) {
+		pr_err("(%s): set_inbound_translation() failed\n",
+		       drv_ctx->drv_name);
+		return ret;
+	}
+
+	atomic_set(&drv_ctx->epf_ctx->core_initialized, 1);
+
+	return ret;
+}
+
+/* Handle link message from @DRV_MODE_EPC. */
+static void
+shutdown_msg_cb(void *data, void *ctx)
+{
+	struct driver_ctx_t *drv_ctx = NULL;
+	struct pci_epf *epf = (struct pci_epf *)ctx;
+	struct comm_msg *msg = (struct comm_msg *)data;
+
+	if (WARN_ON(!msg || !epf))
+		return;
+
+	drv_ctx = epf_get_drvdata(epf);
+	if (!drv_ctx)
+		return;
+
+	if (!atomic_read(&drv_ctx->epf_ctx->epf_initialized)) {
+		pr_err("(%s): Unexpected shutdown msg from nvscic2c-pcie-epc\n",
+		       drv_ctx->drv_name);
+		return;
+	}
+
+	/* schedule deinitialization of epf interfaces. */
+	schedule_work(&drv_ctx->epf_ctx->deinitialization_work);
+}
+
+/*
+ * tasklet/scheduled work for de-initialization of @DRV_MODE_EPF(this)
+ * interfaces. It is done in a tasklet for the following scenario:
+ * @DRV_MODE_EPC can get unloaded(rmmod) and reinserted(insmod) while the
+ * PCIe link with PCIe EP SoC still active. So before we receive
+ * bootstrap message again when @DRV_MODE_EPC is reinserted, we would need
+ * to clean-up all abstractions before they can be reinit again.
+ *
+ * In case of abnormal shutdown of PCIe RP SoC, @DRV_MODE_EPF shall receive
+ * CORE_DEINIT directly from PCIe sub-system without any comm-message from
+ * @DRV_MODE_EPC.
+ */
+static void
+deinit_work(struct work_struct *work)
+{
+	int ret = 0;
+	struct comm_msg msg = {0};
+	struct epf_context_t *epf_ctx =
+		container_of(work, struct epf_context_t, deinitialization_work);
+	struct driver_ctx_t *drv_ctx = (struct driver_ctx_t *)epf_ctx->drv_ctx;
+
+	if (!atomic_read(&drv_ctx->epf_ctx->epf_initialized))
+		return;
+
+	/* local apps can stop processing if they see this.*/
+	pci_client_change_link_status(drv_ctx->pci_client_h,
+				      NVSCIC2C_PCIE_LINK_DOWN);
+	/*
+	 * stop ongoing and pending edma xfers, this edma module shall not
+	 * accept new xfer submissions after this.
+	 */
+	edma_module_stop(drv_ctx);
+
+	/* wait for @DRV_MODE_EPF (local)endpoints to close. */
+	ret = endpoints_waitfor_close(drv_ctx->endpoints_h);
+	if (ret) {
+		pr_err("(%s): Error waiting for endpoints to close\n",
+		       drv_ctx->drv_name);
+	}
+	/* Even in case of error, continue to deinit - cannot be recovered.*/
+
+	/*
+	 * Acknowledge @DRV_MODE_EPC that @DRV_MODE_EPF(this) endpoints are
+	 * closed. If PCIe RP SoC went abnormally away(halt/reset/kernel oops)
+	 * signal anyway (sending signal will not cause local SoC fault when
+	 * PCIe RP SoC (@DRV_MODE_EPC) went abnormally away).
+	 */
+	msg.type = COMM_MSG_TYPE_LINK;
+	msg.u.link.status = NVSCIC2C_PCIE_LINK_DOWN;
+	ret = comm_channel_ctrl_msg_send(drv_ctx->comm_channel_h, &msg);
+	if (ret)
+		pr_err("(%s): Failed to send LINK (DOWN) message\n",
+		       drv_ctx->drv_name);
+
+	endpoints_release(&drv_ctx->endpoints_h);
+	edma_module_deinit(drv_ctx);
+	vmap_deinit(&drv_ctx->vmap_h);
+	clear_outbound_translation(drv_ctx->epf_ctx->epf, &drv_ctx->peer_mem);
+	atomic_set(&drv_ctx->epf_ctx->epf_initialized, 0);
+}
+
+/*
+ * Graceful shutdown: PCIe subsystem calls struct pci_epc_event_ops.core_deinit
+ * when @DRV_MODE_EPC .remove() or .shutdown() handlers are completed/exited.
+ * Abnormal shutdown (when PCIe RP SoC - gets halted, or it's kernel oops):
+ * PCIe subsystem also struct pci_epc_event_ops.core_deinit but
+ * @DRV_MODE_EPC would have already gone then by the time
+ * struct pci_epc_event_ops.core_deinit is called.
+ */
+static int
+nvscic2c_pcie_epf_core_deinit(struct pci_epf *epf)
+{
+	struct driver_ctx_t *drv_ctx = NULL;
+
+	drv_ctx = epf_get_drvdata(epf);
+	if (!drv_ctx)
+		return -EINVAL;
+
+	if (atomic_read(&drv_ctx->epf_ctx->core_initialized)) {
+		/*
+		 * in case of PCIe RP SoC abnormal shutdown, comm-channel
+		 * shutdown message from @DRV_MODE_EPC won't come and
+		 * therefore scheduling the deinit work here is required
+		 * If its already scheduled, it won't be scheduled again.
+		 * Wait for deinit work to complete in either case.
+		 */
+		schedule_work(&drv_ctx->epf_ctx->deinitialization_work);
+		flush_work(&drv_ctx->epf_ctx->deinitialization_work);
+
+		clear_inbound_translation(epf);
+		atomic_set(&drv_ctx->epf_ctx->core_initialized, 0);
+	}
+	wake_up_interruptible_all(&drv_ctx->epf_ctx->core_initialized_waitq);
+
+	return 0;
+}
+
+/* Handle link message from @DRV_MODE_EPC. */
+static void
+link_msg_cb(void *data, void *ctx)
+{
+	struct pci_epf *epf = (struct pci_epf *)ctx;
+	struct comm_msg *msg = (struct comm_msg *)data;
+	struct driver_ctx_t *drv_ctx = (struct driver_ctx_t *)ctx;
+
+	if (WARN_ON(!msg || !epf))
+		return;
+
+	drv_ctx = epf_get_drvdata(epf);
+	if (!drv_ctx)
+		return;
+
+	if (msg->u.link.status != NVSCIC2C_PCIE_LINK_DOWN) {
+		pr_err("(%s): spurious link message received from EPC\n",
+		       drv_ctx->drv_name);
+		return;
+	}
+
+	/* inidicate link status to application.*/
+	pci_client_change_link_status(drv_ctx->pci_client_h,
+				      msg->u.link.status);
+}
+
+/*
+ * ASSUMPTION: applications on and @DRV_MODE_EPC(PCIe RP) must have stopped
+ * communicating with application and @DRV_MODE_EPF (this) before this point.
+ */
+static void
+nvscic2c_pcie_epf_unbind(struct pci_epf *epf)
+{
+	long ret = 0;
+	struct driver_ctx_t *drv_ctx = NULL;
+
+	if (!epf)
+		return;
+
+	drv_ctx = epf_get_drvdata(epf);
+	if (!drv_ctx)
+		return;
+
+	/* timeout should be higher than that of endpoints to close.*/
+	ret = wait_event_interruptible
+			(drv_ctx->epf_ctx->core_initialized_waitq,
+			 !(atomic_read(&drv_ctx->epf_ctx->core_initialized)));
+	if (ret == -ERESTARTSYS)
+		pr_err("(%s): Interrupted waiting for CORE_DEINIT to complete\n",
+		       drv_ctx->drv_name);
+
+	comm_channel_unregister_msg_cb(drv_ctx->comm_channel_h,
+				       COMM_MSG_TYPE_SHUTDOWN);
+	comm_channel_unregister_msg_cb(drv_ctx->comm_channel_h,
+				       COMM_MSG_TYPE_BOOTSTRAP);
+	comm_channel_deinit(&drv_ctx->comm_channel_h);
+	pci_client_deinit(&drv_ctx->pci_client_h);
+	free_outbound_area(epf, &drv_ctx->peer_mem);
+	free_inbound_area(epf, &drv_ctx->self_mem);
+}
+
+static int
+nvscic2c_pcie_epf_bind(struct pci_epf *epf)
+{
+	int ret = 0;
+	size_t win_size = 0;
+	struct pci_epf_bar *epf_bar = NULL;
+	struct driver_ctx_t *drv_ctx = NULL;
+	struct pci_client_params params = {0};
+	struct callback_ops cb_ops = {0};
+
+	if (!epf)
+		return -EINVAL;
+
+	drv_ctx = epf_get_drvdata(epf);
+	if (!drv_ctx)
+		return -EINVAL;
+
+	/*
+	 * device-tree node has edma phandle, user must bind
+	 * the function to the same pcie controller.
+	 */
+	if (drv_ctx->drv_param.edma_np != epf->epc->dev.parent->of_node) {
+		pr_err("epf:(%s) is not bounded to correct controller\n",
+		       epf->name);
+		return -EINVAL;
+	}
+
+	win_size = drv_ctx->drv_param.bar_win_size;
+	ret = allocate_inbound_area(epf, win_size, &drv_ctx->self_mem);
+	if (ret)
+		goto err_alloc_inbound;
+
+	ret = allocate_outbound_area(epf, win_size, &drv_ctx->peer_mem);
+	if (ret)
+		goto err_alloc_outbound;
+
+	params.dev = epf->epc->dev.parent;
+	params.self_mem = &drv_ctx->self_mem;
+	params.peer_mem = &drv_ctx->peer_mem;
+	ret = pci_client_init(&params, &drv_ctx->pci_client_h);
+	if (ret) {
+		pr_err("pci_client_init() failed\n");
+		goto err_pci_client;
+	}
+	pci_client_save_driver_ctx(drv_ctx->pci_client_h, drv_ctx);
+	/*
+	 * setup of comm-channel must be done in bind() for @DRV_MODE_EPC
+	 * to share bootstrap message. register for message from @DRV_MODE_EPC
+	 * (PCIe RP).
+	 */
+	ret = comm_channel_init(drv_ctx, &drv_ctx->comm_channel_h);
+	if (ret) {
+		pr_err("Failed to initialize comm-channel\n");
+		goto err_comm_init;
+	}
+
+	/* register for bootstrap message from @DRV_MODE_EPC (PCIe RP).*/
+	cb_ops.callback = bootstrap_msg_cb;
+	cb_ops.ctx = (void *)epf;
+	ret = comm_channel_register_msg_cb(drv_ctx->comm_channel_h,
+					   COMM_MSG_TYPE_BOOTSTRAP, &cb_ops);
+	if (ret) {
+		pr_err("Failed to register for bootstrap message from RP\n");
+		goto err_register_msg;
+	}
+
+	/* register for shutdown message from @DRV_MODE_EPC (PCIe RP).*/
+	memset(&cb_ops, 0x0, sizeof(cb_ops));
+	cb_ops.callback = shutdown_msg_cb;
+	cb_ops.ctx = (void *)epf;
+	ret = comm_channel_register_msg_cb(drv_ctx->comm_channel_h,
+					   COMM_MSG_TYPE_SHUTDOWN, &cb_ops);
+	if (ret) {
+		pr_err("Failed to register for shutdown message from RP\n");
+		goto err_register_msg;
+	}
+
+	/* register for link message from @DRV_MODE_EPC (PCIe RP).*/
+	memset(&cb_ops, 0x0, sizeof(cb_ops));
+	cb_ops.callback = link_msg_cb;
+	cb_ops.ctx = (void *)epf;
+	ret = comm_channel_register_msg_cb(drv_ctx->comm_channel_h,
+					   COMM_MSG_TYPE_LINK, &cb_ops);
+	if (ret) {
+		pr_err("Failed to register for link message from RP\n");
+		goto err_register_msg;
+	}
+
+	/* BAR:0 settings. - done here to save time in CORE_INIT.*/
+	epf_bar = &epf->bar[BAR_0];
+	epf_bar->phys_addr = drv_ctx->self_mem.dma_handle;
+	epf_bar->size = drv_ctx->self_mem.size;
+	epf_bar->barno = BAR_0;
+	epf_bar->flags |= PCI_BASE_ADDRESS_SPACE_MEMORY |
+			  PCI_BASE_ADDRESS_MEM_TYPE_64 |
+			  PCI_BASE_ADDRESS_MEM_PREFETCH;
+
+	return ret;
+
+err_register_msg:
+	comm_channel_deinit(&drv_ctx->comm_channel_h);
+
+err_comm_init:
+	pci_client_deinit(&drv_ctx->pci_client_h);
+
+err_pci_client:
+	free_outbound_area(epf, &drv_ctx->peer_mem);
+
+err_alloc_outbound:
+	free_inbound_area(epf, &drv_ctx->self_mem);
+
+err_alloc_inbound:
+	return ret;
+}
+
+static void
+nvscic2c_pcie_epf_remove(struct pci_epf *epf)
+{
+	struct driver_ctx_t *drv_ctx = epf_get_drvdata(epf);
+
+	if (!drv_ctx)
+		return;
+
+	cancel_work_sync(&drv_ctx->epf_ctx->deinitialization_work);
+	cancel_work_sync(&drv_ctx->epf_ctx->initialization_work);
+	epf->header = NULL;
+	kfree(drv_ctx->epf_ctx);
+
+	dt_release(&drv_ctx->drv_param);
+
+	epf_set_drvdata(epf, NULL);
+	kfree_const(drv_ctx->drv_name);
+	kfree(drv_ctx);
+}
+
+static kernel_ulong_t
+get_driverdata(const struct pci_epf_device_id *id,
+	       const struct pci_epf *epf)
+{
+	while (id->name[0]) {
+		if (strcmp(epf->name, id->name) == 0)
+			return id->driver_data;
+		id++;
+	}
+
+	return 0;
+}
+
+static const struct pci_epc_event_ops nvscic2c_event_ops = {
+	.core_init = nvscic2c_pcie_epf_core_init,
+	.core_deinit = nvscic2c_pcie_epf_core_deinit,
+};
+
+static int
+nvscic2c_pcie_epf_probe(struct pci_epf *epf)
+{
+	int ret = 0;
+	char *name = NULL;
+	u32 pci_dev_id = 0x0;
+	struct driver_ctx_t *drv_ctx = NULL;
+	struct epf_context_t *epf_ctx = NULL;
+
+	/* get pci device id from epf name.*/
+	pci_dev_id = (u32)get_driverdata(nvscic2c_pcie_epf_ids, epf);
+	if (!pci_dev_id)
+		return -EINVAL;
+
+	/* allocate module context.*/
+	drv_ctx = kzalloc(sizeof(*drv_ctx), GFP_KERNEL);
+	if (WARN_ON(!drv_ctx))
+		return -ENOMEM;
+
+	name = kasprintf(GFP_KERNEL, "%s-%x", DRIVER_NAME_EPF, pci_dev_id);
+	if (WARN_ON(!name)) {
+		kfree(drv_ctx);
+		return -ENOMEM;
+	}
+
+	drv_ctx->drv_mode = DRV_MODE_EPF;
+	drv_ctx->drv_name = name;
+	epf_set_drvdata(epf, drv_ctx);
+
+	/* check for the device tree node against this Id, must be only one.*/
+	ret = dt_parse(pci_dev_id, drv_ctx->drv_mode, &drv_ctx->drv_param);
+	if (ret)
+		goto err_dt_parse;
+
+	/* allocate nvscic2c-pcie epf only context.*/
+	epf_ctx = kzalloc(sizeof(*epf_ctx), GFP_KERNEL);
+	if (WARN_ON(!epf_ctx)) {
+		ret = -ENOMEM;
+		goto err_alloc_epf_ctx;
+	}
+	drv_ctx->epf_ctx = epf_ctx;
+	epf_ctx->header.vendorid = PCI_VENDOR_ID_NVIDIA;
+	epf_ctx->header.deviceid = pci_dev_id;
+	epf_ctx->header.baseclass_code = PCI_BASE_CLASS_COMMUNICATION;
+	epf_ctx->header.interrupt_pin = PCI_INTERRUPT_INTA;
+
+	epf->event_ops = &nvscic2c_event_ops;
+	epf->header = &epf_ctx->header;
+
+	/* to initialize NvSciC2cPcie interfaces on bootstrap msg.*/
+	epf_ctx->drv_ctx = drv_ctx;
+	epf_ctx->epf = epf;
+	INIT_WORK(&epf_ctx->initialization_work, init_work);
+	INIT_WORK(&epf_ctx->deinitialization_work, deinit_work);
+
+	/* to synchronize deinit, unbind.*/
+	atomic_set(&epf_ctx->core_initialized, 0);
+	atomic_set(&drv_ctx->epf_ctx->epf_initialized, 0);
+	init_waitqueue_head(&epf_ctx->core_initialized_waitq);
+
+	return ret;
+
+err_alloc_epf_ctx:
+	dt_release(&drv_ctx->drv_param);
+
+err_dt_parse:
+	epf_set_drvdata(epf, NULL);
+	kfree_const(drv_ctx->drv_name);
+	kfree(drv_ctx);
+
+	return ret;
+}
+
+static struct pci_epf_ops ops = {
+	.unbind = nvscic2c_pcie_epf_unbind,
+	.bind   = nvscic2c_pcie_epf_bind,
+};
+
+static struct pci_epf_driver nvscic2c_pcie_epf_driver = {
+	.driver.name = DRIVER_NAME_EPF,
+	.probe       = nvscic2c_pcie_epf_probe,
+	.remove      = nvscic2c_pcie_epf_remove,
+	.id_table    = nvscic2c_pcie_epf_ids,
+	.ops         = &ops,
+	.owner       = THIS_MODULE,
+};
+
+static int
+__init nvscic2c_pcie_epf_init(void)
+{
+	return pci_epf_register_driver(&nvscic2c_pcie_epf_driver);
+}
+module_init(nvscic2c_pcie_epf_init);
+
+static void
+__exit nvscic2c_pcie_epf_deinit(void)
+{
+	pci_epf_unregister_driver(&nvscic2c_pcie_epf_driver);
+}
+module_exit(nvscic2c_pcie_epf_deinit);
+
+#define DRIVER_LICENSE		"GPL v2"
+#define DRIVER_DESCRIPTION	"NVIDIA Chip-to-Chip transfer module for PCIeEP"
+#define DRIVER_AUTHOR		"Nvidia Corporation"
+MODULE_DESCRIPTION(DRIVER_DESCRIPTION);
+MODULE_LICENSE(DRIVER_LICENSE);
+MODULE_AUTHOR(DRIVER_AUTHOR);

drivers/misc/nvscic2c-pcie/iova-alloc.c

@@ -0,0 +1,146 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+#define pr_fmt(fmt)	"nvscic2c-pcie: iova-alloc: " fmt
+#include <linux/iommu.h>
+#include <linux/types.h>
+#include <linux/pci.h>
+#include <linux/slab.h>
+
+#include "iova-alloc.h"
+
+struct iova_alloc_domain_t {
+	struct device *dev;
+	struct iommu_domain *domain;
+	struct iova_domain iovad;
+};
+
+static int
+iovad_init(struct device *dev, struct iova_alloc_domain_t **ivd_h)
+{
+	int ret = 0;
+	dma_addr_t start = 0;
+	unsigned long order = 0;
+	struct iova_alloc_domain_t *ivd_ctx = NULL;
+	struct iommu_domain_geometry *geometry = NULL;
+
+	ivd_ctx = kzalloc(sizeof(*ivd_ctx), GFP_KERNEL);
+	if (WARN_ON(!ivd_ctx))
+		return -ENOMEM;
+
+	ivd_ctx->dev = dev;
+	ret = iova_cache_get();
+	if (ret < 0)
+		return ret;
+
+	ivd_ctx->domain = iommu_get_domain_for_dev(dev);
+	if (!ivd_ctx->domain) {
+		ret = -EINVAL;
+		pr_err("iommu_get_domain_for_dev() failed.\n");
+		goto put_cache;
+	}
+
+	geometry = &ivd_ctx->domain->geometry;
+	start = geometry->aperture_start & dev->coherent_dma_mask;
+	order = __ffs(ivd_ctx->domain->pgsize_bitmap);
+	pr_debug("Order of address allocation for IOVA domain: %lu\n", order);
+	init_iova_domain(&ivd_ctx->iovad, 1UL << order, start >> order);
+
+	*ivd_h = ivd_ctx;
+
+	return ret;
+put_cache:
+	iova_cache_put();
+	return ret;
+}
+
+static void
+iovad_deinit(struct iova_alloc_domain_t **ivd_h)
+{
+	struct iova_alloc_domain_t *ivd_ctx = NULL;
+
+	ivd_ctx = (struct iova_alloc_domain_t *)(*ivd_h);
+	put_iova_domain(&ivd_ctx->iovad);
+	iova_cache_put();
+	kfree(ivd_ctx);
+	*ivd_h = NULL;
+}
+
+int
+iova_alloc_init(struct device *dev, size_t size, dma_addr_t *dma_handle,
+		struct iova_alloc_domain_t **ivd_h)
+{
+	int ret = 0;
+	unsigned long shift = 0U;
+	unsigned long iova = 0U;
+	unsigned long iova_len = 0U;
+	dma_addr_t dma_limit = 0x0;
+	struct iova_alloc_domain_t *ivd_ctx = NULL;
+	struct iommu_domain_geometry *geometry = NULL;
+
+	if (WARN_ON(!dev || !dma_handle || !ivd_h || *ivd_h))
+		return -EINVAL;
+
+	ret = iovad_init(dev, &ivd_ctx);
+	if (ret < 0) {
+		pr_err("Failed in allocating IOVA domain: %d\n", ret);
+		return ret;
+	}
+
+	geometry = &ivd_ctx->domain->geometry;
+	dma_limit = ivd_ctx->dev->coherent_dma_mask;
+	shift = iova_shift(&ivd_ctx->iovad);
+	iova_len = size >> shift;
+
+	/* Recommendation is to allocate in power of 2.*/
+	if (iova_len < (1 << (IOVA_RANGE_CACHE_MAX_SIZE - 1)))
+		iova_len = roundup_pow_of_two(iova_len);
+
+	if (*ivd_ctx->dev->dma_mask)
+		dma_limit &= *ivd_ctx->dev->dma_mask;
+
+	if (geometry->force_aperture)
+		dma_limit = min(dma_limit, geometry->aperture_end);
+
+	/* Try to get PCI devices a SAC address */
+	if (dma_limit > DMA_BIT_MASK(32) && dev_is_pci(ivd_ctx->dev))
+		iova = alloc_iova_fast(&ivd_ctx->iovad, iova_len,
+				       DMA_BIT_MASK(32) >> shift, false);
+	if (!iova)
+		iova = alloc_iova_fast(&ivd_ctx->iovad, iova_len,
+				       dma_limit >> shift, true);
+
+	*dma_handle = (dma_addr_t)iova << shift;
+	*ivd_h = ivd_ctx;
+
+	return ret;
+}
+
+void
+iova_alloc_deinit(dma_addr_t dma_handle, size_t size,
+		  struct iova_alloc_domain_t **ivd_h)
+{
+	struct iova_domain *iovad = NULL;
+	struct iova_alloc_domain_t *ivd_ctx = NULL;
+
+	if (!ivd_h || !(*ivd_h) || !dma_handle)
+		return;
+
+	ivd_ctx = *ivd_h;
+	iovad = &ivd_ctx->iovad;
+
+	free_iova_fast(iovad, iova_pfn(iovad, dma_handle),
+		       size >> iova_shift(iovad));
+
+	iovad_deinit(ivd_h);
+}

drivers/misc/nvscic2c-pcie/iova-alloc.h

@@ -0,0 +1,48 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+#ifndef __IOVA_ALLOC_H__
+#define __IOVA_ALLOC_H__
+
+#include <linux/device.h>
+#include <linux/iova.h>
+
+struct iova_alloc_domain_t;
+
+/*
+ * iova_alloc_init
+ *
+ * With NvSciC2c usecase IOVA range needs to be allocated without
+ * physical backing. Existing DMA API framework does not allow this.
+ * Hence allocate new IOVA domain to allocate IOVA range.
+ *
+ * Allocate IOVA range using new IOVA domain.
+ * Use this IOVA range in iommu_map() with existing IOMMU domain.
+ */
+int
+iova_alloc_init(struct device *dev, size_t size, dma_addr_t *dma_handle,
+		struct iova_alloc_domain_t **ivd_h);
+
+/*
+ * iova_alloc_deinit
+ *
+ * Free IOVA range allocated using iova_alloc_init.
+ * Client needs to make sure that if physical mapping was created
+ * then it is released before calling iova_alloc_deinit.
+
+ * Release IOVA domain allocated in iova_alloc_init.
+ */
+void
+iova_alloc_deinit(dma_addr_t dma_handle, size_t size,
+		  struct iova_alloc_domain_t **ivd_h);
+#endif //__IOVA_ALLOC_H__

drivers/misc/nvscic2c-pcie/iova-mngr.c

@@ -0,0 +1,389 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#define pr_fmt(fmt)	"nvscic2c-pcie: iova-mgr: " fmt
+
+#include <linux/errno.h>
+#include <linux/mutex.h>
+#include <linux/printk.h>
+#include <linux/slab.h>
+#include <linux/types.h>
+
+#include "common.h"
+#include "iova-mngr.h"
+
+/*
+ * INTERNAL DataStructure that define a single IOVA block/chunk
+ * in a pool of IOVA region managed by IOVA manager.
+ *
+ * IOVA manager chunks entire IOVA space into these blocks/chunks.
+ *
+ * The chunk/block is also a node for linking to previous and next
+ * nodes of a circular doubly linked list - free or reserved.
+ */
+struct block_t {
+	/* for management of this chunk in either avail or reserve lists.*/
+	struct list_head node;
+
+	/* block address.*/
+	u64 address;
+
+	/* block size.*/
+	size_t size;
+};
+
+/*
+ * INTERNAL datastructure for IOVA space manager.
+ *
+ * IOVA space manager would fragment and manage the IOVA region
+ * using two circular doubly linked lists - reserved list and
+ * free list. These lists contain blocks/chunks reserved
+ * or free for use by clients (callers) from the overall
+ * IOVA region the IOVA manager was configured with.
+ */
+struct mngr_ctx_t {
+	/*
+	 * For debug purpose only, all other usage prohibited.
+	 * In the event there are multiple iova_managers within
+	 * an LKM instance, name helps in identification.
+	 */
+	char name[NAME_MAX];
+
+	/*
+	 * Circular doubly linked list of blocks indicating
+	 * available/free IOVA space(s). When IOVA manager is
+	 * initialised all of the IOVA space is marked as available
+	 * to begin with.
+	 */
+	struct list_head *free_list;
+
+	/*
+	 * Book-keeping of the user IOVA blocks in a circular double
+	 * linked list.
+	 */
+	struct list_head *reserved_list;
+
+	/* Ensuring reserve, free and the list operations are serialized.*/
+	struct mutex lock;
+
+	/* base address memory manager is configured with. */
+	u64 base_address;
+};
+
+/*
+ * Reserves a block from the free IOVA regions. Once reserved, the block
+ * is marked reserved and appended in the reserved list (no ordering
+ * required and trying to do so shall increase the time)
+ */
+int
+iova_mngr_block_reserve(void *mngr_handle, size_t size,
+			u64 *address, size_t *offset,
+			void **block_handle)
+{
+	struct mngr_ctx_t *ctx = (struct mngr_ctx_t *)(mngr_handle);
+	struct block_t *reserve = NULL, *curr = NULL, *best = NULL;
+	int ret = 0;
+
+	if (WARN_ON(!ctx || *block_handle || !size))
+		return -EINVAL;
+
+	mutex_lock(&ctx->lock);
+
+	/* if there are no free blocks to reserve. */
+	if (list_empty(ctx->free_list)) {
+		ret = -ENOMEM;
+		pr_err("(%s): No memory available to reserve block of size:(%lu)\n",
+		       ctx->name, size);
+		goto err;
+	}
+
+	/* find the best of all free bocks to reserve.*/
+	list_for_each_entry(curr, ctx->free_list, node) {
+		if (curr->size >= size) {
+			if (!best)
+				best = curr;
+			else if (curr->size < best->size)
+				best = curr;
+		}
+	}
+
+	/* if there isn't any free block of requested size. */
+	if (!best) {
+		ret = -ENOMEM;
+		pr_err("(%s): No enough mem available to reserve block sz:(%lu)\n",
+		       ctx->name, size);
+		goto err;
+	} else {
+		struct block_t *found = NULL;
+
+		/* perfect fit.*/
+		if (best->size == size) {
+			list_del(&best->node);
+			list_add_tail(&best->node, ctx->reserved_list);
+			found = best;
+		} else {
+			/* chunk out a new block, adjust the free block.*/
+			reserve = kzalloc(sizeof(*reserve), GFP_KERNEL);
+			if (WARN_ON(!reserve)) {
+				ret = -ENOMEM;
+				goto err;
+			}
+			reserve->address = best->address;
+			reserve->size = size;
+			best->address += size;
+			best->size -= size;
+			list_add_tail(&reserve->node, ctx->reserved_list);
+			found = reserve;
+		}
+		*block_handle = (void *)(found);
+
+		if (address)
+			*address = found->address;
+		if (offset)
+			*offset = (found->address - ctx->base_address);
+	}
+err:
+	mutex_unlock(&ctx->lock);
+	return ret;
+}
+
+/*
+ * Release an already reserved IOVA block/chunk by the caller back to
+ * free list.
+ */
+int
+iova_mngr_block_release(void *mngr_handle, void **block_handle)
+{
+	struct mngr_ctx_t *ctx = (struct mngr_ctx_t *)(mngr_handle);
+	struct block_t *release = (struct block_t *)(*block_handle);
+	struct block_t *curr = NULL, *prev = NULL;
+	bool done = false;
+	int ret = 0;
+
+	if (!ctx || !release)
+		return -EINVAL;
+
+	mutex_lock(&ctx->lock);
+
+	list_for_each_entry(curr, ctx->free_list, node) {
+		if (release->address < curr->address) {
+			/* if the immediate next node is available for merge.*/
+			if (curr->address == release->address + release->size) {
+				curr->address = release->address;
+				curr->size += release->size;
+				list_del(&release->node);
+				kfree(release);
+				/*
+				 * if the immediate previous node is also
+				 * available for merge.
+				 */
+				if ((prev) &&
+				    ((prev->address + prev->size)
+				      == curr->address)) {
+					prev->size += curr->size;
+					list_del(&curr->node);
+					kfree(curr);
+				}
+			} else if ((prev) &&
+				   ((prev->address + prev->size)
+				    == release->address)) {
+				/*
+				 * if only the immediate prev node is available
+				 */
+				prev->size += release->size;
+				list_del(&release->node);
+				kfree(release);
+			} else {
+				/*
+				 * cannot be merged with either the immediate
+				 * prev or the immediate next node. Add it in the
+				 * free list before the current node.
+				 */
+				list_del(&release->node);
+				list_add_tail(&release->node, &curr->node);
+			}
+			done = true;
+			break;
+		}
+		prev = curr;
+	}
+
+	/*
+	 * Even if after traversing each entry in list, we could not
+	 * add the block to be released back in free list, because:
+	 */
+	if (!done) {
+		if (!list_empty(ctx->free_list)) {
+			/*
+			 * The block to be freed has the highest order
+			 * address of all the existing blocks in free list.
+			 */
+			struct block_t *last =
+			list_last_entry(ctx->free_list, struct block_t, node);
+			if ((last->address + last->size) == release->address) {
+				/* can be merged with last node of list.*/
+				last->size += release->size;
+				list_del(&release->node);
+				kfree(release);
+			} else {
+				/* cannot be merged, add as the last node.*/
+				list_del(&release->node);
+				list_add_tail(&release->node, ctx->free_list);
+			}
+		} else {
+			/* free list was empty.*/
+			list_del(&release->node);
+			list_add_tail(&release->node, ctx->free_list);
+		}
+	}
+	*block_handle = NULL;
+
+	mutex_unlock(&ctx->lock);
+	return ret;
+}
+
+/*
+ * iova_mngr_print
+ *
+ * DEBUG only.
+ *
+ * Helper function to print all the reserved and free blocks with
+ * their names, size and start address.
+ */
+void
+iova_mngr_print(void *mngr_handle)
+{
+	struct mngr_ctx_t *ctx = (struct mngr_ctx_t *)(mngr_handle);
+	struct block_t *block = NULL;
+
+	if (ctx) {
+		mutex_lock(&ctx->lock);
+		pr_debug("(%s): Reserved\n", ctx->name);
+		list_for_each_entry(block, ctx->reserved_list, node) {
+			pr_debug("\t\t (%s): address = 0x%pa[p], size = 0x%lx\n",
+				 ctx->name, &block->address, block->size);
+		}
+		pr_debug("(%s): Free\n", ctx->name);
+		list_for_each_entry(block, ctx->free_list, node) {
+			pr_debug("\t\t (%s): address = 0x%pa[p], size = 0x%lx\n",
+				 ctx->name, &block->address, block->size);
+		}
+		mutex_unlock(&ctx->lock);
+	}
+}
+
+/*
+ * Initialises the IOVA space manager with the base address + size
+ * provided. IOVA manager would use two lists for book-keeping reserved
+ * memory blocks and free memory blocks.
+ *
+ * When initialised all of the IOVA region: base_address + size is free.
+ */
+int
+iova_mngr_init(char *name, u64 base_address, size_t size, void **mngr_handle)
+{
+	int ret = 0;
+	struct block_t *block = NULL;
+	struct mngr_ctx_t *ctx = NULL;
+
+	if (WARN_ON(!base_address || !size ||
+		    !mngr_handle || *mngr_handle || !name))
+		return -EINVAL;
+
+	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
+	if (WARN_ON(!ctx)) {
+		ret = -ENOMEM;
+		goto err;
+	}
+
+	ctx->free_list = kzalloc(sizeof(*ctx->free_list), GFP_KERNEL);
+	if (WARN_ON(!ctx->free_list)) {
+		ret = -ENOMEM;
+		goto err;
+	}
+
+	ctx->reserved_list = kzalloc(sizeof(*ctx->reserved_list), GFP_KERNEL);
+	if (WARN_ON(!ctx->reserved_list)) {
+		ret = -ENOMEM;
+		goto err;
+	}
+
+	if (strlen(name) > (NAME_MAX - 1)) {
+		ret = -EINVAL;
+		pr_err("name: (%s) long, max char:(%u)\n", name, (NAME_MAX - 1));
+		goto err;
+	}
+	strcpy(ctx->name, name);
+	INIT_LIST_HEAD(ctx->reserved_list);
+	INIT_LIST_HEAD(ctx->free_list);
+	mutex_init(&ctx->lock);
+	ctx->base_address = base_address;
+
+	/* add the base_addrss+size as one whole free block.*/
+	block = kzalloc(sizeof(*block), GFP_KERNEL);
+	if (WARN_ON(!block)) {
+		ret = -ENOMEM;
+		goto err;
+	}
+	block->address = base_address;
+	block->size = size;
+	list_add(&block->node, ctx->free_list);
+
+	*mngr_handle = ctx;
+	return ret;
+err:
+	iova_mngr_deinit((void **)(&ctx));
+	return ret;
+}
+
+/*
+ * iova_mngr_deinit
+ *
+ * deinitialize the IOVA space manager. Any blocks unreturned from the client
+ * (caller) shall become dangling.
+ */
+void
+iova_mngr_deinit(void **mngr_handle)
+{
+	struct block_t *block = NULL;
+	struct list_head *curr = NULL, *next = NULL;
+	struct mngr_ctx_t *ctx = (struct mngr_ctx_t *)(*mngr_handle);
+
+	if (ctx) {
+		/* debug only to ensure, lists do not have dangling data left.*/
+		iova_mngr_print(*mngr_handle);
+
+		/* ideally, all blocks should have returned before this.*/
+		if (!list_empty(ctx->reserved_list)) {
+			list_for_each_safe(curr, next, ctx->reserved_list) {
+				block = list_entry(curr, struct block_t, node);
+				iova_mngr_block_release(*mngr_handle,
+							(void **)(&block));
+			}
+		}
+
+		/* ideally, just one whole free block should remain as free.*/
+		list_for_each_safe(curr, next, ctx->free_list) {
+			block = list_entry(curr, struct block_t, node);
+			list_del(&block->node);
+			kfree(block);
+		}
+
+		mutex_destroy(&ctx->lock);
+		kfree(ctx->reserved_list);
+		kfree(ctx->free_list);
+		kfree(ctx);
+		*mngr_handle = NULL;
+	}
+}

drivers/misc/nvscic2c-pcie/iova-mngr.h

@@ -0,0 +1,72 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#ifndef __IOVA_MNGR_H__
+#define __IOVA_MNGR_H__
+
+#include <linux/types.h>
+
+/*
+ * iova_mngr_block_reserve
+ *
+ * Reserves a block from the free IOVA regions. Once reserved, the block
+ * is marked reserved and appended in the reserved list. Use
+ * iova_mngr_block_get_address to fetch the address of the block reserved.
+ */
+int
+iova_mngr_block_reserve(void *mngr_handle, size_t size,
+			u64 *address, size_t *offset,
+			void **block_handle);
+
+/*
+ * iova_mngr_block_release
+ *
+ * Release an already reserved IOVA block/chunk by the caller back to
+ * free list.
+ */
+int
+iova_mngr_block_release(void *mngr_handle, void **block_handle);
+
+/*
+ * iova_mngr_print
+ *
+ * DEBUG only.
+ *
+ * Helper function to print all the reserved and free blocks with
+ * their names, size and start address.
+ */
+void iova_mngr_print(void *handle);
+
+/*
+ * iova_mngr_init
+ *
+ * Initialises the IOVA space manager with the base address + size
+ * provided. IOVA manager would use two lists for book-keeping reserved
+ * memory blocks and free memory blocks.
+ *
+ * When initialised all of the IOVA region: base_address + size is free.
+ */
+int
+iova_mngr_init(char *name, u64 base_address, size_t size, void **mngr_handle);
+
+/*
+ * iova_mngr_deinit
+ *
+ * deinitialize the IOVA space manager. Any blocks unreturned from the client
+ * (caller) shall become dangling.
+ */
+void
+iova_mngr_deinit(void **handle);
+
+#endif //__IOVA_MNGR_H__

drivers/misc/nvscic2c-pcie/module.h

@@ -0,0 +1,172 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+/*
+ * Internal to nvscic2c-pcie module. This file is not supposed to be included
+ * by any other external modules.
+ */
+#ifndef __MODULE_H__
+#define __MODULE_H__
+
+#include <linux/completion.h>
+#include <linux/device.h>
+#include <linux/types.h>
+#include <linux/pci-epc.h>
+#include <linux/pci-epf.h>
+
+#include "common.h"
+
+/* forward declaration.*/
+struct device_node;
+struct platform_device;
+
+/*
+ * Parameters for the nvscic2c-pcie module and it's endpoints.
+ *
+ * These are read-only for the rest of the nvscic2c-pcie module.
+ */
+struct driver_param_t {
+	/* This is (nvscic2c-pcie) device tree node as found in device tree.*/
+	struct device_node *of_node;
+
+	/* driver mode as parsed from compatible string in device-tree.*/
+	enum drv_mode_t drv_mode;
+
+	/* nvscic2c-pcie device tree node.*/
+	struct platform_device *pdev;
+
+	/* host1x device node and pdev via phandle.*/
+	struct device_node *host1x_np;
+	struct platform_device *host1x_pdev;
+
+	/* device node of pcie controller via phandle.*/
+	struct device_node *edma_np;
+
+	/* pci endpoint device id.*/
+	u32 pci_dev_id;
+
+	/* bar window size. - applicable only for epf.*/
+	u32 bar_win_size;
+
+	/* node information, Board+SoC Id.*/
+	struct node_info_t local_node;
+	struct node_info_t peer_node;
+
+	/*
+	 * Properties that each endpoint shall be configured with.
+	 * These properties are populated from device tree node.
+	 */
+	u8 nr_endpoint;
+	struct endpoint_prop_t {
+		/* Endpoint ID as populated from device tree node. */
+		u8 id;
+
+		/*
+		 * Human readable name of the endpoint - char dev node shall be
+		 * instanced using this name.
+		 */
+		char name[NAME_MAX];
+
+		/* Frames and per frame size.*/
+		u8 nframes;
+		u32 frame_sz;
+	} endpoint_props[MAX_ENDPOINTS];
+};
+
+/* nvscic2c-pcie epf specific context. */
+struct epf_context_t {
+	bool notifier_registered;
+	struct pci_epf_header header;
+	struct pci_epf *epf;
+	void *drv_ctx;
+	struct work_struct initialization_work;
+	struct work_struct deinitialization_work;
+	atomic_t core_initialized;
+	atomic_t epf_initialized;
+	wait_queue_head_t core_initialized_waitq;
+};
+
+/* nvscic2c-pcie epc specific context. */
+struct epc_context_t {
+	struct completion epf_ready_cmpl;
+	struct completion epf_shutdown_cmpl;
+};
+
+/*
+ * nvscic2c-pcie module context.
+ * Contains all the information for all
+ *  - Configuration parameters per device-tree.
+ */
+struct driver_ctx_t {
+	/* driver mode as parsed from compatible string in device-tree.*/
+	enum drv_mode_t drv_mode;
+	char *drv_name;
+
+	/* the configuration for module and it's endpoints.*/
+	struct driver_param_t drv_param;
+
+	/*
+	 * Visible region to peer SoC for PCIe writes. In nvscic2c-pcie
+	 * use-cases, it is backed by physical memory allocated for
+	 * comm-channel, endpoints and stream-objs(mem and sync), etc.
+	 *
+	 * Peer's write lands here to be read by local/self.
+	 */
+	struct dma_buff_t self_mem;
+
+	/*
+	 * Point to peer's visible region for data-writes. This is a PCIe
+	 * aperture which allows local/self to write into peer's memory.
+	 */
+	struct pci_aper_t peer_mem;
+
+	/* pci-client abstraction handle.*/
+	void *pci_client_h;
+
+	/* comm-channel abstraction. */
+	void *comm_channel_h;
+
+	/* vmap abstraction, this can be moved within endpoints.*/
+	void *vmap_h;
+
+	/* tegra-pcie-edma module handle.*/
+	void *edma_h;
+
+	/* endpoint absraction handle.*/
+	void *endpoints_h;
+
+	/* DRV_MODE specific.*/
+	struct epf_context_t *epf_ctx;
+	struct epc_context_t *epc_ctx;
+
+	/* peer cpu */
+	enum peer_cpu_t  peer_cpu;
+
+	/* IOVA alloc abstraction.*/
+	struct iova_alloc_domain_t *ivd_h;
+};
+
+/*
+ * Look-up device tree node for the compatible string. Check for the
+ * pci-dev-id within the compatible node, if more than one such node found also
+ * return error.
+ */
+int
+dt_parse(u32 pci_dev_id, enum drv_mode_t drv_mode,
+	 struct driver_param_t *drv_param);
+
+/* Release any memory allocated during dt_parse().*/
+int
+dt_release(struct driver_param_t *drv_param);
+#endif //__MODULE_H__

drivers/misc/nvscic2c-pcie/pci-client.c

@@ -0,0 +1,693 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+#define pr_fmt(fmt)	"nvscic2c-pcie: pci-client: " fmt
+
+#include <linux/dma-buf.h>
+#include <linux/dma-iommu.h>
+#include <linux/dma-map-ops.h>
+#include <linux/errno.h>
+#include <linux/kernel.h>
+#include <linux/pci.h>
+#include <linux/pci-epc.h>
+#include <linux/pci-epf.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/types.h>
+
+#include <asm/cacheflush.h>
+
+#include <uapi/misc/nvscic2c-pcie-ioctl.h>
+#include <linux/tegra-pcie-edma.h>
+
+#include "common.h"
+#include "iova-mngr.h"
+#include "pci-client.h"
+
+/* Anticipate as many users as endpoints in worst-case. */
+#define MAX_LINK_EVENT_USERS	(MAX_ENDPOINTS)
+
+/*
+ * The size of  x86/peer edma rx descriptor
+ */
+#define EDMA_CH_DESC_SZ		(60 * SZ_1K)
+
+/* Internal private data-structure as PCI client. */
+struct pci_client_t {
+	struct device *dev;
+	struct iommu_domain *domain;
+
+	/* Recv area. Peer's write reflect here. */
+	struct dma_buff_t *self_mem;
+
+	/* Send area. PCIe aperture area. Self's Write reach Peer via this.*/
+	struct pci_aper_t *peer_mem;
+
+	/* PCI link status memory. mmap() to user-space.*/
+	atomic_t link_status;
+	struct cpu_buff_t link_status_mem;
+
+	/*
+	 * Lock to guard users getting un/registered and link status change
+	 * invocation at the same time. Also, to protect table from concurrent
+	 * access.
+	 */
+	struct mutex event_tbl_lock;
+
+	/* Table of users registered for change in PCI link status. */
+	struct event_t {
+		/* is taken.*/
+		atomic_t in_use;
+
+		/* callback to invoke when change in status is seen.*/
+		struct callback_ops cb_ops;
+	} event_tbl[MAX_LINK_EVENT_USERS];
+
+	/*
+	 * Skip reserved iova for use. This area in BAR0 aperture is reserved for
+	 * GIC SPI interrupt mechanism. As the allocation, fragmentration
+	 * of iova must be identical on both @DRV_MODE_EPF and @DRV_MODE_EPC
+	 * skip this area for use in @DRV_MODE_EPC also. We skip by reserving
+	 * the iova region and thereby marking it as unusable.
+	 */
+	dma_addr_t edma_ch_desc_iova;
+	bool edma_ch_desc_iova_mapped;
+	void *edma_ch_desc_pva;
+	void *skip_iova;
+	void *skip_meta;
+	void *edma_ch_desc_iova_h;
+	/*
+	 * iova-mngr instance for managing the reserved iova region.
+	 * application allocated objs and endpoints allocated physical memory
+	 * are pinned to this address.
+	 */
+	void *mem_mngr_h;
+
+	/*
+	 * the context of DRV_MODE_EPC/DRV_MODE_EPF
+	 */
+	struct driver_ctx_t *drv_ctx;
+
+};
+
+static void
+free_link_status_mem(struct pci_client_t *ctx)
+{
+	if (!ctx || !ctx->link_status_mem.pva)
+		return;
+
+	kfree(ctx->link_status_mem.pva);
+	ctx->link_status_mem.pva = NULL;
+}
+
+static int
+allocate_link_status_mem(struct pci_client_t *ctx)
+{
+	int ret = 0;
+	struct cpu_buff_t *mem = &ctx->link_status_mem;
+
+	mem->size = PAGE_ALIGN(sizeof(enum nvscic2c_pcie_link));
+	mem->pva  = kzalloc(mem->size, GFP_KERNEL);
+	if (WARN_ON(!mem->pva))
+		return -ENOMEM;
+
+	atomic_set(&ctx->link_status, NVSCIC2C_PCIE_LINK_DOWN);
+	*((enum nvscic2c_pcie_link *)mem->pva) = NVSCIC2C_PCIE_LINK_DOWN;
+
+	/* physical address to be mmap() in user-space.*/
+	mem->phys_addr = virt_to_phys(mem->pva);
+
+	return ret;
+}
+
+/* Free edma rx desc_iova that originally mapped from bar0 */
+static void
+free_edma_rx_desc_iova(struct pci_client_t *ctx)
+{
+	if (!ctx)
+		return;
+
+	if (ctx->skip_iova) {
+		iova_mngr_block_release(ctx->mem_mngr_h, &ctx->skip_iova);
+		ctx->skip_iova = NULL;
+	}
+
+	if (ctx->edma_ch_desc_iova_mapped) {
+		iommu_unmap(ctx->domain, ctx->edma_ch_desc_iova, EDMA_CH_DESC_SZ);
+		ctx->edma_ch_desc_iova_mapped = false;
+	}
+
+	if (ctx->edma_ch_desc_iova) {
+		iova_mngr_block_release(ctx->mem_mngr_h,
+					&ctx->edma_ch_desc_iova_h);
+		ctx->edma_ch_desc_iova_h = NULL;
+	}
+
+	if (ctx->edma_ch_desc_pva) {
+		free_pages_exact(ctx->edma_ch_desc_pva, EDMA_CH_DESC_SZ);
+		ctx->edma_ch_desc_pva = NULL;
+	}
+
+	if (ctx->skip_meta) {
+		iova_mngr_block_release(ctx->mem_mngr_h, &ctx->skip_meta);
+		ctx->skip_meta = NULL;
+	}
+}
+
+/* Allocate desc_iova and mapping to bar0 for remote edma, x86-orin c2c only */
+static int
+allocate_edma_rx_desc_iova(struct pci_client_t *ctx)
+{
+	int ret = 0;
+	int prot = 0;
+	u64 phys_addr = 0;
+
+	if (WARN_ON(!ctx))
+		return -EINVAL;
+
+	/*
+	 *bar0 first 128K [-------128k-------]
+	 *                [-4k-][-60k-][-64k-]
+	 *first 4K reserved for meta data communication
+	 *next 60k for x86/peer edma rx descriptor
+	 *next 64K resered for sys-sw
+	 */
+	ret = iova_mngr_block_reserve(ctx->mem_mngr_h, SZ_4K,
+				      NULL, NULL, &ctx->skip_meta);
+	if (ret) {
+		pr_err("Failed to skip the 4K reserved iova region\n");
+		goto err;
+	}
+	ctx->edma_ch_desc_pva = alloc_pages_exact(EDMA_CH_DESC_SZ, (GFP_KERNEL | __GFP_ZERO));
+	if (!ctx->edma_ch_desc_pva) {
+		pr_err("Failed to allocate a page with size of 60K\n");
+		ret = -ENOMEM;
+		goto err;
+	}
+	phys_addr = page_to_phys(virt_to_page(ctx->edma_ch_desc_pva));
+	if (!phys_addr) {
+		pr_err("Failed to retrieve physical address\n");
+		ret = -ENOMEM;
+		goto err;
+	}
+	ret = iova_mngr_block_reserve(ctx->mem_mngr_h, EDMA_CH_DESC_SZ,
+				      &ctx->edma_ch_desc_iova, NULL,
+				      &ctx->edma_ch_desc_iova_h);
+	if (ret) {
+		pr_err("Failed to reserve 60K iova space for remote edma desc\n");
+		goto err;
+	}
+	prot = (IOMMU_CACHE | IOMMU_READ | IOMMU_WRITE);
+	ret = iommu_map(ctx->domain, ctx->edma_ch_desc_iova, phys_addr, EDMA_CH_DESC_SZ, prot);
+	if (ret) {
+		pr_err("pci client failed to map iova to 60K physical backing\n");
+		goto err;
+	}
+	ctx->edma_ch_desc_iova_mapped = true;
+
+	/* bar0+64K - bar0+126K  reserved for sys-sw  */
+	ret = iova_mngr_block_reserve(ctx->mem_mngr_h, SZ_64K,
+				      NULL, NULL, &ctx->skip_iova);
+	if (ret) {
+		pr_err("Failed to skip the 64K reserved iova region\n");
+		goto err;
+	}
+	return ret;
+
+err:
+	free_edma_rx_desc_iova(ctx);
+	return ret;
+}
+
+int
+pci_client_init(struct pci_client_params *params, void **pci_client_h)
+{
+	u32 i = 0;
+	int ret = 0;
+	struct pci_client_t *ctx = NULL;
+
+	/* should not be an already instantiated pci client context. */
+	if (WARN_ON(!pci_client_h || *pci_client_h ||
+		    !params || !params->self_mem || !params->peer_mem ||
+		    !params->dev))
+		return -EINVAL;
+
+	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
+	if (WARN_ON(!ctx))
+		return -ENOMEM;
+	ctx->dev = params->dev;
+	ctx->self_mem = params->self_mem;
+	ctx->peer_mem = params->peer_mem;
+	mutex_init(&ctx->event_tbl_lock);
+
+	/* for link event notifications. */
+	for (i = 0; i < MAX_LINK_EVENT_USERS; i++)
+		atomic_set(&ctx->event_tbl[i].in_use, 0);
+
+	ret = allocate_link_status_mem(ctx);
+	if (ret)
+		goto err;
+
+	/*
+	 * for mapping application objs and endpoint physical memory to remote
+	 * visible area.
+	 */
+	ctx->domain = iommu_get_domain_for_dev(ctx->dev);
+	if (WARN_ON(!ctx->domain)) {
+		ret = -ENODEV;
+		pr_err("iommu not available for the pci device\n");
+		goto err;
+	}
+
+	/* assumption : PCIe to be fully IO Coherent. Validate the assumption.*/
+	if (WARN_ON(!dev_is_dma_coherent(ctx->dev))) {
+		ret = -ENODEV;
+		pr_err("io-coherency not enabled for the pci device\n");
+		goto err;
+	}
+
+	/*
+	 * configure iova manager for inbound/self_mem. Application
+	 * supplied objs shall be pinned to this area.
+	 */
+	ret = iova_mngr_init("self_mem",
+			     ctx->self_mem->dma_handle, ctx->self_mem->size,
+			     &ctx->mem_mngr_h);
+	if (ret) {
+		pr_err("Failed to initialize iova memory manager\n");
+		goto err;
+	}
+
+	/*
+	 * Skip reserved iova for any use. This area in BAR0 is reserved for
+	 * GIC SPI interrupt mechanism. As the allocation, fragmentration
+	 * of iova must be identical on both @DRV_MODE_EPF and @DRV_MODE_EPC
+	 * skip this area for use in @DRV_MODE_EPC also. We skip by reserving
+	 * the iova region and thereby marking it as unusable for others.
+	 */
+	/* remote edma on x86 */
+	ret = allocate_edma_rx_desc_iova(ctx);
+	if (ret) {
+		pr_err("Failed to skip the reserved iova region\n");
+		goto err;
+	}
+
+	*pci_client_h = ctx;
+	return ret;
+
+err:
+	pci_client_deinit((void **)&ctx);
+	return ret;
+}
+
+void
+pci_client_deinit(void **pci_client_h)
+{
+	struct pci_client_t *ctx = (struct pci_client_t *)(*pci_client_h);
+
+	if (!ctx)
+		return;
+
+	free_edma_rx_desc_iova(ctx);
+
+	if (ctx->mem_mngr_h) {
+		iova_mngr_deinit(&ctx->mem_mngr_h);
+		ctx->mem_mngr_h = NULL;
+	}
+
+	free_link_status_mem(ctx);
+	mutex_destroy(&ctx->event_tbl_lock);
+	kfree(ctx);
+
+	*pci_client_h = NULL;
+}
+
+int
+pci_client_alloc_iova(void *pci_client_h, size_t size, u64 *iova,
+		      size_t *offset, void **block_h)
+{
+	struct pci_client_t *ctx = (struct pci_client_t *)pci_client_h;
+
+	if (WARN_ON(!ctx))
+		return -EINVAL;
+
+	return iova_mngr_block_reserve(ctx->mem_mngr_h, size,
+				       iova, offset, block_h);
+}
+
+int
+pci_client_free_iova(void *pci_client_h, void **block_h)
+{
+	struct pci_client_t *ctx = (struct pci_client_t *)pci_client_h;
+
+	if (!ctx || !block_h)
+		return -EINVAL;
+
+	return iova_mngr_block_release(ctx->mem_mngr_h, block_h);
+}
+
+int
+pci_client_map_addr(void *pci_client_h, u64 to_iova, phys_addr_t paddr,
+		    size_t size, int prot)
+{
+	struct pci_client_t *ctx = (struct pci_client_t *)pci_client_h;
+
+	if (WARN_ON(!ctx || !to_iova || !paddr || !size))
+		return -EINVAL;
+
+	return iommu_map(ctx->domain, to_iova, paddr, size, prot);
+}
+
+size_t
+pci_client_unmap_addr(void *pci_client_h, u64 from_iova, size_t size)
+{
+	struct pci_client_t *ctx = (struct pci_client_t *)pci_client_h;
+
+	if (!ctx || !ctx->domain)
+		return 0;
+
+	return iommu_unmap(ctx->domain, from_iova, size);
+}
+
+int
+pci_client_get_peer_aper(void *pci_client_h, size_t offsetof, size_t size,
+			 phys_addr_t *phys_addr)
+{
+	struct pci_client_t *ctx = (struct pci_client_t *)pci_client_h;
+
+	if (WARN_ON(!ctx || !size || !phys_addr))
+		return -EINVAL;
+
+	if (ctx->peer_mem->size < (offsetof + size))
+		return -ENOMEM;
+
+	*phys_addr = ctx->peer_mem->aper + offsetof;
+
+	return 0;
+}
+
+struct dma_buf_attachment *
+pci_client_dmabuf_attach(void *pci_client_h, struct dma_buf *dmabuff)
+{
+	struct pci_client_t *ctx = (struct pci_client_t *)pci_client_h;
+
+	if (WARN_ON(!ctx || !dmabuff))
+		return ERR_PTR(-EINVAL);
+
+	return dma_buf_attach(dmabuff, ctx->dev);
+}
+
+void
+pci_client_dmabuf_detach(void *pci_client_h, struct dma_buf *dmabuff,
+			 struct dma_buf_attachment *attach)
+{
+	struct pci_client_t *ctx = (struct pci_client_t *)pci_client_h;
+
+	if (!ctx || !dmabuff || !attach)
+		return;
+
+	return dma_buf_detach(dmabuff, attach);
+}
+
+/* Helper function to mmap the PCI link status memory to user-space.*/
+int
+pci_client_mmap_link_mem(void *pci_client_h, struct vm_area_struct *vma)
+{
+	int ret = 0;
+	struct pci_client_t *ctx = (struct pci_client_t *)pci_client_h;
+
+	if (WARN_ON(!vma))
+		return -EINVAL;
+
+	if (WARN_ON(!ctx || !ctx->link_status_mem.pva))
+		return -EINVAL;
+
+	if ((vma->vm_end - vma->vm_start) != ctx->link_status_mem.size)
+		return -EINVAL;
+
+	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+	ret = remap_pfn_range(vma,
+			      vma->vm_start,
+			      PFN_DOWN(ctx->link_status_mem.phys_addr),
+			      ctx->link_status_mem.size,
+			      vma->vm_page_prot);
+	if (ret)
+		pr_err("remap_pfn_range returns error: (%d) for Link mem\n", ret);
+
+	return ret;
+}
+
+/* Query PCI link status. */
+enum nvscic2c_pcie_link
+pci_client_query_link_status(void *pci_client_h)
+{
+	struct pci_client_t *ctx = (struct pci_client_t *)pci_client_h;
+
+	if (WARN_ON(!ctx))
+		return NVSCIC2C_PCIE_LINK_DOWN;
+
+	return atomic_read(&ctx->link_status);
+}
+
+/*
+ * Users/Units can register for PCI link events as received by
+ * @@DRV_MODE_EPF or @DRV_MODE_EPC module sbstraction.
+ */
+int
+pci_client_register_for_link_event(void *pci_client_h,
+				   struct callback_ops *ops, u32 *id)
+{
+	u32 i = 0;
+	int ret = 0;
+	struct event_t *event = NULL;
+	struct pci_client_t *ctx = (struct pci_client_t *)pci_client_h;
+
+	if (WARN_ON(!ctx))
+		return -EINVAL;
+
+	if (WARN_ON(!id || !ops || !ops->callback))
+		return -EINVAL;
+
+	mutex_lock(&ctx->event_tbl_lock);
+	for (i = 0; i < MAX_LINK_EVENT_USERS; i++) {
+		event = &ctx->event_tbl[i];
+		if (!atomic_read(&event->in_use)) {
+			event->cb_ops.callback = ops->callback;
+			event->cb_ops.ctx = ops->ctx;
+			atomic_set(&event->in_use, 1);
+			*id = i;
+			break;
+		}
+	}
+	if (i == MAX_LINK_EVENT_USERS) {
+		ret = -ENOMEM;
+		pr_err("PCI link event registration full\n");
+	}
+	mutex_unlock(&ctx->event_tbl_lock);
+
+	return ret;
+}
+
+/* Unregister for PCI link events. - teardown only. */
+int
+pci_client_unregister_for_link_event(void *pci_client_h, u32 id)
+{
+	int ret = 0;
+	struct event_t *event = NULL;
+	struct pci_client_t *ctx = (struct pci_client_t *)pci_client_h;
+
+	if (WARN_ON(!ctx))
+		return -EINVAL;
+
+	if (WARN_ON(id >= MAX_LINK_EVENT_USERS))
+		return -EINVAL;
+
+	mutex_lock(&ctx->event_tbl_lock);
+	event = &ctx->event_tbl[id];
+	if (atomic_read(&event->in_use)) {
+		atomic_set(&event->in_use, 0);
+		event->cb_ops.callback = NULL;
+		event->cb_ops.ctx = NULL;
+	}
+	mutex_unlock(&ctx->event_tbl_lock);
+
+	return ret;
+}
+
+/*
+ * Update the PCI link status as received in @DRV_MODE_EPF or @DRV_MODE_EPC
+ * module abstraction. Propagate the link status to registered users.
+ */
+int
+pci_client_change_link_status(void *pci_client_h,
+			      enum nvscic2c_pcie_link status)
+{
+	u32 i = 0;
+	int ret = 0;
+	struct page *page = NULL;
+	struct event_t *event = NULL;
+	struct callback_ops *ops = NULL;
+	struct pci_client_t *ctx = (struct pci_client_t *)pci_client_h;
+
+	if (WARN_ON(!ctx))
+		return -EINVAL;
+
+	if (WARN_ON(status < NVSCIC2C_PCIE_LINK_DOWN ||
+		    status > NVSCIC2C_PCIE_LINK_UP))
+		return -EINVAL;
+
+	/*
+	 * Reflect the status for user-space to see.
+	 * For consistent view of same phys_addr by user-space, flush the update
+	 * Call is arm64 specific.
+	 */
+	atomic_set(&ctx->link_status, status);
+	*((enum nvscic2c_pcie_link *)ctx->link_status_mem.pva) = status;
+	page = virt_to_page(ctx->link_status_mem.pva);
+	flush_dcache_page(page);
+
+	/* interrupt registered users. */
+	mutex_lock(&ctx->event_tbl_lock);
+	for (i = 0; i < MAX_LINK_EVENT_USERS; i++) {
+		event = &ctx->event_tbl[i];
+		if (atomic_read(&event->in_use)) {
+			ops = &event->cb_ops;
+			ops->callback(NULL, ops->ctx);
+		}
+	}
+	mutex_unlock(&ctx->event_tbl_lock);
+
+	return ret;
+}
+
+/*
+ * Helper functions to set and get driver context from  pci_client t
+ *
+ */
+/*Set driver context of DRV_MODE_EPF or DRV_MODE_EPC */
+int
+pci_client_save_driver_ctx(void *pci_client_h, struct driver_ctx_t *drv_ctx)
+{
+	int ret = 0;
+	struct pci_client_t *pci_client_ctx = (struct pci_client_t *)pci_client_h;
+
+	if (WARN_ON(!pci_client_ctx))
+		return -EINVAL;
+	if (WARN_ON(!drv_ctx))
+		return -EINVAL;
+	pci_client_ctx->drv_ctx = drv_ctx;
+	return ret;
+}
+
+/*Get the driver context of DRV_MODE_EPF or DRV_MODE_EPC */
+struct driver_ctx_t *
+pci_client_get_driver_ctx(void *pci_client_h)
+{
+	struct pci_client_t *pci_client_ctx = (struct pci_client_t *)pci_client_h;
+	struct driver_ctx_t *drv_ctx = NULL;
+
+	if (WARN_ON(!pci_client_ctx))
+		return NULL;
+	drv_ctx = pci_client_ctx->drv_ctx;
+	if (WARN_ON(!drv_ctx))
+		return NULL;
+	return drv_ctx;
+}
+
+/* get drv mode  */
+enum drv_mode_t
+pci_client_get_drv_mode(void *pci_client_h)
+{
+	struct pci_client_t *pci_client_ctx = (struct pci_client_t *)pci_client_h;
+	struct driver_ctx_t *drv_ctx = NULL;
+
+	if (WARN_ON(!pci_client_ctx))
+		return DRV_MODE_INVALID;
+	drv_ctx = pci_client_ctx->drv_ctx;
+	if (WARN_ON(!drv_ctx))
+		return NVCPU_MAXIMUM;
+	return drv_ctx->drv_mode;
+}
+
+/* save the peer cup orin/x86_64 */
+int
+pci_client_save_peer_cpu(void *pci_client_h, enum peer_cpu_t peer_cpu)
+{
+	int ret = 0;
+	struct pci_client_t *pci_client_ctx = (struct pci_client_t *)pci_client_h;
+	struct driver_ctx_t *drv_ctx = NULL;
+
+	if (WARN_ON(!pci_client_ctx))
+		return -EINVAL;
+	drv_ctx = pci_client_ctx->drv_ctx;
+	if (WARN_ON(!drv_ctx))
+		return -EINVAL;
+	drv_ctx->peer_cpu = peer_cpu;
+	return ret;
+}
+
+/* get the peer cup orin/x86_64 */
+enum peer_cpu_t
+pci_client_get_peer_cpu(void *pci_client_h)
+{
+	struct pci_client_t *pci_client_ctx = (struct pci_client_t *)pci_client_h;
+	struct driver_ctx_t *drv_ctx = NULL;
+
+	if (WARN_ON(!pci_client_ctx))
+		return NVCPU_MAXIMUM;
+	drv_ctx = pci_client_ctx->drv_ctx;
+	if (WARN_ON(!drv_ctx))
+		return NVCPU_MAXIMUM;
+	return drv_ctx->peer_cpu;
+}
+
+/* get  the iova allocated for x86 peer tegra-pcie-emda rx descriptor   */
+dma_addr_t
+pci_client_get_edma_rx_desc_iova(void *pci_client_h)
+{
+	struct pci_client_t *ctx = (struct pci_client_t *)pci_client_h;
+
+	if (ctx)
+		return ctx->edma_ch_desc_iova;
+	else
+		return 0;
+}
+
+int
+pci_client_raise_irq(void *pci_client_h, enum pci_epc_irq_type type, u16 num)
+{
+	int ret = 0;
+	struct pci_client_t *pci_client_ctx = (struct pci_client_t *)pci_client_h;
+	struct driver_ctx_t *drv_ctx = NULL;
+	struct epf_context_t *epf_ctx = NULL;
+
+	if (WARN_ON(!pci_client_ctx))
+		return -EINVAL;
+
+	drv_ctx = pci_client_ctx->drv_ctx;
+	if (WARN_ON(!drv_ctx))
+		return -EINVAL;
+
+	epf_ctx = drv_ctx->epf_ctx;
+	if (WARN_ON(!epf_ctx))
+		return -EINVAL;
+	if (WARN_ON(!epf_ctx->epf))
+		return -EINVAL;
+	if (WARN_ON(drv_ctx->drv_mode != DRV_MODE_EPF))
+		return -EINVAL;
+
+	ret = pci_epc_raise_irq(epf_ctx->epf->epc, epf_ctx->epf->func_no,
+				PCIE_VFNO, type, num);
+	return ret;
+}
+

drivers/misc/nvscic2c-pcie/pci-client.h

@@ -0,0 +1,136 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#ifndef __PCI_CLIENT_H__
+#define __PCI_CLIENT_H__
+
+#include <uapi/misc/nvscic2c-pcie-ioctl.h>
+
+#include "common.h"
+#include "module.h"
+
+/* forward declaration.*/
+struct vm_area_struct;
+struct dma_buf;
+struct dma_buf_attachment;
+
+/*
+ * PCI client initialization parameters. The fields must remain persistent
+ * till deinitialization (exit).
+ */
+struct pci_client_params {
+	struct dma_buff_t *self_mem;
+	struct pci_aper_t *peer_mem;
+	/*
+	 * @DRV_MODE_EPC: &pci_dev->dev
+	 * @DRV_MODE_EPF: epf->epc->dev.parent.
+	 */
+	struct device *dev;
+};
+
+/* Initialize PCI client either for @DRV_MODE_EPF or @DRV_MODE_EPC. */
+int
+pci_client_init(struct pci_client_params *params, void **pci_client_h);
+
+/* Teardown of PCI client. */
+void
+pci_client_deinit(void **pci_client_h);
+
+/* reserve iova using the iova-manager.*/
+int
+pci_client_alloc_iova(void *pci_client_h, size_t size, u64 *address,
+		      size_t *offset, void **block_h);
+
+/* free the reserved iova.*/
+int
+pci_client_free_iova(void *pci_client_h, void **block_h);
+
+int
+pci_client_map_addr(void *pci_client_h, u64 to_iova, phys_addr_t paddr,
+		    size_t size, int prot);
+
+size_t
+pci_client_unmap_addr(void *pci_client_h, u64 from_iova, size_t size);
+
+/* get the pci aperture for a given offset.*/
+int
+pci_client_get_peer_aper(void *pci_client_h, size_t offsetof, size_t size,
+			 phys_addr_t *phys_addr);
+
+/* attach dma-buf to pci device.*/
+struct dma_buf_attachment *
+pci_client_dmabuf_attach(void *pci_client_h, struct dma_buf *dmabuff);
+
+/* detach dma-buf to pci device.*/
+void
+pci_client_dmabuf_detach(void *pci_client_h, struct dma_buf *dmabuff,
+			 struct dma_buf_attachment *attach);
+/*
+ * Users/Units can register for PCI link events as received by
+ * @DRV_MODE_EPF or @DRV_MODE_EPC module sbstraction.
+ */
+int
+pci_client_register_for_link_event(void *pci_client_h,
+				   struct callback_ops *ops, u32 *id);
+
+/* Unregister for PCI link events. - teardown only. */
+int
+pci_client_unregister_for_link_event(void *pci_client_h, u32 id);
+
+/*
+ * Update the PCI link status as received in @DRV_MODE_EPF or @DRV_MODE_EPC
+ * module abstraction. Propagate the link status to registered users.
+ */
+int
+pci_client_change_link_status(void *pci_client_h,
+			      enum nvscic2c_pcie_link status);
+
+/* Helper function to mmap the PCI link status memory to user-space.*/
+int
+pci_client_mmap_link_mem(void *pci_client_h, struct vm_area_struct *vma);
+
+/* Query PCI link status. */
+enum nvscic2c_pcie_link
+pci_client_query_link_status(void *pci_client_h);
+
+/*
+ * Helper functions to set and set driver context from pci_client_h
+ */
+
+/* Save driver context of DRV_MODE_EPF or DRV_MODE_EPC */
+int
+pci_client_save_driver_ctx(void *pci_client_h, struct driver_ctx_t *drv_ctx);
+
+/* Getter driver context of DRV_MODE_EPF or DRV_MODE_EPC */
+struct driver_ctx_t *
+pci_client_get_driver_ctx(void *pci_client_h);
+/*getter drv mode   */
+enum drv_mode_t
+pci_client_get_drv_mode(void *pci_client_h);
+/* save peer cpu sent by boottrap msg  */
+int
+pci_client_save_peer_cpu(void *pci_client_h, enum peer_cpu_t peer_cpu);
+
+/* Getter the soc/arch of rp */
+enum peer_cpu_t
+pci_client_get_peer_cpu(void *pci_client_h);
+
+/* Get allocated edma rx desc iova  */
+dma_addr_t
+pci_client_get_edma_rx_desc_iova(void *pci_client_h);
+
+/* pci client raise irq to rp */
+int
+pci_client_raise_irq(void *pci_client_h, enum pci_epc_irq_type type, u16 num);
+#endif // __PCI_CLIENT_H__

drivers/misc/nvscic2c-pcie/stream-extensions.h

@@ -0,0 +1,51 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+/*
+ * Internal to gos-nvscic2c module. This file is not supposed to be included
+ * by any other external modules.
+ */
+#ifndef __STREAM_EXTENSIONS_H__
+#define __STREAM_EXTENSIONS_H__
+
+#include <linux/types.h>
+
+#include "common.h"
+
+/* forward declaration. */
+struct driver_ctx_t;
+
+/* params to instantiate a stream-extension instance.*/
+struct stream_ext_params {
+	struct node_info_t *local_node;
+	struct node_info_t *peer_node;
+	u32 ep_id;
+	char *ep_name;
+	struct platform_device *host1x_pdev;
+	enum drv_mode_t drv_mode;
+	void *pci_client_h;
+	void *comm_channel_h;
+	void *vmap_h;
+	void *edma_h;
+};
+
+int
+stream_extension_ioctl(void *stream_ext_h, unsigned int cmd, void *arg);
+
+int
+stream_extension_init(struct stream_ext_params *params, void **handle);
+
+void
+stream_extension_deinit(void **handle);
+#endif //__STREAM_EXTENSIONS_H__

drivers/misc/nvscic2c-pcie/vmap-internal.h

@@ -0,0 +1,190 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#ifndef __VMAP_INTERNAL_H__
+#define __VMAP_INTERNAL_H__
+
+#include <linux/dma-iommu.h>
+#include <linux/dma-buf.h>
+#include <linux/pci.h>
+
+#include "common.h"
+#include "vmap.h"
+
+/* forward declaration. */
+struct vmap_ctx_t;
+
+struct memobj_pin_t {
+	/* Input param fd -> dma_buf to be mapped.*/
+	struct dma_buf *dmabuf;
+
+	enum vmap_mngd mngd;
+	enum vmap_obj_prot prot;
+	enum vmap_obj_type type;
+
+	/* Input dmabuf mapped to pci-dev(dev mngd) or dummy dev(client mngd).*/
+	struct dma_buf_attachment *attach;
+	struct sg_table *sgt;
+	enum dma_data_direction dir;
+
+	/*
+	 * [OUT]contiguous iova region obtained from client (iova-mngr).
+	 * which input dmabuf is mapped to.
+	 */
+	void *iova_block_h;
+	struct vmap_obj_attributes attrib;
+
+	/*
+	 * [OUT]
+	 * Per scatter-list nent mapping -  used during free.
+	 * Used for client-managed map only.
+	 */
+	u32 nr_nents;
+	struct iova_nent {
+		u64 iova;
+		size_t len;
+		bool mapped_iova;
+	} *nents;
+};
+
+struct syncobj_pin_t {
+	s32 fd;
+	u32 syncpt_id;
+	struct host1x_syncpt *sp;
+	phys_addr_t phy_addr;
+
+	enum vmap_mngd mngd;
+	enum vmap_obj_prot prot;
+	enum vmap_obj_type type;
+
+	/* local sync objs do not require pinning to pcie address space.*/
+	bool pin_reqd;
+
+	/*
+	 * [OUT]contiguous iova region obtained from client (iova-mngr)
+	 * which syncpoint shim aper is mapped to.
+	 */
+	void *iova_block_h;
+	struct vmap_obj_attributes attrib;
+	bool mapped_iova;
+};
+
+struct importobj_reg_t {
+	/*
+	 * export descriptor and whereabouts of exported obj
+	 * as received from remote end.
+	 */
+	u64 export_desc;
+
+	/* times exported by remote, imported by local.*/
+	u32 nr_export;
+	u32 nr_import;
+
+	struct vmap_obj_attributes attrib;
+};
+
+/* virtual mapping information for Mem obj.*/
+struct memobj_map_ref {
+	s32 obj_id;
+	struct kref refcount;
+	struct memobj_pin_t pin;
+	struct vmap_ctx_t *vmap_ctx;
+};
+
+/* virtual mapping information for Sync obj. */
+struct syncobj_map_ref {
+	s32 obj_id;
+	struct kref refcount;
+	struct syncobj_pin_t pin;
+	struct vmap_ctx_t *vmap_ctx;
+};
+
+/* virtual mapping information for Imported obj. */
+struct importobj_map_ref {
+	s32 obj_id;
+	struct kref refcount;
+	struct importobj_reg_t reg;
+	struct vmap_ctx_t *vmap_ctx;
+};
+
+/* vmap subunit/abstraction context. */
+struct vmap_ctx_t {
+	/* pci-client abstraction handle.*/
+	void *pci_client_h;
+
+	/* comm-channel abstraction. */
+	void *comm_channel_h;
+
+	/* host1x platform device for syncpoint interfaces.*/
+	struct platform_device *host1x_pdev;
+
+	/*
+	 * dummy platform device. - This has smmu disabled to get the
+	 * physical addresses of exported Mem objects when using client
+	 * managed mapping.
+	 */
+	struct platform_device *dummy_pdev;
+	bool dummy_pdev_init;
+
+	/*
+	 * Management of Mem/Sync object Ids.
+	 *
+	 * All objects mapped are identified by - pin_id. IDR mechanism
+	 * generates these IDs. We maintain separate book-keeping for
+	 * Mem, Sync and Import objects. The ID shall overalap between
+	 * Mem, Sync and Import objects.
+	 *
+	 * ID is the pinned handle returned to other units.
+	 */
+	struct idr mem_idr;
+	struct idr sync_idr;
+	struct idr import_idr;
+
+	/* exclusive access to mem idr.*/
+	struct mutex mem_idr_lock;
+	/* exclusive access to sync idr.*/
+	struct mutex sync_idr_lock;
+	/* exclusive access to import idr.*/
+	struct mutex import_idr_lock;
+};
+
+void
+memobj_devmngd_unpin(struct vmap_ctx_t *vmap_ctx,
+		     struct memobj_pin_t *pin);
+int
+memobj_devmngd_pin(struct vmap_ctx_t *vmap_ctx,
+		   struct memobj_pin_t *pin);
+void
+memobj_clientmngd_unpin(struct vmap_ctx_t *vmap_ctx,
+			struct memobj_pin_t *pin);
+int
+memobj_clientmngd_pin(struct vmap_ctx_t *vmap_ctx,
+		      struct memobj_pin_t *pin);
+void
+memobj_unpin(struct vmap_ctx_t *vmap_ctx,
+	     struct memobj_pin_t *pin);
+int
+memobj_pin(struct vmap_ctx_t *vmap_ctx,
+	   struct memobj_pin_t *pin);
+void
+syncobj_clientmngd_unpin(struct vmap_ctx_t *vmap_ctx,
+			 struct syncobj_pin_t *pin);
+void
+syncobj_unpin(struct vmap_ctx_t *vmap_ctx,
+	      struct syncobj_pin_t *pin);
+int
+syncobj_pin(struct vmap_ctx_t *vmap_ctx,
+	    struct syncobj_pin_t *pin);
+
+#endif //__VMAP_INTERNAL_H__

drivers/misc/nvscic2c-pcie/vmap-pin.c

@@ -0,0 +1,391 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#define pr_fmt(fmt)	"nvscic2c-pcie: vmap-pin: " fmt
+
+#include <linux/device.h>
+#include <linux/dma-iommu.h>
+#include <linux/dma-buf.h>
+#include <linux/errno.h>
+#include <linux/host1x-next.h>
+#include <linux/kernel.h>
+#include <linux/pci.h>
+#include <linux/platform_device.h>
+#include <linux/printk.h>
+#include <linux/types.h>
+#include <drm/tegra_drm-next.h>
+
+#include "common.h"
+#include "module.h"
+#include "pci-client.h"
+#include "vmap.h"
+#include "vmap-internal.h"
+
+void
+memobj_devmngd_unpin(struct vmap_ctx_t *vmap_ctx,
+		     struct memobj_pin_t *pin)
+{
+	if (!vmap_ctx)
+		return;
+
+	if (!pin)
+		return;
+
+	if (!(IS_ERR_OR_NULL(pin->sgt))) {
+		dma_buf_unmap_attachment(pin->attach, pin->sgt,	pin->dir);
+		pin->sgt = NULL;
+	}
+
+	if (!(IS_ERR_OR_NULL(pin->attach))) {
+		pci_client_dmabuf_detach(vmap_ctx->pci_client_h,
+					 pin->dmabuf, pin->attach);
+		pin->attach = NULL;
+	}
+}
+
+int
+memobj_devmngd_pin(struct vmap_ctx_t *vmap_ctx,
+		   struct memobj_pin_t *pin)
+{
+	int ret = 0;
+	u32 sg_index = 0;
+	struct scatterlist *sg = NULL;
+
+	if (pin->prot == VMAP_OBJ_PROT_WRITE)
+		pin->dir = DMA_FROM_DEVICE;
+	else
+		pin->dir = DMA_TO_DEVICE;
+
+	pin->attach = pci_client_dmabuf_attach(vmap_ctx->pci_client_h,
+					       pin->dmabuf);
+	if (IS_ERR_OR_NULL(pin->attach)) {
+		ret = PTR_ERR(pin->attach);
+		goto err;
+	}
+
+	pin->sgt = dma_buf_map_attachment(pin->attach, pin->dir);
+	if (IS_ERR_OR_NULL(pin->sgt)) {
+		ret = PTR_ERR(pin->sgt);
+		goto err;
+	}
+
+	/* dma address (for all nents) are deemed contiguous for smmu=on.*/
+	pin->attrib.iova = sg_dma_address(pin->sgt->sgl);
+	for_each_sg(pin->sgt->sgl, sg, pin->sgt->nents, sg_index) {
+		pin->attrib.size += sg->length;
+	}
+
+	/*
+	 * dev mngd used in local mem or remote mem (when exporting from
+	 * Tegra PCIe RP), in both cases, offsetof is not needed.
+	 */
+	pin->attrib.offsetof = 0;
+
+	return ret;
+err:
+	memobj_devmngd_unpin(vmap_ctx, pin);
+	return ret;
+}
+
+void
+memobj_clientmngd_unpin(struct vmap_ctx_t *vmap_ctx,
+			struct memobj_pin_t *pin)
+{
+	u32 i = 0;
+
+	if (!vmap_ctx)
+		return;
+
+	if (!pin)
+		return;
+
+	if (pin->nents) {
+		for (i = 0; i < pin->nr_nents; i++) {
+			if (pin->nents[i].mapped_iova) {
+				pci_client_unmap_addr(vmap_ctx->pci_client_h,
+						      pin->nents[i].iova,
+						      pin->nents[i].len);
+				pin->nents[i].mapped_iova = false;
+			}
+		}
+		kfree(pin->nents);
+		pin->nents = NULL;
+	}
+
+	if (pin->iova_block_h) {
+		pci_client_free_iova(vmap_ctx->pci_client_h,
+				     &pin->iova_block_h);
+		pin->iova_block_h = NULL;
+	}
+
+	if (!(IS_ERR_OR_NULL(pin->sgt))) {
+		dma_buf_unmap_attachment(pin->attach, pin->sgt, pin->dir);
+		pin->sgt = NULL;
+	}
+
+	if (!(IS_ERR_OR_NULL(pin->attach))) {
+		dma_buf_detach(pin->dmabuf, pin->attach);
+		pin->attach = NULL;
+	}
+}
+
+int
+memobj_clientmngd_pin(struct vmap_ctx_t *vmap_ctx,
+		      struct memobj_pin_t *pin)
+{
+	int ret = 0;
+	u64 iova = 0;
+	u32 sg_index = 0;
+	int prot = IOMMU_WRITE;
+	struct scatterlist *sg = NULL;
+
+	if (pin->prot == VMAP_OBJ_PROT_WRITE) {
+		prot = IOMMU_WRITE;
+		pin->dir = DMA_FROM_DEVICE;
+	} else {
+		prot = IOMMU_READ;
+		pin->dir = DMA_TO_DEVICE;
+	}
+
+	/*
+	 * pin to dummy device (which has smmu disabled) to get scatter-list
+	 * of phys addr.
+	 */
+	pin->attach = dma_buf_attach(pin->dmabuf, &vmap_ctx->dummy_pdev->dev);
+	if (IS_ERR_OR_NULL(pin->attach)) {
+		ret = PTR_ERR(pin->attach);
+		pr_err("client_mngd dma_buf_attach failed\n");
+		goto err;
+	}
+	pin->sgt = dma_buf_map_attachment(pin->attach, pin->dir);
+	if (IS_ERR_OR_NULL(pin->sgt)) {
+		ret = PTR_ERR(pin->sgt);
+		pr_err("client_mngd dma_buf_attachment failed\n");
+		goto err;
+	}
+
+	for_each_sg(pin->sgt->sgl, sg, pin->sgt->nents, sg_index)
+		pin->attrib.size += sg->length;
+
+	/* get one contiguous iova.*/
+	ret = pci_client_alloc_iova(vmap_ctx->pci_client_h, pin->attrib.size,
+				    &pin->attrib.iova, &pin->attrib.offsetof,
+				    &pin->iova_block_h);
+	if (ret) {
+		pr_err("Failed to reserve iova block of size: (%lu)\n",
+		       pin->attrib.size);
+		goto err;
+	}
+
+	/* pin the scatter list to contiguous iova.*/
+	pin->nr_nents = pin->sgt->nents;
+	pin->nents = kzalloc((sizeof(*pin->nents) * pin->nr_nents), GFP_KERNEL);
+	if (WARN_ON(!pin->nents)) {
+		ret = -ENOMEM;
+		goto err;
+	}
+
+	iova = pin->attrib.iova;
+	for_each_sg(pin->sgt->sgl, sg, pin->sgt->nents, sg_index) {
+		phys_addr_t paddr = (phys_addr_t)(sg_phys(sg));
+
+		pin->nents[sg_index].iova = iova;
+		pin->nents[sg_index].len = sg->length;
+		ret = pci_client_map_addr(vmap_ctx->pci_client_h,
+					  pin->nents[sg_index].iova, paddr,
+					  pin->nents[sg_index].len,
+					  (IOMMU_CACHE | prot));
+		if (ret < 0) {
+			pr_err("Failed: to iommu_map sg_nent: (%u), size: (%u)\n",
+			       sg_index, sg->length);
+			goto err;
+		}
+		pin->nents[sg_index].mapped_iova = true;
+
+		/* store information for unmap.*/
+		iova += sg->length;
+	}
+
+	return ret;
+err:
+	memobj_clientmngd_unpin(vmap_ctx, pin);
+	return ret;
+}
+
+void
+memobj_unpin(struct vmap_ctx_t *vmap_ctx,
+	     struct memobj_pin_t *pin)
+{
+	if (!vmap_ctx)
+		return;
+
+	if (!pin)
+		return;
+
+	if (pin->mngd == VMAP_MNGD_CLIENT)
+		memobj_clientmngd_unpin(vmap_ctx, pin);
+	else
+		memobj_devmngd_unpin(vmap_ctx, pin);
+
+	dma_buf_put(pin->dmabuf); // get_dma_buf();
+}
+
+int
+memobj_pin(struct vmap_ctx_t *vmap_ctx,
+	   struct memobj_pin_t *pin)
+{
+	int ret = 0;
+
+	/* ref count till we unmap. */
+	get_dma_buf(pin->dmabuf);
+
+	if (pin->mngd == VMAP_MNGD_CLIENT)
+		ret = memobj_clientmngd_pin(vmap_ctx, pin);
+	else
+		ret = memobj_devmngd_pin(vmap_ctx, pin);
+
+	if (ret)
+		memobj_unpin(vmap_ctx, pin);
+
+	return ret;
+}
+
+void
+syncobj_clientmngd_unpin(struct vmap_ctx_t *vmap_ctx,
+			 struct syncobj_pin_t *pin)
+{
+	if (!vmap_ctx)
+		return;
+
+	if (!pin)
+		return;
+
+	if (pin->mapped_iova) {
+		pci_client_unmap_addr(vmap_ctx->pci_client_h,
+				      pin->attrib.iova, pin->attrib.size);
+		pin->mapped_iova = false;
+	}
+
+	if (pin->iova_block_h) {
+		pci_client_free_iova(vmap_ctx->pci_client_h,
+				     &pin->iova_block_h);
+		pin->iova_block_h = NULL;
+	}
+}
+
+static int
+syncobj_clientmngd_pin(struct vmap_ctx_t *vmap_ctx,
+		       struct syncobj_pin_t *pin)
+{
+	int ret = 0;
+
+	if (pin->prot != VMAP_OBJ_PROT_WRITE) {
+		pr_err("Pinning syncobj with read access not supported\n");
+		return -EOPNOTSUPP;
+	}
+
+	ret = pci_client_alloc_iova(vmap_ctx->pci_client_h, pin->attrib.size,
+				    &pin->attrib.iova, &pin->attrib.offsetof,
+				    &pin->iova_block_h);
+	if (ret) {
+		pr_err("Failed to reserve iova block of size: (%lu)\n",
+		       pin->attrib.size);
+		goto err;
+	}
+
+	ret = pci_client_map_addr(vmap_ctx->pci_client_h, pin->attrib.iova,
+				  pin->phy_addr, pin->attrib.size,
+				  (IOMMU_CACHE | IOMMU_WRITE));
+	if (ret) {
+		pr_err("Failed to pin syncpoint physical addr to client iova\n");
+		goto err;
+	}
+	pin->mapped_iova = true;
+
+	return ret;
+err:
+	syncobj_clientmngd_unpin(vmap_ctx, pin);
+	return ret;
+}
+
+void
+syncobj_unpin(struct vmap_ctx_t *vmap_ctx,
+	      struct syncobj_pin_t *pin)
+{
+	if (!vmap_ctx)
+		return;
+
+	if (!pin)
+		return;
+
+	if (pin->pin_reqd) {
+		if (pin->mngd == VMAP_MNGD_DEV)
+			return;
+
+		syncobj_clientmngd_unpin(vmap_ctx, pin);
+	}
+
+	if (pin->sp) {
+		host1x_syncpt_put(pin->sp);
+		pin->sp = NULL;
+	}
+}
+
+int
+syncobj_pin(struct vmap_ctx_t *vmap_ctx,
+	    struct syncobj_pin_t *pin)
+{
+	int ret = 0;
+	struct host1x *host1x = NULL;
+
+	host1x = platform_get_drvdata(vmap_ctx->host1x_pdev);
+	if (!host1x) {
+		ret = -EINVAL;
+		pr_err("Could not get host1x handle from host1x_pdev.");
+		goto err;
+	}
+
+	/*
+	 * Get host1x_syncpt using syncpoint id and fd.
+	 * Takes ref on syncpoint.
+	 */
+	pin->sp = tegra_drm_get_syncpt(pin->fd, pin->syncpt_id);
+	if (IS_ERR_OR_NULL(pin->sp)) {
+		ret = PTR_ERR(pin->sp);
+		pr_err("Failed to get syncpoint from id\n");
+		goto err;
+	}
+
+	pin->attrib.syncpt_id = pin->syncpt_id;
+	pin->attrib.size = SP_MAP_SIZE;
+	if (pin->pin_reqd) {
+		pin->phy_addr = get_syncpt_shim_offset(pin->syncpt_id);
+		/*
+		 * remote/export sync obj are mapped to an iova of client
+		 * choice always and we should not come here for local sync objs
+		 */
+		if (pin->mngd == VMAP_MNGD_DEV) {
+			ret = -EOPNOTSUPP;
+			goto err;
+		}
+
+		ret = syncobj_clientmngd_pin(vmap_ctx, pin);
+	}
+
+err:
+	if (ret)
+		syncobj_unpin(vmap_ctx, pin);
+
+	return ret;
+}

drivers/misc/nvscic2c-pcie/vmap.c

@@ -0,0 +1,833 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#define pr_fmt(fmt)	"nvscic2c-pcie: vmap: " fmt
+
+#include "comm-channel.h"
+#include "common.h"
+#include "descriptor.h"
+#include "module.h"
+#include "pci-client.h"
+#include "vmap.h"
+#include "vmap-internal.h"
+
+#include <linux/device.h>
+#include <linux/errno.h>
+#include <linux/kernel.h>
+#include <linux/of.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+#include <linux/printk.h>
+#include <linux/types.h>
+
+/*
+ * *_START must be > 0 to avoid interference with idr_for_each().
+ */
+#define MEMOBJ_START	(1)
+#define SYNCOBJ_START	(1)
+#define IMPORTOBJ_START	(1)
+#define MEMOBJ_END	(MAX_STREAM_MEMOBJS)
+#define SYNCOBJ_END	(MAX_STREAM_SYNCOBJS)
+#define IMPORTOBJ_END	(MAX_STREAM_MEMOBJS + MAX_STREAM_SYNCOBJS)
+
+static int
+match_dmabuf(int id, void *entry, void *data)
+{
+	struct memobj_map_ref *map = (struct memobj_map_ref *)entry;
+
+	if (map->pin.dmabuf == (struct dma_buf *)data)
+		return id;
+
+	/* 0 shall pick-up next entry.*/
+	return 0;
+}
+
+static int
+memobj_map(struct vmap_ctx_t *vmap_ctx,
+	   struct vmap_memobj_map_params *params,
+	   struct vmap_obj_attributes *attrib)
+{
+	int ret = 0;
+	s32 id_exist = 0;
+	struct memobj_map_ref *map = NULL;
+	struct dma_buf *dmabuf = NULL;
+
+	dmabuf = dma_buf_get(params->fd);
+	if (IS_ERR_OR_NULL(dmabuf)) {
+		pr_err("Failed to get dma_buf for Mem Fd: (%d)\n",
+		       params->fd);
+		return -EFAULT;
+	}
+
+	mutex_lock(&vmap_ctx->mem_idr_lock);
+
+	/* check if the dma_buf is already mapped ? */
+	id_exist = idr_for_each(&vmap_ctx->mem_idr, match_dmabuf, &dmabuf);
+	if (id_exist > 0)
+		map = idr_find(&vmap_ctx->mem_idr, id_exist);
+
+	if (map) {
+		/* already mapped.*/
+		/*
+		 * requested mapping type != already mapped type.
+		 * e.g: mem obj previously mapped with dev mngd and now
+		 * as client mngd.
+		 */
+		if (params->mngd != map->pin.mngd) {
+			pr_err("Memobj: Already mapped with another mngd\n");
+			ret = -EINVAL;
+			goto err;
+		}
+		/*
+		 * add a validation later when rid=sid is enabled, where it
+		 * shall be dev_mngd in both case but dev shall be different.
+		 */
+		kref_get(&map->refcount);
+	} else {
+		map = kzalloc(sizeof(*map), GFP_KERNEL);
+		if (WARN_ON(!map)) {
+			ret = -ENOMEM;
+			goto err;
+		}
+
+		map->vmap_ctx = vmap_ctx;
+		kref_init(&map->refcount);
+		map->pin.dmabuf = dmabuf;
+		map->pin.prot = params->prot;
+		map->pin.mngd = params->mngd;
+		map->obj_id = idr_alloc(&vmap_ctx->mem_idr, map,
+					MEMOBJ_START, MEMOBJ_END,
+					GFP_KERNEL);
+		if (map->obj_id <= 0) {
+			ret = map->obj_id;
+			pr_err("Failed to idr alloc for mem obj\n");
+			kfree(map);
+			goto err;
+		}
+
+		/* populates map->pin.attrib within.*/
+		ret = memobj_pin(vmap_ctx, &map->pin);
+		if (ret) {
+			pr_err("Failed to pin mem obj fd: (%d)\n", params->fd);
+			idr_remove(&vmap_ctx->mem_idr, map->obj_id);
+			kfree(map);
+			goto err;
+		}
+	}
+
+	attrib->type = VMAP_OBJ_TYPE_MEM;
+	attrib->id = map->obj_id;
+	attrib->iova = map->pin.attrib.iova;
+	attrib->size = map->pin.attrib.size;
+	attrib->offsetof = map->pin.attrib.offsetof;
+err:
+	mutex_unlock(&vmap_ctx->mem_idr_lock);
+	dma_buf_put(dmabuf); //dma_buf_get()
+	return ret;
+}
+
+/* must be called with idr lock held.*/
+static void
+memobj_free(struct kref *kref)
+{
+	struct memobj_map_ref *map = NULL;
+
+	if (!kref)
+		return;
+
+	map = container_of(kref, struct memobj_map_ref, refcount);
+	if (map) {
+		memobj_unpin(map->vmap_ctx, &map->pin);
+		idr_remove(&map->vmap_ctx->mem_idr, map->obj_id);
+		kfree(map);
+	}
+}
+
+static int
+memobj_unmap(struct vmap_ctx_t *vmap_ctx, s32 obj_id)
+{
+	struct memobj_map_ref *map = NULL;
+
+	mutex_lock(&vmap_ctx->mem_idr_lock);
+	map = idr_find(&vmap_ctx->mem_idr, obj_id);
+	if (!map) {
+		mutex_unlock(&vmap_ctx->mem_idr_lock);
+		return -EBADF;
+	}
+
+	kref_put(&map->refcount, memobj_free);
+	mutex_unlock(&vmap_ctx->mem_idr_lock);
+
+	return 0;
+}
+
+static int
+memobj_putref(struct vmap_ctx_t *vmap_ctx, s32 obj_id)
+{
+	return memobj_unmap(vmap_ctx, obj_id);
+}
+
+static int
+memobj_getref(struct vmap_ctx_t *vmap_ctx, s32 obj_id)
+{
+	struct memobj_map_ref *map = NULL;
+
+	mutex_lock(&vmap_ctx->mem_idr_lock);
+	map = idr_find(&vmap_ctx->mem_idr, obj_id);
+	if (WARN_ON(!map)) {
+		mutex_unlock(&vmap_ctx->mem_idr_lock);
+		return -EBADF;
+	}
+
+	kref_get(&map->refcount);
+	mutex_unlock(&vmap_ctx->mem_idr_lock);
+
+	return 0;
+}
+
+static int
+match_syncpt_id(int id, void *entry, void *data)
+{
+	struct syncobj_map_ref *map = (struct syncobj_map_ref *)entry;
+
+	if (map->pin.syncpt_id == *((u32 *)data))
+		return id;
+
+	/* 0 shall pick-up next entry.*/
+	return 0;
+}
+
+static int
+syncobj_map(struct vmap_ctx_t *vmap_ctx,
+	    struct vmap_syncobj_map_params *params,
+	    struct vmap_obj_attributes *attrib)
+{
+	int ret = 0;
+	s32 id_exist = 0;
+	u32 syncpt_id = 0;
+	struct syncobj_map_ref *map = NULL;
+
+	syncpt_id = params->id;
+	mutex_lock(&vmap_ctx->sync_idr_lock);
+
+	/* check if the syncpt is already mapped ? */
+	id_exist = idr_for_each(&vmap_ctx->sync_idr, match_syncpt_id,
+				&syncpt_id);
+	if (id_exist > 0)
+		map = idr_find(&vmap_ctx->sync_idr, id_exist);
+
+	if (map) {
+		/* mapping again a SYNC obj(local or remote) is not permitted.*/
+		ret = -EPERM;
+		goto err;
+	} else {
+		map = kzalloc(sizeof(*map), GFP_KERNEL);
+		if (WARN_ON(!map)) {
+			ret = -ENOMEM;
+			goto err;
+		}
+
+		map->vmap_ctx = vmap_ctx;
+		kref_init(&map->refcount);
+		map->obj_id = idr_alloc(&vmap_ctx->sync_idr, map,
+					SYNCOBJ_START, SYNCOBJ_END,
+					GFP_KERNEL);
+		if (map->obj_id <= 0) {
+			ret = map->obj_id;
+			pr_err("Failed to idr alloc for sync obj\n");
+			kfree(map);
+			goto err;
+		}
+
+		/* local syncobjs do not need to be pinned to pcie iova.*/
+		map->pin.fd = params->fd;
+		map->pin.syncpt_id = syncpt_id;
+		map->pin.pin_reqd = params->pin_reqd;
+		map->pin.prot = params->prot;
+		map->pin.mngd = params->mngd;
+		ret = syncobj_pin(vmap_ctx, &map->pin);
+		if (ret) {
+			pr_err("Failed to pin sync obj Id: (%d)\n",
+			       syncpt_id);
+			idr_remove(&vmap_ctx->sync_idr, map->obj_id);
+			kfree(map);
+			goto err;
+		}
+		attrib->type = VMAP_OBJ_TYPE_SYNC;
+		attrib->id = map->obj_id;
+		attrib->iova = map->pin.attrib.iova;
+		attrib->size = map->pin.attrib.size;
+		attrib->offsetof = map->pin.attrib.offsetof;
+		attrib->syncpt_id = map->pin.attrib.syncpt_id;
+	}
+err:
+	mutex_unlock(&vmap_ctx->sync_idr_lock);
+
+	return ret;
+}
+
+/* must be called with idr lock held.*/
+static void
+syncobj_free(struct kref *kref)
+{
+	struct syncobj_map_ref *map = NULL;
+
+	if (!kref)
+		return;
+
+	map = container_of(kref, struct syncobj_map_ref, refcount);
+	if (map) {
+		syncobj_unpin(map->vmap_ctx, &map->pin);
+		idr_remove(&map->vmap_ctx->sync_idr, map->obj_id);
+		kfree(map);
+	}
+}
+
+static int
+syncobj_unmap(struct vmap_ctx_t *vmap_ctx, s32 obj_id)
+{
+	struct syncobj_map_ref *map = NULL;
+
+	mutex_lock(&vmap_ctx->sync_idr_lock);
+	map = idr_find(&vmap_ctx->sync_idr, obj_id);
+	if (!map) {
+		mutex_unlock(&vmap_ctx->sync_idr_lock);
+		return -EBADF;
+	}
+
+	kref_put(&map->refcount, syncobj_free);
+	mutex_unlock(&vmap_ctx->sync_idr_lock);
+
+	return 0;
+}
+
+static int
+syncobj_putref(struct vmap_ctx_t *vmap_ctx, s32 obj_id)
+{
+	return syncobj_unmap(vmap_ctx, obj_id);
+}
+
+static int
+syncobj_getref(struct vmap_ctx_t *vmap_ctx, s32 obj_id)
+{
+	struct memobj_map_ref *map = NULL;
+
+	if (!vmap_ctx)
+		return -EINVAL;
+
+	mutex_lock(&vmap_ctx->sync_idr_lock);
+	map = idr_find(&vmap_ctx->sync_idr, obj_id);
+	if (WARN_ON(!map)) {
+		mutex_unlock(&vmap_ctx->sync_idr_lock);
+		return -EBADF;
+	}
+
+	kref_get(&map->refcount);
+	mutex_unlock(&vmap_ctx->sync_idr_lock);
+
+	return 0;
+}
+
+static int
+match_export_desc(int id, void *entry, void *data)
+{
+	struct importobj_map_ref *map = (struct importobj_map_ref *)entry;
+
+	if (map->reg.export_desc == *((u64 *)data))
+		return id;
+
+	/* 0 shall pick-up next entry.*/
+	return 0;
+}
+
+static int
+importobj_map(struct vmap_ctx_t *vmap_ctx,
+	      struct vmap_importobj_map_params *params,
+	      struct vmap_obj_attributes *attrib)
+{
+	int ret = 0;
+	s32 id_exist = 0;
+	struct importobj_map_ref *map = NULL;
+
+	mutex_lock(&vmap_ctx->import_idr_lock);
+
+	/* check if we have export descriptor from remote already ? */
+	id_exist = idr_for_each(&vmap_ctx->import_idr, match_export_desc,
+				&params->export_desc);
+	if (id_exist > 0)
+		map = idr_find(&vmap_ctx->import_idr, id_exist);
+
+	if (!map) {
+		ret = -EAGAIN;
+		pr_debug("Failed to find descriptor: (%llu), try again\n",
+			 params->export_desc);
+		goto err;
+	} else {
+		/* importing beyond the export from remote is not permitted.*/
+		if (map->reg.nr_import == map->reg.nr_export) {
+			ret = -EPERM;
+			goto err;
+		}
+		map->reg.nr_import++;
+
+		attrib->type = VMAP_OBJ_TYPE_IMPORT;
+		attrib->id = map->obj_id;
+		attrib->iova = map->reg.attrib.iova;
+		attrib->size = map->reg.attrib.size;
+		attrib->offsetof = map->reg.attrib.offsetof;
+	}
+err:
+	mutex_unlock(&vmap_ctx->import_idr_lock);
+
+	return ret;
+}
+
+/* must be called with idr lock held.*/
+static void
+importobj_free(struct kref *kref)
+{
+	struct importobj_map_ref *map = NULL;
+
+	if (!kref)
+		return;
+
+	map = container_of(kref, struct importobj_map_ref, refcount);
+	if (map) {
+		idr_remove(&map->vmap_ctx->import_idr, map->obj_id);
+		kfree(map);
+	}
+}
+
+static int
+importobj_unmap(struct vmap_ctx_t *vmap_ctx, s32 obj_id)
+{
+	struct importobj_map_ref *map = NULL;
+	struct comm_msg msg = {0};
+
+	mutex_lock(&vmap_ctx->import_idr_lock);
+
+	map = idr_find(&vmap_ctx->import_idr, obj_id);
+	if (!map) {
+		mutex_unlock(&vmap_ctx->import_idr_lock);
+		return -EINVAL;
+	}
+	if (WARN_ON(!map->reg.nr_import)) {
+		pr_err("Import ObjId: (%d) wasn't imported yet\n", obj_id);
+		mutex_unlock(&vmap_ctx->import_idr_lock);
+		return -EINVAL;
+	}
+
+	/*
+	 * Each import corresponds to an export, if we unmap an imported
+	 * object, it's exported instance is also refcounted. Remote must
+	 * export again for it to be imported on local SoC again.
+	 */
+	msg.type = COMM_MSG_TYPE_UNREGISTER;
+	msg.u.unreg.export_desc = map->reg.export_desc;
+	msg.u.unreg.iova = map->reg.attrib.iova;
+	msg.u.unreg.size = map->reg.attrib.size;
+	msg.u.unreg.offsetof = map->reg.attrib.offsetof;
+	comm_channel_msg_send(vmap_ctx->comm_channel_h, &msg);
+
+	kref_put(&map->refcount, importobj_free);
+	mutex_unlock(&vmap_ctx->import_idr_lock);
+
+	return 0;
+}
+
+static int
+importobj_putref(struct vmap_ctx_t *vmap_ctx, s32 obj_id)
+{
+	return importobj_unmap(vmap_ctx, obj_id);
+}
+
+static int
+importobj_getref(struct vmap_ctx_t *vmap_ctx, s32 obj_id)
+{
+	struct memobj_map_ref *map = NULL;
+
+	mutex_lock(&vmap_ctx->import_idr_lock);
+	map = idr_find(&vmap_ctx->import_idr, obj_id);
+	if (WARN_ON(!map)) {
+		mutex_unlock(&vmap_ctx->import_idr_lock);
+		return -EBADF;
+	}
+
+	kref_get(&map->refcount);
+	mutex_unlock(&vmap_ctx->import_idr_lock);
+
+	return 0;
+}
+
+int
+vmap_obj_map(void *vmap_h, struct vmap_obj_map_params *params,
+	     struct vmap_obj_attributes *attrib)
+{
+	int ret = 0;
+	struct vmap_ctx_t *vmap_ctx = (struct vmap_ctx_t *)vmap_h;
+
+	if (WARN_ON(!vmap_ctx || !params || !attrib))
+		return -EINVAL;
+
+	switch (params->type) {
+	case VMAP_OBJ_TYPE_MEM:
+		ret = memobj_map(vmap_ctx, &params->u.memobj, attrib);
+		break;
+	case VMAP_OBJ_TYPE_SYNC:
+		ret = syncobj_map(vmap_ctx, &params->u.syncobj, attrib);
+		break;
+	case VMAP_OBJ_TYPE_IMPORT:
+		ret = importobj_map(vmap_ctx, &params->u.importobj, attrib);
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+	return ret;
+}
+
+int
+vmap_obj_unmap(void *vmap_h, enum vmap_obj_type type, s32 obj_id)
+{
+	int ret = 0;
+	struct vmap_ctx_t *vmap_ctx = (struct vmap_ctx_t *)vmap_h;
+
+	if (WARN_ON(!vmap_ctx))
+		return -EINVAL;
+
+	switch (type) {
+	case VMAP_OBJ_TYPE_MEM:
+		ret = memobj_unmap(vmap_ctx, obj_id);
+		break;
+	case VMAP_OBJ_TYPE_SYNC:
+		ret = syncobj_unmap(vmap_ctx, obj_id);
+		break;
+	case VMAP_OBJ_TYPE_IMPORT:
+		ret = importobj_unmap(vmap_ctx, obj_id);
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+	return ret;
+}
+
+int
+vmap_obj_getref(void *vmap_h, enum vmap_obj_type type, s32 obj_id)
+{
+	int ret = 0;
+	struct vmap_ctx_t *vmap_ctx = (struct vmap_ctx_t *)vmap_h;
+
+	if (WARN_ON(!vmap_ctx))
+		return -EINVAL;
+
+	switch (type) {
+	case VMAP_OBJ_TYPE_MEM:
+		ret = memobj_getref(vmap_ctx, obj_id);
+		break;
+	case VMAP_OBJ_TYPE_SYNC:
+		ret = syncobj_getref(vmap_ctx, obj_id);
+		break;
+	case VMAP_OBJ_TYPE_IMPORT:
+		ret = importobj_getref(vmap_ctx, obj_id);
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+	return ret;
+}
+
+int
+vmap_obj_putref(void *vmap_h, enum vmap_obj_type type, s32 obj_id)
+{
+	int ret = 0;
+	struct vmap_ctx_t *vmap_ctx = (struct vmap_ctx_t *)vmap_h;
+
+	if (WARN_ON(!vmap_ctx))
+		return -EINVAL;
+
+	switch (type) {
+	case VMAP_OBJ_TYPE_MEM:
+		ret = memobj_putref(vmap_ctx, obj_id);
+		break;
+	case VMAP_OBJ_TYPE_SYNC:
+		ret = syncobj_putref(vmap_ctx, obj_id);
+		break;
+	case VMAP_OBJ_TYPE_IMPORT:
+		ret = importobj_putref(vmap_ctx, obj_id);
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+	return ret;
+}
+
+static void
+vmap_importobj_unregister(void *data, void *ctx)
+{
+	struct vmap_ctx_t *vmap_ctx = (struct vmap_ctx_t *)ctx;
+	struct comm_msg *msg = (struct comm_msg *)data;
+	union descriptor_t desc;
+
+	WARN_ON(!vmap_ctx);
+	WARN_ON(!msg);
+	WARN_ON(msg->type != COMM_MSG_TYPE_UNREGISTER);
+
+	desc.value = msg->u.unreg.export_desc;
+	pr_debug("Unregister Desc: (%llu)\n", desc.value);
+	if (desc.bit.handle_type == STREAM_OBJ_TYPE_MEM)
+		vmap_obj_putref(vmap_ctx, VMAP_OBJ_TYPE_MEM,
+				desc.bit.handle_id);
+	else
+		vmap_obj_putref(vmap_ctx, VMAP_OBJ_TYPE_SYNC,
+				desc.bit.handle_id);
+}
+
+static void
+vmap_importobj_register(void *data, void *ctx)
+{
+	struct vmap_ctx_t *vmap_ctx = (struct vmap_ctx_t *)ctx;
+	struct comm_msg *msg = (struct comm_msg *)data;
+	struct importobj_map_ref *map = NULL;
+	s32 id_exist = 0;
+
+	WARN_ON(!vmap_ctx);
+	WARN_ON(!msg);
+	WARN_ON(msg->type != COMM_MSG_TYPE_REGISTER);
+
+	mutex_lock(&vmap_ctx->import_idr_lock);
+
+	/* check if we have export descriptor from remote already ? */
+	id_exist = idr_for_each(&vmap_ctx->import_idr, match_export_desc,
+				&msg->u.reg.export_desc);
+	if (id_exist > 0)
+		map = idr_find(&vmap_ctx->import_idr, id_exist);
+
+	if (map) {
+		if (msg->u.reg.iova != map->reg.attrib.iova) {
+			pr_err("attrib:iova doesn't match for export desc\n");
+			goto err;
+		}
+		if (msg->u.reg.size != map->reg.attrib.size) {
+			pr_err("attrib:size doesn't match for export desc\n");
+			goto err;
+		}
+		if (msg->u.reg.offsetof != map->reg.attrib.offsetof) {
+			pr_err("attrib:offsetof doesn't match for export desc\n");
+			goto err;
+		}
+		map->reg.nr_export++;
+		kref_get(&map->refcount);
+		pr_debug("Registered descriptor again: (%llu)\n",
+			 map->reg.export_desc);
+	} else {
+		/* map for the first time.*/
+		map = kzalloc(sizeof(*map), GFP_KERNEL);
+		if (WARN_ON(!map))
+			goto err;
+
+		map->vmap_ctx = vmap_ctx;
+		kref_init(&map->refcount);
+		map->reg.nr_export = 1;
+		map->reg.export_desc = msg->u.reg.export_desc;
+		map->reg.attrib.iova = msg->u.reg.iova;
+		map->reg.attrib.size = msg->u.reg.size;
+		map->reg.attrib.offsetof = msg->u.reg.offsetof;
+		map->obj_id = idr_alloc(&vmap_ctx->import_idr, map,
+					IMPORTOBJ_START, IMPORTOBJ_END,
+					GFP_KERNEL);
+		if (map->obj_id <= 0) {
+			pr_err("Failed to idr alloc for import obj\n");
+			kfree(map);
+			goto err;
+		}
+		pr_debug("Registered descriptor: (%llu)\n", map->reg.export_desc);
+	}
+err:
+	mutex_unlock(&vmap_ctx->import_idr_lock);
+}
+
+/* Entry point for the virtual mapping sub-module/abstraction. */
+int
+vmap_init(struct driver_ctx_t *drv_ctx, void **vmap_h)
+{
+	int ret = 0;
+	struct callback_ops cb_ops = {0};
+	struct vmap_ctx_t *vmap_ctx = NULL;
+
+	/* should not be an already instantiated vmap context. */
+	if (WARN_ON(!drv_ctx || !vmap_h || *vmap_h))
+		return -EINVAL;
+
+	vmap_ctx = kzalloc(sizeof(*vmap_ctx), GFP_KERNEL);
+	if (WARN_ON(!vmap_ctx))
+		return -ENOMEM;
+
+	vmap_ctx->host1x_pdev = drv_ctx->drv_param.host1x_pdev;
+	vmap_ctx->comm_channel_h = drv_ctx->comm_channel_h;
+	vmap_ctx->pci_client_h = drv_ctx->pci_client_h;
+	idr_init(&vmap_ctx->mem_idr);
+	idr_init(&vmap_ctx->sync_idr);
+	idr_init(&vmap_ctx->import_idr);
+	mutex_init(&vmap_ctx->mem_idr_lock);
+	mutex_init(&vmap_ctx->sync_idr_lock);
+	mutex_init(&vmap_ctx->import_idr_lock);
+
+	vmap_ctx->dummy_pdev = platform_device_alloc(drv_ctx->drv_name, -1);
+	if (!vmap_ctx->dummy_pdev) {
+		ret = -ENOMEM;
+		pr_err("Failed to allocate dummy platform device\n");
+		goto err;
+	}
+	ret = platform_device_add(vmap_ctx->dummy_pdev);
+	if (ret) {
+		platform_device_put(vmap_ctx->dummy_pdev);
+		pr_err("Failed to add the dummy platform device\n");
+		goto err;
+	}
+	ret = dma_set_mask(&vmap_ctx->dummy_pdev->dev, DMA_BIT_MASK(39));
+	if (ret) {
+		platform_device_del(vmap_ctx->dummy_pdev);
+		platform_device_put(vmap_ctx->dummy_pdev);
+		pr_err("Failed to set mask for dummy platform device\n");
+		goto err;
+	}
+	vmap_ctx->dummy_pdev_init = true;
+
+	/* comm-channel for registering and unregistering import objects.*/
+	cb_ops.callback = vmap_importobj_register;
+	cb_ops.ctx = (void *)vmap_ctx;
+	ret = comm_channel_register_msg_cb(vmap_ctx->comm_channel_h,
+					   COMM_MSG_TYPE_REGISTER, &cb_ops);
+	if (ret) {
+		pr_err("Failed to add callback with for Register msg\n");
+		goto err;
+	}
+
+	cb_ops.callback = vmap_importobj_unregister;
+	cb_ops.ctx = (void *)vmap_ctx;
+	ret = comm_channel_register_msg_cb(vmap_ctx->comm_channel_h,
+					   COMM_MSG_TYPE_UNREGISTER, &cb_ops);
+	if (ret) {
+		pr_err("Failed to add callback with for Unregister msg\n");
+		goto err;
+	}
+
+	*vmap_h = vmap_ctx;
+	return ret;
+err:
+	vmap_deinit((void **)&vmap_ctx);
+	return ret;
+}
+
+/* Exit point only. */
+static int
+memobj_release(s32 obj_id, void *ptr, void *data)
+{
+	struct memobj_map_ref *map = (struct memobj_map_ref *)(ptr);
+	struct vmap_ctx_t *vmap_ctx = (struct vmap_ctx_t *)(data);
+
+	/* release irrespective of reference counts. */
+	if (map) {
+		memobj_unpin(vmap_ctx, &map->pin);
+		kfree(map);
+	}
+
+	return 0;
+}
+
+/* Exit point only. */
+static int
+syncobj_release(s32 obj_id, void *ptr, void *data)
+{
+	struct syncobj_map_ref *map = (struct syncobj_map_ref *)(ptr);
+	struct vmap_ctx_t *vmap_ctx = (struct vmap_ctx_t *)(data);
+
+	/* release irrespective of reference counts. */
+	if (map) {
+		syncobj_unpin(vmap_ctx, &map->pin);
+		kfree(map);
+	}
+
+	return 0;
+}
+
+/* Exit point only. */
+static int
+importobj_release(s32 obj_id, void *ptr, void *data)
+{
+	struct importobj_map_ref *map = (struct importobj_map_ref *)(ptr);
+	struct vmap_ctx_t *vmap_ctx = (struct vmap_ctx_t *)(data);
+	struct comm_msg msg = {0};
+
+	if (map) {
+		msg.type = COMM_MSG_TYPE_UNREGISTER;
+		msg.u.unreg.export_desc = map->reg.export_desc;
+		msg.u.unreg.iova = map->reg.attrib.iova;
+		msg.u.unreg.size = map->reg.attrib.size;
+		msg.u.unreg.offsetof = map->reg.attrib.offsetof;
+		comm_channel_msg_send(vmap_ctx->comm_channel_h, &msg);
+
+		kfree(map);
+	}
+
+	return 0;
+}
+
+/* Exit point for nvscic2c-pcie vmap sub-module/abstraction. */
+void
+vmap_deinit(void **vmap_h)
+{
+	struct vmap_ctx_t *vmap_ctx = (struct vmap_ctx_t *)(*vmap_h);
+
+	if (!vmap_ctx)
+		return;
+
+	comm_channel_unregister_msg_cb(vmap_ctx->comm_channel_h,
+				       COMM_MSG_TYPE_REGISTER);
+	comm_channel_unregister_msg_cb(vmap_ctx->comm_channel_h,
+				       COMM_MSG_TYPE_UNREGISTER);
+	/*
+	 * free all the allocations still idr allocated IDR.
+	 *
+	 * ideally, this should not be the case, however in scenario:
+	 * application went away and the remote missed to free the imported
+	 * target handle, then during module unload (PCIe link shall be down)
+	 * we shall free all the pinned + yet to be unpinned handles.
+	 */
+	mutex_lock(&vmap_ctx->mem_idr_lock);
+	idr_for_each(&vmap_ctx->mem_idr, memobj_release, vmap_ctx);
+	idr_destroy(&vmap_ctx->mem_idr);
+	mutex_unlock(&vmap_ctx->mem_idr_lock);
+
+	mutex_lock(&vmap_ctx->sync_idr_lock);
+	idr_for_each(&vmap_ctx->sync_idr, syncobj_release, vmap_ctx);
+	idr_destroy(&vmap_ctx->sync_idr);
+	mutex_unlock(&vmap_ctx->sync_idr_lock);
+
+	mutex_lock(&vmap_ctx->import_idr_lock);
+	idr_for_each(&vmap_ctx->import_idr, importobj_release, vmap_ctx);
+	idr_destroy(&vmap_ctx->import_idr);
+	mutex_unlock(&vmap_ctx->import_idr_lock);
+
+	if (vmap_ctx->dummy_pdev_init) {
+		platform_device_unregister(vmap_ctx->dummy_pdev);
+		vmap_ctx->dummy_pdev_init = false;
+	}
+
+	kfree(vmap_ctx);
+	*vmap_h = NULL;
+}

drivers/misc/nvscic2c-pcie/vmap.h

@@ -0,0 +1,259 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#ifndef __VMAP_H__
+#define __VMAP_H__
+
+#include "common.h"
+
+/*
+ * Virtual mapping abstraction offers pinning functionalities for the very
+ * the specific use-case(s) to achieve NvStreams streaming over NvSciC2cPcie
+ * on Tegra PCIe EP or Tegra PCIe RP. It's scope is specific to/for and limited
+ * to NvSciC2cPcie.
+ *
+ * Virtual mapping interfaces primarily offer mapping(also referred as pinning
+ * in tegra context) and unmapping(also referred as unpinning in tegra context)
+ * dma_buf backed NvRmMemHandles or Syncpoint Shim backed NvRmHost1xSyncpoints.
+ * In this abstraction, Memory/Mem objects/objs are NvRmMemHandles(or it's FD),
+ * and Sync objects/objs are NvRmHost1xSyncpointHandles (or it's FD).
+ *
+ * This SW shall run either on Tegra PCIe RP or Tegra PCIe EP.
+ *
+ * ASSUMPTION: Once pages are mapped/pinned, on tegra they shall not swap out.
+ *
+ * On tegra, Mem objects are the NvRmMemHandles with struct dma_buf backing.
+ * On tegra, Sync objects are the NvRmHost1xSyncpointHandles with Syncpoint
+ * shim (aperture) backing.
+ *
+ * Each object is considered one of the following:
+ *
+ * 1. Local - The object visibility is limited to local SoC on which it is
+ *    allocated. Also sometimes referred as Unexported. These objects are mapped
+ *    to PCIe device address space and it's whereabouts are NOT shared with
+ *    remote SoC.
+ * 2. Export/Exported - The object visibility is across PCIe to remote SoC
+ *    and subsequently remote SoC can initiate writes to it. For tegra,
+ *    exported objects are never read over PCIe. These objects are mapped to
+ *    PCIe device address space and it's whereabouts shall be shared with remote
+ *    SoC.
+ * 3. Import/Imported - This is a virtual object which points to the
+ *    corresponding object exported by remote SoC. Being virtual, it shall be
+ *    similar for both Mem and Sync objects - therefore Imported object is
+ *    just an import/imported object and NOT imported Mem/Sync obj.
+ *
+ * Protection/Permission flags:
+ * a. Local Mem objects map to PCIe device with READ access.
+ * b. Export Mem objects map to PCIe device with WRITE access (We export for
+ *    remote to write to it via CPU or PCIe eDMA(zero-copy).
+ * c. Local Sync objects are not mapped to PCIe device as NvSciC2cPcie KMD
+ *    shall signal(write) them via NvHost1x interface.
+ * d. Export Sync objects map to PCIe device with WRITE access (We export for
+ *    remote to signal(write) to it via CPU or PCIe eDMA).
+ *
+ * Mapping:
+ * Tegra PCIe EP exposes three BAR memory windows towards PCIe RP. Of these,
+ * Only one (1) (BAR0) is available for NvSciC2cPcie access. Therefore, all
+ * Export objects must be mapped to a PCIe address which this PCIe EP BAR0
+ * translation is programmed with. With the overall PCIe address space being
+ * much bigger than the PCIe BAR0 space, there is an explicit need to stitch
+ * all Exported objects to a single region. This requires Export objects be
+ * mapped using iommu apis to achieve BAR stitching and this mapping is
+ * referred as client managed with NvSciC2cPcie managing the iova region.
+ *
+ * Tegra PCIe EP has limited set of translation registers for it's CPU to raise
+ * PCIe transactions towards a PCIe RP. Therefore, when Sync objects are
+ * exported from PCIe RP towards PCIe EP to CPU signal them, they must be mapped
+ * to a single iova region which PCIe EP has setup for it's translaction
+ * registers. This is strictly not applicable for Exported Mem objects as they
+ * are written by eDMA always by the importing SoC. However, to keep parity and
+ * symmetry in the behavior of Exported Mem objects from Tegra PCIe EE->PCIe RP,
+ * Exported Mem objects from Tegra PCIe RP->Tegra PCIe EP shall also be mapped
+ * to a client managed iova region.
+ *
+ * For Local Mem objects which are accessed by local SoC PCIe eDMA, they
+ * can be mapped to any pcie address outside the reserved iova region by
+ * NvSciC2cPcie for exports. This doesn't require any iova management by
+ * client and is prefectly fine to use pci device (smmu) managed iova.
+ * This mapping is referred as device(dev) managed mapping. Only on Tegra PCIe
+ * RP, Exported Mem objects can be mapped using dev managed iova as Tegra PCIe
+ * EP shall write them using eDMA, but as stated before, to keep parity and
+ * symmetry in the behavior of Exported Mem objects from Tegra PCIe EE->PCIe RP,
+ * Exported Mem objects from Tegra PCIe RP->Tegra PCIe EP shall also be mapped
+ * to a client managed iova region.
+ *
+ * All Sync objects (local or export) are mapped to NvSciC2cPcie for signalling
+ * (Write access), based on QNX security policy, there shall be only one
+ * signaler allowed and, therefore, Sync objects are pinned/mapped just once.
+ * Export Mem objects are mapped to NvSciC2cPcie for remote SoC to produce data
+ * (Write access), also in the lack of N producer -> 1 consumer user-case,
+ * remote Mem objects are pinned/mapped just once. However, Local Mem objects
+ * which have Read access, can be mapped/pinned again. Essentially, all objects
+ * requiring Write access by NvSciC2cPcie (PCIe device) are pinned/mapped once.
+ *
+ * Summary:
+ *  i.   Types:
+ *        a. Local Mem objects.
+ *        b. Export Mem objects.
+ *        c. Local Sync objects.
+ *        d. Export Sync objects.
+ *        e. Import objects.
+ *  ii.  Mapping:
+ *        a. Local Mem objects - dev managed (READ only).
+ *        b. Export Mem objects - client managed (WRITE only).
+ *            On Tegra PCIe EP:- compulsarily client managed.
+ *            On Tegra PCIe RP:- Can be either dev managed or client managed.
+ *            Choose client manged to be in symmetry with Tegra PCIe EP.
+ *        c. Local Sync objects - Not mapped but pinned (tracked).
+ *        d. Export Sync objects - client managed (WRITE ONLY).
+ *            On Tegra PCIe EP:- compulsarily client managed.
+ *            On Tegra PCIe RP:- Can be either dev managed (if Tegra PCIe RP use
+ *            only eDMA for signaling, but we have use-cases for CPU signaling
+ *            also from Tegra PCIe RP. Therefore choose client managed as it can
+ *            satisfy CPU and eDMA signaling needs from remote Tegra PCIe EP.
+ *        e. Import Objects - virtual objects pointing to exported objects by
+ *            remote. Every exported object from a SoC must have a corresponding
+ *            import object on remote SoC. Not mapped but pinned(tracked).
+ */
+
+/* forward declaration. */
+struct driver_ctx_t;
+
+/* Object type that shall be virtually mapped for PCIe access. */
+enum vmap_obj_type {
+	/* NvRmMemHandle (struct dma_buf *), aka, memobj.*/
+	VMAP_OBJ_TYPE_MEM = STREAM_OBJ_TYPE_MEM,
+
+	/* NvRmHost1xSyncpointHandle ( syncpt id), aka, memobj.*/
+	VMAP_OBJ_TYPE_SYNC = STREAM_OBJ_TYPE_SYNC,
+
+	/*(virtual) objects imported from remote SoC.*/
+	VMAP_OBJ_TYPE_IMPORT,
+};
+
+/*
+ * Permissions for pin/mapping Buff/Sync objs to PCIe device
+ *
+ * at the moment, WRITE for all EXPORT*
+ * at the moment, READ for all LOCAL*
+ */
+enum vmap_obj_prot {
+	/* read-only access by PCIe device. */
+	VMAP_OBJ_PROT_READ = 1,
+
+	/* write only access to PCIe device. */
+	VMAP_OBJ_PROT_WRITE = 2,
+
+	/*
+	 * no use-case known today.
+	 * VMAP_OBJ_PERM_READWRITE = 4,
+	 */
+};
+
+/* Which IOVA to use for mapping Mem/Sync objs.*/
+enum vmap_mngd {
+	/*
+	 * Stitching of all exported objects is done by reserving an IOVA region
+	 * and mapping Mem and Sync objs to it. The reserved IOVA region is
+	 * managed by client/user (NvSciC2cPcie) and use iommu apis to map Mem
+	 * or Sync objects to the specific IOVA.
+	 */
+	VMAP_MNGD_CLIENT = 0,
+
+	/*
+	 * The IOVA is managed by pci dev, therefore dev managed. Used only for
+	 * Mem objects (Local and possible for exported too).
+	 */
+	VMAP_MNGD_DEV,
+};
+
+/* Returned object attributes after mapping.*/
+struct vmap_obj_attributes {
+	enum vmap_obj_type type;
+	s32 id;
+	u64 iova;
+	size_t size;
+	size_t offsetof;
+	/* only for local sync obj.*/
+	u32 syncpt_id;
+};
+
+/* Parameters to map Mem object.*/
+struct vmap_memobj_map_params {
+	s32 fd;
+	/* To allow mapping Export Mem objects as dev-managed - Tegra PCIe RP.*/
+	enum vmap_mngd mngd;
+	/* local/source mem as read-only, remote/export as write-only.*/
+	enum vmap_obj_prot prot;
+};
+
+/* Parameters to map Sync object.*/
+struct vmap_syncobj_map_params {
+	s32 fd;
+	u32 id;
+	/* client mngd only.*/
+	enum vmap_mngd mngd;
+	/* write-only.*/
+	enum vmap_obj_prot prot;
+	/* local sync objs will not be pinned to pcie address space.*/
+	bool pin_reqd;
+};
+
+/* Parameters to map Import object.*/
+struct vmap_importobj_map_params {
+	u64 export_desc;
+};
+
+struct vmap_obj_map_params {
+	enum vmap_obj_type type;
+	union vmap_params {
+		struct vmap_memobj_map_params memobj;
+		struct vmap_syncobj_map_params syncobj;
+		struct vmap_importobj_map_params importobj;
+	} u;
+};
+
+/* Parameters to register Import object, as received from remote NvSciC2cPcie.*/
+struct vmap_importobj_reg_params {
+	u64 export_desc;
+	u64 iova;
+	size_t size;
+	size_t offsetof;
+};
+
+/* Entry point for the virtual mapping sub-module/abstraction. */
+int
+vmap_init(struct driver_ctx_t *drv_ctx, void **vmap_h);
+
+/* Exit point for nvscic2c-pcie vmap sub-module/abstraction. */
+void
+vmap_deinit(void **vmap_h);
+
+/* Map objects to pcie device.*/
+int
+vmap_obj_map(void *vmap_h, struct vmap_obj_map_params *params,
+	     struct vmap_obj_attributes *attrib);
+
+/* Unmap objects from pcie device.*/
+int
+vmap_obj_unmap(void *vmap_h, enum vmap_obj_type type, s32 obj_id);
+
+/* Increment reference count for objects. */
+int
+vmap_obj_getref(void *vmap_h, enum vmap_obj_type type, s32 obj_id);
+
+/* Decrement reference count for objects. */
+int
+vmap_obj_putref(void *vmap_h, enum vmap_obj_type type, s32 obj_id);
+#endif // __VMAP_H__

include/uapi/misc/nvscic2c-pcie-ioctl.h

@@ -0,0 +1,269 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#ifndef __UAPI_NVSCIC2C_PCIE_IOCTL_H__
+#define __UAPI_NVSCIC2C_PCIE_IOCTL_H__
+
+#include <linux/ioctl.h>
+#include <linux/types.h>
+
+#if !defined(__KERNEL__)
+#define __user
+#endif
+
+#define MAX_NAME_SZ		(32)
+
+/* Link status between the two peers - encapsulates PCIe link also.*/
+enum nvscic2c_pcie_link {
+	NVSCIC2C_PCIE_LINK_DOWN = 0,
+	NVSCIC2C_PCIE_LINK_UP,
+};
+
+/**
+ * stream extensions - object type.
+ */
+enum nvscic2c_pcie_obj_type {
+	NVSCIC2C_PCIE_OBJ_TYPE_INVALID = 0,
+
+	/* local NvRmMemHandle(tegra) /NvRmHandle(x86) obj. */
+	NVSCIC2C_PCIE_OBJ_TYPE_SOURCE_MEM,
+
+	/* Exported NvRmMemHandle(tegra) /NvRmHandle(x86) obj. */
+	NVSCIC2C_PCIE_OBJ_TYPE_TARGET_MEM,
+
+	/* local NvRmHost1xSyncpoint(tegra) /GPU Semaphore(x86) obj. */
+	NVSCIC2C_PCIE_OBJ_TYPE_LOCAL_SYNC,
+
+	/* Exported NvRmHost1xSyncpoint(tegra) /GPU Semaphore(x86) obj. */
+	NVSCIC2C_PCIE_OBJ_TYPE_REMOTE_SYNC,
+
+	/* (virtual) objects imported from remote SoC. */
+	NVSCIC2C_PCIE_OBJ_TYPE_IMPORT,
+
+	NVSCIC2C_PCIE_OBJ_TYPE_MAXIMUM,
+};
+
+/**
+ * PCIe aperture and PCIe shared memory
+ * are divided in different C2C endpoints.
+ * Data structure represents endpoint's
+ * physical address and size.
+ */
+struct nvscic2c_pcie_endpoint_mem_info {
+	/* would be one of the enum nvscic2c_mem_type.*/
+	__u32 offset;
+
+	/* size of this memory type device would like user-space to map.*/
+	__u32 size;
+};
+
+/**
+ * NvSciIpc endpoint information relayed to UMD. This information
+ * is per endpoint which shall allow UMD to mmap the endpoint's
+ * send, recv and pcie link area in user-space.
+ */
+struct nvscic2c_pcie_endpoint_info {
+	__u32 nframes;
+	__u32 frame_size;
+	struct nvscic2c_pcie_endpoint_mem_info peer;
+	struct nvscic2c_pcie_endpoint_mem_info self;
+	struct nvscic2c_pcie_endpoint_mem_info link;
+};
+
+/**
+ * stream extensions - Pin/Map.
+ */
+struct nvscic2c_pcie_map_in_arg {
+	/*
+	 * Mem obj - NvRmMemHandle FD. Sync obj - NvRmHost1xSyncpointHandle FD.
+	 */
+	__s32 fd;
+	/*
+	 * Mem obj - padding. Sync obj - Syncpoint Id.
+	 */
+	__u32 id;
+};
+
+struct nvscic2c_pcie_map_out_arg {
+	__s32 handle;
+	__u32 pad;
+};
+
+struct nvscic2c_pcie_map_obj_args {
+	__s32 obj_type;
+	__u32 pad;
+	struct nvscic2c_pcie_map_in_arg in;
+	struct nvscic2c_pcie_map_out_arg out;
+};
+
+/**
+ * stream extensions - Export.
+ */
+struct nvscic2c_pcie_export_in_arg {
+	__s32 handle;
+	__u32 pad;
+};
+
+struct nvscic2c_pcie_export_out_arg {
+	__u64 desc;
+};
+
+struct nvscic2c_pcie_export_obj_args {
+	__s32 obj_type;
+	__u32 pad;
+	struct nvscic2c_pcie_export_in_arg in;
+	struct nvscic2c_pcie_export_out_arg out;
+};
+
+/**
+ * stream extensions - Import.
+ */
+struct nvscic2c_pcie_import_in_arg {
+	__u64 desc;
+};
+
+struct nvscic2c_pcie_import_out_arg {
+	__s32 handle;
+	__u32 pad;
+};
+
+struct nvscic2c_pcie_import_obj_args {
+	__s32 obj_type;
+	__u32 pad;
+	struct nvscic2c_pcie_import_in_arg in;
+	struct nvscic2c_pcie_import_out_arg out;
+};
+
+/**
+ * stream extensions - Free Pinned Or Imported objects.
+ */
+struct nvscic2c_pcie_free_obj_args {
+	__s32 obj_type;
+	__s32 handle;
+};
+
+/**
+ * stream extensions - one transfer/copy unit.
+ */
+struct nvscic2c_pcie_flush_range {
+	__s32 src_handle;
+	__s32 dst_handle;
+	__u64 offset;
+	__u64 size;
+};
+
+/*
+ * @local_post_fences: user memory atleast of size:
+ *  num_local_post_fences * sizeof(__s32) - local sync handles
+ *
+ * @remote_post_fences: user memory atleast of size:
+ *  num_remote_post_fences * sizeof(__s32) - import sync handles
+ *
+ * @copy_requests: user memory atleast of size:
+ *  num_flush_ranges * sizeof(struct nvscic2c_pcie_flush_range)
+ */
+struct nvscic2c_pcie_submit_copy_args {
+	__u64 num_local_post_fences;
+	__u64 local_post_fences;
+	__u64 num_remote_post_fences;
+	__u64 remote_post_fences;
+	__u64 num_flush_ranges;
+	__u64 flush_ranges;
+	__u64 remote_post_fence_values;
+};
+
+/**
+ * stream extensions - Pass upper limit for the total possible outstanding
+ * submit copy requests.
+ * @max_copy_requests: Maximum outstanding @nvscic2c_pcie_submit_copy_args.
+ * @max_flush_ranges: Maximum @nvscic2c_pcie_flush_range possible for each
+ *  of the @max_copy_requests (@nvscic2c_pcie_submit_copy_args)
+ * @max_post_fences: Maximum post-fences possible for each of the
+ *  @max_copy_requests (@nvscic2c_pcie_submit_copy_args)
+ */
+struct nvscic2c_pcie_max_copy_args {
+	__u64 max_copy_requests;
+	__u64 max_flush_ranges;
+	__u64 max_post_fences;
+};
+
+/* Only to facilitate calculation of maximum size of ioctl arguments.*/
+union nvscic2c_pcie_ioctl_arg_max_size {
+	struct nvscic2c_pcie_max_copy_args mc;
+	struct nvscic2c_pcie_submit_copy_args cr;
+	struct nvscic2c_pcie_free_obj_args fo;
+	struct nvscic2c_pcie_import_obj_args io;
+	struct nvscic2c_pcie_export_obj_args eo;
+	struct nvscic2c_pcie_map_obj_args mp;
+	struct nvscic2c_pcie_endpoint_info ep;
+};
+
+/* IOCTL magic number - seen available in ioctl-number.txt*/
+#define NVSCIC2C_PCIE_IOCTL_MAGIC    0xC2
+
+#define NVSCIC2C_PCIE_IOCTL_GET_INFO \
+	_IOWR(NVSCIC2C_PCIE_IOCTL_MAGIC, 1,\
+	      struct nvscic2c_pcie_endpoint_info)
+
+/**
+ * notify remote
+ */
+#define NVSCIC2C_PCIE_IOCTL_NOTIFY_REMOTE \
+	_IO(NVSCIC2C_PCIE_IOCTL_MAGIC, 2)
+
+/**
+ * Pin/Map Mem or Sync objects.
+ */
+#define NVSCIC2C_PCIE_IOCTL_MAP \
+	_IOWR(NVSCIC2C_PCIE_IOCTL_MAGIC, 3,\
+	      struct nvscic2c_pcie_map_obj_args)
+
+/**
+ * Get Export descriptor for Target/Remote Mem/Sync objects.
+ */
+#define NVSCIC2C_PCIE_IOCTL_GET_AUTH_TOKEN \
+	_IOWR(NVSCIC2C_PCIE_IOCTL_MAGIC, 4,\
+	      struct nvscic2c_pcie_export_obj_args)
+
+/**
+ * Get Handle from the imported export descriptor.
+ */
+#define NVSCIC2C_PCIE_IOCTL_GET_HANDLE \
+	_IOWR(NVSCIC2C_PCIE_IOCTL_MAGIC, 5,\
+	      struct nvscic2c_pcie_import_obj_args)
+
+/**
+ * Free the Mapped/Pinned Source, Target or Imported Mem or Sync object handle.
+ */
+#define NVSCIC2C_PCIE_IOCTL_FREE \
+	_IOW(NVSCIC2C_PCIE_IOCTL_MAGIC, 6,\
+	      struct nvscic2c_pcie_free_obj_args)
+
+/**
+ * Submit a Copy request for transfer.
+ */
+#define NVSCIC2C_PCIE_IOCTL_SUBMIT_COPY_REQUEST \
+	_IOW(NVSCIC2C_PCIE_IOCTL_MAGIC, 7,\
+	      struct nvscic2c_pcie_submit_copy_args)
+
+/**
+ * Set the maximum possible outstanding copy requests that can be submitted.
+ */
+#define NVSCIC2C_PCIE_IOCTL_MAX_COPY_REQUESTS \
+	_IOW(NVSCIC2C_PCIE_IOCTL_MAGIC, 8,\
+	      struct nvscic2c_pcie_max_copy_args)
+
+#define NVSCIC2C_PCIE_IOCTL_NUMBER_MAX 8
+
+#endif /*__UAPI_NVSCIC2C_PCIE_IOCTL_H__*/