linux-nv-oot/drivers/platform/tegra/rtcpu/ivc-bus.c

// SPDX-License-Identifier: GPL-2.0-only
// SPDX-FileCopyrightText: Copyright (c) 2022-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.

#include <nvidia/conftest.h>

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/of.h>
#include <linux/of_device.h>
#include <linux/pm_runtime.h>
#include <linux/slab.h>
#if defined(NV_TEGRA_IVC_USE_IVC_EXT_DRIVER)
#include <soc/tegra/ivc_ext.h>
#else
#include <soc/tegra/ivc-priv.h>
#endif
#include <linux/tegra-ivc-bus.h>
#include <linux/bitops.h>
#include <linux/version.h>
#include "soc/tegra/camrtc-channels.h"

#if defined(NV_TEGRA_IVC_STRUCT_HAS_IOSYS_MAP)
#include <linux/iosys-map.h>
#endif

#define NV(p) "nvidia," #p

#define CAMRTC_IVC_CONFIG_SIZE	4096

struct tegra_ivc_region {
	uintptr_t base;
	size_t size;
	dma_addr_t iova;
	size_t config_size;
	size_t ivc_size;
};

struct tegra_ivc_bus {
	struct device dev;
	struct tegra_ivc_channel *chans;
	unsigned num_regions;
	struct tegra_ivc_region regions[];
};

static void tegra_ivc_channel_ring(struct tegra_ivc *ivc, void *data)
{
	struct tegra_ivc_channel *chan =
		container_of(ivc, struct tegra_ivc_channel, ivc);
	struct camrtc_hsp *camhsp = (struct camrtc_hsp *) data;

	camrtc_hsp_group_ring(camhsp, chan->group);
}

struct device_type tegra_ivc_channel_type = {
	.name = "tegra-ivc-channel",
};
EXPORT_SYMBOL(tegra_ivc_channel_type);

int tegra_ivc_channel_runtime_get(struct tegra_ivc_channel *ch)
{
	BUG_ON(ch == NULL);

	return pm_runtime_get_sync(&ch->dev);
}
EXPORT_SYMBOL(tegra_ivc_channel_runtime_get);

void tegra_ivc_channel_runtime_put(struct tegra_ivc_channel *ch)
{
	BUG_ON(ch == NULL);

	pm_runtime_put(&ch->dev);
}
EXPORT_SYMBOL(tegra_ivc_channel_runtime_put);

static void tegra_ivc_channel_release(struct device *dev)
{
	struct tegra_ivc_channel *chan =
		container_of(dev, struct tegra_ivc_channel, dev);

	of_node_put(dev->of_node);
	kfree(chan);
}

static struct tegra_ivc_channel *tegra_ivc_channel_create(
		struct tegra_ivc_bus *bus, struct device_node *ch_node,
		struct tegra_ivc_region *region,
		struct camrtc_hsp *camhsp)
{
	struct camrtc_tlv_ivc_setup *tlv;
#if defined(NV_TEGRA_IVC_STRUCT_HAS_IOSYS_MAP)
	struct iosys_map rx_map, tx_map;
#endif
	struct {
		u32 rx;
		u32 tx;
	} start, end;
	u32 version, channel_group, nframes, frame_size, queue_size;
	const char *service;
	int ret;
	u32 mul = 0U, sum = 0U;

	struct tegra_ivc_channel *chan = kzalloc(sizeof(*chan), GFP_KERNEL);
	if (unlikely(chan == NULL))
		return ERR_PTR(-ENOMEM);

	chan->dev.parent = &bus->dev;
	chan->dev.type = &tegra_ivc_channel_type;
	chan->dev.bus = &tegra_ivc_bus_type;
	chan->dev.of_node = of_node_get(ch_node);
	chan->dev.release = tegra_ivc_channel_release;
	dev_set_name(&chan->dev, "%s:%s", dev_name(&bus->dev),
			kbasename(ch_node->full_name));
	device_initialize(&chan->dev);
	pm_runtime_no_callbacks(&chan->dev);
	pm_runtime_enable(&chan->dev);

	ret = of_property_read_string(ch_node, NV(service), &service);
	if (ret) {
		dev_err(&chan->dev, "missing <%s> property\n",
			NV(service));
		goto error;
	}

	ret = of_property_read_u32(ch_node, NV(version), &version);
	if (ret)
		version = 0;

	ret = of_property_read_u32(ch_node, NV(group), &channel_group);
	if (ret) {
		dev_err(&chan->dev, "missing <%s> property\n", NV(group));
		goto error;
	}

	/* We have 15 channel group bits available */
	if ((channel_group & 0x7FFFU) != channel_group) {
		dev_err(&chan->dev, "invalid property %s = 0x%x\n",
			NV(group), channel_group);
		goto error;
	}

	ret = of_property_read_u32(ch_node, NV(frame-count), &nframes);
	if (ret || !nframes) {
		dev_err(&chan->dev, "missing <%s> property\n",
			NV(frame-count));
		goto error;
	}
	nframes = 1 << fls(nframes - 1); /* Round up to a power of two */

	ret = of_property_read_u32(ch_node, NV(frame-size), &frame_size);
	if (ret || !frame_size) {
		dev_err(&chan->dev, "missing <%s> property\n", NV(frame-size));
		goto error;
	}

	if (__builtin_add_overflow(region->config_size, sizeof(*tlv), &sum)) {
		dev_err(&chan->dev, "IVC config size overflow\n");
		ret = -EOVERFLOW;
		goto error;
	}

	if (sum > CAMRTC_IVC_CONFIG_SIZE) {
		dev_err(&chan->dev, "IVC config size exceeded\n");
		ret = -ENOSPC;
		goto error;
	}

	if (__builtin_mul_overflow(nframes, frame_size, &mul)) {
		dev_err(&chan->dev, "IVC frame size overflow\n");
		ret = -EOVERFLOW;
		goto error;
	}
	queue_size = tegra_ivc_total_queue_size(mul);

	if (__builtin_mul_overflow(2U, queue_size, &mul)) {
		dev_err(&chan->dev, "IVC queue size overflow\n");
		ret = -EOVERFLOW;
		goto error;
	}

	if (__builtin_add_overflow(region->ivc_size, mul, &sum)) {
		dev_err(&chan->dev, "IVC size overflow\n");
		ret = -EOVERFLOW;
		goto error;
	}

	if (sum > region->size) {
		dev_err(&chan->dev, "buffers exceed IVC region\n");
		ret = -ENOSPC;
		goto error;
	}

	start.rx = region->ivc_size;
	region->ivc_size += queue_size;
	end.rx = region->ivc_size;

	start.tx = end.rx;
	region->ivc_size += queue_size;
	end.tx = region->ivc_size;

#if defined(NV_TEGRA_IVC_STRUCT_HAS_IOSYS_MAP)
	iosys_map_set_vaddr(&rx_map, (void *)(region->base + start.rx));
	iosys_map_set_vaddr(&tx_map, (void *)(region->base + start.tx));

	/* Init IVC */
	ret = tegra_ivc_init(&chan->ivc,
			NULL,
			&rx_map,
			region->iova + start.rx,
			&tx_map,
			region->iova + start.tx,
			nframes, frame_size,
			tegra_ivc_channel_ring,
			(void *)camhsp);
#else
	/* Init IVC */
	ret = tegra_ivc_init(&chan->ivc,
			NULL,
			(void *)(region->base + start.rx),
			region->iova + start.rx,
			(void *)(region->base + start.tx),
			region->iova + start.tx,
			nframes, frame_size,
			tegra_ivc_channel_ring,
			(void *)camhsp);
#endif
	if (ret) {
		dev_err(&chan->dev, "IVC initialization error: %d\n", ret);
		goto error;
	}

	chan->group = channel_group;

#if defined(NV_TEGRA_IVC_USE_IVC_EXT_DRIVER)
	tegra_ivc_channel_reset(&chan->ivc);
#else
	tegra_ivc_reset(&chan->ivc);
#endif

	/* Fill channel descriptor */
	tlv = (struct camrtc_tlv_ivc_setup *)
		(region->base + region->config_size);

	tlv->tag = CAMRTC_TAG_IVC_SETUP;
	tlv->len = sizeof(*tlv);
	if (__builtin_add_overflow(region->iova, start.rx, &tlv->rx_iova)) {
		dev_err(&chan->dev, "IVC setup overflow\n");
		ret = -EOVERFLOW;
		goto error;
	}

	tlv->rx_frame_size = frame_size;
	tlv->rx_nframes = nframes;
	if (__builtin_add_overflow(region->iova, start.tx, &tlv->tx_iova)) {
		dev_err(&chan->dev, "IVC setup overflow\n");
		ret = -EOVERFLOW;
		goto error;
	}

	tlv->tx_frame_size = frame_size;
	tlv->tx_nframes = nframes;
	tlv->channel_group = channel_group;
	tlv->ivc_version = version;
	if (strscpy(tlv->ivc_service, service, sizeof(tlv->ivc_service)) < 0)
		dev_warn(&chan->dev, "service name <%s> too long\n", service);

	region->config_size += sizeof(*tlv);
	(++tlv)->tag = 0; /* terminator */

	dev_info(&chan->dev,
		"%s: ver=%u grp=%u RX[%ux%u]=0x%x-0x%x TX[%ux%u]=0x%x-0x%x\n",
		ch_node->name, version, channel_group,
		nframes, frame_size, start.rx, end.rx,
		nframes, frame_size, start.tx, end.tx);

	ret = device_add(&chan->dev);
	if (ret) {
		dev_err(&chan->dev, "channel device error: %d\n", ret);
		goto error;
	}

	return chan;
error:
	put_device(&chan->dev);
	return ERR_PTR(ret);
}

static void tegra_ivc_channel_notify(struct tegra_ivc_channel *chan)
{
	const struct tegra_ivc_channel_ops *ops;

#if defined(NV_TEGRA_IVC_USE_IVC_EXT_DRIVER)
	if (tegra_ivc_channel_notified(&chan->ivc) != 0)
#else
	if (tegra_ivc_notified(&chan->ivc) != 0)
#endif
		return;

	if (!chan->is_ready)
		return;

	rcu_read_lock();
	ops = rcu_dereference(chan->ops);

	if (ops != NULL && ops->notify != NULL)
		ops->notify(chan);
	rcu_read_unlock();
}

void tegra_ivc_bus_notify(struct tegra_ivc_bus *bus, u16 group)
{
	struct tegra_ivc_channel *chan;

	for (chan = bus->chans; chan != NULL; chan = chan->next) {
		if ((chan->group & group) != 0)
			tegra_ivc_channel_notify(chan);
	}
}
EXPORT_SYMBOL(tegra_ivc_bus_notify);

struct device_type tegra_ivc_bus_dev_type = {
	.name = "tegra-ivc-bus",
};
EXPORT_SYMBOL(tegra_ivc_bus_dev_type);

static void tegra_ivc_bus_release(struct device *dev)
{
	struct tegra_ivc_bus *bus =
		container_of(dev, struct tegra_ivc_bus, dev);
	int i;

	of_node_put(dev->of_node);

	for (i = 0; i < bus->num_regions; i++) {
		if (!bus->regions[i].base)
			continue;

		dma_free_coherent(dev->parent, bus->regions[i].size,
				(void *)bus->regions[i].base,
				bus->regions[i].iova);
	}

	kfree(bus);
}

#if defined(NV_BUS_TYPE_STRUCT_MATCH_HAS_CONST_DRV_ARG)
static int tegra_ivc_bus_match(struct device *dev, const struct device_driver *drv)
#else
static int tegra_ivc_bus_match(struct device *dev, struct device_driver *drv)
#endif
{
	struct tegra_ivc_driver *ivcdrv = to_tegra_ivc_driver(drv);

	if (dev->type != ivcdrv->dev_type)
		return 0;
	return of_driver_match_device(dev, drv);
}

static void tegra_ivc_bus_stop(struct tegra_ivc_bus *bus)
{
	while (bus->chans != NULL) {
		struct tegra_ivc_channel *chan = bus->chans;

		bus->chans = chan->next;
		pm_runtime_disable(&chan->dev);
		device_unregister(&chan->dev);
	}
}

static int tegra_ivc_bus_start(struct tegra_ivc_bus *bus,
	struct camrtc_hsp *camhsp)
{
	struct device_node *dn = bus->dev.parent->of_node;
	struct of_phandle_args reg_spec;
	const char *status;
	int i, ret;

	for (i = 0;
		of_parse_phandle_with_fixed_args(dn, NV(ivc-channels), 3,
							i, &reg_spec) == 0;
		i++) {
		struct device_node *ch_node;

		for_each_child_of_node(reg_spec.np, ch_node) {
			struct tegra_ivc_channel *chan;

			ret = of_property_read_string(ch_node,
					"status", &status);

			if (ret == 0) {
				ret = strcmp(status, "disabled");

				if (ret == 0)
					continue;
			}

			chan = tegra_ivc_channel_create(bus, ch_node,
							&bus->regions[i],
							camhsp);
			if (IS_ERR(chan)) {
				ret = PTR_ERR(chan);
				of_node_put(ch_node);
				goto error;
			}

			chan->next = bus->chans;
			bus->chans = chan;
		}
	}

	return 0;
error:
	tegra_ivc_bus_stop(bus);
	return ret;
}

/*
 * This is called during RTCPU boot to synchronize
 * (or re-synchronize in the case of PM resume).
 */
int tegra_ivc_bus_boot_sync(struct tegra_ivc_bus *bus,
	int (*iovm_setup)(struct device*, dma_addr_t))
{
	int i;

	if (IS_ERR_OR_NULL(bus))
		return 0;

	for (i = 0; i < bus->num_regions; i++) {
		int ret = (*iovm_setup)(bus->dev.parent,
				bus->regions[i].iova);
		if (ret != 0) {
			dev_info(&bus->dev, "IOVM setup error: %d\n", ret);
			return -EIO;
		}
	}

	return 0;
}
EXPORT_SYMBOL(tegra_ivc_bus_boot_sync);

static int tegra_ivc_bus_probe(struct device *dev)
{
	int ret = -ENXIO;

	if (dev->type == &tegra_ivc_channel_type) {
		struct tegra_ivc_driver *drv = to_tegra_ivc_driver(dev->driver);
		struct tegra_ivc_channel *chan = to_tegra_ivc_channel(dev);
		const struct tegra_ivc_channel_ops *ops = drv->ops.channel;

		mutex_init(&chan->ivc_wr_lock);

		BUG_ON(ops == NULL);
		if (ops->probe != NULL) {
			ret = ops->probe(chan);
			if (ret)
				return ret;
		}

		rcu_assign_pointer(chan->ops, ops);
		ret = 0;

	}

	return ret;
}

#if defined(NV_BUS_TYPE_STRUCT_REMOVE_HAS_INT_RETURN_TYPE) /* Linux v5.15 */
static int tegra_ivc_bus_remove(struct device *dev)
#else
static void tegra_ivc_bus_remove(struct device *dev)
#endif
{
	if (dev->type == &tegra_ivc_channel_type) {
		struct tegra_ivc_driver *drv = to_tegra_ivc_driver(dev->driver);
		struct tegra_ivc_channel *chan = to_tegra_ivc_channel(dev);
		const struct tegra_ivc_channel_ops *ops = drv->ops.channel;

		WARN_ON(rcu_access_pointer(chan->ops) != ops);
		RCU_INIT_POINTER(chan->ops, NULL);
		synchronize_rcu();

		if (ops->remove != NULL)
			ops->remove(chan);

	}

#if defined(NV_BUS_TYPE_STRUCT_REMOVE_HAS_INT_RETURN_TYPE) /* Linux v5.15 */
	return 0;
#endif
}

static int tegra_ivc_bus_ready_child(struct device *dev, void *data)
{
	struct tegra_ivc_driver *drv = to_tegra_ivc_driver(dev->driver);
	bool is_ready = (data != NULL) ? *(bool *)data : true;

	if (dev->type == &tegra_ivc_channel_type) {
		struct tegra_ivc_channel *chan = to_tegra_ivc_channel(dev);
		const struct tegra_ivc_channel_ops *ops;

		chan->is_ready = is_ready;
		if (!is_ready)
			atomic_inc(&chan->bus_resets);
		smp_wmb();

		if (drv != NULL) {
			rcu_read_lock();
			ops = rcu_dereference(chan->ops);
			if (ops->ready != NULL)
				ops->ready(chan, is_ready);
			rcu_read_unlock();
		} else {
			dev_warn(dev, "ivc channel driver missing\n");
		}
	}

	return 0;
}

struct bus_type tegra_ivc_bus_type = {
	.name	= "tegra-ivc-bus",
	.match	= tegra_ivc_bus_match,
	.probe	= tegra_ivc_bus_probe,
	.remove	= tegra_ivc_bus_remove,
};
EXPORT_SYMBOL(tegra_ivc_bus_type);

int tegra_ivc_driver_register(struct tegra_ivc_driver *drv)
{
	return driver_register(&drv->driver);
}
EXPORT_SYMBOL(tegra_ivc_driver_register);

void tegra_ivc_driver_unregister(struct tegra_ivc_driver *drv)
{
	return driver_unregister(&drv->driver);
}
EXPORT_SYMBOL(tegra_ivc_driver_unregister);

static int tegra_ivc_bus_parse_regions(struct tegra_ivc_bus *bus,
					struct device_node *dev_node)
{
	struct of_phandle_args reg_spec;
	int i;
	u32 mul = 0U, sum = 0U;

	/* Parse out all regions in a node */
	for (i = 0;
		of_parse_phandle_with_fixed_args(dev_node, NV(ivc-channels), 3,
							i, &reg_spec) == 0;
		i++) {
		struct device_node *ch_node;
		struct tegra_ivc_region *region = &bus->regions[i];
		u32 nframes, frame_size, size = CAMRTC_IVC_CONFIG_SIZE;
		int ret = -ENODEV;

		if (reg_spec.args_count < 3) {
			of_node_put(reg_spec.np);
			dev_err(&bus->dev, "invalid region specification\n");
			return -EINVAL;
		}

		for_each_child_of_node(reg_spec.np, ch_node) {
			ret = of_property_read_u32(ch_node, NV(frame-count),
						&nframes);
			if (ret || !nframes) {
				dev_err(&bus->dev, "missing <%s> property\n",
					NV(frame-count));
				break;
			}
			/* Round up to a power of two */
			nframes = 1 << fls(nframes - 1);

			ret = of_property_read_u32(ch_node, NV(frame-size),
						&frame_size);
			if (ret || !frame_size) {
				dev_err(&bus->dev, "missing <%s> property\n",
					NV(frame-size));
				break;
			}

			if (__builtin_mul_overflow(nframes, frame_size, &mul)) {
				dev_err(&bus->dev, "IVC frame size overflow\n");
				break;
			}

			if (__builtin_mul_overflow(2U, tegra_ivc_total_queue_size(mul), &mul)) {
				dev_err(&bus->dev, "IVC queue size overflow\n");
				break;
			}

			if (__builtin_add_overflow(size, mul, &size)) {
				dev_err(&bus->dev, "IVC size overflow\n");
				break;
			}
		}
		of_node_put(reg_spec.np);

		if (ret)
			return ret;

		region->base =
			(uintptr_t)dma_alloc_coherent(bus->dev.parent,
						size, &region->iova,
						GFP_KERNEL | __GFP_ZERO);
		if (!region->base)
			return -ENOMEM;

		region->size = size;
		region->config_size = 0;
		region->ivc_size = CAMRTC_IVC_CONFIG_SIZE;

		(void)(__builtin_add_overflow(region->iova, size, &sum));
		(void)(__builtin_sub_overflow(sum, 1U, &sum));
		dev_info(&bus->dev, "region %u: iova=0x%x-0x%x size=%u\n",
			i, (u32)region->iova, sum,
			size);
	}

	return 0;
}

static unsigned tegra_ivc_bus_count_regions(const struct device_node *dev_node)
{
	unsigned i;

	for (i = 0; of_parse_phandle_with_fixed_args(dev_node,
			NV(ivc-channels), 3, i, NULL) == 0; i++)
		;

	return i;
}

struct tegra_ivc_bus *tegra_ivc_bus_create(struct device *dev,
	struct camrtc_hsp *camhsp)
{
	struct tegra_ivc_bus *bus;
	unsigned num;
	int ret;

	num = tegra_ivc_bus_count_regions(dev->of_node);

	bus = kzalloc(sizeof(*bus) + num * sizeof(*bus->regions), GFP_KERNEL);
	if (unlikely(bus == NULL))
		return ERR_PTR(-ENOMEM);

	bus->num_regions = num;
	bus->dev.parent = dev;
	bus->dev.type = &tegra_ivc_bus_dev_type;
	bus->dev.bus = &tegra_ivc_bus_type;
	bus->dev.of_node = of_get_child_by_name(dev->of_node, "hsp");
	bus->dev.release = tegra_ivc_bus_release;
	dev_set_name(&bus->dev, "%s:ivc-bus", dev_name(dev));
	device_initialize(&bus->dev);
	pm_runtime_no_callbacks(&bus->dev);
	pm_runtime_enable(&bus->dev);

	ret = tegra_ivc_bus_parse_regions(bus, dev->of_node);
	if (ret) {
		dev_err(&bus->dev, "IVC regions setup failed: %d\n", ret);
		goto error;
	}

	ret = device_add(&bus->dev);
	if (ret) {
		dev_err(&bus->dev, "IVC instance error: %d\n", ret);
		goto error;
	}

	ret = tegra_ivc_bus_start(bus, camhsp);
	if (ret) {
		dev_err(&bus->dev, "bus start failed: %d\n", ret);
		goto error;
	}

	return bus;

error:
	put_device(&bus->dev);
	return ERR_PTR(ret);
}
EXPORT_SYMBOL(tegra_ivc_bus_create);

/*
 * Communicate RTCPU UP/DOWN state to IVC devices.
 */
void tegra_ivc_bus_ready(struct tegra_ivc_bus *bus, bool online)
{
	if (IS_ERR_OR_NULL(bus))
		return;

	device_for_each_child(&bus->dev, &online, tegra_ivc_bus_ready_child);

	if (online)
		tegra_ivc_bus_notify(bus, 0xFFFFU);
}
EXPORT_SYMBOL(tegra_ivc_bus_ready);

void tegra_ivc_bus_destroy(struct tegra_ivc_bus *bus)
{
	if (IS_ERR_OR_NULL(bus))
		return;

	pm_runtime_disable(&bus->dev);
	tegra_ivc_bus_stop(bus);
	device_unregister(&bus->dev);
}
EXPORT_SYMBOL(tegra_ivc_bus_destroy);

static __init int tegra_ivc_bus_init(void)
{
	return bus_register(&tegra_ivc_bus_type);
}

static __exit void tegra_ivc_bus_exit(void)
{
	bus_unregister(&tegra_ivc_bus_type);
}

module_init(tegra_ivc_bus_init);
module_exit(tegra_ivc_bus_exit);
MODULE_AUTHOR("Remi Denis-Courmont <remid@nvidia.com>");
MODULE_DESCRIPTION("NVIDIA Tegra IVC generic bus driver");
MODULE_LICENSE("GPL v2");