From 872cead7b68b5160ab7e247aaa42c8e1a4d17323 Mon Sep 17 00:00:00 2001 From: amitabhd Date: Tue, 14 May 2024 05:13:59 +0000 Subject: [PATCH] host1x: Emulated Syncpoint KMD Driver 1. Update Host1x-Syncpt - add syncpt-list in syncpoint-pool structure. - add exported "host1x-syncpt-*" interface and supporting private interface. 2. Update Host1x-NvHost - add support for nvhost-syncpt and nvhost-client interface and supporting private interface. - falcon and pm interface not supported 3. Update Host1x-Fence - add exported "host1x-fence-*" interface and supporting private interface. 4. Add Host1x-Poll - Per Pool polling workequeue created - add poll supporting private interface. 5. Add new IOCTL Jira HOSTX-5514 Change-Id: I187fa7ed89a3479e51b79f9d8d438ede75e1a36a Reviewed-on: https://git-master.nvidia.com/r/c/linux-nv-oot/+/3136834 Tested-by: Amitabh Dutta Reviewed-by: Bitan Biswas GVS: buildbot_gerritrpt Reviewed-by: Amitabh Dutta --- drivers/Makefile | 1 + drivers/gpu/host1x-emu/Makefile | 35 ++ drivers/gpu/host1x-emu/debug.c | 136 ++++ drivers/gpu/host1x-emu/debug.h | 43 ++ drivers/gpu/host1x-emu/dev.c | 383 +++++++++++ drivers/gpu/host1x-emu/dev.h | 99 +++ drivers/gpu/host1x-emu/fence.c | 182 ++++++ drivers/gpu/host1x-emu/fence.h | 33 + drivers/gpu/host1x-emu/hw/host1xEMU.c | 17 + drivers/gpu/host1x-emu/hw/host1xEMU.h | 12 + drivers/gpu/host1x-emu/hw/syncpt_hw.c | 71 +++ .../gpu/host1x-emu/include/linux/host1x-emu.h | 83 +++ .../gpu/host1x-emu/include/linux/nvhost-emu.h | 257 ++++++++ .../gpu/host1x-emu/include/linux/symbol-emu.h | 17 + drivers/gpu/host1x-emu/ioctl.c | 345 ++++++++++ drivers/gpu/host1x-emu/ioctl.h | 129 ++++ drivers/gpu/host1x-emu/nvhost.c | 594 ++++++++++++++++++ drivers/gpu/host1x-emu/poll.c | 152 +++++ drivers/gpu/host1x-emu/poll.h | 61 ++ drivers/gpu/host1x-emu/syncpt.c | 475 ++++++++++++++ drivers/gpu/host1x-emu/syncpt.h | 146 +++++ 21 files changed, 3271 insertions(+) create mode 100644 drivers/gpu/host1x-emu/Makefile create mode 100644 drivers/gpu/host1x-emu/debug.c create mode 100644 drivers/gpu/host1x-emu/debug.h create mode 100644 drivers/gpu/host1x-emu/dev.c create mode 100644 drivers/gpu/host1x-emu/dev.h create mode 100644 drivers/gpu/host1x-emu/fence.c create mode 100644 drivers/gpu/host1x-emu/fence.h create mode 100644 drivers/gpu/host1x-emu/hw/host1xEMU.c create mode 100644 drivers/gpu/host1x-emu/hw/host1xEMU.h create mode 100644 drivers/gpu/host1x-emu/hw/syncpt_hw.c create mode 100644 drivers/gpu/host1x-emu/include/linux/host1x-emu.h create mode 100644 drivers/gpu/host1x-emu/include/linux/nvhost-emu.h create mode 100644 drivers/gpu/host1x-emu/include/linux/symbol-emu.h create mode 100644 drivers/gpu/host1x-emu/ioctl.c create mode 100644 drivers/gpu/host1x-emu/ioctl.h create mode 100644 drivers/gpu/host1x-emu/nvhost.c create mode 100644 drivers/gpu/host1x-emu/poll.c create mode 100644 drivers/gpu/host1x-emu/poll.h create mode 100644 drivers/gpu/host1x-emu/syncpt.c create mode 100644 drivers/gpu/host1x-emu/syncpt.h diff --git a/drivers/Makefile b/drivers/Makefile index 65e0bb4c..8d09a7e9 100644 --- a/drivers/Makefile +++ b/drivers/Makefile @@ -26,6 +26,7 @@ obj-m += firmware/tegra/ obj-m += gpio/ ifdef CONFIG_TEGRA_HOST1X obj-m += gpu/ +obj-m += gpu/host1x-emu/ endif ifdef CONFIG_HWMON obj-m += hwmon/ diff --git a/drivers/gpu/host1x-emu/Makefile b/drivers/gpu/host1x-emu/Makefile new file mode 100644 index 00000000..54d472eb --- /dev/null +++ b/drivers/gpu/host1x-emu/Makefile @@ -0,0 +1,35 @@ +# SPDX-License-Identifier: GPL-2.0-only +# Copyright (c) 2024 NVIDIA CORPORATION. All rights reserved. + +# Enable below flag for hypervisor build configuration +ifeq ($(NV_BUILD_CONFIGURATION_EMU_SYNCPT),1) +ccflags-y += -DHOST1X_EMU_HYPERVISOR +endif + +ifeq ($(NV_BUILD_CONFIGURATION_EXPOSING_T26X), 1) +LINUXINCLUDE += -I$(srctree.nvidia-oot)/drivers/gpu/host1x-emu/include +# Enable for verification in VDK. Below allow Emulated Syncpoint driver +# to co-exits with "Host1x" driver by exporting modified kernel symbol. +ccflags-y += -DCONFIG_TEGRA_HOST1X_EMU_DBG_SYMBL + +# Enable debug +CONFIG_TEGRA_HOST1X_EMU_SYNCPT_DEBUG := 1 +endif + +host1x-emu-y += \ + dev.o \ + poll.o \ + ioctl.o \ + fence.o \ + nvhost.o \ + syncpt.o \ + hw/host1xEMU.o + +ifdef CONFIG_TEGRA_HOST1X_EMU_SYNCPT_DEBUG +host1x-emu-y += \ + debug.o + +ccflags-y += -DHOST1X_EMU_SYNCPT_DEGUB +endif + +obj-m := host1x-emu.o diff --git a/drivers/gpu/host1x-emu/debug.c b/drivers/gpu/host1x-emu/debug.c new file mode 100644 index 00000000..9d7255de --- /dev/null +++ b/drivers/gpu/host1x-emu/debug.c @@ -0,0 +1,136 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: GPL-2.0-only + */ +#include +#include +#include +#include +#include +#include + +#include "dev.h" +#include "debug.h" +#include "include/linux/symbol-emu.h" + +void host1x_debug_output(struct output *o, const char *fmt, ...) +{ + va_list args; + int len; + + va_start(args, fmt); + len = vsnprintf(o->buf, sizeof(o->buf), fmt, args); + va_end(args); + + o->fn(o->ctx, o->buf, len, false); +} + +void host1x_debug_cont(struct output *o, const char *fmt, ...) +{ + va_list args; + int len; + + va_start(args, fmt); + len = vsnprintf(o->buf, sizeof(o->buf), fmt, args); + va_end(args); + + o->fn(o->ctx, o->buf, len, true); +} + +static void show_syncpts(struct host1x *m, struct output *o, bool show_all) +{ + unsigned long irqflags; + struct list_head *pos; + unsigned int i; + + host1x_debug_output(o, "---- Emulated Syncpts ----\n"); + for (i = 0; i < host1x_syncpt_nb_pts(m); i++) { + u32 max = HOST1X_EMU_EXPORT_CALL(host1x_syncpt_read_max(m->syncpt + i)); + u32 min = host1x_syncpt_load(m->syncpt + i); + unsigned int waiters = 0; + + spin_lock_irqsave(&m->syncpt[i].fences.lock, irqflags); + list_for_each(pos, &m->syncpt[i].fences.list) + waiters++; + spin_unlock_irqrestore(&m->syncpt[i].fences.lock, irqflags); + + if (!kref_read(&m->syncpt[i].ref)) + continue; + + if (!min && !max && !waiters) + continue; + + host1x_debug_output(o, + "id %u (%s) min %d max %d (%d waiters)\n", + i, m->syncpt[i].name, min, max, waiters); + } + host1x_debug_output(o, "\n"); +} + +static int host1x_debug_show(struct seq_file *s, void *unused) +{ + struct output o = { + .fn = write_to_seqfile, + .ctx = s + }; + + show_syncpts(s->private, &o, false); + return 0; +} + +void host1x_debug_dump_syncpts(struct host1x *host1x) +{ + struct output o = { + .fn = write_to_printk + }; + + show_syncpts(host1x, &o, false); +} + +static int host1x_debug_open(struct inode *inode, struct file *file) +{ + return single_open(file, host1x_debug_show, inode->i_private); +} + +static const struct file_operations host1x_debug_fops = { + .open = host1x_debug_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static void host1x_debugfs_init(struct host1x *host1x) +{ + struct dentry *de; + char dir_name[64]; + int numa_node = dev_to_node(host1x->dev); + + if (numa_node != NUMA_NO_NODE) + sprintf(dir_name, "tegra-host1x-emu.%d", numa_node); + else + sprintf(dir_name, "tegra-host1x-emu"); + + de = debugfs_create_dir(dir_name, NULL); + host1x->debugfs = de; + + /* Status File*/ + debugfs_create_file("status", S_IRUGO, de, host1x, &host1x_debug_fops); +} + +static void host1x_debugfs_exit(struct host1x *host1x) +{ + debugfs_remove_recursive(host1x->debugfs); +} + +void host1x_debug_init(struct host1x *host1x) +{ + if (IS_ENABLED(CONFIG_DEBUG_FS)) { + host1x_debugfs_init(host1x); + } +} + +void host1x_debug_deinit(struct host1x *host1x) +{ + if (IS_ENABLED(CONFIG_DEBUG_FS)) + host1x_debugfs_exit(host1x); +} diff --git a/drivers/gpu/host1x-emu/debug.h b/drivers/gpu/host1x-emu/debug.h new file mode 100644 index 00000000..66e3153a --- /dev/null +++ b/drivers/gpu/host1x-emu/debug.h @@ -0,0 +1,43 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: GPL-2.0-only + */ +#ifndef __HOST1X_EMU_DEBUG_H +#define __HOST1X_EMU_DEBUG_H + +#include +#include + +struct host1x; + +struct output { + void (*fn)(void *ctx, const char *str, size_t len, bool cont); + void *ctx; + char buf[256]; +}; + +static inline void write_to_seqfile(void *ctx, const char *str, size_t len, + bool cont) +{ + seq_write((struct seq_file *)ctx, str, len); +} + +static inline void write_to_printk(void *ctx, const char *str, size_t len, + bool cont) +{ + if (cont) + pr_cont("%s", str); + else + pr_info("%s", str); +} + +void __printf(2, 3) host1x_debug_output(struct output *o, const char *fmt, ...); +void __printf(2, 3) host1x_debug_cont(struct output *o, const char *fmt, ...); + +void host1x_debug_init(struct host1x *host1x); +void host1x_debug_deinit(struct host1x *host1x); +void host1x_debug_dump_syncpts(struct host1x *host1x); + +void host1x_sanity_init(struct host1x *host1x); +void nvhost_sanity_init(struct host1x *host1x); +#endif diff --git a/drivers/gpu/host1x-emu/dev.c b/drivers/gpu/host1x-emu/dev.c new file mode 100644 index 00000000..dd1c1eb6 --- /dev/null +++ b/drivers/gpu/host1x-emu/dev.c @@ -0,0 +1,383 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: GPL-2.0-only + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if IS_ENABLED(CONFIG_ARM_DMA_USE_IOMMU) +#include +#endif + +#include + +#include "dev.h" +#include "debug.h" +#include "hw/host1xEMU.h" + +#define HOST1X_POOL_MSEC_PERIOD 70 /*70msec*/ +#define HOST1X_SYNCPT_POOL_BASE(x) (x*2+0) +#define HOST1X_SYNCPT_POOL_SIZE(x) (x*2+1) + +static const struct host1x_info host1xEmu_info = { + .nb_pts = 1024, + .init = host1xEMU_init, + .dma_mask = DMA_BIT_MASK(40), +}; + +static const struct of_device_id host1x_of_match[] = { + { .compatible = "nvidia,tegraEmu-host1x", .data = &host1xEmu_info, }, + { }, +}; +MODULE_DEVICE_TABLE(of, host1x_of_match); + +void host1x_sync_writel(struct host1x *host1x, u32 v, u32 r) +{ +#ifdef HOST1X_EMU_HYPERVISOR + void __iomem *sync_mem = host1x->syncpt_va_apt; + writel(v, (void __iomem*)((u8*)sync_mem + r)); +#else + unsigned int *sync_mem = (unsigned int*)((u8*)host1x->syncpt_va_apt + r); + *sync_mem = v; +#endif +} + +u32 host1x_sync_readl(struct host1x *host1x, u32 r) +{ +#ifdef HOST1X_EMU_HYPERVISOR + void __iomem *sync_mem = host1x->syncpt_va_apt; + return readl((void __iomem*)((u8*)sync_mem + r)); +#else + unsigned int *sync_mem = (unsigned int*)((u8*)host1x->syncpt_va_apt + r); + return(*sync_mem); +#endif +} + +static int host1x_get_assigned_resources(struct host1x *host) +{ + int err; + u32 vals[4]; +#ifndef HOST1X_EMU_HYPERVISOR + unsigned long page_addr = 0; + unsigned int syncpt_pwr_2; +#endif + struct device_node *np = host->dev->of_node; + + err = of_property_read_u32_array(np, "nvidia,syncpoints", vals, 2); + if (err == 0) { + host->syncpt_base = vals[0]; + host->syncpt_end = vals[0] + vals[1]; + } else { + /** + * In case of no syncpoint property defined, use pre-defined syncpoints + * info. + */ + host->syncpt_base = 0; + host->syncpt_end = host->info->nb_pts; + dev_err(host->dev, "Host1x-EMU: invalid/no nvidia,syncpoints property: %d\n", err); + } + + err = of_property_read_u32_array(np, "nvidia,polling-interval", vals, 1); + if (err == 0) { + host->polling_intrval = vals[0]; + } else { + host->polling_intrval = HOST1X_POOL_MSEC_PERIOD; + } + +#ifdef HOST1X_EMU_HYPERVISOR + err = of_property_read_u32_array(np, "nvidia,syncpoints-mem", vals, 4); + if (err == 0) { + host->syncpt_phy_apt = ((uint64_t)vals[0] << 32U) | ((uint64_t)vals[1]); + host->syncpt_page_size = vals[2]; + host->syncpt_count = vals[3]; +#ifdef HOST1X_EMU_SYNCPT_DEGUB + /** + * TODO: Remove debug prints + */ + pr_info("Host1x-EMU: Syncpoint Physical Addr:%llx\n", host->syncpt_phy_apt); + pr_info("Host1x-EMU: Syncpoint Page Size :%u\n", vals[2]); + pr_info("Host1x-EMU: Syncpoint Count :%u\n", vals[3]); + pr_info("Host1x-EMU: Syncpoint Pooling Interval :%u\n", host->polling_intrval); + pr_info("Host1x-EMU: OS Scheduling resolution :%u\n", HZ); +#endif + } else { + dev_err(host->dev, + "Host1x-EMU:invalid nvidia,syncpoints-mem property: %d\n", err); + return err; + } + + if ((host->syncpt_end + host->syncpt_base) > host->syncpt_count) { + dev_err(host->dev, + "Host1x-EMU: Invalid syncpoint property, Syncpoint excedes range: %d\n", -EINVAL ); + return -EINVAL ; + } + + host->syncpt_va_apt = devm_ioremap(host->dev, host->syncpt_phy_apt, + (host->syncpt_count*host->syncpt_page_size)); + if (IS_ERR(host->syncpt_va_apt)) { + return PTR_ERR(host->syncpt_va_apt); + } +#else + /** + * TODO: Check if we can set this from DT. + * Currently for native OS using static value for number of syncpoint + */ + host->syncpt_count = host->info->nb_pts; + if ((host->syncpt_end + host->syncpt_base) > host->syncpt_count) { + dev_err(host->dev, + "Host1x-EMU: Invalid syncpoint property, Syncpoint excedes range: %d\n", -EINVAL ); + return -EINVAL; + } + + syncpt_pwr_2 = order_base_2(host->syncpt_count); + page_addr = __get_free_pages(GFP_KERNEL, syncpt_pwr_2); + if (unlikely((void*)page_addr == NULL)) { + dev_err(host->dev, + "Host1x-EMU: Syncpoint Carveout allocation failed: %d\n", (-ENOMEM)); + return -ENOMEM; + } + host->syncpt_phy_apt = __pa(page_addr); + host->syncpt_va_apt = (void*)page_addr; + host->syncpt_page_size = PAGE_SIZE; + /*Resetting pool to zero value*/ + memset((void*)page_addr, 0, PAGE_SIZE << syncpt_pwr_2); +#endif + + pr_info("Host1x-EMU: Syncpoint-Base:%d Syncpoint-End:%d Syncpoint-Count:%d\n", + host->syncpt_base, host->syncpt_end, host->syncpt_count); + return 0; +} + +static int host1x_get_syncpt_pools(struct host1x *host) +{ + struct device_node *np = host->dev->of_node; + int ret; + int i; + + ret = of_property_count_strings(np, "nvidia,syncpoint-pool-names"); + if (ret < 0) { + /* No pools defined, only read only pool*/ + dev_err(host->dev, "Host1x-EMU: Invalid nvidia,syncpoint-pool-names property: %d\n", ret); + ret = 0; + } + host->num_pools = ret; + + /* + * Adding 1 here for RO-pool, which is used for all RO-syncpoint for this VM. + * By default all syncpoint are assigned to RO-Pool. + * + * Further pool initialization and syncpoint initialization will re-assign + * R/W syncpoint to appropiate pool based on calibration data. + * + * Note: RO-Pool variable "sp_base/sp_end" are not updated to correct syncpoint + * range after all pool/syncpoint initialization due to following reason + * 1. Variable are not used after pool/syncpoint object initialization + * 2. Two Variable are not sufficient to represent fragmented RO range + * Ex |---RO Sync Range1--|--RW range--|---RO Sync Range2--| + */ + host->ro_pool_id = host->num_pools; + host->pools = devm_kcalloc(host->dev, host->num_pools + 1, + sizeof(struct host1x_syncpt_pool), GFP_KERNEL); + if (!host->pools) { + dev_err(host->dev, "Host1x-EMU: Failed allocating pool memory\n"); + return -ENOMEM; + } + host->pools[host->ro_pool_id].sp_base = 0; + host->pools[host->ro_pool_id].sp_end = host->syncpt_count; + + /* Return if only read only pools*/ + if (host->num_pools == 0) { + return 0; + } + + for (i = 0; i < host->num_pools; i++) { + struct host1x_syncpt_pool *pool = &host->pools[i]; + + ret = of_property_read_string_index(np, "nvidia,syncpoint-pool-names", i, &pool->name); + if (ret) { + dev_err(host->dev, "Host1x-EMU: Invalid nvidia,syncpoint-pool-names property: %d\n", ret); + return ret; + } + + ret = of_property_read_u32_index(np, "nvidia,syncpoint-pools", HOST1X_SYNCPT_POOL_BASE(i), &pool->sp_base); + if (!ret) { + ret = of_property_read_u32_index(np, "nvidia,syncpoint-pools", HOST1X_SYNCPT_POOL_SIZE(i), &pool->sp_end); + if (ret) { + dev_err(host->dev, "Host1x-EMU: Invalid nvidia,syncpoint-pools property: %d\n", ret); + return ret; + } + } else { + dev_err(host->dev, "Host1x-EMU: Error in read, invalid nvidia,syncpoint-pools property: %d\n", ret); + return ret; + } + + pool->sp_end = pool->sp_base + pool->sp_end; + if (pool->sp_end > host->syncpt_count) { + pool->sp_end = host->syncpt_count; + } + } + return 0; +} + +static int host1x_probe(struct platform_device *pdev) +{ + int err; + struct host1x *host; + + host = devm_kzalloc(&pdev->dev, sizeof(*host), GFP_KERNEL); + if (!host) + return -ENOMEM; + + host->info = of_device_get_match_data(&pdev->dev); + if (host->info == NULL) { + dev_err(&pdev->dev, "Host1x-EMU: platform match data not found\n"); + return -EINVAL; + } + host->dev = &pdev->dev; + + /* set common host1x device data */ + platform_set_drvdata(pdev, host); + + host->dev->dma_parms = &host->dma_parms; + dma_set_max_seg_size(host->dev, UINT_MAX); + + if (host->info->init) { + err = host->info->init(host); + if (err) + return err; + } + + err = host1x_get_assigned_resources(host); + if (err) + return err; + + err = host1x_get_syncpt_pools(host); + if (err) + return err; + + err = host1x_syncpt_init(host); + if (err) { + dev_err(&pdev->dev, "Host1x-EMU: failed to initialize syncpts\n"); + return err; + } + + err = host1x_poll_init(host); + if (err) { + dev_err(&pdev->dev, "Host1x-EMU: failed to initialize interrupts\n"); + goto deinit_syncpt; + } + + pm_runtime_enable(&pdev->dev); + /* the driver's code isn't ready yet for the dynamic RPM */ + err = pm_runtime_resume_and_get(&pdev->dev); + if (err) { + goto pm_disable; + } + + host1x_user_init(host); + +#ifdef HOST1X_EMU_SYNCPT_DEGUB + host1x_debug_init(host); +#endif + + err = devm_of_platform_populate(&pdev->dev); + if (err < 0) { + pr_info("Host1x-EMU: Failed to populate device from DT\n"); + goto deinit_debugfs; + } + + /* Start pool polling thread*/ + host1x_poll_start(host); + + pr_info("Host1x-EMU: Probe Done\n"); + return 0; + +deinit_debugfs: +#ifdef HOST1X_EMU_SYNCPT_DEGUB + host1x_debug_deinit(host); +#endif + pm_runtime_put_sync_suspend(&pdev->dev); +pm_disable: + pm_runtime_disable(&pdev->dev); +deinit_syncpt: + host1x_syncpt_deinit(host); + return err; +} + +static int host1x_remove(struct platform_device *pdev) +{ + struct host1x *host = platform_get_drvdata(pdev); + +#ifdef HOST1X_EMU_SYNCPT_DEGUB + host1x_debug_deinit(host); +#endif + pm_runtime_force_suspend(&pdev->dev); + host1x_syncpt_deinit(host); + return 0; +} + +static int __maybe_unused host1x_runtime_suspend(struct device *dev) +{ + struct host1x *host = dev_get_drvdata(dev); + + host1x_poll_stop(host); + host1x_syncpt_save(host); + return 0; +} + +static int __maybe_unused host1x_runtime_resume(struct device *dev) +{ + struct host1x *host = dev_get_drvdata(dev); + + host1x_syncpt_restore(host); + host1x_poll_start(host); + return 0; +} + +static const struct dev_pm_ops host1x_pm_ops = { + SET_RUNTIME_PM_OPS(NULL, NULL, NULL) + SET_SYSTEM_SLEEP_PM_OPS(host1x_runtime_suspend, host1x_runtime_resume) +}; + +static struct platform_driver tegra_host1x_driver = { + .driver = { + .name = "tegra-host1x-emu", + .of_match_table = host1x_of_match, + .pm = &host1x_pm_ops, + }, + .probe = host1x_probe, + .remove = host1x_remove, +}; + +static struct platform_driver * const drivers[] = { + &tegra_host1x_driver, +}; + +static int __init tegra_host1x_init(void) +{ + int err; + + err = platform_register_drivers(drivers, ARRAY_SIZE(drivers)); + return err; +} +module_init(tegra_host1x_init); + +static void __exit tegra_host1x_exit(void) +{ + platform_unregister_drivers(drivers, ARRAY_SIZE(drivers)); +} +module_exit(tegra_host1x_exit); + +MODULE_AUTHOR("Amitabh Dutta "); +MODULE_AUTHOR("Amitabh Dutta "); +MODULE_DESCRIPTION("Emulated Host1x Syncpoint Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/gpu/host1x-emu/dev.h b/drivers/gpu/host1x-emu/dev.h new file mode 100644 index 00000000..6b702203 --- /dev/null +++ b/drivers/gpu/host1x-emu/dev.h @@ -0,0 +1,99 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: GPL-2.0-only + */ +#ifndef HOST1X_DEV_H +#define HOST1X_DEV_H + +#include +#include +#include +#include +#include +#include + +#include "poll.h" +#include "syncpt.h" + +struct output; +struct dentry; +struct host1x_syncpt; + +int host1x_user_init(struct host1x *host); +void host1x_sync_writel(struct host1x *host1x, u32 r, u32 v); +u32 host1x_sync_readl(struct host1x *host1x, u32 r); + +struct host1x_syncpt_ops { + void (*restore)(struct host1x_syncpt *syncpt); + u32 (*load)(struct host1x_syncpt *syncpt); + int (*cpu_incr)(struct host1x_syncpt *syncpt); +}; + + +struct host1x_info { + u64 dma_mask; /* mask of addressable memory */ + unsigned int nb_pts; /* host1x: number of syncpoints supported */ + int (*init)(struct host1x *host1x); /* initialize per SoC ops */ +}; + +struct host1x { + const struct host1x_info *info; + + struct device *dev; + struct dentry *debugfs; + struct device_dma_parameters dma_parms; + + /* Charected-Dev*/ + int major; + int next_minor; + struct class *host1x_class; + struct cdev cdev; + struct device *ctrl; + + /* Resources */ + struct host1x_syncpt *syncpt; + struct host1x_syncpt_base *bases; + + /* Resources accessible by this VM */ + unsigned int syncpt_end; + unsigned int syncpt_base; + unsigned int syncpt_count; + unsigned int polling_intrval; +#ifdef HOST1X_EMU_HYPERVISOR + void __iomem *syncpt_va_apt; /* syncpoint apperture mapped in kernel space */ +#else + void *syncpt_va_apt; +#endif + u64 syncpt_phy_apt; /* syncpoint page size */ + unsigned int syncpt_page_size; /* syncpoint page size */ + + /* Restricted syncpoint pools */ + unsigned int num_pools; + unsigned int ro_pool_id; + struct host1x_syncpt_pool *pools; + + /* Resource Ops */ + const struct host1x_syncpt_ops *syncpt_op; + + /* Resources Lock */ + struct mutex syncpt_mutex; +}; + +static inline void host1x_hw_syncpt_restore(struct host1x *host, + struct host1x_syncpt *sp) +{ + host->syncpt_op->restore(sp); +} + +static inline u32 host1x_hw_syncpt_load(struct host1x *host, + struct host1x_syncpt *sp) +{ + return host->syncpt_op->load(sp); +} + +static inline int host1x_hw_syncpt_cpu_incr(struct host1x *host, + struct host1x_syncpt *sp) +{ + return host->syncpt_op->cpu_incr(sp); +} +#endif diff --git a/drivers/gpu/host1x-emu/fence.c b/drivers/gpu/host1x-emu/fence.c new file mode 100644 index 00000000..41e524e6 --- /dev/null +++ b/drivers/gpu/host1x-emu/fence.c @@ -0,0 +1,182 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: GPL-2.0-only + */ +#include +#include +#include +#include +#include +#include +#include +#include "dev.h" +#include "fence.h" +#include "poll.h" +#include "syncpt.h" + +static const char *host1x_syncpt_fence_get_driver_name(struct dma_fence *f) +{ + return "host1x-emu"; +} + +static const char *host1x_syncpt_fence_get_timeline_name(struct dma_fence *f) +{ + return "syncpoint"; +} + +static struct host1x_syncpt_fence *to_host1x_fence(struct dma_fence *f) +{ + return container_of(f, struct host1x_syncpt_fence, dfence); +} + +static bool host1x_syncpt_fence_signaled(struct dma_fence *f) +{ + struct host1x_syncpt_fence *sf = to_host1x_fence(f); + + return host1x_syncpt_is_expired(sf->sp, sf->threshold) || f->error; +} + +static bool host1x_syncpt_fence_enable_signaling(struct dma_fence *f) +{ + struct host1x_syncpt_fence *sf = to_host1x_fence(f); + + if (host1x_syncpt_is_expired(sf->sp, sf->threshold)) + return false; + + /* Reference for poll path. */ + dma_fence_get(f); + + /* + * The dma_fence framework requires the fence driver to keep a + * reference to any fences for which 'enable_signaling' has been + * called (and that have not been signalled). + * + * We cannot currently always guarantee that all fences get signalled + * or cancelled. As such, for such situations, set up a timeout, so + * that long-lasting fences will get reaped eventually. + */ + if (sf->timeout) { + /* Reference for timeout path. */ + dma_fence_get(f); + schedule_delayed_work(&sf->timeout_work, msecs_to_jiffies(30000)); + } + + host1x_poll_add_fence_locked(sf->sp->host, sf); + + return true; +} + +const struct dma_fence_ops host1x_syncpt_fence_ops = { + .get_driver_name = host1x_syncpt_fence_get_driver_name, + .get_timeline_name = host1x_syncpt_fence_get_timeline_name, + .enable_signaling = host1x_syncpt_fence_enable_signaling, + .signaled = host1x_syncpt_fence_signaled, +}; + +static void host1x_fence_timeout_handler(struct work_struct *work) +{ + struct delayed_work *dwork = (struct delayed_work *)work; + struct host1x_syncpt_fence *sf = + container_of(dwork, struct host1x_syncpt_fence, timeout_work); + + if (atomic_xchg(&sf->signaling, 1)) { + /* Already on poll path, drop timeout path reference if any. */ + if (sf->timeout) + dma_fence_put(&sf->dfence); + return; + } + + if (host1x_poll_remove_fence(sf->sp->host, sf)) { + /* + * Managed to remove fence from queue, so it's safe to drop + * the poll path's reference. + */ + dma_fence_put(&sf->dfence); + } + dma_fence_set_error(&sf->dfence, -ETIMEDOUT); + dma_fence_signal(&sf->dfence); + + /* Drop timeout path reference if any. */ + if (sf->timeout) + dma_fence_put(&sf->dfence); +} + +void host1x_fence_signal(struct host1x_syncpt_fence *sf, ktime_t ts) +{ + if (atomic_xchg(&sf->signaling, 1)) { + /* + * Already on timeout path, but we removed the fence before + * timeout path could, so drop poll path reference. + */ + dma_fence_put(&sf->dfence); + return; + } + + if (sf->timeout && cancel_delayed_work(&sf->timeout_work)) { + /* + * We know that the timeout path will not be entered. + * Safe to drop the timeout path's reference now. + */ + dma_fence_put(&sf->dfence); + } + + dma_fence_signal_timestamp_locked(&sf->dfence, ts); + + /*Drop poll path reference*/ + dma_fence_put(&sf->dfence); +} + +HOST1X_EMU_EXPORT_DECL(struct dma_fence*, host1x_fence_create(struct host1x_syncpt *sp, + u32 threshold, + bool timeout)) +{ + struct host1x_syncpt_fence *fence; + + if (!tegra_platform_is_silicon()) { + dev_info_once(sp->host->dev, "fence timeout disabled due to pre-silicon platform\n"); + timeout = false; + } + + /* Allocate memory for Host1x-Fence*/ + fence = kzalloc(sizeof(*fence), GFP_KERNEL); + if (!fence) { + return ERR_PTR(-ENOMEM); + } + fence->sp = sp; + fence->threshold = threshold; + fence->timeout = timeout; + + dma_fence_init(&fence->dfence, + &host1x_syncpt_fence_ops, + &sp->fences.lock, + dma_fence_context_alloc(1), + 0); + INIT_DELAYED_WORK(&fence->timeout_work, host1x_fence_timeout_handler); + return &fence->dfence; +} +HOST1X_EMU_EXPORT_SYMBOL(host1x_fence_create); + +HOST1X_EMU_EXPORT_DECL(int, host1x_fence_extract(struct dma_fence *dfence, u32 *id, u32 *threshold)) +{ + struct host1x_syncpt_fence *sf; + + if (dfence->ops != &host1x_syncpt_fence_ops) + return -EINVAL; + + sf = to_host1x_fence(dfence); + + *id = sf->sp->id; + *threshold = sf->threshold; + + return 0; +} +HOST1X_EMU_EXPORT_SYMBOL(host1x_fence_extract); + +HOST1X_EMU_EXPORT_DECL(void, host1x_fence_cancel(struct dma_fence *dfence)) +{ + struct host1x_syncpt_fence *sf = to_host1x_fence(dfence); + + schedule_delayed_work(&sf->timeout_work, 0); + flush_delayed_work(&sf->timeout_work); +} +HOST1X_EMU_EXPORT_SYMBOL(host1x_fence_cancel); diff --git a/drivers/gpu/host1x-emu/fence.h b/drivers/gpu/host1x-emu/fence.h new file mode 100644 index 00000000..820bac86 --- /dev/null +++ b/drivers/gpu/host1x-emu/fence.h @@ -0,0 +1,33 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: GPL-2.0-only + */ +#ifndef HOST1X_EMU_FENCE_H +#define HOST1X_EMU_FENCE_H + +#include +#include +#include +#include +#include +#include + +struct host1x; +struct host1x_syncpt; + +struct host1x_syncpt_fence { + bool timeout; + atomic_t signaling; + u32 threshold; + struct dma_fence dfence; + struct host1x_syncpt *sp; + struct delayed_work timeout_work; + + /** + * Used for adding into syncpoint fence-list + */ + struct list_head list; +}; + +void host1x_fence_signal(struct host1x_syncpt_fence *fence, ktime_t ts); +#endif diff --git a/drivers/gpu/host1x-emu/hw/host1xEMU.c b/drivers/gpu/host1x-emu/hw/host1xEMU.c new file mode 100644 index 00000000..7f8b24ee --- /dev/null +++ b/drivers/gpu/host1x-emu/hw/host1xEMU.c @@ -0,0 +1,17 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: GPL-2.0-only + */ + +/* include hw specification */ +#include "host1xEMU.h" + +#include "../dev.h" + +#include "syncpt_hw.c" + +int host1xEMU_init(struct host1x *host) +{ + host->syncpt_op = &host1x_syncpt_ops; + return 0; +} diff --git a/drivers/gpu/host1x-emu/hw/host1xEMU.h b/drivers/gpu/host1x-emu/hw/host1xEMU.h new file mode 100644 index 00000000..90329b65 --- /dev/null +++ b/drivers/gpu/host1x-emu/hw/host1xEMU.h @@ -0,0 +1,12 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: GPL-2.0-only + */ +#ifndef HOST1X_HOST1X08_H +#define HOST1X_HOST1X08_H + +struct host1x; + +int host1xEMU_init(struct host1x *host); + +#endif diff --git a/drivers/gpu/host1x-emu/hw/syncpt_hw.c b/drivers/gpu/host1x-emu/hw/syncpt_hw.c new file mode 100644 index 00000000..72e4607b --- /dev/null +++ b/drivers/gpu/host1x-emu/hw/syncpt_hw.c @@ -0,0 +1,71 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: GPL-2.0-only + */ +#include + +#include "../dev.h" +#include "../syncpt.h" + +#define HOST1X_SYNC_SYNCPT(h, x) ((h) * (x)) + +/* + * Write the current syncpoint value back to hw. + */ +static void syncpt_restore(struct host1x_syncpt *sp) +{ + u32 min = HOST1X_EMU_EXPORT_CALL(host1x_syncpt_read_min(sp)); + struct host1x *host = sp->host; + + host1x_sync_writel(host, min, HOST1X_SYNC_SYNCPT(host->syncpt_page_size, sp->id)); +} + +/* + * Updates the last value read from hardware. + */ +static u32 syncpt_load(struct host1x_syncpt *sp) +{ + u32 old, live; + struct host1x *host = sp->host; + + /* Loop in case there's a race writing to min_val */ + do { + old = HOST1X_EMU_EXPORT_CALL(host1x_syncpt_read_min(sp)); + live = host1x_sync_readl(host, HOST1X_SYNC_SYNCPT(host->syncpt_page_size, sp->id)); + } while ((u32)atomic_cmpxchg(&sp->min_val, old, live) != old); + + if (!host1x_syncpt_check_max(sp, live)) + dev_err(host->dev, "%s failed: id=%u, min=%d, max=%d\n", + __func__, sp->id, HOST1X_EMU_EXPORT_CALL(host1x_syncpt_read_min(sp)), + HOST1X_EMU_EXPORT_CALL(host1x_syncpt_read_max(sp))); + + return live; +} + +/* + * Write a cpu syncpoint increment to the hardware, without touching + * the cache. + */ +static int syncpt_cpu_incr(struct host1x_syncpt *sp) +{ + struct host1x *host = sp->host; + u32 live; + + if (!host1x_syncpt_client_managed(sp) && host1x_syncpt_idle(sp)) { + dev_err(host->dev, "%s failed: Syncpoint id=%u increment\n", + __func__,sp->id); + return -EINVAL; + } + + live = host1x_sync_readl(host, HOST1X_SYNC_SYNCPT(host->syncpt_page_size, sp->id)); + host1x_sync_writel(host, (live + 1U), + HOST1X_SYNC_SYNCPT(host->syncpt_page_size, sp->id)); + wmb(); + return 0; +} + +static const struct host1x_syncpt_ops host1x_syncpt_ops = { + .restore = syncpt_restore, + .load = syncpt_load, + .cpu_incr = syncpt_cpu_incr, +}; diff --git a/drivers/gpu/host1x-emu/include/linux/host1x-emu.h b/drivers/gpu/host1x-emu/include/linux/host1x-emu.h new file mode 100644 index 00000000..76388b75 --- /dev/null +++ b/drivers/gpu/host1x-emu/include/linux/host1x-emu.h @@ -0,0 +1,83 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: GPL-2.0-only + */ +#ifndef __LINUX_HOST1X_H +#define __LINUX_HOST1X_H + +#include +#include +#include +#include +#include +#include +#include + +/** + * TODO: Remove header after pre-silicon verification + * This header is required till emulation driver verification is done on Tho-r + * VDK platform. Since we are exporting new kernel symbol, header with declarion + * of modified symbols is required + * + * Changed cannot be added in orignal Host1x driver header files ("linux/nvhost.h" + * and "host1x-next.h") due to Host1x driver and other dependent drivers. Note in + * Tho-r verification config both the driver will co-exist. + * + * Some data structure are also re-delacred in this header for case where client + * driver wants to verify over emulation driver. + */ + +/* + * host1x syncpoints + */ +#define HOST1X_SYNCPT_CLIENT_MANAGED (1 << 0) +#define HOST1X_SYNCPT_HAS_BASE (1 << 1) +#define HOST1X_SYNCPT_GPU (1 << 2) + +struct host1x; +struct host1x_syncpt; + +HOST1X_EMU_EXPORT_DECL(u64, host1x_get_dma_mask(struct host1x *host1x)); + +HOST1X_EMU_EXPORT_DECL(struct host1x_syncpt*, host1x_syncpt_get_by_id( + struct host1x *host, u32 id)); + +HOST1X_EMU_EXPORT_DECL(struct host1x_syncpt*, host1x_syncpt_get_by_id_noref( + struct host1x *host, u32 id)); + +HOST1X_EMU_EXPORT_DECL(struct host1x_syncpt*, host1x_syncpt_get( + struct host1x_syncpt *sp)); + +HOST1X_EMU_EXPORT_DECL(u32, host1x_syncpt_id(struct host1x_syncpt *sp)); + +HOST1X_EMU_EXPORT_DECL(u32, host1x_syncpt_read_min(struct host1x_syncpt *sp)); + +HOST1X_EMU_EXPORT_DECL(u32, host1x_syncpt_read_max(struct host1x_syncpt *sp)); + +HOST1X_EMU_EXPORT_DECL(u32, host1x_syncpt_read(struct host1x_syncpt *sp)); + +HOST1X_EMU_EXPORT_DECL(int, host1x_syncpt_incr(struct host1x_syncpt *sp)); + +HOST1X_EMU_EXPORT_DECL(u32, host1x_syncpt_incr_max(struct host1x_syncpt *sp, + u32 incrs)); + +HOST1X_EMU_EXPORT_DECL(int, host1x_syncpt_wait_ts(struct host1x_syncpt *sp, + u32 thresh, long timeout, u32 *value, ktime_t *ts)); + +HOST1X_EMU_EXPORT_DECL(int, host1x_syncpt_wait(struct host1x_syncpt *sp, + u32 thresh, long timeout, u32 *value)); + +HOST1X_EMU_EXPORT_DECL(void, host1x_syncpt_put(struct host1x_syncpt *sp)); + +HOST1X_EMU_EXPORT_DECL(struct host1x_syncpt*, host1x_syncpt_alloc(struct host1x *host, + unsigned long flags, + const char *name)); + +HOST1X_EMU_EXPORT_DECL(struct dma_fence*, host1x_fence_create(struct host1x_syncpt *sp, u32 threshold, + bool timeout)); + +HOST1X_EMU_EXPORT_DECL(int, host1x_fence_extract(struct dma_fence *fence, u32 *id, u32 *threshold)); + +HOST1X_EMU_EXPORT_DECL(void, host1x_fence_cancel(struct dma_fence *fence)); + +#endif diff --git a/drivers/gpu/host1x-emu/include/linux/nvhost-emu.h b/drivers/gpu/host1x-emu/include/linux/nvhost-emu.h new file mode 100644 index 00000000..4b1e0292 --- /dev/null +++ b/drivers/gpu/host1x-emu/include/linux/nvhost-emu.h @@ -0,0 +1,257 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: GPL-2.0-only + */ +#ifndef __LINUX_NVHOST_H +#define __LINUX_NVHOST_H + +#include +#include +#include +#include +#include + +/** + * TODO: Remove header after pre-silicon verification + * This header is required till emulation driver verification is done on Tho-r + * VDK platform. Since we are exporting new kernel symbol, header with declarion + * of modified symbols is required + * + * Changed cannot be added in orignal Host1x driver header files ("linux/nvhost.h" + * and "host1x-next.h") due to Host1x driver and other dependent drivers. Note in + * Tho-r verification config both the driver will co-exist. + * + * Some data structure are also re-delacred in this header for case where client + * driver wants to verify over emulation driver. + */ + +struct nvhost_ctrl_sync_fence_info; +struct nvhost_fence; + +#define NVHOST_MODULE_MAX_CLOCKS 8 +#define NVHOST_MODULE_MAX_IORESOURCE_MEM 5 + +enum tegra_emc_request_type { + TEGRA_SET_EMC_FLOOR, /* lower bound */ +}; + +struct nvhost_clock { + char *name; + unsigned long default_rate; + u32 moduleid; + enum tegra_emc_request_type request_type; + bool disable_scaling; + unsigned long devfreq_rate; +}; + +struct nvhost_vm_hwid { + u64 addr; + bool dynamic; + u32 shift; +}; + +/* + * Defines HW and SW class identifiers. + * + * This is module ID mapping between userspace and kernelspace. + * The values of enum entries' are referred from NvRmModuleID enum defined + * in below userspace file: + * $TOP/vendor/nvidia/tegra/core/include/nvrm_module.h + * Please make sure each entry below has same value as set in above file. + */ +enum nvhost_module_identifier { + /* Specifies external memory (DDR RAM, etc) */ + NVHOST_MODULE_ID_EXTERNAL_MEMORY_CONTROLLER = 75, +}; + +enum nvhost_resource_policy { + RESOURCE_PER_DEVICE = 0, + RESOURCE_PER_CHANNEL_INSTANCE, +}; + +struct nvhost_device_data { + int version; /* ip version number of device */ + void __iomem *aperture[NVHOST_MODULE_MAX_IORESOURCE_MEM]; + + u32 moduleid; /* Module id for user space API */ + + /* interrupt ISR routine for falcon based engines */ + int (*flcn_isr)(struct platform_device *dev); + int irq; + int module_irq; /* IRQ bit from general intr reg for module intr */ + bool self_config_flcn_isr; /* skip setting up falcon interrupts */ + + u32 class; /* Device class */ + bool keepalive; /* Do not power gate when opened */ + bool serialize; /* Serialize submits in the channel */ + bool push_work_done; /* Push_op done into push buffer */ + bool poweron_reset; /* Reset the engine before powerup */ + char *devfs_name; /* Name in devfs */ + char *devfs_name_family; /* Core of devfs name */ + + char *firmware_name; /* Name of firmware */ + bool firmware_not_in_subdir; /* Firmware is not located in + chip subdirectory */ + + bool engine_can_cg; /* True if CG is enabled */ + bool can_powergate; /* True if module can be power gated */ + int autosuspend_delay; /* Delay before power gated */ + struct nvhost_clock clocks[NVHOST_MODULE_MAX_CLOCKS];/* Clock names */ + + int num_clks; /* Number of clocks opened for dev */ + struct clk_bulk_data *clks; + struct mutex lock; /* Power management lock */ + + int num_channels; /* Max num of channel supported */ + int num_ppc; /* Number of pixels per clock cycle */ + dev_t cdev_region; + + /* device node for ctrl block */ + struct class *nvhost_class; + struct device *ctrl_node; + struct cdev ctrl_cdev; + const struct file_operations *ctrl_ops; /* ctrl ops for the module */ + + struct kobject clk_cap_kobj; + struct kobj_attribute *clk_cap_attrs; + struct dentry *debugfs; /* debugfs directory */ + + /* Marks if the device is booted when pm runtime is disabled */ + bool booted; + + void *private_data; /* private platform data */ + void *falcon_data; /* store the falcon info */ + struct platform_device *pdev; /* owner platform_device */ + struct host1x *host1x; /* host1x device */ + + /* Finalize power on. Can be used for context restore. */ + int (*finalize_poweron)(struct platform_device *dev); + + /* Preparing for power off. Used for context save. */ + int (*prepare_poweroff)(struct platform_device *dev); + + /* paring for power off. Used for context save. */ + int (*aggregate_constraints)(struct platform_device *dev, + int clk_index, + unsigned long floor_rate, + unsigned long pixel_rate, + unsigned long bw_rate); + + /* Used to add platform specific masks on reloc address */ + dma_addr_t (*get_reloc_phys_addr)(dma_addr_t phys_addr, u32 reloc_type); + + /* engine specific init functions */ + int (*pre_virt_init)(struct platform_device *pdev); + int (*post_virt_init)(struct platform_device *pdev); + + /* Information related to engine-side synchronization */ + void *syncpt_unit_interface; + + u64 transcfg_addr; + u32 transcfg_val; + struct nvhost_vm_hwid vm_regs[13]; + + /* Should we map channel at submit time? */ + bool resource_policy; + /* Should we enable context isolation for this device? */ + bool isolate_contexts; + + /* reset control for this device */ + struct reset_control *reset_control; + + /* icc client id for emc requests */ + int icc_id; + + /* icc_path handle handle */ + struct icc_path *icc_path_handle; + + /* bandwidth manager client id for emc requests */ + int bwmgr_client_id; +}; + +static inline +struct nvhost_device_data *nvhost_get_devdata(struct platform_device *pdev) +{ + return (struct nvhost_device_data *)platform_get_drvdata(pdev); +} + +extern const struct dev_pm_ops nvhost_module_pm_ops; + +HOST1X_EMU_EXPORT_DECL(u32, host1x_readl(struct platform_device *pdev, u32 r)); + +HOST1X_EMU_EXPORT_DECL(void, host1x_writel(struct platform_device *pdev, u32 r, u32 v)); + +/* public api to return platform_device ptr to the default host1x instance */ +HOST1X_EMU_EXPORT_DECL(struct platform_device*, nvhost_get_default_device(void)); + +/* common runtime pm and power domain APIs */ +HOST1X_EMU_EXPORT_DECL(int, nvhost_module_init(struct platform_device *ndev)); + +HOST1X_EMU_EXPORT_DECL(void, nvhost_module_deinit(struct platform_device *dev)); + +HOST1X_EMU_EXPORT_DECL(void, nvhost_module_reset(struct platform_device *dev, bool reboot)); + +HOST1X_EMU_EXPORT_DECL(int, nvhost_module_busy(struct platform_device *dev)); + +HOST1X_EMU_EXPORT_DECL(void, nvhost_module_idle(struct platform_device *dev)); + +HOST1X_EMU_EXPORT_DECL(void, nvhost_module_idle_mult(struct platform_device *pdev, int refs)); + +/* common device management APIs */ +HOST1X_EMU_EXPORT_DECL(int, nvhost_client_device_get_resources(struct platform_device *dev)); + +HOST1X_EMU_EXPORT_DECL(int, nvhost_client_device_release(struct platform_device *dev)); + +HOST1X_EMU_EXPORT_DECL(int, nvhost_client_device_init(struct platform_device *dev)); + +/* public host1x sync-point management APIs */ +HOST1X_EMU_EXPORT_DECL(u32, nvhost_get_syncpt_host_managed(struct platform_device *pdev, + u32 param, const char *syncpt_name)); + +HOST1X_EMU_EXPORT_DECL(u32, nvhost_get_syncpt_client_managed(struct platform_device *pdev, + const char *syncpt_name)); + +HOST1X_EMU_EXPORT_DECL(u32, nvhost_get_syncpt_gpu_managed(struct platform_device *pdev, + const char *syncpt_name)); + +HOST1X_EMU_EXPORT_DECL(void, nvhost_syncpt_put_ref_ext(struct platform_device *pdev, u32 id)); + +HOST1X_EMU_EXPORT_DECL(bool, nvhost_syncpt_is_valid_pt_ext(struct platform_device *dev, u32 id)); + +HOST1X_EMU_EXPORT_DECL(int, nvhost_syncpt_is_expired_ext(struct platform_device *dev, u32 id, + u32 thresh)); + +HOST1X_EMU_EXPORT_DECL(void, nvhost_syncpt_set_min_update(struct platform_device *pdev, u32 id, u32 val)); + +HOST1X_EMU_EXPORT_DECL(int, nvhost_syncpt_read_ext_check(struct platform_device *dev, u32 id, u32 *val)); + +HOST1X_EMU_EXPORT_DECL(u32, nvhost_syncpt_read_maxval(struct platform_device *dev, u32 id)); + +HOST1X_EMU_EXPORT_DECL(u32, nvhost_syncpt_incr_max_ext(struct platform_device *dev, u32 id, u32 incrs)); + +HOST1X_EMU_EXPORT_DECL(int, nvhost_syncpt_unit_interface_init(struct platform_device *pdev)); + +HOST1X_EMU_EXPORT_DECL(void, nvhost_syncpt_unit_interface_deinit(struct platform_device *pdev)); + +HOST1X_EMU_EXPORT_DECL(dma_addr_t, nvhost_syncpt_address(struct platform_device *engine_pdev, u32 id)); + +/* public host1x interrupt management APIs */ +HOST1X_EMU_EXPORT_DECL(int, nvhost_intr_register_notifier(struct platform_device *pdev, + u32 id, u32 thresh, + void (*callback)(void *), + void *private_data)); + +/* public host1x sync-point management APIs */ +HOST1X_EMU_EXPORT_DECL(struct host1x*, nvhost_get_host1x(struct platform_device *pdev)); + +HOST1X_EMU_EXPORT_DECL(int, nvhost_syncpt_unit_interface_get_aperture( + struct platform_device *host_pdev, + phys_addr_t *base, + size_t *size)); + +HOST1X_EMU_EXPORT_DECL(u32, nvhost_syncpt_unit_interface_get_byte_offset(u32 syncpt_id)); + +HOST1X_EMU_EXPORT_DECL(u32, nvhost_syncpt_unit_interface_get_byte_offset_ext( + struct platform_device *host_pdev, + u32 syncpt_id)); +#endif diff --git a/drivers/gpu/host1x-emu/include/linux/symbol-emu.h b/drivers/gpu/host1x-emu/include/linux/symbol-emu.h new file mode 100644 index 00000000..23362bfe --- /dev/null +++ b/drivers/gpu/host1x-emu/include/linux/symbol-emu.h @@ -0,0 +1,17 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: GPL-2.0-only + */ +#ifndef __HOST1X_EMU_SYMMBOL_H +#define __HOST1X_EMU_SYMMBOL_H + +#ifdef CONFIG_TEGRA_HOST1X_EMU_DBG_SYMBL +#define HOST1X_EMU_EXPORT_CALL(...) Dbg_ ## __VA_ARGS__ +#define HOST1X_EMU_EXPORT_DECL(ret, ...) ret Dbg_ ## __VA_ARGS__ +#define HOST1X_EMU_EXPORT_SYMBOL(f) EXPORT_SYMBOL( Dbg_## f) +#else +#define HOST1X_EMU_EXPORT_CALL(...) __VA_ARGS__ +#define HOST1X_EMU_EXPORT_DECL(ret, ...) ret __VA_ARGS__ +#define HOST1X_EMU_EXPORT_SYMBOL(f) EXPORT_SYMBOL(f) +#endif +#endif /*__HOST1X_EMU_SYMMBOL_H*/ diff --git a/drivers/gpu/host1x-emu/ioctl.c b/drivers/gpu/host1x-emu/ioctl.c new file mode 100644 index 00000000..c018dfa3 --- /dev/null +++ b/drivers/gpu/host1x-emu/ioctl.c @@ -0,0 +1,345 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: GPL-2.0-only + */ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "dev.h" +#include "fence.h" +#include "poll.h" +#include "syncpt.h" +#include "ioctl.h" + + +#define IFACE_NAME "host1x-emu" +#define TRACE_MAX_LENGTH 128U + +/* convenience,shorter err/fn/dbg_info */ +#if defined(HOST1X_EMU_SYNCPT_DEBUG) +extern u32 host1x_emu_dbg_mask; +extern u32 host1x_emu_dbg_ftrace; +#define host1x_emu_dbg(dbg_mask, format, arg...) \ +do { \ + if (unlikely((dbg_mask) & nvhost_dbg_mask)) { \ + pr_info("host1x-emu %s: " format "\n", \ + __func__, ##arg); \ + } \ +} while (0) +#else /* NVHOST_DEBUG */ +#define host1x_emu_dbg(dbg_mask, format, arg...) \ +do { \ + if (0) \ + printk(KERN_INFO "host1x-emu %s: " format "\n", __func__, ##arg);\ +} while (0) +#endif + +#define host1x_emu_err(d, fmt, arg...) \ + dev_err(d, "%s: " fmt "\n", __func__, ##arg) + +#define host1x_emu_err_ratelimited(d, fmt, arg...) \ + dev_err_ratelimited(d, "%s: " fmt "\n", __func__, ##arg) + +#define host1x_emu_warn(d, fmt, arg...) \ + dev_warn(d, "%s: " fmt "\n", __func__, ##arg) + +#define host1x_emu_dbg_fn(fmt, arg...) \ + host1x_emu__dbg(dbg_fn, fmt, ##arg) + +#define host1x_emu_dbg_info(fmt, arg...) \ + host1x_emu__dbg(dbg_info, fmt, ##arg) + +struct host1x_emu_ctrl_userctx { + struct host1x *host; + struct mutex lock; + struct xarray syncpoints; +}; + +/** + * timeout_abs_to_jiffies - calculate jiffies timeout from absolute time in + * sec/nsec + * + * @timeout_nsec: timeout nsec component in ns, 0 for poll + */ +static signed long timeout_abs_to_jiffies(int64_t timeout_nsec) +{ + ktime_t now; + ktime_t abs_timeout; + u64 timeout_ns; + u64 timeout_jiffies64; + + /* make 0 timeout means poll - absolute 0 doesn't seem valid */ + if (timeout_nsec == 0) + return 0; + + abs_timeout = ns_to_ktime(timeout_nsec); + now = ktime_get(); + + if (!ktime_after(abs_timeout, now)) + return 0; + + timeout_ns = ktime_to_ns(ktime_sub(abs_timeout, now)); + + timeout_jiffies64 = nsecs_to_jiffies64(timeout_ns); + /* clamp timeout to avoid infinite timeout */ + if (timeout_jiffies64 >= MAX_SCHEDULE_TIMEOUT - 1) + return MAX_SCHEDULE_TIMEOUT - 1; + + return timeout_jiffies64 + 1; +} + +static int host1x_ctrlopen(struct inode *inode, struct file *filp) +{ + struct host1x *host = container_of(inode->i_cdev, struct host1x, cdev); + struct host1x_emu_ctrl_userctx *priv = NULL; + + priv = kzalloc(sizeof(*priv), GFP_KERNEL); + if (priv == NULL) { + host1x_emu_err(host->dev, "failed to allocate host1x user context"); + kfree(priv); + return -ENOMEM; + } + xa_init(&priv->syncpoints); + mutex_init(&priv->lock); + + priv->host = host; + filp->private_data = priv; + return 0; +} + +static int host1x_ctrlrelease(struct inode *inode, struct file *filp) +{ + unsigned long id; + struct host1x_emu_ctrl_userctx *priv = filp->private_data; + struct host1x_syncpt *sp; + + filp->private_data = NULL; + + xa_for_each(&priv->syncpoints, id, sp) + HOST1X_EMU_EXPORT_CALL(host1x_syncpt_put(sp)); + + xa_destroy(&priv->syncpoints); + kfree(priv); + return 0; +} + +static int host1x_ioctl_syncpoint_allocate( + struct host1x_emu_ctrl_userctx *ctx, + struct host1x_emu_ctrl_alloc_syncpt_args *args) +{ + int err; + struct host1x *host = ctx->host; + struct host1x_syncpt *sp; + + if (args->id) + return -EINVAL; + + sp = HOST1X_EMU_EXPORT_CALL(host1x_syncpt_alloc(host, + HOST1X_SYNCPT_CLIENT_MANAGED, current->comm)); + if (!sp) + return -EBUSY; + + args->id = HOST1X_EMU_EXPORT_CALL(host1x_syncpt_id(sp)); + + err = xa_insert(&ctx->syncpoints, args->id, sp, GFP_KERNEL); + if (err) { + HOST1X_EMU_EXPORT_CALL(host1x_syncpt_put(sp)); + return err; + } + + return 0; +} + +static int host1x_ioctl_syncpoint_free( + struct host1x_emu_ctrl_userctx *ctx, + struct host1x_emu_ctrl_free_syncpt_args *args) +{ + struct host1x_syncpt *sp; + + mutex_lock(&ctx->lock); + sp = xa_erase(&ctx->syncpoints, args->id); + mutex_unlock(&ctx->lock); + if (!sp) + return -EINVAL; + + HOST1X_EMU_EXPORT_CALL(host1x_syncpt_put(sp)); + return 0; +} + +static int host1x_ioctl_ctrl_syncpt_read( + struct host1x_emu_ctrl_userctx *ctx, + struct host1x_emu_ctrl_syncpt_read_args *args) +{ + struct host1x *host = ctx->host; + struct host1x_syncpt *sp; + + sp = HOST1X_EMU_EXPORT_CALL(host1x_syncpt_get_by_id_noref(host, args->id)); + if (!sp) + return -EINVAL; + + args->value = HOST1X_EMU_EXPORT_CALL(host1x_syncpt_read(sp)); + return 0; +} + +static int host1x_ioctl_ctrl_syncpt_incr( + struct host1x_emu_ctrl_userctx *ctx, + struct host1x_emu_ctrl_syncpt_incr_args *args) +{ + int err; + uint32_t idx; + struct host1x_syncpt *sp; + + sp = xa_load(&ctx->syncpoints, args->id); + if (!sp) + return -EINVAL; + + for(idx = 0; idx < args->val; idx++) { + err = HOST1X_EMU_EXPORT_CALL(host1x_syncpt_incr(sp)); + if (err < 0) { + return err; + } + } + return err; +} + +static int host1x_ioctl_syncpoint_wait( + struct host1x_emu_ctrl_userctx *ctx, + struct host1x_emu_ctrl_syncpt_wait_args *args) + +{ + signed long timeout_jiffies; + struct host1x_syncpt *sp; + struct host1x *host = ctx->host; + + sp = HOST1X_EMU_EXPORT_CALL(host1x_syncpt_get_by_id_noref(host, args->id)); + if (!sp) + return -EINVAL; + timeout_jiffies = timeout_abs_to_jiffies(args->timeout_ns); + + return HOST1X_EMU_EXPORT_CALL(host1x_syncpt_wait(sp, + args->threshold, + timeout_jiffies, + &args->value)); +} + +static long host1x_ctrlctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + struct host1x_emu_ctrl_userctx *priv = filp->private_data; + u8 buf[HOST1X_EMU_SYNCPT_IOCTL_CTRL_MAX_ARG_SIZE] __aligned(sizeof(u64)); + int err = 0; + + if ((_IOC_TYPE(cmd) != HOST1X_EMU_SYNCPT_IOCTL_MAGIC) || + (_IOC_NR(cmd) == 0) || + (_IOC_NR(cmd) > HOST1X_EMU_SYNCPT_IOCTL_CTRL_LAST) || + (_IOC_SIZE(cmd) > HOST1X_EMU_SYNCPT_IOCTL_CTRL_MAX_ARG_SIZE)) { + host1x_emu_err_ratelimited(NULL, "invalid cmd 0x%x", cmd); + return -ENOIOCTLCMD; + } + + if (_IOC_DIR(cmd) & _IOC_WRITE) { + if (copy_from_user(buf, (void __user *)arg, _IOC_SIZE(cmd))) { + host1x_emu_err(NULL, "failed to copy from user arg=%px", + (void __user *)arg); + return -EFAULT; + } + } + + switch (cmd) { + case HOST1X_EMU_SYNCPT_IOCTL_CTRL_ALLOC_SYNCPT: + err = host1x_ioctl_syncpoint_allocate(priv, (void *)buf); + break; + case HOST1X_EMU_SYNCPT_IOCTL_CTRL_FREE_SYNCPT: + err = host1x_ioctl_syncpoint_free(priv, (void *)buf); + break; + case HOST1X_EMU_SYNCPT_IOCTL_CTRL_SYNCPT_READ: + err = host1x_ioctl_ctrl_syncpt_read(priv, (void *)buf); + break; + case HOST1X_EMU_SYNCPT_IOCTL_CTRL_SYNCPT_INCR: + err = host1x_ioctl_ctrl_syncpt_incr(priv, (void *)buf); + break; + case HOST1X_EMU_SYNCPT_IOCTL_CTRL_SYNCPT_WAIT: + err = host1x_ioctl_syncpoint_wait(priv, (void *)buf); + break; + default: + host1x_emu_err(priv->host->dev, "invalid cmd 0x%x", cmd); + err = -ENOIOCTLCMD; + break; + } + + if ((err == 0) && (_IOC_DIR(cmd) & _IOC_READ)) { + err = copy_to_user((void __user *)arg, buf, _IOC_SIZE(cmd)); + if (err) { + host1x_emu_err(priv->host->dev, + "failed to copy to user"); + err = -EFAULT; + } + } + + return err; +} + +static const struct file_operations host1x_ctrlops = { + .owner = THIS_MODULE, + .release = host1x_ctrlrelease, + .open = host1x_ctrlopen, + .unlocked_ioctl = host1x_ctrlctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = host1x_ctrlctl, +#endif +}; + +int host1x_user_init(struct host1x *host) +{ + int err; + dev_t devno; + +#if defined(NV_CLASS_CREATE_HAS_NO_OWNER_ARG) /* Linux v6.4 */ + host->host1x_class = class_create(dev_name(host->dev)); +#else + host->host1x_class = class_create(THIS_MODULE, dev_name(host->dev)); +#endif + if (IS_ERR(host->host1x_class)) { + err = PTR_ERR(host->host1x_class); + dev_err(host->dev, "failed to create class\n"); + goto fail; + } + + err = alloc_chrdev_region(&devno, 0, 1, IFACE_NAME); + if (err < 0) { + dev_err(host->dev, "failed to reserve chrdev region\n"); + goto fail; + } + + host->major = MAJOR(devno); + host->next_minor += 1; + + cdev_init(&host->cdev, &host1x_ctrlops); + host->cdev.owner = THIS_MODULE; + err = cdev_add(&host->cdev, devno, 1); + if (err < 0) { + host1x_emu_err(host->dev, "failed to add cdev"); + goto fail; + } + host->ctrl = device_create(host->host1x_class, host->dev, devno, + NULL, IFACE_NAME "-%s", "ctrl"); + if (IS_ERR(host->ctrl)) { + err = PTR_ERR(host->ctrl); + dev_err(host->dev, "failed to create ctrl device\n"); + goto fail; + } + + return 0; +fail: + return err; +} diff --git a/drivers/gpu/host1x-emu/ioctl.h b/drivers/gpu/host1x-emu/ioctl.h new file mode 100644 index 00000000..12bc55f1 --- /dev/null +++ b/drivers/gpu/host1x-emu/ioctl.h @@ -0,0 +1,129 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: GPL-2.0-only + */ +#ifndef __UAPI_LINUX_EMU_HOST1X_SYNCPT_IOCTL_H +#define __UAPI_LINUX_EMU_HOST1X_SYNCPT_IOCTL_H + +#include +#include + +#if !defined(__KERNEL__) +#define __user +#endif + +#define HOST1X_EMU_SYNCPT_IOCTL_MAGIC 'E' +#define HOST1X_EMU_SYNCPT_INVALID_SYNCPOINT 0xFFFFFFFF +#define HOST1X_EMU_SYNCPT_SUBMIT_MAX_NUM_SYNCPT_INCRS 10 + +struct host1x_emu_ctrl_alloc_syncpt_args { + /** + * @id: [out] + * + * ID of allocated syncpoint. + */ + __u32 id; + __u32 padding; +}; + +struct host1x_emu_ctrl_free_syncpt_args { + /** + * @id: [in] + * + * ID of syncpoint to free. + */ + __u32 id; + __u32 padding; +}; + +struct host1x_emu_ctrl_syncpt_read_args { + /** + * @id: + * + * ID of the syncpoint to read the current value from. + */ + __u32 id; + + /** + * @value: + * + * The current syncpoint value. Set by the kernel upon successful + * completion of the IOCTL. + */ + __u32 value; +}; + +struct host1x_emu_ctrl_syncpt_incr_args { + /** + * @id: + * + * ID of the syncpoint to increment. + */ + __u32 id; + + /** + * @val: + * + * Syncpoint increment value. + */ + __u32 val; +}; + +struct host1x_emu_ctrl_syncpt_wait_args { + /** + * @timeout: [in] + * + * Absolute timestamp at which the wait will time out. + */ + __s64 timeout_ns; + + /** + * @id: [in] + * + * ID of syncpoint to wait on. + */ + __u32 id; + + /** + * @threshold: [in] + * + * Threshold to wait for. + */ + __u32 threshold; + + /** + * @value: [out] + * + * Value of the syncpoint upon wait completion. + */ + __u32 value; + + /** + * @timestamp: [out] + * + * CLOCK_MONOTONIC timestamp in nanoseconds taken when the wait completes. + */ + __u64 timestamp; +}; + +#define HOST1X_EMU_SYNCPT_IOCTL_CTRL_ALLOC_SYNCPT \ + _IOWR(HOST1X_EMU_SYNCPT_IOCTL_MAGIC, 1, struct host1x_emu_ctrl_alloc_syncpt_args) + +#define HOST1X_EMU_SYNCPT_IOCTL_CTRL_FREE_SYNCPT \ + _IOWR(HOST1X_EMU_SYNCPT_IOCTL_MAGIC, 2, struct host1x_emu_ctrl_free_syncpt_args) + +#define HOST1X_EMU_SYNCPT_IOCTL_CTRL_SYNCPT_READ \ + _IOWR(HOST1X_EMU_SYNCPT_IOCTL_MAGIC, 3, struct host1x_emu_ctrl_syncpt_read_args) + +#define HOST1X_EMU_SYNCPT_IOCTL_CTRL_SYNCPT_INCR \ + _IOW(HOST1X_EMU_SYNCPT_IOCTL_MAGIC, 4, struct host1x_emu_ctrl_syncpt_incr_args) + +#define HOST1X_EMU_SYNCPT_IOCTL_CTRL_SYNCPT_WAIT \ + _IOW(HOST1X_EMU_SYNCPT_IOCTL_MAGIC, 5, struct host1x_emu_ctrl_syncpt_wait_args) + +#define HOST1X_EMU_SYNCPT_IOCTL_CTRL_LAST \ + _IOC_NR(HOST1X_EMU_SYNCPT_IOCTL_CTRL_SYNCPT_WAIT) + +#define HOST1X_EMU_SYNCPT_IOCTL_CTRL_MAX_ARG_SIZE \ + sizeof(struct host1x_emu_ctrl_syncpt_wait_args) +#endif diff --git a/drivers/gpu/host1x-emu/nvhost.c b/drivers/gpu/host1x-emu/nvhost.c new file mode 100644 index 00000000..cbfaf57a --- /dev/null +++ b/drivers/gpu/host1x-emu/nvhost.c @@ -0,0 +1,594 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: GPL-2.0-only + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef CONFIG_TEGRA_HOST1X_EMU_DBG_SYMBL +#include +#include +#else +#include +#include +#include +#endif + +#include "dev.h" + +#define NVHOST_NUM_CDEV 1 + +struct nvhost_syncpt_interface { + dma_addr_t base; + size_t size; + uint32_t page_size; +}; + +struct nvhost_host1x_cb { + struct dma_fence_cb cb; + struct work_struct work; + void (*notifier)(void *data); + void *notifier_data; +}; + +static const struct of_device_id host1x_match[] = { + { .compatible = "nvidia,tegraEmu-host1x", }, + {}, +}; + +static struct device *nvhost_client_device_create(struct platform_device *pdev, + struct cdev *cdev, + const char *cdev_name, + dev_t devno, + const struct file_operations *ops) +{ + struct nvhost_device_data *pdata = platform_get_drvdata(pdev); + struct device *dev; + int err; + +#if defined(NV_CLASS_CREATE_HAS_NO_OWNER_ARG) /* Linux v6.4 */ + pdata->nvhost_class = class_create(pdev->dev.of_node->name); +#else + pdata->nvhost_class = class_create(THIS_MODULE, pdev->dev.of_node->name); +#endif + if (IS_ERR(pdata->nvhost_class)) { + dev_err(&pdev->dev, "failed to create class\n"); + return ERR_CAST(pdata->nvhost_class); + } + + cdev_init(cdev, ops); + cdev->owner = THIS_MODULE; + + err = cdev_add(cdev, devno, 1); + if (err < 0) { + dev_err(&pdev->dev, "failed to add cdev\n"); + class_destroy(pdata->nvhost_class); + return ERR_PTR(err); + } + + dev = device_create(pdata->nvhost_class, &pdev->dev, devno, NULL, + (pdev->id <= 0) ? "nvhost-%s%s" : "nvhost-%s%s.%d", + cdev_name, pdev->dev.of_node->name, pdev->id); + + if (IS_ERR(dev)) { + dev_err(&pdev->dev, "failed to create %s device\n", cdev_name); + class_destroy(pdata->nvhost_class); + cdev_del(cdev); + } + + return dev; +} + +static void nvhost_host1x_cb_func(struct dma_fence *f, struct dma_fence_cb *cb) +{ + struct nvhost_host1x_cb *host1x_cb; + + host1x_cb = container_of(cb, struct nvhost_host1x_cb, cb); + schedule_work(&host1x_cb->work); + dma_fence_put(f); +} + +static void nvhost_intr_do_work(struct work_struct *work) +{ + struct nvhost_host1x_cb *host1x_cb; + + host1x_cb = container_of(work, struct nvhost_host1x_cb, work); + host1x_cb->notifier(host1x_cb->notifier_data); +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)) + kfree_rcu_mightsleep(host1x_cb); +#else + kfree_rcu(host1x_cb); +#endif +} + +/*Exported Function*/ +HOST1X_EMU_EXPORT_DECL(u32, host1x_readl(struct platform_device *pdev, u32 r)) +{ + return 0; +} +HOST1X_EMU_EXPORT_SYMBOL(host1x_readl); + +HOST1X_EMU_EXPORT_DECL(void, host1x_writel(struct platform_device *pdev, u32 r, u32 v)) +{ + return; +} +HOST1X_EMU_EXPORT_SYMBOL(host1x_writel); + +HOST1X_EMU_EXPORT_DECL(struct platform_device*, nvhost_get_default_device(void)) +{ + struct device_node *np; + struct platform_device *host1x_pdev; + + np = of_find_matching_node(NULL, host1x_match); + if (!np) + return NULL; + + host1x_pdev = of_find_device_by_node(np); + if (!host1x_pdev) + return NULL; + + return host1x_pdev; +} +HOST1X_EMU_EXPORT_SYMBOL(nvhost_get_default_device); + +HOST1X_EMU_EXPORT_DECL(struct host1x*, nvhost_get_host1x(struct platform_device *pdev)) +{ + struct host1x *host1x; + struct platform_device *host1x_pdev; + + host1x_pdev = HOST1X_EMU_EXPORT_CALL(nvhost_get_default_device()); + if (!host1x_pdev) { + dev_dbg(&pdev->dev, "host1x device not available\n"); + return NULL; + } + + host1x = platform_get_drvdata(host1x_pdev); + if (!host1x) { + dev_warn(&pdev->dev, "No platform data for host1x!\n"); + return NULL; + } + + return host1x; +} +HOST1X_EMU_EXPORT_SYMBOL(nvhost_get_host1x); + +HOST1X_EMU_EXPORT_DECL(int, nvhost_client_device_get_resources(struct platform_device *pdev)) +{ + struct nvhost_device_data *pdata = platform_get_drvdata(pdev); + int err; + u32 i; + + pdata->host1x = HOST1X_EMU_EXPORT_CALL(nvhost_get_host1x(pdev)); + if (!pdata->host1x) { + dev_warn(&pdev->dev, "No platform data for host1x!\n"); + return -ENODEV; + } + + for (i = 0; i < pdev->num_resources; i++) { + void __iomem *regs = NULL; + struct resource *r; + + r = platform_get_resource(pdev, IORESOURCE_MEM, i); + /* We've run out of mem resources */ + if (!r) + break; + + regs = devm_ioremap_resource(&pdev->dev, r); + if (IS_ERR(regs)) { + err = PTR_ERR(regs); + goto fail; + } + + pdata->aperture[i] = regs; + } + + return 0; + +fail: + dev_err(&pdev->dev, "failed to get register memory\n"); + + return err; + +} +HOST1X_EMU_EXPORT_SYMBOL(nvhost_client_device_get_resources); + +HOST1X_EMU_EXPORT_DECL(int, nvhost_client_device_init(struct platform_device *pdev)) +{ + struct nvhost_device_data *pdata = platform_get_drvdata(pdev); + dev_t devno; + int err; + + err = alloc_chrdev_region(&devno, 0, NVHOST_NUM_CDEV, "nvhost"); + if (err < 0) { + dev_err(&pdev->dev, "failed to reserve chrdev region\n"); + return err; + } + + pdata->ctrl_node = nvhost_client_device_create(pdev, &pdata->ctrl_cdev, + "ctrl-", devno, + pdata->ctrl_ops); + if (IS_ERR(pdata->ctrl_node)) + return PTR_ERR(pdata->ctrl_node); + + pdata->cdev_region = devno; + + return 0; +} +HOST1X_EMU_EXPORT_SYMBOL(nvhost_client_device_init); + +HOST1X_EMU_EXPORT_DECL(int, nvhost_client_device_release(struct platform_device *pdev)) +{ + struct nvhost_device_data *pdata = platform_get_drvdata(pdev); + + if (!IS_ERR_OR_NULL(pdata->ctrl_node)) { + device_destroy(pdata->nvhost_class, pdata->ctrl_cdev.dev); + cdev_del(&pdata->ctrl_cdev); + class_destroy(pdata->nvhost_class); + } + + unregister_chrdev_region(pdata->cdev_region, NVHOST_NUM_CDEV); + + return 0; +} +HOST1X_EMU_EXPORT_SYMBOL(nvhost_client_device_release); + +HOST1X_EMU_EXPORT_DECL(u32, nvhost_get_syncpt_host_managed(struct platform_device *pdev, + u32 param, const char *syncpt_name)) +{ + struct host1x_syncpt *sp; + struct nvhost_device_data *pdata = platform_get_drvdata(pdev); + + sp = HOST1X_EMU_EXPORT_CALL(host1x_syncpt_alloc(pdata->host1x, 0, + syncpt_name ? syncpt_name : dev_name(&pdev->dev))); + if (!sp) + return 0; + + return HOST1X_EMU_EXPORT_CALL(host1x_syncpt_id(sp)); +} +HOST1X_EMU_EXPORT_SYMBOL(nvhost_get_syncpt_host_managed); + +HOST1X_EMU_EXPORT_DECL(u32, nvhost_get_syncpt_client_managed(struct platform_device *pdev, + const char *syncpt_name)) +{ + struct host1x_syncpt *sp; + struct nvhost_device_data *pdata = platform_get_drvdata(pdev); + + sp = HOST1X_EMU_EXPORT_CALL(host1x_syncpt_alloc(pdata->host1x, + HOST1X_SYNCPT_CLIENT_MANAGED, syncpt_name ? syncpt_name : dev_name(&pdev->dev))); + if (!sp) + return 0; + + return HOST1X_EMU_EXPORT_CALL(host1x_syncpt_id(sp)); +} +HOST1X_EMU_EXPORT_SYMBOL(nvhost_get_syncpt_client_managed); + +HOST1X_EMU_EXPORT_DECL(u32, nvhost_get_syncpt_gpu_managed(struct platform_device *pdev, + const char *syncpt_name)) +{ + return HOST1X_EMU_EXPORT_CALL(nvhost_get_syncpt_client_managed(pdev, syncpt_name)); +} +HOST1X_EMU_EXPORT_SYMBOL(nvhost_get_syncpt_gpu_managed); + +HOST1X_EMU_EXPORT_DECL(void, nvhost_syncpt_put_ref_ext(struct platform_device *pdev, u32 id)) +{ + struct nvhost_device_data *pdata = platform_get_drvdata(pdev); + struct host1x_syncpt *sp; + + sp = HOST1X_EMU_EXPORT_CALL(host1x_syncpt_get_by_id_noref(pdata->host1x, id)); + if (WARN_ON(!sp)) + return; + + HOST1X_EMU_EXPORT_CALL(host1x_syncpt_put(sp)); +} +HOST1X_EMU_EXPORT_SYMBOL(nvhost_syncpt_put_ref_ext); + +HOST1X_EMU_EXPORT_DECL(bool, nvhost_syncpt_is_valid_pt_ext(struct platform_device *pdev, u32 id)) +{ + struct nvhost_device_data *pdata = platform_get_drvdata(pdev); + struct host1x_syncpt *sp; + + if (!pdata || !pdata->host1x) + return -ENODEV; + + sp = HOST1X_EMU_EXPORT_CALL(host1x_syncpt_get_by_id_noref(pdata->host1x, id)); + + return sp ? true : false; +} +HOST1X_EMU_EXPORT_SYMBOL(nvhost_syncpt_is_valid_pt_ext); + +HOST1X_EMU_EXPORT_DECL(int, nvhost_syncpt_is_expired_ext(struct platform_device *pdev, u32 id, + u32 thresh)) +{ + struct nvhost_device_data *pdata = platform_get_drvdata(pdev); + struct host1x_syncpt *sp; + + sp = HOST1X_EMU_EXPORT_CALL(host1x_syncpt_get_by_id_noref(pdata->host1x, id)); + if (WARN_ON(!sp)) + return true; + + if (HOST1X_EMU_EXPORT_CALL(host1x_syncpt_wait(sp, thresh, 0, NULL))) + return false; + + return true; +} +HOST1X_EMU_EXPORT_SYMBOL(nvhost_syncpt_is_expired_ext); + +HOST1X_EMU_EXPORT_DECL(void, nvhost_syncpt_set_min_update(struct platform_device *pdev, u32 id, u32 val)) +{ + struct nvhost_device_data *pdata = platform_get_drvdata(pdev); + struct host1x_syncpt *sp; + u32 cur; + + /** + * TODO: Use host1x_syncpt_get_by_id(), otherwise anyone can update + * syncpoint without allocating + */ + //sp = HOST1X_EMU_EXPORT_CALL(host1x_syncpt_get_by_id(pdata->host1x, id)); + sp = HOST1X_EMU_EXPORT_CALL(host1x_syncpt_get_by_id_noref(pdata->host1x, id)); + if (WARN_ON(!sp)) + return; + + cur = HOST1X_EMU_EXPORT_CALL(host1x_syncpt_read(sp)); + + while (cur++ != val) + HOST1X_EMU_EXPORT_CALL(host1x_syncpt_incr(sp)); + //HOST1X_EMU_EXPORT_CALL(host1x_syncpt_put(sp)); + + HOST1X_EMU_EXPORT_CALL(host1x_syncpt_read(sp)); +} +HOST1X_EMU_EXPORT_SYMBOL(nvhost_syncpt_set_min_update); + +HOST1X_EMU_EXPORT_DECL(int, nvhost_syncpt_read_ext_check(struct platform_device *pdev, u32 id, u32 *val)) +{ + struct nvhost_device_data *pdata = platform_get_drvdata(pdev); + struct host1x_syncpt *sp; + + sp = HOST1X_EMU_EXPORT_CALL(host1x_syncpt_get_by_id_noref(pdata->host1x, id)); + if (!sp) + return -EINVAL; + + *val = HOST1X_EMU_EXPORT_CALL(host1x_syncpt_read(sp)); + return 0; +} +HOST1X_EMU_EXPORT_SYMBOL(nvhost_syncpt_read_ext_check); + +HOST1X_EMU_EXPORT_DECL(u32, nvhost_syncpt_read_maxval(struct platform_device *pdev, u32 id)) +{ + struct nvhost_device_data *pdata = platform_get_drvdata(pdev); + struct host1x_syncpt *sp; + + sp = HOST1X_EMU_EXPORT_CALL(host1x_syncpt_get_by_id_noref(pdata->host1x, id)); + if (WARN_ON(!sp)) + return 0; + + return HOST1X_EMU_EXPORT_CALL(host1x_syncpt_read_max(sp)); +} +HOST1X_EMU_EXPORT_SYMBOL(nvhost_syncpt_read_maxval); + +HOST1X_EMU_EXPORT_DECL(u32, nvhost_syncpt_incr_max_ext(struct platform_device *pdev, u32 id, u32 incrs)) +{ + struct nvhost_device_data *pdata = platform_get_drvdata(pdev); + struct host1x_syncpt *sp; + + sp = HOST1X_EMU_EXPORT_CALL(host1x_syncpt_get_by_id_noref(pdata->host1x, id)); + if (WARN_ON(!sp)) + return 0; + + return HOST1X_EMU_EXPORT_CALL(host1x_syncpt_incr_max(sp, incrs)); +} +HOST1X_EMU_EXPORT_SYMBOL(nvhost_syncpt_incr_max_ext); + +HOST1X_EMU_EXPORT_DECL(u32, nvhost_syncpt_unit_interface_get_byte_offset_ext(struct platform_device *pdev, + u32 syncpt_id)) +{ + struct nvhost_device_data *pdata = platform_get_drvdata(pdev); + + if (WARN_ON(!pdata)) + return 0; + if (syncpt_id >= pdata->host1x->syncpt_count) { + pr_info("Invalid syncpoint ID!\n"); + return 0; + } + + return syncpt_id * pdata->host1x->syncpt_page_size; +} +HOST1X_EMU_EXPORT_SYMBOL(nvhost_syncpt_unit_interface_get_byte_offset_ext); + +HOST1X_EMU_EXPORT_DECL(u32, nvhost_syncpt_unit_interface_get_byte_offset(u32 syncpt_id)) +{ + struct platform_device *host1x_pdev; + struct host1x *host1x; + + host1x_pdev = HOST1X_EMU_EXPORT_CALL(nvhost_get_default_device()); + if (WARN_ON(!host1x_pdev)) + return 0; + + host1x = platform_get_drvdata(host1x_pdev); + if (!host1x) { + pr_info("No platform data for host1x!\n"); + return 0; + } + + if (syncpt_id >= host1x->syncpt_count) { + pr_info("Invalid syncpoint ID!\n"); + return 0; + } + return (syncpt_id * host1x->syncpt_page_size); +} +HOST1X_EMU_EXPORT_SYMBOL(nvhost_syncpt_unit_interface_get_byte_offset); + +HOST1X_EMU_EXPORT_DECL(int, nvhost_syncpt_unit_interface_get_aperture(struct platform_device *pdev, + u64 *base, size_t *size)) +{ + struct nvhost_device_data *pdata = platform_get_drvdata(pdev); + + *base = pdata->host1x->syncpt_phy_apt; + *size = pdata->host1x->syncpt_page_size * pdata->host1x->syncpt_count; + return 0; +} +HOST1X_EMU_EXPORT_SYMBOL(nvhost_syncpt_unit_interface_get_aperture); + +HOST1X_EMU_EXPORT_DECL(int, nvhost_syncpt_unit_interface_init(struct platform_device *pdev)) +{ + struct nvhost_device_data *pdata = platform_get_drvdata(pdev); + struct nvhost_syncpt_interface *syncpt_if; + + + syncpt_if = devm_kzalloc(&pdev->dev, sizeof(*syncpt_if), GFP_KERNEL); + if (!syncpt_if) + return -ENOMEM; + + if (pdata->host1x == NULL) { + dev_info(&pdev->dev,"Client device resource not initialized\n"); + return -ENODEV; + } + + syncpt_if->base = (dma_addr_t)(pdata->host1x->syncpt_phy_apt); + syncpt_if->size = pdata->host1x->syncpt_page_size * pdata->host1x->syncpt_count; + syncpt_if->page_size = pdata->host1x->syncpt_page_size; + + /* If IOMMU is enabled, map it into the device memory */ + if (iommu_get_domain_for_dev(&pdev->dev)) { + syncpt_if->base = dma_map_resource(&pdev->dev, pdata->host1x->syncpt_phy_apt, + syncpt_if->size, + DMA_BIDIRECTIONAL, + DMA_ATTR_SKIP_CPU_SYNC); + if (dma_mapping_error(&pdev->dev, syncpt_if->base)) + return -ENOMEM; + } + + pdata->syncpt_unit_interface = syncpt_if; + dev_info(&pdev->dev, "syncpt_unit_base %llx syncpt_unit_size %zx size %x\n", + pdata->host1x->syncpt_phy_apt, + syncpt_if->size, + syncpt_if->page_size); + return 0; +} +HOST1X_EMU_EXPORT_SYMBOL(nvhost_syncpt_unit_interface_init); + +HOST1X_EMU_EXPORT_DECL(void, nvhost_syncpt_unit_interface_deinit(struct platform_device *pdev)) +{ + struct nvhost_syncpt_interface *syncpt_if; + struct nvhost_device_data *pdata; + + pdata = platform_get_drvdata(pdev); + if (pdata == NULL) { + return; + } + syncpt_if = pdata->syncpt_unit_interface; + + if (syncpt_if != NULL) { + if (iommu_get_domain_for_dev(&pdev->dev)) { + dma_unmap_resource(&pdev->dev, syncpt_if->base, syncpt_if->size, + DMA_BIDIRECTIONAL, DMA_ATTR_SKIP_CPU_SYNC); + } + devm_kfree(&pdev->dev, syncpt_if); + } +} +HOST1X_EMU_EXPORT_SYMBOL(nvhost_syncpt_unit_interface_deinit); + +HOST1X_EMU_EXPORT_DECL(dma_addr_t, nvhost_syncpt_address(struct platform_device *pdev, u32 id)) +{ + struct nvhost_device_data *pdata = platform_get_drvdata(pdev); + struct nvhost_syncpt_interface *syncpt_if = pdata->syncpt_unit_interface; + + if (syncpt_if == NULL) { + dev_info(&pdev->dev,"Syncpoint unit interfac not initialized\n"); + return (dma_addr_t)NULL; + } + + if (id >= pdata->host1x->syncpt_count) { + dev_info(&pdev->dev,"Invalid Syncpoint ID\n"); + return (dma_addr_t)NULL; + } + return syncpt_if->base + syncpt_if->page_size * id; +} +HOST1X_EMU_EXPORT_SYMBOL(nvhost_syncpt_address); + +HOST1X_EMU_EXPORT_DECL(int, nvhost_intr_register_notifier(struct platform_device *pdev, + u32 id, u32 thresh, + void (*callback)(void *data), + void *private_data)) +{ + int err; + struct dma_fence *fence; + struct host1x_syncpt *sp; + struct nvhost_host1x_cb *cb; + struct nvhost_device_data *pdata = platform_get_drvdata(pdev); + + sp = HOST1X_EMU_EXPORT_CALL(host1x_syncpt_get_by_id_noref(pdata->host1x, id)); + if (!sp) + return -EINVAL; + + fence = HOST1X_EMU_EXPORT_CALL(host1x_fence_create(sp, thresh, true)); + if (IS_ERR(fence)) { + pr_err("error %d during construction of fence!", (int)PTR_ERR(fence)); + return PTR_ERR(fence); + } + + cb = kzalloc(sizeof(*cb), GFP_KERNEL); + if (!cb) { + dma_fence_put(fence); + return -ENOMEM; + } + + INIT_WORK(&cb->work, nvhost_intr_do_work); + cb->notifier = callback; + cb->notifier_data = private_data; + + err = dma_fence_add_callback(fence, &cb->cb, nvhost_host1x_cb_func); + if (err < 0) { + dma_fence_put(fence); + kfree(cb); + } + + return err; +} +HOST1X_EMU_EXPORT_SYMBOL(nvhost_intr_register_notifier); + +HOST1X_EMU_EXPORT_DECL(int, nvhost_module_init(struct platform_device *pdev)) +{ + return -EOPNOTSUPP; +} +HOST1X_EMU_EXPORT_SYMBOL(nvhost_module_init); + +HOST1X_EMU_EXPORT_DECL(void, nvhost_module_deinit(struct platform_device *pdev)) +{ + return; +} +HOST1X_EMU_EXPORT_SYMBOL(nvhost_module_deinit); + +HOST1X_EMU_EXPORT_DECL(void, nvhost_module_reset(struct platform_device *pdev, bool reboot)) +{ + return; +} +HOST1X_EMU_EXPORT_SYMBOL(nvhost_module_reset); + +HOST1X_EMU_EXPORT_DECL(int, nvhost_module_busy(struct platform_device *dev)) +{ + return -EOPNOTSUPP; +} +HOST1X_EMU_EXPORT_SYMBOL(nvhost_module_busy); + +HOST1X_EMU_EXPORT_DECL(inline void, nvhost_module_idle(struct platform_device *pdev)) +{ + return; +} +HOST1X_EMU_EXPORT_SYMBOL(nvhost_module_idle); + +HOST1X_EMU_EXPORT_DECL(void, nvhost_module_idle_mult(struct platform_device *pdev, int refs)) +{ + return; +} +HOST1X_EMU_EXPORT_SYMBOL(nvhost_module_idle_mult); \ No newline at end of file diff --git a/drivers/gpu/host1x-emu/poll.c b/drivers/gpu/host1x-emu/poll.c new file mode 100644 index 00000000..fcdf9967 --- /dev/null +++ b/drivers/gpu/host1x-emu/poll.c @@ -0,0 +1,152 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: GPL-2.0-only + */ +#include "dev.h" +#include "fence.h" +#include "poll.h" + +static void host1x_poll_add_fence_to_list(struct host1x_fence_list *list, + struct host1x_syncpt_fence *fence) +{ + struct host1x_syncpt_fence *fence_in_list; + + list_for_each_entry_reverse(fence_in_list, &list->list, list) { + if ((s32)(fence_in_list->threshold - fence->threshold) <= 0) { + /* Fence in list is before us, we can insert here */ + list_add(&fence->list, &fence_in_list->list); + return; + } + } + + /* Add as first in list */ + list_add(&fence->list, &list->list); +} + +void host1x_poll_add_fence_locked(struct host1x *host, struct host1x_syncpt_fence *fence) +{ + struct host1x_fence_list *fence_list = &fence->sp->fences; + + INIT_LIST_HEAD(&fence->list); + host1x_poll_add_fence_to_list(fence_list, fence); +} + +bool host1x_poll_remove_fence(struct host1x *host, struct host1x_syncpt_fence *fence) +{ + struct host1x_fence_list *fence_list = &fence->sp->fences; + unsigned long irqflags; + + spin_lock_irqsave(&fence_list->lock, irqflags); + + if (list_empty(&fence->list)) { + spin_unlock_irqrestore(&fence_list->lock, irqflags); + return false; + } + list_del_init(&fence->list); + + spin_unlock_irqrestore(&fence_list->lock, irqflags); + return true; +} + +static void host1x_pool_timeout_handler(struct work_struct *work) +{ + struct delayed_work *dwork = (struct delayed_work *)work; + struct host1x_syncpt_pool *pool = container_of(dwork, + struct host1x_syncpt_pool, pool_work); + struct host1x_syncpt *sp; + struct host1x_syncpt *tmp_spt; + struct host1x *host = pool->host; + ktime_t ts = ktime_get(); + + spin_lock(&pool->syncpt_list.lock); + list_for_each_entry_safe(sp, tmp_spt, &pool->syncpt_list.list, list) { + struct host1x_syncpt_fence *fence, *tmp; + unsigned int value; + + value = host1x_syncpt_load(sp); + + spin_lock(&sp->fences.lock); + list_for_each_entry_safe(fence, tmp, &sp->fences.list, list) { + if (((value - fence->threshold) & 0x80000000U) != 0U) { + /* Fence is not yet expired, we are done */ + break; + } + + list_del_init(&fence->list); + host1x_fence_signal(fence ,ts); + } + spin_unlock(&sp->fences.lock); + } + spin_unlock(&pool->syncpt_list.lock); + + /** + * TODO: Optimize pool polling mechanism + */ + schedule_delayed_work(&pool->pool_work, + msecs_to_jiffies(host->polling_intrval)); +} + +int host1x_poll_init(struct host1x *host) +{ + unsigned int id; + + for (id = 0; id < host->num_pools; ++id) { + struct host1x_syncpt_pool *syncpt_pool = &host->pools[id]; + + syncpt_pool->host = host; + spin_lock_init(&syncpt_pool->syncpt_list.lock); + INIT_LIST_HEAD(&syncpt_pool->syncpt_list.list); + + INIT_DELAYED_WORK(&syncpt_pool->pool_work, host1x_pool_timeout_handler); + } + + /* Initialize RO-Pool*/ + host->pools[host->ro_pool_id].host = host; + spin_lock_init(&host->pools[host->ro_pool_id].syncpt_list.lock); + INIT_LIST_HEAD(&host->pools[host->ro_pool_id].syncpt_list.list); + INIT_DELAYED_WORK(&host->pools[host->ro_pool_id].pool_work, + host1x_pool_timeout_handler); + + for (id = 0; id < host1x_syncpt_nb_pts(host); ++id) { + struct host1x_syncpt *syncpt = &host->syncpt[id]; + struct host1x_syncpt_pool *pool = syncpt->pool; + + spin_lock_init(&syncpt->fences.lock); + INIT_LIST_HEAD(&syncpt->fences.list); + INIT_LIST_HEAD(&syncpt->list); + + /* Add syncpoint to pool list*/ + list_add(&syncpt->list, &pool->syncpt_list.list); + } + + return 0; +} + +void host1x_poll_start(struct host1x *host) +{ + int id; + + /*Loop till "host->num_pools + 1" to include Ro-Pool*/ + for (id = 0; id < host->num_pools + 1; ++id) { + struct host1x_syncpt_pool *syncpt_pool = &host->pools[id]; + + schedule_delayed_work(&syncpt_pool->pool_work, msecs_to_jiffies(host->polling_intrval)); + } +} + +void host1x_poll_stop(struct host1x *host) +{ + int id; + + /*Loop till "host->num_pools + 1" to include Ro-Pool*/ + for (id = 0; id < host->num_pools + 1; ++id) { + struct host1x_syncpt_pool *syncpt_pool = &host->pools[id]; + + //Schedule delayed work immediately + schedule_delayed_work(&syncpt_pool->pool_work, 0); + //Wait for schedule work to complete + flush_delayed_work(&syncpt_pool->pool_work); + //Cancel the work as it reschedule itself + cancel_delayed_work(&syncpt_pool->pool_work); + } +} diff --git a/drivers/gpu/host1x-emu/poll.h b/drivers/gpu/host1x-emu/poll.h new file mode 100644 index 00000000..070e995c --- /dev/null +++ b/drivers/gpu/host1x-emu/poll.h @@ -0,0 +1,61 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: GPL-2.0-only + */ +#ifndef __HOST1X_EMU_POLL_H +#define __HOST1X_EMU_POLL_H + +#include +#include +#include + +struct host1x; +struct host1x_syncpt_fence; + +struct host1x_fence_list { + spinlock_t lock; + struct list_head list; +}; + +struct host1x_syncpt_list { + spinlock_t lock; + struct list_head list; +}; + +/** + * Description: Initialize host1x syncpoint and pool for polling + * + */ +int host1x_poll_init(struct host1x *host); + +/** + * Description: Deinitialize host1x syncpoint and pool for polling + */ +void host1x_poll_deinit(struct host1x *host); + +/** + * Description: Enable host1x syncpoint pool polling + */ +void host1x_poll_start(struct host1x *host); + +/** + * Description: Disable host1x syncpoint pool polling + */ +void host1x_poll_stop(struct host1x *host); + +/** + * Description: Polling Thread Handler + */ +void host1x_poll_handle_timeout(struct host1x *host, unsigned int id, ktime_t ts); + +/** + * Description: Add fence to syncpoint object fence-list + */ +void host1x_poll_add_fence_locked(struct host1x *host, struct host1x_syncpt_fence *fence); + +/** + * Description: Remove fence from syncpoint object fence-list + */ +bool host1x_poll_remove_fence(struct host1x *host, struct host1x_syncpt_fence *fence); + +#endif diff --git a/drivers/gpu/host1x-emu/syncpt.c b/drivers/gpu/host1x-emu/syncpt.c new file mode 100644 index 00000000..39b27d72 --- /dev/null +++ b/drivers/gpu/host1x-emu/syncpt.c @@ -0,0 +1,475 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: GPL-2.0-only + */ +#include +#include +#include +#include +#include +#include + +#include "dev.h" +#include "syncpt.h" + +static void syncpt_release(struct kref *ref) +{ + struct host1x_syncpt *sp = container_of(ref, struct host1x_syncpt, ref); + + atomic_set(&sp->max_val, HOST1X_EMU_EXPORT_CALL(host1x_syncpt_read(sp))); + sp->locked = false; + + mutex_lock(&sp->host->syncpt_mutex); + kfree(sp->name); + sp->name = NULL; + sp->client_managed = false; + mutex_unlock(&sp->host->syncpt_mutex); +} + +void host1x_syncpt_restore(struct host1x *host) +{ + struct host1x_syncpt *sp_base = host->syncpt; + unsigned int i; + + for (i = host->syncpt_base; i < host->syncpt_end; i++) { + host1x_hw_syncpt_restore(host, sp_base + i); + } + wmb(); +} + +void host1x_syncpt_save(struct host1x *host) +{ + struct host1x_syncpt *sp_base = host->syncpt; + unsigned int i; + + for (i = 0; i < host1x_syncpt_nb_pts(host); i++) { + if (host1x_syncpt_client_managed(sp_base + i)) + host1x_hw_syncpt_load(host, sp_base + i); + else + WARN_ON(!host1x_syncpt_idle(sp_base + i)); + } +} + +/** + * Updates the cached syncpoint value by reading a new value + * from the memory + */ +u32 host1x_syncpt_load(struct host1x_syncpt *sp) +{ + u32 val; + + val = host1x_hw_syncpt_load(sp->host, sp); + return val; +} + +/** + * Returns true if syncpoint is expired, false if we may need to wait + */ +bool host1x_syncpt_is_expired(struct host1x_syncpt *sp, u32 thresh) +{ + u32 current_val; + + smp_rmb(); + + current_val = (u32)atomic_read(&sp->min_val); + return ((current_val - thresh) & 0x80000000U) == 0U; +} + +int host1x_syncpt_init(struct host1x *host) +{ + unsigned int i; + struct host1x_syncpt *syncpt; + + syncpt = devm_kcalloc(host->dev, host->syncpt_count, sizeof(*syncpt), + GFP_KERNEL); + if (!syncpt) { + pr_info("Host1x-EMU: Memory allocation for syncpoint structure failed\n"); + return -ENOMEM; + } + + for (i = 0; i < host->syncpt_count; i++) { + syncpt[i].id = i; + syncpt[i].host = host; + syncpt[i].client_managed = true; + /*Setting default syncpoint read-only pool*/ + syncpt[i].pool = &host->pools[host->ro_pool_id]; + } + + for (i = 0; i < host->num_pools; i++) { + struct host1x_syncpt_pool *pool = &host->pools[i]; + unsigned int j; + + for (j = pool->sp_base; j < pool->sp_end; j++) + syncpt[j].pool = pool; + } + + mutex_init(&host->syncpt_mutex); + host->syncpt = syncpt; + + return 0; +} + +void host1x_syncpt_deinit(struct host1x *host) +{ + struct host1x_syncpt *sp = host->syncpt; + unsigned int i; + + for (i = 0; i < host->syncpt_count; i++, sp++) + kfree(sp->name); + + /** + * Deallocating syncpoint array. + * Syncpoint deinit is invoked from drvier remove callback + * or drvier probe failure. + */ + kfree(host->syncpt); +} + +unsigned int host1x_syncpt_nb_pts(struct host1x *host) +{ + return host->syncpt_count; +} + +/** + * host1x_get_dma_mask() - query the supported DMA mask for host1x + * @host1x: host1x instance + * + * Note that this returns the supported DMA mask for host1x, which can be + * different from the applicable DMA mask under certain circumstances. + */ +HOST1X_EMU_EXPORT_DECL(u64, host1x_get_dma_mask(struct host1x *host1x)) +{ + return host1x->info->dma_mask; +} +HOST1X_EMU_EXPORT_SYMBOL(host1x_get_dma_mask); + +/** + * @brief Increment syncpoint refcount + * + * @sp: syncpoint + */ +HOST1X_EMU_EXPORT_DECL(struct host1x_syncpt*, host1x_syncpt_get( + struct host1x_syncpt *sp)) +{ + kref_get(&sp->ref); + + return sp; +} +HOST1X_EMU_EXPORT_SYMBOL(host1x_syncpt_get); + +/** + * @brief Obtain a syncpoint by ID + * + * @host: host1x controller + * @id: syncpoint ID + */ +HOST1X_EMU_EXPORT_DECL(struct host1x_syncpt*, host1x_syncpt_get_by_id( + struct host1x *host, unsigned int id)) +{ + if (id >= host->syncpt_count) + return NULL; + + if (kref_get_unless_zero(&host->syncpt[id].ref)) + return &host->syncpt[id]; + else + return NULL; +} +HOST1X_EMU_EXPORT_SYMBOL(host1x_syncpt_get_by_id); + +/** + * @brief Obtain a syncpoint by ID but don't increase the refcount. + * + * @host: host1x controller + * @id: syncpoint ID + */ +HOST1X_EMU_EXPORT_DECL(struct host1x_syncpt*, host1x_syncpt_get_by_id_noref(struct host1x *host, unsigned int id)) +{ + if (id >= host->syncpt_count) + return NULL; + + return &host->syncpt[id]; +} +HOST1X_EMU_EXPORT_SYMBOL(host1x_syncpt_get_by_id_noref); + +/** + * @brief Read the current syncpoint value + * + * @sp: host1x syncpoint + */ +HOST1X_EMU_EXPORT_DECL(u32, host1x_syncpt_read(struct host1x_syncpt *sp)) +{ + return host1x_syncpt_load(sp); +} +HOST1X_EMU_EXPORT_SYMBOL(host1x_syncpt_read); + +/** + * @brief Read minimum syncpoint value. + * + * The minimum syncpoint value is a shadow of the current sync point value + * in syncpoint-memory. + * + * @sp: host1x syncpoint + * + */ +HOST1X_EMU_EXPORT_DECL(u32, host1x_syncpt_read_min(struct host1x_syncpt *sp)) +{ + smp_rmb(); + + return (u32)atomic_read(&sp->min_val); +} +HOST1X_EMU_EXPORT_SYMBOL(host1x_syncpt_read_min); + +/** + * @brief Read maximum syncpoint value. + * + * The maximum syncpoint value indicates how many operations there are in queue, + * either in channel or in a software thread. + * + * @sp: host1x syncpoint + * + */ +HOST1X_EMU_EXPORT_DECL(u32, host1x_syncpt_read_max(struct host1x_syncpt *sp)) +{ + smp_rmb(); + return (u32)atomic_read(&sp->max_val); +} +HOST1X_EMU_EXPORT_SYMBOL(host1x_syncpt_read_max); + +/** + * @brief Increment syncpoint value from CPU, updating cache + * @sp: host1x syncpoint + */ +HOST1X_EMU_EXPORT_DECL(int, host1x_syncpt_incr(struct host1x_syncpt *sp)) +{ + return host1x_hw_syncpt_cpu_incr(sp->host, sp); +} +HOST1X_EMU_EXPORT_SYMBOL(host1x_syncpt_incr); + +/** + * @brief Update the value sent to hardware + * + * @sp: host1x syncpoint + * @incrs: number of increments + */ +HOST1X_EMU_EXPORT_DECL(u32, host1x_syncpt_incr_max(struct host1x_syncpt *sp, u32 incrs)) +{ + return (u32)atomic_add_return(incrs, &sp->max_val); +} +HOST1X_EMU_EXPORT_SYMBOL(host1x_syncpt_incr_max); + +/** + * @brief Allocate a syncpoint + * + * Allocates a hardware syncpoint for the caller's use. The caller then has + * the sole authority to mutate the syncpoint's value until it is freed again. + * + * If no free syncpoints are available, or a NULL name was specified, returns + * NULL. + * + * @host: host1x device data + * @flags: bitfield of HOST1X_SYNCPT_* flags + * @name: name for the syncpoint for use in debug prints + */ +HOST1X_EMU_EXPORT_DECL(struct host1x_syncpt*, host1x_syncpt_alloc(struct host1x *host, + unsigned long flags, + const char *name)) +{ + struct host1x_syncpt *sp = host->syncpt + host->syncpt_base; + struct host1x_syncpt_pool *pool = NULL; + char *full_name; + unsigned int i; + + if (!name) { + dev_err(host->dev, "syncpoints name null\n"); + return NULL; + } + + /* Only Read only pool*/ + if (host->num_pools == 0) { + dev_err(host->dev, + "Syncpoints alloc fail, only RO-Only pool avialable\n"); + return NULL; + } + + /** + * TODO: Update this based on new pools logic + */ + if (flags & HOST1X_SYNCPT_GPU) { + for (i = 0; i < host->num_pools; i++) { + if (!strcmp(host->pools[i].name, "gpu")) { + pool = &host->pools[i]; + break; + } + } + } + + /** + * TODO: syncpt_mutex is for entire synpoint list + * maybe, update this to syncpoint-pool level lock + */ + mutex_lock(&host->syncpt_mutex); + + /** + * TODO: Optimize syncpoint allocation, serial allocation + * dosen't effectively utilize per pool polling thread. + */ + for (i = host->syncpt_base; i < host->syncpt_end; i++, sp++) { + + /* Do pool verification if pool selected */ + if ((pool != NULL) && (sp->pool != pool)) + continue; + + /* Skip if pool is read only pool */ + if (sp->pool == &host->pools[host->ro_pool_id]) + continue; + + if (kref_read(&sp->ref) == 0) { + break; + } + } + + if (i >= host->syncpt_end) { + goto unlock; + } + + full_name = kasprintf(GFP_KERNEL, "%u-%s", sp->id, name); + if (!full_name) { + goto unlock; + } + sp->name = full_name; + + if (flags & HOST1X_SYNCPT_CLIENT_MANAGED) + sp->client_managed = true; + else + sp->client_managed = false; + + kref_init(&sp->ref); + + mutex_unlock(&host->syncpt_mutex); + return sp; + +unlock: + mutex_unlock(&host->syncpt_mutex); + return NULL; +} +HOST1X_EMU_EXPORT_SYMBOL(host1x_syncpt_alloc); + +/** + * @brief Free a requested syncpoint + * + * Release a syncpoint previously allocated using host1x_syncpt_request(). + * A host1x client driver should call this when the syncpoint is no longer + * in use. + * + * @sp: host1x syncpoint + */ +HOST1X_EMU_EXPORT_DECL(void, host1x_syncpt_put(struct host1x_syncpt *sp)) +{ + if (!sp) + return; + + kref_put(&sp->ref, syncpt_release); +} +HOST1X_EMU_EXPORT_SYMBOL(host1x_syncpt_put); + +/** + * @brief Retrieve syncpoint ID + * @sp: host1x syncpoint + * + * Given a pointer to a struct host1x_syncpt, retrieves its ID. This ID is + * often used as a value to program into registers that control how hardware + * blocks interact with syncpoints. + */ +HOST1X_EMU_EXPORT_DECL(u32, host1x_syncpt_id(struct host1x_syncpt *sp)) +{ + return sp->id; +} +HOST1X_EMU_EXPORT_SYMBOL(host1x_syncpt_id); + +/** + * @brief Wait for a syncpoint to reach a given threshold value. + * + * @sp: host1x syncpoint + * @thresh: threshold + * @timeout: maximum time to wait for the syncpoint to reach the given value + * @value: return location for the syncpoint value + * @ts: return location for completion timestamp + */ +HOST1X_EMU_EXPORT_DECL(int, host1x_syncpt_wait_ts(struct host1x_syncpt *sp, + u32 thresh, long timeout, u32 *value, ktime_t *ts)) +{ + ktime_t spin_timeout; + ktime_t time; + struct dma_fence *fence; + long wait_err; + + if (timeout < 0) + timeout = LONG_MAX; + + /* + * Even 1 jiffy is longer than 50us, so assume timeout is over 50us + * always except for polls (timeout=0) + */ + spin_timeout = ktime_add_us(ktime_get(), timeout > 0 ? 50 : 0); + for (;;) { + host1x_hw_syncpt_load(sp->host, sp); + time = ktime_get(); + if (value) + *value = host1x_syncpt_load(sp); + if (ts) + *ts = time; + if (host1x_syncpt_is_expired(sp, thresh)) + return 0; + if (ktime_compare(time, spin_timeout) > 0) + break; + udelay(5); + } + + if (timeout == 0) + return -EAGAIN; + + fence = HOST1X_EMU_EXPORT_CALL(host1x_fence_create(sp, thresh, false)); + if (IS_ERR(fence)) + return PTR_ERR(fence); + + wait_err = dma_fence_wait_timeout(fence, true, timeout); + if (wait_err == 0) + HOST1X_EMU_EXPORT_CALL(host1x_fence_cancel(fence)); + + if (value) + *value = host1x_syncpt_load(sp); + if (ts) + *ts = fence->timestamp; + + dma_fence_put(fence); + + /* + * Don't rely on dma_fence_wait_timeout return value, + * since it returns zero both on timeout and if the + * wait completed with 0 jiffies left. + */ + host1x_hw_syncpt_load(sp->host, sp); + if (wait_err == 0 && !host1x_syncpt_is_expired(sp, thresh)) + return -EAGAIN; + else if (wait_err < 0) + return wait_err; + else + return 0; +} +HOST1X_EMU_EXPORT_SYMBOL(host1x_syncpt_wait_ts); + +/** + * @brief Wait for a syncpoint to reach a given threshold value + * + * @sp: host1x syncpoint + * @thresh: threshold + * @timeout: maximum time to wait for the syncpoint to reach the given value + * @value: return location for the syncpoint value + */ +HOST1X_EMU_EXPORT_DECL(int, host1x_syncpt_wait(struct host1x_syncpt *sp, + u32 thresh, long timeout, u32 *value)) +{ + return HOST1X_EMU_EXPORT_CALL(host1x_syncpt_wait_ts(sp, thresh, + timeout, value, NULL)); +} +HOST1X_EMU_EXPORT_SYMBOL(host1x_syncpt_wait); diff --git a/drivers/gpu/host1x-emu/syncpt.h b/drivers/gpu/host1x-emu/syncpt.h new file mode 100644 index 00000000..2b7f5c95 --- /dev/null +++ b/drivers/gpu/host1x-emu/syncpt.h @@ -0,0 +1,146 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: GPL-2.0-only + */ +#ifndef __HOST1X_EMU_SYNCPT_H +#define __HOST1X_EMU_SYNCPT_H + +#include +#include +#include +#include +#ifdef CONFIG_TEGRA_HOST1X_EMU_DBG_SYMBL +#include +#else +#include +#include "include/linux/symbol-emu.h" +#endif + +#include "fence.h" +#include "poll.h" + +/* Reserved for replacing an expired wait with a NOP */ +struct host1x; + +struct host1x_syncpt_pool { + const char *name; + unsigned int sp_base; + unsigned int sp_end; + struct host1x *host; + struct delayed_work pool_work; + struct host1x_syncpt_list syncpt_list; +}; + +struct host1x_syncpt { + struct kref ref; + unsigned int id; + const char *name; + atomic_t min_val; + atomic_t max_val; + bool client_managed; + + struct host1x *host; + struct host1x_syncpt_pool *pool; + struct host1x_fence_list fences; + bool locked; + + /** + * Used for adding syncpoint to pool->syncpt-list + */ + struct list_head list; +}; + +/** + * Description: Initialize sync point array + */ +int host1x_syncpt_init(struct host1x *host); + +/** + * Description: Free sync point array + */ +void host1x_syncpt_deinit(struct host1x *host); + +/** + * Description: Return number of sync point supported. + */ +unsigned int host1x_syncpt_nb_pts(struct host1x *host); + +/** + * Description: Return true if sync point is client managed. + */ +static inline bool host1x_syncpt_client_managed(struct host1x_syncpt *sp) +{ + return sp->client_managed; +} + +/** + * Description: Check sync point sanity. + * + * If max is larger than min, there have too many sync point increments. + * Client managed sync point are not tracked. + */ +static inline bool host1x_syncpt_check_max(struct host1x_syncpt *sp, u32 real) +{ + u32 max; + if (sp->client_managed) + return true; + + max = HOST1X_EMU_EXPORT_CALL(host1x_syncpt_read_max(sp)); + return (s32)(max - real) >= 0; +} + +/** + * Description: Returns true if syncpoint min == max, which means that there are + * no outstanding operations. + */ +static inline bool host1x_syncpt_idle(struct host1x_syncpt *sp) +{ + int min, max; + smp_rmb(); + min = atomic_read(&sp->min_val); + max = atomic_read(&sp->max_val); + return (min == max); +} + +/** + * Description: Load current value from hardware to the shadow register. + */ +u32 host1x_syncpt_load(struct host1x_syncpt *sp); + +/** + * Description: Check if the given syncpoint value has already passed + */ +bool host1x_syncpt_is_expired(struct host1x_syncpt *sp, u32 thresh); + +/** + * Description: Save host1x sync point state into shadow registers. + */ +void host1x_syncpt_save(struct host1x *host); + +/** + * Description: Reset host1x sync point state from shadow registers. + */ +void host1x_syncpt_restore(struct host1x *host); + +/** + * Description: Indicate future operations by incrementing the sync point max. + */ +HOST1X_EMU_EXPORT_DECL(u32, host1x_syncpt_incr_max(struct host1x_syncpt *sp, + u32 incrs)); + +/** + * Description: Check if sync point id is valid. + */ +static inline int host1x_syncpt_is_valid(struct host1x_syncpt *sp) +{ + return sp->id < host1x_syncpt_nb_pts(sp->host); +} + +/** + * Description: Set sync as locked. + */ +static inline void host1x_syncpt_set_locked(struct host1x_syncpt *sp) +{ + sp->locked = true; +} +#endif