diff --git a/drivers/Makefile b/drivers/Makefile index c19519b9..c00ae4be 100644 --- a/drivers/Makefile +++ b/drivers/Makefile @@ -43,6 +43,7 @@ obj-m += misc/ obj-m += net/ obj-m += nvpps/ obj-m += nvpmodel/ +obj-m += nv-p2p/ ifdef CONFIG_PCI obj-m += pci/ ifdef CONFIG_TTY diff --git a/drivers/nv-p2p/Makefile b/drivers/nv-p2p/Makefile new file mode 100644 index 00000000..3d40ac6d --- /dev/null +++ b/drivers/nv-p2p/Makefile @@ -0,0 +1,5 @@ +# +# Nvidia Tegra RDMA driver. +# +ccflags-y += -Werror +obj-m += nvidia-p2p.o diff --git a/drivers/nv-p2p/nvidia-p2p.c b/drivers/nv-p2p/nvidia-p2p.c new file mode 100644 index 00000000..f785daa8 --- /dev/null +++ b/drivers/nv-p2p/nvidia-p2p.c @@ -0,0 +1,373 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +#include +#include +#include +#include +#include + +MODULE_DESCRIPTION("Nvidia Tegra P2P Driver"); +MODULE_AUTHOR("Preetham Chandru pchandru@nvidia.com"); +MODULE_LICENSE("GPL"); + +static inline u64 safe_cast_s32_to_u64(s32 si_a) +{ + if (si_a < 0) { + WARN_ON(1); + return 0UL; + } else { + return (u64)si_a; + } +} + +static inline s32 safe_cast_u64_to_s32(u64 ul_a) +{ + if (ul_a > safe_cast_s32_to_u64(INT_MAX)) { + WARN_ON(1); + return 0; + } else { + return (s32)ul_a; + } +} + +static inline u32 safe_cast_s32_to_u32(s32 si_a) +{ + if (si_a < 0) { + WARN_ON(1); + return 0U; + } else { + return (u32)si_a; + } +} + +static inline s32 safe_cast_u32_to_s32(u32 ui_a) +{ + if (ui_a > safe_cast_s32_to_u32(INT_MAX)) { + WARN_ON(1); + return 0; + } else { + return (s32)ui_a; + } +} + +static inline s32 safe_cast_s64_to_s32(s64 sl_a) +{ + if ((sl_a > INT_MAX) || (sl_a < INT_MIN)) { + WARN_ON(1); + return 0; + } else { + return (s32)sl_a; + } +} + +static void nvidia_p2p_mn_release(struct mmu_notifier *mn, + struct mm_struct *mm) +{ + struct nvidia_p2p_page_table *page_table = container_of(mn, + struct nvidia_p2p_page_table, + mn); + + page_table->free_callback(page_table->data); +} + +static void nvidia_p2p_mn_invl_range_start_legacy(struct mmu_notifier *mn, + struct mm_struct *mm, unsigned long start, unsigned long end) +{ + struct nvidia_p2p_page_table *page_table = container_of(mn, + struct nvidia_p2p_page_table, + mn); + + u64 vaddr = 0; + u64 size = 0; + + vaddr = page_table->vaddr; + size = page_table->size; + + if (vaddr >= start && vaddr <= end) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0) + mmu_notifier_put(&page_table->mn); +#else + mmu_notifier_unregister_no_release(&page_table->mn, page_table->mm); +#endif + page_table->free_callback(page_table->data); + } +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 20, 0) +static int nvidia_p2p_mn_invl_range_start(struct mmu_notifier *mn, + const struct mmu_notifier_range *range) +{ + nvidia_p2p_mn_invl_range_start_legacy(mn, NULL, range->start, range->end); + + return 0; +} +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0) +static void nvidia_p2p_free_notifier(struct mmu_notifier *mn) +{ +} +#endif + +static struct mmu_notifier_ops nvidia_p2p_mmu_ops = { + .release = nvidia_p2p_mn_release, +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 20, 0) + .invalidate_range_start = nvidia_p2p_mn_invl_range_start, +#else + .invalidate_range_start = nvidia_p2p_mn_invl_range_start_legacy, +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0) + .free_notifier = nvidia_p2p_free_notifier, +#endif +}; + +int nvidia_p2p_get_pages(u64 vaddr, u64 size, + struct nvidia_p2p_page_table **page_table, + void (*free_callback)(void *data), void *data) +{ + int ret = 0; + int user_pages = 0; + int locked = 0; + int nr_pages = safe_cast_u64_to_s32(size >> PAGE_SHIFT); + struct page **pages; + + if (nr_pages <= 0) { + return -EINVAL; + } + + *page_table = kzalloc(sizeof(**page_table), GFP_KERNEL); + if (!*page_table) { + return -ENOMEM; + } + + pages = kcalloc(nr_pages, sizeof(*pages), GFP_KERNEL); + if (!pages) { + ret = -ENOMEM; + goto free_page_table; + } +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0) + down_read(¤t->mm->mmap_lock); +#else + down_read(¤t->mm->mmap_sem); +#endif + locked = 1; + user_pages = safe_cast_s64_to_s32(get_user_pages_locked(vaddr & PAGE_MASK, nr_pages, + FOLL_WRITE | FOLL_FORCE, pages, &locked)); +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0) + up_read(¤t->mm->mmap_lock); +#else + up_read(¤t->mm->mmap_sem); +#endif + if (user_pages != nr_pages) { + ret = user_pages < 0 ? user_pages : -ENOMEM; + goto free_pages; + } + + (*page_table)->version = NVIDIA_P2P_PAGE_TABLE_VERSION; + (*page_table)->pages = pages; + (*page_table)->entries = user_pages; + (*page_table)->page_size = NVIDIA_P2P_PAGE_SIZE_4KB; + (*page_table)->size = size; + + (*page_table)->mn.ops = &nvidia_p2p_mmu_ops; + (*page_table)->mm = current->mm; + (*page_table)->free_callback = free_callback; + (*page_table)->data = data; + (*page_table)->vaddr = vaddr; + mutex_init(&(*page_table)->lock); + (*page_table)->mapped = NVIDIA_P2P_PINNED; + + ret = mmu_notifier_register(&(*page_table)->mn, (*page_table)->mm); + if (ret) { + goto free_pages; + } + + return 0; +free_pages: + while (--user_pages >= 0) { + put_page(pages[user_pages]); + } + kfree(pages); +free_page_table: + kfree(*page_table); + *page_table = NULL; + return ret; +} +EXPORT_SYMBOL(nvidia_p2p_get_pages); + +int nvidia_p2p_put_pages(struct nvidia_p2p_page_table *page_table) +{ + if (!page_table) { + return -EINVAL; + } + + mmu_notifier_unregister(&page_table->mn, page_table->mm); + + return 0; +} +EXPORT_SYMBOL(nvidia_p2p_put_pages); + +int nvidia_p2p_free_page_table(struct nvidia_p2p_page_table *page_table) +{ + int user_pages = 0; + struct page **pages = NULL; + + if (!page_table) { + return 0; + } + + mutex_lock(&page_table->lock); + + if (page_table->mapped & NVIDIA_P2P_MAPPED) { + WARN(1, "Attempting to free unmapped pages"); + } + + if (page_table->mapped & NVIDIA_P2P_PINNED) { + pages = page_table->pages; + user_pages = safe_cast_u32_to_s32(page_table->entries); + + while (--user_pages >= 0) { + put_page(pages[user_pages]); + } + + kfree(pages); + page_table->mapped &= ~NVIDIA_P2P_PINNED; + } + + mutex_unlock(&page_table->lock); + + return 0; +} +EXPORT_SYMBOL(nvidia_p2p_free_page_table); + +int nvidia_p2p_dma_map_pages(struct device *dev, + struct nvidia_p2p_page_table *page_table, + struct nvidia_p2p_dma_mapping **dma_mapping, + enum dma_data_direction direction) +{ + struct sg_table *sgt = NULL; + struct scatterlist *sg; + struct page **pages = NULL; + u32 nr_pages = 0; + int ret = 0; + int i, count; + + if (!page_table) { + return -EINVAL; + } + + mutex_lock(&page_table->lock); + + pages = page_table->pages; + nr_pages = page_table->entries; + if (nr_pages <= 0) { + mutex_unlock(&page_table->lock); + return -EINVAL; + } + + *dma_mapping = kzalloc(sizeof(**dma_mapping), GFP_KERNEL); + if (!*dma_mapping) { + mutex_unlock(&page_table->lock); + return -ENOMEM; + } + sgt = kzalloc(sizeof(*sgt), GFP_KERNEL); + if (!sgt) { + ret = -ENOMEM; + goto free_dma_mapping; + } + ret = sg_alloc_table_from_pages(sgt, pages, + nr_pages, 0, page_table->size, GFP_KERNEL); + if (ret) { + goto free_sgt; + } + + (*dma_mapping)->version = NVIDIA_P2P_DMA_MAPPING_VERSION; + (*dma_mapping)->sgt = sgt; + (*dma_mapping)->dev = dev; + (*dma_mapping)->direction = direction; + (*dma_mapping)->page_table = page_table; + + count = dma_map_sg(dev, sgt->sgl, sgt->nents, direction); + if (count < 1) { + goto free_sg_table; + } + + (*dma_mapping)->entries = count; + + (*dma_mapping)->hw_address = kcalloc(count, sizeof(u64), GFP_KERNEL); + if (!((*dma_mapping)->hw_address)) { + ret = -ENOMEM; + goto unmap_sg; + } + (*dma_mapping)->hw_len = kcalloc(count, sizeof(u64), GFP_KERNEL); + if (!((*dma_mapping)->hw_len)) { + ret = -ENOMEM; + goto free_hw_address; + } + + for_each_sg(sgt->sgl, sg, count, i) { + (*dma_mapping)->hw_address[i] = sg_dma_address(sg); + (*dma_mapping)->hw_len[i] = sg_dma_len(sg); + } + (*dma_mapping)->page_table->mapped |= NVIDIA_P2P_MAPPED; + mutex_unlock(&page_table->lock); + + return 0; +free_hw_address: + kfree((*dma_mapping)->hw_address); +unmap_sg: + dma_unmap_sg(dev, sgt->sgl, + safe_cast_u32_to_s32(sgt->nents), direction); +free_sg_table: + sg_free_table(sgt); +free_sgt: + kfree(sgt); +free_dma_mapping: + kfree(*dma_mapping); + *dma_mapping = NULL; + mutex_unlock(&page_table->lock); + + return ret; +} +EXPORT_SYMBOL(nvidia_p2p_dma_map_pages); + +int nvidia_p2p_dma_unmap_pages(struct nvidia_p2p_dma_mapping *dma_mapping) +{ + struct nvidia_p2p_page_table *page_table = NULL; + + if (!dma_mapping) { + return -EINVAL; + } + + page_table = dma_mapping->page_table; + if (!page_table) { + return -EFAULT; + } + + mutex_lock(&page_table->lock); + if (page_table->mapped & NVIDIA_P2P_MAPPED) { + kfree(dma_mapping->hw_len); + kfree(dma_mapping->hw_address); + if (dma_mapping->entries) + dma_unmap_sg(dma_mapping->dev, + dma_mapping->sgt->sgl, + safe_cast_u32_to_s32(dma_mapping->sgt->nents), + dma_mapping->direction); + sg_free_table(dma_mapping->sgt); + kfree(dma_mapping->sgt); + kfree(dma_mapping); + page_table->mapped &= ~NVIDIA_P2P_MAPPED; + } + mutex_unlock(&page_table->lock); + + return 0; +} +EXPORT_SYMBOL(nvidia_p2p_dma_unmap_pages); + +int nvidia_p2p_free_dma_mapping(struct nvidia_p2p_dma_mapping *dma_mapping) +{ + return nvidia_p2p_dma_unmap_pages(dma_mapping); +} +EXPORT_SYMBOL(nvidia_p2p_free_dma_mapping); diff --git a/include/linux/nv-p2p.h b/include/linux/nv-p2p.h new file mode 100644 index 00000000..c8a7d274 --- /dev/null +++ b/include/linux/nv-p2p.h @@ -0,0 +1,190 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +#ifndef __NVIDIA_P2P_H__ +#define __NVIDIA_P2P_H__ + +#include +#include + +#define NVIDIA_P2P_UNINITIALIZED 0x0 +#define NVIDIA_P2P_PINNED 0x1U +#define NVIDIA_P2P_MAPPED 0x2U + +#define NVIDIA_P2P_MAJOR_VERSION_MASK 0xffff0000 +#define NVIDIA_P2P_MINOR_VERSION_MASK 0x0000ffff + +#define NVIDIA_P2P_MAJOR_VERSION(v) \ + (((v) & NVIDIA_P2P_MAJOR_VERSION_MASK) >> 16) + +#define NVIDIA_P2P_MINOR_VERSION(v) \ + (((v) & NVIDIA_P2P_MINOR_VERSION_MASK)) + +#define NVIDIA_P2P_MAJOR_VERSION_MATCHES(p, v) \ + (NVIDIA_P2P_MAJOR_VERSION((p)->version) == NVIDIA_P2P_MAJOR_VERSION(v)) + +#define NVIDIA_P2P_VERSION_COMPATIBLE(p, v) \ + (NVIDIA_P2P_MAJOR_VERSION_MATCHES(p, v) && \ + (NVIDIA_P2P_MINOR_VERSION((p)->version) >= \ + (NVIDIA_P2P_MINOR_VERSION(v)))) + +enum nvidia_p2p_page_size_type { + NVIDIA_P2P_PAGE_SIZE_4KB = 0, + NVIDIA_P2P_PAGE_SIZE_64KB, + NVIDIA_P2P_PAGE_SIZE_128KB, + NVIDIA_P2P_PAGE_SIZE_COUNT +}; + +typedef struct nvidia_p2p_page_table { + u32 version; + u32 page_size; + u64 size; + u32 entries; + struct page **pages; + + u64 vaddr; + u32 mapped; + + struct mm_struct *mm; + struct mmu_notifier mn; + struct mutex lock; + void (*free_callback)(void *data); + void *data; +} nvidia_p2p_page_table_t; + +typedef struct nvidia_p2p_dma_mapping { + u32 version; + dma_addr_t *hw_address; + u32 *hw_len; + u32 entries; + + struct sg_table *sgt; + struct device *dev; + struct nvidia_p2p_page_table *page_table; + enum dma_data_direction direction; +} nvidia_p2p_dma_mapping_t; + +#define NVIDIA_P2P_PAGE_TABLE_VERSION 0x00010000 + +#define NVIDIA_P2P_PAGE_TABLE_VERSION_COMPATIBLE(p) \ + NVIDIA_P2P_VERSION_COMPATIBLE(p, NVIDIA_P2P_PAGE_TABLE_VERSION) + +/* + * @brief + * Make the pages underlying a range of GPU virtual memory + * accessible to a third-party device. + * + * @param[in] vaddr + * A GPU Virtual Address + * @param[in] size + * The size of the requested mapping. + * Size must be a multiple of Page size. + * @param[out] **page_table + * A pointer to struct nvidia_p2p_page_table + * @param[in] free_callback + * A non-NULL pointer to the function to be invoked when the pages + * underlying the virtual address range are freed + * implicitly. Must be non NULL. + * @param[in] data + * A non-NULL opaque pointer to private data to be passed to the + * callback function. + * + * @return + * 0 upon successful completion. + * Negative number if any error + */ +int nvidia_p2p_get_pages(u64 vaddr, u64 size, + struct nvidia_p2p_page_table **page_table, + void (*free_callback)(void *data), void *data); +/* + * @brief + * Release the pages previously made accessible to + * a third-party device. + * + * @param[in] *page_table + * A pointer to struct nvidia_p2p_page_table + * + * @return + * 0 upon successful completion. + * -ENOMEM if the driver failed to allocate memory or if + * insufficient resources were available to complete the operation. + * Negative number if any other error + */ +int nvidia_p2p_put_pages(struct nvidia_p2p_page_table *page_table); + +/* + * @brief + * Release the pages previously made accessible to + * a third-party device. This is called during the + * execution of the free_callback(). + * + * @param[in] *page_table + * A pointer to struct nvidia_p2p_page_table + * + * @return + * 0 upon successful completion. + * -ENOMEM if the driver failed to allocate memory or if + * insufficient resources were available to complete the operation. + * Negative number if any other error + */ +int nvidia_p2p_free_page_table(struct nvidia_p2p_page_table *page_table); + +#define NVIDIA_P2P_DMA_MAPPING_VERSION 0x00010000 + +#define NVIDIA_P2P_DMA_MAPPING_VERSION_COMPATIBLE(p) \ + NVIDIA_P2P_VERSION_COMPATIBLE(p, NVIDIA_P2P_DMA_MAPPING_VERSION) + +/* + * @brief + * Map the pages retrieved using nvidia_p2p_get_pages and + * pass the dma address to a third-party device. + * + * @param[in] *dev + * The peer device that needs to DMA to/from the + * mapping. + * @param[in] *page_table + * A pointer to struct nvidia_p2p_page_table + * @param[out] **map + * A pointer to struct nvidia_p2p_dma_mapping. + * The DMA mapping containing the DMA addresses to use. + * @param[in] direction + * DMA direction + * + * @return + * 0 upon successful completion. + * Negative number if any other error + */ +int nvidia_p2p_dma_map_pages(struct device *dev, + struct nvidia_p2p_page_table *page_table, + struct nvidia_p2p_dma_mapping **map, + enum dma_data_direction direction); +/* + * @brief + * Unmap the pages previously mapped using nvidia_p2p_dma_map_pages + * + * @param[in] *map + * A pointer to struct nvidia_p2p_dma_mapping. + * The DMA mapping containing the DMA addresses to use. + * + * @return + * 0 upon successful completion. + * Negative number if any other error + */ +int nvidia_p2p_dma_unmap_pages(struct nvidia_p2p_dma_mapping *map); + +/* + * @brief + * Unmap the pages previously mapped using nvidia_p2p_dma_map_pages. + * This is called during the execution of the free_callback(). + * + * @param[in] *map + * A pointer to struct nvidia_p2p_dma_mapping. + * The DMA mapping containing the DMA addresses to use. + * + * @return + * 0 upon successful completion. + * Negative number if any other error + */ +int nvidia_p2p_free_dma_mapping(struct nvidia_p2p_dma_mapping *dma_mapping); + +#endif