diff --git a/commitFile.txt b/commitFile.txt index d464da5..34e4d98 100644 --- a/commitFile.txt +++ b/commitFile.txt @@ -21,7 +21,7 @@ d580300f41118dacc5569fffa9f47e78c5883141 - kernel-open/common/inc/nv-modeset-int 5bc7a748c7d3dfa6559ca4f9fe6199e17098ec8f - kernel-open/common/inc/nv-lock.h b249abc0a7d0c9889008e98cb2f8515a9d310b85 - kernel-open/common/inc/nvgputypes.h e4a4f57abb8769d204468b2f5000c81f5ea7c92f - kernel-open/common/inc/nv-procfs.h -d9c22a9048dcc9eaf0548ba0f89b6861b01aa716 - kernel-open/common/inc/nv.h +3906bee59dc2433131bf58d12bdbeb7fedb8465f - kernel-open/common/inc/nv.h ede1f77acb43e28391bceac058e00a7a8d799b0d - kernel-open/common/inc/nvmisc.h ae374d3e438f8d3b60df8c4602618c58564b73f9 - kernel-open/common/inc/rm-gpu-ops.h 3f7b20e27e6576ee1f2f0557d269697a0b8af7ec - kernel-open/common/inc/nv-firmware-registry.h @@ -42,7 +42,7 @@ b986bc6591ba17a74ad81ec4c93347564c6d5165 - kernel-open/common/inc/nvkms-format.h 143051f69a53db0e7c5d2f846a9c14d666e264b4 - kernel-open/common/inc/nv-kref.h 56f432032bef4683c2801f46bec5065923475fb1 - kernel-open/common/inc/nv-kthread-q.h b4c5d759f035b540648117b1bff6b1701476a398 - kernel-open/common/inc/nvCpuUuid.h -c551c92e7de645fda78572fcf882cb5249f0f13c - kernel-open/common/inc/nv-linux.h +5d7fdbf9c5edea6088b471e27a9ce87a500d8f7e - kernel-open/common/inc/nv-linux.h 7c7888550b12eeb98128ea9ac771b897327f538e - kernel-open/common/inc/nv-hypervisor.h f9cb3701681994ff6f32833892d900b0da2b89f6 - kernel-open/common/inc/nv-pgprot.h b8700a911ac85770bf25d70b9692308af63966bd - kernel-open/common/inc/nvstatuscodes.h @@ -80,7 +80,7 @@ dcf4427b83cce7737f2b784d410291bf7a9612dc - kernel-open/nvidia/nv-reg.h 6b09b5ef8a37f78c8e82074b06b40ef593c81807 - kernel-open/nvidia/libspdm_rsa.c b8d361216db85fe897cbced2a9600507b7708c61 - kernel-open/nvidia/libspdm_hkdf_sha.c 66e2bfc490fb77e0b72a8192b719d3dc74d25d59 - kernel-open/nvidia/nv-pat.c -26a30f2d26c2a97a6e2ee457d97d32f48b0bf25b - kernel-open/nvidia/nv-vm.c +3abdd4528338680e0e85d2d7758cc7ba5b21ba79 - kernel-open/nvidia/nv-vm.c b8a770cea0629c57d8b0b3d7414d7b0f043ee8cf - kernel-open/nvidia/libspdm_ecc.c 4c183eb39251cd78d90868ec6f75ebc7a37e6644 - kernel-open/nvidia/os-usermap.c 8c30b6230439edcbec62636cc93be512bca8637f - kernel-open/nvidia/nv-usermap.c @@ -92,7 +92,7 @@ ef8fd76c55625aeaa71c9b789c4cf519ef6116b2 - kernel-open/nvidia/libspdm_hkdf.c f16e6a33b5004566333fb8b99504a0fb95d51226 - kernel-open/nvidia/nv-gpio.c 8ed2c3b93eeaa52342d944e794180fd5d386688a - kernel-open/nvidia/libspdm_rsa_ext.c 2e5d18118835c19c5ca7edee9bceeae613b9d7f9 - kernel-open/nvidia/nv-procfs.c -17502601d63ff2f353e2a65a016ee9310fa65f23 - kernel-open/nvidia/nv.c +ef2f1033d97fa5509d75feb223af5880c0384719 - kernel-open/nvidia/nv.c 65fe797fb5d4af2db67544ddb79d49ab1b7ca859 - kernel-open/nvidia/nv-dsi-parse-panel-props.c e3efae4ed920545062a2d06064df8be1a2a42135 - kernel-open/nvidia/nv-caps-imex.h 8c64e75aaaa9ac6f17aae7ed62db23eb2e5b9953 - kernel-open/nvidia/nv_uvm_interface.c @@ -108,7 +108,7 @@ c762aa186dc72ed0b9183492f9bd187c301d33d3 - kernel-open/nvidia/nv-kthread-q.c 70bece14e12b9ffc92816ee8159a4ce596579d78 - kernel-open/nvidia/os-pci.c a677049bb56fa5ebe22fe43b0c4a12acd58a6677 - kernel-open/nvidia/nv-p2p.c e4d12f027cb5f74124da71bbbc23bcb33651834a - kernel-open/nvidia/nv-pci-table.c -1034de1ab2403c30bb22d6a65bd2134a540590f4 - kernel-open/nvidia/nv-pci.c +8b66b2577aa817d8b06712a3f4b92228708d1194 - kernel-open/nvidia/nv-pci.c 6dfc57ac42bed97c6ff81d82e493f05b369e0b84 - kernel-open/nvidia/nvspdm_cryptlib_extensions.h bba706cfbc04b3a880b5e661066f92e765fad663 - kernel-open/nvidia/nv-caps-imex.c ed3c83f62e4ccc4b53d886eedd4b47518a361393 - kernel-open/nvidia/nv-dmabuf.c @@ -849,7 +849,7 @@ c5f16fdf43ca3d2845d120c219d1da11257072b0 - src/nvidia/nv-kernel.ld dcf4427b83cce7737f2b784d410291bf7a9612dc - src/nvidia/arch/nvalloc/unix/include/nv-reg.h 4750735d6f3b334499c81d499a06a654a052713d - src/nvidia/arch/nvalloc/unix/include/nv-caps.h 3c61881e9730a8a1686e422358cdfff59616b670 - src/nvidia/arch/nvalloc/unix/include/nv_escape.h -1650cc4d3013f9f91ef1708ef305ae129723721f - src/nvidia/arch/nvalloc/unix/include/nv.h +aa399178ed2525bd2c9cf9d097e5c51ef96d1264 - src/nvidia/arch/nvalloc/unix/include/nv.h 81592e5c17bebad04cd11d73672c859baa070329 - src/nvidia/arch/nvalloc/unix/include/nv-chardev-numbers.h e69045379ed58dc0110d16d17eb39a6f600f0d1d - src/nvidia/arch/nvalloc/unix/include/nv-ioctl-lockless-diag.h d1b1a1bc1fa30c1a966e95447f7831a06340d2d0 - src/nvidia/arch/nvalloc/unix/include/nv-priv.h @@ -864,7 +864,7 @@ b3ecb82f142a50bdc37eafaeb86d67f10fbcf73f - src/nvidia/arch/nvalloc/unix/include/ af45762b6eeae912cc2602acf7dc31d30775ade7 - src/nvidia/arch/nvalloc/unix/include/nv-kernel-rmapi-ops.h 107d1ecb8a128044260915ea259b1e64de3defea - src/nvidia/arch/nvalloc/unix/include/nv-ioctl-numbers.h 3a26838c4edd3525daa68ac6fc7b06842dc6fc07 - src/nvidia/arch/nvalloc/unix/include/nv-gpu-info.h -70ca9726e45c08ec2436b6f9f6e52d4c59cdf470 - src/nvidia/arch/nvalloc/unix/src/os.c +aa6a617b459fba57cd11dec4c669e30aae309405 - src/nvidia/arch/nvalloc/unix/src/os.c a659a503a6fcffdcacd2b76ae6b1f156b4b9216c - src/nvidia/arch/nvalloc/unix/src/osmemdesc.c b5ae9b8d551a3e5489605c13686fb6cce4579598 - src/nvidia/arch/nvalloc/unix/src/power-management-tegra.c a17aae37486b325442e447489b64add3694ab8b0 - src/nvidia/arch/nvalloc/unix/src/osunix.c @@ -892,7 +892,7 @@ b9903d23010ea9d63117c27d5fe0cfba09849fa4 - src/nvidia/generated/g_context_dma_nv c5cad88aa7de5a04a3b6f9836f355347448d6a7b - src/nvidia/generated/g_rmconfig_util.h db1d1e047d00780efbe4c1c1ae6e4fecd3ab49e8 - src/nvidia/generated/g_os_desc_mem_nvoc.h 1ec59322d0874153252a387dcb50bf6d7328d56e - src/nvidia/generated/g_system_mem_nvoc.c -b141335ea49d7ef1b4ca267ed6db2803add4021d - src/nvidia/generated/g_gpu_nvoc.c +5d15a2256b4b06cca30e9d52e13f58c1bdecc7f6 - src/nvidia/generated/g_gpu_nvoc.c 4613f3d42dbc899b278fca71c3aaae79159d7dbe - src/nvidia/generated/g_gpu_user_shared_data_nvoc.c b55573cb02ff8129aa4f5aa050ac53d1f4fcfdb2 - src/nvidia/generated/g_rs_resource_nvoc.h 16c8d551a3a908ec194d39c88c5603cea436c9b7 - src/nvidia/generated/g_binary_api_nvoc.c @@ -910,7 +910,7 @@ abc769851bd523ee08cf829bf3864cf5475066ec - src/nvidia/generated/g_subdevice_nvoc 255c404719b18c2a3aec2a47948c0fbcf4affd4b - src/nvidia/generated/rmconfig.h c7fda8cbe109ad2736694ce9ec0e2ab93d0e3f2c - src/nvidia/generated/g_mem_list_nvoc.h f9bdef39159a8475626a0edcbc3a53505a0ff80a - src/nvidia/generated/g_os_hal.h -e3a0a139faf20b6d7c1242efa1bcd081596bbce3 - src/nvidia/generated/g_mem_desc_nvoc.h +8a05682a80229723be9cc37bcf9509b6091e8a83 - src/nvidia/generated/g_mem_desc_nvoc.h 1702c9d021149c0f5c73ebeda7bea29e246af31d - src/nvidia/generated/g_nv_name_released.h 2e0c45e4186d44774286a71daf797c980c2ddf7a - src/nvidia/generated/g_objtmr_nvoc.c 9b78bc02a8fe0ec297167bb4bdb7f8255b94198b - src/nvidia/generated/g_disp_capabilities_nvoc.h @@ -975,7 +975,7 @@ cc7ec616b034ec01da1c5176b6c62759c3f31a06 - src/nvidia/generated/g_subdevice_nvoc 3b0e038829647cfe0d8807579db33416a420d1d2 - src/nvidia/generated/g_chips2halspec.h a1fad555b8ad36437992afdd6e3e08d236167ac7 - src/nvidia/generated/g_journal_nvoc.h d210a82e3dda39239201cfc1c2fcb2e971915c1e - src/nvidia/generated/g_device_nvoc.h -836f88914b046eadad9435786e1b474ee6690f5f - src/nvidia/generated/g_gpu_nvoc.h +f2752ea5d78a8dc2d90e92bb019def5fc3021fff - src/nvidia/generated/g_gpu_nvoc.h ea0d27b0f05818e2e44be7d04b31f8843e1d05b7 - src/nvidia/generated/g_io_vaspace_nvoc.c 10529db24fb0501aa7f2aae25e0a87247ab5405c - src/nvidia/generated/g_resource_nvoc.h 5d47bed309c731bfee4144f61093192e7efcaa55 - src/nvidia/generated/g_disp_channel_nvoc.c @@ -1343,7 +1343,7 @@ f6e518524581b772f8fdbc80418a2018570940ca - src/nvidia/src/kernel/gpu/timer/timer 10a8bfd47ce609763c07a0d61be2f71f9f91889e - src/nvidia/src/kernel/gpu/mem_mgr/mem_ctrl.c bfc82499a8b9b8ce10411f6c391b0e575dc7c0d6 - src/nvidia/src/kernel/gpu/mem_mgr/context_dma.c a62f423d6cf69e96b0523a233ec00353d63ee8bd - src/nvidia/src/kernel/gpu/mem_mgr/mem_utils.c -b68459a8edae4d51a7068e37e49e924680e690b4 - src/nvidia/src/kernel/gpu/mem_mgr/mem_desc.c +7bd464742552f52a6e509659fde2a914e8194f26 - src/nvidia/src/kernel/gpu/mem_mgr/mem_desc.c 4a95b73f744807d96510b0ad7181eae5b12839ce - src/nvidia/src/kernel/gpu/mem_mgr/arch/turing/mem_mgr_tu102_base.c c3ce2ab15dcf7b0ae2dd89a62d73d16c0aba9687 - src/nvidia/src/kernel/gpu/dce_client/dce_client_rpc.c cebb9eee63e23bb934881b3313e422b50fb38abb - src/nvidia/src/kernel/gpu/dce_client/dce_client.c @@ -1357,7 +1357,7 @@ fe618e428d9a172a0fd9412f5a20df64d7270418 - src/nvidia/src/kernel/gpu_mgr/gpu_mgr 5a5e689cf264134ae8c4300d986c209c04167743 - src/nvidia/src/kernel/mem_mgr/vaspace.c 5b9048e62581a3fbb0227d1a46c4ee8d8397bf5b - src/nvidia/src/kernel/mem_mgr/mem_mgr_internal.h 630200d06b6588d7fa8c5b1ea16146e8281163d7 - src/nvidia/src/kernel/mem_mgr/io_vaspace.c -04876ed2dedf0ac3228ec6261a0f3f79609e44a5 - src/nvidia/src/kernel/mem_mgr/system_mem.c +d2bf2a33ee9ee9bfbe360506a34b9b4a91c9c6fa - src/nvidia/src/kernel/mem_mgr/system_mem.c 873de51b330501a86ec7656fcf3f615034c49f8e - src/nvidia/src/kernel/mem_mgr/os_desc_mem.c ed8376f04af08af8da7d47c6340ff38a8910de87 - src/nvidia/src/kernel/mem_mgr/mem.c 08762b3172f6309f1aeab895761193fa19cb176f - src/nvidia/interface/nv_sriov_defines.h diff --git a/kernel-open/common/inc/nv-linux.h b/kernel-open/common/inc/nv-linux.h index 506c130..4a57262 100644 --- a/kernel-open/common/inc/nv-linux.h +++ b/kernel-open/common/inc/nv-linux.h @@ -1055,6 +1055,7 @@ typedef struct nv_alloc_s { NvBool unencrypted : 1; NvBool coherent : 1; NvBool carveout : 1; + NvBool no_reclaim : 1; } flags; unsigned int cache_type; unsigned int num_pages; @@ -1540,7 +1541,6 @@ typedef struct nv_linux_state_s { struct nv_pci_tegra_devfreq_dev *nvd_devfreq_dev; struct nv_pci_tegra_devfreq_dev *sys_devfreq_dev; struct nv_pci_tegra_devfreq_dev *pwr_devfreq_dev; - NvU32 tegra_suspend_freq; int (*devfreq_suspend)(struct device *dev); int (*devfreq_resume)(struct device *dev); diff --git a/kernel-open/common/inc/nv.h b/kernel-open/common/inc/nv.h index 836fef5..370e85a 100644 --- a/kernel-open/common/inc/nv.h +++ b/kernel-open/common/inc/nv.h @@ -740,6 +740,20 @@ typedef enum NV_MEMORY_TYPE_DEVICE_MMIO, /* All kinds of MMIO referred by NVRM e.g. BARs and MCFG of device */ } nv_memory_type_t; +typedef struct nv_allocation_request_s +{ + NvU32 count; + NvU64 page_size; + NvBool alloc_type_contiguous; + NvU32 cache_type; + NvBool alloc_type_zeroed; + NvBool unencrypted; + NvBool no_reclaim; + NvS32 node_id; + NvU64 *pte_array; + void **private; +} nv_allocation_request_t; + #define NV_PRIMARY_VGA(nv) ((nv)->primary_vga) #define NV_IS_CTL_DEVICE(nv) ((nv)->flags & NV_FLAG_CONTROL) @@ -911,7 +925,7 @@ nv_state_t* NV_API_CALL nv_get_ctl_state (void); void NV_API_CALL nv_set_dma_address_size (nv_state_t *, NvU32 ); NV_STATUS NV_API_CALL nv_alias_pages (nv_state_t *, NvU32, NvU64, NvU32, NvU32, NvU64, NvU64 *, NvBool, void **); -NV_STATUS NV_API_CALL nv_alloc_pages (nv_state_t *, NvU32, NvU64, NvBool, NvU32, NvBool, NvBool, NvS32, NvU64 *, void **); +NV_STATUS NV_API_CALL nv_alloc_pages (nv_state_t *, nv_allocation_request_t *); NV_STATUS NV_API_CALL nv_free_pages (nv_state_t *, NvU32, NvBool, NvU32, void *); NV_STATUS NV_API_CALL nv_register_user_pages (nv_state_t *, NvU64, NvU64 *, void *, void **, NvBool); diff --git a/kernel-open/nvidia/nv-pci.c b/kernel-open/nvidia/nv-pci.c index b552b43..f6cce2a 100644 --- a/kernel-open/nvidia/nv-pci.c +++ b/kernel-open/nvidia/nv-pci.c @@ -78,13 +78,6 @@ #include #endif -/* - * Set the default devfreq suspend frequency to 315 MHz - * considering the balance between power consumption - * and performance based on various scenarios. - */ -#define NV_PCI_TEGRA_DEVFREQ_SUSPEND_FREQ 315000000 - extern int NVreg_GrdmaPciTopoCheckOverride; static void @@ -658,18 +651,13 @@ nv_pci_gb10b_devfreq_target(struct device *dev, unsigned long *freq, u32 flags) u32 kBps; #endif - // - // GPU can be under suspending/suspended/resuming state defined the runtime - // PM (RPM) framework. When device is under either of these states, DVFS based - // on GPU load information should be disabled. - // - // Complete load-based DVFS cycle involve GPU load query through rmapi and - // clock scaling through BPMP MRQ_CLK mailbox request, which will awake the - // GPU and contradict the suspended state. - // - if (!pm_runtime_active(&pdev->dev)) + /* + * If the device is suspended, skip the frequency scaling + * because the clocks may be unavailable. + * Otherwise, set the clocks to the target frequency. + */ + if (pm_runtime_suspended(&pdev->dev)) { - *freq = tdev->devfreq->scaling_min_freq; return 0; } @@ -753,10 +741,13 @@ nv_pci_tegra_devfreq_get_cur_freq(struct device *dev, unsigned long *freq) // will further call the get_dev_status callback function. // if (pm_runtime_active(dev->parent)) + { *freq = clk_get_rate(tdev->clk); + } else - *freq = tdev->devfreq->scaling_min_freq; - + { + *freq = tdev->devfreq->previous_freq; + } return 0; } @@ -786,7 +777,7 @@ nv_pci_tegra_devfreq_get_dev_status(struct device *dev, { stat->total_time = 100; stat->busy_time = 0; - stat->current_frequency = tdev->devfreq->scaling_min_freq; + stat->current_frequency = tdev->devfreq->previous_freq; return 0; } @@ -1198,13 +1189,6 @@ nv_pci_gb10b_register_devfreq(struct pci_dev *pdev) nvl->gpc_devfreq_dev->devfreq = NULL; goto error_slave_teardown; } -#if defined(NV_DEVFREQ_HAS_SUSPEND_FREQ) - else - { - // Set the devfreq suspend frequency for the GPC devfreq device. - nvl->gpc_devfreq_dev->devfreq->suspend_freq = nvl->tegra_suspend_freq; - } -#endif if (nvl->sys_devfreq_dev != NULL) { list_add_tail(&nvl->sys_devfreq_dev->gpc_cluster, &nvl->gpc_devfreq_dev->gpc_cluster); @@ -1226,13 +1210,6 @@ nv_pci_gb10b_register_devfreq(struct pci_dev *pdev) nvl->nvd_devfreq_dev->devfreq = NULL; goto error_slave_teardown; } -#if defined(NV_DEVFREQ_HAS_SUSPEND_FREQ) - else - { - // Set the devfreq suspend frequency for the NVD devfreq device. - nvl->nvd_devfreq_dev->devfreq->suspend_freq = nvl->tegra_suspend_freq; - } -#endif if (nvl->sys_devfreq_dev != NULL) { list_add_tail(&nvl->sys_devfreq_dev->nvd_cluster, &nvl->nvd_devfreq_dev->nvd_cluster); @@ -1285,6 +1262,10 @@ nv_pci_gb10b_suspend_devfreq(struct device *dev) if (nvl->gpc_devfreq_dev != NULL && nvl->gpc_devfreq_dev->devfreq != NULL) { + mutex_lock(&nvl->gpc_devfreq_dev->devfreq->lock); + nvl->gpc_devfreq_dev->devfreq->suspend_freq = nvl->gpc_devfreq_dev->devfreq->previous_freq; + mutex_unlock(&nvl->gpc_devfreq_dev->devfreq->lock); + err = devfreq_suspend_device(nvl->gpc_devfreq_dev->devfreq); if (err) { @@ -1301,6 +1282,10 @@ nv_pci_gb10b_suspend_devfreq(struct device *dev) if (nvl->nvd_devfreq_dev != NULL && nvl->nvd_devfreq_dev->devfreq != NULL) { + mutex_lock(&nvl->nvd_devfreq_dev->devfreq->lock); + nvl->nvd_devfreq_dev->devfreq->suspend_freq = nvl->nvd_devfreq_dev->devfreq->previous_freq; + mutex_unlock(&nvl->nvd_devfreq_dev->devfreq->lock); + err = devfreq_suspend_device(nvl->nvd_devfreq_dev->devfreq); if (err) { @@ -1332,6 +1317,17 @@ nv_pci_gb10b_resume_devfreq(struct device *dev) { return err; } + /* + * During GPU runtime suspended state, switching the devfreq governor + * which doesn't poll for GPU utilization could lead to devfreq + * frequency not being updated after runtime resume. + * + * Manually trigger the devfreq update here to ensure the + * frequency is compliant with the devfreq governor. + */ + mutex_lock(&nvl->gpc_devfreq_dev->devfreq->lock); + update_devfreq(nvl->gpc_devfreq_dev->devfreq); + mutex_unlock(&nvl->gpc_devfreq_dev->devfreq->lock); } if (nvl->nvd_devfreq_dev != NULL && nvl->nvd_devfreq_dev->devfreq != NULL) @@ -1341,6 +1337,9 @@ nv_pci_gb10b_resume_devfreq(struct device *dev) { return err; } + mutex_lock(&nvl->nvd_devfreq_dev->devfreq->lock); + update_devfreq(nvl->nvd_devfreq_dev->devfreq); + mutex_unlock(&nvl->nvd_devfreq_dev->devfreq->lock); } return err; @@ -1592,10 +1591,6 @@ nv_pci_tegra_register_devfreq(struct pci_dev *pdev) nv_linux_state_t *nvl = pci_get_drvdata(pdev); const struct nv_pci_tegra_data *tegra_data = NULL; int err; -#if defined(NV_DEVFREQ_HAS_SUSPEND_FREQ) - struct device_node *np = pdev->dev.of_node; - NvU32 suspend_freq = 0; -#endif tegra_data = nv_pci_get_tegra_igpu_data(pdev); @@ -1611,15 +1606,6 @@ nv_pci_tegra_register_devfreq(struct pci_dev *pdev) nvl->devfreq_enable_boost = tegra_data->devfreq_enable_boost; nvl->devfreq_disable_boost = tegra_data->devfreq_disable_boost; -#if defined(NV_DEVFREQ_HAS_SUSPEND_FREQ) - of_property_read_u32(np, "nvidia,suspend-freq", &suspend_freq); - if (suspend_freq == 0) - { - suspend_freq = NV_PCI_TEGRA_DEVFREQ_SUSPEND_FREQ; - } - nvl->tegra_suspend_freq = suspend_freq; -#endif - err = tegra_data->devfreq_register(pdev); if (err != 0) { diff --git a/kernel-open/nvidia/nv-vm.c b/kernel-open/nvidia/nv-vm.c index 2896dbd..b5f8861 100644 --- a/kernel-open/nvidia/nv-vm.c +++ b/kernel-open/nvidia/nv-vm.c @@ -278,6 +278,12 @@ static unsigned int nv_compute_gfp_mask( if (at->order > 0) gfp_mask |= __GFP_COMP; + if (at->flags.no_reclaim) + { + gfp_mask &= ~(__GFP_RETRY_MAYFAIL | __GFP_RECLAIM); + gfp_mask |= __GFP_NORETRY; + } + return gfp_mask; } diff --git a/kernel-open/nvidia/nv.c b/kernel-open/nvidia/nv.c index cdb1a6c..d222448 100644 --- a/kernel-open/nvidia/nv.c +++ b/kernel-open/nvidia/nv.c @@ -3717,15 +3717,7 @@ void NV_API_CALL nv_free_kernel_mapping( NV_STATUS NV_API_CALL nv_alloc_pages( nv_state_t *nv, - NvU32 page_count, - NvU64 page_size, - NvBool contiguous, - NvU32 cache_type, - NvBool zeroed, - NvBool unencrypted, - NvS32 node_id, - NvU64 *pte_array, - void **priv_data + nv_allocation_request_t *alloc_request ) { nv_alloc_t *at; @@ -3734,6 +3726,16 @@ NV_STATUS NV_API_CALL nv_alloc_pages( NvBool will_remap = NV_FALSE; NvU32 i; struct device *dev = NULL; + NvU32 page_count = alloc_request->count; + NvU64 page_size = alloc_request->page_size; + NvBool contiguous = alloc_request->alloc_type_contiguous; + NvU32 cache_type = alloc_request->cache_type; + NvBool zeroed = alloc_request->alloc_type_zeroed; + NvBool unencrypted = alloc_request->unencrypted; + NvBool no_reclaim = alloc_request->no_reclaim; + NvS32 node_id = alloc_request->node_id; + NvU64 *pte_array = alloc_request->pte_array; + void **priv_data = alloc_request->private; nv_printf(NV_DBG_MEMINFO, "NVRM: VM: nv_alloc_pages: %d pages, nodeid %d\n", page_count, node_id); nv_printf(NV_DBG_MEMINFO, "NVRM: VM: contig %d cache_type %d\n", @@ -3776,6 +3778,8 @@ NV_STATUS NV_API_CALL nv_alloc_pages( at->node_id = node_id; } + at->flags.no_reclaim = no_reclaim; + if (at->flags.contig) { status = nv_alloc_contig_pages(nv, at); diff --git a/push_info.txt b/push_info.txt index 2707cc1..b1c7920 100644 --- a/push_info.txt +++ b/push_info.txt @@ -1 +1 @@ -rel-38_eng_2025-12-05 +rel-38_eng_2025-12-16 diff --git a/src/nvidia/arch/nvalloc/unix/include/nv.h b/src/nvidia/arch/nvalloc/unix/include/nv.h index dd35d8e..6a5bea7 100644 --- a/src/nvidia/arch/nvalloc/unix/include/nv.h +++ b/src/nvidia/arch/nvalloc/unix/include/nv.h @@ -740,6 +740,20 @@ typedef enum NV_MEMORY_TYPE_DEVICE_MMIO, /* All kinds of MMIO referred by NVRM e.g. BARs and MCFG of device */ } nv_memory_type_t; +typedef struct nv_allocation_request_s +{ + NvU32 count; + NvU64 page_size; + NvBool alloc_type_contiguous; + NvU32 cache_type; + NvBool alloc_type_zeroed; + NvBool unencrypted; + NvBool no_reclaim; + NvS32 node_id; + NvU64 *pte_array; + void **private; +} nv_allocation_request_t; + #define NV_PRIMARY_VGA(nv) ((nv)->primary_vga) #define NV_IS_CTL_DEVICE(nv) ((nv)->flags & NV_FLAG_CONTROL) @@ -911,7 +925,7 @@ nv_state_t* NV_API_CALL nv_get_ctl_state (void); void NV_API_CALL nv_set_dma_address_size (nv_state_t *, NvU32 ); NV_STATUS NV_API_CALL nv_alias_pages (nv_state_t *, NvU32, NvU64, NvU32, NvU32, NvU64, NvU64 *, NvBool, void **); -NV_STATUS NV_API_CALL nv_alloc_pages (nv_state_t *, NvU32, NvU64, NvBool, NvU32, NvBool, NvBool, NvS32, NvU64 *, void **); +NV_STATUS NV_API_CALL nv_alloc_pages (nv_state_t *, nv_allocation_request_t *); NV_STATUS NV_API_CALL nv_free_pages (nv_state_t *, NvU32, NvBool, NvU32, void *); NV_STATUS NV_API_CALL nv_register_user_pages (nv_state_t *, NvU64, NvU64 *, void *, void **, NvBool); diff --git a/src/nvidia/arch/nvalloc/unix/src/os.c b/src/nvidia/arch/nvalloc/unix/src/os.c index b05077d..d753450 100644 --- a/src/nvidia/arch/nvalloc/unix/src/os.c +++ b/src/nvidia/arch/nvalloc/unix/src/os.c @@ -894,18 +894,24 @@ NV_STATUS osAllocPagesInternal( if (status != NV_OK) goto done; - status = nv_alloc_pages( - NV_GET_NV_STATE(pGpu), - osPageCount, // TODO: This call needs to receive the page count param at the requested page size. - pageSize, - memdescGetContiguity(pMemDesc, AT_CPU), - cpuCacheAttrib, - pSys->getProperty(pSys, - PDB_PROP_SYS_INITIALIZE_SYSTEM_MEMORY_ALLOCATIONS), - unencrypted, - nodeId, - memdescGetPteArray(pMemDesc, AT_CPU), - &pMemData); + { + nv_allocation_request_t alloc_request = {0}; + alloc_request.count = osPageCount; // TODO: This call needs to receive the page count param at the requested page size. + alloc_request.page_size = pageSize; + alloc_request.alloc_type_contiguous = memdescGetContiguity(pMemDesc, AT_CPU); + alloc_request.cache_type = cpuCacheAttrib; + alloc_request.alloc_type_zeroed = pSys->getProperty(pSys, + PDB_PROP_SYS_INITIALIZE_SYSTEM_MEMORY_ALLOCATIONS); + alloc_request.unencrypted = unencrypted; + alloc_request.no_reclaim = memdescGetFlag(pMemDesc, MEMDESC_FLAGS_ALLOC_NO_RECLAIM); + alloc_request.node_id = nodeId; + alloc_request.pte_array = memdescGetPteArray(pMemDesc, AT_CPU); + alloc_request.private = &pMemData; + + status = nv_alloc_pages( + NV_GET_NV_STATE(pGpu), + &alloc_request); + } if (nv && nv->force_dma32_alloc) nv->force_dma32_alloc = NV_FALSE; diff --git a/src/nvidia/generated/g_gpu_nvoc.c b/src/nvidia/generated/g_gpu_nvoc.c index e0bad0e..469b1ae 100644 --- a/src/nvidia/generated/g_gpu_nvoc.c +++ b/src/nvidia/generated/g_gpu_nvoc.c @@ -429,6 +429,9 @@ void __nvoc_init_dataField_OBJGPU(OBJGPU *pThis) { pThis->bInstanceMemoryAlwaysCached = NV_TRUE; } + // Hal field -- bAPageSizeAllocRetryEnabled + pThis->bAPageSizeAllocRetryEnabled = NV_FALSE; + // Hal field -- bComputePolicyTimesliceSupported // default { diff --git a/src/nvidia/generated/g_gpu_nvoc.h b/src/nvidia/generated/g_gpu_nvoc.h index 6e8d613..4ab342e 100644 --- a/src/nvidia/generated/g_gpu_nvoc.h +++ b/src/nvidia/generated/g_gpu_nvoc.h @@ -1293,6 +1293,7 @@ struct OBJGPU { NvBool bClientRmAllocatedCtxBuffer; NvBool bEccPageRetirementWithSliAllowed; NvBool bInstanceMemoryAlwaysCached; + NvBool bAPageSizeAllocRetryEnabled; NvBool bUseRpcSimEscapes; NvBool bRmProfilingPrivileged; NvBool bGeforceSmb; diff --git a/src/nvidia/generated/g_mem_desc_nvoc.h b/src/nvidia/generated/g_mem_desc_nvoc.h index d2b16f9..8906de2 100644 --- a/src/nvidia/generated/g_mem_desc_nvoc.h +++ b/src/nvidia/generated/g_mem_desc_nvoc.h @@ -381,7 +381,11 @@ typedef struct ADDRESS_TRANSLATION_ *ADDRESS_TRANSLATION; // 32-bit addressable. #define MEMDESC_FLAGS_ALLOC_32BIT_ADDRESSABLE NVBIT64(41) -// unused NVBIT64(42) +// +// This flag causes linux to not try as hard to reclaim used pages. Useful when +// we are retrying with successively smaller page sizes as in sysmemConstruct. +// +#define MEMDESC_FLAGS_ALLOC_NO_RECLAIM NVBIT64(42) // // If this flag is set then it indicates that the memory associated with diff --git a/src/nvidia/src/kernel/gpu/mem_mgr/mem_desc.c b/src/nvidia/src/kernel/gpu/mem_mgr/mem_desc.c index 2aef019..1ec1666 100644 --- a/src/nvidia/src/kernel/gpu/mem_mgr/mem_desc.c +++ b/src/nvidia/src/kernel/gpu/mem_mgr/mem_desc.c @@ -1051,10 +1051,17 @@ memdescAlloc } // Actually allocate the memory - NV_CHECK_OK(status, LEVEL_ERROR, _memdescAllocInternal(pMemDesc)); - + status = _memdescAllocInternal(pMemDesc); if (status != NV_OK) { + if (status == NV_ERR_NO_MEMORY && pGpu->bAPageSizeAllocRetryEnabled && pMemDesc->_pageSize != RM_PAGE_SIZE) + { + NV_PRINTF(LEVEL_INFO, "Failed to allocate memory due to insufficient memory with page size 0x%llx, retrying with page size 0x%x\n", pMemDesc->_pageSize, RM_PAGE_SIZE); + } + else + { + NV_PRINTF(LEVEL_ERROR, "Failed to allocate memory with page size 0x%llx\n status: %x", pMemDesc->_pageSize, status); + } pMemDesc->pHeap = NULL; } diff --git a/src/nvidia/src/kernel/mem_mgr/system_mem.c b/src/nvidia/src/kernel/mem_mgr/system_mem.c index 56afabc..66dcb5e 100644 --- a/src/nvidia/src/kernel/mem_mgr/system_mem.c +++ b/src/nvidia/src/kernel/mem_mgr/system_mem.c @@ -154,7 +154,9 @@ sysmemConstruct_IMPL MEMORY_DESCRIPTOR *pMemDesc; NvU32 flags; RM_ATTR_PAGE_SIZE pageSizeAttr; - NvBool bRetry = NV_FALSE; + NvBool bLastAttempt = NV_TRUE; + + pGpu->bAPageSizeAllocRetryEnabled = NV_FALSE; NV_ASSERT_OR_RETURN(pRmClient != NULL, NV_ERR_INVALID_CLIENT); @@ -203,7 +205,8 @@ sysmemConstruct_IMPL if (FLD_TEST_DRF(OS32, _ATTR, _PAGE_SIZE, _DEFAULT, pAllocData->attr) && (GPU_GET_MEMORY_MANAGER(pGpu)->bSysmemPageSizeDefaultAllowLargePages)) { - bRetry = NV_TRUE; + pGpu->bAPageSizeAllocRetryEnabled = NV_TRUE; + bLastAttempt = NV_FALSE; } do @@ -247,6 +250,8 @@ sysmemConstruct_IMPL if (FLD_TEST_DRF(OS32, _ATTR2, _NISO_DISPLAY, _YES, pAllocData->attr2)) memdescSetFlag(pMemDesc, MEMDESC_FLAGS_MEMORY_TYPE_DISPLAY_NISO, NV_TRUE); + memdescSetFlag(pMemDesc, MEMDESC_FLAGS_ALLOC_NO_RECLAIM, !bLastAttempt); + memdescSetFlag(pMemDesc, MEMDESC_FLAGS_SYSMEM_OWNED_BY_CLIENT, NV_TRUE); if (FLD_TEST_DRF(OS32, _ATTR2, _FIXED_NUMA_NODE_ID, _YES, pAllocData->attr2)) @@ -273,13 +278,21 @@ sysmemConstruct_IMPL memdescTagAlloc(rmStatus, NV_FB_ALLOC_RM_INTERNAL_OWNER_UNNAMED_TAG_132, pMemDesc); if (rmStatus != NV_OK) { - if (bRetry) + if (pGpu->bAPageSizeAllocRetryEnabled) { NvU64 pageSize; pageSize = _sysmemGetNextSmallerPageSize(pGpu, &pAllocData->attr, &pAllocData->attr2); if (pageSize == 0) { - NV_CHECK_OK_OR_GOTO(rmStatus, LEVEL_ERROR, rmStatus, failed_destroy_memdesc); + NV_CHECK_OR_GOTO(LEVEL_ERROR, !bLastAttempt, failed_destroy_memdesc); + bLastAttempt = NV_TRUE; + NV_PRINTF(LEVEL_INFO, + "Sysmem alloc failed at 4K page size, retrying with reclamation enabled.\n"); + } + else + { + NV_PRINTF(LEVEL_INFO, + "Sysmem alloc failed, retrying with page size 0x%llx.\n", pageSize); } NV_PRINTF(LEVEL_INFO, "Sysmem alloc failed, retrying with page size 0x%llx.\n", pageSize); @@ -297,9 +310,10 @@ sysmemConstruct_IMPL else { // Got a valid allocation, set retry to false. - bRetry = NV_FALSE; + pGpu->bAPageSizeAllocRetryEnabled = NV_FALSE; } - } while (bRetry); + } while (pGpu->bAPageSizeAllocRetryEnabled); + pGpu->bAPageSizeAllocRetryEnabled = NV_FALSE; // ClientDB can set the pagesize for memdesc. // With GPU SMMU mapping, this needs to be set on the SMMU memdesc.