diff --git a/arch/nvgpu-common.yaml b/arch/nvgpu-common.yaml index 0db83af2a..6a55318b4 100644 --- a/arch/nvgpu-common.yaml +++ b/arch/nvgpu-common.yaml @@ -664,6 +664,11 @@ mm: include/nvgpu/mm.h, include/nvgpu/gops/mm.h ] deps: [ ] + ipa_pa_cache: + safe: yes + sources: [ common/mm/ipa_pa_cache.c, + include/nvgpu/ipa_pa_cache.h ] + deps: [ ] nvgpu_mem: safe: yes sources: [ common/mm/nvgpu_mem.c, diff --git a/drivers/gpu/nvgpu/Makefile b/drivers/gpu/nvgpu/Makefile index bb606cf4f..ad5f44540 100644 --- a/drivers/gpu/nvgpu/Makefile +++ b/drivers/gpu/nvgpu/Makefile @@ -571,6 +571,7 @@ nvgpu-y += \ common/mm/vm_area.o \ common/mm/nvgpu_mem.o \ common/mm/nvgpu_sgt.o \ + common/mm/ipa_pa_cache.o \ common/mm/mm.o \ common/mm/dma.o \ common/vbios/bios.o \ diff --git a/drivers/gpu/nvgpu/Makefile.sources b/drivers/gpu/nvgpu/Makefile.sources index 8e9034c7d..d389e5940 100644 --- a/drivers/gpu/nvgpu/Makefile.sources +++ b/drivers/gpu/nvgpu/Makefile.sources @@ -109,6 +109,7 @@ srcs += common/device.c \ common/mm/vm_area.c \ common/mm/nvgpu_mem.c \ common/mm/nvgpu_sgt.c \ + common/mm/ipa_pa_cache.c \ common/mm/mm.c \ common/mm/dma.c \ common/therm/therm.c \ diff --git a/drivers/gpu/nvgpu/common/init/nvgpu_init.c b/drivers/gpu/nvgpu/common/init/nvgpu_init.c index 5ecaabbe7..2718b6d4f 100644 --- a/drivers/gpu/nvgpu/common/init/nvgpu_init.c +++ b/drivers/gpu/nvgpu/common/init/nvgpu_init.c @@ -613,6 +613,13 @@ static int nvgpu_init_slcg_acb_load_gating_prod(struct gk20a *g) return 0; } +#ifdef CONFIG_TEGRA_HV_MANAGER +static int nvgpu_ipa_pa_rwsem_init(struct gk20a *g) +{ + nvgpu_rwsem_init(&(g->ipa_pa_cache.ipa_pa_rw_lock)); + return 0; +} +#endif static int nvgpu_init_interrupt_setup(struct gk20a *g) { /** @@ -666,6 +673,9 @@ static int nvgpu_early_init(struct gk20a *g) * prior to enabling interrupts for corresponding units. */ NVGPU_INIT_TABLE_ENTRY(g->ops.ecc.ecc_init_support, NO_FLAG), +#ifdef CONFIG_TEGRA_HV_MANAGER + NVGPU_INIT_TABLE_ENTRY(&nvgpu_ipa_pa_rwsem_init, NO_FLAG), +#endif NVGPU_INIT_TABLE_ENTRY(&nvgpu_device_init, NO_FLAG), #ifdef CONFIG_NVGPU_DGPU NVGPU_INIT_TABLE_ENTRY(g->ops.bios.bios_sw_init, NO_FLAG), diff --git a/drivers/gpu/nvgpu/common/mm/ipa_pa_cache.c b/drivers/gpu/nvgpu/common/mm/ipa_pa_cache.c new file mode 100644 index 000000000..f81c28325 --- /dev/null +++ b/drivers/gpu/nvgpu/common/mm/ipa_pa_cache.c @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ +#ifdef CONFIG_TEGRA_HV_MANAGER +#include +#include +#include +#include + +static u64 nvgpu_ipa_to_pa_cache_lookup(struct gk20a *g, u64 ipa, + u64 *pa_len) +{ + struct nvgpu_ipa_desc *desc; + struct nvgpu_ipa_pa_cache *ipa_cache; + u32 i = 0U; + u64 pa = 0UL; + + ipa_cache = &(g->ipa_pa_cache); + desc = &(ipa_cache->ipa[0]); + + for (i = 0; i < ipa_cache->num_ipa_desc; ++i, ++desc) { + if (ipa >= desc->ipa_base && + (ipa < (desc->ipa_base + desc->ipa_size))) { + pa = ipa - desc->ipa_base + desc->pa_base; + if (pa_len != NULL) { + *pa_len = desc->ipa_size -(ipa - desc->ipa_base); + } + + return pa; + } + } + + return 0U; +} + +u64 nvgpu_ipa_to_pa_cache_lookup_locked(struct gk20a *g, u64 ipa, + u64 *pa_len) +{ + struct nvgpu_ipa_pa_cache *ipa_cache; + u64 pa = 0UL; + + ipa_cache = &(g->ipa_pa_cache); + nvgpu_rwsem_down_read(&(ipa_cache->ipa_pa_rw_lock)); + pa = nvgpu_ipa_to_pa_cache_lookup(g, ipa, pa_len); + nvgpu_rwsem_up_read(&(ipa_cache->ipa_pa_rw_lock)); + return pa; +} + +void nvgpu_ipa_to_pa_add_to_cache(struct gk20a *g, u64 ipa, u64 pa, + struct hyp_ipa_pa_info *info) +{ + struct nvgpu_ipa_pa_cache *ipa_cache; + struct nvgpu_ipa_desc *desc = NULL; + u64 pa_cached = 0U; + + ipa_cache = &(g->ipa_pa_cache); + nvgpu_rwsem_down_write(&(ipa_cache->ipa_pa_rw_lock)); + pa_cached = nvgpu_ipa_to_pa_cache_lookup(g, ipa, NULL); + if (pa_cached != 0UL) { + /* Check any other context insert the translation + * already and return. + */ + nvgpu_assert(pa_cached == pa); + nvgpu_rwsem_up_write(&(ipa_cache->ipa_pa_rw_lock)); + return; + } + + if (ipa_cache->num_ipa_desc >= MAX_IPA_PA_CACHE) { + desc = &ipa_cache->ipa[nvgpu_current_time_ns() % + MAX_IPA_PA_CACHE]; + } else { + desc = &ipa_cache->ipa[ipa_cache->num_ipa_desc++]; + } + + desc->ipa_base = ipa - info->offset; + desc->ipa_size = info->size; + desc->pa_base = info->base; + nvgpu_rwsem_up_write(&(ipa_cache->ipa_pa_rw_lock)); + +} +#endif diff --git a/drivers/gpu/nvgpu/include/nvgpu/gk20a.h b/drivers/gpu/nvgpu/include/nvgpu/gk20a.h index 6902cd855..b589e948b 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/gk20a.h +++ b/drivers/gpu/nvgpu/include/nvgpu/gk20a.h @@ -146,6 +146,9 @@ enum nvgpu_profiler_pm_reservation_scope; #include #include #include +#ifdef CONFIG_TEGRA_HV_MANAGER +#include +#endif #include #include @@ -830,6 +833,10 @@ struct gk20a { /** Pointer to struct storing CIC-RM's data */ struct nvgpu_cic_rm *cic_rm; +#ifdef CONFIG_TEGRA_HV_MANAGER + /** Cache to store IPA to PA translations. */ + struct nvgpu_ipa_pa_cache ipa_pa_cache; +#endif }; /** diff --git a/drivers/gpu/nvgpu/include/nvgpu/ipa_pa_cache.h b/drivers/gpu/nvgpu/include/nvgpu/ipa_pa_cache.h new file mode 100644 index 000000000..f7d37b0b7 --- /dev/null +++ b/drivers/gpu/nvgpu/include/nvgpu/ipa_pa_cache.h @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef NVGPU_IPAPACACHE_H +#define NVGPU_IPAPACACHE_H + +#ifdef CONFIG_TEGRA_HV_MANAGER +#include + +struct hyp_ipa_pa_info; +struct gk20a; + +#define MAX_IPA_PA_CACHE 256U + +struct nvgpu_ipa_desc { + u64 ipa_base; + u64 ipa_size; + u64 pa_base; +}; + +struct nvgpu_ipa_pa_cache { + struct nvgpu_rwsem ipa_pa_rw_lock; + struct nvgpu_ipa_desc ipa[MAX_IPA_PA_CACHE]; + u32 num_ipa_desc; +}; + +u64 nvgpu_ipa_to_pa_cache_lookup_locked(struct gk20a *g, u64 ipa, + u64 *pa_len); + +void nvgpu_ipa_to_pa_add_to_cache(struct gk20a *g, u64 ipa, + u64 pa, struct hyp_ipa_pa_info *info); +#endif +#endif /* NVGPU_IPAPACACHE_H */ diff --git a/drivers/gpu/nvgpu/os/linux/soc.c b/drivers/gpu/nvgpu/os/linux/soc.c index af1b9393f..2e0dd6efd 100644 --- a/drivers/gpu/nvgpu/os/linux/soc.c +++ b/drivers/gpu/nvgpu/os/linux/soc.c @@ -18,6 +18,7 @@ #include #ifdef CONFIG_TEGRA_HV_MANAGER #include +#include #endif #include @@ -73,6 +74,11 @@ static u64 nvgpu_tegra_hv_ipa_pa(struct gk20a *g, u64 ipa, u64 *pa_len) int err; u64 pa = 0ULL; + pa = nvgpu_ipa_to_pa_cache_lookup_locked(g, ipa, pa_len); + if (pa != 0UL) { + return pa; + } + err = hyp_read_ipa_pa_info(&info, platform->vmid, ipa); if (err < 0) { nvgpu_err(g, "ipa=%llx translation failed vmid=%u err=%d", @@ -92,6 +98,11 @@ static u64 nvgpu_tegra_hv_ipa_pa(struct gk20a *g, u64 ipa, u64 *pa_len) ipa, platform->vmid, pa, info.base, info.offset, info.size); } + + if (pa != 0U) { + nvgpu_ipa_to_pa_add_to_cache(g, ipa, pa, &info); + } + return pa; } #endif