diff --git a/drivers/gpu/nvgpu/common/clk_arb/clk_arb.c b/drivers/gpu/nvgpu/common/clk_arb/clk_arb.c index ccc2fe307..b54badec7 100644 --- a/drivers/gpu/nvgpu/common/clk_arb/clk_arb.c +++ b/drivers/gpu/nvgpu/common/clk_arb/clk_arb.c @@ -120,7 +120,7 @@ int nvgpu_clk_arb_update_vf_table(struct nvgpu_clk_arb *arb) struct clk_set_info *p0_info; - table = NV_ACCESS_ONCE(arb->current_vf_table); + table = NV_READ_ONCE(arb->current_vf_table); /* make flag visible when all data has resolved in the tables */ nvgpu_smp_rmb(); table = (table == &arb->vf_table_pool[0]) ? &arb->vf_table_pool[1] : @@ -279,7 +279,7 @@ u32 nvgpu_clk_arb_notify(struct nvgpu_clk_dev *dev, l_notification = &arb->notification_queue. clk_q_notifications[((u64)index + 1ULL) % size]; - alarm_detected = NV_ACCESS_ONCE( + alarm_detected = NV_READ_ONCE( l_notification->clk_notification); if ((enabled_mask & alarm_detected) == 0U) { @@ -289,7 +289,7 @@ u32 nvgpu_clk_arb_notify(struct nvgpu_clk_dev *dev, queue_index++; dev->queue.clk_q_notifications[ queue_index % dev->queue.size].timestamp = - NV_ACCESS_ONCE(l_notification->timestamp); + NV_READ_ONCE(l_notification->timestamp); dev->queue.clk_q_notifications[queue_index % dev->queue.size].clk_notification = @@ -628,7 +628,7 @@ void nvgpu_clk_arb_schedule_vf_table_update(struct gk20a *g) */ u32 nvgpu_clk_arb_get_current_pstate(struct gk20a *g) { - return NV_ACCESS_ONCE(g->clk_arb->actual->pstate); + return NV_READ_ONCE(g->clk_arb->actual->pstate); } void nvgpu_clk_arb_pstate_change_lock(struct gk20a *g, bool lock) diff --git a/drivers/gpu/nvgpu/common/clk_arb/clk_arb_gp10b.c b/drivers/gpu/nvgpu/common/clk_arb/clk_arb_gp10b.c index bede94c8c..44b9cf582 100644 --- a/drivers/gpu/nvgpu/common/clk_arb/clk_arb_gp10b.c +++ b/drivers/gpu/nvgpu/common/clk_arb/clk_arb_gp10b.c @@ -344,7 +344,7 @@ void gp10b_clk_arb_run_arbiter_cb(struct nvgpu_clk_arb *arb) goto exit_arb; } - actual = ((NV_ACCESS_ONCE(arb->actual)) == &arb->actual_pool[0] ? + actual = ((NV_READ_ONCE(arb->actual)) == &arb->actual_pool[0] ? &arb->actual_pool[1] : &arb->actual_pool[0]); /* do not reorder this pointer */ diff --git a/drivers/gpu/nvgpu/common/clk_arb/clk_arb_gv100.c b/drivers/gpu/nvgpu/common/clk_arb/clk_arb_gv100.c index dc2b1417c..7b78232ed 100644 --- a/drivers/gpu/nvgpu/common/clk_arb/clk_arb_gv100.c +++ b/drivers/gpu/nvgpu/common/clk_arb/clk_arb_gv100.c @@ -429,7 +429,7 @@ void gv100_clk_arb_run_arbiter_cb(struct nvgpu_clk_arb *arb) goto exit_arb; } - actual = NV_ACCESS_ONCE(arb->actual) == &arb->actual_pool[0] ? + actual = NV_READ_ONCE(arb->actual) == &arb->actual_pool[0] ? &arb->actual_pool[1] : &arb->actual_pool[0]; /* do not reorder this pointer */ diff --git a/drivers/gpu/nvgpu/common/mm/allocators/lockless_allocator.c b/drivers/gpu/nvgpu/common/mm/allocators/lockless_allocator.c index ed4090e14..395c61a87 100644 --- a/drivers/gpu/nvgpu/common/mm/allocators/lockless_allocator.c +++ b/drivers/gpu/nvgpu/common/mm/allocators/lockless_allocator.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2019, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2016-2020, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -68,9 +68,9 @@ static u64 nvgpu_lockless_alloc(struct nvgpu_allocator *a, u64 len) return 0; } - head = NV_ACCESS_ONCE(pa->head); + head = NV_READ_ONCE(pa->head); while (head >= 0) { - new_head = NV_ACCESS_ONCE(pa->next[head]); + new_head = NV_READ_ONCE(pa->next[head]); ret = cmpxchg(&pa->head, head, new_head); if (ret == head) { addr = pa->base + U64(head) * pa->blk_size; @@ -79,7 +79,7 @@ static u64 nvgpu_lockless_alloc(struct nvgpu_allocator *a, u64 len) addr); break; } - head = NV_ACCESS_ONCE(pa->head); + head = NV_READ_ONCE(pa->head); } if (addr != 0ULL) { @@ -102,8 +102,8 @@ static void nvgpu_lockless_free(struct nvgpu_allocator *a, u64 addr) alloc_dbg(a, "Free node # %llu @ addr 0x%llx", cur_idx, addr); while (true) { - head = NV_ACCESS_ONCE(pa->head); - NV_ACCESS_ONCE(pa->next[cur_idx]) = head; + head = NV_READ_ONCE(pa->head); + NV_WRITE_ONCE(pa->next[cur_idx], head); nvgpu_assert(cur_idx <= U64(INT_MAX)); ret = cmpxchg(&pa->head, head, (int)cur_idx); if (ret == head) { diff --git a/drivers/gpu/nvgpu/common/pmu/clk/clk_vf_point.c b/drivers/gpu/nvgpu/common/pmu/clk/clk_vf_point.c index b946ed52f..a34492b65 100644 --- a/drivers/gpu/nvgpu/common/pmu/clk/clk_vf_point.c +++ b/drivers/gpu/nvgpu/common/pmu/clk/clk_vf_point.c @@ -399,7 +399,7 @@ int nvgpu_clk_arb_find_slave_points(struct nvgpu_clk_arb *arb, do { gpc2clk_target = vf_point->gpc_mhz; - table = NV_ACCESS_ONCE(arb->current_vf_table); + table = NV_READ_ONCE(arb->current_vf_table); /* pointer to table can be updated by callback */ nvgpu_smp_rmb(); @@ -436,7 +436,7 @@ int nvgpu_clk_arb_find_slave_points(struct nvgpu_clk_arb *arb, vf_point->gpc_mhz = gpc2clk_target; } } while ((table == NULL) || - (NV_ACCESS_ONCE(arb->current_vf_table) != table)); + (NV_READ_ONCE(arb->current_vf_table) != table)); return status; diff --git a/drivers/gpu/nvgpu/common/pmu/pg/pmu_pg.c b/drivers/gpu/nvgpu/common/pmu/pg/pmu_pg.c index 815edcb5b..8a9e3c44d 100644 --- a/drivers/gpu/nvgpu/common/pmu/pg/pmu_pg.c +++ b/drivers/gpu/nvgpu/common/pmu/pg/pmu_pg.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2019, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2016-2020, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -165,7 +165,8 @@ static void pmu_handle_pg_elpg_msg(struct gk20a *g, struct pmu_msg *msg, pmu->pg->initialized = true; nvgpu_pmu_fw_state_change(g, pmu, PMU_FW_STATE_STARTED, true); - WRITE_ONCE(pmu->pg->mscg_stat, PMU_MSCG_DISABLED); + NV_WRITE_ONCE(pmu->pg->mscg_stat, + PMU_MSCG_DISABLED); /* make status visible */ nvgpu_smp_mb(); } else { diff --git a/drivers/gpu/nvgpu/include/nvgpu/barrier.h b/drivers/gpu/nvgpu/include/nvgpu/barrier.h index 91ef99a20..bd0d820e6 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/barrier.h +++ b/drivers/gpu/nvgpu/include/nvgpu/barrier.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -44,7 +44,8 @@ #define nvgpu_smp_rmb() nvgpu_smp_rmb_impl() #define nvgpu_smp_wmb() nvgpu_smp_wmb_impl() -#define NV_ACCESS_ONCE(x) NV_ACCESS((x)) +#define NV_READ_ONCE(x) NV_READ_ONCE_IMPL((x)) +#define NV_WRITE_ONCE(x, y) NV_WRITE_ONCE_IMPL((x), (y)) /* * Sometimes we want to prevent speculation. diff --git a/drivers/gpu/nvgpu/include/nvgpu/linux/barrier.h b/drivers/gpu/nvgpu/include/nvgpu/linux/barrier.h index 0d6c88c34..c10889fb6 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/linux/barrier.h +++ b/drivers/gpu/nvgpu/include/nvgpu/linux/barrier.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, @@ -27,7 +27,8 @@ #define nvgpu_smp_rmb_impl() smp_rmb() #define nvgpu_smp_wmb_impl() smp_wmb() -#define NV_ACCESS(x) ACCESS_ONCE(x) +#define NV_READ_ONCE_IMPL(x) READ_ONCE(x) +#define NV_WRITE_ONCE_IMPL(x, y) WRITE_ONCE(x, y) #define nvgpu_speculation_barrier_impl() speculation_barrier() diff --git a/drivers/gpu/nvgpu/include/nvgpu/posix/barrier.h b/drivers/gpu/nvgpu/include/nvgpu/posix/barrier.h index 20ef10db3..3f5c8c658 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/posix/barrier.h +++ b/drivers/gpu/nvgpu/include/nvgpu/posix/barrier.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -23,7 +23,9 @@ #ifndef NVGPU_POSIX_BARRIER_H #define NVGPU_POSIX_BARRIER_H -#define ACCESS_ONCE(x) (*(volatile __typeof__(x) *)&x) +#include + +#define READ_ONCE(x) (*(volatile __typeof__(x) *)&x) /* * TODO: implement all these! @@ -36,6 +38,7 @@ #define nvgpu_smp_rmb_impl() #define nvgpu_smp_wmb_impl() -#define NV_ACCESS(x) ACCESS_ONCE(x) +#define NV_READ_ONCE_IMPL(x) READ_ONCE(x) +#define NV_WRITE_ONCE_IMPL(x, y) WRITE_ONCE(x, y) #endif /* NVGPU_POSIX_BARRIER_H */ diff --git a/drivers/gpu/nvgpu/os/linux/ioctl_clk_arb.c b/drivers/gpu/nvgpu/os/linux/ioctl_clk_arb.c index f0c63e7d9..62a01aeef 100644 --- a/drivers/gpu/nvgpu/os/linux/ioctl_clk_arb.c +++ b/drivers/gpu/nvgpu/os/linux/ioctl_clk_arb.c @@ -516,7 +516,7 @@ static int nvgpu_clk_arb_stats_show(struct seq_file *s, void *unused) u64 num; s64 tmp, avg, std, max, min; - debug = NV_ACCESS_ONCE(arb->debug); + debug = READ_ONCE(arb->debug); /* Make copy of structure and ensure no reordering */ nvgpu_smp_rmb(); if (!debug) diff --git a/drivers/gpu/nvgpu/os/linux/sysfs.c b/drivers/gpu/nvgpu/os/linux/sysfs.c index 1f93b974d..07681dfc0 100644 --- a/drivers/gpu/nvgpu/os/linux/sysfs.c +++ b/drivers/gpu/nvgpu/os/linux/sysfs.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2019, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2011-2020, NVIDIA CORPORATION. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, @@ -553,7 +553,7 @@ static ssize_t mscg_enable_store(struct device *dev, g->mscg_enabled = true; if (nvgpu_pmu_is_lpwr_feature_supported(g, PMU_PG_LPWR_FEATURE_MSCG)) { - if (!ACCESS_ONCE(pmu->pg->mscg_stat)) { + if (!READ_ONCE(pmu->pg->mscg_stat)) { WRITE_ONCE(pmu->pg->mscg_stat, PMU_MSCG_ENABLED); /* make status visible */