nv-tegra-mirror/linux-nvgpu
1
0
Fork 0
mirror of git://nv-tegra.nvidia.com/linux-nvgpu.git synced 2025-12-24 02:22:34 +03:00
Code Issues Packages Projects Releases Wiki Activity

Open source GPL/LGPL release

Browse Source
This commit is contained in:
svcmobrel-release
2022-07-21 16:03:29 -07:00
commit f338182221
2260 changed files with 576813 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

110
drivers/gpu/nvgpu/common/utils/enabled.c Normal file
View File

@@ -0,0 +1,110 @@
/*
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
#include <nvgpu/enabled.h>
#include <nvgpu/bitops.h>
#include <nvgpu/log.h>
#include <nvgpu/gk20a.h>
#include <nvgpu/static_analysis.h>
#include <nvgpu/utils.h>
/**
* Array of flag names
*/
#define DEFINE_FLAG(flag, desc) [flag] = nvgpu_stringify(flag)
static const char *enabled_flag_names[NVGPU_MAX_ENABLED_BITS + 1U] = {
ENABLED_FLAGS
};
#undef DEFINE_FLAG
/**
* Array of flag descriptions
*/
#define DEFINE_FLAG(flag, desc) [flag] = desc
static const char *enabled_flag_desc[NVGPU_MAX_ENABLED_BITS + 1U] = {
ENABLED_FLAGS
};
#undef DEFINE_FLAG
void nvgpu_print_enabled_flags(struct gk20a *g)
{
u32 i;
nvgpu_log(g, gpu_dbg_info, "NVGPU support flags status");
nvgpu_log(g, gpu_dbg_info, "%-55.55s %-6.6s %s",
"Flag", "Status", "Description");
nvgpu_log(g, gpu_dbg_info, "%-55.55s %-6.6s %s",
"----", "------", "-----------");
for (i = 0U; i < U32(NVGPU_MAX_ENABLED_BITS); i++) {
nvgpu_log(g, gpu_dbg_info, "%-55.55s %-6.6s %s",
enabled_flag_names[i],
nvgpu_is_enabled(g, i) ? "true" : "false",
enabled_flag_desc[i]);
}
}
int nvgpu_init_enabled_flags(struct gk20a *g)
{
/*
* Zero all flags initially. Flags that should be set to non-zero states
* can be done so during driver init.
*/
g->enabled_flags = nvgpu_kzalloc(g,
BITS_TO_LONGS(U32(NVGPU_MAX_ENABLED_BITS)) *
sizeof(unsigned long));
if (g->enabled_flags == NULL) {
return -ENOMEM;
}
return 0;
}
/*
* Call this on driver shutdown!
*/
void nvgpu_free_enabled_flags(struct gk20a *g)
{
nvgpu_kfree(g, g->enabled_flags);
}
bool nvgpu_is_enabled(struct gk20a *g, u32 flag)
{
if (flag < NVGPU_MAX_ENABLED_BITS) {
return nvgpu_test_bit(flag, g->enabled_flags);
} else {
return 0;
}
}
void nvgpu_set_enabled(struct gk20a *g, u32 flag, bool state)
{
if (flag >= NVGPU_MAX_ENABLED_BITS) {
return;
}
if (state) {
nvgpu_set_bit(flag, g->enabled_flags);
} else {
nvgpu_clear_bit(flag, g->enabled_flags);
}
}

125
drivers/gpu/nvgpu/common/utils/errata.c Normal file
View File

@@ -0,0 +1,125 @@
/*
* Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
#include <nvgpu/errata.h>
#include <nvgpu/bitops.h>
#include <nvgpu/log.h>
#include <nvgpu/gk20a.h>
#include <nvgpu/static_analysis.h>
#include <nvgpu/utils.h>
/**
* Array of flag names
*/
#define DEFINE_ERRATA(flag, chip, desc) [flag] = nvgpu_stringify(flag)
static const char *errata_flag_names[NVGPU_MAX_ERRATA_BITS + 1U] = {
ERRATA_FLAGS_NEXT
ERRATA_FLAGS
};
#undef DEFINE_ERRATA
/**
* Array of flag descriptions
*/
#define DEFINE_ERRATA(flag, chip, desc) [flag] = desc
static const char *errata_flag_desc[NVGPU_MAX_ERRATA_BITS + 1U] = {
ERRATA_FLAGS_NEXT
ERRATA_FLAGS
};
#undef DEFINE_ERRATA
/**
* Array of chips where errata was first discovered
*/
#define DEFINE_ERRATA(flag, chip, desc) [flag] = chip
static const char *errata_flag_chip[NVGPU_MAX_ERRATA_BITS + 1U] = {
ERRATA_FLAGS_NEXT
ERRATA_FLAGS
};
#undef DEFINE_ERRATA
void nvgpu_print_errata_flags(struct gk20a *g)
{
u32 i;
nvgpu_log(g, gpu_dbg_info, "NVGPU Erratas present in chip");
nvgpu_log(g, gpu_dbg_info, "%-55.55s %-5.5s %s",
"Flag", "Chip", "Description");
nvgpu_log(g, gpu_dbg_info, "%-55.55s %-5.5s %s",
"----", "-----", "-----------");
for (i = 0U; i < U32(NVGPU_MAX_ERRATA_BITS); i++) {
/* Only print erratas present in chip */
if (nvgpu_is_errata_present(g, i)) {
nvgpu_log(g, gpu_dbg_info, "%-55.55s %-5.5s %s",
errata_flag_names[i],
errata_flag_chip[i],
errata_flag_desc[i]);
}
}
}
int nvgpu_init_errata_flags(struct gk20a *g)
{
/*
* Zero all flags initially. Flags that should be set to non-zero states
* can be done so during hal init.
*/
g->errata_flags = nvgpu_kzalloc(g,
BITS_TO_LONGS(U32(NVGPU_MAX_ERRATA_BITS)) *
sizeof(unsigned long));
if (g->errata_flags == NULL) {
return -ENOMEM;
}
return 0;
}
/*
* Call this on driver shutdown!
*/
void nvgpu_free_errata_flags(struct gk20a *g)
{
nvgpu_kfree(g, g->errata_flags);
}
bool nvgpu_is_errata_present(struct gk20a *g, u32 flag)
{
if (flag < NVGPU_MAX_ERRATA_BITS) {
return nvgpu_test_bit(flag, g->errata_flags);
} else {
return 0;
}
}
void nvgpu_set_errata(struct gk20a *g, u32 flag, bool state)
{
if (flag >= NVGPU_MAX_ERRATA_BITS) {
return;
}
if (state) {
nvgpu_set_bit(flag, g->errata_flags);
} else {
nvgpu_clear_bit(flag, g->errata_flags);
}
}

493
drivers/gpu/nvgpu/common/utils/rbtree.c Normal file
View File

@@ -0,0 +1,493 @@
/*
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
#include <nvgpu/rbtree.h>
#include <nvgpu/bug.h>
/*
* rotate node x to left
*/
static void rotate_left(struct nvgpu_rbtree_node **root,
struct nvgpu_rbtree_node *x)
{
struct nvgpu_rbtree_node *y = x->right;
/* establish x->right link */
x->right = y->left;
if (y->left != NULL) {
y->left->parent = x;
}
/* establish y->parent link */
y->parent = x->parent;
if (x->parent != NULL) {
if (x == x->parent->left) {
x->parent->left = y;
} else {
x->parent->right = y;
}
} else {
*root = y;
}
/* link x and y */
y->left = x;
x->parent = y;
}
/*
* rotate node x to right
*/
static void rotate_right(struct nvgpu_rbtree_node **root,
struct nvgpu_rbtree_node *x)
{
struct nvgpu_rbtree_node *y = x->left;
/* establish x->left link */
x->left = y->right;
if (y->right != NULL) {
y->right->parent = x;
}
/* establish y->parent link */
y->parent = x->parent;
if (x->parent != NULL) {
if (x == x->parent->right) {
x->parent->right = y;
} else {
x->parent->left = y;
}
} else {
*root = y;
}
/* link x and y */
y->right = x;
x->parent = y;
}
/*
* maintain red-black tree balance after inserting node x
*/
static void insert_fixup(struct nvgpu_rbtree_node **root,
struct nvgpu_rbtree_node *x)
{
/* check red-black properties */
while ((x != *root) && x->parent->is_red) {
/* we have a violation */
if (x->parent == x->parent->parent->left) {
struct nvgpu_rbtree_node *y = x->parent->parent->right;
if ((y != NULL) && (y->is_red)) {
/* uncle is RED */
x->parent->is_red = false;
y->is_red = false;
x->parent->parent->is_red = true;
x = x->parent->parent;
} else {
/* uncle is BLACK */
if (x == x->parent->right) {
/* make x a left child */
x = x->parent;
rotate_left(root, x);
}
/* recolor and rotate */
x->parent->is_red = false;
x->parent->parent->is_red = true;
rotate_right(root, x->parent->parent);
}
} else {
/* mirror image of above code */
struct nvgpu_rbtree_node *y = x->parent->parent->left;
if ((y != NULL) && (y->is_red)) {
/* uncle is RED */
x->parent->is_red = false;
y->is_red = false;
x->parent->parent->is_red = true;
x = x->parent->parent;
} else {
/* uncle is BLACK */
if (x == x->parent->left) {
x = x->parent;
rotate_right(root, x);
}
x->parent->is_red = false;
x->parent->parent->is_red = true;
rotate_left(root, x->parent->parent);
}
}
}
(*root)->is_red = false;
}
void nvgpu_rbtree_insert(struct nvgpu_rbtree_node *new_node,
struct nvgpu_rbtree_node **root)
{
struct nvgpu_rbtree_node *curr;
struct nvgpu_rbtree_node *parent;
/* find future parent */
curr = *root;
parent = NULL;
while (curr != NULL) {
parent = curr;
if (new_node->key_start < curr->key_start) {
curr = curr->left;
} else if (new_node->key_start > curr->key_start) {
curr = curr->right;
} else {
return; /* duplicate entry */
}
}
/* the caller allocated the node already, just fix the links */
new_node->parent = parent;
new_node->left = NULL;
new_node->right = NULL;
new_node->is_red = true;
/* insert node in tree */
if (parent != NULL) {
if (new_node->key_start < parent->key_start) {
parent->left = new_node;
} else {
parent->right = new_node;
}
} else {
*root = new_node;
}
insert_fixup(root, new_node);
}
/*
* helper function for delete_fixup_*_child to test if node has no red
* children
*/
static bool has_no_red_children(struct nvgpu_rbtree_node *w)
{
return (w == NULL) ||
(((w->left == NULL) || (!w->left->is_red)) &&
((w->right == NULL) || (!w->right->is_red)));
}
/* delete_fixup handling if x is the left child */
static void delete_fixup_left_child(struct nvgpu_rbtree_node **root,
struct nvgpu_rbtree_node *parent_of_x,
struct nvgpu_rbtree_node **x)
{
struct nvgpu_rbtree_node *w = parent_of_x->right;
if ((w != NULL) && (w->is_red)) {
w->is_red = false;
parent_of_x->is_red = true;
rotate_left(root, parent_of_x);
w = parent_of_x->right;
}
if (has_no_red_children(w)) {
if (w != NULL) {
w->is_red = true;
}
*x = parent_of_x;
} else {
if ((w->right == NULL) || (!w->right->is_red)) {
w->left->is_red = false;
w->is_red = true;
rotate_right(root, w);
w = parent_of_x->right;
}
w->is_red = parent_of_x->is_red;
parent_of_x->is_red = false;
w->right->is_red = false;
rotate_left(root, parent_of_x);
*x = *root;
}
}
/* delete_fixup handling if x is the right child */
static void delete_fixup_right_child(struct nvgpu_rbtree_node **root,
struct nvgpu_rbtree_node *parent_of_x,
struct nvgpu_rbtree_node **x)
{
struct nvgpu_rbtree_node *w = parent_of_x->left;
if ((w != NULL) && (w->is_red)) {
w->is_red = false;
parent_of_x->is_red = true;
rotate_right(root, parent_of_x);
w = parent_of_x->left;
}
if (has_no_red_children(w)) {
if (w != NULL) {
w->is_red = true;
}
*x = parent_of_x;
} else {
if ((w->left == NULL) || (!w->left->is_red)) {
w->right->is_red = false;
w->is_red = true;
rotate_left(root, w);
w = parent_of_x->left;
}
w->is_red = parent_of_x->is_red;
parent_of_x->is_red = false;
w->left->is_red = false;
rotate_right(root, parent_of_x);
*x = *root;
}
}
/*
* maintain red-black tree balance after deleting node x
*/
static void delete_fixup(struct nvgpu_rbtree_node **root,
struct nvgpu_rbtree_node *parent_of_x,
struct nvgpu_rbtree_node *x)
{
while ((x != *root) && ((x == NULL) || (!x->is_red))) {
/*
* NULL nodes are sentinel nodes. If we delete a sentinel
* node (x==NULL) it must have a parent node (or be the root).
* Hence, parent_of_x == NULL with
* x==NULL is never possible (tree invariant)
*/
if (parent_of_x == NULL) {
nvgpu_assert(x != NULL);
parent_of_x = x->parent;
continue;
}
if (x == parent_of_x->left) {
delete_fixup_left_child(root, parent_of_x, &x);
} else {
delete_fixup_right_child(root, parent_of_x, &x);
}
parent_of_x = x->parent;
}
if (x != NULL) {
x->is_red = false;
}
}
static void swap_in_new_child(struct nvgpu_rbtree_node *old,
struct nvgpu_rbtree_node *new,
struct nvgpu_rbtree_node **root)
{
if (old->parent != NULL) {
if (old == old->parent->left) {
old->parent->left = new;
} else {
old->parent->right = new;
}
} else {
*root = new;
}
}
static void adopt_children(struct nvgpu_rbtree_node *old,
struct nvgpu_rbtree_node *new)
{
new->left = old->left;
if (old->left != NULL) {
old->left->parent = new;
}
new->right = old->right;
if (old->right != NULL) {
old->right->parent = new;
}
}
void nvgpu_rbtree_unlink(struct nvgpu_rbtree_node *node,
struct nvgpu_rbtree_node **root)
{
struct nvgpu_rbtree_node *x;
struct nvgpu_rbtree_node *y;
struct nvgpu_rbtree_node *z;
struct nvgpu_rbtree_node *parent_of_x;
bool y_was_black;
z = node;
/* unlink */
if ((z->left == NULL) || (z->right == NULL)) {
/* y has a SENTINEL node as a child */
y = z;
} else {
/* find tree successor */
y = z->right;
while (y->left != NULL) {
y = y->left;
}
}
/* x is y's only child */
if (y->left != NULL) {
x = y->left;
} else {
x = y->right;
}
/* remove y from the parent chain */
parent_of_x = y->parent;
if (x != NULL) {
x->parent = parent_of_x;
}
/* update the parent's links */
swap_in_new_child(y, x, root);
y_was_black = !y->is_red;
if (y != z) {
/* we need to replace z with y so
* the memory for z can be freed
*/
y->parent = z->parent;
swap_in_new_child(z, y, root);
y->is_red = z->is_red;
adopt_children(z, y);
if (parent_of_x == z) {
parent_of_x = y;
}
}
if (y_was_black) {
delete_fixup(root, parent_of_x, x);
}
}
void nvgpu_rbtree_search(u64 key_start, struct nvgpu_rbtree_node **node,
struct nvgpu_rbtree_node *root)
{
struct nvgpu_rbtree_node *curr = root;
while (curr != NULL) {
if (key_start < curr->key_start) {
curr = curr->left;
} else if (key_start > curr->key_start) {
curr = curr->right;
} else {
*node = curr;
return;
}
}
*node = NULL;
}
void nvgpu_rbtree_range_search(u64 key,
struct nvgpu_rbtree_node **node,
struct nvgpu_rbtree_node *root)
{
struct nvgpu_rbtree_node *curr = root;
while (curr != NULL) {
if ((key >= curr->key_start) &&
(key < curr->key_end)) {
*node = curr;
return;
} else if (key < curr->key_start) {
curr = curr->left;
} else {
curr = curr->right;
}
}
*node = NULL;
}
void nvgpu_rbtree_less_than_search(u64 key_start,
struct nvgpu_rbtree_node **node,
struct nvgpu_rbtree_node *root)
{
struct nvgpu_rbtree_node *curr = root;
while (curr != NULL) {
if (key_start <= curr->key_start) {
curr = curr->left;
} else {
*node = curr;
curr = curr->right;
}
}
}
void nvgpu_rbtree_enum_start(u64 key_start, struct nvgpu_rbtree_node **node,
struct nvgpu_rbtree_node *root)
{
*node = NULL;
if (root != NULL) {
struct nvgpu_rbtree_node *curr = root;
while (curr != NULL) {
if (key_start < curr->key_start) {
*node = curr;
curr = curr->left;
} else if (key_start > curr->key_start) {
curr = curr->right;
} else {
*node = curr;
break;
}
}
}
}
void nvgpu_rbtree_enum_next(struct nvgpu_rbtree_node **node,
struct nvgpu_rbtree_node *root)
{
struct nvgpu_rbtree_node *curr = NULL;
if ((root != NULL) && (*node != NULL)) {
/* if we don't have a right subtree return the parent */
curr = *node;
/* pick the leftmost node of the right subtree ? */
if (curr->right != NULL) {
curr = curr->right;
while (curr->left != NULL) {
curr = curr->left;
}
} else {
/* go up until we find the right inorder node */
for (curr = curr->parent;
curr != NULL;
curr = curr->parent) {
if (curr->key_start > (*node)->key_start) {
break;
}
}
}
}
*node = curr;
}

94
drivers/gpu/nvgpu/common/utils/string.c Normal file
View File

@@ -0,0 +1,94 @@
/*
* Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
#include <nvgpu/string.h>
#include <nvgpu/log.h>
#include <nvgpu/static_analysis.h>
void
nvgpu_memcpy(u8 *destb, const u8 *srcb, size_t n)
{
(void) memcpy(destb, srcb, n);
}
int
nvgpu_memcmp(const u8 *b1, const u8 *b2, size_t n)
{
return memcmp(b1, b2, n);
}
int nvgpu_strnadd_u32(char *dst, const u32 value, size_t size, u32 radix)
{
int n;
u32 v;
char *p;
u32 digit;
if ((radix < 2U) || (radix > 16U)) {
return 0;
}
if (size > ((u64)(INT_MAX))) {
return 0;
}
/* how many digits do we need ? */
n = 0;
v = value;
do {
n = nvgpu_safe_add_s32(n, 1);
v = v / radix;
} while (v > 0U);
/* bail out if there is not room for '\0' */
if (n >= (s32)size) {
return 0;
}
/* number of digits (not including '\0') */
p = dst + n;
/* terminate with '\0' */
*p = '\0';
p--;
v = value;
do {
digit = v % radix;
*p = "0123456789abcdef"[digit];
v = v / radix;
p--;
}
while (v > 0U);
return n;
}
bool nvgpu_mem_is_word_aligned(struct gk20a *g, u8 *addr)
{
if (((unsigned long)addr % 4UL) != 0UL) {
nvgpu_log_info(g, "addr not 4-byte aligned");
return false;
}
return true;
}

298
drivers/gpu/nvgpu/common/utils/worker.c Normal file
View File

@@ -0,0 +1,298 @@
/*
* Copyright (c) 2019-2021, NVIDIA CORPORATION. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
#include <nvgpu/log.h>
#include <nvgpu/bug.h>
#include <nvgpu/worker.h>
static void nvgpu_worker_pre_process(struct nvgpu_worker *worker)
{
if (worker->ops->pre_process != NULL) {
worker->ops->pre_process(worker);
}
}
static bool nvgpu_worker_wakeup_condition(struct nvgpu_worker *worker)
{
bool ret = false;
if (worker->ops->wakeup_condition != NULL) {
ret = worker->ops->wakeup_condition(worker);
}
return ret;
}
static u32 nvgpu_worker_wakeup_timeout(
struct nvgpu_worker *worker)
{
u32 timeout = 0U;
if (worker->ops->wakeup_timeout != NULL) {
timeout = worker->ops->wakeup_timeout(
worker);
}
return timeout;
}
static bool nvgpu_worker_wakeup_early_exit(struct nvgpu_worker *worker)
{
bool ret = false;
if (worker->ops->wakeup_early_exit != NULL) {
ret = worker->ops->wakeup_early_exit(worker);
}
return ret;
}
static void nvgpu_worker_wakeup_process_item(struct nvgpu_worker *worker,
struct nvgpu_list_node *work_item)
{
nvgpu_assert(worker->ops->wakeup_process_item != NULL);
worker->ops->wakeup_process_item(work_item);
}
static void nvgpu_worker_wakeup_post_process(
struct nvgpu_worker *worker)
{
if (worker->ops->wakeup_post_process != NULL) {
worker->ops->wakeup_post_process(worker);
}
}
/**
* Tell the worker that potentially more work needs to be done.
*
* Increase the work counter to synchronize the worker with the new work. Wake
* up the worker. If the worker was already running, it will handle this work
* before going to sleep.
*/
static int nvgpu_worker_wakeup(struct nvgpu_worker *worker)
{
int put;
struct gk20a *g = worker->g;
nvgpu_log_fn(g, " ");
put = nvgpu_atomic_inc_return(&worker->put);
nvgpu_cond_signal_interruptible(&worker->wq);
return put;
}
static bool nvgpu_worker_pending(struct nvgpu_worker *worker, int get)
{
bool pending = nvgpu_atomic_read(&worker->put) != get;
/* We don't need barriers because they are implicit in locking */
return pending;
}
/**
* Process the queued works for the worker thread serially.
*
* Flush all the work items in the queue one by one. This may block timeout
* handling for a short while, as these are serialized.
*/
static void nvgpu_worker_process(struct nvgpu_worker *worker, int *get)
{
struct gk20a *g = worker->g;
while (nvgpu_worker_pending(worker, *get)) {
struct nvgpu_list_node *work_item = NULL;
nvgpu_spinlock_acquire(&worker->items_lock);
if (!nvgpu_list_empty(&worker->items)) {
work_item = worker->items.next;
nvgpu_list_del(work_item);
}
nvgpu_spinlock_release(&worker->items_lock);
if (work_item == NULL) {
/*
* Woke up for some other reason, but there are no
* other reasons than a work item added in the items
* list currently, so warn and ack the message.
*/
nvgpu_info(g, "Spurious worker event!");
++*get;
break;
}
nvgpu_worker_wakeup_process_item(worker, work_item);
++*get;
}
}
/*
* Process all work items found in the work queue.
*/
static int nvgpu_worker_poll_work(void *arg)
{
struct nvgpu_worker *worker = (struct nvgpu_worker *)arg;
int get = 0;
nvgpu_worker_pre_process(worker);
while (!nvgpu_thread_should_stop(&worker->poll_task)) {
int ret;
ret = NVGPU_COND_WAIT_INTERRUPTIBLE(
&worker->wq,
nvgpu_worker_pending(worker, get) ||
nvgpu_worker_wakeup_condition(worker),
nvgpu_worker_wakeup_timeout(worker));
if (nvgpu_worker_wakeup_early_exit(worker)) {
break;
}
if (ret == 0) {
nvgpu_worker_process(worker, &get);
}
nvgpu_worker_wakeup_post_process(worker);
}
return 0;
}
static int nvgpu_worker_start(struct nvgpu_worker *worker)
{
int err = 0;
if (nvgpu_thread_is_running(&worker->poll_task)) {
return err;
}
nvgpu_mutex_acquire(&worker->start_lock);
/*
* Mutexes have implicit barriers, so there is no risk of a thread
* having a stale copy of the poll_task variable as the call to
* thread_is_running is volatile
*/
if (nvgpu_thread_is_running(&worker->poll_task)) {
nvgpu_mutex_release(&worker->start_lock);
return err;
}
err = nvgpu_thread_create(&worker->poll_task, worker,
nvgpu_worker_poll_work, worker->thread_name);
nvgpu_mutex_release(&worker->start_lock);
return err;
}
bool nvgpu_worker_should_stop(struct nvgpu_worker *worker)
{
return nvgpu_thread_should_stop(&worker->poll_task);
}
int nvgpu_worker_enqueue(struct nvgpu_worker *worker,
struct nvgpu_list_node *work_item)
{
int err;
struct gk20a *g = worker->g;
/*
* Warn if worker thread cannot run
*/
err = nvgpu_worker_start(worker);
if (err != 0) {
nvgpu_do_assert_print(g, "nvgpu_worker %s cannot run!",
worker->thread_name);
return -1;
}
nvgpu_spinlock_acquire(&worker->items_lock);
if (!nvgpu_list_empty(work_item)) {
/*
* Already queued, so will get processed eventually.
* The worker is probably awake already.
*/
nvgpu_spinlock_release(&worker->items_lock);
return -1;
}
nvgpu_list_add_tail(work_item, &worker->items);
nvgpu_spinlock_release(&worker->items_lock);
(void) nvgpu_worker_wakeup(worker);
return 0;
}
void nvgpu_worker_init_name(struct nvgpu_worker *worker,
const char* worker_name, const char *gpu_name)
{
/*
* Maximum character size of worker thread name
* Note: 1 is subtracted to account for null character
*/
size_t worker_name_size = sizeof(worker->thread_name) - 1U;
/* Number of characters that can be used for thread name */
size_t num_free_chars = worker_name_size;
/* Terminate thread name with NULL character */
worker->thread_name[0] = '\0';
(void) strncat(worker->thread_name, worker_name, num_free_chars);
num_free_chars = worker_name_size - strlen(worker->thread_name);
(void) strncat(worker->thread_name, "_", num_free_chars);
num_free_chars = worker_name_size - strlen(worker->thread_name);
(void) strncat(worker->thread_name, gpu_name, num_free_chars);
}
int nvgpu_worker_init(struct gk20a *g, struct nvgpu_worker *worker,
const struct nvgpu_worker_ops *ops)
{
int err;
worker->g = g;
nvgpu_atomic_set(&worker->put, 0);
(void) nvgpu_cond_init(&worker->wq);
nvgpu_init_list_node(&worker->items);
nvgpu_spinlock_init(&worker->items_lock);
nvgpu_mutex_init(&worker->start_lock);
worker->ops = ops;
err = nvgpu_worker_start(worker);
if (err != 0) {
nvgpu_err(g, "failed to start worker poller thread %s",
worker->thread_name);
return err;
}
return 0;
}
void nvgpu_worker_deinit(struct nvgpu_worker *worker)
{
nvgpu_mutex_acquire(&worker->start_lock);
nvgpu_thread_stop(&worker->poll_task);
nvgpu_mutex_release(&worker->start_lock);
}