gpu: nvgpu: unit: required tests system

When running unit testing, we need to ensure that all expected unit
tests were actually run. This used to be done by an external Python
script, but it is not suitable for some OSes. This patch brings this
mechanism into the unit testing framework, and makes use of an INI
file to provide the list of expected tests.

JIRA NVGPU-3072

Change-Id: I9ed590de9005fe10324c594ea33ebc8fda482019
Signed-off-by: Nicolas Benech <nbenech@nvidia.com>
Reviewed-on: https://git-master.nvidia.com/r/c/linux-nvgpu/+/2317220
Reviewed-by: automaticguardword <automaticguardword@nvidia.com>
Reviewed-by: Alex Waterman <alexw@nvidia.com>
Reviewed-by: mobile promotions <svcmobile_promotions@nvidia.com>
Tested-by: mobile promotions <svcmobile_promotions@nvidia.com>
GVS: Gerrit_Virtual_Submit

userspace/Makefile.sources

@@ -35,6 +35,7 @@ CORE_OBJS :=	\
 	$(CORE_OUT)/args.o			\
 	$(CORE_OUT)/io.o			\
 	$(CORE_OUT)/module.o			\
+	$(CORE_OUT)/required_tests.o		\
 	$(CORE_OUT)/results.o			\
 	$(CORE_OUT)/exec.o
 

userspace/Makefile.tmk

@@ -29,6 +29,7 @@ NVGPU_UNIT_COMMON_SRCS		:= \
                                 src/args.c \
                                 src/io.c \
                                 src/module.c \
+                                src/required_tests.c \
                                 src/results.c \
                                 src/exec.c
 NVGPU_UNIT_COMMON_INCLUDES	:= \
@@ -61,7 +62,7 @@ _NV_TOOLCHAIN_CFLAGS            += -rdynamic
 NV_UNIT_SH=unit.sh
 NV_SUBMIT_UNIT_SH=nvgpu_submit_unit.sh
 NV_TESTLIST_PY=testlist.py
-NV_REQ_TESTS_JSON=required_tests.json
+NV_REQ_TESTS_INI=required_tests.ini
 NV_NETD_IMG=NETD_img.bin
 NV_FECS_IMG=fecs.bin
 NV_FECS_SIG_IMG=fecs_sig.bin
@@ -75,7 +76,7 @@ NV_UNIT_REQ_FIRMWARE_DIR        := $(NV_COMPONENT_SYSTEMIMAGE_DIR)/firmware/gv11
 systemimage:: $(NV_COMPONENT_SYSTEMIMAGE_DIR) $(NV_COMPONENT_SYSTEMIMAGE_DIR)/$(NV_UNIT_SH) \
 	$(NV_SYSTEMIMAGE_TEST_EXECUTABLE_DIR)/$(NV_SUBMIT_UNIT_SH) \
 	$(NV_COMPONENT_SYSTEMIMAGE_DIR)/$(NV_TESTLIST_PY) \
-	$(NV_COMPONENT_SYSTEMIMAGE_DIR)/$(NV_REQ_TESTS_JSON) \
+	$(NV_COMPONENT_SYSTEMIMAGE_DIR)/$(NV_REQ_TESTS_INI) \
 	$(NV_UNIT_REQ_FIRMWARE_DIR) $(NV_UNIT_REQ_FIRMWARE_DIR)/$(NV_NETD_IMG) \
 	$(NV_UNIT_REQ_FIRMWARE_DIR)/$(NV_FECS_IMG) \
 	$(NV_UNIT_REQ_FIRMWARE_DIR)/$(NV_FECS_SIG_IMG) \
@@ -99,7 +100,7 @@ $(NV_SYSTEMIMAGE_TEST_EXECUTABLE_DIR)/$(NV_SUBMIT_UNIT_SH) : $(NV_COMPONENT_DIR)
 	$(CP) $< $@
 $(NV_COMPONENT_SYSTEMIMAGE_DIR)/$(NV_TESTLIST_PY) : $(NV_COMPONENT_DIR)/$(NV_TESTLIST_PY) $(NV_COMPONENT_SYSTEMIMAGE_DIR)
 	$(CP) $< $@
-$(NV_COMPONENT_SYSTEMIMAGE_DIR)/$(NV_REQ_TESTS_JSON) : $(NV_COMPONENT_DIR)/$(NV_REQ_TESTS_JSON) $(NV_COMPONENT_SYSTEMIMAGE_DIR)
+$(NV_COMPONENT_SYSTEMIMAGE_DIR)/$(NV_REQ_TESTS_INI) : $(NV_COMPONENT_DIR)/$(NV_REQ_TESTS_INI) $(NV_COMPONENT_SYSTEMIMAGE_DIR)
 	$(CP) $< $@
 $(NV_UNIT_REQ_FIRMWARE_DIR)/$(NV_NETD_IMG) : $(NV_COMPONENT_DIR)/firmware/gv11b/$(NV_NETD_IMG) $(NV_UNIT_REQ_FIRMWARE_DIR)
 	$(CP) $< $@

userspace/include/unit/args.h

@@ -53,6 +53,7 @@ struct unit_fw_args {
 	const char	*unit_name;
 	const char	*unit_load_path;
 	const char	*unit_to_run;
+	const char	*required_tests_file;
 };
 
 int core_parse_args(struct unit_fw *fw, int argc, char **argv);

userspace/include/unit/required_tests.h

@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef __REQUIRED_TESTS_H__
+#define __REQUIRED_TESTS_H__
+
+#define MAX_LINE_SIZE	256
+
+int parse_req_file(struct unit_fw *fw, const char *ini_file);
+int check_executed_tests(struct unit_fw *fw);
+
+#endif

userspace/src/args.c

@@ -40,10 +40,11 @@ static struct option core_opts[] = {
 	{ "num-threads",	1, NULL, 'j' },
 	{ "test-level",		1, NULL, 't' },
 	{ "debug",		0, NULL, 'd' },
+	{ "required",		0, NULL, 'r' },
 	{ NULL,			0, NULL,  0  }
 };
 
-static const char *core_opts_str = "hvqCnQL:j:t:d";
+static const char *core_opts_str = "hvqCnQL:j:t:dr:";
 
 void core_print_help(struct unit_fw *fw)
 {
@@ -74,6 +75,8 @@ void core_print_help(struct unit_fw *fw)
 "                         Test plan level. 0=L0, 1=L1. default: 1\n",
 "  -d, --debug            Disable signal handling to facilitate debug of",
 "                         crashes.\n",
+"  -r, --required <FILE>  Path to a file with a list of required tests to\n"
+"                         check if all were executed.\n",
 "\n",
 "Note: mandatory arguments to long arguments are mandatory for short\n",
 "arguments as well.\n",
@@ -92,6 +95,7 @@ static void set_arg_defaults(struct unit_fw_args *args)
 	args->unit_load_path = DEFAULT_ARG_UNIT_LOAD_PATH;
 	args->thread_count = 1;
 	args->test_lvl = TEST_PLAN_MAX;
+	args->required_tests_file = NULL;
 }
 
 /*
@@ -166,6 +170,9 @@ int core_parse_args(struct unit_fw *fw, int argc, char **argv)
 		case 'd':
 			args->debug = true;
 			break;
+		case 'r':
+			args->required_tests_file = optarg;
+			break;
 		case '?':
 			args->help = true;
 			return -1;

userspace/src/required_tests.c

@@ -0,0 +1,333 @@
+/*
+ * Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include <unit/io.h>
+#include <unit/core.h>
+#include <unit/unit.h>
+#include <unit/required_tests.h>
+#include <unit/results.h>
+#include <nvgpu/list.h>
+
+struct test_list_node {
+	char *test_subtest_name;
+	long test_level;
+	struct nvgpu_list_node list;
+};
+struct unit_list_node {
+	char *unit_name;
+	struct nvgpu_list_node test_list_head;
+	struct nvgpu_list_node list;
+};
+
+static struct nvgpu_list_node unit_list_head;
+
+static inline struct test_list_node *
+test_list_node_from_list(struct nvgpu_list_node *node)
+{
+	return (struct test_list_node *)
+		((uintptr_t)node - offsetof(struct test_list_node, list));
+};
+static inline struct unit_list_node *
+unit_list_node_from_list(struct nvgpu_list_node *node)
+{
+	return (struct unit_list_node *)
+		((uintptr_t)node - offsetof(struct unit_list_node, list));
+};
+
+/*
+ * Add a unit in the list of units. The node will hold the name of the unit and
+ * another list of tests.
+ */
+static struct unit_list_node *add_unit(struct nvgpu_list_node *head,
+	const char *unit_name)
+{
+	struct unit_list_node *node = malloc(sizeof(struct unit_list_node));
+
+	if (node == NULL) {
+		return NULL;
+	}
+
+	node->unit_name = malloc(strlen(unit_name)+1);
+	if (node->unit_name == NULL) {
+		free(node);
+		return NULL;
+	}
+
+	strcpy(node->unit_name, unit_name);
+	nvgpu_init_list_node(&node->test_list_head);
+	nvgpu_list_add(&node->list, head);
+
+	return node;
+}
+
+/*
+ * From a unit list node, add a test_subtest string along with its test level.
+ */
+static struct test_list_node *add_test_subtest_level(
+	struct unit_list_node *unit, const char *test_subtest_name, long level)
+{
+	struct test_list_node *node = malloc(sizeof(struct test_list_node));
+
+	if (node == NULL) {
+		return NULL;
+	}
+
+	node->test_level = level;
+	node->test_subtest_name = malloc(strlen(test_subtest_name)+1);
+	if (node->test_subtest_name == NULL) {
+		free(node);
+		return NULL;
+	}
+
+	strcpy(node->test_subtest_name, test_subtest_name);
+
+	nvgpu_list_add(&node->list, &unit->test_list_head);
+
+	return node;
+}
+
+/*
+ * Simple helper to sanitize the input and remove un-needed characters.
+ */
+static void sanitize(char *dest, const char *src)
+{
+	char *dest_start = dest;
+
+	if (dest == NULL || src == NULL)
+		return;
+
+	/* Trim leading spaces */
+	while (*src == ' ' || *src == '\t')
+		src++;
+
+	/* Copy the rest of the string */
+	while (*src != '\0') {
+		if (*src == '#') {
+			/* Rest of the line is a comment, discard it */
+			break;
+		}
+		*dest++ = *src++;
+	}
+	*dest = '\0';
+
+	/* Backtrack to remove line returns and trim spaces at the end */
+	while (dest_start <= dest) {
+		if ((*dest == '\0') || (*dest == '\r') || (*dest == '\n')
+			|| (*dest == ' ')) {
+			*dest = '\0';
+		} else {
+			return;
+		}
+		dest--;
+	}
+}
+
+/* Parse a sting looking for a long number, along with error handling. */
+static bool parse_long(const char *str, long *val)
+{
+	char *tmp;
+	bool result = true;
+	errno = 0;
+	*val = strtol(str, &tmp, 10);
+
+	if (tmp == str || *tmp != '\0' ||
+		((*val == LONG_MIN || *val == LONG_MAX) && errno == ERANGE)) {
+		result = false;
+	}
+
+	return result;
+}
+
+/*
+ * Load the INI file that contains the list of required tests. This manages a
+ * list of units, and each unit node has a list of test cases.
+ */
+int parse_req_file(struct unit_fw *fw, const char *ini_file)
+{
+	char rawline[MAX_LINE_SIZE];
+	char line[MAX_LINE_SIZE];
+	char tmp[MAX_LINE_SIZE];
+	char *open_bracket, *close_bracket, *equal;
+	long value, len;
+	struct unit_list_node *current_unit = NULL;
+	FILE *file = fopen(ini_file, "r");
+
+	if (file == NULL) {
+		perror("Error reading INI file: ");
+		return -1;
+	}
+
+	nvgpu_init_list_node(&unit_list_head);
+
+	while (fgets(rawline, sizeof(rawline), file)) {
+		line[0] = '\0';
+		tmp[0] = '\0';
+		sanitize(line, rawline);
+		if (strlen(line) == 0) {
+			continue;
+		}
+		open_bracket = strchr(line, '[');
+		close_bracket = strchr(line, ']');
+		equal = strchr(line, '=');
+
+		if (equal != NULL) {
+			/* This is a key/value pair */
+			if (current_unit == NULL) {
+				continue;
+			}
+			strncpy(tmp, line, (equal-line));
+			tmp[equal-line] = '\0';
+			if (!parse_long(equal+1, &value)) {
+				core_err(fw, "Conversion error:\n%s\n",
+					rawline);
+				return -1;
+			}
+			if (add_test_subtest_level(current_unit, tmp, value)
+				== NULL) {
+					return -ENOMEM;
+				}
+		} else if ((open_bracket != NULL) && (close_bracket != NULL)) {
+			/* This is a section */
+			if (open_bracket++ > close_bracket) {
+				continue;
+			}
+			len = close_bracket-open_bracket;
+			strncpy(tmp, open_bracket, len);
+			tmp[len] = '\0';
+			current_unit = add_unit(&unit_list_head, tmp);
+			if (current_unit == NULL) {
+				return -ENOMEM;
+			}
+		} else {
+			/* Unknown line or syntax error */
+			core_err(fw, "Syntax error parsing:\n%s\n", rawline);
+			return -1;
+		}
+	}
+
+	fclose(file);
+
+	return 0;
+}
+
+/*
+ * Helper that takes a test function name and a subcase name, combines them to
+ * be in the same format as the INI file, and compare the result to a given
+ * string. Returns true if it matches.
+ */
+static bool cmp_test_name(const char *exec_fn_name, const char* exec_case_name,
+	const char *ini_test_subtest_name)
+{
+	char exec_test_subtest_name[MAX_LINE_SIZE];
+
+	exec_test_subtest_name[0] = '\0';
+	strcat(exec_test_subtest_name, exec_fn_name);
+	strcat(exec_test_subtest_name, ".");
+	strcat(exec_test_subtest_name, exec_case_name);
+
+	if (strcmp(ini_test_subtest_name, exec_test_subtest_name) == 0) {
+		return true;
+	}
+
+	return false;
+}
+
+/*
+ * Check the tests that were executed and compare them to the list of tests
+ * from the INI file. This only check passing tests; if there are failed tests,
+ * the failure will be handled somewhere else, and skipped tests are handled by
+ * the test level in the INI file.
+ * Returns the number of required tests that were not executed.
+ */
+int check_executed_tests(struct unit_fw *fw)
+{
+	struct unit_test_record *rec;
+	struct unit_list_node *unit_node, *found_unit_node = NULL;
+	struct test_list_node *test_node, *found_test_node = NULL;
+	struct unit_test_list *passing_tests = &fw->results->passing;
+	int missing_count = 0;
+
+	for_record_in_test_list(passing_tests, rec) {
+		found_unit_node = NULL;
+		found_test_node = NULL;
+		/* Search for the unit name */
+		nvgpu_list_for_each_entry(unit_node, &unit_list_head,
+				unit_list_node, list) {
+			if (strcmp(unit_node->unit_name, rec->mod->name) != 0) {
+				continue;
+			}
+
+			found_unit_node = unit_node;
+			/* Search for the fn_name.case_name */
+			nvgpu_list_for_each_entry(test_node,
+					&unit_node->test_list_head,
+					test_list_node, list) {
+				if (cmp_test_name(rec->test->fn_name,
+						rec->test->case_name,
+						test_node->test_subtest_name)) {
+					found_test_node = test_node;
+					break;
+				}
+			}
+			break;
+		}
+
+		if (found_test_node == NULL) {
+			/* Test should be added to the INI file */
+			core_err(fw,
+				"Test not in required tests: [%s] %s.%s\n",
+				rec->mod->name, rec->test->fn_name,
+				rec->test->case_name);
+		} else {
+			/* Found it, remove it from the list. */
+			nvgpu_list_del(&found_test_node->list);
+			if (nvgpu_list_empty(
+					&found_unit_node->test_list_head)) {
+				/* No more tests, remove the unit itself */
+				nvgpu_list_del(&found_unit_node->list);
+			}
+		}
+	}
+
+	/*
+	 * Now that all the executed tests were removed from the list, any test
+	 * that is leftover is a required test that was not executed.
+	 */
+	nvgpu_list_for_each_entry(unit_node, &unit_list_head, unit_list_node,
+					list) {
+		nvgpu_list_for_each_entry(test_node, &unit_node->test_list_head,
+						test_list_node, list) {
+			if (test_node->test_level <= fw->args->test_lvl) {
+				core_err(fw, "Required test not run: [%s] %s\n",
+					unit_node->unit_name,
+					test_node->test_subtest_name);
+				missing_count++;
+			}
+		}
+	}
+
+	return missing_count;
+}
+

userspace/src/unit_main.c

@@ -32,6 +32,7 @@
 #include <unit/args.h>
 #include <unit/module.h>
 #include <unit/results.h>
+#include <unit/required_tests.h>
 
 int main(int argc, char **argv)
 {
@@ -91,5 +92,22 @@ int main(int argc, char **argv)
 		return -2;
 	}
 
+	if (fw->args->required_tests_file != NULL) {
+		ret = parse_req_file(fw, fw->args->required_tests_file);
+		if (ret != 0) {
+			core_err(fw,
+				"Failed to load the required tests file.\n");
+			return -1;
+		}
+
+		ret = check_executed_tests(fw);
+		if (ret != 0) {
+			core_err(fw,
+				"Found %d required tests that were not run!\n",
+				ret);
+			return -1;
+		}
+	}
+
 	return 0;
 }

userspace/unit.sh

@@ -46,7 +46,7 @@ else
         # running on host
         LD_LIBRARY_PATH="build:build/units"
         # On host, must run single-threaded to avoid high VAs
-        NVGPU_UNIT="./build/nvgpu_unit --num-threads 1"
+        NVGPU_UNIT="./build/nvgpu_unit --num-threads 1 -r required_tests.ini"
 fi
 export LD_LIBRARY_PATH
 
@@ -70,10 +70,6 @@ if [ $rc -eq "0" ]; then
                 esac
                 shift
         done
-        echo $testlevelparam
-	echo "Checking executed tests against list of required tests:"
-	./testlist.py --html $testlevelparam
-	rc=$?
 fi
 popd
 exit $rc