diff --git a/userspace/Makefile.tmk b/userspace/Makefile.tmk index 1a47d48b6..841593674 100644 --- a/userspace/Makefile.tmk +++ b/userspace/Makefile.tmk @@ -48,8 +48,12 @@ NV_COMPONENT_CFLAGS += -D__NVGPU_POSIX__ _NV_TOOLCHAIN_CFLAGS += -rdynamic NV_UNIT_SH=unit.sh +NV_TESTLIST_PY=testlist.py +NV_REQ_TESTS_JSON=required_tests.json NV_COMPONENT_SYSTEMIMAGE_DIR := $(NV_SYSTEMIMAGE_TEST_EXECUTABLE_DIR)/nvgpu_unit -systemimage:: $(NV_COMPONENT_SYSTEMIMAGE_DIR) $(NV_COMPONENT_SYSTEMIMAGE_DIR)/$(NV_UNIT_SH) +systemimage:: $(NV_COMPONENT_SYSTEMIMAGE_DIR) $(NV_COMPONENT_SYSTEMIMAGE_DIR)/$(NV_UNIT_SH) \ + $(NV_COMPONENT_SYSTEMIMAGE_DIR)/$(NV_TESTLIST_PY) \ + $(NV_COMPONENT_SYSTEMIMAGE_DIR)/$(NV_REQ_TESTS_JSON) #make the output directory $(NV_COMPONENT_SYSTEMIMAGE_DIR) : $(NV_SYSTEMIMAGE_TEST_EXECUTABLE_DIR) @@ -58,6 +62,11 @@ $(NV_COMPONENT_SYSTEMIMAGE_DIR) : $(NV_SYSTEMIMAGE_TEST_EXECUTABLE_DIR) #copy the test script $(NV_COMPONENT_SYSTEMIMAGE_DIR)/$(NV_UNIT_SH) : $(NV_COMPONENT_DIR)/$(NV_UNIT_SH) $(NV_COMPONENT_SYSTEMIMAGE_DIR) $(CP) $< $@ +$(NV_COMPONENT_SYSTEMIMAGE_DIR)/$(NV_TESTLIST_PY) : $(NV_COMPONENT_DIR)/$(NV_TESTLIST_PY) $(NV_COMPONENT_SYSTEMIMAGE_DIR) + $(CP) $< $@ +$(NV_COMPONENT_SYSTEMIMAGE_DIR)/$(NV_REQ_TESTS_JSON) : $(NV_COMPONENT_DIR)/$(NV_REQ_TESTS_JSON) $(NV_COMPONENT_SYSTEMIMAGE_DIR) + $(CP) $< $@ + include $(NV_BUILD_NVTEST_EXECUTABLE) diff --git a/userspace/install-unit.sh b/userspace/install-unit.sh index 57df54acb..6a28b889f 100755 --- a/userspace/install-unit.sh +++ b/userspace/install-unit.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright (c) 2018, NVIDIA CORPORATION. All Rights Reserved. +# Copyright (c) 2019, NVIDIA CORPORATION. All Rights Reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the "Software"), @@ -118,6 +118,9 @@ jcp $nvgpu_bins/userspace-l4t_64/libnvgpu_unit-lib.so nvgpu_unit/libnvgpu-uni jcp $nvgpu_bins/drivers/gpu/nvgpu-l4t_64/libnvgpu-drv.so nvgpu_unit/libnvgpu-drv.so jcp $TOP/kernel/nvgpu/userspace/unit.sh nvgpu_unit/unit.sh jcp $TOP/kernel/nvgpu/userspace/gcov.sh nvgpu_unit/gcov.sh +jcp $TOP/kernel/nvgpu/userspace/testlist.py nvgpu_unit/testlist.py +jcp $TOP/kernel/nvgpu/userspace/required_tests.json \ + nvgpu_unit/required_tests.json find $nvgpu_bins/userspace/units -name "*.so" -not -path "*unit.so" \ -not -path "*drv.so" -exec ls {} \; | while read unit_so ; do diff --git a/userspace/required_tests.json b/userspace/required_tests.json new file mode 100644 index 000000000..96fe28c5b --- /dev/null +++ b/userspace/required_tests.json @@ -0,0 +1,616 @@ +[ + { + "test": "cache_default", + "unit": "fault_injection" + }, + { + "test": "cache_delayed_disable", + "unit": "fault_injection" + }, + { + "test": "cache_delayed_enable", + "unit": "fault_injection" + }, + { + "test": "cache_enabled", + "unit": "fault_injection" + }, + { + "test": "dma_alloc_default", + "unit": "fault_injection" + }, + { + "test": "dma_alloc_delayed_enable", + "unit": "fault_injection" + }, + { + "test": "dma_alloc_enabled", + "unit": "fault_injection" + }, + { + "test": "dma_alloc_init", + "unit": "fault_injection" + }, + { + "test": "fault_injection_init", + "unit": "fault_injection" + }, + { + "test": "init", + "unit": "fault_injection" + }, + { + "test": "kmalloc_default", + "unit": "fault_injection" + }, + { + "test": "kmalloc_delayed_disable", + "unit": "fault_injection" + }, + { + "test": "kmalloc_delayed_enable", + "unit": "fault_injection" + }, + { + "test": "kmalloc_enabled", + "unit": "fault_injection" + }, + { + "test": "fuse_gm20b_basic_fuses", + "unit": "fuse" + }, + { + "test": "fuse_gm20b_check_fmodel", + "unit": "fuse" + }, + { + "test": "fuse_gm20b_check_gcplex_fail", + "unit": "fuse" + }, + { + "test": "fuse_gm20b_check_non_sec", + "unit": "fuse" + }, + { + "test": "fuse_gm20b_check_sec", + "unit": "fuse" + }, + { + "test": "fuse_gm20b_check_sec_invalid_gcplex", + "unit": "fuse" + }, + { + "test": "fuse_gm20b_cleanup", + "unit": "fuse" + }, + { + "test": "fuse_gm20b_init", + "unit": "fuse" + }, + { + "test": "fuse_gp10b_check_fmodel", + "unit": "fuse" + }, + { + "test": "fuse_gp10b_check_gcplex_fail", + "unit": "fuse" + }, + { + "test": "fuse_gp10b_check_non_sec", + "unit": "fuse" + }, + { + "test": "fuse_gp10b_check_sec", + "unit": "fuse" + }, + { + "test": "fuse_gp10b_check_sec_invalid_gcplex", + "unit": "fuse" + }, + { + "test": "fuse_gp10b_cleanup", + "unit": "fuse" + }, + { + "test": "fuse_gp10b_ecc", + "unit": "fuse" + }, + { + "test": "fuse_gp10b_feature_override_disable", + "unit": "fuse" + }, + { + "test": "fuse_gp10b_init", + "unit": "fuse" + }, + { + "test": "fuse_gv100_cleanup", + "unit": "fuse" + }, + { + "test": "fuse_gv100_init", + "unit": "fuse" + }, + { + "test": "fuse_gv100_vin_cal_rev", + "unit": "fuse" + }, + { + "test": "fuse_gv100_vin_cal_slope_intercept", + "unit": "fuse" + }, + { + "test": "list_all_head", + "unit": "list" + }, + { + "test": "list_all_tail", + "unit": "list" + }, + { + "test": "list_move", + "unit": "list" + }, + { + "test": "list_replace", + "unit": "list" + }, + { + "test": "alloc_destroy", + "unit": "nvgpu_allocator" + }, + { + "test": "alloc_ops", + "unit": "nvgpu_allocator" + }, + { + "test": "common_init", + "unit": "nvgpu_allocator" + }, + { + "test": "gmmu_clean", + "unit": "nvgpu_gmmu" + }, + { + "test": "gmmu_init", + "unit": "nvgpu_gmmu" + }, + { + "test": "gmmu_map_unmap_iommu_sysmem", + "unit": "nvgpu_gmmu" + }, + { + "test": "gmmu_map_unmap_iommu_sysmem_adv_big_pages", + "unit": "nvgpu_gmmu" + }, + { + "test": "gmmu_map_unmap_iommu_sysmem_adv_big_pages_batched", + "unit": "nvgpu_gmmu" + }, + { + "test": "gmmu_map_unmap_iommu_sysmem_adv_big_pages_offset", + "unit": "nvgpu_gmmu" + }, + { + "test": "gmmu_map_unmap_iommu_sysmem_adv_ctag", + "unit": "nvgpu_gmmu" + }, + { + "test": "gmmu_map_unmap_iommu_sysmem_adv_kernel_pages", + "unit": "nvgpu_gmmu" + }, + { + "test": "gmmu_map_unmap_iommu_sysmem_adv_sgl_skip", + "unit": "nvgpu_gmmu" + }, + { + "test": "gmmu_map_unmap_iommu_sysmem_adv_small_pages_sparse", + "unit": "nvgpu_gmmu" + }, + { + "test": "gmmu_map_unmap_iommu_sysmem_coh", + "unit": "nvgpu_gmmu" + }, + { + "test": "gmmu_map_unmap_iommu_sysmem_ro", + "unit": "nvgpu_gmmu" + }, + { + "test": "gmmu_map_unmap_iommu_sysmem_ro_f", + "unit": "nvgpu_gmmu" + }, + { + "test": "gmmu_map_unmap_no_iommu_sysmem", + "unit": "nvgpu_gmmu" + }, + { + "test": "gmmu_map_unmap_no_iommu_sysmem_adv_big_pages_offset_large", + "unit": "nvgpu_gmmu" + }, + { + "test": "gmmu_map_unmap_no_iommu_sysmem_noncacheable", + "unit": "nvgpu_gmmu" + }, + { + "test": "gmmu_map_unmap_sgt_iommu_sysmem", + "unit": "nvgpu_gmmu" + }, + { + "test": "gmmu_map_unmap_unmapped", + "unit": "nvgpu_gmmu" + }, + { + "test": "gmmu_map_unmap_vidmem", + "unit": "nvgpu_gmmu" + }, + { + "test": "gmmu_set_pte", + "unit": "nvgpu_gmmu" + }, + { + "test": "init_page_table_fail", + "unit": "nvgpu_gmmu" + }, + { + "test": "map_fail_fi_null_sgt", + "unit": "nvgpu_gmmu" + }, + { + "test": "map_fail_fi_vm_alloc", + "unit": "nvgpu_gmmu" + }, + { + "test": "map_fail_pd_allocate", + "unit": "nvgpu_gmmu" + }, + { + "test": "map_fail_pd_allocate_child", + "unit": "nvgpu_gmmu" + }, + { + "test": "flat", + "unit": "nvgpu_runlist" + }, + { + "test": "flat_oversize_end", + "unit": "nvgpu_runlist" + }, + { + "test": "flat_oversize_onehalf", + "unit": "nvgpu_runlist" + }, + { + "test": "flat_oversize_single", + "unit": "nvgpu_runlist" + }, + { + "test": "flat_oversize_tiny", + "unit": "nvgpu_runlist" + }, + { + "test": "flat_oversize_two", + "unit": "nvgpu_runlist" + }, + { + "test": "interleaving", + "unit": "nvgpu_runlist" + }, + { + "test": "interleaving_l0", + "unit": "nvgpu_runlist" + }, + { + "test": "interleaving_l0_l1", + "unit": "nvgpu_runlist" + }, + { + "test": "interleaving_l0_l2", + "unit": "nvgpu_runlist" + }, + { + "test": "interleaving_l1", + "unit": "nvgpu_runlist" + }, + { + "test": "interleaving_l1_l2", + "unit": "nvgpu_runlist" + }, + { + "test": "interleaving_l2", + "unit": "nvgpu_runlist" + }, + { + "test": "interleaving_oversize_l2", + "unit": "nvgpu_runlist" + }, + { + "test": "interleaving_oversize_l2_l1_l2", + "unit": "nvgpu_runlist" + }, + { + "test": "interleaving_oversize_l2_l1_l2_l1", + "unit": "nvgpu_runlist" + }, + { + "test": "interleaving_oversize_l2_l1_l2_l1_l2", + "unit": "nvgpu_runlist" + }, + { + "test": "interleaving_oversize_tiny", + "unit": "nvgpu_runlist" + }, + { + "test": "tsg_format_ch1", + "unit": "nvgpu_runlist" + }, + { + "test": "tsg_format_ch1_timeslice", + "unit": "nvgpu_runlist" + }, + { + "test": "tsg_format_ch2", + "unit": "nvgpu_runlist" + }, + { + "test": "tsg_format_ch3_inactive2", + "unit": "nvgpu_runlist" + }, + { + "test": "tsg_format_ch5", + "unit": "nvgpu_runlist" + }, + { + "test": "sgt_alignment_non_iommu", + "unit": "nvgpu_sgt" + }, + { + "test": "sgt_alignment_with_iommu", + "unit": "nvgpu_sgt" + }, + { + "test": "sgt_basic_apis", + "unit": "nvgpu_sgt" + }, + { + "test": "sgt_get_next", + "unit": "nvgpu_sgt" + }, + { + "test": "alloc_1024x256B_x11x3", + "unit": "pd_cache" + }, + { + "test": "alloc_1024x256B_x16x1", + "unit": "pd_cache" + }, + { + "test": "alloc_1024x256B_x16x15", + "unit": "pd_cache" + }, + { + "test": "alloc_1024x256B_x32x1", + "unit": "pd_cache" + }, + { + "test": "alloc_1x1024B", + "unit": "pd_cache" + }, + { + "test": "alloc_1x2048B", + "unit": "pd_cache" + }, + { + "test": "alloc_1x256B", + "unit": "pd_cache" + }, + { + "test": "alloc_1x512B", + "unit": "pd_cache" + }, + { + "test": "alloc_direct_1024x16PAGE", + "unit": "pd_cache" + }, + { + "test": "alloc_direct_1024xPAGE", + "unit": "pd_cache" + }, + { + "test": "alloc_direct_1024xPAGE_x16x1", + "unit": "pd_cache" + }, + { + "test": "alloc_direct_1024xPAGE_x16x15", + "unit": "pd_cache" + }, + { + "test": "alloc_direct_1024xPAGE_x16x4", + "unit": "pd_cache" + }, + { + "test": "alloc_direct_1024xPAGE_x32x24", + "unit": "pd_cache" + }, + { + "test": "alloc_direct_1x16PAGE", + "unit": "pd_cache" + }, + { + "test": "alloc_direct_1xPAGE", + "unit": "pd_cache" + }, + { + "test": "alloc_direct_oom", + "unit": "pd_cache" + }, + { + "test": "alloc_oom", + "unit": "pd_cache" + }, + { + "test": "env_init", + "unit": "pd_cache" + }, + { + "test": "fini", + "unit": "pd_cache" + }, + { + "test": "free_empty", + "unit": "pd_cache" + }, + { + "test": "init", + "unit": "pd_cache" + }, + { + "test": "invalid_pd_alloc", + "unit": "pd_cache" + }, + { + "test": "pd_packing", + "req": "NVGPU-RQCD-68.C3", + "vc": "V4", + "uid": "6439202", + "unit": "pd_cache" + }, + { + "test": "valid_alloc", + "req": "NVGPU-RQCD-68.C1,2", + "vc": "V4", + "uid": "6439202", + "unit": "pd_cache" + }, + { + "test": "bit_clear", + "unit": "posix_bitops" + }, + { + "test": "bit_set", + "unit": "posix_bitops" + }, + { + "test": "bitmap_clear", + "unit": "posix_bitops" + }, + { + "test": "bitmap_set", + "unit": "posix_bitops" + }, + { + "test": "ffs", + "unit": "posix_bitops" + }, + { + "test": "ffz", + "unit": "posix_bitops" + }, + { + "test": "find_first_bit", + "unit": "posix_bitops" + }, + { + "test": "find_first_zero_bit", + "unit": "posix_bitops" + }, + { + "test": "find_next_bit", + "unit": "posix_bitops" + }, + { + "test": "find_zero_area", + "unit": "posix_bitops" + }, + { + "test": "fls", + "unit": "posix_bitops" + }, + { + "test": "info", + "unit": "posix_bitops" + }, + { + "test": "single_bitops", + "unit": "posix_bitops" + }, + { + "test": "test_and_clear_bit", + "unit": "posix_bitops" + }, + { + "test": "test_and_set_bit", + "unit": "posix_bitops" + }, + { + "test": "endianness", + "unit": "posix_env" + }, + { + "test": "ptr_in_u64", + "unit": "posix_env" + }, + { + "test": "signage", + "unit": "posix_env" + }, + { + "test": "sizes", + "unit": "posix_env" + }, + { + "test": "__readl", + "unit": "posix_mockio" + }, + { + "test": "bar1_readl", + "unit": "posix_mockio" + }, + { + "test": "bar1_writel", + "unit": "posix_mockio" + }, + { + "test": "readl", + "unit": "posix_mockio" + }, + { + "test": "register_io_callbacks", + "unit": "posix_mockio" + }, + { + "test": "test_register_space", + "unit": "posix_mockio" + }, + { + "test": "usermode_writel", + "unit": "posix_mockio" + }, + { + "test": "writel", + "unit": "posix_mockio" + }, + { + "test": "writel_check", + "unit": "posix_mockio" + }, + { + "test": "nvgpu_pramin_dying", + "unit": "pramin" + }, + { + "test": "nvgpu_pramin_free_test_env", + "unit": "pramin" + }, + { + "test": "nvgpu_pramin_memset", + "unit": "pramin" + }, + { + "test": "nvgpu_pramin_rd_n_1_sgl", + "unit": "pramin" + }, + { + "test": "nvgpu_pramin_wr_n_3_sgl", + "unit": "pramin" + } +] \ No newline at end of file diff --git a/userspace/src/results.c b/userspace/src/results.c index 4c30c4dbe..f83feedd8 100644 --- a/userspace/src/results.c +++ b/userspace/src/results.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -114,6 +115,40 @@ done: return err; } +static void dump_test_record(FILE *logfile, struct unit_test_record *rec, + bool status, bool first) +{ + first ? fprintf(logfile, "\t{") : fprintf(logfile, ",\n\t{"); + fprintf(logfile, "\"unit\": \"%s\", ", rec->mod->name); + fprintf(logfile, "\"test\": \"%s\", ", rec->test->name); + fprintf(logfile, "\"status\": %s, ", status ? "true":"false"); + fprintf(logfile, "\"uid\": \"%s\", ", rec->test->jama.unique_id); + fprintf(logfile, "\"vc\": \"%s\", ", + rec->test->jama.verification_criteria); + fprintf(logfile, "\"req\": \"%s\"", rec->test->jama.requirement); + fprintf(logfile, "}"); +} + +static void dump_test_log(struct unit_fw *fw, struct unit_test_list + *passing_tests, struct unit_test_list *failing_tests) +{ + struct unit_test_record *rec; + int count = 0; + FILE *logfile = fopen("results.json", "w+"); + + fprintf(logfile, "[\n"); + for_record_in_test_list(passing_tests, rec) { + dump_test_record(logfile, rec, true, count == 0); + count++; + } + for_record_in_test_list(failing_tests, rec) { + dump_test_record(logfile, rec, false, count == 0); + count++; + } + fprintf(logfile, "\n]\n"); + fclose(logfile); +} + void core_print_test_status(struct unit_fw *fw) { struct unit_test_list *failing_tests = &fw->results->failing; @@ -139,4 +174,6 @@ void core_print_test_status(struct unit_fw *fw) rec->mod->name, rec->test->name); } + + dump_test_log(fw, &fw->results->passing, &fw->results->failing); } diff --git a/userspace/testlist.py b/userspace/testlist.py new file mode 100755 index 000000000..688773f34 --- /dev/null +++ b/userspace/testlist.py @@ -0,0 +1,229 @@ +#!/usr/bin/python +# +# Copyright (c) 2019, NVIDIA CORPORATION. All Rights Reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. + +import argparse, os +import json, md5 + +RESULTS_FILE = 'results.json' +HTML_FILE = 'results.html' +REQ_FILE = 'required_tests.json' + +HTML_HEAD=''' + + + + + + + + +

Results

+''' + +#Build a dictionary-based structure to easily access tests by unit and name. +def build_results_dict(results): + test_dict = {} + #Iterate through the results and group them by unit name + for result in results: + unit = result['unit'] + test = result['test'] + if unit not in test_dict: + test_dict[unit] = {} + if test in test_dict[unit]: + print("WARNING: %s - %s already exists." % (unit, test)) + test_dict[unit][test] = result + return test_dict + +def regen(): + output_list = [] + if not os.path.exists(RESULTS_FILE): + print("File 'results.json' not found. Run unit.sh first.") + exit(1) + with open(RESULTS_FILE) as infile: + results = json.loads(infile.read()) + + test_dict = build_results_dict(results) + + #Generate the output by alphabetical order + test_count = 0 + for unit, tests in sorted(test_dict.items(), key=lambda kv: kv[0], reverse=False): + for test in sorted(tests.items()): + entry = {"unit": unit, "test": test[0]} + if test[1]['uid'] != "(null)": + entry['uid'] = test[1]['uid'] + entry['vc'] = test[1]['vc'] + entry['req'] = test[1]['req'] + output_list.append(entry) + test_count += 1 + with open(REQ_FILE, "w+") as outfile: + json_text = json.dumps(output_list, indent=4).replace(" \n", "\n") + outfile.write(json_text) + print("Generated updated version of \"%s\" based on tests in \"%s\"." % (REQ_FILE, RESULTS_FILE)) + print("Required tests: %d" % test_count) + +def get_RGB(s): + hash = md5.md5(s).hexdigest() + rgb = '#' + hash[0:2] + hash[2:4] + hash[4:6] + return rgb + +def format_html_test(unit, test, status, error, uid, req, vc): + if status: + status = "green" + else: + status = "red" + color = get_RGB(unit) + ret = "%s%s%s" % (color, unit, test, status, error) + jamalink = "" + class_name = "invalid-uid" + if uid != "": + jamalink = "https://nvidia.jamacloud.com/perspective.req#/items/%s?projectId=20460" % uid + jamalink = "%s" % (jamalink, uid) + class_name = "valid-uid" + ret += "%s\n" % (class_name, jamalink) + ret += "%s%s\n" % (req, vc) + ret += "\n" + return ret + +def check(html = False): + #Check that tests in results.json cover required_tests.json + with open(RESULTS_FILE) as results_file: + results = json.loads(results_file.read()) + with open(REQ_FILE) as req_file: + reqd_tests = json.loads(req_file.read()) + + test_dict = build_results_dict(results) + req_dict = build_results_dict(reqd_tests) + + errors = 0 + test_count = 0 + log = "" + html = "" + + #First make sure that all required tests were run and PASSED. + for reqd_test in reqd_tests: + unit = reqd_test['unit'] + test = reqd_test['test'] + error = "" + status = False + if unit not in test_dict: + error = ("ERROR: Required unit %s is not in test results.\n" % unit) + log += error + errors += 1 + elif test not in test_dict[unit]: + error = ("ERROR: Required test %s - %s is not in test results.\n" % (unit, test)) + log += error + errors += 1 + elif test_dict[unit][test]['status'] is False: + log += ("ERROR: Required test %s - %s FAILED.\n" % (unit, test)) + error = "FAILED" + errors += 1 + else: + status = True + error = "PASS" + html += format_html_test(unit, test, status, error, reqd_test.get('uid', ''), reqd_test.get('req', ''), reqd_test.get('vc', '')) + test_count += 1 + + #As a helpful hint, check to see if any tests were run without being in the list of required + #tests. This should not cause a failure, but just a warning for the developer to add the + #test to the required list. + for result in results: + unit = result['unit'] + test = result['test'] + if unit not in req_dict: + log += ("WARNING: Tested unit %s is not in required tests. Use testlist.py --regen\n" % unit) + elif test not in req_dict[unit]: + log +=("WARNING: Test %s - %s is not in required tests. Use testlist.py --regen\n" % (unit, test)) + + if log != "": + print(log) + + if html: + with open(HTML_FILE, "w+") as outfile: + outfile.write(HTML_HEAD) + outfile.write("\n") + outfile.write("\n") + outfile.write("\n") + outfile.write("\n") + outfile.write(html) + outfile.write("
UnitTestResultJAMA UID [Toggle]RequirementVC
\n") + outfile.write("") + print("Wrote %s" % HTML_FILE) + + if errors > 0: + return 1 + else: + print("PASS: All %d tests found in result log." % test_count) + return 0 + +def html(): + return check(html = True) + +parser = argparse.ArgumentParser() +parser.add_argument("--regen", help="Regenerate list of expected test cases.", action="store_true") +parser.add_argument("--check", help="Make sure all expected test cases were run.", action="store_true") +parser.add_argument("--html", help="Perform --check and export results in an HTML file.", action="store_true") + +args = parser.parse_args() +if args.regen: + regen() + exit(0) +if args.check: + exit(check()) +if args.html: + exit(html()) +else: + parser.print_help() + diff --git a/userspace/unit.sh b/userspace/unit.sh index 3832f316f..db762af7e 100755 --- a/userspace/unit.sh +++ b/userspace/unit.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright (c) 2018, NVIDIA CORPORATION. All Rights Reserved. +# Copyright (c) 2019, NVIDIA CORPORATION. All Rights Reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the "Software"), @@ -47,5 +47,10 @@ echo "$ $NVGPU_UNIT $*" $NVGPU_UNIT $* rc=$? +if [ $rc -eq "0" ]; then + echo "Checking executed tests against list of required tests:" + ./testlist.py --html + rc=$? +fi popd exit $rc