gpu: nvgpu: unit: propagate fault injection to threads

Change approach to how the fault injection state is stored to facilitate
propagating fault injection state to child-threads. Rather than each
unit maintaining a thread-local object, there is a thread-local
container stored in the posix-fault-injection itself. This container is
initialized for each test module so that is independent of other other
test modules (for parallel test module execution). When child threads
are created with nvgpu_create_thread(), the fault injection container is
configured for the child.

JIRA NVGPU-3981

Change-Id: I9b580dc7f1621a7770eef8eba796f3918f2738bf
Signed-off-by: Philip Elcan <pelcan@nvidia.com>
Reviewed-on: https://git-master.nvidia.com/r/2238474
Reviewed-by: mobile promotions <svcmobile_promotions@nvidia.com>
Tested-by: mobile promotions <svcmobile_promotions@nvidia.com>

drivers/gpu/nvgpu/include/nvgpu/posix/posix-fault-injection.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2018-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -23,19 +23,83 @@
 #ifndef NVGPU_POSIX_FAULT_INJECTION_H
 #define NVGPU_POSIX_FAULT_INJECTION_H
 
+/**
+ * State for fault injection instance.
+ */
 struct nvgpu_posix_fault_inj {
 	bool enabled;
 	unsigned int counter;
 };
 
 /**
- * nvgpu_posix_init_fault_injection - Initialize the fault injection object
+ * Container for all fault injection instances.
- * to be used by a module that will report errors.
- *
- * @fi - pointer to the fault_inj object to initialize
  */
-void nvgpu_posix_init_fault_injection(struct nvgpu_posix_fault_inj *fi);
+struct nvgpu_posix_fault_inj_container {
+	/* nvgpu-core */
+	struct nvgpu_posix_fault_inj thread_fi;
+	struct nvgpu_posix_fault_inj cond_fi;
+	struct nvgpu_posix_fault_inj fstat_op;
+	struct nvgpu_posix_fault_inj fread_op;
+	struct nvgpu_posix_fault_inj kmem_fi;
+	struct nvgpu_posix_fault_inj nvgpu_fi;
+	struct nvgpu_posix_fault_inj dma_fi;
+	struct nvgpu_posix_fault_inj queue_out_fi;
 
+	/* qnx */
+	struct nvgpu_posix_fault_inj nvgpu_nvrmread_fi;
+	struct nvgpu_posix_fault_inj sdl_nvguard_fi;
+	struct nvgpu_posix_fault_inj clock_gettime_fi;
+	struct nvgpu_posix_fault_inj chip_id;
+	struct nvgpu_posix_fault_inj nvdt_node_compatible;
+	struct nvgpu_posix_fault_inj nvdt_read_prop_array_by_name_fi;
+	struct nvgpu_posix_fault_inj nvdt_read_prop_array_by_index_var;
+	struct nvgpu_posix_fault_inj nvdt_node_get_prop_fi;
+	struct nvgpu_posix_fault_inj nvgpu_platform_fpga_var;
+	struct nvgpu_posix_fault_inj nvgpu_platform_simulation_var;
+	struct nvgpu_posix_fault_inj check_os_native_var;
+	struct nvgpu_posix_fault_inj get_chip_revision_var;
+	struct nvgpu_posix_fault_inj nvdt_reg_prop_fi;
+	struct nvgpu_posix_fault_inj nvdt_reg_prop_zero_fi;
+	struct nvgpu_posix_fault_inj nvdt_reg_prop_size_zero_fi;
+	struct nvgpu_posix_fault_inj nvmap_alloc_fi;
+	struct nvgpu_posix_fault_inj nvmap_physinfo_fi;
+	struct nvgpu_posix_fault_inj nvmap_mmap_fi;
+	struct nvgpu_posix_fault_inj nvmap_handleparams_fi;
+	struct nvgpu_posix_fault_inj nvmap_useddesc_fi;
+	struct nvgpu_posix_fault_inj nvmap_munmap_fi;
+	struct nvgpu_posix_fault_inj nvmap_importid_fi;
+	struct nvgpu_posix_fault_inj nvgpu_hv_ipa_pa_var;
+	struct nvgpu_posix_fault_inj nvgpu_io_map_var;
+	struct nvgpu_posix_fault_inj nvgpu_readl_fi;
+	struct nvgpu_posix_fault_inj nvgpu_readl_impl_fi;
+	struct nvgpu_posix_fault_inj report_error_fi;
+	struct nvgpu_posix_fault_inj nvhost1x_open_fi;
+	struct nvgpu_posix_fault_inj nvhost1x_syncpointallocate_fi;
+	struct nvgpu_posix_fault_inj nvhost1x_getid_fi;
+	struct nvgpu_posix_fault_inj nvhost1x_waiterallocate_fi;
+	struct nvgpu_posix_fault_inj nvhost1x_syncpointread_fi;
+};
+
+/**
+ * @brief Initialize fault injection container for this thread.
+ *
+ * c [in]	Pointer to fault injection container to use in this thread.
+ *
+ * The fault injection container stores the fault injection state for all
+ * the fault injection users. This container is in thread-local-storage and
+ * must be initialized for a unit test thread and any child threads that need
+ * to have the same fault injection state.
+ */
+void nvgpu_posix_init_fault_injection
+				(struct nvgpu_posix_fault_inj_container *c);
+
+/**
+ * @brief Get the fault injection container for this thread.
+ *
+ * @return A pointer to the fault injection container in use for this thread.
+ */
+struct nvgpu_posix_fault_inj_container
+	*nvgpu_posix_fault_injection_get_container(void);
 
 /**
  * nvgpu_posix_enable_fault_injection - Enable/Disable fault injection for the

drivers/gpu/nvgpu/include/nvgpu/posix/thread.h

@@ -76,6 +76,9 @@ struct nvgpu_posix_thread_data {
 	 * Data to be passed to the nvgpu thread function.
 	 */
 	void *data;
+#ifdef NVGPU_UNITTEST_FAULT_INJECTION_ENABLEMENT
+	struct nvgpu_posix_fault_inj_container *fi_container;
+#endif
 };
 
 struct nvgpu_thread {

drivers/gpu/nvgpu/libnvgpu-drv_safe.export

@@ -357,6 +357,7 @@ nvgpu_posix_cleanup
 nvgpu_posix_enable_fault_injection
 nvgpu_posix_ffs
 nvgpu_posix_fls
+nvgpu_posix_init_fault_injection
 nvgpu_posix_io_add_reg_space
 nvgpu_posix_io_check_sequence
 nvgpu_posix_io_delete_reg_space

drivers/gpu/nvgpu/os/posix/cond.c

@@ -27,12 +27,12 @@
 #endif
 
 #ifdef NVGPU_UNITTEST_FAULT_INJECTION_ENABLEMENT
-_Thread_local struct nvgpu_posix_fault_inj cond_fi = {
-					.enabled = false,.counter = 0U,};
-
 struct nvgpu_posix_fault_inj *nvgpu_cond_get_fault_injection(void)
 {
-	return &cond_fi;
+	struct nvgpu_posix_fault_inj_container *c =
+			nvgpu_posix_fault_injection_get_container();
+
+	return &c->cond_fi;
 }
 #endif
 
@@ -41,7 +41,8 @@ int nvgpu_cond_init(struct nvgpu_cond *cond)
 	int ret;
 
 #ifdef NVGPU_UNITTEST_FAULT_INJECTION_ENABLEMENT
-	if (nvgpu_posix_fault_injection_handle_call(&cond_fi)) {
+	if (nvgpu_posix_fault_injection_handle_call(
+					nvgpu_cond_get_fault_injection())) {
 		return -EINVAL;
 	}
 #endif

drivers/gpu/nvgpu/os/posix/file_ops.c

@@ -30,24 +30,28 @@
 
 
 #ifdef NVGPU_UNITTEST_FAULT_INJECTION_ENABLEMENT
-_Thread_local struct nvgpu_posix_fault_inj fstat_op;
-_Thread_local struct nvgpu_posix_fault_inj fread_op;
-
 struct nvgpu_posix_fault_inj *nvgpu_file_ops_get_fstat_injection(void)
 {
-	return &fstat_op;
+	struct nvgpu_posix_fault_inj_container *c =
+			nvgpu_posix_fault_injection_get_container();
+
+	return &c->fstat_op;
 }
 
 struct nvgpu_posix_fault_inj *nvgpu_file_ops_get_fread_injection(void)
 {
-	return &fread_op;
+	struct nvgpu_posix_fault_inj_container *c =
+			nvgpu_posix_fault_injection_get_container();
+
+	return &c->fread_op;
 }
 #endif
 
 int nvgpu_fstat(int fd, struct stat *buf)
 {
 #ifdef NVGPU_UNITTEST_FAULT_INJECTION_ENABLEMENT
-	if (nvgpu_posix_fault_injection_handle_call(&fstat_op)) {
+	if (nvgpu_posix_fault_injection_handle_call(
+					nvgpu_file_ops_get_fstat_injection())) {
 		return -1;
 	}
 #endif
@@ -57,7 +61,8 @@ int nvgpu_fstat(int fd, struct stat *buf)
 ssize_t nvgpu_fread(int fildes, void* buf, size_t nbytes)
 {
 #ifdef NVGPU_UNITTEST_FAULT_INJECTION_ENABLEMENT
-	if (nvgpu_posix_fault_injection_handle_call(&fread_op)) {
+	if (nvgpu_posix_fault_injection_handle_call(
+					nvgpu_file_ops_get_fread_injection())) {
 		errno = -1;
 		return -1;
 	}

drivers/gpu/nvgpu/os/posix/kmem.c

@@ -47,11 +47,12 @@ static nvgpu_atomic_t kmem_cache_id;
 #endif
 
 #ifdef NVGPU_UNITTEST_FAULT_INJECTION_ENABLEMENT
-_Thread_local struct nvgpu_posix_fault_inj kmem_fi;
-
 struct nvgpu_posix_fault_inj *nvgpu_kmem_get_fault_injection(void)
 {
-	return &kmem_fi;
+	struct nvgpu_posix_fault_inj_container *c =
+			nvgpu_posix_fault_injection_get_container();
+
+	return &c->kmem_fi;
 }
 #endif
 
@@ -63,7 +64,8 @@ struct nvgpu_kmem_cache *nvgpu_kmem_cache_create(struct gk20a *g, size_t size)
 {
 	struct nvgpu_kmem_cache *cache;
 #ifdef NVGPU_UNITTEST_FAULT_INJECTION_ENABLEMENT
-	if (nvgpu_posix_fault_injection_handle_call(&kmem_fi)) {
+	if (nvgpu_posix_fault_injection_handle_call(
+					nvgpu_kmem_get_fault_injection())) {
 		return NULL;
 	}
 #endif
@@ -95,7 +97,8 @@ void *nvgpu_kmem_cache_alloc(struct nvgpu_kmem_cache *cache)
 	void *ptr;
 
 #ifdef NVGPU_UNITTEST_FAULT_INJECTION_ENABLEMENT
-	if (nvgpu_posix_fault_injection_handle_call(&kmem_fi)) {
+	if (nvgpu_posix_fault_injection_handle_call(
+					nvgpu_kmem_get_fault_injection())) {
 		return NULL;
 	}
 #endif
@@ -119,7 +122,8 @@ void *nvgpu_kmalloc_impl(struct gk20a *g, size_t size, void *ip)
 	void *ptr;
 
 #ifdef NVGPU_UNITTEST_FAULT_INJECTION_ENABLEMENT
-	if (nvgpu_posix_fault_injection_handle_call(&kmem_fi)) {
+	if (nvgpu_posix_fault_injection_handle_call(
+					nvgpu_kmem_get_fault_injection())) {
 		return NULL;
 	}
 #endif
@@ -145,7 +149,8 @@ void *nvgpu_kzalloc_impl(struct gk20a *g, size_t size, void *ip)
 	void *ptr;
 
 #ifdef NVGPU_UNITTEST_FAULT_INJECTION_ENABLEMENT
-	if (nvgpu_posix_fault_injection_handle_call(&kmem_fi)) {
+	if (nvgpu_posix_fault_injection_handle_call(
+					nvgpu_kmem_get_fault_injection())) {
 		return NULL;
 	}
 #endif
@@ -164,7 +169,8 @@ void *nvgpu_kcalloc_impl(struct gk20a *g, size_t n, size_t size, void *ip)
 	void *ptr;
 
 #ifdef NVGPU_UNITTEST_FAULT_INJECTION_ENABLEMENT
-	if (nvgpu_posix_fault_injection_handle_call(&kmem_fi)) {
+	if (nvgpu_posix_fault_injection_handle_call(
+					nvgpu_kmem_get_fault_injection())) {
 		return NULL;
 	}
 #endif
@@ -215,7 +221,8 @@ void nvgpu_big_free(struct gk20a *g, void *p)
 int nvgpu_kmem_init(struct gk20a *g)
 {
 #ifdef NVGPU_UNITTEST_FAULT_INJECTION_ENABLEMENT
-	if (nvgpu_posix_fault_injection_handle_call(&kmem_fi)) {
+	if (nvgpu_posix_fault_injection_handle_call(
+					nvgpu_kmem_get_fault_injection())) {
 		return -ENOMEM;
 	}
 #endif

drivers/gpu/nvgpu/os/posix/nvgpu.c

@@ -42,11 +42,12 @@
 #endif
 
 #ifdef NVGPU_UNITTEST_FAULT_INJECTION_ENABLEMENT
-_Thread_local struct nvgpu_posix_fault_inj nvgpu_fi;
-
 struct nvgpu_posix_fault_inj *nvgpu_nvgpu_get_fault_injection(void)
 {
-	return &nvgpu_fi;
+	struct nvgpu_posix_fault_inj_container *c =
+			nvgpu_posix_fault_injection_get_container();
+
+	return &c->nvgpu_fi;
 }
 #endif
 
@@ -142,7 +143,8 @@ void gk20a_idle_nosuspend(struct gk20a *g)
 int gk20a_busy(struct gk20a *g)
 {
 #ifdef NVGPU_UNITTEST_FAULT_INJECTION_ENABLEMENT
-	if (nvgpu_posix_fault_injection_handle_call(&nvgpu_fi)) {
+	if (nvgpu_posix_fault_injection_handle_call(
+					nvgpu_nvgpu_get_fault_injection())) {
 		return -ENODEV;
 	}
 #endif
@@ -171,7 +173,8 @@ struct gk20a *nvgpu_posix_probe(void)
 	struct nvgpu_os_posix *p;
 
 #ifdef NVGPU_UNITTEST_FAULT_INJECTION_ENABLEMENT
-	if (nvgpu_posix_fault_injection_handle_call(&nvgpu_fi)) {
+	if (nvgpu_posix_fault_injection_handle_call(
+					nvgpu_nvgpu_get_fault_injection())) {
 		return NULL;
 	}
 #endif

drivers/gpu/nvgpu/os/posix/posix-dma.c

@@ -37,14 +37,12 @@
 #include <nvgpu/posix/posix_vidmem.h>
 #include "os_posix.h"
 
-_Thread_local struct nvgpu_posix_fault_inj dma_fi = {
-						     .enabled = false,
-						     .counter = 0U,
-						    };
-
 struct nvgpu_posix_fault_inj *nvgpu_dma_alloc_get_fault_injection(void)
 {
-	return &dma_fi;
+	struct nvgpu_posix_fault_inj_container *c =
+			nvgpu_posix_fault_injection_get_container();
+
+	return &c->dma_fi;
 }
 
 /*
@@ -57,7 +55,8 @@ static int __nvgpu_do_dma_alloc(struct gk20a *g, unsigned long flags,
 {
 	void *memory;
 #ifdef NVGPU_UNITTEST_FAULT_INJECTION_ENABLEMENT
-	if (nvgpu_posix_fault_injection_handle_call(&dma_fi)) {
+	if (nvgpu_posix_fault_injection_handle_call(
+				nvgpu_dma_alloc_get_fault_injection())) {
 		return -ENOMEM;
 	}
 #endif
@@ -134,7 +133,8 @@ int nvgpu_dma_alloc_flags_vid_at(struct gk20a *g, unsigned long flags,
 	u64 addr;
 	int err;
 #ifdef NVGPU_UNITTEST_FAULT_INJECTION_ENABLEMENT
-	if (nvgpu_posix_fault_injection_handle_call(&dma_fi)) {
+	if (nvgpu_posix_fault_injection_handle_call(
+				nvgpu_dma_alloc_get_fault_injection())) {
 		return -ENOMEM;
 	}
 #endif

drivers/gpu/nvgpu/os/posix/posix-fault-injection.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2018-2019, NVIDIA CORPORATION.  All rights reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -24,10 +24,17 @@
 #include <stdbool.h>
 #include <nvgpu/posix/posix-fault-injection.h>
 
-void nvgpu_posix_init_fault_injection(struct nvgpu_posix_fault_inj *fi)
+_Thread_local struct nvgpu_posix_fault_inj_container *thread_fi;
+
+void nvgpu_posix_init_fault_injection(struct nvgpu_posix_fault_inj_container *c)
 {
-	fi->enabled = false;
+	thread_fi = c;
-	fi->counter = 0U;
+}
+
+struct nvgpu_posix_fault_inj_container
+	*nvgpu_posix_fault_injection_get_container(void)
+{
+	return thread_fi;
 }
 
 void nvgpu_posix_enable_fault_injection(struct nvgpu_posix_fault_inj *fi,

drivers/gpu/nvgpu/os/posix/queue.c

@@ -31,14 +31,12 @@
 #endif
 
 #ifdef NVGPU_UNITTEST_FAULT_INJECTION_ENABLEMENT
-_Thread_local struct nvgpu_posix_fault_inj queue_out_fi = {
-	.enabled = false,
-	.counter = 0U,
-};
-
 struct nvgpu_posix_fault_inj *nvgpu_queue_out_get_fault_injection(void)
 {
-	return &queue_out_fi;
+	struct nvgpu_posix_fault_inj_container *c =
+			nvgpu_posix_fault_injection_get_container();
+
+	return &c->queue_out_fi;
 }
 #endif
 
@@ -206,7 +204,8 @@ int nvgpu_queue_out_locked(struct nvgpu_queue *queue, void *buf,
 		unsigned int len, struct nvgpu_mutex *lock)
 {
 #ifdef NVGPU_UNITTEST_FAULT_INJECTION_ENABLEMENT
-	if (nvgpu_posix_fault_injection_handle_call(&queue_out_fi)) {
+	if (nvgpu_posix_fault_injection_handle_call(
+				nvgpu_queue_out_get_fault_injection())) {
 		return -1;
 	}
 #endif

drivers/gpu/nvgpu/os/posix/thread.c

@@ -28,12 +28,12 @@
 #endif
 
 #ifdef NVGPU_UNITTEST_FAULT_INJECTION_ENABLEMENT
-_Thread_local struct nvgpu_posix_fault_inj thread_fi = {
-					.enabled = false,.counter = 0U,};
-
 struct nvgpu_posix_fault_inj *nvgpu_thread_get_fault_injection(void)
 {
-	return &thread_fi;
+	struct nvgpu_posix_fault_inj_container *c =
+			nvgpu_posix_fault_injection_get_container();
+
+	return &c->thread_fi;
 }
 #endif
 
@@ -52,6 +52,11 @@ static void *nvgpu_posix_thread_wrapper(void *data)
 	struct nvgpu_posix_thread_data *nvgpu =
 				(struct nvgpu_posix_thread_data *)data;
 
+#ifdef NVGPU_UNITTEST_FAULT_INJECTION_ENABLEMENT
+	/* setup the fault injection container from the parent */
+	nvgpu_posix_init_fault_injection(nvgpu->fi_container);
+#endif
+
 	ret = nvgpu->fn(nvgpu->data);
 
 	if (ret != 0L) {
@@ -75,7 +80,8 @@ int nvgpu_thread_create(struct nvgpu_thread *thread,
 	int ret;
 
 #ifdef NVGPU_UNITTEST_FAULT_INJECTION_ENABLEMENT
-	if (nvgpu_posix_fault_injection_handle_call(&thread_fi)) {
+	if (nvgpu_posix_fault_injection_handle_call(
+				nvgpu_thread_get_fault_injection())) {
 		return -EINVAL;
 	}
 #endif
@@ -93,6 +99,11 @@ int nvgpu_thread_create(struct nvgpu_thread *thread,
 
 	thread->nvgpu.data = data;
 	thread->nvgpu.fn = threadfn;
+#ifdef NVGPU_UNITTEST_FAULT_INJECTION_ENABLEMENT
+	/* pass the fault injection container to the child */
+	thread->nvgpu.fi_container =
+				nvgpu_posix_fault_injection_get_container();
+#endif
 
 	nvgpu_atomic_set(&thread->running, 1);
 	ret = pthread_attr_init(&attr);
@@ -143,6 +154,11 @@ int nvgpu_thread_create_priority(struct nvgpu_thread *thread,
 
 	thread->nvgpu.data = data;
 	thread->nvgpu.fn = threadfn;
+#ifdef NVGPU_UNITTEST_FAULT_INJECTION_ENABLEMENT
+	/* pass the fault injection container to the child */
+	thread->nvgpu.fi_container =
+				nvgpu_posix_fault_injection_get_container();
+#endif
 
 	nvgpu_atomic_set(&thread->running, 1);
 
@@ -216,7 +232,8 @@ void nvgpu_thread_stop_graceful(struct nvgpu_thread *thread,
 bool nvgpu_thread_should_stop(struct nvgpu_thread *thread)
 {
 #ifdef NVGPU_UNITTEST_FAULT_INJECTION_ENABLEMENT
-	if (nvgpu_posix_fault_injection_handle_call(&thread_fi)) {
+	if (nvgpu_posix_fault_injection_handle_call(
+				nvgpu_thread_get_fault_injection())) {
 		return true;
 	}
 #endif

userspace/include/unit/core.h

@@ -28,6 +28,7 @@ struct unit_modules;
 struct unit_results;
 
 struct gk20a;
+struct nvgpu_posix_fault_inj_container;
 
 /*
  * The core unit testing framework data structure. Keeps track of global state
@@ -54,7 +55,12 @@ struct unit_fw {
 	struct {
 		struct gk20a	*(*nvgpu_posix_probe)(void);
 		void		 (*nvgpu_posix_cleanup)(struct gk20a *g);
+		void		 (*nvgpu_posix_init_fault_injection)
+				    (struct nvgpu_posix_fault_inj_container *c);
+		void		 (*nvgpu_posix_init_fault_injection_qnx)
+				    (struct nvgpu_posix_fault_inj_container *c);
 	} nvgpu;
+	void			 *nvgpu_qnx_ut;
 };
 
 int core_load_nvgpu(struct unit_fw *fw);

userspace/src/exec.c

@@ -36,6 +36,7 @@
 #include <unit/results.h>
 
 #include <nvgpu/posix/probe.h>
+#include <nvgpu/posix/posix-fault-injection.h>
 
 /*
  * Sempaphore to limit the number of threads
@@ -58,9 +59,28 @@ static void *core_exec_module(void *module_param)
 	unsigned int i;
 	struct unit_module *module = (struct unit_module *) module_param;
 	struct gk20a *g;
+	struct nvgpu_posix_fault_inj_container *fi_container = NULL;
 	clock_t begin;
 	double time_spent;
 
+	/*
+	 * Setup fault injection first so that faults can be injected in
+	 * nvgpu_posix_probe().
+	 */
+	fi_container = (struct nvgpu_posix_fault_inj_container *)
+			malloc(sizeof(struct nvgpu_posix_fault_inj_container));
+	if (fi_container == NULL) {
+		core_msg_color(module->fw, C_RED,
+				"  failed to allocate fault inj: Module %s\n",
+				module->name);
+		goto thread_exit;
+	}
+	memset(fi_container, 0, sizeof(struct nvgpu_posix_fault_inj_container));
+	module->fw->nvgpu.nvgpu_posix_init_fault_injection(fi_container);
+	if (module->fw->nvgpu.nvgpu_posix_init_fault_injection_qnx != NULL) {
+		module->fw->nvgpu.nvgpu_posix_init_fault_injection_qnx(fi_container);
+	}
+
 	g = module->fw->nvgpu.nvgpu_posix_probe();
 
 	if (!g) {
@@ -111,6 +131,9 @@ static void *core_exec_module(void *module_param)
 	core_vbs(module->fw, 1, "Module completed: %s (execution time: %f)\n",
 		 module->name, time_spent);
 thread_exit:
+	if (fi_container != NULL) {
+		free(fi_container);
+	}
 	sem_post(&unit_thread_semaphore);
 	return NULL;
 }

userspace/src/nvgpu.c

@@ -81,5 +81,34 @@ int core_load_nvgpu(struct unit_fw *fw)
 		return -1;
 	}
 
+	fw->nvgpu.nvgpu_posix_init_fault_injection = dlsym(fw->nvgpu_so,
+					    "nvgpu_posix_init_fault_injection");
+	if (fw->nvgpu.nvgpu_posix_init_fault_injection == NULL) {
+		msg = dlerror();
+		core_err(fw, "Failed to resolve nvgpu_posix_init_fault_injection: %s\n",
+			 msg);
+		return -1;
+	}
+
+	if (fw->args->is_qnx != 0) {
+		fw->nvgpu_qnx_ut = dlopen("libnvgpu_ut.so", flag);
+		if (fw->nvgpu_qnx_ut == NULL) {
+			msg = dlerror();
+			core_err(fw, "Failed to load nvgpu_ut: %s\n", msg);
+			return -1;
+		}
+		fw->nvgpu.nvgpu_posix_init_fault_injection_qnx =
+					dlsym(fw->nvgpu_qnx_ut,
+					      "nvgpu_posix_init_fault_injection");
+		if (fw->nvgpu.nvgpu_posix_init_fault_injection_qnx == NULL) {
+			msg = dlerror();
+			core_err(fw, "Failed to resolve nvgpu_posix_init_fault_injection: %s\n",
+				 msg);
+			return -1;
+		}
+	} else {
+		fw->nvgpu.nvgpu_posix_init_fault_injection_qnx = NULL;
+	}
+
 	return 0;
 }