diff --git a/nvsched/include/nvs/nvs-control-interface.h b/nvsched/include/nvs/nvs-control-interface.h new file mode 100644 index 000000000..ce6b96578 --- /dev/null +++ b/nvsched/include/nvs/nvs-control-interface.h @@ -0,0 +1,340 @@ +/* + * Copyright (c) 2022 NVIDIA Corporation. All rights reserved. + * + * NVIDIA Corporation and its licensors retain all intellectual property + * and proprietary rights in and to this software, related documentation + * and any modifications thereto. Any use, reproduction, disclosure or + * distribution of this software and related documentation without an express + * license agreement from NVIDIA Corporation is strictly prohibited. + */ + +#ifndef NVS_CONTROL_INTERFACE_H +#define NVS_CONTROL_INTERFACE_H + +/** + * @file + * @brief NVIDIA GPU domain scheduler asynchronous messaging interface + */ + +#include "nvs-control-messages.h" + +#if defined(__cplusplus) +#include +#else +#include +#endif + +#if defined(__cplusplus) +extern "C" { +#endif + + +/** + * @brief Value for nvs_domain_msg_fifo_control::get to indicate disabled + * flow control. + * + * @since 1.0.0 + */ +#define NVS_DOMAIN_MSG_FIFO_CONTROL_GET_FLOW_CTRL_DISABLED (0xFFFFFFFFU) + +/** + * @brief Least significant bit of field PUT in nvs_domain_msg_fifo_control::put_revolutions + */ +#define NVS_DOMAIN_MSG_FIFO_CONTROL_PUT_REVOLUTIONS_PUT_LSB (0U) + +/** + * @brief Most significant bit of field PUT in nvs_domain_msg_fifo_control::put_revolutions + */ +#define NVS_DOMAIN_MSG_FIFO_CONTROL_PUT_REVOLUTIONS_PUT_MSB (31U) + +/** + * @brief Least significant bit of field REVOLUTIONS in nvs_domain_msg_fifo_control::put_revolutions + */ +#define NVS_DOMAIN_MSG_FIFO_CONTROL_PUT_REVOLUTIONS_REVOLUTIONS_LSB (32U) + +/** + * @brief Most significant bit of field REVOLUTIONS in nvs_domain_msg_fifo_control::put_revolutions + */ +#define NVS_DOMAIN_MSG_FIFO_CONTROL_PUT_REVOLUTIONS_REVOLUTIONS_MSB (63U) + +/** + * @brief The FIFO ring buffer control block + * + * FIFO ring buffer initialization + * ------------------------------- + * + * The FIFO ring buffer control block should be initialized on allocation as + * follows: + * - client --> scheduler: + * - all zeroes + * - scheduler --> client: + * - nvs_domain_msg_fifo_control::get = + * #NVS_DOMAIN_MSG_FIFO_CONTROL_GET_FLOW_CTRL_DISABLED. + * - everything else zeroes + * + * The FIFO size in entries is calculated as follows from the buffer size: + * + * @code + * num_messages = (buffer_size - sizeof(nvs_domain_msg_fifo_control)) / sizeof(nvs_domain_message) + * @endcode + * + * With the current data structure layouts, this formula with sizeof substitutions is as: + * + * @code + * num_messages = (buffer_size - 128U) / 64U + * @endcode + * + * For example, the number of messages in the FIFO is 1022 for a buffer of size 64K. + * + * + * Adding a message in the FIFO ring buffer + * ---------------------------------------- + * + * The sender is suggested to maintain the following variables locally: + * - @c local_put --- local copy of put index + * - @c local_revolution_count --- local copy of revolution count + * - @c local_num_dropped_messages --- local copy of number of dropped messages + * - @c local_fifo_size_entries --- local copy of the FIFO ring buffer size + * + * When the sender is initialized, the local variables should be initialized + * from the fields in #nvs_domain_msg_fifo_control, except for the FIFO size. The + * FIFO size is received from nvgpu-rm as part of the physical buffer + * parameters. + * + * The sequence for adding a message in the FIFO ring buffer is as follows: + * + * 1. Read nvs_domain_msg_fifo_control::get as @c get + * 2. Determine whether there is space for the message. There are two cases: + * - @c get == #NVS_DOMAIN_MSG_FIFO_CONTROL_GET_FLOW_CTRL_DISABLED: flow control disabled, space always available + * - otherwise: space is available when + * ((local_put + 1) % local_fifo_size_entries) != get + * 3. If out of space, then: + * - increment @c local_num_dropped_messages + * - write it to nvs_domain_msg_fifo_control::num_dropped_messages + * - do not proceed with this sequence + * - Notes: + * - scheduler --> client: nvs_domain_msg_fifo_control::num_dropped_messages + * indicates that messages were dropped. Logging an + * overflow event may be applicable. + * - client --> scheduler: The usermode client should return an error. + * 4. Read-write memory barrier + * 5. Write the message to nvs_domain_msg_fifo::messages[local_put] + * - NOTE: if the message size is less than the size of the array entry, it + * should be appended with 0 to fill the entry. This allows extending the + * messages with new fields in later protocol versions in a backwards + * compatible manner. + * - NOTE: this update does not need to be atomic + * 6. Write memory barrier + * 7. Increment @c local_put (modulo fifo size). In case of a wrap-around, + * increment also @c local_revolution_count + * 8. Write local_put | (local_revolution_count << 32) + * to nvs_domain_msg_fifo_control::put_revolutions. This write + * should be atomic 64-bit write. + * + * + * Receiving a message in the FIFO ring buffer (read-write access client) + * --------------------------------------------------------------- + * + * The receiver is suggested to maintain the following variables locally: + * - @c local_get --- local copy of get index + * - @c local_fifo_size_entries --- local copy of the FIFO ring buffer size + * + * When the reader is initialized, it should: + * + * 1. atomic read nvs_domain_msg_fifo_control::put_revolutions and store the + * 32-bit lower bits as @c local_get + * 2. store @c local_get to nvs_domain_msg_fifo_control::get. This enables + * flow control. + * + * When the reader exits, it should write + * #NVS_DOMAIN_MSG_FIFO_CONTROL_GET_FLOW_CTRL_DISABLED in + * nvs_domain_msg_fifo_control::get. This disables flow control. Further, + * nvgpu-rm should also write + * #NVS_DOMAIN_MSG_FIFO_CONTROL_GET_FLOW_CTRL_DISABLED in + * nvs_domain_msg_fifo_control::get when the R/W reader client exits. This + * is to ensure that abnormal client exit (e.g., process crash) disables flow + * control. + * + * The sequence for reading a message: + * + * 1. Atomic read the bottom 32 bits (or whole field) of + * nvs_domain_msg_fifo_control::put_revolutions as @c put + * 2. If local_get == put, there are no more messages. Exit this + * sequence. + * 3. Read memory barrier + * 4. Read the message from nvs_domain_msg_fifo::messages[local_get] + * (non-atomic read ok) + * 5. Read-write memory barrier + * 6. Increment local_get (mod FIFO size) + * 7. Atomic write nvs_domain_msg_fifo_control::get + * + * + * Receiving a message in the FIFO ring buffer (read-only access client) + * --------------------------------------------------------------- + * + * NOTE: The read-only reader client should not be used for + * safety-critical operation. It does not have flow control and it is subject to + * FIFO overruns. The read-only client is intended for diagnostics and tracing. + * + * The receiver is suggested to maintain the following variables locally: + * - @c local_get --- local copy of get index + * - @c local_revolutions --- local copy of FIFO revolutions + * - @c local_fifo_size_entries --- local copy of the FIFO ring buffer size + * + * When the reader is initialized, it should: + * + * 1. atomic read nvs_domain_msg_fifo_control::put_revolutions and store the + * lower 32 bits as @c local_get, and the upper 32 bits as @c + * local_revolutions + * + * The sequence for reading a message: + * + * 1. Atomic read nvs_domain_msg_fifo_control::put_revolutions as @c put and + * @c revolutions + * 2. Determine whether the reader is more than a full revolution behind the + * writer. One way to do this is to calculate the total number of messages + * read and written, and then calculate the cyclic difference of the + * totals. + * @code + * cycleSize = FIFO_entries * (1 << 32) ; num messages until 'revolutions' wraps around + * messagesRead = local_get + (local_revolutions * FIFO_entries) + * messagesWritten = put + (revolutions * FIFO_entries) + * unreadMessages = (messagesWritten - messagesRead) mod cycleSize + * @endcode + * In case unreadMessages > FIFO_entries, a FIFO overrun has + * occurred. Note that in this formulation, care must be taken to avoid + * integer overflows during computation. + * + * It is up to the implementation what to do on FIFO overflow. Possibly, + * report an error and reset local_revolutions = revolutions and + * local_get = @c put. + * 3. If local_get == put, the are no unread messages. Exit this + * sequence. + * 4. Read memory barrier + * 5. Read the message from nvs_domain_msg_fifo::messages[local_get] + * (non-atomic read ok) + * 6. Read memory barrier + * 7. Atomic read nvs_domain_msg_fifo_control::put_revolutions as @c put and + * @c revolutions + * 8. Perform the FIFO overrun check again as in step 2. In case overrun is + * detected, then the message read on step 5 may have been overwritten + * while reading. + * + * It is up to the implementation what to do on FIFO overflow. Possibly, + * report an error and reset local_revolutions = revolutions and + * local_get = @c put. + * 9. Increment local_get (mod FIFO_entries). If there is a wrap-around, + * increment @c local_revolutions. + * + * @since 1.0.0 + */ +struct nvs_domain_msg_fifo_control { + + /** + * @brief Get index (updated by the RW consumer) + * + * @remark Special value + * #NVS_DOMAIN_MSG_FIFO_CONTROL_GET_FLOW_CTRL_DISABLED means no flow + * control (scheduler --> client buffers only) + * + * @since 1.0.0 + */ + uint32_t get; + + /** + * @brief Padding to fill up 64B + */ + uint32_t reserved0[15]; + + /** + * @brief Message put index and revolution count (updated by the producer) + * + * This member consists of two fields: + * + * - The lower 32 bits is the put index + * - The upper 32 bits is the revolution count, i.e., how many times the + * put index has wrapped around + * + * For example, this field would be incremented as follows for a FIFO of size 5: + * + * - 0x0000'0000'0000'0000 (initial value) + * - 0x0000'0000'0000'0001 + * - 0x0000'0000'0000'0002 + * - 0x0000'0000'0000'0003 + * - 0x0000'0000'0000'0004 + * - 0x0000'0001'0000'0000 (wrap-around, revolution count incremented) + * - 0x0000'0001'0000'0001 + * + * The intention of the revolution count is to provide read-only + * observers a mechanism to detect dropped messages. + * + * @sa NVS_DOMAIN_MSG_FIFO_CONTROL_PUT_REVOLUTIONS_PUT_LSB + * @sa NVS_DOMAIN_MSG_FIFO_CONTROL_PUT_REVOLUTIONS_PUT_MSB + * @sa NVS_DOMAIN_MSG_FIFO_CONTROL_PUT_REVOLUTIONS_REVOLUTIONS_LSB + * @sa NVS_DOMAIN_MSG_FIFO_CONTROL_PUT_REVOLUTIONS_REVOLUTIONS_MSB + * + * @since 1.0.0 + */ + uint64_t put_revolutions; + + /** + * @brief Number of dropped messages due to overrun (updated by the + * producer) + * + * @since 1.0.0 + */ + uint64_t num_dropped_messages; // number of lost messages due to buffer overrun + + /** + * @brief Padding to fill up 64B + */ + uint32_t reserved1[12]; +}; + +#if (defined(__cplusplus) && (__cplusplus >= 201103L)) +/* Double-check that the example in the documentation is correct */ +static_assert( + sizeof(nvs_domain_msg_fifo_control) == 128U, + "Verify the documented substitution (1)"); +static_assert( + sizeof(nvs_domain_message) == 64U, + "Verify the documented substitution (2)"); +static_assert( + (65536U - sizeof(nvs_domain_msg_fifo_control)) / sizeof(nvs_domain_message) == 1022U, + "Verify the documented example"); +#endif + +/** + * @brief The general FIFO ring buffer format + * + * The following FIFO ring buffers are specified: + * + * - client --> scheduler control request messages + * - scheduler --> client control response messages + * + * The FIFO must be aligned by 64 bytes. + * + * @since 1.0.0 + */ +struct nvs_domain_msg_fifo { + + /** + * @brief Message ring buffer control + * + * @since 1.0.0 + */ + struct nvs_domain_msg_fifo_control control; + + /** + * @brief Message ring buffer data + * + * @since 1.0.0 + */ + struct nvs_domain_message messages[]; +}; + +#if defined(__cplusplus) +} // extern "C" +#endif + +#endif diff --git a/nvsched/include/nvs/nvs-control-messages.h b/nvsched/include/nvs/nvs-control-messages.h new file mode 100644 index 000000000..5d0705ea1 --- /dev/null +++ b/nvsched/include/nvs/nvs-control-messages.h @@ -0,0 +1,438 @@ +/* + * Copyright (c) 2022 NVIDIA Corporation. All rights reserved. + * + * NVIDIA Corporation and its licensors retain all intellectual property + * and proprietary rights in and to this software, related documentation + * and any modifications thereto. Any use, reproduction, disclosure or + * distribution of this software and related documentation without an express + * license agreement from NVIDIA Corporation is strictly prohibited. + */ + +#ifndef NVS_CONTROL_MESSAGES_H +#define NVS_CONTROL_MESSAGES_H + +/** + * @file + * @brief NVIDIA GPU domain scheduler asynchronous message definitions + */ + +#if defined(__cplusplus) +#include +#else +#include +#endif + +#if defined(__cplusplus) +extern "C" { +#endif + +/** + * @brief Communication error + * + * @sa nvs_domain_msg_ctrl_error_resp + * + * @since 1.0.0 + */ +#define NVS_DOMAIN_MSG_TYPE_CTRL_ERROR (0U) + +/** + * @brief Unknown/undefined error + * + * @sa nvs_domain_msg_ctrl_error_resp::error_code + * + * @since 1.0.0 + */ +#define NVS_DOMAIN_MSG_CTRL_ERROR_UNKNOWN (0U) + +/** + * @brief Unhandled message + * + * The scheduler received a message that it does not know how to handle. + * + * @sa nvs_domain_msg_ctrl_error_resp::error_code + * + * @since 1.0.0 + */ +#define NVS_DOMAIN_MSG_CTRL_ERROR_UNHANDLED_MESSAGE (1U) + +/** + * @brief Communication error + * + * This control message is sent as a response to an erroneous request. + * + * @sa NVS_DOMAIN_MSG_TYPE_CTRL_ERROR + * + * @since 1.0.0 + */ +struct nvs_domain_msg_ctrl_error_resp { + + /** + * @brief Communication error code + * + * See NVS_DOMAIN_MSG_CTRL_ERROR_* error codes. + * + * @since 1.0.0 + */ + uint32_t error_code; +}; + +/** + * @brief Scheduler capability query + * + * @sa nvs_domain_msg_ctrl_get_caps_req + * @sa nvs_domain_msg_ctrl_get_caps_resp + * + * @since 1.0.0 + */ +#define NVS_DOMAIN_MSG_TYPE_CTRL_GET_CAPS_INFO (1U) + +/** + * @brief Scheduler capability query request + * + * The client version is used by the scheduler to check the client + * compatibility, following the semantic versioning specification. (See + * #NVS_DOMAIN_SCHED_VERSION_MAJOR for details.) + * + * @sa NVS_DOMAIN_MSG_TYPE_CTRL_GET_CAPS_INFO + * @sa nvs_domain_msg_ctrl_get_caps_resp::client_version_status + * + * + * @since 1.0.0 + */ +struct nvs_domain_msg_ctrl_get_caps_req { + + /** + * @brief Client major version + * + * @since 1.0.0 + */ + uint8_t client_version_major; + + /** + * @brief Client minor version + * + * @since 1.0.0 + */ + uint8_t client_version_minor; + + /** + * @brief Client patch level + * + * @since 1.0.0 + */ + uint8_t client_version_patch; +}; + +/** + * @brief Major version of the domain scheduler interface + * + * The versioning scheme follows the Semantic Versioning 2.0.0 + * specification. See https://semver.org/spec/v2.0.0.html + * + * @since 1.0.0 + */ +#define NVS_DOMAIN_SCHED_VERSION_MAJOR (1U) + +/** + * @brief Minor version of the domain scheduler interface + * + * The versioning scheme follows the Semantic Versioning 2.0.0 + * specification. See https://semver.org/spec/v2.0.0.html + * + * @since 1.0.0 + */ +#define NVS_DOMAIN_SCHED_VERSION_MINOR (0U) + +/** + * @brief Patch version of the domain scheduler interface + * + * The versioning scheme follows the Semantic Versioning 2.0.0 + * specification. See https://semver.org/spec/v2.0.0.html + * + * @since 1.0.0 + */ +#define NVS_DOMAIN_SCHED_VERSION_PATCH (0U) + +/** + * @brief Client version check failed + * + * The client should not proceed with scheduler communication. The + * scheduler is not expected to be compatible. + */ +#define NVS_DOMAIN_MSG_CTRL_GET_CAPS_RESP_CLIENT_VERSION_STATUS_FAILED (0U) + +/** + * @brief Client version check passed + */ +#define NVS_DOMAIN_MSG_CTRL_GET_CAPS_RESP_CLIENT_VERSION_STATUS_OK (1U) + +/** + * @brief Scheduler capability query response + * + * @sa NVS_DOMAIN_MSG_TYPE_CTRL_GET_CAPS_INFO + * + * @since 1.0.0 + */ +struct nvs_domain_msg_ctrl_get_caps_resp { + + /** + * @brief Scheduler major version + * + * @since 1.0.0 + */ + uint8_t sched_version_major; + + /** + * @brief Scheduler minor version + * + * @since 1.0.0 + */ + uint8_t sched_version_minor; + + /** + * @brief Scheduler patch level + * + * @since 1.0.0 + */ + uint8_t sched_version_patch; + + /** + * @brief Client version check status + * + * This field indicates the scheduler-side check for client + * version compatibility. In general, the version numbering + * scheme follows the semantic versioning specification (see + * #NVS_DOMAIN_SCHED_VERSION_MAJOR). Additionally, the scheduler is + * allowed to fail this check for known incompatible or otherwise bad + * versions of the client. + * + * In general: + * - When the client and scheduler major version numbers match, + * #NVS_DOMAIN_MSG_CTRL_GET_CAPS_RESP_CLIENT_VERSION_STATUS_OK + * is returned. + * - When the client and scheduler major versions do not match, + * #NVS_DOMAIN_MSG_CTRL_GET_CAPS_RESP_CLIENT_VERSION_STATUS_FAILED + * is returned. Future note: If the client supports multiple + * scheduler major versions, the client is allowed to send + * #nvs_domain_msg_ctrl_get_caps_req again with a compatible + * version number. + * + * @since 1.0.0 + */ + uint8_t client_version_status; +}; + +/** + * @brief Switch to another domain + * + * @sa nvs_domain_msg_ctrl_switch_domain_req + * @sa nvs_domain_msg_ctrl_switch_domain_resp + * + * @since 1.0.0 + */ +#define NVS_DOMAIN_MSG_TYPE_CTRL_SWITCH_DOMAIN (2U) + +/** + * @brief Pseudo-domain ID for all TSGs over all domains + * + * @sa nvs_domain_msg_ctrl_switch_domain_req::domain_id + * + * @since 1.0.0 + */ +#define NVS_DOMAIN_CTRL_DOMAIN_ID_ALL (~(uint64_t)0U) + +/** + * @brief Scheduler domain switch request + * + * @sa NVS_DOMAIN_MSG_TYPE_CTRL_SWITCH_DOMAIN + * + * @since 1.0.0 + */ +struct nvs_domain_msg_ctrl_switch_domain_req { + + /** + * @brief Domain id + * + * @remark Domain id #NVS_DOMAIN_CTRL_DOMAIN_ID_ALL has a special meaning. This + * is a request to switch to a runlist that contains all TSGs in all domains. + * + * @since 1.0.0 + */ + uint64_t domain_id; +}; + +/** + * @brief Scheduler domain switch succeeded + * + * @sa nvs_domain_msg_ctrl_switch_domain_resp::status + * + * @since 1.0.0 + */ +#define NVS_DOMAIN_MSG_TYPE_CTRL_SWITCH_DOMAIN_STATUS_SUCCESS (0U) + +/** + * @brief Scheduler domain switch response + * + * @sa NVS_DOMAIN_MSG_TYPE_CTRL_SWITCH_DOMAIN + * + * @since 1.0.0 + */ +struct nvs_domain_msg_ctrl_switch_domain_resp { + + /** + * @brief Domain switch status + * + * @remark The domain scheduler may return a status code that + * is not listed below. In this case, domain switch failed and + * the status code is returned for diagnostic and debugging + * purposes. + * + * @sa NVS_DOMAIN_MSG_TYPE_CTRL_SWITCH_DOMAIN_STATUS_SUCCESS + * + * @since 1.0.0 + */ + uint8_t status; + + /** + * @brief Padding to ensure 8B alignment for the next field + * for 32/64-bit compatibility + */ + uint8_t reserved[7]; + + /** + * @brief Time it took to preempt and switch to the new domain (nanoseconds) + * + * @since 1.0.0 + */ + uint64_t switch_ns; +}; + + +/** + * @brief Maximum payload size for #nvs_domain_message. + * + * @sa nvs_domain_message::payload + * + * @since 1.0.0 + */ +#define NVS_DOMAIN_MESSAGE_MAX_PAYLOAD_SIZE (48U) + + +/** + * @brief Payload union + * + * Union that contains all defined messages. + * + * @since 1.0.0 + */ +union nvs_domain_msg_payload_union { + /** + * @brief Control response message: communication error + * @since 1.0.0 + */ + struct nvs_domain_msg_ctrl_error_resp resp_error; + + /** + * @brief Control request message: get_caps + * @since 1.0.0 + */ + struct nvs_domain_msg_ctrl_get_caps_req req_get_caps; + + /** + * @brief Control response message: get_caps + * @since 1.0.0 + */ + struct nvs_domain_msg_ctrl_get_caps_resp resp_get_caps; + + /** + * @brief Control request message: switch_domain + * @since 1.0.0 + */ + struct nvs_domain_msg_ctrl_switch_domain_req req_switch_domain; + + /** + * @brief Control response message: switch_domain + * @since 1.0.0 + */ + struct nvs_domain_msg_ctrl_switch_domain_resp resp_switch_domain; + + + /** + * @brief Raw message data + * + * @since 1.0.0 + */ + uint8_t raw_data[NVS_DOMAIN_MESSAGE_MAX_PAYLOAD_SIZE]; +}; + +#if (defined(__cplusplus) && (__cplusplus >= 201103L)) +static_assert( + sizeof(nvs_domain_msg_payload_union) == NVS_DOMAIN_MESSAGE_MAX_PAYLOAD_SIZE, + "Check for expected payload size"); +#endif + +/** + * @brief The generic NVS domain message envelope + * + * @since 1.0.0 + */ +struct nvs_domain_message { + + /** + * @brief FIFO-specific message type + * + * Message type defines: + * - control messages: NVS_DOMAIN_MSG_TYPE_CTRL_* + * + * @since 1.0.0 + */ + uint32_t type; + + /** + * @brief Client-provided message tag + * + * - **Request-response messages.** The client provides a tag + * which the scheduler copies to the corresponding + * response. + * + * @remark It is recommended that the client uses a sequence + * to generate the tags for easier request/response tracing. + * + * @since 1.0.0 + */ + uint32_t sequence_tag; + + /** + * @brief Message timestamp (nanoseconds) + * + * client --> scheduler: host CPU time (CLOCK_MONOTONIC) + * + * scheduler --> client: scheduler local time + * + * @remark The timestamp field is primarily intended for + * message tracing purposes for allowing the reconstruction of + * the timeline of events. For operational purposes such as + * measuring the request/response round-trips as a health + * indicator, the usermode software should use local CPU + * host-side clock sampling, instead. The GSP clock is not + * guaranteed to use the same time domain with the host + * CPU. Further, drift may occur between the host CPU and GSP + * time domains. + * + * @since 1.0.0 + */ + uint64_t timestamp_ns; + + /** + * @brief Payload union + * + * @since 1.0.0 + */ + union nvs_domain_msg_payload_union payload; +}; + +#if defined(__cplusplus) +} // extern "C" +#endif + +#endif