/**
 * @file        scp_private.c
 * @brief       Private interfaces for internal use by the SCP driver.
 *
 * Copyright (c) 2021-2022, NVIDIA CORPORATION. All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * * Redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer.
 * * Redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the distribution.
 * * Neither the name of NVIDIA CORPORATION nor the names of its
 * contributors may be used to endorse or promote products derived
 * from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#if !NVRISCV_FEATURE_SCP
#error "Attempting to build scp_private.c but the SCP driver is disabled!"
#endif // NVRISCV_FEATURE_SCP


/* Compiler headers */
#include <stdbool.h>                    // for bool, true, false
#include <stddef.h>                     // for size_t, NULL
#include <stdint.h>                     // for uintptr_t, uint64_t, etc.

/* Early FSP headers */
#include <misc/ct-assert.h>             // for HEADER_CHECK

/* Hardware headers */

/* Late FSP headers */
#include <error/common-errors.h>        // for error_t, common error codes
#if NVRISCV_DRIVER_SCP_SHORTCUT_ENABLED
#include <fbdma/fbdma.h>                // for fbdma_scp_to_extmem
#endif // NVRISCV_DRIVER_SCP_SHORTCUT_ENABLED
#include <fbdma/memutils.h>             // for memutils_riscv_pa_to_target_offset
#include <misc/macros.h>                // for INLINE_RFD
#include <misc/nvmisc_drf.h>            // for DRF macros

/* Module-specific FSP headers */
#include <scp/scp-errors.h>             // for SCP error codes
#include <scp/scp_common.h>             // for scp_register_index_t, etc.
#include <scp/scp_direct.h>             // for SCP primitives
#include <scp/scp_private.h>            // for SCP_SECRET_INDEX_CLEAR, etc.

/*
 * Compile-time check for FSP header files
 *   Each FSP header file contains a signature unique to that file, and the
 *   FSP project.  The HEADER_CHECK macro (contained in misc/ct-assert.h) can
 *   check for this signature.  If it does not exist, then the build will
 *   abort.
 *
 *   This is a trap for projects which have their own include files of the
 *   same names, but different contents.  This trap ensures that only the
 *   files from the FSP project, are built into the FSP source code.
 */
HEADER_CHECK(FSP__ERROR__COMMON_ERRORS_H)
#if NVRISCV_DRIVER_SCP_SHORTCUT_ENABLED
HEADER_CHECK(FSP__FBDMA__FBDMA_H)
#endif // NVRISCV_DRIVER_SCP_SHORTCUT_ENABLED
HEADER_CHECK(FSP__FBDMA__MEMUTILS_H)
HEADER_CHECK(FSP__MISC__MACROS_H)
HEADER_CHECK(FSP__MISC__NVMISC_DRF_H)
HEADER_CHECK(FSP__SCP__SCP_ERRORS_H)
HEADER_CHECK(FSP__SCP__SCP_COMMON_H)
HEADER_CHECK(FSP__SCP__SCP_DIRECT_H)
HEADER_CHECK(FSP__SCP__SCP_PRIVATE_H)


/**
 * @brief The chmod value for a fully-open register ACL.
 */
#define SCP_REGISTER_ACL_ALL 0x1FU

/**
 * @brief The maximum supported size for a single SCPDMA transfer, in bytes.
 */
#define SCP_TRANSFER_LIMIT 256U


//
// Prototypes for local helper functions.
// See definitions for details.
//
static error_t s_scp_Unlock_Reg(scp_register_index_t reg_index);
static error_t s_scp_Zero_Reg(scp_register_index_t reg_index);


/**
 * @copydoc scp_pvt_Clear_Reg
 */
error_t
scp_pvt_Clear_Reg
(
    scp_register_index_t reg_index
)
{
    error_t status;

    // Clear the register's contents.
    status = s_scp_Zero_Reg(reg_index);
    if (status != E_SUCCESS)
    {
        return status;
    }

    //
    // In secure contexts, s_scp_Zero_Reg() will have locked out insecure access
    // to the target register, so restore it here. In insecure contexts, the
    // register is assumed to already be accessible (or else s_scp_Zero_Reg()
    // would have triggered an ACL violation).
    //
    if (scp_pvt_Is_Secure())
    {
        status = s_scp_Unlock_Reg(reg_index);
        if (status != E_SUCCESS)
        {
            return status;
        }
    }

    // Register cleared successfully.
    return E_SUCCESS;
}

/**
 * @copydoc scp_pvt_Clear_All
 */
error_t
scp_pvt_Clear_All(void)
{
    error_t                 current_status, final_status = E_SUCCESS;
    scp_register_index_t    reg_index;

    //
    // Clear each register in turn. To ensure we always clear as much data as
    // possible, we avoid exiting immediately on failure here and instead
    // simply track any errors for later.
    //
    for (reg_index = SCP_REGISTER_INDEX_0;
         reg_index < SCP_REGISTER_INDEX_COUNT;
         reg_index++)
    {
        current_status = scp_pvt_Clear_Reg(reg_index);
        if (current_status != E_SUCCESS)
        {
            final_status = current_status;
        }
    }

    // Registers cleared.
    return final_status;
}

/**
 * @copydoc scp_pvt_Unlock_All
 */
error_t
scp_pvt_Unlock_All(void)
{
    error_t                 current_status, final_status = E_SUCCESS;
    scp_register_index_t    reg_index;

    //
    // Unlock each register in turn. To ensure we always unlock as many GPRs as
    // possible, we avoid exiting immediately on failure here and instead
    // simply track any errors for later.
    //
    for (reg_index = SCP_REGISTER_INDEX_0;
         reg_index < SCP_REGISTER_INDEX_COUNT;
         reg_index++)
    {
        current_status = s_scp_Unlock_Reg(reg_index);
        if (current_status != E_SUCCESS)
        {
            final_status = current_status;
        }
    }

    // Registers unlocked.
    return final_status;
}

/**
 * @copydoc scp_pvt_Zero_All
 */
error_t
scp_pvt_Zero_All(void)
{
    error_t                 current_status, final_status = E_SUCCESS;
    scp_register_index_t    reg_index;

    //
    // Zero each register in turn. To ensure we always zero as many GPRs as
    // possible, we avoid exiting immediately on failure here and instead
    // simply track any errors for later.
    //
    for (reg_index = SCP_REGISTER_INDEX_0;
         reg_index < SCP_REGISTER_INDEX_COUNT;
         reg_index++)
    {
        current_status = s_scp_Zero_Reg(reg_index);
        if (current_status != E_SUCCESS)
        {
            final_status = current_status;
        }
    }

    // Registers zeroed.
    return final_status;
}

/**
 * @copydoc scp_pvt_Load_Buffer
 */
error_t
scp_pvt_Load_Buffer
(
    uintptr_t               source_pa,
    scp_register_index_t    reg_index
)
{
    error_t status;

    // Load into the selected register.
    scp_load(source_pa, reg_index);

    // Wait for completion.
    scp_wait_dma();

    // Check for any failures.
    status = scp_check_dma(true);
    if (status == E_FAULT)
    {
        // Bubble up IO-PMP errors.
        return status;
    }
    else if (status != E_SUCCESS)
    {
        // No other errors are expected.
        return E_SCP_GENERIC;
    }

    // Data was loaded successfully.
    return E_SUCCESS;
}

/**
 * @copydoc scp_pvt_Store_Buffer
 */
error_t
scp_pvt_Store_Buffer
(
    scp_register_index_t    reg_index,
    uintptr_t               dest_pa
)
{
    error_t status;

    // Write to the destination buffer.
    scp_store(reg_index, dest_pa);

#if NVRISCV_DRIVER_SCP_SHORTCUT_ENABLED
    riscv_mem_target_t  dest_target;
    uint64_t            dest_offset;

    // Obtain the global target/offset pair for the destination buffer.
    if (memutils_riscv_pa_to_target_offset(dest_pa, &dest_target, &dest_offset)
            != E_SUCCESS)
    {
        // This should have been validated by the caller.
        return E_SCP_GENERIC;
    }

    // Complete the follow-up transfer to external memory, if applicable.
    if ((dest_target == RISCV_MEM_TARGET_FBGPA) ||
        (dest_target == RISCV_MEM_TARGET_SYSGPA))
    {
        if (fbdma_scp_to_extmem(dest_offset, SCP_REGISTER_SIZE,
                g_scp_state.dma_index) != E_SUCCESS)
        {
            //
            // This should never happen so long as the caller has properly
            // sanitized dest_pa and ensured that the shortcut-DMA path has
            // been configured.
            //
            return E_SCP_GENERIC;
        }
    }
#endif // NVRISCV_DRIVER_SCP_SHORTCUT_ENABLED

    // Wait for completion.
    scp_wait_dma();

    // Check for any failures.
    status = scp_check_dma(true);
    if (status == E_FAULT)
    {
        // Bubble up IO-PMP errors.
        return status;
    }
    else if (status != E_SUCCESS)
    {
        // No other errors are expected.
        return E_SCP_GENERIC;
    }

    // Data was stored successfully.
    return E_SUCCESS;
}

/**
 * @copydoc scp_pvt_Validate_Buffer
 */
error_t
scp_pvt_Validate_Buffer
(
    uintptr_t   base_pa,
    size_t      size,
    bool        b_allow_external
)
{
    riscv_mem_target_t  base_target, end_target;
    uint64_t            base_offset;

    // Check base address for zero/NULL.
    if (base_pa == (uintptr_t)NULL)
    {
        return E_SCP_INVALID_POINTER;
    }

    // Convert to global target/offset pair.
    if (memutils_riscv_pa_to_target_offset(base_pa, &base_target, &base_offset)
            != E_SUCCESS)
    {
        return E_SCP_INVALID_BASE;
    }

    // Ensure the target memory region is supported.
    switch (base_target)
    {
        // IMEM and DMEM are always supported.
        case RISCV_MEM_TARGET_IMEM:
        case RISCV_MEM_TARGET_DMEM:
            break;

        //
        // FB and SYSMEM are supported only for output buffers and only if the
        // shortcut-DMA path has been enabled and configured.
        //
        case RISCV_MEM_TARGET_FBGPA:
        case RISCV_MEM_TARGET_SYSGPA:
            {
                if (!b_allow_external)
                {
                    return E_SCP_INVALID_BASE;
                }

                if (!NVRISCV_DRIVER_SCP_SHORTCUT_ENABLED)
                {
                    return E_SCP_FEATURE_NOT_ENABLED;
                }

                if (!g_scp_state.b_shortcut_configured)
                {
                    return E_SCP_INVALID_DMA_SPECIFIER;
                }
            }
            break;

        // No other locations are supported.
        default:
            return E_SCP_INVALID_BASE;
    }

    //
    // Obtain the offset of the final byte in the buffer (accounting for the
    // edge case of a zero-sized buffer).
    //
    const size_t end_index = (size - (size > 0U));

    // Ensure the buffer does not overflow uintptr_t.
    if ((end_index > UINTPTR_MAX) || ((UINTPTR_MAX - end_index) < base_pa))
    {
        return E_SCP_OUT_OF_RANGE;
    }

    // Obtain the physical address of the final byte in the buffer.
    const uintptr_t end_pa = base_pa + end_index;

    // Ensure the buffer does not overflow the target memory region.
    if ((memutils_riscv_pa_to_target_offset(end_pa, &end_target, NULL) !=
        E_SUCCESS) || (end_target != base_target))
    {
        return E_SCP_OUT_OF_RANGE;
    }

    // Ensure the offset into the target memory region is correctly aligned.
    if (!NV_IS_ALIGNED64(base_offset, SCP_BUFFER_ALIGNMENT))
    {
        return E_SCP_INVALID_ADDRESS;
    }

    // Buffer validated successfully.
    return E_SUCCESS;
}

/**
 * @copydoc scp_pvt_Run_Sequence
 */
error_t
scp_pvt_Run_Sequence
(
    uintptr_t   source_pa,
    uintptr_t   dest_pa,
    size_t      size
)
{
    riscv_mem_target_t  dest_target;
    uint64_t            source_offset, dest_offset;

    // Track which parameters are present.
    const bool b_has_input  = (source_pa != (uintptr_t)NULL);
    const bool b_has_output = (dest_pa   != (uintptr_t)NULL);

    // Obtain the aperture offset of the source buffer, if any.
    if (b_has_input)
    {
        if (memutils_riscv_pa_to_target_offset(source_pa, NULL,
                &source_offset) != E_SUCCESS)
        {
            // This should have been validated by the caller.
            return E_SCP_GENERIC;
        }
    }

    // Obtain the global target/offset pair for the destination buffer, if any.
    if (b_has_output)
    {
        if (memutils_riscv_pa_to_target_offset(dest_pa, &dest_target,
                &dest_offset) != E_SUCCESS)
        {
            // This should have been validated by the caller.
            return E_SCP_GENERIC;
        }
    }

#if NVRISCV_DRIVER_SCP_SHORTCUT_ENABLED
    //
    // Track whether the destination buffer (if any) is located in an external
    // memory region.
    //
    const bool b_external_dest = (b_has_output &&
        ((dest_target == RISCV_MEM_TARGET_FBGPA) ||
         (dest_target == RISCV_MEM_TARGET_SYSGPA)));
#endif // NVRISCV_DRIVER_SCP_SHORTCUT_ENABLED

    // Continue processing until no data remains.
    while (size > 0U)
    {
        scp_transfer_size_t transfer_size;
        uint8_t             num_iterations;

        //
        // Determine the maximum number of bytes we can process this iteration
        // based on the alignment of the source and destination buffers (as
        // applicable), and the constraints of the SCPDMA unit.
        //
        // Note that the cast to uint32_t is safe so long as SCP_TRANSFER_LIMIT
        // is less than 2^32, which we can reasonably assume will always be
        // true.
        //
        INLINE_RFD(CERTC, DEVIATE, INT31_C, "Bug: TODO, DR: TODO - the cast to uint32_t is safe so long as SCP_TRANSFER_LIMIT is less than 2^32");
        uint32_t batch_size = (uint32_t)LOWESTBIT(
                                  (b_has_input  ? source_offset : 0U) |
                                  (b_has_output ? dest_offset   : 0U) |
                                  SCP_TRANSFER_LIMIT);

        //
        // Ensure that the computed size does not exceed the quantity of data
        // remaining to be processed.
        //
        while (batch_size > size)
        {
            batch_size >>= 1U;
        }

        //
        // Convert to an equivalent SCPDMA transfer size and corresponding
        // number of sequencer iterations (assuming at most one push and/or
        // fetch instruction per loop).
        //
        switch (batch_size)
        {
            case (16U * SCP_REGISTER_SIZE):
                transfer_size  = SCP_TRANSFER_SIZE_16R;
                num_iterations = 16U;
                break;

            case (8U * SCP_REGISTER_SIZE):
                transfer_size  = SCP_TRANSFER_SIZE_8R;
                num_iterations = 8U;
                break;

            case (4U * SCP_REGISTER_SIZE):
                transfer_size  = SCP_TRANSFER_SIZE_4R;
                num_iterations = 4U;
                break;

            case (2U * SCP_REGISTER_SIZE):
                transfer_size  = SCP_TRANSFER_SIZE_2R;
                num_iterations = 2U;
                break;

            case (1U * SCP_REGISTER_SIZE):
                transfer_size  = SCP_TRANSFER_SIZE_1R;
                num_iterations = 1U;
                break;

            default:
                //
                // This should never happen unless the caller has failed to
                // ensure that source_pa, dest_pa, and size are aligned
                // properly.
                //
                return E_SCP_GENERIC;
        }

        // Queue the next batch of data to read from the source (if any).
        if (b_has_input)
        {
            scp_queue_read(source_pa, transfer_size);
        }

        // Queue the next batch of data to write to the destination (if any).
        if (b_has_output)
        {
            scp_queue_write(dest_pa, transfer_size);
        }

        // Execute the computed number of sequencer iterations.
        scp_loop_trace0(num_iterations);

#if NVRISCV_DRIVER_SCP_SHORTCUT_ENABLED
        // Complete the follow-up transfer to external memory, if applicable.
        if (b_external_dest)
        {
            if (fbdma_scp_to_extmem(dest_offset, batch_size,
                    g_scp_state.dma_index) != E_SUCCESS)
            {
                //
                // This should never happen so long as the caller has properly
                // sanitized dest_pa and ensured that the shortcut-DMA path has
                // been configured.
                //
                return E_SCP_GENERIC;
            }
        }
#endif // NVRISCV_DRIVER_SCP_SHORTCUT_ENABLED

        // Wait for the current batch to finish processing.
        scp_wait_dma();

        // Check for any failures.
        error_t status = scp_check_dma(true);
        if (status == E_FAULT)
        {
            // Bubble up IO-PMP errors.
            return status;
        }
        else if (status != E_SUCCESS)
        {
            // No other errors are expected.
            return E_SCP_GENERIC;
        }

        // Update pointers.
        source_pa += batch_size;
        dest_pa += batch_size;

        // Update aperture offsets.
        if (b_has_input)
        {
            source_offset += batch_size;
        }

        if (b_has_output)
        {
            dest_offset += batch_size;
        }

        // Update bytes remaining.
        size -= batch_size;
    }

    // Data was processed successfully.
    return E_SUCCESS;
}

///////////////////////////////////////////////////////////////////////////////

/**
 * @brief Resets the ACL of the specified SCP GPR to a fully-open state.
 *
 * @pre The SCP hardware must have been properly initialized.
 *
 * @param[in] reg_index The index of the SCP register to unlock.
 *
 * @retval E_SUCCESS        if the register was unlocked successfully.
 * @retval E_SCP_GENERIC    if an unexpected error occurred.
 *
 * @note This function must only be called from secure contexts and requires
 *       that both secure permissions (Fs and Ks) are already present for the
 *       register being unlocked.
 *
 * @note As a precaution, the target register's contents are cleared after
 *       unlocking. It is recommended that callers also clear the register
 *       before unlocking by calling s_scp_Zero_Reg() and checking that it
 *       succeeded.
 */
static error_t
s_scp_Unlock_Reg
(
    scp_register_index_t reg_index
)
{
    //
    // Enable all permissions in the ACL of the specified register.
    //
    // Some concerns were raised that this creates a dangerous potential ROP
    // gadget for an attacker to leverage, but we don't have any other options
    // for recovering the insecure-write (Wi) permission at this point.
    //
    // We did consider reducing the number of permissions we grant here, but
    // that would require sprinkling additional code throughout other parts of
    // the SCP driver in order to recover the remaining insecure-fetch (Fi) and
    // insecure-key (Ki) permissions on-demand. It's also unlikely that such an
    // alternative approach would buy us much addditional security as any
    // attacker with ROP capabilities can likely also manipulate function
    // arguments anyway.
    //
    scp_chmod(SCP_REGISTER_ACL_ALL, reg_index);

    //
    // Clear the register's contents. This gives us some protection against the
    // aforementioned ROP threat as it makes leaking sensitive information a
    // bit harder.
    //
    // Note that we don't need to go through the extra steps that
    // s_scp_Zero_Reg() follows here since the target register is expected to be
    // fetchable already.
    //
    scp_xor(reg_index, reg_index);

    // Wait for completion.
    scp_wait_dma();

    //
    // Check for failure. The SCPDMA itself should never fail here (we'd get
    // an interrupt instead if the actual chmod or xor instructions failed),
    // but we check anyway to be safe.
    //
    if (scp_check_dma(true) != E_SUCCESS)
    {
        return E_SCP_GENERIC;
    }

    // Register unlocked successfully.
    return E_SUCCESS;
}

/**
 * @brief Zeroes the specified SCP GPR.
 *
 * @pre The SCP hardware must have been properly initialized.
 * @pre The target register must be writable and fetchable if called from an
 *      insecure context.
 *
 * @param[in] reg_index The index of the SCP register to zero.
 *
 * @retval E_SUCCESS                    if the register was zeroed successfully.
 *
 * @retval E_SCP_INSUFFICIENT_RESOURCES if the application lacks access to
 *                                      secret-index zero (required in secure
 *                                      contexts).
 *
 * @retval E_SCP_GENERIC                if an unexpected error occurred.
 *
 * @note Calling this function from a secure context will leave the specified
 *       SCP GPR inaccessible from insecure contexts until explicitly unlocked
 *       later.
 */
static error_t
s_scp_Zero_Reg
(
    scp_register_index_t reg_index
)
{
    //
    // In order for the xor operation below to succeed, the target register
    // must have its fetchable attribute set in its associated ACL. We can
    // ensure this in secure contexts by first loading a fetchable hardware
    // secret into the register, which we do here.
    //
    // Insecure contexts do not have this option but are also not expected to
    // ever encounter a non-fetchable register unless the application has been
    // misusing certain direct-mode interfaces (in which case we'll simply end
    // up triggering an ACL violation).
    //
    if (scp_pvt_Is_Secure())
    {
        // Load secret-index zero to ensure the target register is fetchable.
        scp_secret(SCP_SECRET_INDEX_CLEAR, reg_index);

        // Wait for completion.
        scp_wait_dma();

        // Check for failures.
        error_t status = scp_check_dma(true);

        //
        // Catch denied access to the hardware secret. We return a unique
        // error code here to help differentiate this failure from other
        // permissions errors that higher-level interfaces using this function
        // may return.
        //
        if (status == E_PERM)
        {
            return E_SCP_INSUFFICIENT_RESOURCES;
        }
        //
        // Other failures shouldn't be possible here so just return a generic
        // error on the off chance we hit one somehow.
        //
        else if (status != E_SUCCESS)
        {
            return E_SCP_GENERIC;
        }
    }

    // Clear the register's contents.
    scp_xor(reg_index, reg_index);

    // Wait for completion.
    scp_wait_dma();

    //
    // Check for failure. The SCPDMA itself should never fail here (we'd get
    // an interrupt instead if the actual xor instruction failed), but we
    // check anyway to be safe.
    //
    if (scp_check_dma(true) != E_SUCCESS)
    {
        return E_SCP_GENERIC;
    }

    // Register was zeroed successfully.
    return E_SUCCESS;
}

///////////////////////////////////////////////////////////////////////////////

/**
 * @copydoc g_scp_state
 */
scp_state_t g_scp_state =
{
    // Not initialized until scp_Init() is called.
    .b_initialized          = false,

    // Not shutdown until scp_Shutdown() is called.
    .b_shutdown             = false,

    // Direct mode must be requested by the application.
    .b_direct_mode_active   = false,

    // RNG hardware must be configured explicitly.
    .b_rand_configured      = false,

    // Shortcut path must be configured explicitly.
    .b_shortcut_configured  = false,
    .dma_index              = UINT8_MAX,
};
