/* * Copyright (c) 2015, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER * DEALINGS IN THE SOFTWARE. */ #include "nvidia-drm-conftest.h" /* NV_DRM_ATOMIC_MODESET_AVAILABLE */ #if defined(NV_DRM_ATOMIC_MODESET_AVAILABLE) #include "nvidia-drm-priv.h" #include "nvidia-drm-modeset.h" #include "nvidia-drm-crtc.h" #include "nvidia-drm-os-interface.h" #include "nvidia-drm-helper.h" #if defined(NV_DRM_DRMP_H_PRESENT) #include #endif #if defined(NV_DRM_DRM_VBLANK_H_PRESENT) #include #endif #include #include #include struct nv_drm_atomic_state { struct NvKmsKapiRequestedModeSetConfig config; struct drm_atomic_state base; }; static inline struct nv_drm_atomic_state *to_nv_atomic_state( struct drm_atomic_state *state) { return container_of(state, struct nv_drm_atomic_state, base); } struct drm_atomic_state *nv_drm_atomic_state_alloc(struct drm_device *dev) { struct nv_drm_atomic_state *nv_state = nv_drm_calloc(1, sizeof(*nv_state)); if (nv_state == NULL || drm_atomic_state_init(dev, &nv_state->base) < 0) { nv_drm_free(nv_state); return NULL; } return &nv_state->base; } void nv_drm_atomic_state_clear(struct drm_atomic_state *state) { drm_atomic_state_default_clear(state); } void nv_drm_atomic_state_free(struct drm_atomic_state *state) { struct nv_drm_atomic_state *nv_state = to_nv_atomic_state(state); drm_atomic_state_default_release(state); nv_drm_free(nv_state); } /** * __will_generate_flip_event - Check whether event is going to be generated by * hardware when it flips from old crtc/plane state to current one. This * function is called after drm_atomic_helper_swap_state(), therefore new state * is swapped into current state. */ static void __get_events_plane_mask(struct drm_crtc *crtc, struct drm_crtc_state *old_crtc_state, unsigned int *plane_mask, unsigned int *pending_events_plane_mask) { struct drm_crtc_state *new_crtc_state = crtc->state; struct drm_plane_state *old_plane_state = NULL; struct drm_plane *plane = NULL; int i; *plane_mask = 0x0; *pending_events_plane_mask = 0x0; if (!old_crtc_state->active && !new_crtc_state->active) { /* * crtc is not active in old and new states therefore all planes are * disabled, hardware can not generate flip events. */ return; } /* Find out whether primary & overlay flip done events will be generated. */ nv_drm_for_each_plane_in_state(old_crtc_state->state, plane, old_plane_state, i) { if ((old_plane_state->crtc != crtc) && (plane->state->crtc != crtc)) { continue; } if (plane->type == DRM_PLANE_TYPE_CURSOR) { continue; } *plane_mask |= NVBIT(plane->index); /* * Hardware generates flip event for only those * planes which were active previously. */ if (old_crtc_state->active && old_plane_state->fb != NULL) { *pending_events_plane_mask |= NVBIT(plane->index); } } } static int __nv_drm_put_back_post_fence_fd( struct nv_drm_plane_state *plane_state, const struct NvKmsKapiLayerReplyConfig *layer_reply_config) { int fd = layer_reply_config->postSyncptFd; int ret = 0; if ((fd >= 0) && (plane_state->fd_user_ptr != NULL)) { ret = copy_to_user(plane_state->fd_user_ptr, &fd, sizeof(fd)); if (ret != 0) { return ret; } /*! set back to Null and let set_property specify it again */ plane_state->fd_user_ptr = NULL; } return ret; } static int __nv_drm_get_syncpt_data( struct nv_drm_device *nv_dev, struct drm_crtc *crtc, struct drm_crtc_state *old_crtc_state, struct NvKmsKapiRequestedModeSetConfig *requested_config, struct NvKmsKapiModeSetReplyConfig *reply_config) { struct nv_drm_crtc *nv_crtc = to_nv_crtc(crtc); struct NvKmsKapiHeadReplyConfig *head_reply_config; struct nv_drm_plane_state *plane_state; struct drm_crtc_state *new_crtc_state = crtc->state; struct drm_plane_state *old_plane_state = NULL; struct drm_plane_state *new_plane_state = NULL; struct drm_plane *plane = NULL; int i, ret; if (!old_crtc_state->active && !new_crtc_state->active) { /* * crtc is not active in old and new states therefore all planes are * disabled, exit early. */ return 0; } head_reply_config = &reply_config->headReplyConfig[nv_crtc->head]; nv_drm_for_each_plane_in_state(old_crtc_state->state, plane, old_plane_state, i) { struct nv_drm_plane *nv_plane = to_nv_plane(plane); if (plane->type == DRM_PLANE_TYPE_CURSOR || old_plane_state->crtc != crtc) { continue; } new_plane_state = plane->state; if (new_plane_state->crtc != crtc) { continue; } plane_state = to_nv_drm_plane_state(new_plane_state); ret = __nv_drm_put_back_post_fence_fd( plane_state, &head_reply_config->layerReplyConfig[nv_plane->layer_idx]); if (ret != 0) { return ret; } } return 0; } /** * nv_drm_atomic_commit - validate/commit modeset config * @dev: DRM device * @state: atomic state tracking atomic update * @commit: commit/check modeset config associated with atomic update * * @state tracks atomic update and modeset objects affected * by the atomic update, but the state of the modeset objects it contains * depends on the current stage of the update. * At the commit stage, the proposed state is already stored in the current * state, and @state contains old state for all affected modeset objects. * At the check/validation stage, @state contains the proposed state for * all affected objects. * * Sequence of atomic update - * 1. The check/validation of proposed atomic state, * 2. Do any other steps that might fail, * 3. Put the proposed state into the current state pointers, * 4. Actually commit the hardware state, * 5. Cleanup old state. * * The function nv_drm_atomic_apply_modeset_config() is getting called * at stages (1) and (4) after drm_atomic_helper_swap_state(). */ static int nv_drm_atomic_apply_modeset_config(struct drm_device *dev, struct drm_atomic_state *state, bool commit) { struct nv_drm_device *nv_dev = to_nv_device(dev); struct NvKmsKapiRequestedModeSetConfig *requested_config = &(to_nv_atomic_state(state)->config); struct NvKmsKapiModeSetReplyConfig reply_config = { }; struct drm_crtc *crtc; struct drm_crtc_state *crtc_state; int i; int ret; /* * If sub-owner permission was granted to another NVKMS client, disallow * modesets through the DRM interface. */ if (nv_dev->subOwnershipGranted) { return -EINVAL; } memset(requested_config, 0, sizeof(*requested_config)); /* Loop over affected crtcs and construct NvKmsKapiRequestedModeSetConfig */ nv_drm_for_each_crtc_in_state(state, crtc, crtc_state, i) { /* * When committing a state, the new state is already stored in * crtc->state. When checking a proposed state, the proposed state is * stored in crtc_state. */ struct drm_crtc_state *new_crtc_state = commit ? crtc->state : crtc_state; struct nv_drm_crtc *nv_crtc = to_nv_crtc(crtc); requested_config->headRequestedConfig[nv_crtc->head] = to_nv_crtc_state(new_crtc_state)->req_config; requested_config->headsMask |= 1 << nv_crtc->head; if (commit) { struct drm_crtc_state *old_crtc_state = crtc_state; struct nv_drm_crtc_state *nv_new_crtc_state = to_nv_crtc_state(new_crtc_state); if (!old_crtc_state->active && new_crtc_state->active) { drm_crtc_vblank_on(crtc); } nv_new_crtc_state->nv_flip->event = new_crtc_state->event; new_crtc_state->event = NULL; __get_events_plane_mask(crtc, old_crtc_state, &nv_new_crtc_state->nv_flip->plane_mask, &nv_new_crtc_state->nv_flip->pending_events_plane_mask); if (nv_new_crtc_state->nv_flip->event != NULL) { drm_crtc_vblank_get(crtc); } nv_drm_crtc_enqueue_flip(nv_crtc, nv_new_crtc_state->nv_flip); nv_new_crtc_state->nv_flip = NULL; #if defined(NV_DRM_CRTC_STATE_HAS_VRR_ENABLED) requested_config->headRequestedConfig[nv_crtc->head].modeSetConfig.vrrEnabled = new_crtc_state->vrr_enabled; #endif if (new_crtc_state->mode_changed) { struct drm_vblank_crtc *vblank = &dev->vblank[drm_crtc_index(crtc)]; spin_lock_irq(&dev->vbl_lock); vblank->inmodeset = 0x2; spin_unlock_irq(&dev->vbl_lock); } } } if (commit && nvKms->systemInfo.bAllowWriteCombining) { /* * XXX This call is required only if dumb buffer is going * to be presented. */ nv_drm_write_combine_flush(); nv_kthread_q_flush(&nv_dev->nv_kthread_q); } if (!nvKms->applyModeSetConfig(nv_dev->pDevice, requested_config, &reply_config, commit)) { return -EINVAL; } if (commit) { nv_drm_for_each_crtc_in_state(state, crtc, crtc_state, i) { struct drm_crtc_state *new_crtc_state = crtc->state; struct drm_crtc_state *old_crtc_state = crtc_state; if (new_crtc_state->mode_changed) { struct drm_vblank_crtc *vblank = &dev->vblank[drm_crtc_index(crtc)]; spin_lock_irq(&dev->vbl_lock); vblank->inmodeset = 0x0; spin_unlock_irq(&dev->vbl_lock); } if (old_crtc_state->active && !new_crtc_state->active) { drm_crtc_vblank_off(crtc); } } } if (commit && nv_dev->supportsSyncpts) { nv_drm_for_each_crtc_in_state(state, crtc, crtc_state, i) { /*! loop over affected crtcs and get NvKmsKapiModeSetReplyConfig */ ret = __nv_drm_get_syncpt_data( nv_dev, crtc, crtc_state, requested_config, &reply_config); if (ret != 0) { return ret; } } } return 0; } int nv_drm_atomic_check(struct drm_device *dev, struct drm_atomic_state *state) { int ret = 0; if ((ret = drm_atomic_helper_check(dev, state)) != 0) { goto done; } ret = nv_drm_atomic_apply_modeset_config(dev, state, false /* commit */); done: return ret; } static void nv_drm_update_vblank_timestamp(struct nv_drm_crtc *nv_crtc) { /* If high-precision query is unsupported, DRM core resets vblank * timestamp to 0 to mark it invalid in drm_crtc_vblank_on. It's * possible that the initial event sending to userspace with 0 * timestamp before 1st valid vblank timestamp updated by * drm_handle_vblank. Update timestamp for such case only. */ struct drm_crtc *crtc = &nv_crtc->base; if (!crtc->funcs->get_vblank_timestamp) { ktime_t t_vblank = 0; drm_crtc_vblank_count_and_time(crtc, &t_vblank); if (t_vblank == 0) { struct drm_vblank_crtc *vblank = &crtc->dev->vblank[drm_crtc_index(crtc)]; write_seqlock(&vblank->seqlock); vblank->time = ktime_get(); write_sequnlock(&vblank->seqlock); } } } static void nv_drm_send_completion_event(struct nv_drm_crtc *nv_crtc, struct nv_drm_flip *nv_flip_pos) { WARN_ON(nv_flip_pos->pending_events_plane_mask); if (nv_flip_pos->event != NULL) { nv_drm_update_vblank_timestamp(nv_crtc); drm_crtc_send_vblank_event(&nv_crtc->base, nv_flip_pos->event); drm_crtc_vblank_put(&nv_crtc->base); } list_del(&nv_flip_pos->list_entry); nv_drm_free(nv_flip_pos); } /** * __nv_drm_handle_flip_event - handle flip occurred event * @nv_crtc: crtc on which flip has been occurred * * This handler dequeues the first nv_drm_flip from the crtc's flip_list, * generates an event if requested at flip time, and frees the nv_drm_flip. */ void nv_drm_handle_flip_event(struct nv_drm_device *nv_dev, NvU32 head, NvU32 layer, bool process_pending) { struct nv_drm_crtc *nv_crtc = nv_drm_crtc_lookup(nv_dev, head); struct nv_drm_plane *nv_plane = nv_drm_plane_lookup(nv_dev, head, layer); struct drm_device *dev = nv_crtc->base.dev; struct nv_drm_flip *nv_flip_pos, *nv_flip_n; const uint32_t plane_mask = NVBIT(nv_plane->base.index); bool process_pending_crtc = true; spin_lock(&dev->event_lock); list_for_each_entry_safe(nv_flip_pos, nv_flip_n, &nv_crtc->flip_list, list_entry) { if (!(nv_flip_pos->plane_mask & plane_mask)) { if (nv_flip_pos->plane_mask) { process_pending_crtc = false; } if (process_pending_crtc && !nv_flip_pos->plane_mask) { nv_drm_send_completion_event(nv_crtc, nv_flip_pos); } continue; } if (!process_pending && (nv_flip_pos->pending_events_plane_mask & plane_mask)) { break; } if (nv_flip_pos->pending_events_plane_mask & plane_mask) { process_pending = false; } nv_flip_pos->pending_events_plane_mask &= ~plane_mask; nv_flip_pos->plane_mask &= ~plane_mask; if (!nv_flip_pos->plane_mask) { nv_drm_send_completion_event(nv_crtc, nv_flip_pos); } else { process_pending_crtc = false; } } spin_unlock(&dev->event_lock); wake_up(&nv_dev->flip_event_wq); WARN_ON(process_pending); } static void nv_drm_crtc_handle_completion_event(struct nv_drm_device *nv_dev, struct nv_drm_crtc *nv_crtc) { struct drm_device *dev = nv_crtc->base.dev; struct nv_drm_flip *nv_flip_pos, *nv_flip_n; spin_lock(&dev->event_lock); list_for_each_entry_safe(nv_flip_pos, nv_flip_n, &nv_crtc->flip_list, list_entry) { if (nv_flip_pos->plane_mask) { break; } else { nv_drm_send_completion_event(nv_crtc, nv_flip_pos); } } spin_unlock(&dev->event_lock); wake_up(&nv_dev->flip_event_wq); } int nv_drm_atomic_commit(struct drm_device *dev, struct drm_atomic_state *state, bool nonblock) { int ret = -EBUSY; int i; struct drm_crtc *crtc = NULL; struct drm_crtc_state *crtc_state = NULL; struct drm_plane *plane = NULL; struct drm_plane_state *plane_state = NULL; struct nv_drm_device *nv_dev = to_nv_device(dev); /* * drm_mode_config_funcs::atomic_commit() mandates to return -EBUSY * for nonblocking commit if previous updates (commit tasks/flip event) are * pending. In case of blocking commits it mandates to wait for previous * updates to complete. */ if (nonblock) { nv_drm_for_each_crtc_in_state(state, crtc, crtc_state, i) { struct nv_drm_crtc *nv_crtc = to_nv_crtc(crtc); /* * Here you aren't required to hold nv_drm_crtc::flip_list_lock * because: * * The core DRM driver acquires lock for all affected crtcs before * calling into ->commit() hook, therefore it is not possible for * other threads to call into ->commit() hook affecting same crtcs * and enqueue flip objects into flip_list - * * nv_drm_atomic_commit_internal() * |-> nv_drm_atomic_apply_modeset_config(commit=true) * |-> nv_drm_crtc_enqueue_flip() * * Only possibility is list_empty check races with code path * dequeuing flip object - * * __nv_drm_handle_flip_event() * |-> nv_drm_crtc_dequeue_flip() * * But this race condition can't lead list_empty() to return * incorrect result. nv_drm_crtc_dequeue_flip() in the middle of * updating the list could not trick us into thinking the list is * empty when it isn't. */ if (!list_empty(&nv_crtc->flip_list)) { return -EBUSY; } } } #if defined(NV_DRM_ATOMIC_HELPER_SWAP_STATE_HAS_STALL_ARG) /* * nv_drm_atomic_commit_internal() * implements blocking/non-blocking atomic commit using * nv_drm_crtc::flip_list, it does not require any help from core DRM * helper functions to stall commit processing. Therefore passing false to * 'stall' parameter. * In this context, failure from drm_atomic_helper_swap_state() is not * expected. */ #if defined(NV_DRM_ATOMIC_HELPER_SWAP_STATE_RETURN_INT) ret = drm_atomic_helper_swap_state(state, false /* stall */); if (WARN_ON(ret != 0)) { return ret; } #else drm_atomic_helper_swap_state(state, false /* stall */); #endif #else drm_atomic_helper_swap_state(dev, state); #endif /* * nv_drm_atomic_commit_internal() must not return failure after * calling drm_atomic_helper_swap_state(). */ if ((ret = nv_drm_atomic_apply_modeset_config( dev, state, true /* commit */)) != 0) { NV_DRM_DEV_LOG_ERR( nv_dev, "Failed to apply atomic modeset. Error code: %d", ret); goto done; } nv_drm_for_each_plane_in_state(state, plane, plane_state, i) { struct nv_drm_plane *nv_plane = to_nv_plane(plane); if (plane->type == DRM_PLANE_TYPE_CURSOR) { continue; } nv_drm_handle_flip_event(nv_dev, nv_plane->head, nv_plane->layer_idx, false); } nv_drm_for_each_crtc_in_state(state, crtc, crtc_state, i) { struct nv_drm_crtc *nv_crtc = to_nv_crtc(crtc); nv_drm_crtc_handle_completion_event(nv_dev, nv_crtc); } nv_drm_for_each_crtc_in_state(state, crtc, crtc_state, i) { struct nv_drm_crtc *nv_crtc = to_nv_crtc(crtc); if (!nonblock) { /* * Here you aren't required to hold nv_drm_crtc::flip_list_lock * because: * * The core DRM driver acquires lock for all affected crtcs before * calling into ->commit() hook, therefore it is not possible for * other threads to call into ->commit() hook affecting same crtcs * and enqueue flip objects into flip_list - * * nv_drm_atomic_commit_internal() * |-> nv_drm_atomic_apply_modeset_config(commit=true) * |-> nv_drm_crtc_enqueue_flip() * * Only possibility is list_empty check races with code path * dequeuing flip object - * * __nv_drm_handle_flip_event() * |-> nv_drm_crtc_dequeue_flip() * * But this race condition can't lead list_empty() to return * incorrect result. nv_drm_crtc_dequeue_flip() in the middle of * updating the list could not trick us into thinking the list is * empty when it isn't. */ if (wait_event_timeout( nv_dev->flip_event_wq, list_empty(&nv_crtc->flip_list), 3 * HZ /* 3 second */) == 0) { NV_DRM_DEV_LOG_ERR( nv_dev, "Flip event timeout on head %u", nv_crtc->head); } } } done: #if defined(NV_DRM_ATOMIC_STATE_REF_COUNTING_PRESENT) /* * If ref counting is present, state will be freed when the caller * drops its reference after we return. */ #else drm_atomic_state_free(state); #endif return 0; } #endif