Improved XR event processing logic. Particularly, handling session stop/start in a more predictable, less crashy manner. Added disabling of rendering when XR session is not running.

apps/openmw/mwvr/openxrmanager.cpp

@@ -32,6 +32,13 @@ namespace MWVR
         return false;
     }
 
+    bool OpenXRManager::frameShouldRender()
+    {
+        if (realized())
+            return impl().frameShouldRender();
+        return false;
+    }
+
     void OpenXRManager::handleEvents()
     {
         if (realized())
@@ -86,6 +93,16 @@ namespace MWVR
         return impl().disablePredictions();
     }
 
+    void OpenXRManager::xrResourceAcquired()
+    {
+        return impl().xrResourceAcquired();
+    }
+
+    void OpenXRManager::xrResourceReleased()
+    {
+        return impl().xrResourceReleased();
+    }
+
     std::array<View, 2> OpenXRManager::getPredictedViews(int64_t predictedDisplayTime, ReferenceSpace space)
     {
         return impl().getPredictedViews(predictedDisplayTime, space);

apps/openmw/mwvr/openxrmanager.hpp

@@ -56,6 +56,9 @@ namespace MWVR
         //! Whether the openxr session is currently in a running state
         bool xrSessionRunning();
 
+        //! Whether frames should be rendered in the current state
+        bool frameShouldRender();
+
         //! Process all openxr events
         void handleEvents();
 
@@ -68,6 +71,12 @@ namespace MWVR
         //! Disable pose predictions.
         void disablePredictions();
 
+        //! Must be called every time an openxr resource is acquired to keep track
+        void xrResourceAcquired();
+
+        //! Must be called every time an openxr resource is released to keep track
+        void xrResourceReleased();
+
         //! Get poses and fov of both eyes at the predicted time, relative to the given reference space. \note Will throw if predictions are disabled.
         std::array<View, 2> getPredictedViews(int64_t predictedDisplayTime, ReferenceSpace space);
 

apps/openmw/mwvr/openxrmanagerimpl.cpp

@@ -214,7 +214,7 @@ namespace MWVR
     inline XrResult CheckXrResult(XrResult res, const char* originator, const char* sourceLocation) {
         if (XR_FAILED(res)) {
             std::stringstream ss;
-            ss << sourceLocation << ": OpenXR[" << to_string(res) << "]: " << originator;
+            ss << sourceLocation << ": OpenXR[Error: " << to_string(res) << "][Thread: " << std::this_thread::get_id() << "][DC: " << wglGetCurrentDC() << "][GLRC: " << wglGetCurrentContext() << "]: " << originator;
             Log(Debug::Error) << ss.str();
             throw std::runtime_error(ss.str().c_str());
         }
@@ -471,39 +471,58 @@ namespace MWVR
     {
         std::unique_lock<std::mutex> lock(mEventMutex);
 
-        // React to events
+        xrQueueEvents();
+
         while (auto* event = nextEvent())
         {
-            Log(Debug::Verbose) << "OpenXR: Event received: " << to_string(event->type);
+            if (!processEvent(event))
-            switch (event->type)
             {
-            case XR_TYPE_EVENT_DATA_SESSION_STATE_CHANGED:
+                // Do not consider processing an event optional.
-            {
+                // Retry once per frame until every event has been successfully processed
-                const auto* stateChangeEvent = reinterpret_cast<const XrEventDataSessionStateChanged*>(event);
+                return;
-                HandleSessionStateChanged(*stateChangeEvent);
-                break;
-            }
-            case XR_TYPE_EVENT_DATA_INSTANCE_LOSS_PENDING:
-            case XR_TYPE_EVENT_DATA_INTERACTION_PROFILE_CHANGED:
-            case XR_TYPE_EVENT_DATA_REFERENCE_SPACE_CHANGE_PENDING:
-            default: {
-                Log(Debug::Verbose) << "OpenXR: Event ignored";
-                break;
-            }
             }
+            popEvent();
         }
     }
 
-    void
+    const XrEventDataBaseHeader* OpenXRManagerImpl::nextEvent()
-        OpenXRManagerImpl::HandleSessionStateChanged(
+    {
+        if (mEventQueue.size() > 0)
+            return reinterpret_cast<XrEventDataBaseHeader*> (&mEventQueue.front());
+        return nullptr;
+    }
+
+    bool OpenXRManagerImpl::processEvent(const XrEventDataBaseHeader* header)
+    {
+        Log(Debug::Verbose) << "OpenXR: Event received: " << to_string(header->type);
+        switch (header->type)
+        {
+        case XR_TYPE_EVENT_DATA_SESSION_STATE_CHANGED:
+        {
+            const auto* stateChangeEvent = reinterpret_cast<const XrEventDataSessionStateChanged*>(header);
+            return handleSessionStateChanged(*stateChangeEvent);
+            break;
+        }
+        case XR_TYPE_EVENT_DATA_INSTANCE_LOSS_PENDING:
+        case XR_TYPE_EVENT_DATA_INTERACTION_PROFILE_CHANGED:
+        case XR_TYPE_EVENT_DATA_REFERENCE_SPACE_CHANGE_PENDING:
+        default:
+        {
+            Log(Debug::Verbose) << "OpenXR: Event ignored";
+            break;
+        }
+        }
+        return true;
+    }
+
+    bool
+        OpenXRManagerImpl::handleSessionStateChanged(
             const XrEventDataSessionStateChanged& stateChangedEvent)
     {
         auto oldState = mSessionState;
         auto newState = stateChangedEvent.state;
-        mSessionState = newState;
-
         Log(Debug::Verbose) << "XrEventDataSessionStateChanged: state " << to_string(oldState) << "->" << to_string(newState);
-
+        bool success = true;
 
         switch (newState)
         {
@@ -517,21 +536,45 @@ namespace MWVR
         }
         case XR_SESSION_STATE_STOPPING:
         {
-            mSessionRunning = false;
+            if (checkStopCondition())
-            CHECK_XRCMD(xrEndSession(mSession));
+            {
+                CHECK_XRCMD(xrEndSession(mSession));
+                mSessionStopRequested = false;
+                mSessionRunning = false;
+            }
+            else
+            {
+                mSessionStopRequested = true;
+                success = false;
+            }
             break;
         }
         default:
-            Log(Debug::Verbose) << "XrEventDataSessionStateChanged: Ignoring new strate " << to_string(newState);
+            Log(Debug::Verbose) << "XrEventDataSessionStateChanged: Ignoring new state " << to_string(newState);
         }
+
+        if (success)
+        {
+            mSessionState = newState;
+        }
+        else
+        {
+            Log(Debug::Verbose) << "XrEventDataSessionStateChanged: Conditions for state " << to_string(newState) << " not met, retrying next frame";
+        }
+
+        return success;
     }
 
-    const XrEventDataBaseHeader*
+    bool OpenXRManagerImpl::checkStopCondition()
-        OpenXRManagerImpl::nextEvent()
     {
-        XrEventDataBaseHeader* baseHeader = reinterpret_cast<XrEventDataBaseHeader*>(&mEventDataBuffer);
+        return mAcquiredResources == 0;
+    }
+
+    bool OpenXRManagerImpl::xrNextEvent(XrEventDataBuffer& eventBuffer)
+    {
+        XrEventDataBaseHeader* baseHeader = reinterpret_cast<XrEventDataBaseHeader*>(&eventBuffer);
         *baseHeader = { XR_TYPE_EVENT_DATA_BUFFER };
-        const XrResult result = xrPollEvent(mInstance, &mEventDataBuffer);
+        const XrResult result = xrPollEvent(mInstance, &eventBuffer);
         if (result == XR_SUCCESS)
         {
             if (baseHeader->type == XR_TYPE_EVENT_DATA_EVENTS_LOST) {
@@ -547,6 +590,22 @@ namespace MWVR
         return nullptr;
     }
 
+    void OpenXRManagerImpl::popEvent()
+    {
+        if (mEventQueue.size() > 0)
+            mEventQueue.pop();
+    }
+
+    void
+        OpenXRManagerImpl::xrQueueEvents()
+    {
+        XrEventDataBuffer eventBuffer;
+        while (xrNextEvent(eventBuffer))
+        {
+            mEventQueue.push(eventBuffer);
+        }
+    }
+
     MWVR::Pose fromXR(XrPosef pose)
     {
         return MWVR::Pose{ fromXR(pose.position), fromXR(pose.orientation) };
@@ -582,6 +641,26 @@ namespace MWVR
         return mEnabledExtensions.count(extensionName) != 0;
     }
 
+    bool OpenXRManagerImpl::frameShouldRender()
+    {
+        return xrSessionRunning() && !xrSessionStopRequested();
+    }
+
+    void OpenXRManagerImpl::xrResourceAcquired()
+    {
+        mAcquiredResources++;
+    }
+
+    void OpenXRManagerImpl::xrResourceReleased()
+    {
+        mAcquiredResources--;
+    }
+
+    bool OpenXRManagerImpl::xrSessionStopRequested()
+    {
+        return mSessionStopRequested;
+    }
+
     void OpenXRManagerImpl::enablePredictions()
     {
         mPredictionsEnabled = true;

apps/openmw/mwvr/openxrmanagerimpl.hpp

@@ -21,6 +21,7 @@
 #include <iostream>
 #include <thread>
 #include <chrono>
+#include <queue>
 
 namespace MWVR
 {
@@ -70,13 +71,22 @@ namespace MWVR
         XrSession xrSession() const { return mSession; };
         XrInstance xrInstance() const { return mInstance; };
         bool xrExtensionIsEnabled(const char* extensionName) const;
+        bool xrSessionStopRequested();
+        bool frameShouldRender();
+        void xrResourceAcquired();
+        void xrResourceReleased();
 
     protected:
         void LogLayersAndExtensions();
         void LogInstanceInfo();
         void LogReferenceSpaces();
+        bool xrNextEvent(XrEventDataBuffer& eventBuffer);
+        void xrQueueEvents();
         const XrEventDataBaseHeader* nextEvent();
-        void HandleSessionStateChanged(const XrEventDataSessionStateChanged& stateChangedEvent);
+        bool processEvent(const XrEventDataBaseHeader* header);
+        void popEvent();
+        bool handleSessionStateChanged(const XrEventDataSessionStateChanged& stateChangedEvent);
+        bool checkStopCondition();
 
     private:
 
@@ -94,13 +104,15 @@ namespace MWVR
         std::array<XrViewConfigurationView, 2> mConfigViews{ { {XR_TYPE_VIEW_CONFIGURATION_VIEW}, {XR_TYPE_VIEW_CONFIGURATION_VIEW} } };
         XrSpace mReferenceSpaceView = XR_NULL_HANDLE;
         XrSpace mReferenceSpaceStage = XR_NULL_HANDLE;
-        XrEventDataBuffer mEventDataBuffer{ XR_TYPE_EVENT_DATA_BUFFER };
         XrFrameState mFrameState{};
         XrSessionState mSessionState = XR_SESSION_STATE_UNKNOWN;
+        bool mSessionStopRequested = false;
         bool mSessionRunning = false;
+        uint32_t mAcquiredResources = 0;
         std::mutex mFrameStateMutex{};
         std::mutex mEventMutex{};
         std::set<std::string> mEnabledExtensions;
+        std::queue<XrEventDataBuffer> mEventQueue;
 
         std::array<XrCompositionLayerDepthInfoKHR, 2> mLayerDepth;
     };

apps/openmw/mwvr/openxrswapchainimpl.cpp

@@ -165,6 +165,7 @@ namespace MWVR {
 
     void OpenXRSwapchainImpl::acquire(osg::GraphicsContext*)
     {
+        auto xr = Environment::get().getManager();
         XrSwapchainImageAcquireInfo acquireInfo{ XR_TYPE_SWAPCHAIN_IMAGE_ACQUIRE_INFO };
         // I am trusting that the openxr runtime won't diverge these indices so long as these are always called together.
         // If some dumb ass implementation decides to violate this we'll just have to work around that if it actually happens.
@@ -177,18 +178,23 @@ namespace MWVR {
         XrSwapchainImageWaitInfo waitInfo{ XR_TYPE_SWAPCHAIN_IMAGE_WAIT_INFO };
         waitInfo.timeout = XR_INFINITE_DURATION;
         CHECK_XRCMD(xrWaitSwapchainImage(mSwapchain, &waitInfo));
+        xr->xrResourceAcquired();
         CHECK_XRCMD(xrWaitSwapchainImage(mSwapchainDepth, &waitInfo));
+        xr->xrResourceAcquired();
 
         mIsAcquired = true;
     }
 
     void OpenXRSwapchainImpl::release(osg::GraphicsContext*)
     {
+        auto xr = Environment::get().getManager();
         mIsAcquired = false;
 
         XrSwapchainImageReleaseInfo releaseInfo{ XR_TYPE_SWAPCHAIN_IMAGE_RELEASE_INFO };
         CHECK_XRCMD(xrReleaseSwapchainImage(mSwapchain, &releaseInfo));
+        xr->xrResourceReleased();
         CHECK_XRCMD(xrReleaseSwapchainImage(mSwapchainDepth, &releaseInfo));
+        xr->xrResourceReleased();
     }
     void OpenXRSwapchainImpl::checkAcquired() const
     {

apps/openmw/mwvr/vrsession.cpp

@@ -113,6 +113,7 @@ namespace MWVR
     }
 
     bool VRSession::isRunning() const {
+        return true;
         auto* xr = Environment::get().getManager();
         return xr->xrSessionRunning();
     }
@@ -124,12 +125,11 @@ namespace MWVR
         beginPhase(FramePhase::Swap);
 
         auto* frameMeta = getFrame(FramePhase::Swap).get();
+        auto leftView = viewer.getView("LeftEye");
+        auto rightView = viewer.getView("RightEye");
 
-        if (frameMeta->mShouldRender && isRunning())
+        if (frameMeta->mShouldRender)
         {
-            auto leftView = viewer.getView("LeftEye");
-            auto rightView = viewer.getView("RightEye");
-
             viewer.blitEyesToMirrorTexture(gc);
             gc->swapBuffersImplementation();
             leftView->swapBuffers(gc);
@@ -145,6 +145,7 @@ namespace MWVR
 
             Log(Debug::Debug) << frameMeta->mFrameNo << ": EndFrame " << std::this_thread::get_id();
             xr->endFrame(frameMeta->mPredictedDisplayTime, 1, layerStack);
+            xr->xrResourceReleased();
         }
 
         {
@@ -167,7 +168,6 @@ namespace MWVR
             }
 
             getFrame(FramePhase::Swap) = nullptr;
-            mFramesInFlight--;
         }
         mCondition.notify_one();
     }
@@ -292,8 +292,9 @@ namespace MWVR
         frame->mPredictedDisplayTime = predictedDisplayTime;
         frame->mFrameNo = mFrames;
         frame->mPredictedPoses = predictedPoses;
-        frame->mShouldRender = isRunning();
+        frame->mShouldRender = xr->frameShouldRender();
-        mFramesInFlight++;
+        if (frame->mShouldRender)
+            xr->xrResourceAcquired();
     }
 
     const PoseSet& VRSession::predictedPoses(FramePhase phase)

apps/openmw/mwvr/vrsession.hpp

@@ -77,7 +77,6 @@ namespace MWVR
         osg::Matrix viewMatrix(FramePhase phase, Side side);
         osg::Matrix projectionMatrix(FramePhase phase, Side side);
 
-        int mFramesInFlight{ 0 };
         std::array<std::unique_ptr<VRFrameMeta>, (int)FramePhase::NumPhases> mFrame{ nullptr };
 
     private:

apps/openmw/mwvr/vrview.cpp

@@ -57,12 +57,8 @@ namespace MWVR {
 
     void VRView::prerenderCallback(osg::RenderInfo& renderInfo)
     {
-
+        if(Environment::get().getSession()->getFrame(VRSession::FramePhase::Draw)->mShouldRender)
-        if (Environment::get().getManager()->xrSessionRunning())
-        {
             mSwapchain->beginFrame(renderInfo.getState()->getGraphicsContext());
-        }
-
     }
 
     void VRView::InitialDrawCallback::operator()(osg::RenderInfo& renderInfo) const
@@ -86,18 +82,33 @@ namespace MWVR {
         auto* camera = slave._camera.get();
         auto name = camera->getName();
 
-        Side side = Side::RIGHT_SIDE;
+        // Update current cached cull mask of camera if it is active
-        if (name == "LeftEye")
+        auto mask = camera->getCullMask();
-            side = Side::LEFT_SIDE;
+        if (mask == 0)
+            camera->setCullMask(mCullMask);
+        else
+            mCullMask = mask;
 
-        auto* session = Environment::get().getSession();
+        if (Environment::get().getSession()->getFrame(VRSession::FramePhase::Update)->mShouldRender)
-        auto viewMatrix = view.getCamera()->getViewMatrix();
+        {
+            Side side = Side::RIGHT_SIDE;
+            if (name == "LeftEye")
+                side = Side::LEFT_SIDE;
 
-        auto modifiedViewMatrix = viewMatrix * session->viewMatrix(VRSession::FramePhase::Update, side);
+            auto* session = Environment::get().getSession();
-        auto projectionMatrix = session->projectionMatrix(VRSession::FramePhase::Update, side);
+            auto viewMatrix = view.getCamera()->getViewMatrix();
 
-        camera->setViewMatrix(modifiedViewMatrix);
+            auto modifiedViewMatrix = viewMatrix * session->viewMatrix(VRSession::FramePhase::Update, side);
-        camera->setProjectionMatrix(projectionMatrix);
+            auto projectionMatrix = session->projectionMatrix(VRSession::FramePhase::Update, side);
+
+            camera->setViewMatrix(modifiedViewMatrix);
+            camera->setProjectionMatrix(projectionMatrix);
+        }
+        else
+        {
+            // If the session is not active, we do not want to waste resources rendering frames.
+            camera->setCullMask(0);
+        }
         slave.updateSlaveImplementation(view);
     }
 

apps/openmw/mwvr/vrview.hpp

@@ -23,16 +23,11 @@ namespace MWVR
         class UpdateSlaveCallback : public osg::View::Slave::UpdateSlaveCallback
         {
         public:
-            UpdateSlaveCallback(osg::ref_ptr<VRView> view, osg::GraphicsContext* gc)
-                : mView(view), mGC(gc)
-            {}
-
             void updateSlave(osg::View& view, osg::View::Slave& slave) override;
 
         private:
-            osg::ref_ptr<OpenXRManager> mXR;
             osg::ref_ptr<VRView> mView;
-            osg::ref_ptr<osg::GraphicsContext> mGC;
+            osg::Node::NodeMask mCullMask;
         };
 
     public:

apps/openmw/mwvr/vrviewer.cpp

@@ -107,7 +107,7 @@ namespace MWVR
             mViewer->addSlave(camera, true);
             auto* slave = mViewer->findSlaveForCamera(camera);
             assert(slave);
-            slave->_updateSlaveCallback = new VRView::UpdateSlaveCallback(view, context);
+            slave->_updateSlaveCallback = new VRView::UpdateSlaveCallback();
 
             if (mirror)
                 mMsaaResolveMirrorTexture[i].reset(new VRFramebuffer(context->getState(),