Fixed jitter bugs with the new player tracking.

apps/openmw/mwbase/world.hpp

@@ -51,6 +51,7 @@ namespace ESM
 namespace MWRender
 {
     class Animation;
+    class RenderingManager;
 }
 
 namespace MWMechanics
@@ -135,6 +136,8 @@ namespace MWBase
             virtual MWWorld::Ptr getPlayerPtr() = 0;
             virtual MWWorld::ConstPtr getPlayerConstPtr() const = 0;
 
+            virtual MWRender::RenderingManager& getRenderingManager() = 0;
+
             virtual const MWWorld::ESMStore& getStore() const = 0;
 
             virtual std::vector<ESM::ESMReader>& getEsmReader() = 0;

apps/openmw/mwphysics/physicssystem.cpp

@@ -448,7 +448,7 @@ namespace MWPhysics
                     }
                 }
 
-                // Best effort attempt and not losing any tracking
+                // Best effort attempt at not losing any tracking
                 osg::Vec3 moved = newPosition - position;
                 inputManager->mHeadOffset.x() -= moved.x();
                 inputManager->mHeadOffset.y() -= moved.y();

apps/openmw/mwrender/camera.cpp

@@ -111,6 +111,11 @@ namespace MWRender
         return position;
     }
 
+    void Camera::updateCamera()
+    {
+        updateCamera(mCamera);
+    }
+
     void Camera::updateCamera(osg::Camera *cam)
     {
         if (mTrackingPtr.isEmpty())
@@ -125,9 +130,6 @@ namespace MWRender
         if (inputManager)
         {
             position += inputManager->mHeadOffset;
-            // To show the body, we'll need a neck offset for comfort
-            // This won't do as it will mess with tracking when the player turns his head
-            //position += orient * osg::Vec3(0, 15, 0);
         }
 #else
         osg::Vec3d offset = orient * osg::Vec3d(0, isFirstPerson() ? 0 : -mCameraDistance, 0);
@@ -138,10 +140,7 @@ namespace MWRender
         osg::Vec3d forward = orient * osg::Vec3d(0,1,0);
         osg::Vec3d up = orient * osg::Vec3d(0,0,1);
 
-        osg::Matrix lookAt = osg::Matrix::lookAt(position, position + forward, up);
+        cam->setViewMatrixAsLookAt(position, position + forward, up);
-
-        //cam->setViewMatrixAsLookAt(position, position + forward, up);
-        cam->setViewMatrix(lookAt);
     }
 
     void Camera::reset()
@@ -163,6 +162,10 @@ namespace MWRender
         setYaw(yaw);
         setPitch(pitch);
         setRoll(roll);
+
+        // This might happen mid-update traversal because of openxr input management.
+        // It is essential to VR comfort that this be effective immediately and not next frame.
+        updateCamera();
     }
 
     void Camera::attachTo(const MWWorld::Ptr &ptr)

apps/openmw/mwrender/camera.hpp

@@ -65,6 +65,9 @@ namespace MWRender
         /// Update the view matrix of \a cam
         void updateCamera(osg::Camera* cam);
 
+        /// Update the view matrix of the current camera
+        void updateCamera();
+
         /// Reset to defaults
         void reset();
 

apps/openmw/mwvr/openxranimation.cpp

@@ -165,9 +165,13 @@ OpenXRAnimation::OpenXRAnimation(
     : MWRender::NpcAnimation(ptr, parentNode, resourceSystem, disableSounds, VM_Normal, 55.f)
     , mSession(xrSession)
     , mIndexFingerControllers{nullptr, nullptr}
+    // The player model needs to be pushed back a little to make sure the player's view point is naturally protruding 
+    // Pushing the camera forward instead would produce an unnatural extra movement when rotating the player model.
+    , mModelOffset(new osg::MatrixTransform(osg::Matrix::translate(osg::Vec3(0,-15,0))))
 {
     mIndexFingerControllers[0] = osg::ref_ptr<FingerController> (new FingerController(osg::Quat(0, 0, 0, 1)));
     mIndexFingerControllers[1] = osg::ref_ptr<FingerController> (new FingerController(osg::Quat(0, 0, 0, 1)));
+    mModelOffset->setName("ModelOffset");
 }
 
 OpenXRAnimation::~OpenXRAnimation() {};
@@ -256,6 +260,17 @@ void OpenXRAnimation::addControllers()
             mActiveControllers.insert(std::make_pair(node, mForearmControllers[i]));
         }
     }
+
+
+    auto parent = mObjectRoot->getParent(0);
+
+    if (parent->getName() == "Player Root")
+    {
+        auto group = parent->asGroup();
+        group->removeChildren(0, parent->getNumChildren());
+        group->addChild(mModelOffset);
+        mModelOffset->addChild(mObjectRoot);
+    }
 }
 void OpenXRAnimation::enableHeadAnimation(bool)
 {

apps/openmw/mwvr/openxranimation.hpp

@@ -60,6 +60,7 @@ private:
     ForearmController* mForearmControllers[2]{};
     HandController* mHandControllers[2]{};
     osg::ref_ptr<FingerController> mIndexFingerControllers[2];
+    osg::ref_ptr<osg::MatrixTransform> mModelOffset;
 };
 
 }

apps/openmw/mwvr/openxrinputmanager.cpp

@@ -30,6 +30,9 @@
 #include "../mwworld/esmstore.hpp"
 #include "../mwmechanics/actorutil.hpp"
 
+#include "../mwrender/renderingmanager.hpp"
+#include "../mwrender/camera.hpp"
+
 #include <openxr/openxr.h>
 
 #include <osg/Camera>
@@ -795,12 +798,9 @@ namespace MWVR
 
         auto* session = MWBase::Environment::get().getXRSession();
 
-        updateHead();
-
         OpenXRActionEvent event{};
         while (mXRInput->nextActionEvent(event))
         {
-            //Log(Debug::Verbose) << "ActionEvent action=" << event.action << " onPress=" << event.onPress;
             processEvent(event);
         }
 
@@ -969,7 +969,6 @@ namespace MWVR
         auto player = mPlayer->getPlayer();
 
         auto* session = MWBase::Environment::get().getXRSession();
-
         auto currentHeadPose = session->predictedPoses().head[(int)TrackedSpace::STAGE];
         session->mXR->playerScale(currentHeadPose);
         currentHeadPose.position *= session->unitsPerMeter();
@@ -986,21 +985,25 @@ namespace MWVR
         if (mRecenter)
         {
             mHeadOffset = osg::Vec3(0, 0, 0);
+            // Z should not be affected
+            mHeadOffset.z() = currentHeadPose.position.z();
             mRecenter = false;
         }
         else
         {
             osg::Quat gameworldYaw = osg::Quat(mYaw, osg::Vec3(0, 0, -1));
-            mHeadOffset += gameworldYaw * vrMovement;
+            mHeadOffset += vrMovement;
 
-            float rot[3];
+            Log(Debug::Verbose) << "Set head offset";
-            rot[0] = pitch;
+
-            rot[1] = roll;
+            mVrAngles[0] = pitch;
-            rot[2] = yaw;
+            mVrAngles[1] = roll;
-            world->rotateObject(player, rot[0], rot[1], rot[2], MWBase::RotationFlag_none);
+            mVrAngles[2] = yaw;
+            world->rotateObject(player, mVrAngles[0], mVrAngles[1], mVrAngles[2], MWBase::RotationFlag_none);
+            Log(Debug::Verbose) << "yaw=" << yaw << ", pitch=" << pitch << ", roll=" << roll;
+            
+            MWBase::Environment::get().getWorld()->getRenderingManager().getCamera()->updateCamera();
         }
 
-        // Z should not be affected
-        mHeadOffset.z() = currentHeadPose.position.z();
     }
 }

apps/openmw/mwvr/openxrinputmanager.hpp

@@ -49,6 +49,8 @@ namespace MWVR
         osg::Vec3 mHeadOffset{ 0,0,0 };
         bool mRecenter{ true };
         float mYaw{ 0.f };
+
+        float mVrAngles[3]{ 0.f,0.f,0.f };
     };
 }
 

apps/openmw/mwvr/openxrlayer.cpp

@@ -12,11 +12,14 @@ namespace MWVR {
     {
         LayerHeaders headers{};
         for (auto* layerObject: mLayerObjects)
+        {
+            if (layerObject)
             {
                 auto* header = layerObject->layer();
                 if (header)
                     headers.push_back(header);
             }
+        }
 
         return headers;
     }

apps/openmw/mwvr/openxrmenu.cpp

@@ -1,78 +1,100 @@
 #include "openxrmenu.hpp"
 #include "openxrmanagerimpl.hpp"
 #include <openxr/openxr.h>
+#include <osg/texturerectangle>
 
 namespace MWVR
 {
 
     OpenXRMenu::OpenXRMenu(
-        osg::ref_ptr<OpenXRManager> XR,
+        osg::ref_ptr<osg::Group> root,
-        OpenXRSwapchain::Config config, 
-        osg::ref_ptr<osg::State> state, 
         const std::string& title,
-        osg::Vec2 extent_meters)
+        osg::Vec2 extent_meters,
-        : OpenXRView(XR, title, config, state)
+        Pose pose,
-        , mTitle(title)
+        int width,
-        , mExtent(extent_meters)
+        int height,
+        const osg::Vec4& clearColor,
+        osg::GraphicsContext* gc)
+        : mTitle(title)
+        , mRoot(root)
     {
+        osg::ref_ptr<osg::Vec3Array> vertices{ new osg::Vec3Array(4) };
+        osg::ref_ptr<osg::Vec2Array> texCoords{ new osg::Vec2Array(4) };
+        osg::ref_ptr<osg::Vec3Array> normals{ new osg::Vec3Array(1) };
+        osg::ref_ptr<osg::Vec4Array> colors{ new osg::Vec4Array(1) };
+        osg::Vec3 top_left(0, 1, 1);
+        osg::Vec3 top_right(1, 1, 1);
+        osg::Vec3 bottom_left(0, 1, 0);
+        osg::Vec3 bottom_right(1, 1, 0);
+        (*vertices)[0] = top_left;
+        (*vertices)[1] = top_right;
+        (*vertices)[2] = bottom_left;
+        (*vertices)[3] = bottom_right;
+        (*texCoords)[0].set(0.0f, 0.0f);
+        (*texCoords)[1].set(0.0f, 1.0f);
+        (*texCoords)[2].set(1.0f, 0.0f);
+        (*texCoords)[3].set(1.0f, 1.0f);
+        (*normals)[0].set(0.0f, -1.0f, 0.0f);
+        (*colors)[0].set(1.0f, 1.0f, 1.0f, 0.0f);
+        mGeometry->setVertexArray(vertices);
+        mGeometry->setTexCoordArray(0, texCoords);
+        mGeometry->setNormalArray(normals, osg::Array::BIND_OVERALL);
+        mGeometry->setColorArray(colors, osg::Array::BIND_OVERALL);
+        mGeometry->addPrimitiveSet(new osg::DrawArrays(GL_QUADS, 0, 4));
+        mGeometry->setUseDisplayList(false);
 
-        //updatePosition();
+        mMenuCamera->setClearColor(clearColor);
+        mMenuCamera->setClearMask(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
+        mMenuCamera->setRenderTargetImplementation(osg::Camera::FRAME_BUFFER_OBJECT);
+        mMenuCamera->setRenderOrder(osg::Camera::PRE_RENDER, 0);
+        mMenuCamera->setComputeNearFarMode(osg::CullSettings::DO_NOT_COMPUTE_NEAR_FAR);
+        mMenuCamera->setAllowEventFocus(false);
+        mMenuCamera->setReferenceFrame(osg::Transform::ABSOLUTE_RF);
+        mMenuCamera->setViewport(0, 0, width, height);
+        mMenuCamera->setGraphicsContext(gc);
+
+        //mMenuImage->setFilter(osg::Texture2D::MIN_FILTER, osg::Texture2D::LINEAR);
+        //mMenuImage->setFilter(osg::Texture2D::MAG_FILTER, osg::Texture2D::LINEAR);
+        //mMenuImage->setWrap(osg::Texture::WRAP_S, osg::Texture::CLAMP_TO_EDGE);
+        //mMenuImage->setWrap(osg::Texture::WRAP_T, osg::Texture::CLAMP_TO_EDGE);
+        //mMenuImage->setTextureSize(width, height);
+        //mMenuImage->setSourceFormat(GL_SRGB8_ALPHA8);
+
+        mTransform->setScale(osg::Vec3(extent_meters.x(), 1.f, extent_meters.y()));
+        mTransform->setAttitude(pose.orientation);
+        mTransform->setPosition(pose.position);
+
+        mGeode->addDrawable(mGeometry);
+        mTransform->addChild(mGeode);
+        mRoot->addChild(mTransform);
     }
 
     OpenXRMenu::~OpenXRMenu()
     {
     }
 
-    const XrCompositionLayerBaseHeader* 
+    void OpenXRMenu::updateCallback()
-        OpenXRMenu::layer()
     {
+    }
 
-        if (mPositionNeedsUpdate)
+    void OpenXRMenu::postRenderCallback(osg::RenderInfo& renderInfo)
     {
-            // I position menus half a meter in front of the player, facing the player.
+        if (!mTexture || mMenuCamera->getAttachmentMapModifiedCount() > 0)
-            mPose = predictedPose();
-            mPose.position += mPose.orientation * osg::Vec3(0, 0.5, 0);
-            mPose.orientation = -mPose.orientation;
-
-            Log(Debug::Verbose) << "Menu pose updated to: " << mPose;
-            mPositionNeedsUpdate = false;
-        }
-        
-        if (mLayer)
         {
-            mLayer->pose.position = osg::toXR(mPose.position);
+            auto image = mMenuCamera->getBufferAttachmentMap()[osg::Camera::COLOR_BUFFER]._image;
-            mLayer->pose.orientation = osg::toXR(mPose.orientation);
+            mTexture = new osg::TextureRectangle(image);
+            auto texMat = new osg::TexMat();
+            texMat->setScaleByTextureRectangleSize(true);
+
+            auto* stateSet = mGeometry->getOrCreateStateSet();
+            stateSet->setTextureAttributeAndModes(0, mTexture, osg::StateAttribute::ON);
+            stateSet->setTextureAttributeAndModes(0, texMat, osg::StateAttribute::ON);
+            stateSet->setMode(GL_LIGHTING, osg::StateAttribute::OFF);
+        }
     }
 
-        if(mVisible)
+    void OpenXRMenu::preRenderCallback(osg::RenderInfo& renderInfo)
-            return reinterpret_cast<XrCompositionLayerBaseHeader*>(mLayer.get());
-        return nullptr;
-    }
-
-    void OpenXRMenu::updatePosition()
     {
-        mPositionNeedsUpdate = true;
+        // Doesn't need to do anything
-    }
-
-    void OpenXRMenu::swapBuffers(
-        osg::GraphicsContext* gc)
-    {
-        OpenXRView::swapBuffers(gc);
-
-        if (!mLayer)
-        {
-            mLayer.reset(new XrCompositionLayerQuad);
-            mLayer->type = XR_TYPE_COMPOSITION_LAYER_QUAD;
-            mLayer->next = nullptr;
-            mLayer->layerFlags = 0;
-            mLayer->space = mXR->impl().mReferenceSpaceStage;
-            mLayer->eyeVisibility = XR_EYE_VISIBILITY_BOTH;
-            mLayer->subImage = swapchain().subImage();
-            mLayer->size.width = mExtent.x();
-            mLayer->size.height = mExtent.y();
-
-            // Orientation needs a norm of 1 to be accepted by OpenXR, so we default it to 0,0,0,1
-            mLayer->pose.orientation.w = 1.f;
-        }
     }
 }

apps/openmw/mwvr/openxrmenu.hpp

@@ -1,30 +1,46 @@
 #ifndef OPENXR_MENU_HPP
 #define OPENXR_MENU_HPP
 
+#include <osg/Geometry>
+#include <osg/TexMat>
+#include <osg/PositionAttitudeTransform>
+
 #include "openxrview.hpp"
 #include "openxrlayer.hpp"
 
 struct XrCompositionLayerQuad;
 namespace MWVR
 {
-    class OpenXRMenu : public OpenXRView, public OpenXRLayer
+    class OpenXRMenu
     {
-
     public:
-        OpenXRMenu(osg::ref_ptr<OpenXRManager> XR, OpenXRSwapchain::Config config, osg::ref_ptr<osg::State> state, const std::string& title, osg::Vec2 extent_meters);
+        OpenXRMenu(
+            osg::ref_ptr<osg::Group> root,
+            const std::string& title, 
+            osg::Vec2 extent_meters,
+            Pose pose,
+            int width,
+            int height,
+            const osg::Vec4& clearColor, 
+            osg::GraphicsContext* gc);
         ~OpenXRMenu();
-        const XrCompositionLayerBaseHeader* layer() override;
         const std::string& title() const { return mTitle; }
-        void updatePosition();
+        void updateCallback();
 
-        void swapBuffers(osg::GraphicsContext* gc) override;
+        void preRenderCallback(osg::RenderInfo& renderInfo);
+        void postRenderCallback(osg::RenderInfo& renderInfo);
+        osg::Camera* camera() { return mMenuCamera; }
 
     protected:
-        bool mPositionNeedsUpdate{ true };
-        std::unique_ptr<XrCompositionLayerQuad> mLayer = nullptr;
         std::string mTitle;
-        Pose mPose{ {}, {0,0,0,1}, {} };
+        osg::ref_ptr<osg::Group> mRoot;
-        osg::Vec2 mExtent;
+        osg::ref_ptr<osg::Texture> mTexture{ nullptr };
+        osg::ref_ptr<osg::TexMat> mTexMat{ nullptr };
+        osg::ref_ptr<osg::Camera> mMenuCamera{ new osg::Camera() };
+        //osg::ref_ptr<osg::Texture2D> mMenuImage{ new osg::Texture2D() };
+        osg::ref_ptr<osg::Geometry> mGeometry{ new osg::Geometry() };
+        osg::ref_ptr<osg::Geode> mGeode{ new osg::Geode };
+        osg::ref_ptr<osg::PositionAttitudeTransform> mTransform{ new osg::PositionAttitudeTransform() };
     };
 }
 

apps/openmw/mwvr/openxrsession.cpp

@@ -55,6 +55,7 @@ namespace MWVR
             return;
 
         for (auto layer : mLayerStack.layerObjects())
+            if(layer)
                 layer->swapBuffers(gc);
 
         timer.checkpoint("Rendered");
@@ -79,26 +80,26 @@ namespace MWVR
 
     void OpenXRSession::showMenu(bool show)
     {
-        auto* menuLayer = dynamic_cast<OpenXRMenu*>(mLayerStack.layerObjects()[OpenXRLayerStack::MENU_VIEW_LAYER]);
+        //auto* menuLayer = dynamic_cast<OpenXRMenu*>(mLayerStack.layerObjects()[OpenXRLayerStack::MENU_VIEW_LAYER]);
-        if (menuLayer)
+        //if (menuLayer)
-        {
+        //{
-            bool change = show != menuLayer->isVisible();
+        //    bool change = show != menuLayer->isVisible();
-            menuLayer->setVisible(show);
+        //    menuLayer->setVisible(show);
 
-            // Automatically update position of menu whenever the menu opens.
+        //    // Automatically update position of menu whenever the menu opens.
-            // This ensures menus are always opened near the player.
+        //    // This ensures menus are always opened near the player.
-            if(change)
+        //    if(change)
-                menuLayer->updatePosition();
+        //        menuLayer->updatePosition();
-        }
+        //}
     }
 
     void OpenXRSession::updateMenuPosition(void)
     {
-        auto* menuLayer = dynamic_cast<OpenXRMenu*>(mLayerStack.layerObjects()[OpenXRLayerStack::MENU_VIEW_LAYER]);
+        //auto* menuLayer = dynamic_cast<OpenXRMenu*>(mLayerStack.layerObjects()[OpenXRLayerStack::MENU_VIEW_LAYER]);
-        if (menuLayer)
+        //if (menuLayer)
-        {
+        //{
-            menuLayer->updatePosition();
+        //    menuLayer->updatePosition();
-        }
+        //}
     }
 
 
@@ -168,6 +169,11 @@ namespace MWVR
         mPredictedPoses.eye[(int)TrackedSpace::VIEW][(int)Side::LEFT_HAND] = fromXR(hmdViews[(int)Side::LEFT_HAND].pose);
         mPredictedPoses.eye[(int)TrackedSpace::STAGE][(int)Side::RIGHT_HAND] = fromXR(stageViews[(int)Side::RIGHT_HAND].pose);
         mPredictedPoses.eye[(int)TrackedSpace::VIEW][(int)Side::RIGHT_HAND] = fromXR(hmdViews[(int)Side::RIGHT_HAND].pose);
+
+        auto newpos = mPredictedPoses.head[(int)TrackedSpace::STAGE].position * mUnitsPerMeter;
+        auto oldpos = previousHeadPose.position * mUnitsPerMeter;
+
+        Log(Debug::Verbose) << "Head stage: " << newpos << ", diff=" << (newpos - oldpos);
     }
 }
 

apps/openmw/mwvr/openxrview.cpp

@@ -52,6 +52,7 @@ namespace MWVR {
 
     void OpenXRView::prerenderCallback(osg::RenderInfo& renderInfo)
     {
+        Log(Debug::Verbose) << "Prerender";
         if (mSwapchain)
         {
             mSwapchain->beginFrame(renderInfo.getState()->getGraphicsContext());

apps/openmw/mwvr/openxrviewer.cpp

@@ -42,6 +42,10 @@ namespace MWVR
         mRightHandTransform->setName("tracker r hand");
         mRightHandTransform->setUpdateCallback(new TrackedNodeUpdateCallback(this));
         this->addChild(mRightHandTransform);
+
+        // TODO: Left off here
+        mMenusRoot = new osg::Group();
+        mMenusRoot->setName("XR Menus Root");
     }
 
     OpenXRViewer::~OpenXRViewer(void)
@@ -151,30 +155,33 @@ namespace MWVR
         // It's just convenient.
         mMirrorTextureSwapchain.reset(new OpenXRSwapchain(mXR, context->getState(), config));
 
-        auto menuView = new OpenXRMenu(mXR, config, context->getState(), "MainMenu", osg::Vec2(1.f, 1.f));
+        //auto menuView = new OpenXRMenu(mXR, config, context->getState(), "MainMenu", osg::Vec2(1.f, 1.f));
-        mViews["MenuView"] = menuView;
+        //mViews["MenuView"] = menuView;
+        //auto menuCamera = mCameras["MenuView"] = menuView->createCamera(2, clearColor, context);
 
-        auto menuCamera = mCameras["MenuView"] = menuView->createCamera(2, clearColor, context);
+        //mMenus.reset(new OpenXRMenu(mMenusRoot, "MainMenu", osg::Vec2(1.f, 1.f), MWVR::Pose(), config.width, config.height, osg::Vec4(0.f, 0.f, 0.f, 0.f), context));
-        menuCamera->setCullMask(MWRender::Mask_GUI);
+        //auto menuCamera = mMenus->camera();
-        menuCamera->setName("MenuView");
+        //menuCamera->setCullMask(MWRender::Mask_GUI);
-        menuCamera->setPreDrawCallback(mPreDraw);
+        //menuCamera->setName("MenuView");
-        menuCamera->setPostDrawCallback(mPostDraw);
+        //menuCamera->setPreDrawCallback(mPreDraw);
+        //menuCamera->setPostDrawCallback(mPostDraw);
 
 
-        mViewer->addSlave(menuCamera, true);
+        //mViewer->addSlave(menuCamera, true);
         mViewer->getSlave(0)._updateSlaveCallback = new OpenXRWorldView::UpdateSlaveCallback(mXR, session, leftView, context);
         mViewer->getSlave(1)._updateSlaveCallback = new OpenXRWorldView::UpdateSlaveCallback(mXR, session, rightView, context);
 
         mainCamera->getGraphicsContext()->setSwapCallback(new OpenXRViewer::SwapBuffersCallback(this));
         mainCamera->setGraphicsContext(nullptr);
         session->setLayer(OpenXRLayerStack::WORLD_VIEW_LAYER, this);
-        session->setLayer(OpenXRLayerStack::MENU_VIEW_LAYER, dynamic_cast<OpenXRLayer*>(mViews["MenuView"].get()));
+        //session->setLayer(OpenXRLayerStack::MENU_VIEW_LAYER, dynamic_cast<OpenXRLayer*>(mViews["MenuView"].get()));
         mConfigured = true;
 
     }
 
     void OpenXRViewer::blitEyesToMirrorTexture(osg::GraphicsContext* gc, bool includeMenu)
     {
+        includeMenu = false;
         mMirrorTextureSwapchain->beginFrame(gc);
 
         int mirror_width = 0;
@@ -187,8 +194,8 @@ namespace MWVR
         mViews["RightEye"]->swapchain().renderBuffer()->blit(gc, 0, 0, mirror_width, mMirrorTextureSwapchain->height());
         mViews["LeftEye"]->swapchain().renderBuffer()->blit(gc, mirror_width, 0, 2 * mirror_width, mMirrorTextureSwapchain->height());
 
-        if(includeMenu)
+        //if(includeMenu)
-            mViews["MenuView"]->swapchain().renderBuffer()->blit(gc, 2 * mirror_width, 0, 3 * mirror_width, mMirrorTextureSwapchain->height());
+        //    mViews["MenuView"]->swapchain().renderBuffer()->blit(gc, 2 * mirror_width, 0, 3 * mirror_width, mMirrorTextureSwapchain->height());
 
         mMirrorTextureSwapchain->endFrame(gc);
 
@@ -222,12 +229,13 @@ namespace MWVR
         rightView->swapBuffers(gc);
         timer.checkpoint("Views");
 
-        mCompositionLayerProjectionViews[0].pose = toXR(leftView->predictedPose());
+        mCompositionLayerProjectionViews[0].pose = toXR(MWBase::Environment::get().getXRSession()->predictedPoses().eye[(int)TrackedSpace::STAGE][(int)Side::LEFT_HAND]);
-        mCompositionLayerProjectionViews[1].pose = toXR(rightView->predictedPose());
+        mCompositionLayerProjectionViews[1].pose = toXR(MWBase::Environment::get().getXRSession()->predictedPoses().eye[(int)TrackedSpace::STAGE][(int)Side::RIGHT_HAND]);
+
         timer.checkpoint("Poses");
 
         // TODO: Keep track of these in the session too.
-        auto stageViews = mXR->impl().getPredictedViews(mXR->impl().frameState().predictedDisplayTime, TrackedSpace::VIEW);
+        auto stageViews = mXR->impl().getPredictedViews(mXR->impl().frameState().predictedDisplayTime, TrackedSpace::STAGE);
         mCompositionLayerProjectionViews[0].fov = stageViews[0].fov;
         mCompositionLayerProjectionViews[1].fov = stageViews[1].fov;
         timer.checkpoint("Fovs");
@@ -237,7 +245,7 @@ namespace MWVR
         {
             mLayer.reset(new XrCompositionLayerProjection);
             mLayer->type = XR_TYPE_COMPOSITION_LAYER_PROJECTION;
-            mLayer->space = mXR->impl().mReferenceSpaceView;
+            mLayer->space = mXR->impl().mReferenceSpaceStage;
             mLayer->viewCount = 2;
             mLayer->views = mCompositionLayerProjectionViews.data();
         }
@@ -273,8 +281,8 @@ namespace MWVR
             {
                 mXR->beginFrame();
                 auto& poses = MWBase::Environment::get().getXRSession()->predictedPoses();
-                auto menuPose = poses.head[(int)TrackedSpace::STAGE];
+                //auto menuPose = poses.head[(int)TrackedSpace::STAGE];
-                mViews["MenuView"]->setPredictedPose(menuPose);
+                //mViews["MenuView"]->setPredictedPose(menuPose);
             }
         }
 

apps/openmw/mwvr/openxrviewer.hpp

@@ -121,11 +121,14 @@ namespace MWVR
         PredrawCallback* mPreDraw{ nullptr };
         PostdrawCallback* mPostDraw{ nullptr };
 
-        std::unique_ptr<OpenXRSwapchain> mMirrorTextureSwapchain = nullptr;
+        std::unique_ptr<OpenXRSwapchain> mMirrorTextureSwapchain{ nullptr };
 
         std::mutex mMutex;
 
-        bool mConfigured = false;
+        bool mConfigured{ false };
+
+        osg::ref_ptr<osg::Group> mMenusRoot{ new osg::Group };
+        std::unique_ptr<OpenXRMenu> mMenus{ nullptr };
     };
 }
 

apps/openmw/mwvr/openxrworldview.cpp

@@ -13,6 +13,9 @@
 
 #include "../mwrender/vismask.hpp"
 #include "../mwbase/environment.hpp"
+#include "../mwbase/world.hpp"
+#include "../mwrender/renderingmanager.hpp"
+#include "../mwrender/camera.hpp"
 
 #include <osg/Camera>
 #include <osgViewer/Renderer>
@@ -82,7 +85,7 @@ namespace MWVR
     osg::Matrix OpenXRWorldView::projectionMatrix()
     {
         // TODO: Get this from the session predictions instead
-        auto hmdViews = mXR->impl().getPredictedViews(mXR->impl().frameState().predictedDisplayTime, TrackedSpace::VIEW);
+        auto hmdViews = mXR->impl().getPredictedViews(mXR->impl().frameState().predictedDisplayTime, TrackedSpace::STAGE);
 
         float near = Settings::Manager::getFloat("near clip", "Camera");
         float far = Settings::Manager::getFloat("viewing distance", "Camera");
@@ -99,6 +102,16 @@ namespace MWVR
         osg::Vec3 position = pose.position * mUnitsPerMeter;
         osg::Quat orientation = pose.orientation;
 
+        float y = position.y();
+        float z = position.z();
+        position.y() = z;
+        position.z() = -y;
+
+        y = orientation.y();
+        z = orientation.z();
+        orientation.y() = z;
+        orientation.z() = -y;
+
         osg::Matrix viewMatrix;
         viewMatrix.setTrans(-position);
         viewMatrix.postMultRotate(orientation.conj());
@@ -133,14 +146,6 @@ namespace MWVR
         }
     }
 
-    void OpenXRWorldView::prerenderCallback(osg::RenderInfo& renderInfo)
-    {
-        OpenXRView::prerenderCallback(renderInfo);
-        auto* view = renderInfo.getView();
-        auto* camera = renderInfo.getCurrentCamera();
-        auto name = camera->getName();
-    }
-
     void
         OpenXRWorldView::UpdateSlaveCallback::updateSlave(
             osg::View& view,
@@ -152,27 +157,42 @@ namespace MWVR
 
         auto& poses = mSession->predictedPoses();
 
+        Pose viewPose{};
+        Pose stagePose{};
+
         if (name == "LeftEye")
         {
-            mXR->handleEvents();
             mSession->waitFrame();
-            auto leftEyePose = poses.eye[(int)TrackedSpace::VIEW][(int)Side::LEFT_HAND];
+
-            mView->setPredictedPose(leftEyePose);
+            // Updating the head pose needs to happen after waitFrame(),
+            // But i can't call waitFrame from the input manager since it might
+            // not always be active.
+            auto inputManager = MWBase::Environment::get().getXRInputManager();
+            if (inputManager)
+                inputManager->updateHead();
+
+            stagePose = poses.eye[(int)TrackedSpace::STAGE][(int)Side::LEFT_HAND];
+            viewPose = poses.eye[(int)TrackedSpace::VIEW][(int)Side::LEFT_HAND];
+            mView->setPredictedPose(viewPose);
         }
         else
         {
-            auto rightEyePose = poses.eye[(int)TrackedSpace::VIEW][(int)Side::RIGHT_HAND];
+            stagePose = poses.eye[(int)TrackedSpace::STAGE][(int)Side::RIGHT_HAND];
-            mView->setPredictedPose(rightEyePose);
+            viewPose = poses.eye[(int)TrackedSpace::VIEW][(int)Side::RIGHT_HAND];
+            mView->setPredictedPose(viewPose);
         }
         if (!mXR->sessionRunning())
             return;
 
-        auto viewMatrix = view.getCamera()->getViewMatrix() * mView->viewMatrix();
+        auto viewMatrix = view.getCamera()->getViewMatrix();
+        auto modifiedViewMatrix = viewMatrix * mView->viewMatrix();
         auto projectionMatrix = mView->projectionMatrix();
 
-        camera->setViewMatrix(viewMatrix);
+        camera->setViewMatrix(modifiedViewMatrix);
         camera->setProjectionMatrix(projectionMatrix);
-
         slave.updateSlaveImplementation(view);
+
+
+
     }
 }

apps/openmw/mwvr/openxrworldview.hpp

@@ -29,8 +29,6 @@ namespace MWVR
         OpenXRWorldView(osg::ref_ptr<OpenXRManager> XR, std::string name, osg::ref_ptr<osg::State> state, OpenXRSwapchain::Config config, float unitsPerMeter);
         ~OpenXRWorldView();
 
-        //! Prepare for render (update matrices)
-        void prerenderCallback(osg::RenderInfo& renderInfo) override;
         //! Projection offset for this view
         osg::Matrix projectionMatrix();
         //! View offset for this view

apps/openmw/mwworld/worldimp.cpp

@@ -3511,6 +3511,11 @@ namespace MWWorld
         return mPlayer->getConstPlayer();
     }
 
+    MWRender::RenderingManager& World::getRenderingManager()
+    {
+        return *mRendering;
+    }
+
     void World::updateDialogueGlobals()
     {
         MWWorld::Ptr player = getPlayerPtr();

apps/openmw/mwworld/worldimp.hpp

@@ -243,6 +243,8 @@ namespace MWWorld
             MWWorld::Ptr getPlayerPtr() override;
             MWWorld::ConstPtr getPlayerConstPtr() const override;
 
+            MWRender::RenderingManager& getRenderingManager() override;
+
             const MWWorld::ESMStore& getStore() const override;
 
             std::vector<ESM::ESMReader>& getEsmReader() override;