Pointing beam to visualize what the player is pointing at

apps/openmw/mwbase/world.hpp

@@ -7,6 +7,7 @@
 #include <map>
 #include <set>
 #include <deque>
+#include <osg/node>
 
 #include <components/esm/cellid.hpp>
 
@@ -260,7 +261,11 @@ namespace MWBase
             virtual MWWorld::Ptr  getFacedObject() = 0;
             ///< Return pointer to the object the player is looking at, if it is within activation range
 
+            virtual std::pair<MWWorld::Ptr, osg::Node*> getPointedAtObject() = 0;
+            ///< Return pointer to the object and/or node the player is currently pointing at
+
             virtual float getDistanceToFacedObject() = 0;
+            virtual float getDistanceToPointedAtObject() = 0;
 
             virtual float getMaxActivationDistance() = 0;
 

apps/openmw/mwrender/renderingmanager.cpp

@@ -1019,6 +1019,7 @@ namespace MWRender
         RenderingManager::RayResult result;
         result.mHit = false;
         result.mRatio = 0;
+        result.mHitNode = nullptr;
         if (intersector->containsIntersections())
         {
             result.mHit = true;
@@ -1028,6 +1029,9 @@ namespace MWRender
             result.mHitNormalWorld = intersection.getWorldIntersectNormal();
             result.mRatio = intersection.ratio;
 
+            if(!intersection.nodePath.empty())
+                result.mHitNode = intersection.nodePath.back();
+
             PtrHolder* ptrHolder = nullptr;
             for (osg::NodePath::const_iterator it = intersection.nodePath.begin(); it != intersection.nodePath.end(); ++it)
             {

apps/openmw/mwrender/renderingmanager.hpp

@@ -152,6 +152,7 @@ namespace MWRender
             osg::Vec3f mHitNormalWorld;
             osg::Vec3f mHitPointWorld;
             MWWorld::Ptr mHitObject;
+            osg::Node* mHitNode;
             float mRatio;
         };
 

apps/openmw/mwvr/openxranimation.cpp

@@ -6,6 +6,7 @@
 #include <osg/Depth>
 #include <osg/Drawable>
 #include <osg/Object>
+#include <osg/BlendFunc>
 
 #include <osgUtil/RenderBin>
 #include <osgUtil/CullVisitor>
@@ -51,6 +52,15 @@
 #include "../mwrender/rotatecontroller.hpp"
 #include "../mwrender/renderbin.hpp"
 #include "../mwrender/vismask.hpp"
+#include "../mwrender/renderingmanager.hpp"
+#include "../mwrender/objects.hpp"
+
+#include "../mwphysics/collisiontype.hpp"
+#include "../mwphysics/physicssystem.hpp"
+
+#include "openxrenvironment.hpp"
+#include "openxrviewer.hpp"
+#include "openxrinputmanager.hpp"
 
 namespace MWVR
 {
@@ -59,7 +69,7 @@ namespace MWVR
 // existing animation system, implementing this as an animation source.
 // But I'm not sure it would be since these are not classical animations.
 // It would make it easier to control priority, and later allow for users to add their own stuff to animations based on VR/touch input.
-// But openmw doesn't really have any concepts for user animation overrides.
+// But openmw doesn't really have any concepts for user animation overrides as far as i can tell.
 
 
 /// Implements dummy control of the forearm, to control mesh/bone deformation of the hand.
@@ -142,18 +152,44 @@ void FingerController::operator()(osg::Node* node, osg::NodeVisitor* nv)
         return;
     }
 
+    // This update needs to hard override all other animation updates.
+    // To do this i need to make sure no further update calls are made.
+    // Therefore i do not traverse normally but instead explicitly fetch 
+    // the children i want to update and update them here.
+
+    // I'm sure this could be done in a cleaner way
+
+
+    // First, update the base of the finger to the overriding orientation
     auto matrixTransform = node->asTransform()->asMatrixTransform();
     auto matrix = matrixTransform->getMatrix();
     matrix.setRotate(mRotate);
     matrixTransform->setMatrix(matrix);
 
-    auto tip = matrixTransform->getChild(0);
+    // Next update the tip.
-    auto tipMatrixTransform = tip->asTransform()->asMatrixTransform();
+    // Note that for now both tips are just given osg::Quat(0,0,0,1) as that amounts to pointing forward.
-    matrix = tipMatrixTransform->getMatrix();
+    auto tip = matrixTransform->getChild(0)->asTransform()->asMatrixTransform();
+    matrix = tip->getMatrix();
     matrix.setRotate(mRotate);
-    tipMatrixTransform->setMatrix(matrix);
+    tip->setMatrix(matrix);
 
-    //traverse(node, nv);
+    // Finally, if pointing forward is enabled we need to intersect the scene to find where the player is pointing
+    // So that we can display a beam to visualize where the player is pointing.
+
+    // Dig up the pointer transform
+    SceneUtil::FindByNameVisitor findPointerVisitor("Pointer Transform", osg::NodeVisitor::TRAVERSE_ALL_CHILDREN);
+    tip->accept(findPointerVisitor);
+    auto pointerTransform = findPointerVisitor.mFoundNode;
+    if (pointerTransform)
+    {
+        // Get distance to pointer intersection
+        auto pat = pointerTransform->asTransform()->asPositionAttitudeTransform();
+        // TODO: Using the cached value from the input manager makes this off by one frame
+        float intersected_distance = MWBase::Environment::get().getWorld()->getDistanceToPointedAtObject();
+
+        // Stretch beam to point of intersection.
+        pat->setScale(osg::Vec3(1.f, 1.f, intersected_distance));
+    }
 }
 
 
@@ -172,6 +208,7 @@ OpenXRAnimation::OpenXRAnimation(
     mIndexFingerControllers[0] = osg::ref_ptr<FingerController> (new FingerController(osg::Quat(0, 0, 0, 1)));
     mIndexFingerControllers[1] = osg::ref_ptr<FingerController> (new FingerController(osg::Quat(0, 0, 0, 1)));
     mModelOffset->setName("ModelOffset");
+    createPointer();
 }
 
 OpenXRAnimation::~OpenXRAnimation() {};
@@ -213,11 +250,99 @@ void OpenXRAnimation::setPointForward(bool enabled)
     auto found00 = mNodeMap.find("bip01 r finger1");
     if (found00 != mNodeMap.end())
     {
-        found00->second->removeUpdateCallback(mIndexFingerControllers[0]);
+        auto base_joint = found00->second;
+        auto second_joint = base_joint->getChild(0)->asTransform()->asMatrixTransform();
+        assert(second_joint);
+
+        second_joint->removeChild(mPointerTransform);
+        base_joint->removeUpdateCallback(mIndexFingerControllers[0]);
         if (enabled)
-            found00->second->addUpdateCallback(mIndexFingerControllers[0]);
+        {
+            second_joint->addChild(mPointerTransform);
+            base_joint->addUpdateCallback(mIndexFingerControllers[0]);
+        }
     }
 }
+
+void OpenXRAnimation::createPointer(void)
+{
+    mPointerGeometry = createPointerGeometry();
+    mPointerTransform = new osg::PositionAttitudeTransform();
+    mPointerTransform->addChild(mPointerGeometry);
+    mPointerTransform->asPositionAttitudeTransform()->setAttitude(osg::Quat(osg::DegreesToRadians(90.f), osg::Vec3(0.f, 1.f, 0.f)));
+    mPointerTransform->asPositionAttitudeTransform()->setPosition(osg::Vec3(0.f, 0.f, 0.f));
+    mPointerTransform->asPositionAttitudeTransform()->setScale(osg::Vec3(1.f, 1.f, 1.f));
+    mPointerTransform->setName("Pointer Transform");
+}
+
+osg::ref_ptr<osg::Geometry> OpenXRAnimation::createPointerGeometry(void)
+{
+    osg::ref_ptr<osg::Geometry> geometry = new osg::Geometry();
+
+    // Create pointer geometry, which will point from the tip of the player's finger.
+    // The geometry will be a Four sided pyramid, with the top at the player's fingers
+
+    osg::Vec3 vertices[]{
+        {0, 0, 0}, // origin
+        {-1, 1, 1}, // top_left
+        {-1, -1, 1}, // bottom_left
+        {1, -1, 1}, // bottom_right
+        {1, 1, 1}, // top_right
+    };
+
+    osg::Vec4 colors[]{
+        osg::Vec4(1.0f, 0.0f, 0.0f, 0.0f),
+        osg::Vec4(1.0f, 0.0f, 0.0f, 1.0f),
+        osg::Vec4(1.0f, 0.0f, 0.0f, 1.0f),
+        osg::Vec4(1.0f, 0.0f, 0.0f, 1.0f),
+        osg::Vec4(1.0f, 0.0f, 0.0f, 1.0f),
+    };
+
+    const int origin = 0;
+    const int top_left = 1;
+    const int bottom_left = 2;
+    const int bottom_right = 3;
+    const int top_right = 4;
+
+    const int triangles[] =
+    {
+        bottom_right, top_right, top_left,
+        bottom_right, top_left, bottom_left,
+        origin, top_left, top_right,
+        origin, top_right, bottom_right,
+        origin, bottom_left, top_left,
+        origin, bottom_right, bottom_left,
+    };
+    int numVertices = sizeof(triangles) / sizeof(*triangles);
+    osg::ref_ptr<osg::Vec3Array> vertexArray = new osg::Vec3Array(numVertices);
+    osg::ref_ptr<osg::Vec4Array> colorArray = new osg::Vec4Array(numVertices);
+    for (int i = 0; i < numVertices; i++)
+    {
+        (*vertexArray)[i] = vertices[triangles[i]];
+        (*colorArray)[i] = colors[triangles[i]];
+    }
+
+    osg::ref_ptr<osg::Vec3Array> normals = new osg::Vec3Array;
+    normals->push_back(osg::Vec3(0.0f, -1.0f, 0.0f));
+
+
+
+    geometry->setVertexArray(vertexArray);
+    geometry->setColorArray(colorArray, osg::Array::BIND_PER_VERTEX);
+    geometry->addPrimitiveSet(new osg::DrawArrays(osg::PrimitiveSet::TRIANGLES, 0, numVertices));
+    geometry->setDataVariance(osg::Object::DYNAMIC);
+    geometry->setSupportsDisplayList(false);
+    geometry->setCullingActive(false);
+
+    auto stateset = geometry->getOrCreateStateSet();
+    stateset->setMode(GL_LIGHTING, osg::StateAttribute::OFF);
+    stateset->setMode(GL_BLEND, osg::StateAttribute::ON);
+    stateset->setAttributeAndModes(new osg::BlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA));
+    stateset->setRenderingHint(osg::StateSet::TRANSPARENT_BIN);
+
+    return geometry;
+}
+
 osg::Vec3f OpenXRAnimation::runAnimation(float timepassed)
 {
     return NpcAnimation::runAnimation(timepassed);

apps/openmw/mwvr/openxranimation.hpp

@@ -55,12 +55,19 @@ public:
     /// (Used to visualize direction of activation action)
     void setPointForward(bool enabled);
 
+public:
+    void createPointer(void);
+    static osg::ref_ptr<osg::Geometry> createPointerGeometry(void);
+
 private:
     std::shared_ptr<OpenXRSession> mSession;
     ForearmController* mForearmControllers[2]{};
     HandController* mHandControllers[2]{};
     osg::ref_ptr<FingerController> mIndexFingerControllers[2];
     osg::ref_ptr<osg::MatrixTransform> mModelOffset;
+    osg::ref_ptr<osg::Geometry> mPointerGeometry{ nullptr };
+    osg::ref_ptr<osg::Transform> mPointerTransform{ nullptr };
+    
 };
 
 }

apps/openmw/mwvr/openxrinputmanager.cpp

@@ -799,9 +799,15 @@ namespace MWVR
         bool disableControls,
         bool disableEvents)
     {
-
         mXRInput->updateControls();
 
+        auto* world = MWBase::Environment::get().getWorld();
+        if (world)
+        {
+            auto pointedAt = world->getPointedAtObject();
+            // TODO: Left off here
+        }
+
         OpenXRActionEvent event{};
         while (mXRInput->nextActionEvent(event))
         {
@@ -820,7 +826,7 @@ namespace MWVR
 
     void OpenXRInputManager::processEvent(const OpenXRActionEvent& event)
     {
-        auto* session = OpenXREnvironment::get().getSession();
+        //auto* session = OpenXREnvironment::get().getSession();
         auto* xrMenuManager = OpenXREnvironment::get().getMenuManager();
         switch (event.action)
         {

apps/openmw/mwvr/openxrinputmanager.hpp

@@ -8,6 +8,8 @@
 #include <array>
 #include <iostream>
 
+#include "../mwworld/ptr.hpp"
+
 namespace MWVR
 {
     struct OpenXRInput;

apps/openmw/mwvr/openxrmenu.cpp

@@ -2,6 +2,7 @@
 #include "openxrenvironment.hpp"
 #include "openxrsession.hpp"
 #include "openxrmanagerimpl.hpp"
+#include "openxranimation.hpp"
 #include <openxr/openxr.h>
 #include <osg/Texture2D>
 #include <osg/ClipNode>
@@ -30,7 +31,7 @@ public:
 
 private:
 
-    osg::observer_ptr<osg::Texture2D> mTexture;
+    osg::ref_ptr<osg::Texture2D> mTexture;
 };
 
 /// RTT camera used to draw the osg GUI to a texture
@@ -143,6 +144,7 @@ private:
         mGeometry->addPrimitiveSet(new osg::DrawArrays(GL_QUADS, 0, 4));
         mGeometry->setDataVariance(osg::Object::DYNAMIC);
         mGeometry->setSupportsDisplayList(false);
+        mGeometry->setName("XR Menu Geometry");
         //mGeode->addDrawable(mGeometry);
 
         // Define the camera that will render the menu texture
@@ -162,6 +164,7 @@ private:
         mTransform->setAttitude(pose.orientation);
         mTransform->setPosition(pose.position);
         mTransform->addChild(mGeometry);
+        //mTransform->addChild(OpenXRAnimation::createPointerGeometry());
 
         // Add to scene graph
         mGeometryRoot->addChild(mTransform);

apps/openmw/mwvr/openxrview.cpp

@@ -61,8 +61,6 @@ namespace MWVR {
 
     void OpenXRView::postrenderCallback(osg::RenderInfo& renderInfo)
     {
-        //    mSwapchain->endFrame(renderInfo.getState()->getGraphicsContext());
-
         mTimer.checkpoint("Postrender");
         auto state = renderInfo.getState();
         auto gl = osg::GLExtensions::Get(state->getContextID(), false);
@@ -76,6 +74,5 @@ namespace MWVR {
     void  OpenXRView::setPredictedPose(const Pose& pose) 
     { 
         mPredictedPose = pose; 
-        //Log(Debug::Verbose) << mName << " predicted pose updated to " << pose;
     };
 }

apps/openmw/mwvr/openxrviewer.cpp

@@ -244,7 +244,6 @@ namespace MWVR
         if (!mConfigured)
             return;
 
-        Log(Debug::Verbose) << "OpenXRViewer: swapBuffers";
         auto* session = OpenXREnvironment::get().getSession();
         auto* xr = OpenXREnvironment::get().getManager();
 
@@ -311,9 +310,6 @@ namespace MWVR
             if (xr->sessionRunning())
             {
                 xr->beginFrame();
-                auto& poses = session->predictedPoses();
-                //auto menuPose = poses.head[(int)TrackedSpace::STAGE];
-                //mViews["MenuView"]->setPredictedPose(menuPose);
             }
         }
 

apps/openmw/mwworld/worldimp.cpp

@@ -23,6 +23,7 @@
 #include <components/resource/resourcesystem.hpp>
 
 #include <components/sceneutil/positionattitudetransform.hpp>
+#include <components/sceneutil/visitor.hpp>
 
 #include <components/detournavigator/debug.hpp>
 #include <components/detournavigator/navigatorimpl.hpp>
@@ -1112,11 +1113,34 @@ namespace MWWorld
         return facedObject;
     }
 
+    std::pair<MWWorld::Ptr, osg::Node*> World::getPointedAtObject()
+    {
+        if (MWBase::Environment::get().getWindowManager()->isGuiMode() &&
+            MWBase::Environment::get().getWindowManager()->isConsoleMode())
+        {
+            return getPointedAtObject(getMaxActivationDistance() * 50, false);
+        }
+        else
+        {
+            auto pointedAtObject = getPointedAtObject(getActivationDistancePlusTelekinesis(), true);
+
+            if (!pointedAtObject.first.isEmpty() && !pointedAtObject.first.getClass().allowTelekinesis(pointedAtObject.first)
+                && mDistanceToFacedObject > getMaxActivationDistance() && !MWBase::Environment::get().getWindowManager()->isGuiMode())
+                return std::pair<MWWorld::Ptr, osg::Node*>();
+            return pointedAtObject;
+        }
+    }
+
    float World::getDistanceToFacedObject()
    {
         return mDistanceToFacedObject;
    }
 
+   float World::getDistanceToPointedAtObject()
+   {
+       return mDistanceToPointedAtObject;
+   }
+
     osg::Matrixf World::getActorHeadTransform(const MWWorld::ConstPtr& actor) const
     {
         const MWRender::Animation *anim = mRendering->getAnimation(actor);
@@ -2046,6 +2070,46 @@ namespace MWWorld
         return facedObject;
     }
 
+    std::pair<MWWorld::Ptr, osg::Node*> World::getPointedAtObject(float maxDistance, bool ignorePlayer)
+    {
+        auto sceneRoot = mRendering->getLightRoot();
+
+        // Find the transform giving the finger's pointing direction.
+        SceneUtil::FindByNameVisitor findPointerVisitor("Pointer Transform", osg::NodeVisitor::TRAVERSE_ALL_CHILDREN);
+        sceneRoot->accept(findPointerVisitor);
+        auto pointerTransform = findPointerVisitor.mFoundNode;
+        if (pointerTransform)
+        {
+            auto pat = pointerTransform->asTransform()->asPositionAttitudeTransform();
+
+            // Discover the world position of the finger
+            // (This is actually the base of the last joint bone but so long as it is pointing forward it serves the same purpose).
+            osg::Matrix worldMatrix = osg::computeLocalToWorld(pointerTransform->getParentalNodePaths()[0]);
+            pat->computeLocalToWorldMatrix(worldMatrix, nullptr);
+            osg::Vec3 translate;
+            osg::Quat rotation;
+            osg::Vec3 scale;
+            osg::Quat scaleRotation;
+            worldMatrix.decompose(translate, rotation, scale, scaleRotation);
+            osg::Vec3f direction = rotation * osg::Vec3f(-1, 0, 0);
+            direction.normalize();
+
+            osg::Vec3f raySource = translate;
+            osg::Vec3f rayTarget = translate + direction * maxDistance;
+
+            auto rayToObject = mRendering->castRay(raySource, rayTarget, ignorePlayer, false);
+            if (rayToObject.mHit)
+                mDistanceToPointedAtObject = (rayToObject.mHitPointWorld - raySource).length();
+            else
+            // Leave a very large but not too large number to permit visualizing a beam going into "infinity"
+                mDistanceToPointedAtObject = 10000.f;
+
+            return { rayToObject.mHitObject, rayToObject.mHitNode };
+        }
+
+        return std::pair<MWWorld::Ptr, osg::Node*>();
+    }
+
     bool World::isCellExterior() const
     {
         const CellStore *currentCell = mWorldScene->getCurrentCell();

apps/openmw/mwworld/worldimp.hpp

@@ -140,6 +140,8 @@ namespace MWWorld
 
             MWWorld::Ptr getFacedObject(float maxDistance, bool ignorePlayer=true);
 
+            std::pair<MWWorld::Ptr, osg::Node*> getPointedAtObject(float maxDistance, bool ignorePlayer=true);
+
     public: // FIXME
             void addContainerScripts(const Ptr& reference, CellStore* cell) override;
             void removeContainerScripts(const Ptr& reference) override;
@@ -174,6 +176,7 @@ namespace MWWorld
             float mSwimHeightScale;
 
             float mDistanceToFacedObject;
+            float mDistanceToPointedAtObject;
 
             bool mTeleportEnabled;
             bool mLevitationEnabled;
@@ -374,7 +377,11 @@ namespace MWWorld
             MWWorld::Ptr getFacedObject() override;
             ///< Return pointer to the object the player is looking at, if it is within activation range
 
+            std::pair<MWWorld::Ptr, osg::Node*> getPointedAtObject() override;
+            ///< Return pointer to the object and/or node the player is currently pointing at
+
             float getDistanceToFacedObject() override;
+            float getDistanceToPointedAtObject() override;
 
             /// Returns a pointer to the object the provided object would hit (if within the
             /// specified distance), and the point where the hit occurs. This will attempt to

components/myguiplatform/myguirendermanager.cpp

@@ -99,7 +99,6 @@ public:
     // Stage 2: execute the draw calls. Run during the Draw traversal. May run in parallel with the update traversal of the next frame.
     virtual void drawImplementation(osg::RenderInfo &renderInfo) const
     {
-        std::cout << "DrawImplementation" << std::endl;
         osg::State *state = renderInfo.getState();
 
         state->pushStateSet(mStateSet);