Object placement raycasts should use the rendering meshes

apps/openmw/mwrender/renderingmanager.cpp

@@ -438,21 +438,6 @@ namespace MWRender
         mViewer->getCamera()->setCullMask(oldCullMask);
     }
 
-    void RenderingManager::getCameraToViewportRay(float nX, float nY, osg::Vec3f &origin, osg::Vec3f &dest)
-    {
-        osg::Matrix viewProj = mViewer->getCamera()->getViewMatrix() * mViewer->getCamera()->getProjectionMatrix();
-        osg::Matrix invViewProj = viewProj.inverse(viewProj);
-
-        nX = nX * 2 - 1;
-        nY = nY * -2 + 1;
-
-        osg::Vec3f start (nX, nY, -1.f);
-        osg::Vec3f end (nX, nY, 1.f);
-
-        origin = invViewProj.preMult(start);
-        dest = invViewProj.preMult(end);
-    }
-
     osg::Vec4f RenderingManager::getScreenBounds(const MWWorld::Ptr& ptr)
     {
         if (!ptr.getRefData().getBaseNode())
@@ -487,7 +472,61 @@ namespace MWRender
         return osg::Vec4f(min_x, min_y, max_x, max_y);
     }
 
-    MWWorld::Ptr RenderingManager::getFacedObject(const float nX, const float nY, float maxDistance, bool ignorePlayer)
+    RenderingManager::RayResult getIntersectionResult (osgUtil::LineSegmentIntersector* intersector)
+    {
+        RenderingManager::RayResult result;
+        result.mHit = false;
+        if (intersector->containsIntersections())
+        {
+            result.mHit = true;
+            osgUtil::LineSegmentIntersector::Intersection intersection = intersector->getFirstIntersection();
+
+            result.mHitPointWorld = intersection.getWorldIntersectPoint();
+            result.mHitNormalWorld = intersection.getWorldIntersectNormal();
+
+            PtrHolder* ptrHolder = NULL;
+            for (osg::NodePath::const_iterator it = intersection.nodePath.begin(); it != intersection.nodePath.end(); ++it)
+            {
+                osg::UserDataContainer* userDataContainer = (*it)->getUserDataContainer();
+                if (!userDataContainer)
+                    continue;
+                for (unsigned int i=0; i<userDataContainer->getNumUserObjects(); ++i)
+                {
+                    if (PtrHolder* p = dynamic_cast<PtrHolder*>(userDataContainer->getUserObject(i)))
+                        ptrHolder = p;
+                }
+            }
+
+            if (ptrHolder)
+                result.mHitObject = ptrHolder->mPtr;
+        }
+
+        return result;
+
+    }
+
+    RenderingManager::RayResult RenderingManager::castRay(const osg::Vec3f& origin, const osg::Vec3f& dest, bool ignorePlayer, bool ignoreActors)
+    {
+        osg::ref_ptr<osgUtil::LineSegmentIntersector> intersector (new osgUtil::LineSegmentIntersector(osgUtil::LineSegmentIntersector::MODEL,
+            origin, dest));
+        intersector->setIntersectionLimit(osgUtil::LineSegmentIntersector::LIMIT_NEAREST);
+
+        osgUtil::IntersectionVisitor intersectionVisitor(intersector);
+        int mask = intersectionVisitor.getTraversalMask();
+        mask &= ~(Mask_RenderToTexture|Mask_Sky|Mask_Debug|Mask_Effect|Mask_Water);
+        if (ignorePlayer)
+            mask &= ~(Mask_Player);
+        if (ignoreActors)
+            mask &= ~(Mask_Actor|Mask_Player);
+
+        intersectionVisitor.setTraversalMask(mask);
+
+        mRootNode->accept(intersectionVisitor);
+
+        return getIntersectionResult(intersector);
+    }
+
+    RenderingManager::RayResult RenderingManager::castCameraToViewportRay(const float nX, const float nY, float maxDistance, bool ignorePlayer, bool ignoreActors)
     {
         osg::ref_ptr<osgUtil::LineSegmentIntersector> intersector (new osgUtil::LineSegmentIntersector(osgUtil::LineSegmentIntersector::PROJECTION,
                                                                                                        nX * 2.f - 1.f, nY * (-2.f) + 1.f));
@@ -506,33 +545,14 @@ namespace MWRender
         mask &= ~(Mask_RenderToTexture|Mask_Sky|Mask_Debug|Mask_Effect|Mask_Water);
         if (ignorePlayer)
             mask &= ~(Mask_Player);
+        if (ignoreActors)
+            mask &= ~(Mask_Actor|Mask_Player);
 
         intersectionVisitor.setTraversalMask(mask);
 
         mViewer->getCamera()->accept(intersectionVisitor);
 
-        if (intersector->containsIntersections())
+        return getIntersectionResult(intersector);
-        {
-            osgUtil::LineSegmentIntersector::Intersection intersection = intersector->getFirstIntersection();
-
-            PtrHolder* ptrHolder = NULL;
-            for (osg::NodePath::const_iterator it = intersection.nodePath.begin(); it != intersection.nodePath.end(); ++it)
-            {
-                osg::UserDataContainer* userDataContainer = (*it)->getUserDataContainer();
-                if (!userDataContainer)
-                    continue;
-                for (unsigned int i=0; i<userDataContainer->getNumUserObjects(); ++i)
-                {
-                    if (PtrHolder* p = dynamic_cast<PtrHolder*>(userDataContainer->getUserObject(i)))
-                        ptrHolder = p;
-                }
-            }
-
-            if (ptrHolder)
-                return ptrHolder->mPtr;
-        }
-
-        return MWWorld::Ptr();
     }
 
     void RenderingManager::updatePtr(const MWWorld::Ptr &old, const MWWorld::Ptr &updated)

apps/openmw/mwrender/renderingmanager.hpp

@@ -85,16 +85,23 @@ namespace MWRender
         /// Take a screenshot of w*h onto the given image, not including the GUI.
         void screenshot(osg::Image* image, int w, int h);
 
+        struct RayResult
+        {
+            bool mHit;
+            osg::Vec3f mHitNormalWorld;
+            osg::Vec3f mHitPointWorld;
+            MWWorld::Ptr mHitObject;
+        };
+
+        RayResult castRay(const osg::Vec3f& origin, const osg::Vec3f& dest, bool ignorePlayer, bool ignoreActors=false);
+
         /// Return the object under the mouse cursor / crosshair position, given by nX and nY normalized screen coordinates,
         /// where (0,0) is the top left corner.
-        MWWorld::Ptr getFacedObject(const float nX, const float nY, float maxDistance, bool ignorePlayer);
+        RayResult castCameraToViewportRay(const float nX, const float nY, float maxDistance, bool ignorePlayer, bool ignoreActors=false);
 
         /// Get the bounding box of the given object in screen coordinates as (minX, minY, maxX, maxY), with (0,0) being the top left corner.
         osg::Vec4f getScreenBounds(const MWWorld::Ptr& ptr);
 
-        /// Get a camera to viewport ray for normalized screen coordinates nX and nY, with the top left corner being at (0,0)
-        void getCameraToViewportRay(float nX, float nY, osg::Vec3f& origin, osg::Vec3f& dest);
-
         void setSkyEnabled(bool enabled);
 
         bool toggleRenderMode(RenderMode mode);

apps/openmw/mwworld/worldimp.cpp

@@ -1662,11 +1662,11 @@ namespace MWWorld
         {
             float x, y;
             MWBase::Environment::get().getWindowManager()->getMousePosition(x, y);
-            return mRendering->getFacedObject(x, y, maxDistance, ignorePlayer);
+            return mRendering->castCameraToViewportRay(x, y, maxDistance, ignorePlayer).mHitObject;
         }
         else
         {
-            return mRendering->getFacedObject(0.5f, 0.5f, maxDistance, ignorePlayer);
+            return mRendering->castCameraToViewportRay(0.5f, 0.5f, maxDistance, ignorePlayer).mHitObject;
         }
     }
 
@@ -1790,25 +1790,18 @@ namespace MWWorld
 
     MWWorld::Ptr World::placeObject (const MWWorld::Ptr& object, float cursorX, float cursorY, int amount)
     {
-        osg::Vec3f origin, dest;
-        mRendering->getCameraToViewportRay(cursorX, cursorY, origin, dest);
-
         const float maxDist = 200.f;
-        osg::Vec3f dir = (dest - origin);
-        dir.normalize();
-        dest = origin + dir * maxDist;
 
-        MWPhysics::PhysicsSystem::RayResult result = mPhysics->castRay(origin, dest, MWWorld::Ptr(),
+        MWRender::RenderingManager::RayResult result = mRendering->castCameraToViewportRay(cursorX, cursorY, maxDist, true, true);
-            MWPhysics::CollisionType_World|MWPhysics::CollisionType_HeightMap);
 
         CellStore* cell = getPlayerPtr().getCell();
 
         ESM::Position pos = getPlayerPtr().getRefData().getPosition();
         if (result.mHit)
         {
-            pos.pos[0] = result.mHitPos.x();
+            pos.pos[0] = result.mHitPointWorld.x();
-            pos.pos[1] = result.mHitPos.y();
+            pos.pos[1] = result.mHitPointWorld.y();
-            pos.pos[2] = result.mHitPos.z();
+            pos.pos[2] = result.mHitPointWorld.z();
         }
         // We want only the Z part of the player's rotation
         pos.rot[0] = 0;
@@ -1828,21 +1821,13 @@ namespace MWWorld
 
     bool World::canPlaceObject(float cursorX, float cursorY)
     {
-        osg::Vec3f origin, dest;
-        mRendering->getCameraToViewportRay(cursorX, cursorY, origin, dest);
-
         const float maxDist = 200.f;
-        osg::Vec3f dir = (dest - origin);
+        MWRender::RenderingManager::RayResult result = mRendering->castCameraToViewportRay(cursorX, cursorY, maxDist, true, true);
-        dir.normalize();
-        dest = origin + dir * maxDist;
-
-        MWPhysics::PhysicsSystem::RayResult result = mPhysics->castRay(origin, dest, MWWorld::Ptr(),
-            MWPhysics::CollisionType_World|MWPhysics::CollisionType_HeightMap);
 
         if (result.mHit)
         {
             // check if the wanted position is on a flat surface, and not e.g. against a vertical wall
-            if (std::acos(result.mHitNormal * osg::Vec3f(0,0,1)) >= osg::DegreesToRadians(30.f))
+            if (std::acos(result.mHitNormalWorld * osg::Vec3f(0,0,1)) >= osg::DegreesToRadians(30.f))
                 return false;
 
             return true;
@@ -1925,10 +1910,9 @@ namespace MWWorld
 
         float len = 100.0;
 
-        MWPhysics::PhysicsSystem::RayResult result = mPhysics->castRay(orig, orig+dir*len, MWWorld::Ptr(),
+        MWRender::RenderingManager::RayResult result = mRendering->castRay(orig, orig+dir*len, true, true);
-            MWPhysics::CollisionType_World|MWPhysics::CollisionType_HeightMap);
         if (result.mHit)
-            pos.pos[2] = result.mHitPos.z();
+            pos.pos[2] = result.mHitPointWorld.z();
 
         // copy the object and set its count
         int origCount = object.getRefData().getCount();