video playback

apps/openmw/CMakeLists.txt

@@ -16,7 +16,7 @@ source_group(game FILES ${GAME} ${GAME_HEADER})
 add_openmw_dir (mwrender
     renderingmanager debugging sky player animation npcanimation creatureanimation actors objects
     renderinginterface localmap occlusionquery terrain terrainmaterial water shadows
-    compositors characterpreview externalrendering globalmap
+    compositors characterpreview externalrendering globalmap videoplayer
     )
 
 add_openmw_dir (mwinput
@@ -108,6 +108,11 @@ target_link_libraries(openmw
     components
 )
 
+if (USE_FFMPEG)
+    target_link_libraries(openmw
+    "swscale")
+endif()
+
 # Fix for not visible pthreads functions for linker with glibc 2.15
 if (UNIX AND NOT APPLE)
 target_link_libraries(openmw ${CMAKE_THREAD_LIBS_INIT})

apps/openmw/mwbase/world.hpp

@@ -291,6 +291,10 @@ namespace MWBase
             /// 1 - only waiting \n
             /// 2 - player is underwater \n
             /// 3 - enemies are nearby (not implemented)
+
+
+            /// \todo this does not belong here
+            virtual void playVideo(const std::string& name) = 0;
     };
 }
 

apps/openmw/mwgui/mode.hpp

@@ -41,7 +41,9 @@ namespace MWGui
       GM_Loading,
       GM_LoadingWallpaper,
 
-      GM_QuickKeysMenu
+      GM_QuickKeysMenu,
+
+      GM_Video
     };
 
   // Windows shown in inventory mode

apps/openmw/mwgui/windowmanagerimp.cpp

@@ -371,6 +371,9 @@ void WindowManager::updateVisible()
         case GM_Loading:
             MyGUI::PointerManager::getInstance().setVisible(false);
             break;
+        case GM_Video:
+            mHud->setVisible(false);
+            break;
         default:
             // Unsupported mode, switch back to game
             break;

apps/openmw/mwrender/renderingmanager.cpp

@@ -36,6 +36,7 @@
 #include "npcanimation.hpp"
 #include "externalrendering.hpp"
 #include "globalmap.hpp"
+#include "videoplayer.hpp"
 
 using namespace MWRender;
 using namespace Ogre;
@@ -159,6 +160,8 @@ RenderingManager::RenderingManager (OEngine::Render::OgreRenderer& _rend, const
 
     mOcclusionQuery = new OcclusionQuery(&mRendering, mSkyManager->getSunNode());
 
+    mVideoPlayer = new VideoPlayer(mRendering.getScene ());
+
     mSun = 0;
 
     mDebugging = new Debugging(mMwRoot, engine);
@@ -180,6 +183,7 @@ RenderingManager::~RenderingManager ()
     delete mOcclusionQuery;
     delete mCompositors;
     delete mWater;
+    delete mVideoPlayer;
 }
 
 MWRender::SkyManager* RenderingManager::getSkyManager()
@@ -339,6 +343,8 @@ void RenderingManager::update (float duration)
 
     mRendering.update(duration);
 
+    mVideoPlayer->update ();
+
     MWWorld::RefData &data = 
         MWBase::Environment::get()
             .getWorld()
@@ -898,4 +904,9 @@ void RenderingManager::setupExternalRendering (MWRender::ExternalRendering& rend
     rendering.setup (mRendering.getScene());
 }
 
+void RenderingManager::playVideo(const std::string& name)
+{
+    mVideoPlayer->play ("video/" + name);
+}
+
 } // namespace

apps/openmw/mwrender/renderingmanager.hpp

@@ -45,6 +45,7 @@ namespace MWRender
     class Compositors;
     class ExternalRendering;
     class GlobalMap;
+    class VideoPlayer;
 
 class RenderingManager: private RenderingInterface, public Ogre::WindowEventListener {
 
@@ -195,6 +196,8 @@ class RenderingManager: private RenderingInterface, public Ogre::WindowEventList
 
     void setupExternalRendering (MWRender::ExternalRendering& rendering);
 
+    void playVideo(const std::string& name);
+
   protected:
 	virtual void windowResized(Ogre::RenderWindow* rw);
     virtual void windowClosed(Ogre::RenderWindow* rw);
@@ -248,6 +251,8 @@ class RenderingManager: private RenderingInterface, public Ogre::WindowEventList
     MWRender::Shadows* mShadows;
 
     MWRender::Compositors* mCompositors;
+
+    VideoPlayer* mVideoPlayer;
 };
 
 }

apps/openmw/mwrender/videoplayer.cpp

@@ -0,0 +1,395 @@
+#include "videoplayer.hpp"
+
+//#ifdef OPENMW_USE_FFMPEG
+
+#include <OgreMaterialManager.h>
+#include <OgreSceneManager.h>
+#include <OgreMaterial.h>
+#include <OgreHardwarePixelBuffer.h>
+
+
+extern "C"
+{
+#include <libavformat/avformat.h>
+#include <libavcodec/avcodec.h>
+#include <libswscale/swscale.h>
+}
+
+#include "../mwbase/windowmanager.hpp"
+#include "../mwbase/environment.hpp"
+
+
+namespace MWRender
+{
+
+    int OgreResource_Read(void *opaque, uint8_t *buf, int buf_size)
+    {
+        Ogre::DataStreamPtr stream = *((Ogre::DataStreamPtr*)opaque);
+
+        int num_read = stream->size() - stream->tell();
+
+        if (num_read > buf_size)
+            num_read = buf_size;
+
+        stream->read(buf, num_read);
+        return num_read;
+    }
+
+    int OgreResource_Write(void *opaque, uint8_t *buf, int buf_size)
+    {
+        Ogre::DataStreamPtr stream = *((Ogre::DataStreamPtr*)opaque);
+
+        int num_write = stream->size() - stream->tell();
+
+        if (num_write > buf_size)
+            num_write = buf_size;
+
+        stream->write (buf, num_write);
+        return num_write;
+    }
+
+    int64_t OgreResource_Seek(void *opaque, int64_t offset, int whence)
+    {
+        Ogre::DataStreamPtr stream = *((Ogre::DataStreamPtr*)opaque);
+
+        switch (whence)
+        {
+        case SEEK_SET:
+            stream->seek(offset);
+        case SEEK_CUR:
+            stream->seek(stream->tell() + offset);
+        case SEEK_END:
+            stream->seek(stream->size() + offset);
+        case AVSEEK_SIZE:
+            return stream->size();
+        default:
+            return -1;
+        }
+
+        return stream->tell();
+    }
+
+    VideoPlayer::VideoPlayer(Ogre::SceneManager *sceneMgr)
+        : mAvContext(NULL)
+        , mVideoStreamId(-1)
+    {
+        Ogre::MaterialPtr videoMaterial = Ogre::MaterialManager::getSingleton ().create("VideoMaterial", "General");
+        videoMaterial->getTechnique(0)->getPass(0)->setDepthWriteEnabled(false);
+        videoMaterial->getTechnique(0)->getPass(0)->setDepthCheckEnabled(false);
+        videoMaterial->getTechnique(0)->getPass(0)->setLightingEnabled(false);
+        mTextureUnit = videoMaterial->getTechnique(0)->getPass(0)->createTextureUnitState();
+
+        mRectangle = new Ogre::Rectangle2D(true);
+        mRectangle->setCorners(-1.0, 1.0, 1.0, -1.0);
+        mRectangle->setMaterial("VideoMaterial");
+        mRectangle->setRenderQueueGroup(Ogre::RENDER_QUEUE_OVERLAY+1);
+        // Use infinite AAB to always stay visible
+        Ogre::AxisAlignedBox aabInf;
+        aabInf.setInfinite();
+        mRectangle->setBoundingBox(aabInf);
+        // Attach background to the scene
+        Ogre::SceneNode* node = sceneMgr->getRootSceneNode()->createChildSceneNode();
+        node->attachObject(mRectangle);
+        mRectangle->setVisible(false);
+        mRectangle->setVisibilityFlags (0x1);
+    }
+
+    VideoPlayer::~VideoPlayer()
+    {
+        if (mAvContext)
+            deleteContext();
+
+        delete mRectangle;
+    }
+
+    void VideoPlayer::play (const std::string& resourceName)
+    {
+        mStream = Ogre::ResourceGroupManager::getSingleton ().openResource (resourceName);
+
+
+        mVideoStreamId = -1;
+        mEOF = false;
+
+        // if something is already playing, close it
+        if (mAvContext)
+            close();
+
+        mRectangle->setVisible(true);
+
+        MWBase::Environment::get().getWindowManager ()->pushGuiMode (MWGui::GM_Video);
+
+
+        // BASIC INITIALIZATION
+
+        // Load all the decoders
+        av_register_all();
+
+        AVIOContext	 *ioContext = 0;
+
+        int err = 0;
+
+        mAvContext = avformat_alloc_context();
+        if (!mAvContext)
+            throwError(0);
+
+        ioContext = avio_alloc_context(NULL, 0, 0, &mStream, OgreResource_Read, OgreResource_Write, OgreResource_Seek);
+        if (!ioContext)
+            throwError(0);
+
+        mAvContext->pb = ioContext;
+
+        err = avformat_open_input(&mAvContext, resourceName.c_str(), NULL, NULL);
+        if (err != 0)
+            throwError(err);
+
+        err = avformat_find_stream_info(mAvContext, 0);
+        if (err < 0)
+            throwError(err);
+
+
+
+
+        // INIT VIDEO
+
+        // Find the video stream among the different streams
+        for (unsigned int i = 0; i < mAvContext->nb_streams; i++)
+        {
+            if (mAvContext->streams[i]->codec->codec_type == AVMEDIA_TYPE_VIDEO)
+            {
+                mVideoStreamId = i;
+                break;
+            }
+        }
+
+        if (-1 == mVideoStreamId)
+            throw std::runtime_error("No video stream found in the video");
+
+        // Get the video codec
+        mVideoCodecContext = mAvContext->streams[mVideoStreamId]->codec;
+        if (!mVideoCodecContext)
+            throw std::runtime_error("Stream doesn't have a codec");
+
+        // Get the video decoder
+        mVideoCodec = avcodec_find_decoder(mVideoCodecContext->codec_id);
+        if (NULL == mVideoCodec)
+            throw std::runtime_error("No decoder found");
+
+        // Load the video codec
+        err = avcodec_open2(mVideoCodecContext, mVideoCodec, 0);
+        if (err < 0)
+            throwError (err);
+
+        // Create the frame buffers
+        mRawFrame = avcodec_alloc_frame();
+        mRGBAFrame = avcodec_alloc_frame();
+        if (!mRawFrame || !mRGBAFrame)
+        {
+            throw std::runtime_error("Can't allocate video frames");
+        }
+
+
+        avpicture_alloc ((AVPicture *)mRGBAFrame, PIX_FMT_RGBA, mVideoCodecContext->width, mVideoCodecContext->height);
+
+
+        // Setup the image scaler
+        // All this does is convert from YUV to RGB - note it would be faster to do this in a shader,
+        // but i'm not worried about performance just yet
+        mSwsContext = sws_getContext(mVideoCodecContext->width, mVideoCodecContext->height,
+            mVideoCodecContext->pix_fmt,
+            mVideoCodecContext->width, mVideoCodecContext->height,
+            PIX_FMT_RGBA,
+            SWS_BICUBIC, NULL, NULL, NULL);
+        if (!mSwsContext)
+            throw std::runtime_error("Can't create SWS Context");
+
+        // Get the frame time we need for this video
+        AVRational r = mAvContext->streams[mVideoStreamId]->avg_frame_rate;
+        AVRational r2 = mAvContext->streams[mVideoStreamId]->r_frame_rate;
+        if ((!r.num || !r.den) &&
+            (!r2.num || !r2.den))
+        {
+            std::cerr << "Warning - unable to get the video frame rate. Using standard NTSC frame rate : 29.97 fps." << std::endl;
+            mWantedFrameTime = 1.f / 29.97f;
+        }
+        else
+        {
+            if (r.num && r.den)
+                mWantedFrameTime = 1.f/((float)r.num / r.den);
+            else
+                mWantedFrameTime = 1.f/((float)r2.num / r2.den);
+        }
+
+
+        mTextureUnit->setTextureName ("");
+
+        if (!Ogre::TextureManager::getSingleton ().getByName("VideoTexture").isNull())
+            Ogre::TextureManager::getSingleton().remove("VideoTexture");
+
+        mVideoTexture = Ogre::TextureManager::getSingleton().createManual(
+                    "VideoTexture",
+            Ogre::ResourceGroupManager::DEFAULT_RESOURCE_GROUP_NAME,
+            Ogre::TEX_TYPE_2D,
+            mVideoCodecContext->width, mVideoCodecContext->height,
+            0,
+            Ogre::PF_BYTE_RGBA,
+            Ogre::TU_DEFAULT);
+
+        mTextureUnit->setTextureName ("VideoTexture");
+
+    }
+
+    void VideoPlayer::throwError(int error)
+    {
+        char buffer[4096] = {0};
+
+        if (0 == av_strerror(error, buffer, sizeof(buffer)))
+        {
+            std::stringstream msg;
+            msg << "FFMPEG error: ";
+            msg << buffer << std::endl;
+            throw std::runtime_error(msg.str());
+        }
+        else
+            throw std::runtime_error("Unknown FFMPEG error");
+    }
+
+    bool VideoPlayer::readFrameAndQueue ()
+    {
+        bool ret = true;
+        AVPacket *pkt = NULL;
+
+        // check we're not at eof
+        if (mEOF)
+            ret = false;
+        else
+        {
+            // read frame
+            pkt = (AVPacket *)av_malloc(sizeof(*pkt));
+            int res = av_read_frame(mAvContext, pkt);
+
+            // check we didn't reach eof right now
+            if (res < 0)
+            {
+                mEOF = true;
+                ret = false;
+                av_free(pkt);
+            }
+            else
+            {
+                // When a frame has been read, save it
+                if (!saveFrame(pkt))
+                {
+                    // we failed to save it, which means it was unknown type of frame - delete it here
+                    av_free_packet(pkt);
+                    av_free(pkt);
+                }
+            }
+        }
+
+        return ret;
+    }
+
+    bool VideoPlayer::saveFrame(AVPacket* frame)
+    {
+        bool saved = false;
+
+        if (frame->stream_index == mVideoStreamId)
+        {
+            // If it was a video frame...
+            mVideoPacketQueue.push(frame);
+            saved = true;
+        }
+
+        return saved;
+
+    }
+
+    void VideoPlayer::update()
+    {
+        if (!mAvContext)
+            return;
+
+
+        if (!mVideoPacketQueue.size() && mEOF)
+            close();
+
+        Ogre::Timer timer;
+
+        if (!mVideoPacketQueue.size())
+        {
+            if (readFrameAndQueue())
+                decodeFrontFrame();
+        }
+        else
+            decodeFrontFrame();
+
+        mDecodingTime = timer.getMilliseconds ()/1000.f;
+    }
+
+    void VideoPlayer::decodeFrontFrame ()
+    {
+        int didDecodeFrame = 0;
+
+        // Make sure there is something to decode
+        if (!mVideoPacketQueue.size())
+            return;
+
+        // Get the front frame and decode it
+        AVPacket *videoPacket = mVideoPacketQueue.front();
+        int res;
+        res = avcodec_decode_video2(mVideoCodecContext, mRawFrame, &didDecodeFrame, videoPacket);
+
+        if (res < 0 || !didDecodeFrame)
+            throw std::runtime_error ("an error occured while decoding the video frame");
+
+        // Convert the frame to RGB
+        sws_scale(mSwsContext,
+            mRawFrame->data, mRawFrame->linesize,
+            0, mVideoCodecContext->height,
+            mRGBAFrame->data, mRGBAFrame->linesize);
+
+
+        Ogre::HardwarePixelBufferSharedPtr pixelBuffer = mVideoTexture->getBuffer();
+        Ogre::PixelBox pb(mVideoCodecContext->width, mVideoCodecContext->height, 1, Ogre::PF_BYTE_RGBA, mRGBAFrame->data[0]);
+        pixelBuffer->blitFromMemory(pb);
+
+        //m_displayedFrameCount++;
+        av_free_packet(mVideoPacketQueue.front());
+        av_free(mVideoPacketQueue.front());
+        mVideoPacketQueue.pop();
+    }
+
+    void VideoPlayer::close ()
+    {
+        mRectangle->setVisible (false);
+        MWBase::Environment::get().getWindowManager ()->removeGuiMode (MWGui::GM_Video);
+        deleteContext();
+    }
+
+    void VideoPlayer::deleteContext()
+    {
+        while (mVideoPacketQueue.size())
+        {
+            av_free_packet(mVideoPacketQueue.front());
+            av_free(mVideoPacketQueue.front());
+            mVideoPacketQueue.pop();
+        }
+
+        avcodec_close(mVideoCodecContext);
+
+        avpicture_free((AVPicture *)mRGBAFrame);
+
+        if (mRawFrame)
+            av_free(mRawFrame);
+        if (mRGBAFrame)
+            av_free(mRGBAFrame);
+
+        sws_freeContext(mSwsContext);
+
+        avformat_close_input(&mAvContext);
+
+        mAvContext = NULL;
+    }
+}
+
+//#endif

apps/openmw/mwrender/videoplayer.hpp

@@ -0,0 +1,105 @@
+#ifndef MWRENDER_VIDEOPLAYER_H
+#define MWRENDER_VIDEOPLAYER_H
+
+//#ifdef OPENMW_USE_FFMPEG
+
+
+
+#include <string>
+
+#include <OgreDataStream.h>
+#include <OgreTexture.h>
+
+namespace Ogre
+{
+    class Rectangle2D;
+    class SceneManager;
+    class TextureUnitState;
+}
+
+struct AVFormatContext;
+struct AVCodecContext;
+struct AVCodec;
+struct AVFrame;
+struct SwsContext;
+struct AVPacket;
+
+namespace MWRender
+{
+
+    class VideoPlayer
+    {
+    public:
+        VideoPlayer(Ogre::SceneManager* sceneMgr);
+        ~VideoPlayer();
+
+        void play (const std::string& resourceName);
+
+        void update();
+
+    private:
+        Ogre::Rectangle2D* mRectangle;
+        Ogre::TextureUnitState* mTextureUnit;
+
+        Ogre::DataStreamPtr mStream;
+
+        Ogre::TexturePtr mVideoTexture;
+
+
+    private:
+
+        AVFormatContext* mAvContext;
+
+
+        bool mEOF;
+
+        // VIDEO
+        AVCodecContext* mVideoCodecContext;
+        AVCodec* mVideoCodec;
+        int mVideoStreamId;
+        AVFrame* mRawFrame;
+        AVFrame* mRGBAFrame;
+        SwsContext* mSwsContext;
+        float mWantedFrameTime;
+        float mDecodingTime;
+        std::queue <AVPacket *> mVideoPacketQueue;
+
+
+        bool readFrameAndQueue();
+        bool saveFrame(AVPacket* frame);
+
+        void decodeFrontFrame();
+
+        void close();
+        void deleteContext();
+
+
+        void throwError(int error);
+    };
+
+}
+
+//#else
+/*
+
+
+// If FFMPEG not available, dummy implentation that does nothing
+
+namespace MWRender
+{
+
+    class VideoPlayer
+    {
+    public:
+        VideoPlayer(Ogre::SceneManager* sceneMgr){}
+
+        void play (const std::string& resourceName) {}
+        void update() {}
+    };
+
+}
+*/
+//#endif
+
+
+#endif

apps/openmw/mwscript/docs/vmformat.txt

@@ -205,5 +205,6 @@ op 0x200019e: PlaceAtMe Explicit
 op 0x200019f: GetPcSleep
 op 0x20001a0: ShowMap
 op 0x20001a1: FillMap
-opcodes 0x20001a2-0x3ffffff unused
+op 0x20001a2: PlayBink
+opcodes 0x20001a3-0x3ffffff unused
 

apps/openmw/mwscript/miscextensions.cpp

@@ -21,6 +21,19 @@ namespace MWScript
 {
     namespace Misc
     {
+        class OpPlayBink : public Interpreter::Opcode0
+        {
+        public:
+
+            virtual void execute (Interpreter::Runtime& runtime)
+            {
+                std::string name = runtime.getStringLiteral (runtime[0].mInteger);
+                runtime.pop();
+
+                MWBase::Environment::get().getWorld ()->playVideo (name);
+            }
+        };
+
         class OpGetPcSleep : public Interpreter::Opcode0
         {
         public:
@@ -261,6 +274,7 @@ namespace MWScript
         const int opcodeDontSaveObject = 0x2000153;
         const int opcodeToggleVanityMode = 0x2000174;
         const int opcodeGetPcSleep = 0x200019f;
+        const int opcodePlayBink = 0x20001a2;
 
         void registerExtensions (Compiler::Extensions& extensions)
         {
@@ -286,6 +300,7 @@ namespace MWScript
             extensions.registerInstruction ("togglevanitymode", "", opcodeToggleVanityMode);
             extensions.registerInstruction ("tvm", "", opcodeToggleVanityMode);
             extensions.registerFunction ("getpcsleep", 'l', "", opcodeGetPcSleep);
+            extensions.registerInstruction ("playbink", "S", opcodePlayBink);
         }
 
         void installOpcodes (Interpreter::Interpreter& interpreter)
@@ -307,6 +322,7 @@ namespace MWScript
             interpreter.installSegment5 (opcodeDontSaveObject, new OpDontSaveObject);
             interpreter.installSegment5 (opcodeToggleVanityMode, new OpToggleVanityMode);
             interpreter.installSegment5 (opcodeGetPcSleep, new OpGetPcSleep);
+            interpreter.installSegment5 (opcodePlayBink, new OpPlayBink);
         }
     }
 }

apps/openmw/mwworld/worldimp.cpp

@@ -1281,4 +1281,9 @@ namespace MWWorld
         return 0;
 
     }
+
+    void World::playVideo (const std::string &name)
+    {
+        mRendering->playVideo(name);
+    }
 }

apps/openmw/mwworld/worldimp.hpp

@@ -322,6 +322,9 @@ namespace MWWorld
             /// 1 - only waiting \n
             /// 2 - player is underwater \n
             /// 3 - enemies are nearby (not implemented)
+
+            /// \todo this does not belong here
+            virtual void playVideo(const std::string& name);
     };
 }
 

files/gbuffer/gbuffer.compositor

@@ -72,7 +72,7 @@ compositor gbufferFinalizer
             pass render_scene
             {
                 first_render_queue 51
-                last_render_queue 100
+                last_render_queue 105
             }
         }
         target_output