Implement frame drop support in the videoplayer (Fixes #1343)

apps/openmw/mwrender/videoplayer.cpp

@@ -44,23 +44,15 @@ extern "C"
         #include <libavutil/time.h>
     #endif
 
-    // From libavutil version 52.2.0 and onward the declaration of
-    // AV_CH_LAYOUT_* is removed from libavcodec/avcodec.h and moved to
-    // libavutil/channel_layout.h
     #if AV_VERSION_INT(52, 2, 0) <= AV_VERSION_INT(LIBAVUTIL_VERSION_MAJOR, \
         LIBAVUTIL_VERSION_MINOR, LIBAVUTIL_VERSION_MICRO)
         #include <libavutil/channel_layout.h>
     #endif
 
-    // WARNING: avcodec versions up to 54.54.100 potentially crashes on Windows 64bit.
-
-    // From version 54.56 binkaudio encoding format changed from S16 to FLTP. See:
-    // https://gitorious.org/ffmpeg/ffmpeg/commit/7bfd1766d1c18f07b0a2dd042418a874d49ea60d
-    // http://ffmpeg.zeranoe.com/forum/viewtopic.php?f=15&t=872
     #ifdef HAVE_LIBSWRESAMPLE
     #include <libswresample/swresample.h>
     #else
-    /* FIXME: remove this section once libswresample is available on all platforms */
+    // FIXME: remove this section once libswresample is packaged for Debian
     #include <libavresample/avresample.h>
     #include <libavutil/opt.h>
     #define SwrContext AVAudioResampleContext
@@ -75,12 +67,12 @@ extern "C"
 #define MAX_VIDEOQ_SIZE (5 * 256 * 1024)
 #define AV_SYNC_THRESHOLD 0.01
 #define AUDIO_DIFF_AVG_NB 20
-#define VIDEO_PICTURE_QUEUE_SIZE 1
+#define VIDEO_PICTURE_QUEUE_SIZE 50
 
 enum {
-    AV_SYNC_AUDIO_MASTER,
+    AV_SYNC_AUDIO_MASTER, // Play audio with no frame drops, sync video to audio
-    AV_SYNC_VIDEO_MASTER,
+    AV_SYNC_VIDEO_MASTER, // Play video with no frame drops, sync audio to video
-    AV_SYNC_EXTERNAL_MASTER,
+    AV_SYNC_EXTERNAL_MASTER, // Sync audio and video to an external clock
 
     AV_SYNC_DEFAULT = AV_SYNC_EXTERNAL_MASTER
 };
@@ -121,10 +113,10 @@ struct VideoState {
       : format_ctx(NULL), av_sync_type(AV_SYNC_DEFAULT)
       , external_clock_base(0.0)
       , audio_st(NULL)
-      , video_st(NULL), frame_last_pts(0.0), frame_last_delay(0.0),
+      , video_st(NULL), frame_last_pts(0.0)
-        video_clock(0.0), sws_context(NULL), rgbaFrame(NULL), pictq_size(0),
+      , video_clock(0.0), sws_context(NULL), rgbaFrame(NULL), pictq_size(0)
-        pictq_rindex(0), pictq_windex(0)
+      , pictq_rindex(0), pictq_windex(0)
-      , refresh_rate_ms(10), refresh(false), quit(false), display_ready(false)
+      , quit(false)
     {
         // Register all formats and codecs
         av_register_all();
@@ -143,15 +135,12 @@ struct VideoState {
     static void video_thread_loop(VideoState *is);
     static void decode_thread_loop(VideoState *is);
 
-    void video_display();
+    void video_display(VideoPicture* vp);
-    void video_refresh_timer();
+    void video_refresh();
 
     int queue_picture(AVFrame *pFrame, double pts);
     double synchronize_video(AVFrame *src_frame, double pts);
 
-    static void video_refresh(VideoState *is);
-
-
     double get_audio_clock()
     { return this->AudioTrack->getTimeOffset(); }
 
@@ -189,7 +178,6 @@ struct VideoState {
 
     AVStream**  video_st;
     double      frame_last_pts;
-    double      frame_last_delay;
     double      video_clock; ///<pts of last decoded frame / predicted pts of next decoded frame
     PacketQueue videoq;
     SwsContext*  sws_context;
@@ -203,12 +191,7 @@ struct VideoState {
     boost::thread parse_thread;
     boost::thread video_thread;
 
-    boost::thread refresh_thread;
-    volatile int refresh_rate_ms;
-
-    volatile bool refresh;
     volatile bool quit;
-    volatile bool display_ready;
 };
 
 
@@ -667,23 +650,8 @@ int64_t VideoState::OgreResource_Seek(void *user_data, int64_t offset, int whenc
     return stream->tell();
 }
 
-
+void VideoState::video_display(VideoPicture *vp)
-void VideoState::video_refresh(VideoState* is)
-{
-    boost::system_time t = boost::get_system_time();
-    while(!is->quit)
-    {
-        t += boost::posix_time::milliseconds(is->refresh_rate_ms);
-        boost::this_thread::sleep(t);
-        is->refresh = true;
-    }
-}
-
-
-void VideoState::video_display()
 {
-    VideoPicture *vp = &this->pictq[this->pictq_rindex];
-
     if((*this->video_st)->codec->width != 0 && (*this->video_st)->codec->height != 0)
     {
 
@@ -698,55 +666,53 @@ void VideoState::video_display()
         Ogre::PixelBox pb((*this->video_st)->codec->width, (*this->video_st)->codec->height, 1, Ogre::PF_BYTE_RGBA, &vp->data[0]);
         Ogre::HardwarePixelBufferSharedPtr buffer = mTexture->getBuffer();
         buffer->blitFromMemory(pb);
-        this->display_ready = true;
     }
 }
 
-void VideoState::video_refresh_timer()
+void VideoState::video_refresh()
 {
-    VideoPicture *vp;
-    double delay;
-
     if(this->pictq_size == 0)
         return;
 
-    vp = &this->pictq[this->pictq_rindex];
+    if (this->av_sync_type == AV_SYNC_VIDEO_MASTER)
-
-    delay = vp->pts - this->frame_last_pts; /* the pts from last time */
-    if(delay <= 0 || delay >= 1.0) {
-        /* if incorrect delay, use previous one */
-        delay = this->frame_last_delay;
-    }
-    /* save for next time */
-    this->frame_last_delay = delay;
-    this->frame_last_pts = vp->pts;
-
-    /* FIXME: Syncing should be done in the decoding stage, where frames can be
-     * skipped or duplicated as needed. */
-    /* update delay to sync to audio if not master source */
-    if(this->av_sync_type != AV_SYNC_VIDEO_MASTER)
     {
-        double diff = this->get_video_clock() - this->get_master_clock();
+        VideoPicture* vp = &this->pictq[this->pictq_rindex];
-
+        this->video_display(vp);
-        /* Skip or repeat the frame. Take delay into account
+        this->pictq_rindex = (pictq_rindex+1) % VIDEO_PICTURE_QUEUE_SIZE;
-         * FFPlay still doesn't "know if this is the best guess." */
+        this->frame_last_pts = vp->pts;
-        double sync_threshold = std::max(delay, AV_SYNC_THRESHOLD);
+        this->pictq_mutex.lock();
-        if(diff <= -sync_threshold)
+        this->pictq_size--;
-            delay = 0;
+        this->pictq_cond.notify_one();
-        else if(diff >= sync_threshold)
+        this->pictq_mutex.unlock();
-            delay = 2 * delay;
     }
+    else
+    {
+        const float threshold = 0.03;
+        if (this->pictq[pictq_rindex].pts > this->get_master_clock() + threshold)
+            return; // not ready yet to show this picture
+        int i=0;
+        for (; i<this->pictq_size-1; ++i)
+        {
+            if (this->pictq[pictq_rindex].pts + threshold <= this->get_master_clock())
+                this->pictq_rindex = (this->pictq_rindex+1) % VIDEO_PICTURE_QUEUE_SIZE; // not enough time to show this picture
+            else
+                break;
+        }
 
-    this->refresh_rate_ms = std::max<int>(1, (int)(delay*1000.0));
+        VideoPicture* vp = &this->pictq[this->pictq_rindex];
-    /* show the picture! */
-    this->video_display();
 
-    /* update queue for next picture! */
+        this->video_display(vp);
-    this->pictq_rindex = (this->pictq_rindex+1) % VIDEO_PICTURE_QUEUE_SIZE;
+
-    this->pictq_mutex.lock();
+        this->frame_last_pts = vp->pts;
-    this->pictq_size--;
+
-    this->pictq_cond.notify_one();
+        this->pictq_mutex.lock();
-    this->pictq_mutex.unlock();
+        this->pictq_size -= i;
+        // update queue for next picture
+        this->pictq_size--;
+        this->pictq_rindex++;
+        this->pictq_cond.notify_one();
+        this->pictq_mutex.unlock();
+    }
 }
 
 
@@ -916,7 +882,7 @@ void VideoState::decode_thread_loop(VideoState *self)
         while(!self->quit)
         {
             // EOF reached, all packets processed, we can exit now
-            if(self->audioq.nb_packets == 0 && self->videoq.nb_packets == 0)
+            if(self->audioq.nb_packets == 0 && self->videoq.nb_packets == 0 && self->pictq_size == 0)
                 break;
             boost::this_thread::sleep(boost::posix_time::milliseconds(100));
         }
@@ -937,11 +903,7 @@ bool VideoState::update()
     if(this->quit)
         return false;
 
-    if(this->refresh)
+    this->video_refresh();
-    {
-        this->refresh = false;
-        this->video_refresh_timer();
-    }
     return true;
 }
 
@@ -982,12 +944,9 @@ int VideoState::stream_open(int stream_index, AVFormatContext *pFormatCtx)
     case AVMEDIA_TYPE_VIDEO:
         this->video_st = pFormatCtx->streams + stream_index;
 
-        this->frame_last_delay = 40e-3;
-
         codecCtx->get_buffer = our_get_buffer;
         codecCtx->release_buffer = our_release_buffer;
         this->video_thread = boost::thread(video_thread_loop, this);
-        this->refresh_thread = boost::thread(video_refresh, this);
         break;
 
     default:
@@ -1004,8 +963,6 @@ void VideoState::init(const std::string& resourceName)
     unsigned int i;
 
     this->av_sync_type = AV_SYNC_DEFAULT;
-    this->refresh_rate_ms = 10;
-    this->refresh = false;
     this->quit = false;
 
     this->stream = Ogre::ResourceGroupManager::getSingleton().openResource(resourceName);
@@ -1108,8 +1065,6 @@ void VideoState::deinit()
         this->parse_thread.join();
     if (this->video_thread.joinable())
         this->video_thread.join();
-    if (this->refresh_thread.joinable())
-        this->refresh_thread.join();
 
     if(this->audio_st)
         avcodec_close((*this->audio_st)->codec);