Load loudness data asynchronously

Currently abuses the output audio streams' background processing thread to do
the work, since there's no generalized threaded processing mechanism.

apps/openmw/mwsound/loudness.cpp

@@ -52,6 +52,8 @@ void Sound_Loudness::analyzeLoudness(const std::vector< char >& data, int sample
         mSamples.push_back(rms);
         ++segment;
     }
+
+    mReady = true;
 }
 
 

apps/openmw/mwsound/loudness.hpp

@@ -12,9 +12,10 @@ class Sound_Loudness {
     // Loudness sample info
     float mSamplesPerSec;
     std::vector<float> mSamples;
+    volatile bool mReady;
 
 public:
-    Sound_Loudness() : mSamplesPerSec(0.0f) { }
+    Sound_Loudness() : mSamplesPerSec(0.0f), mReady(false) { }
 
     /**
      * Analyzes the energy (closely related to loudness) of a sound buffer.
@@ -30,6 +31,7 @@ public:
                          ChannelConfig chans, SampleType type,
                          float valuesPerSecond);
 
+    bool isReady() { return mReady; }
     float getLoudnessAtTime(float sec) const;
 };
 

apps/openmw/mwsound/openal_output.cpp

@@ -23,6 +23,13 @@
 #define MAKE_PTRID(id) ((void*)(uintptr_t)id)
 #define GET_PTRID(ptr) ((ALuint)(uintptr_t)ptr)
 
+namespace
+{
+
+const int sLoudnessFPS = 20; // loudness values per second of audio
+
+}
+
 namespace MWSound
 {
 
@@ -217,6 +224,10 @@ public:
 struct OpenAL_Output::StreamThread {
     typedef std::vector<OpenAL_SoundStream*> StreamVec;
     StreamVec mStreams;
+
+    typedef std::vector<std::pair<DecoderPtr,Sound_Loudness*> > DecoderLoudnessVec;
+    DecoderLoudnessVec mDecoderLoudness;
+
     volatile bool mQuitNow;
     boost::mutex mMutex;
     boost::condition_variable mCondVar;
@@ -248,6 +259,33 @@ struct OpenAL_Output::StreamThread {
                 else
                     ++iter;
             }
+
+            // Only do one loudness decode at a time, in case it takes particularly long we don't
+            // want to block up anything.
+            DecoderLoudnessVec::iterator dliter = mDecoderLoudness.begin();
+            if(dliter != mDecoderLoudness.end())
+            {
+                DecoderPtr decoder = dliter->first;
+                Sound_Loudness *loudness = dliter->second;
+                mDecoderLoudness.erase(dliter);
+                lock.unlock();
+
+                std::vector<char> data;
+                ChannelConfig chans = ChannelConfig_Mono;
+                SampleType type = SampleType_Int16;
+                int srate = 48000;
+                try {
+                    decoder->getInfo(&srate, &chans, &type);
+                    decoder->readAll(data);
+                }
+                catch(std::exception &e) {
+                    std::cerr<< "Failed to decode audio: "<<e.what() <<std::endl;
+                }
+
+                loudness->analyzeLoudness(data, srate, chans, type, static_cast<float>(sLoudnessFPS));
+                lock.lock();
+                continue;
+            }
             mCondVar.timed_wait(lock, boost::posix_time::milliseconds(50));
         }
     }
@@ -274,6 +312,15 @@ struct OpenAL_Output::StreamThread {
     {
         boost::lock_guard<boost::mutex> lock(mMutex);
         mStreams.clear();
+        mDecoderLoudness.clear();
+    }
+
+    void add(DecoderPtr decoder, Sound_Loudness *loudness)
+    {
+        boost::unique_lock<boost::mutex> lock(mMutex);
+        mDecoderLoudness.push_back(std::make_pair(decoder, loudness));
+        lock.unlock();
+        mCondVar.notify_all();
     }
 
 private:
@@ -1052,6 +1099,12 @@ void OpenAL_Output::resumeSounds(int types)
 }
 
 
+void OpenAL_Output::loadLoudnessAsync(DecoderPtr decoder, Sound_Loudness *loudness)
+{
+    mStreamThread->add(decoder, loudness);
+}
+
+
 OpenAL_Output::OpenAL_Output(SoundManager &mgr)
   : Sound_Output(mgr), mDevice(0), mContext(0), mLastEnvironment(Env_Normal),
     mStreamThread(new StreamThread)

apps/openmw/mwsound/openal_output.hpp

@@ -57,6 +57,8 @@ namespace MWSound
         virtual void pauseSounds(int types);
         virtual void resumeSounds(int types);
 
+        virtual void loadLoudnessAsync(DecoderPtr decoder, Sound_Loudness *loudness);
+
         OpenAL_Output& operator=(const OpenAL_Output &rhs);
         OpenAL_Output(const OpenAL_Output &rhs);
 

apps/openmw/mwsound/sound_output.hpp

@@ -47,6 +47,11 @@ namespace MWSound
         virtual void pauseSounds(int types) = 0;
         virtual void resumeSounds(int types) = 0;
 
+        // HACK: The sound output implementation really shouldn't be handling
+        // asynchronous loudness data loading, but it's currently where we have
+        // a background processing thread.
+        virtual void loadLoudnessAsync(DecoderPtr decoder, Sound_Loudness *loudness) = 0;
+
         Sound_Output& operator=(const Sound_Output &rhs);
         Sound_Output(const Sound_Output &rhs);
 

apps/openmw/mwsound/soundmanagerimp.cpp

@@ -30,11 +30,6 @@
 #endif
 
 
-namespace
-{
-    const int sLoudnessFPS = 20; // loudness values per second of audio
-}
-
 namespace MWSound
 {
     SoundManager::SoundManager(const VFS::Manager* vfs, bool useSound)
@@ -234,27 +229,38 @@ namespace MWSound
             return decoder;
         }
 
-        ChannelConfig chans;
+        mVoiceLipBuffers.insert(mVoiceLipBuffers.end(), Sound_Loudness());
-        SampleType type;
-        int srate;
-        decoder->getInfo(&srate, &chans, &type);
-
-        std::vector<char> data;
-        decoder->readAll(data);
-
-        Sound_Loudness loudness;
-        loudness.analyzeLoudness(data, srate, chans, type, static_cast<float>(sLoudnessFPS));
-
-        mVoiceLipBuffers.insert(mVoiceLipBuffers.end(), loudness);
         lipiter = mVoiceLipNameMap.insert(
             std::make_pair(voicefile, &mVoiceLipBuffers.back())
         ).first;
 
-        decoder->rewind();
+        mOutput->loadLoudnessAsync(decoder, lipiter->second);
+
         *lipdata = lipiter->second;
         return decoder;
     }
 
+    MWBase::SoundPtr SoundManager::playVoice(DecoderPtr decoder, const osg::Vec3f &pos, bool playlocal)
+    {
+        MWBase::World* world = MWBase::Environment::get().getWorld();
+        static const float fAudioMinDistanceMult = world->getStore().get<ESM::GameSetting>().find("fAudioMinDistanceMult")->getFloat();
+        static const float fAudioMaxDistanceMult = world->getStore().get<ESM::GameSetting>().find("fAudioMaxDistanceMult")->getFloat();
+        static const float fAudioVoiceDefaultMinDistance = world->getStore().get<ESM::GameSetting>().find("fAudioVoiceDefaultMinDistance")->getFloat();
+        static const float fAudioVoiceDefaultMaxDistance = world->getStore().get<ESM::GameSetting>().find("fAudioVoiceDefaultMaxDistance")->getFloat();
+        static float minDistance = std::max(fAudioVoiceDefaultMinDistance * fAudioMinDistanceMult, 1.0f);
+        static float maxDistance = std::max(fAudioVoiceDefaultMaxDistance * fAudioMaxDistanceMult, minDistance);
+
+        float basevol = volumeFromType(Play_TypeVoice);
+        if(playlocal)
+            return mOutput->streamSound(decoder,
+                basevol, 1.0f, Play_Normal|Play_TypeVoice|Play_2D
+            );
+        return mOutput->streamSound3D(decoder,
+            pos, 1.0f, basevol, 1.0f, minDistance, maxDistance,
+            Play_Normal|Play_TypeVoice|Play_3D
+        );
+    }
+
 
     // Gets the combined volume settings for the given sound type
     float SoundManager::volumeFromType(PlayType type) const
@@ -375,16 +381,7 @@ namespace MWSound
             return;
         try
         {
-            MWBase::World* world = MWBase::Environment::get().getWorld();
-            static const float fAudioMinDistanceMult = world->getStore().get<ESM::GameSetting>().find("fAudioMinDistanceMult")->getFloat();
-            static const float fAudioMaxDistanceMult = world->getStore().get<ESM::GameSetting>().find("fAudioMaxDistanceMult")->getFloat();
-            static const float fAudioVoiceDefaultMinDistance = world->getStore().get<ESM::GameSetting>().find("fAudioVoiceDefaultMinDistance")->getFloat();
-            static const float fAudioVoiceDefaultMaxDistance = world->getStore().get<ESM::GameSetting>().find("fAudioVoiceDefaultMaxDistance")->getFloat();
-            static float minDistance = std::max(fAudioVoiceDefaultMinDistance * fAudioMinDistanceMult, 1.0f);
-            static float maxDistance = std::max(fAudioVoiceDefaultMaxDistance * fAudioMaxDistanceMult, minDistance);
-
             std::string voicefile = "Sound/"+filename;
-            float basevol = volumeFromType(Play_TypeVoice);
             const ESM::Position &pos = ptr.getRefData().getPosition();
             const osg::Vec3f objpos(pos.asVec3());
 
@@ -392,18 +389,14 @@ namespace MWSound
             mVFS->normalizeFilename(voicefile);
             DecoderPtr decoder = loadVoice(voicefile, &loudness);
 
-            MWBase::SoundPtr sound;
+            if(!loudness->isReady())
-            if(ptr == MWMechanics::getPlayer())
+                mPendingSaySounds[ptr] = std::make_pair(decoder, loudness);
-                sound = mOutput->streamSound(decoder,
-                    basevol, 1.0f, Play_Normal|Play_TypeVoice|Play_2D
-                );
             else
-                sound = mOutput->streamSound3D(decoder,
+            {
-                    objpos, 1.0f, basevol, 1.0f, minDistance, maxDistance,
+                MWBase::SoundPtr sound = playVoice(decoder, objpos, (ptr == MWMechanics::getPlayer()));
-                    Play_Normal|Play_TypeVoice|Play_3D
-                );
                 mActiveSaySounds[ptr] = std::make_pair(sound, loudness);
             }
+        }
         catch(std::exception &e)
         {
             std::cout <<"Sound Error: "<<e.what()<< std::endl;
@@ -432,17 +425,19 @@ namespace MWSound
         try
         {
             std::string voicefile = "Sound/"+filename;
-            float basevol = volumeFromType(Play_TypeVoice);
 
             Sound_Loudness *loudness;
             mVFS->normalizeFilename(voicefile);
             DecoderPtr decoder = loadVoice(voicefile, &loudness);
 
-            MWBase::SoundPtr sound = mOutput->streamSound(decoder,
+            if(!loudness->isReady())
-                basevol, 1.0f, Play_Normal|Play_TypeVoice|Play_2D
+                mPendingSaySounds[MWWorld::Ptr()] = std::make_pair(decoder, loudness);
-            );
+            else
+            {
+                MWBase::SoundPtr sound = playVoice(decoder, osg::Vec3f(), true);
                 mActiveSaySounds[MWWorld::Ptr()] = std::make_pair(sound, loudness);
             }
+        }
         catch(std::exception &e)
         {
             std::cout <<"Sound Error: "<<e.what()<< std::endl;
@@ -456,9 +451,10 @@ namespace MWSound
         {
             if(snditer->second.first->isPlaying())
                 return false;
-        }
             return true;
         }
+        return mPendingSaySounds.find(ptr) == mPendingSaySounds.end();
+    }
 
     void SoundManager::stopSay(const MWWorld::Ptr &ptr)
     {
@@ -468,6 +464,7 @@ namespace MWSound
             snditer->second.first->stop();
             mActiveSaySounds.erase(snditer);
         }
+        mPendingSaySounds.erase(ptr);
     }
 
 
@@ -832,6 +829,35 @@ namespace MWSound
                 ++snditer;
         }
 
+        SayDecoderMap::iterator penditer = mPendingSaySounds.begin();
+        while(penditer != mPendingSaySounds.end())
+        {
+            Sound_Loudness *loudness = penditer->second.second;
+            if(loudness->isReady())
+            {
+                try {
+                    DecoderPtr decoder = penditer->second.first;
+                    decoder->rewind();
+
+                    MWWorld::Ptr ptr = penditer->first;
+                    const ESM::Position &pos = ptr.getRefData().getPosition();
+                    const osg::Vec3f objpos(pos.asVec3());
+
+                    MWBase::SoundPtr sound = playVoice(decoder,
+                        objpos, (ptr == MWMechanics::getPlayer())
+                    );
+                    mActiveSaySounds[ptr] = std::make_pair(sound, loudness);
+                }
+                catch(std::exception &e) {
+                    std::cerr<< "Sound Error: "<<e.what() <<std::endl;
+                }
+
+                mPendingSaySounds.erase(penditer++);
+            }
+            else
+                ++penditer;
+        }
+
         SaySoundMap::iterator sayiter = mActiveSaySounds.begin();
         while(sayiter != mActiveSaySounds.end())
         {
@@ -947,6 +973,13 @@ namespace MWSound
             mActiveSaySounds.erase(sayiter);
             mActiveSaySounds[updated] = sndlist;
         }
+        SayDecoderMap::iterator penditer = mPendingSaySounds.find(old);
+        if(penditer != mPendingSaySounds.end())
+        {
+            DecoderLoudnessPair dl = penditer->second;
+            mPendingSaySounds.erase(penditer);
+            mPendingSaySounds[updated] = dl;
+        }
     }
 
     // Default readAll implementation, for decoders that can't do anything
@@ -1033,6 +1066,7 @@ namespace MWSound
         for(;sayiter != mActiveSaySounds.end();++sayiter)
             sayiter->second.first->stop();
         mActiveSaySounds.clear();
+        mPendingSaySounds.clear();
         mUnderwaterSound.reset();
         stopMusic();
     }

apps/openmw/mwsound/soundmanagerimp.hpp

@@ -90,6 +90,10 @@ namespace MWSound
         typedef std::map<MWWorld::Ptr,SoundLoudnessPair> SaySoundMap;
         SaySoundMap mActiveSaySounds;
 
+        typedef std::pair<DecoderPtr,Sound_Loudness*> DecoderLoudnessPair;
+        typedef std::map<MWWorld::Ptr,DecoderLoudnessPair> SayDecoderMap;
+        SayDecoderMap mPendingSaySounds;
+
         MWBase::SoundPtr mUnderwaterSound;
 
         bool mListenerUnderwater;
@@ -104,10 +108,12 @@ namespace MWSound
         Sound_Buffer *lookupSound(const std::string &soundId) const;
         Sound_Buffer *loadSound(const std::string &soundId);
 
-        // Ensures the loudness/"lip" data is loaded, and returns a decoder to
+        // Ensures the loudness/"lip" data gets loaded, and returns a decoder
-        // start streaming
+        // to start streaming
         DecoderPtr loadVoice(const std::string &voicefile, Sound_Loudness **lipdata);
 
+        MWBase::SoundPtr playVoice(DecoderPtr decoder, const osg::Vec3f &pos, bool playlocal);
+
         void streamMusicFull(const std::string& filename);
         bool updateSound(MWBase::SoundPtr sound, const MWWorld::Ptr &ptr, float duration);
         void updateSounds(float duration);