Analyze the loudness data as the stream is decoded for playback

Instead of getting the loudness data for the whole file in advance, we now get it piece by piece as the sound is streamed.

The benefit is that we need to decode the audio just once instead of twice.

We no longer need to rewind() the stream when the first decoding is done, this should hopefully fix bug #3453 .
coverity_scan^2
scrawl 9 years ago
parent d2d201cf6d
commit 965aaebbdb

@ -8,15 +8,16 @@
namespace MWSound
{
void Sound_Loudness::analyzeLoudness(const std::vector< char >& data, int sampleRate, ChannelConfig chans, SampleType type, float valuesPerSecond)
void Sound_Loudness::analyzeLoudness(const std::vector< char >& data)
{
int samplesPerSegment = static_cast<int>(sampleRate / valuesPerSecond);
int numSamples = bytesToFrames(data.size(), chans, type);
int advance = framesToBytes(1, chans, type);
mQueue.insert( mQueue.end(), data.begin(), data.end() );
if (!mQueue.size())
return;
int samplesPerSegment = static_cast<int>(mSampleRate / mSamplesPerSec);
int numSamples = bytesToFrames(mQueue.size(), mChannelConfig, mSampleType);
int advance = framesToBytes(1, mChannelConfig, mSampleType);
mSamplesPerSec = valuesPerSecond;
mSamples.clear();
mSamples.reserve(numSamples/samplesPerSegment);
int segment=0;
int sample=0;
@ -28,16 +29,16 @@ void Sound_Loudness::analyzeLoudness(const std::vector< char >& data, int sample
{
// get sample on a scale from -1 to 1
float value = 0;
if (type == SampleType_UInt8)
value = ((char)(data[sample*advance]^0x80))/128.f;
else if (type == SampleType_Int16)
if (mSampleType == SampleType_UInt8)
value = ((char)(mQueue[sample*advance]^0x80))/128.f;
else if (mSampleType == SampleType_Int16)
{
value = *reinterpret_cast<const int16_t*>(&data[sample*advance]);
value = *reinterpret_cast<const int16_t*>(&mQueue[sample*advance]);
value /= float(std::numeric_limits<int16_t>::max());
}
else if (type == SampleType_Float32)
else if (mSampleType == SampleType_Float32)
{
value = *reinterpret_cast<const float*>(&data[sample*advance]);
value = *reinterpret_cast<const float*>(&mQueue[sample*advance]);
value = std::max(-1.f, std::min(1.f, value)); // Float samples *should* be scaled to [-1,1] already.
}
@ -53,7 +54,7 @@ void Sound_Loudness::analyzeLoudness(const std::vector< char >& data, int sample
++segment;
}
mReady = true;
mQueue.erase(mQueue.begin(), mQueue.begin() + sample*advance);
}

@ -2,6 +2,7 @@
#define GAME_SOUND_LOUDNESS_H
#include <vector>
#include <deque>
#include "sound_decoder.hpp"
@ -9,29 +10,45 @@ namespace MWSound
{
class Sound_Loudness {
// Loudness sample info
float mSamplesPerSec;
int mSampleRate;
ChannelConfig mChannelConfig;
SampleType mSampleType;
// Loudness sample info
std::vector<float> mSamples;
volatile bool mReady;
std::deque<char> mQueue;
public:
Sound_Loudness() : mSamplesPerSec(0.0f), mReady(false) { }
/**
* @param samplesPerSecond How many loudness values per second of audio to compute.
* @param sampleRate the sample rate of the sound buffer
* @param chans channel layout of the buffer
* @param type sample type of the buffer
*/
Sound_Loudness(float samplesPerSecond, int sampleRate, ChannelConfig chans, SampleType type)
: mSamplesPerSec(samplesPerSecond)
, mSampleRate(sampleRate)
, mChannelConfig(chans)
, mSampleType(type)
{ }
/**
* Analyzes the energy (closely related to loudness) of a sound buffer.
* The buffer will be divided into segments according to \a valuesPerSecond,
* and for each segment a loudness value in the range of [0,1] will be computed.
* The computed values are then added to the mSamples vector. This method should be called continuously
* with chunks of audio until the whole audio file is processed.
* If the size of \a data does not exactly fit a number of loudness samples, the remainder
* will be kept in the mQueue and used in the next call to analyzeLoudness.
* @param data the sound buffer to analyze, containing raw samples
* @param sampleRate the sample rate of the sound buffer
* @param chans channel layout of the buffer
* @param type sample type of the buffer
* @param valuesPerSecond How many loudness values per second of audio to compute.
*/
void analyzeLoudness(const std::vector<char>& data, int sampleRate,
ChannelConfig chans, SampleType type,
float valuesPerSecond);
void analyzeLoudness(const std::vector<char>& data);
bool isReady() { return mReady; }
/**
* Get loudness at a particular time. Before calling this, the stream has to be analyzed up to that point in time (see analyzeLoudness()).
*/
float getLoudnessAtTime(float sec) const;
};

@ -2,6 +2,7 @@
#include <stdexcept>
#include <iostream>
#include <vector>
#include <osg/Timer>
#include <stdint.h>
@ -212,6 +213,8 @@ private:
DecoderPtr mDecoder;
std::auto_ptr<Sound_Loudness> mLoudnessAnalyzer;
volatile bool mIsFinished;
void updateAll(bool local);
@ -229,6 +232,8 @@ public:
double getStreamDelay() const;
double getStreamOffset() const;
float getCurrentLoudness() const;
bool process();
ALint refillQueue();
};
@ -242,9 +247,6 @@ struct OpenAL_Output::StreamThread {
typedef std::vector<OpenAL_SoundStream*> StreamVec;
StreamVec mStreams;
typedef std::vector<std::pair<DecoderPtr,Sound_Loudness*> > DecoderLoudnessVec;
DecoderLoudnessVec mDecoderLoudness;
volatile bool mQuitNow;
boost::mutex mMutex;
boost::condition_variable mCondVar;
@ -277,32 +279,6 @@ struct OpenAL_Output::StreamThread {
++iter;
}
// Only do one loudness decode at a time, in case it takes particularly long we don't
// want to block up anything.
DecoderLoudnessVec::iterator dliter = mDecoderLoudness.begin();
if(dliter != mDecoderLoudness.end())
{
DecoderPtr decoder = dliter->first;
Sound_Loudness *loudness = dliter->second;
mDecoderLoudness.erase(dliter);
lock.unlock();
std::vector<char> data;
ChannelConfig chans = ChannelConfig_Mono;
SampleType type = SampleType_Int16;
int srate = 48000;
try {
decoder->getInfo(&srate, &chans, &type);
decoder->readAll(data);
}
catch(std::exception &e) {
std::cerr<< "Failed to decode audio: "<<e.what() <<std::endl;
}
loudness->analyzeLoudness(data, srate, chans, type, static_cast<float>(sLoudnessFPS));
lock.lock();
continue;
}
mCondVar.timed_wait(lock, boost::posix_time::milliseconds(50));
}
}
@ -329,15 +305,6 @@ struct OpenAL_Output::StreamThread {
{
boost::lock_guard<boost::mutex> lock(mMutex);
mStreams.clear();
mDecoderLoudness.clear();
}
void add(DecoderPtr decoder, Sound_Loudness *loudness)
{
boost::unique_lock<boost::mutex> lock(mMutex);
mDecoderLoudness.push_back(std::make_pair(decoder, loudness));
lock.unlock();
mCondVar.notify_all();
}
private:
@ -371,6 +338,8 @@ OpenAL_SoundStream::OpenAL_SoundStream(ALuint src, DecoderPtr decoder)
mFrameSize = framesToBytes(1, chans, type);
mBufferSize = static_cast<ALuint>(sBufferLength*srate);
mBufferSize *= mFrameSize;
mLoudnessAnalyzer.reset(new Sound_Loudness(sLoudnessFPS, mSampleRate, chans, type));
}
catch(std::exception&)
{
@ -446,6 +415,15 @@ double OpenAL_SoundStream::getStreamOffset() const
return t;
}
float OpenAL_SoundStream::getCurrentLoudness() const
{
if (!mLoudnessAnalyzer.get())
return 0.f;
float time = getStreamOffset();
return mLoudnessAnalyzer->getLoudnessAtTime(time);
}
bool OpenAL_SoundStream::process()
{
try {
@ -493,6 +471,9 @@ ALint OpenAL_SoundStream::refillQueue()
}
if(got > 0)
{
if (mLoudnessAnalyzer.get())
mLoudnessAnalyzer->analyzeLoudness(data);
ALuint bufid = mBuffers[mCurrentBufIdx];
alBufferData(bufid, mFormat, &data[0], data.size(), mSampleRate);
alSourceQueueBuffers(mSource, 1, &bufid);
@ -1045,6 +1026,14 @@ double OpenAL_Output::getStreamOffset(MWBase::SoundStreamPtr sound)
return stream->getStreamOffset();
}
float OpenAL_Output::getStreamLoudness(MWBase::SoundStreamPtr sound)
{
if(!sound->mHandle) return 0.0;
OpenAL_SoundStream *stream = reinterpret_cast<OpenAL_SoundStream*>(sound->mHandle);
boost::lock_guard<boost::mutex> lock(mStreamThread->mMutex);
return stream->getCurrentLoudness();
}
bool OpenAL_Output::isStreamPlaying(MWBase::SoundStreamPtr sound)
{
if(!sound->mHandle) return false;
@ -1144,12 +1133,6 @@ void OpenAL_Output::resumeSounds(int types)
}
void OpenAL_Output::loadLoudnessAsync(DecoderPtr decoder, Sound_Loudness *loudness)
{
mStreamThread->add(decoder, loudness);
}
OpenAL_Output::OpenAL_Output(SoundManager &mgr)
: Sound_Output(mgr), mDevice(0), mContext(0)
, mListenerPos(0.0f, 0.0f, 0.0f), mListenerEnv(Env_Normal)

@ -67,6 +67,7 @@ namespace MWSound
virtual void finishStream(MWBase::SoundStreamPtr sound);
virtual double getStreamDelay(MWBase::SoundStreamPtr sound);
virtual double getStreamOffset(MWBase::SoundStreamPtr sound);
virtual float getStreamLoudness(MWBase::SoundStreamPtr sound);
virtual bool isStreamPlaying(MWBase::SoundStreamPtr sound);
virtual void updateStream(MWBase::SoundStreamPtr sound);
@ -78,8 +79,6 @@ namespace MWSound
virtual void pauseSounds(int types);
virtual void resumeSounds(int types);
virtual void loadLoudnessAsync(DecoderPtr decoder, Sound_Loudness *loudness);
OpenAL_Output(SoundManager &mgr);
virtual ~OpenAL_Output();
};

@ -12,7 +12,6 @@ namespace MWSound
class SoundManager;
struct Sound_Decoder;
class Sound;
class Sound_Loudness;
// An opaque handle for the implementation's sound buffers.
typedef void *Sound_Handle;
@ -46,6 +45,7 @@ namespace MWSound
virtual void finishStream(MWBase::SoundStreamPtr sound) = 0;
virtual double getStreamDelay(MWBase::SoundStreamPtr sound) = 0;
virtual double getStreamOffset(MWBase::SoundStreamPtr sound) = 0;
virtual float getStreamLoudness(MWBase::SoundStreamPtr sound) = 0;
virtual bool isStreamPlaying(MWBase::SoundStreamPtr sound) = 0;
virtual void updateStream(MWBase::SoundStreamPtr sound) = 0;
@ -57,11 +57,6 @@ namespace MWSound
virtual void pauseSounds(int types) = 0;
virtual void resumeSounds(int types) = 0;
// HACK: The sound output implementation really shouldn't be handling
// asynchronous loudness data loading, but it's currently where we have
// a background processing thread.
virtual void loadLoudnessAsync(DecoderPtr decoder, Sound_Loudness *loudness) = 0;
Sound_Output& operator=(const Sound_Output &rhs);
Sound_Output(const Sound_Output &rhs);

@ -219,7 +219,7 @@ namespace MWSound
return sfx;
}
DecoderPtr SoundManager::loadVoice(const std::string &voicefile, Sound_Loudness **lipdata)
DecoderPtr SoundManager::loadVoice(const std::string &voicefile)
{
DecoderPtr decoder = getDecoder();
// Workaround: Bethesda at some point converted some of the files to mp3, but the references were kept as .wav.
@ -234,21 +234,6 @@ namespace MWSound
decoder->open(file);
}
NameLoudnessRefMap::iterator lipiter = mVoiceLipNameMap.find(voicefile);
if(lipiter != mVoiceLipNameMap.end())
{
*lipdata = lipiter->second;
return decoder;
}
mVoiceLipBuffers.insert(mVoiceLipBuffers.end(), Sound_Loudness());
lipiter = mVoiceLipNameMap.insert(
std::make_pair(voicefile, &mVoiceLipBuffers.back())
).first;
mOutput->loadLoudnessAsync(decoder, lipiter->second);
*lipdata = lipiter->second;
return decoder;
}
@ -400,28 +385,22 @@ namespace MWSound
{
std::string voicefile = "Sound/"+filename;
Sound_Loudness *loudness;
mVFS->normalizeFilename(voicefile);
DecoderPtr decoder = loadVoice(voicefile, &loudness);
DecoderPtr decoder = loadVoice(voicefile);
if(!loudness->isReady())
mPendingSaySounds[ptr] = std::make_pair(decoder, loudness);
else
{
MWBase::World *world = MWBase::Environment::get().getWorld();
const osg::Vec3f pos = world->getActorHeadTransform(ptr).getTrans();
SaySoundMap::iterator oldIt = mActiveSaySounds.find(ptr);
if (oldIt != mActiveSaySounds.end())
{
mOutput->finishStream(oldIt->second.first);
mOutput->finishStream(oldIt->second);
mActiveSaySounds.erase(oldIt);
}
MWBase::SoundStreamPtr sound = playVoice(decoder, pos, (ptr == MWMechanics::getPlayer()));
mActiveSaySounds.insert(std::make_pair(ptr, std::make_pair(sound, loudness)));
}
mActiveSaySounds.insert(std::make_pair(ptr, sound));
}
catch(std::exception &e)
{
@ -434,10 +413,8 @@ namespace MWSound
SaySoundMap::const_iterator snditer = mActiveSaySounds.find(ptr);
if(snditer != mActiveSaySounds.end())
{
MWBase::SoundStreamPtr sound = snditer->second.first;
Sound_Loudness *loudness = snditer->second.second;
float sec = mOutput->getStreamOffset(sound);
return loudness->getLoudnessAtTime(sec);
MWBase::SoundStreamPtr sound = snditer->second;
return mOutput->getStreamLoudness(sound);
}
return 0.0f;
@ -451,24 +428,18 @@ namespace MWSound
{
std::string voicefile = "Sound/"+filename;
Sound_Loudness *loudness;
mVFS->normalizeFilename(voicefile);
DecoderPtr decoder = loadVoice(voicefile, &loudness);
DecoderPtr decoder = loadVoice(voicefile);
if(!loudness->isReady())
mPendingSaySounds[MWWorld::ConstPtr()] = std::make_pair(decoder, loudness);
else
{
SaySoundMap::iterator oldIt = mActiveSaySounds.find(MWWorld::ConstPtr());
if (oldIt != mActiveSaySounds.end())
{
mOutput->finishStream(oldIt->second.first);
mOutput->finishStream(oldIt->second);
mActiveSaySounds.erase(oldIt);
}
mActiveSaySounds.insert(std::make_pair(MWWorld::ConstPtr(),
std::make_pair(playVoice(decoder, osg::Vec3f(), true), loudness)));
}
playVoice(decoder, osg::Vec3f(), true)));
}
catch(std::exception &e)
{
@ -481,11 +452,11 @@ namespace MWSound
SaySoundMap::const_iterator snditer = mActiveSaySounds.find(ptr);
if(snditer != mActiveSaySounds.end())
{
if(mOutput->isStreamPlaying(snditer->second.first))
if(mOutput->isStreamPlaying(snditer->second))
return false;
return true;
}
return mPendingSaySounds.find(ptr) == mPendingSaySounds.end();
return true;
}
void SoundManager::stopSay(const MWWorld::ConstPtr &ptr)
@ -493,10 +464,9 @@ namespace MWSound
SaySoundMap::iterator snditer = mActiveSaySounds.find(ptr);
if(snditer != mActiveSaySounds.end())
{
mOutput->finishStream(snditer->second.first);
mOutput->finishStream(snditer->second);
mActiveSaySounds.erase(snditer);
}
mPendingSaySounds.erase(ptr);
}
@ -691,7 +661,7 @@ namespace MWSound
sayiter->first != MWMechanics::getPlayer() &&
sayiter->first.getCell() == cell)
{
mOutput->finishStream(sayiter->second.first);
mOutput->finishStream(sayiter->second);
}
++sayiter;
}
@ -901,51 +871,11 @@ namespace MWSound
++snditer;
}
SayDecoderMap::iterator penditer = mPendingSaySounds.begin();
while(penditer != mPendingSaySounds.end())
{
Sound_Loudness *loudness = penditer->second.second;
if(loudness->isReady())
{
try {
DecoderPtr decoder = penditer->second.first;
decoder->rewind();
MWBase::SoundStreamPtr sound;
MWWorld::ConstPtr ptr = penditer->first;
SaySoundMap::iterator old = mActiveSaySounds.find(ptr);
if (old != mActiveSaySounds.end())
{
mOutput->finishStream(old->second.first);
mActiveSaySounds.erase(old);
}
if(ptr == MWWorld::ConstPtr())
sound = playVoice(decoder, osg::Vec3f(), true);
else
{
MWBase::World *world = MWBase::Environment::get().getWorld();
const osg::Vec3f pos = world->getActorHeadTransform(ptr).getTrans();
sound = playVoice(decoder, pos, (ptr == MWMechanics::getPlayer()));
}
mActiveSaySounds.insert(std::make_pair(ptr, std::make_pair(sound, loudness)));
}
catch(std::exception &e) {
std::cerr<< "Sound Error: "<<e.what() <<std::endl;
}
mPendingSaySounds.erase(penditer++);
}
else
++penditer;
}
SaySoundMap::iterator sayiter = mActiveSaySounds.begin();
while(sayiter != mActiveSaySounds.end())
{
MWWorld::ConstPtr ptr = sayiter->first;
MWBase::SoundStreamPtr sound = sayiter->second.first;
MWBase::SoundStreamPtr sound = sayiter->second;
if(!ptr.isEmpty() && sound->getIs3D())
{
MWBase::World *world = MWBase::Environment::get().getWorld();
@ -1040,7 +970,7 @@ namespace MWSound
SaySoundMap::iterator sayiter = mActiveSaySounds.begin();
for(;sayiter != mActiveSaySounds.end();++sayiter)
{
MWBase::SoundStreamPtr sound = sayiter->second.first;
MWBase::SoundStreamPtr sound = sayiter->second;
sound->setBaseVolume(volumeFromType(sound->getPlayType()));
mOutput->updateStream(sound);
}
@ -1080,16 +1010,9 @@ namespace MWSound
SaySoundMap::iterator sayiter = mActiveSaySounds.find(old);
if(sayiter != mActiveSaySounds.end())
{
SoundLoudnessPair sndlist = sayiter->second;
MWBase::SoundStreamPtr stream = sayiter->second;
mActiveSaySounds.erase(sayiter);
mActiveSaySounds[updated] = sndlist;
}
SayDecoderMap::iterator penditer = mPendingSaySounds.find(old);
if(penditer != mPendingSaySounds.end())
{
DecoderLoudnessPair dl = penditer->second;
mPendingSaySounds.erase(penditer);
mPendingSaySounds[updated] = dl;
mActiveSaySounds[updated] = stream;
}
}
@ -1175,13 +1098,12 @@ namespace MWSound
mActiveSounds.clear();
SaySoundMap::iterator sayiter = mActiveSaySounds.begin();
for(;sayiter != mActiveSaySounds.end();++sayiter)
mOutput->finishStream(sayiter->second.first);
mOutput->finishStream(sayiter->second);
mActiveSaySounds.clear();
TrackList::iterator trkiter = mActiveTracks.begin();
for(;trkiter != mActiveTracks.end();++trkiter)
mOutput->finishStream(*trkiter);
mActiveTracks.clear();
mPendingSaySounds.clear();
mUnderwaterSound.reset();
stopMusic();
}

@ -11,7 +11,6 @@
#include <components/settings/settings.hpp>
#include "loudness.hpp"
#include "../mwbase/soundmanager.hpp"
namespace VFS
@ -69,12 +68,6 @@ namespace MWSound
typedef std::map<std::string,Sound_Buffer*> NameBufferMap;
NameBufferMap mBufferNameMap;
typedef std::deque<Sound_Loudness> LoudnessList;
LoudnessList mVoiceLipBuffers;
typedef std::map<std::string,Sound_Loudness*> NameLoudnessRefMap;
NameLoudnessRefMap mVoiceLipNameMap;
// NOTE: unused buffers are stored in front-newest order.
typedef std::deque<Sound_Buffer*> SoundList;
SoundList mUnusedBuffers;
@ -84,14 +77,9 @@ namespace MWSound
typedef std::map<MWWorld::ConstPtr,SoundBufferRefPairList> SoundMap;
SoundMap mActiveSounds;
typedef std::pair<MWBase::SoundStreamPtr,Sound_Loudness*> SoundLoudnessPair;
typedef std::map<MWWorld::ConstPtr,SoundLoudnessPair> SaySoundMap;
typedef std::map<MWWorld::ConstPtr,MWBase::SoundStreamPtr> SaySoundMap;
SaySoundMap mActiveSaySounds;
typedef std::pair<DecoderPtr,Sound_Loudness*> DecoderLoudnessPair;
typedef std::map<MWWorld::ConstPtr,DecoderLoudnessPair> SayDecoderMap;
SayDecoderMap mPendingSaySounds;
typedef std::vector<MWBase::SoundStreamPtr> TrackList;
TrackList mActiveTracks;
@ -112,9 +100,8 @@ namespace MWSound
Sound_Buffer *lookupSound(const std::string &soundId) const;
Sound_Buffer *loadSound(const std::string &soundId);
// Ensures the loudness/"lip" data gets loaded, and returns a decoder
// to start streaming
DecoderPtr loadVoice(const std::string &voicefile, Sound_Loudness **lipdata);
// returns a decoder to start streaming
DecoderPtr loadVoice(const std::string &voicefile);
MWBase::SoundStreamPtr playVoice(DecoderPtr decoder, const osg::Vec3f &pos, bool playlocal);

Loading…
Cancel
Save