From 278337116b93e62e7458e03463e83c2055eb15e7 Mon Sep 17 00:00:00 2001 From: Nathan Jeffords Date: Tue, 1 Jan 2013 15:07:33 -0800 Subject: [PATCH] increased performance of ConstrainedDataStream Reimplemented ConstrainedDataStream to use low-level IO calls and a custom buffering scheme to avoid using C++ iostreams. --- components/CMakeLists.txt | 2 +- .../files/constrainedfiledatastream.cpp | 218 ++++++++----- components/files/lowlevelfile.cpp | 299 ++++++++++++++++++ components/files/lowlevelfile.hpp | 54 ++++ 4 files changed, 492 insertions(+), 81 deletions(-) create mode 100644 components/files/lowlevelfile.cpp create mode 100644 components/files/lowlevelfile.hpp diff --git a/components/CMakeLists.txt b/components/CMakeLists.txt index f54efab22..3da09ecb8 100644 --- a/components/CMakeLists.txt +++ b/components/CMakeLists.txt @@ -48,7 +48,7 @@ add_component_dir (misc add_component_dir (files linuxpath windowspath macospath fixedpath multidircollection collections fileops configurationmanager - filelibrary ogreplugin constrainedfiledatastream + filelibrary ogreplugin constrainedfiledatastream lowlevelfile ) add_component_dir (compiler diff --git a/components/files/constrainedfiledatastream.cpp b/components/files/constrainedfiledatastream.cpp index 037ffe6f0..2f75ce41b 100644 --- a/components/files/constrainedfiledatastream.cpp +++ b/components/files/constrainedfiledatastream.cpp @@ -1,112 +1,170 @@ -/* - OpenMW - The completely unofficial reimplementation of Morrowind - Copyright (C) 2008-2010 Nicolay Korslund - Email: < korslund@gmail.com > - WWW: http://openmw.sourceforge.net/ +#include "constrainedfiledatastream.hpp" +#include "lowlevelfile.hpp" - This file (bsa_file.cpp) is part of the OpenMW package. +#include +#include - OpenMW is distributed as free software: you can redistribute it - and/or modify it under the terms of the GNU General Public License - version 3, as published by the Free Software Foundation. +namespace { - This program is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. +class ConstrainedDataStream : public Ogre::DataStream { +public: - You should have received a copy of the GNU General Public License - version 3 along with this program. If not, see - http://www.gnu.org/licenses/ . + static const size_t sBufferSize = 4096; // somewhat arbitrary though 64KB buffers didn't seem to improve performance any + static const size_t sBufferThreshold = 1024; // reads larger than this bypass buffering as cost of memcpy outweighs cost of system call - */ + ConstrainedDataStream(const Ogre::String &fname, size_t start, size_t length) + { + mFile.open (fname.c_str ()); + mSize = length != 0xFFFFFFFF ? length : mFile.size () - start; -#include "constrainedfiledatastream.hpp" + mPos = 0; + mOrigin = start; + mExtent = start + mSize; -#include + mBufferOrigin = 0; + mBufferExtent = 0; + } + -class ConstrainedDataStream : public Ogre::DataStream { - std::ifstream mStream; - const size_t mStart; - size_t mPos; - bool mIsEOF; + size_t read(void* buf, size_t count) + { + assert (mPos <= mSize); -public: - ConstrainedDataStream(const Ogre::String &fname, size_t start, size_t length) - : mStream(fname.c_str(), std::ios_base::binary), mStart(start), mPos(0), mIsEOF(false) - { - mSize = length; - if(!mStream.seekg(mStart, std::ios_base::beg)) - throw std::runtime_error("Error seeking to start of BSA entry"); - } + uint8_t * out = reinterpret_cast (buf); - ConstrainedDataStream(const Ogre::String &name, const Ogre::String &fname, - size_t start, size_t length) - : Ogre::DataStream(name), mStream(fname.c_str(), std::ios_base::binary), - mStart(start), mPos(0), mIsEOF(false) - { - mSize = length; - if(!mStream.seekg(mStart, std::ios_base::beg)) - throw std::runtime_error("Error seeking to start of BSA entry"); - } + size_t posBeg = mOrigin + mPos; + size_t posEnd = posBeg + count; + if (posEnd > mExtent) + posEnd = mExtent; - virtual size_t read(void *buf, size_t count) - { - mStream.clear(); - - if(count > mSize-mPos) - { - count = mSize-mPos; - mIsEOF = true; - } - mStream.read(reinterpret_cast(buf), count); - - count = mStream.gcount(); - mPos += count; - return count; - } + size_t posCur = posBeg; + + while (posCur != posEnd) + { + size_t readLeft = posEnd - posCur; + + if (posCur < mBufferOrigin || posCur >= mBufferExtent) + { + if (readLeft >= sBufferThreshold || (posCur == mOrigin && posEnd == mExtent)) + { + assert (mFile.tell () == mBufferExtent); + + if (posCur != mBufferExtent) + mFile.seek (posCur); + + posCur += mFile.read (out, readLeft); + + mBufferOrigin = mBufferExtent = posCur; + + mPos = posCur - mOrigin; + + return posCur - posBeg; + } + else + { + size_t newBufferOrigin; + + if ((posCur < mBufferOrigin) && (mBufferOrigin - posCur < sBufferSize)) + newBufferOrigin = std::max (mOrigin, mBufferOrigin > sBufferSize ? mBufferOrigin - sBufferSize : 0); + else + newBufferOrigin = posCur; + + fill (newBufferOrigin); + } + } + + size_t xfer = std::min (readLeft, mBufferExtent - posCur); - virtual void skip(long count) + memcpy (out, mBuffer + (posCur - mBufferOrigin), xfer); + + posCur += xfer; + out += xfer; + } + + count = posEnd - posBeg; + mPos += count; + return count; + } + + void skip(long count) { + assert (mPos <= mSize); + if((count >= 0 && (size_t)count <= mSize-mPos) || (count < 0 && (size_t)-count <= mPos)) - { - mStream.clear(); - if(mStream.seekg(count, std::ios_base::cur)) - { - mPos += count; - mIsEOF = false; - } - } + mPos += count; } - virtual void seek(size_t pos) + void seek(size_t pos) { - if(pos < mSize) - { - mStream.clear(); - if(mStream.seekg(pos+mStart, std::ios_base::beg)) - { - mPos = pos; - mIsEOF = false; - } - } + assert (mPos <= mSize); + + if (pos < mSize) + mPos = pos; } virtual size_t tell() const - { return mPos; } + { + assert (mPos <= mSize); + + return mPos; + } virtual bool eof() const - { return mIsEOF; } + { + assert (mPos <= mSize); + + return mPos == mSize; + } virtual void close() - { mStream.close(); } + { + mFile.close(); + } + +private: + + void fill (size_t newOrigin) + { + assert (mFile.tell () == mBufferExtent); + + size_t newExtent = newOrigin + sBufferSize; + + if (newExtent > mExtent) + newExtent = mExtent; + + size_t oldExtent = mBufferExtent; + + if (newOrigin != oldExtent) + mFile.seek (newOrigin); + + mBufferOrigin = mBufferExtent = newOrigin; + + size_t amountRequested = newExtent - newOrigin; + + size_t amountRead = mFile.read (mBuffer, amountRequested); + + if (amountRead != amountRequested) + throw std::runtime_error ("An unexpected condition occurred while reading from a file."); + + mBufferExtent = newExtent; + } + + LowLevelFile mFile; + + size_t mOrigin; + size_t mExtent; + size_t mPos; + + uint8_t mBuffer [sBufferSize]; + size_t mBufferOrigin; + size_t mBufferExtent; }; +} // end of unnamed namespace + Ogre::DataStreamPtr openConstrainedFileDataStream (char const * filename, size_t offset, size_t length) { - assert (length != 0xFFFFFFFF); // reserved for future use... - return Ogre::DataStreamPtr(new ConstrainedDataStream(filename, offset, length)); } diff --git a/components/files/lowlevelfile.cpp b/components/files/lowlevelfile.cpp new file mode 100644 index 000000000..71fd1b523 --- /dev/null +++ b/components/files/lowlevelfile.cpp @@ -0,0 +1,299 @@ +#include "lowlevelfile.hpp" + +#include +#include +#include + +#if FILE_API == FILE_API_POSIX +#include +#include +#include +#endif + +#if FILE_API == FILE_API_STDIO +/* + * + * Implementation of LowLevelFile methods using c stdio + * + */ + +LowLevelFile::LowLevelFile () +{ + mHandle = NULL; +} + +LowLevelFile::~LowLevelFile () +{ + if (mHandle != NULL) + fclose (mHandle); +} + +void LowLevelFile::open (char const * filename) +{ + assert (mHandle == NULL); + + mHandle = fopen (filename, "rb"); + + if (mHandle == NULL) + { + std::ostringstream os; + os << "Failed to open '" << filename << "' for reading."; + throw std::runtime_error (os.str ()); + } +} + +void LowLevelFile::close () +{ + assert (mHandle != NULL); + + fclose (mHandle); + + mHandle = NULL; +} + +size_t LowLevelFile::size () +{ + assert (mHandle != NULL); + + long oldPosition = ftell (mHandle); + + if (oldPosition == -1) + throw std::runtime_error ("A query operation on a file failed."); + + if (fseek (mHandle, 0, SEEK_END) != 0) + throw std::runtime_error ("A query operation on a file failed."); + + long Size = ftell (mHandle); + + if (Size == -1) + throw std::runtime_error ("A query operation on a file failed."); + + if (fseek (mHandle, oldPosition, SEEK_SET) != 0) + throw std::runtime_error ("A query operation on a file failed."); + + return size_t (Size); +} + +void LowLevelFile::seek (size_t Position) +{ + assert (mHandle != NULL); + + if (fseek (mHandle, Position, SEEK_SET) != 0) + throw std::runtime_error ("A seek operation on a file failed."); +} + +size_t LowLevelFile::tell () +{ + assert (mHandle != NULL); + + long Position = ftell (mHandle); + + if (Position == -1) + throw std::runtime_error ("A query operation on a file failed."); + + return size_t (Position); +} + +size_t LowLevelFile::read (void * data, size_t size) +{ + assert (mHandle != NULL); + + int amount = fread (data, 1, size, mHandle); + + if (amount == 0 && ferror (mHandle)) + throw std::runtime_error ("A read operation on a file failed."); + + return amount; +} + +#elif FILE_API == FILE_API_POSIX +/* + * + * Implementation of LowLevelFile methods using posix IO calls + * + */ + +LowLevelFile::LowLevelFile () +{ + mHandle = -1; +} + +LowLevelFile::~LowLevelFile () +{ + if (mHandle != -1) + ::close (mHandle); +} + +void LowLevelFile::open (char const * filename) +{ + assert (mHandle == -1); + +#ifdef O_BINARY + static const int openFlags = O_RDONLY | O_BINARY; +#else + static const int openFlags = O_RDONLY; +#endif + + mHandle = ::open (filename, openFlags, 0); + + if (mHandle == -1) + { + std::ostringstream os; + os << "Failed to open '" << filename << "' for reading."; + throw std::runtime_error (os.str ()); + } +} + +void LowLevelFile::close () +{ + assert (mHandle != -1); + + ::close (mHandle); + + mHandle = -1; +} + +size_t LowLevelFile::size () +{ + assert (mHandle != -1); + + size_t oldPosition = ::lseek (mHandle, 0, SEEK_CUR); + + if (oldPosition == size_t (-1)) + throw std::runtime_error ("A query operation on a file failed."); + + size_t Size = ::lseek (mHandle, 0, SEEK_END); + + if (Size == size_t (-1)) + throw std::runtime_error ("A query operation on a file failed."); + + if (lseek (mHandle, oldPosition, SEEK_SET) == -1) + throw std::runtime_error ("A query operation on a file failed."); + + return Size; +} + +void LowLevelFile::seek (size_t Position) +{ + assert (mHandle != -1); + + if (::lseek (mHandle, Position, SEEK_SET) == -1) + throw std::runtime_error ("A seek operation on a file failed."); +} + +size_t LowLevelFile::tell () +{ + assert (mHandle != -1); + + size_t Position = ::lseek (mHandle, 0, SEEK_CUR); + + if (Position == size_t (-1)) + throw std::runtime_error ("A query operation on a file failed."); + + return Position; +} + +size_t LowLevelFile::read (void * data, size_t size) +{ + assert (mHandle != -1); + + int amount = ::read (mHandle, data, size); + + if (amount == -1) + throw std::runtime_error ("A read operation on a file failed."); + + return amount; +} + +#elif FILE_API == FILE_API_WIN32 +/* + * + * Implementation of LowLevelFile methods using Win32 API calls + * + */ + +LowLevelFile::LowLevelFile () +{ + mHandle = INVALID_HANDLE_VALUE; +} + +LowLevelFile::~LowLevelFile () +{ + if (mHandle == INVALID_HANDLE_VALUE) + CloseHandle (mHandle); +} + +void LowLevelFile::open (char const * filename) +{ + assert (mHandle == INVALID_HANDLE_VALUE); + + HANDLE handle = CreateFileA (filename, GENERIC_READ, FILE_SHARE_READ, 0, OPEN_EXISTING, 0, 0); + + if (handle == NULL) + { + std::ostringstream os; + os << "Failed to open '" << filename << "' for reading."; + throw std::runtime_error (os.str ()); + } + + mHandle = handle; +} + +void LowLevelFile::close () +{ + assert (mHandle != INVALID_HANDLE_VALUE); + + CloseHandle (mHandle); + + mHandle = INVALID_HANDLE_VALUE; +} + +size_t LowLevelFile::size () +{ + assert (mHandle != INVALID_HANDLE_VALUE); + + BY_HANDLE_FILE_INFORMATION info; + + if (!GetFileInformationByHandle (mHandle, &info)) + throw std::runtime_error ("A query operation on a file failed."); + + if (info.nFileSizeHigh != 0) + throw std::runtime_error ("Files greater that 4GB are not supported."); + + return info.nFileSizeLow; +} + +void LowLevelFile::seek (size_t Position) +{ + assert (mHandle != INVALID_HANDLE_VALUE); + + if (SetFilePointer (mHandle, Position, NULL, SEEK_SET) == INVALID_SET_FILE_POINTER) + if (GetLastError () != NO_ERROR) + throw std::runtime_error ("A seek operation on a file failed."); +} + +size_t LowLevelFile::tell () +{ + assert (mHandle != INVALID_HANDLE_VALUE); + + DWORD value = SetFilePointer (mHandle, 0, NULL, SEEK_CUR); + + if (value == INVALID_SET_FILE_POINTER && GetLastError () != NO_ERROR) + throw std::runtime_error ("A query operation on a file failed."); + + return value; +} + +size_t LowLevelFile::read (void * data, size_t size) +{ + assert (mHandle != INVALID_HANDLE_VALUE); + + DWORD read; + + if (!ReadFile (mHandle, data, size, &read, NULL)) + throw std::runtime_error ("A read operation on a file failed."); + + return read; +} + +#endif diff --git a/components/files/lowlevelfile.hpp b/components/files/lowlevelfile.hpp new file mode 100644 index 000000000..a6de6df76 --- /dev/null +++ b/components/files/lowlevelfile.hpp @@ -0,0 +1,54 @@ +#ifndef COMPONENTS_FILES_LOWLEVELFILE_HPP +#define COMPONENTS_FILES_LOWLEVELFILE_HPP + +#include + +#define FILE_API_STDIO 0 +#define FILE_API_POSIX 1 +#define FILE_API_WIN32 2 + +#if OGRE_PLATFORM == OGRE_PLATFORM_LINUX +#define FILE_API FILE_API_POSIX +#elif OGRE_PLATFORM == OGRE_PLATFORM_WIN32 +#define FILE_API FILE_API_WIN32 +#else +#define FILE_API FILE_API_STDIO +#endif + +#if FILE_API == FILE_API_STDIO +#include +#elif FILE_API == FILE_API_POSIX +#elif FILE_API == FILE_API_WIN32 +#include +#else +#error Unsupported File API +#endif + +class LowLevelFile +{ +public: + + LowLevelFile (); + ~LowLevelFile (); + + void open (char const * filename); + void close (); + + size_t size (); + + void seek (size_t Position); + size_t tell (); + + size_t read (void * data, size_t size); + +private: +#if FILE_API == FILE_API_STDIO + FILE* mHandle; +#elif FILE_API == FILE_API_POSIX + int mHandle; +#elif FILE_API == FILE_API_WIN32 + HANDLE mHandle; +#endif +}; + +#endif