increased performance of ConstrainedDataStream

Reimplemented ConstrainedDataStream to use low-level IO calls and a
custom buffering scheme to avoid using C++ iostreams.

components/CMakeLists.txt

@@ -48,7 +48,7 @@ add_component_dir (misc
 
 add_component_dir (files
     linuxpath windowspath macospath fixedpath multidircollection collections fileops configurationmanager
-    filelibrary ogreplugin constrainedfiledatastream
+    filelibrary ogreplugin constrainedfiledatastream lowlevelfile
     )
 
 add_component_dir (compiler

components/files/constrainedfiledatastream.cpp

@@ -1,112 +1,170 @@
-/*
-  OpenMW - The completely unofficial reimplementation of Morrowind
-  Copyright (C) 2008-2010  Nicolay Korslund
-  Email: < korslund@gmail.com >
-  WWW: http://openmw.sourceforge.net/
+#include "constrainedfiledatastream.hpp"
+#include "lowlevelfile.hpp"
 
-  This file (bsa_file.cpp) is part of the OpenMW package.
+#include <stdexcept>
+#include <cassert>
 
-  OpenMW is distributed as free software: you can redistribute it
-  and/or modify it under the terms of the GNU General Public License
-  version 3, as published by the Free Software Foundation.
+namespace {
 
-  This program is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
+class ConstrainedDataStream : public Ogre::DataStream {
+public:
 
-  You should have received a copy of the GNU General Public License
-  version 3 along with this program. If not, see
-  http://www.gnu.org/licenses/ .
+	static const size_t sBufferSize = 4096; // somewhat arbitrary though 64KB buffers didn't seem to improve performance any
+	static const size_t sBufferThreshold = 1024; // reads larger than this bypass buffering as cost of memcpy outweighs cost of system call
 
- */
+    ConstrainedDataStream(const Ogre::String &fname, size_t start, size_t length)
+	{
+		mFile.open (fname.c_str ());
+		mSize  = length != 0xFFFFFFFF ? length : mFile.size () - start;
 
-#include "constrainedfiledatastream.hpp"
+		mPos    = 0;
+		mOrigin = start;
+		mExtent = start + mSize;
 
-#include <stdexcept>
+		mBufferOrigin = 0;
+		mBufferExtent = 0;
+	}
+		
 
-class ConstrainedDataStream : public Ogre::DataStream {
-    std::ifstream mStream;
-    const size_t mStart;
-    size_t mPos;
-    bool mIsEOF;
+	size_t read(void* buf, size_t count)
+	{
+		assert (mPos <= mSize);
 
-public:
-    ConstrainedDataStream(const Ogre::String &fname, size_t start, size_t length)
-      : mStream(fname.c_str(), std::ios_base::binary), mStart(start), mPos(0), mIsEOF(false)
-    {
-        mSize = length;
-        if(!mStream.seekg(mStart, std::ios_base::beg))
-            throw std::runtime_error("Error seeking to start of BSA entry");
-    }
+		uint8_t * out = reinterpret_cast <uint8_t *> (buf);
 
-    ConstrainedDataStream(const Ogre::String &name, const Ogre::String &fname,
-                          size_t start, size_t length)
-      : Ogre::DataStream(name), mStream(fname.c_str(), std::ios_base::binary),
-        mStart(start), mPos(0), mIsEOF(false)
-    {
-        mSize = length;
-        if(!mStream.seekg(mStart, std::ios_base::beg))
-            throw std::runtime_error("Error seeking to start of BSA entry");
-    }
+		size_t posBeg = mOrigin + mPos;
+		size_t posEnd = posBeg + count;
 
+		if (posEnd > mExtent)
+			posEnd = mExtent;
 
-    virtual size_t read(void *buf, size_t count)
-    {
-        mStream.clear();
-
-        if(count > mSize-mPos)
-        {
-            count = mSize-mPos;
-            mIsEOF = true;
-        }
-        mStream.read(reinterpret_cast<char*>(buf), count);
-
-        count = mStream.gcount();
-        mPos += count;
-        return count;
-    }
+		size_t posCur = posBeg;
+
+		while (posCur != posEnd)
+		{
+			size_t readLeft = posEnd - posCur;
+
+			if (posCur < mBufferOrigin || posCur >= mBufferExtent)
+			{
+				if (readLeft >= sBufferThreshold || (posCur == mOrigin && posEnd == mExtent))
+				{
+					assert (mFile.tell () == mBufferExtent);
+
+					if (posCur != mBufferExtent)
+						mFile.seek (posCur);
+
+					posCur += mFile.read (out, readLeft);
+
+					mBufferOrigin = mBufferExtent = posCur;
+
+					mPos = posCur - mOrigin;
+
+					return posCur - posBeg;
+				}
+				else
+				{
+					size_t newBufferOrigin;
+
+					if ((posCur < mBufferOrigin) && (mBufferOrigin - posCur < sBufferSize))
+						newBufferOrigin = std::max (mOrigin, mBufferOrigin > sBufferSize ? mBufferOrigin - sBufferSize : 0);
+					else
+						newBufferOrigin = posCur;
+
+					fill (newBufferOrigin);
+				}
+			}
+
+			size_t xfer = std::min (readLeft, mBufferExtent - posCur);
 
-    virtual void skip(long count)
+			memcpy (out, mBuffer + (posCur - mBufferOrigin), xfer);
+
+			posCur += xfer;
+			out += xfer;
+		}
+
+		count = posEnd - posBeg;
+		mPos += count;
+		return count;
+	}
+
+    void skip(long count)
     {
+		assert (mPos <= mSize);
+
         if((count >= 0 && (size_t)count <= mSize-mPos) ||
            (count < 0 && (size_t)-count <= mPos))
-        {
-            mStream.clear();
-            if(mStream.seekg(count, std::ios_base::cur))
-            {
-                mPos += count;
-                mIsEOF = false;
-            }
-        }
+			mPos += count;
     }
 
-    virtual void seek(size_t pos)
+    void seek(size_t pos)
     {
-        if(pos < mSize)
-        {
-            mStream.clear();
-            if(mStream.seekg(pos+mStart, std::ios_base::beg))
-            {
-                mPos = pos;
-                mIsEOF = false;
-            }
-        }
+		assert (mPos <= mSize);
+
+        if (pos < mSize)
+			mPos = pos;
     }
 
     virtual size_t tell() const
-    { return mPos; }
+    {
+		assert (mPos <= mSize);
+
+		return mPos;
+	}
 
     virtual bool eof() const
-    { return mIsEOF; }
+    {
+		assert (mPos <= mSize);
+
+		return mPos == mSize;
+	}
 
     virtual void close()
-    { mStream.close(); }
+    {
+		mFile.close();
+	}
+
+private:
+
+	void fill (size_t newOrigin)
+	{
+		assert (mFile.tell () == mBufferExtent);
+
+		size_t newExtent = newOrigin + sBufferSize;
+
+		if (newExtent > mExtent)
+			newExtent = mExtent;
+
+		size_t oldExtent = mBufferExtent;
+
+		if (newOrigin != oldExtent)
+			mFile.seek (newOrigin);
+
+		mBufferOrigin = mBufferExtent = newOrigin;
+
+		size_t amountRequested = newExtent - newOrigin;
+
+		size_t amountRead = mFile.read (mBuffer, amountRequested);
+
+		if (amountRead != amountRequested)
+			throw std::runtime_error ("An unexpected condition occurred while reading from a file.");
+
+		mBufferExtent = newExtent;
+	}
+
+	LowLevelFile mFile;
+
+	size_t mOrigin;
+	size_t mExtent;
+	size_t mPos;
+
+	uint8_t mBuffer [sBufferSize];
+	size_t mBufferOrigin;
+	size_t mBufferExtent;
 };
 
+} // end of unnamed namespace
+
 Ogre::DataStreamPtr openConstrainedFileDataStream (char const * filename, size_t offset, size_t length)
 {
-	assert (length != 0xFFFFFFFF); // reserved for future use...
-
 	return Ogre::DataStreamPtr(new ConstrainedDataStream(filename, offset, length));
 }

components/files/lowlevelfile.cpp

@@ -0,0 +1,299 @@
+#include "lowlevelfile.hpp"
+
+#include <stdexcept>
+#include <sstream>
+#include <cassert>
+
+#if FILE_API == FILE_API_POSIX
+#include <sys/types.h>
+#include <unistd.h>
+#include <fcntl.h>
+#endif
+
+#if FILE_API == FILE_API_STDIO
+/*
+ *
+ *	Implementation of LowLevelFile methods using c stdio
+ *
+ */
+
+LowLevelFile::LowLevelFile ()
+{
+	mHandle = NULL;
+}
+
+LowLevelFile::~LowLevelFile ()
+{
+	if (mHandle != NULL)
+		fclose (mHandle);
+}
+
+void LowLevelFile::open (char const * filename)
+{
+	assert (mHandle == NULL);
+
+	mHandle = fopen (filename, "rb");
+
+	if (mHandle == NULL)
+	{
+		std::ostringstream os;
+		os << "Failed to open '" << filename << "' for reading.";
+		throw std::runtime_error (os.str ());
+	}
+}
+
+void LowLevelFile::close ()
+{
+	assert (mHandle != NULL);
+
+	fclose (mHandle);
+
+	mHandle = NULL;
+}
+
+size_t LowLevelFile::size ()
+{
+	assert (mHandle != NULL);
+
+	long oldPosition = ftell (mHandle);
+
+	if (oldPosition == -1)
+		throw std::runtime_error ("A query operation on a file failed.");
+
+	if (fseek (mHandle, 0, SEEK_END) != 0)
+		throw std::runtime_error ("A query operation on a file failed.");
+
+	long Size = ftell (mHandle);
+
+	if (Size == -1)
+		throw std::runtime_error ("A query operation on a file failed.");
+
+	if (fseek (mHandle, oldPosition, SEEK_SET) != 0)
+		throw std::runtime_error ("A query operation on a file failed.");
+
+	return size_t (Size);
+}
+
+void LowLevelFile::seek (size_t Position)
+{
+	assert (mHandle != NULL);
+
+	if (fseek (mHandle, Position, SEEK_SET) != 0)
+		throw std::runtime_error ("A seek operation on a file failed.");
+}
+
+size_t LowLevelFile::tell ()
+{
+	assert (mHandle != NULL);
+
+	long Position = ftell (mHandle);
+
+	if (Position == -1)
+		throw std::runtime_error ("A query operation on a file failed.");
+
+	return size_t (Position);
+}
+
+size_t LowLevelFile::read (void * data, size_t size)
+{
+	assert (mHandle != NULL);
+
+	int amount = fread (data, 1, size, mHandle);
+
+	if (amount == 0 && ferror (mHandle))
+		throw std::runtime_error ("A read operation on a file failed.");
+
+	return amount;
+}
+
+#elif FILE_API == FILE_API_POSIX
+/*
+ *
+ *	Implementation of LowLevelFile methods using posix IO calls
+ *
+ */
+
+LowLevelFile::LowLevelFile ()
+{
+	mHandle = -1;
+}
+
+LowLevelFile::~LowLevelFile ()
+{
+	if (mHandle != -1)
+		::close (mHandle);
+}
+
+void LowLevelFile::open (char const * filename)
+{
+	assert (mHandle == -1);
+
+#ifdef O_BINARY
+	static const int openFlags = O_RDONLY | O_BINARY;
+#else
+	static const int openFlags = O_RDONLY;
+#endif
+
+	mHandle = ::open (filename, openFlags, 0);
+
+	if (mHandle == -1)
+	{
+		std::ostringstream os;
+		os << "Failed to open '" << filename << "' for reading.";
+		throw std::runtime_error (os.str ());
+	}
+}
+
+void LowLevelFile::close ()
+{
+	assert (mHandle != -1);
+
+	::close (mHandle);
+
+	mHandle = -1;
+}
+
+size_t LowLevelFile::size ()
+{
+	assert (mHandle != -1);
+
+	size_t oldPosition = ::lseek (mHandle, 0, SEEK_CUR);
+
+	if (oldPosition == size_t (-1))
+		throw std::runtime_error ("A query operation on a file failed.");
+
+	size_t Size = ::lseek (mHandle, 0, SEEK_END);
+
+	if (Size == size_t (-1))
+		throw std::runtime_error ("A query operation on a file failed.");
+
+	if (lseek (mHandle, oldPosition, SEEK_SET) == -1)
+		throw std::runtime_error ("A query operation on a file failed.");
+
+	return Size;
+}
+
+void LowLevelFile::seek (size_t Position)
+{
+	assert (mHandle != -1);
+
+	if (::lseek (mHandle, Position, SEEK_SET) == -1)
+		throw std::runtime_error ("A seek operation on a file failed.");
+}
+
+size_t LowLevelFile::tell ()
+{
+	assert (mHandle != -1);
+
+	size_t Position = ::lseek (mHandle, 0, SEEK_CUR);
+
+	if (Position == size_t (-1))
+		throw std::runtime_error ("A query operation on a file failed.");
+
+	return Position;
+}
+
+size_t LowLevelFile::read (void * data, size_t size)
+{
+	assert (mHandle != -1);
+
+	int amount = ::read (mHandle, data, size);
+
+	if (amount == -1)
+		throw std::runtime_error ("A read operation on a file failed.");
+
+	return amount;
+}
+
+#elif FILE_API == FILE_API_WIN32
+/*
+ *
+ *	Implementation of LowLevelFile methods using Win32 API calls
+ *
+ */
+
+LowLevelFile::LowLevelFile ()
+{
+	mHandle = INVALID_HANDLE_VALUE;
+}
+
+LowLevelFile::~LowLevelFile ()
+{
+	if (mHandle == INVALID_HANDLE_VALUE)
+		CloseHandle (mHandle);
+}
+
+void LowLevelFile::open (char const * filename)
+{
+	assert (mHandle == INVALID_HANDLE_VALUE);
+
+	HANDLE handle = CreateFileA (filename, GENERIC_READ, FILE_SHARE_READ, 0, OPEN_EXISTING, 0, 0);
+
+	if (handle == NULL)
+	{
+		std::ostringstream os;
+		os << "Failed to open '" << filename << "' for reading.";
+		throw std::runtime_error (os.str ());
+	}
+
+	mHandle = handle;
+}
+
+void LowLevelFile::close ()
+{
+	assert (mHandle != INVALID_HANDLE_VALUE);
+
+	CloseHandle (mHandle);
+
+	mHandle = INVALID_HANDLE_VALUE;
+}
+
+size_t LowLevelFile::size ()
+{
+	assert (mHandle != INVALID_HANDLE_VALUE);
+
+	BY_HANDLE_FILE_INFORMATION info;
+
+	if (!GetFileInformationByHandle (mHandle, &info))
+		throw std::runtime_error ("A query operation on a file failed.");
+
+	if (info.nFileSizeHigh != 0)
+		throw std::runtime_error ("Files greater that 4GB are not supported.");
+
+	return info.nFileSizeLow;
+}
+
+void LowLevelFile::seek (size_t Position)
+{
+	assert (mHandle != INVALID_HANDLE_VALUE);
+
+	if (SetFilePointer (mHandle, Position, NULL, SEEK_SET) == INVALID_SET_FILE_POINTER)
+		if (GetLastError () != NO_ERROR)
+			throw std::runtime_error ("A seek operation on a file failed.");
+}
+
+size_t LowLevelFile::tell ()
+{
+	assert (mHandle != INVALID_HANDLE_VALUE);
+
+	DWORD value = SetFilePointer (mHandle, 0, NULL, SEEK_CUR);
+
+	if (value == INVALID_SET_FILE_POINTER && GetLastError () != NO_ERROR)
+		throw std::runtime_error ("A query operation on a file failed.");
+
+	return value;
+}
+
+size_t LowLevelFile::read (void * data, size_t size)
+{
+	assert (mHandle != INVALID_HANDLE_VALUE);
+
+	DWORD read;
+
+	if (!ReadFile (mHandle, data, size, &read, NULL))
+		throw std::runtime_error ("A read operation on a file failed.");
+
+	return read;
+}
+
+#endif

components/files/lowlevelfile.hpp

@@ -0,0 +1,54 @@
+#ifndef COMPONENTS_FILES_LOWLEVELFILE_HPP
+#define COMPONENTS_FILES_LOWLEVELFILE_HPP
+
+#include <OgrePlatform.h>
+
+#define FILE_API_STDIO	0
+#define FILE_API_POSIX	1
+#define FILE_API_WIN32	2
+
+#if OGRE_PLATFORM == OGRE_PLATFORM_LINUX
+#define FILE_API	FILE_API_POSIX
+#elif OGRE_PLATFORM == OGRE_PLATFORM_WIN32
+#define FILE_API	FILE_API_WIN32
+#else
+#define FILE_API	FILE_API_STDIO
+#endif
+
+#if FILE_API == FILE_API_STDIO
+#include <cstdio>
+#elif FILE_API == FILE_API_POSIX
+#elif FILE_API == FILE_API_WIN32
+#include <windows.h>
+#else
+#error Unsupported File API
+#endif
+
+class LowLevelFile
+{
+public:
+
+	LowLevelFile ();
+	~LowLevelFile ();
+
+	void open (char const * filename);
+	void close ();
+
+	size_t size ();
+
+	void seek (size_t Position);
+	size_t tell ();
+
+	size_t read (void * data, size_t size);
+
+private:
+#if FILE_API == FILE_API_STDIO
+	FILE* mHandle;
+#elif FILE_API == FILE_API_POSIX
+	int mHandle;
+#elif FILE_API == FILE_API_WIN32
+	HANDLE mHandle;
+#endif
+};
+
+#endif