From d81820303db264c045bf44fb11154f3539d7823d Mon Sep 17 00:00:00 2001 From: Armin Preiml Date: Sat, 31 Jul 2010 18:55:52 +0200 Subject: [PATCH 1/2] Added cmake file for the iconv library and extended relevant cmake files to use it. Added convertToUTF8 to esm_reader. getString(int size) uses this method now. --- cmake/FindIconv.cmake | 64 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 cmake/FindIconv.cmake diff --git a/cmake/FindIconv.cmake b/cmake/FindIconv.cmake new file mode 100644 index 0000000000..000e09b845 --- /dev/null +++ b/cmake/FindIconv.cmake @@ -0,0 +1,64 @@ +# - Try to find Iconv +# Once done this will define +# +# ICONV_FOUND - system has Iconv +# ICONV_INCLUDE_DIR - the Iconv include directory +# ICONV_LIBRARIES - Link these to use Iconv +# ICONV_SECOND_ARGUMENT_IS_CONST - the second argument for iconv() is const +# +include(CheckCCompilerFlag) +include(CheckCXXSourceCompiles) + +IF (ICONV_INCLUDE_DIR AND ICONV_LIBRARIES) + # Already in cache, be silent + SET(ICONV_FIND_QUIETLY TRUE) +ENDIF (ICONV_INCLUDE_DIR AND ICONV_LIBRARIES) + +FIND_PATH(ICONV_INCLUDE_DIR iconv.h) + +FIND_LIBRARY(ICONV_LIBRARIES NAMES iconv libiconv c) + +IF(ICONV_INCLUDE_DIR AND ICONV_LIBRARIES) + SET(ICONV_FOUND TRUE) +ENDIF(ICONV_INCLUDE_DIR AND ICONV_LIBRARIES) + +set(CMAKE_REQUIRED_INCLUDES ${ICONV_INCLUDE_DIR}) +set(CMAKE_REQUIRED_LIBRARIES ${ICONV_LIBRARIES}) +IF(ICONV_FOUND) + check_c_compiler_flag("-Werror" ICONV_HAVE_WERROR) + set (CMAKE_C_FLAGS_BACKUP "${CMAKE_C_FLAGS}") + if(ICONV_HAVE_WERROR) + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror") + endif(ICONV_HAVE_WERROR) + check_c_source_compiles(" + #include + int main(){ + iconv_t conv = 0; + const char* in = 0; + size_t ilen = 0; + char* out = 0; + size_t olen = 0; + iconv(conv, &in, &ilen, &out, &olen); + return 0; + } +" ICONV_SECOND_ARGUMENT_IS_CONST ) + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS_BACKUP}") +ENDIF(ICONV_FOUND) +set(CMAKE_REQUIRED_INCLUDES) +set(CMAKE_REQUIRED_LIBRARIES) + +IF(ICONV_FOUND) + IF(NOT ICONV_FIND_QUIETLY) + MESSAGE(STATUS "Found Iconv: ${ICONV_LIBRARIES}") + ENDIF(NOT ICONV_FIND_QUIETLY) +ELSE(ICONV_FOUND) + IF(Iconv_FIND_REQUIRED) + MESSAGE(FATAL_ERROR "Could not find Iconv") + ENDIF(Iconv_FIND_REQUIRED) +ENDIF(ICONV_FOUND) + +MARK_AS_ADVANCED( + ICONV_INCLUDE_DIR + ICONV_LIBRARIES + ICONV_SECOND_ARGUMENT_IS_CONST +) From f3ee9ced5cb2bb06c709ceaf2cc077acf1fd4614 Mon Sep 17 00:00:00 2001 From: Armin Preiml Date: Sat, 31 Jul 2010 19:02:29 +0200 Subject: [PATCH 2/2] Added cmake file for the iconv library and extended relevant cmake files to use it. Added convertToUTF8 to esm_reader. getString(int size) uses this method now. --- CMakeLists.txt | 2 + apps/openmw/CMakeLists.txt | 1 + components/esm/esm_reader.hpp | 88 ++++++++++++++++++++++++++++++++++- 3 files changed, 90 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2a61e61aa2..cd058c570e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -109,9 +109,11 @@ endif (WIN32) find_package(OGRE REQUIRED) find_package(Boost REQUIRED COMPONENTS system filesystem program_options thread) find_package(OIS REQUIRED) +find_package(Iconv REQUIRED) include_directories("." ${OGRE_INCLUDE_DIR} ${OGRE_INCLUDE_DIR}/Ogre ${OIS_INCLUDE_DIR} ${Boost_INCLUDE_DIR} + ${ICONV_INCLUDE_DIR} ${PLATFORM_INCLUDE_DIR} ${CMAKE_HOME_DIRECTORY}/extern/caelum/include ${CMAKE_HOME_DIRECTORY}/extern/mygui_3.0.1/MyGUIEngine/include diff --git a/apps/openmw/CMakeLists.txt b/apps/openmw/CMakeLists.txt index b3d0649a8d..c254dcaef6 100644 --- a/apps/openmw/CMakeLists.txt +++ b/apps/openmw/CMakeLists.txt @@ -104,6 +104,7 @@ target_link_libraries(openmw ${OGRE_LIBRARIES} ${OIS_LIBRARIES} ${Boost_LIBRARIES} + ${ICONV_LIBRARIES} caelum MyGUIEngine MyGUI.OgrePlatform diff --git a/components/esm/esm_reader.hpp b/components/esm/esm_reader.hpp index 7db21323f1..098afc5131 100644 --- a/components/esm/esm_reader.hpp +++ b/components/esm/esm_reader.hpp @@ -8,6 +8,8 @@ #include #include #include +#include +#include #include #include @@ -613,7 +615,91 @@ public: // Convert to std::string and return std::string res(ptr,size); delete[] ptr; - return res; + return convertToUTF8(res); + } + + // Convert a string from ISO-8859-1 encoding to UTF-8 + std::string convertToUTF8(std::string input) + { + std::string output = ""; + + //create convert description + iconv_t cd = iconv_open("UTF-8", "ISO-8859-1"); + + if (cd == (iconv_t)-1) //error handling + { + std::string errMsg = "Creating description for UTF-8 converting failed: "; + + switch (errno) //detailed error messages (maybe it contains too much detail :) + { + case EMFILE: + errMsg += "{OPEN_MAX} files descriptors are currently open in the calling process."; + case ENFILE: + errMsg += "Too many files are currently open in the system."; + case ENOMEM: + errMsg +="Insufficient storage space is available."; + case EINVAL: + errMsg += "The conversion specified by fromcode and tocode is not supported by the implementation."; + + default: + errMsg += "Unknown Error\n"; + } + + fail(errMsg); + + } + else + { + const size_t inputSize = input.size(); + + if (inputSize) //input is not empty + { + //convert function doesn't accept const char *, therefore copy content into an char * + std::vector inputBuffer(input.begin(), input.end()); + char *inputBufferBegin = &inputBuffer[0]; + + size_t inputBytesLeft = inputSize; //bytes to convert + + static const size_t outputSize = 1000; + size_t outputBytesLeft; + + char outputBuffer[outputSize]; + char *outputBufferBegin; + + while (inputBytesLeft > 0 ) + { + outputBytesLeft = outputSize; + outputBufferBegin = outputBuffer; + + if (iconv(cd, &inputBufferBegin, &inputBytesLeft, &outputBufferBegin, &outputBytesLeft) == (size_t)-1) + { + switch (errno) + { + case E2BIG: //outputBuffer is full + output += std::string(outputBuffer, outputSize); + break; + case EILSEQ: + fail("Iconv: Invalid multibyte sequence.\n"); + break; + case EINVAL: + fail("Iconv: Incomplete multibyte sequence.\n"); + break; + default: + fail("Iconv: Unknown Error\n"); + } + + } + } + + //read only relevant bytes from outputBuffer + output += std::string(outputBuffer, outputSize - outputBytesLeft); + + } + } + + iconv_close (cd); + + return output; } void skip(int bytes) { esm->seek(esm->tell()+bytes); }