|
|
|
@ -2,6 +2,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
#include <vector>
|
|
|
|
#include <vector>
|
|
|
|
#include <cassert>
|
|
|
|
#include <cassert>
|
|
|
|
|
|
|
|
#include <iostream>
|
|
|
|
|
|
|
|
#include <iomanip>
|
|
|
|
|
|
|
|
|
|
|
|
/* This file contains the code to translate from WINDOWS-1252 (native
|
|
|
|
/* This file contains the code to translate from WINDOWS-1252 (native
|
|
|
|
charset used in English version of Morrowind) to UTF-8. The library
|
|
|
|
charset used in English version of Morrowind) to UTF-8. The library
|
|
|
|
@ -39,341 +41,298 @@
|
|
|
|
// Generated tables
|
|
|
|
// Generated tables
|
|
|
|
#include "tables_gen.hpp"
|
|
|
|
#include "tables_gen.hpp"
|
|
|
|
|
|
|
|
|
|
|
|
// Shared global buffers, we love you. These initial sizes are large
|
|
|
|
using namespace ToUTF8;
|
|
|
|
// enough to hold the largest books in Morrowind.esm, but we will
|
|
|
|
|
|
|
|
// resize automaticall if necessary.
|
|
|
|
|
|
|
|
static std::vector<char> buf (50*1024);
|
|
|
|
|
|
|
|
static std::vector<char> output (50*1024);
|
|
|
|
|
|
|
|
static int size;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Make sure the given vector is large enough for 'size' bytes,
|
|
|
|
Utf8Encoder::Utf8Encoder(const FromType sourceEncoding):
|
|
|
|
// including a terminating zero after it.
|
|
|
|
mOutput(50*1024)
|
|
|
|
static void resize(std::vector<char> &buf, size_t size)
|
|
|
|
|
|
|
|
{
|
|
|
|
{
|
|
|
|
if(buf.size() <= size)
|
|
|
|
switch (sourceEncoding)
|
|
|
|
// Add some extra padding to reduce the chance of having to resize
|
|
|
|
{
|
|
|
|
// again later.
|
|
|
|
case ToUTF8::WINDOWS_1252:
|
|
|
|
buf.resize(3*size);
|
|
|
|
{
|
|
|
|
|
|
|
|
translationArray = ToUTF8::windows_1252;
|
|
|
|
|
|
|
|
break;
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
case ToUTF8::WINDOWS_1250:
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
translationArray = ToUTF8::windows_1250;
|
|
|
|
|
|
|
|
break;
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
case ToUTF8::WINDOWS_1251:
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
translationArray = ToUTF8::windows_1251;
|
|
|
|
|
|
|
|
break;
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
default:
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
assert(0);
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// And make sure the string is zero terminated
|
|
|
|
std::string Utf8Encoder::getUtf8(const char* input, int size)
|
|
|
|
buf[size] = 0;
|
|
|
|
{
|
|
|
|
|
|
|
|
// Double check that the input string stops at some point (it might
|
|
|
|
|
|
|
|
// contain zero terminators before this, inside its own data, which
|
|
|
|
|
|
|
|
// is also ok.)
|
|
|
|
|
|
|
|
assert(input[size] == 0);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// TODO: The rest of this function is designed for single-character
|
|
|
|
|
|
|
|
// input encodings only. It also assumes that the input the input
|
|
|
|
|
|
|
|
// encoding shares its first 128 values (0-127) with ASCII. These
|
|
|
|
|
|
|
|
// conditions must be checked again if you add more input encodings
|
|
|
|
|
|
|
|
// later.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Compute output length, and check for pure ascii input at the same
|
|
|
|
|
|
|
|
// time.
|
|
|
|
|
|
|
|
bool ascii;
|
|
|
|
|
|
|
|
size_t outlen = getLength(input, ascii);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// If we're pure ascii, then don't bother converting anything.
|
|
|
|
|
|
|
|
if(ascii)
|
|
|
|
|
|
|
|
return std::string(input, outlen);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Make sure the output is large enough
|
|
|
|
|
|
|
|
resize(outlen);
|
|
|
|
|
|
|
|
char *out = &mOutput[0];
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Translate
|
|
|
|
|
|
|
|
while (*input)
|
|
|
|
|
|
|
|
copyFromArray(*(input++), out);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Make sure that we wrote the correct number of bytes
|
|
|
|
|
|
|
|
assert((out-&mOutput[0]) == (int)outlen);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// And make extra sure the output is null terminated
|
|
|
|
|
|
|
|
assert(mOutput.size() > outlen);
|
|
|
|
|
|
|
|
assert(mOutput[outlen] == 0);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Return a string
|
|
|
|
|
|
|
|
return std::string(&mOutput[0], outlen);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// This is just used to spew out a reusable input buffer for the
|
|
|
|
std::string Utf8Encoder::getLegacyEnc(const char *input, int size)
|
|
|
|
// conversion process.
|
|
|
|
|
|
|
|
char *ToUTF8::getBuffer(int s)
|
|
|
|
|
|
|
|
{
|
|
|
|
{
|
|
|
|
// Remember the requested size
|
|
|
|
// Double check that the input string stops at some point (it might
|
|
|
|
size = s;
|
|
|
|
// contain zero terminators before this, inside its own data, which
|
|
|
|
resize(buf, size);
|
|
|
|
// is also ok.)
|
|
|
|
return &buf[0];
|
|
|
|
assert(input[size] == 0);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// TODO: The rest of this function is designed for single-character
|
|
|
|
|
|
|
|
// input encodings only. It also assumes that the input the input
|
|
|
|
|
|
|
|
// encoding shares its first 128 values (0-127) with ASCII. These
|
|
|
|
|
|
|
|
// conditions must be checked again if you add more input encodings
|
|
|
|
|
|
|
|
// later.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Compute output length, and check for pure ascii input at the same
|
|
|
|
|
|
|
|
// time.
|
|
|
|
|
|
|
|
bool ascii;
|
|
|
|
|
|
|
|
size_t outlen = getLength2(input, ascii);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// If we're pure ascii, then don't bother converting anything.
|
|
|
|
|
|
|
|
if(ascii)
|
|
|
|
|
|
|
|
return std::string(input, outlen);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Make sure the output is large enough
|
|
|
|
|
|
|
|
resize(outlen);
|
|
|
|
|
|
|
|
char *out = &mOutput[0];
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Translate
|
|
|
|
|
|
|
|
while(*input)
|
|
|
|
|
|
|
|
copyFromArray2(input, out);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Make sure that we wrote the correct number of bytes
|
|
|
|
|
|
|
|
assert((out-&mOutput[0]) == (int)outlen);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// And make extra sure the output is null terminated
|
|
|
|
|
|
|
|
assert(mOutput.size() > outlen);
|
|
|
|
|
|
|
|
assert(mOutput[outlen] == 0);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Return a string
|
|
|
|
|
|
|
|
return std::string(&mOutput[0], outlen);
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Make sure the output vector is large enough for 'size' bytes,
|
|
|
|
|
|
|
|
// including a terminating zero after it.
|
|
|
|
|
|
|
|
void Utf8Encoder::resize(size_t size)
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
if (mOutput.size() <= size)
|
|
|
|
|
|
|
|
// Add some extra padding to reduce the chance of having to resize
|
|
|
|
|
|
|
|
// again later.
|
|
|
|
|
|
|
|
mOutput.resize(3*size);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// And make sure the string is zero terminated
|
|
|
|
|
|
|
|
mOutput[size] = 0;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/** Get the total length length needed to decode the given string with
|
|
|
|
/** Get the total length length needed to decode the given string with
|
|
|
|
the given translation array. The arrays are encoded with 6 bytes
|
|
|
|
the given translation array. The arrays are encoded with 6 bytes
|
|
|
|
per character, with the first giving the length and the next 5 the
|
|
|
|
per character, with the first giving the length and the next 5 the
|
|
|
|
actual data.
|
|
|
|
actual data.
|
|
|
|
|
|
|
|
|
|
|
|
The function serves a dual purpose for optimization reasons: it
|
|
|
|
The function serves a dual purpose for optimization reasons: it
|
|
|
|
checks if the input is pure ascii (all values are <= 127). If this
|
|
|
|
checks if the input is pure ascii (all values are <= 127). If this
|
|
|
|
is the case, then the ascii parameter is set to true, and the
|
|
|
|
is the case, then the ascii parameter is set to true, and the
|
|
|
|
caller can optimize for this case.
|
|
|
|
caller can optimize for this case.
|
|
|
|
*/
|
|
|
|
*/
|
|
|
|
static size_t getLength(const char *arr, const char* input, bool &ascii)
|
|
|
|
size_t Utf8Encoder::getLength(const char* input, bool &ascii)
|
|
|
|
{
|
|
|
|
{
|
|
|
|
ascii = true;
|
|
|
|
ascii = true;
|
|
|
|
size_t len = 0;
|
|
|
|
size_t len = 0;
|
|
|
|
const char* ptr = input;
|
|
|
|
const char* ptr = input;
|
|
|
|
unsigned char inp = *ptr;
|
|
|
|
unsigned char inp = *ptr;
|
|
|
|
|
|
|
|
|
|
|
|
// Do away with the ascii part of the string first (this is almost
|
|
|
|
// Do away with the ascii part of the string first (this is almost
|
|
|
|
// always the entire string.)
|
|
|
|
// always the entire string.)
|
|
|
|
while(inp && inp < 128)
|
|
|
|
while (inp && inp < 128)
|
|
|
|
inp = *(++ptr);
|
|
|
|
inp = *(++ptr);
|
|
|
|
len += (ptr-input);
|
|
|
|
len += (ptr-input);
|
|
|
|
|
|
|
|
|
|
|
|
// If we're not at the null terminator at this point, then there
|
|
|
|
// If we're not at the null terminator at this point, then there
|
|
|
|
// were some non-ascii characters to deal with. Go to slow-mode for
|
|
|
|
// were some non-ascii characters to deal with. Go to slow-mode for
|
|
|
|
// the rest of the string.
|
|
|
|
// the rest of the string.
|
|
|
|
if(inp)
|
|
|
|
if (inp)
|
|
|
|
{
|
|
|
|
{
|
|
|
|
ascii = false;
|
|
|
|
ascii = false;
|
|
|
|
while(inp)
|
|
|
|
while (inp)
|
|
|
|
{
|
|
|
|
{
|
|
|
|
// Find the translated length of this character in the
|
|
|
|
// Find the translated length of this character in the
|
|
|
|
// lookup table.
|
|
|
|
// lookup table.
|
|
|
|
len += arr[inp*6];
|
|
|
|
len += translationArray[inp*6];
|
|
|
|
inp = *(++ptr);
|
|
|
|
inp = *(++ptr);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return len;
|
|
|
|
return len;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// Translate one character 'ch' using the translation array 'arr', and
|
|
|
|
// Translate one character 'ch' using the translation array 'arr', and
|
|
|
|
// advance the output pointer accordingly.
|
|
|
|
// advance the output pointer accordingly.
|
|
|
|
static void copyFromArray(const char *arr, unsigned char ch, char* &out)
|
|
|
|
void Utf8Encoder::copyFromArray(unsigned char ch, char* &out)
|
|
|
|
{
|
|
|
|
{
|
|
|
|
// Optimize for ASCII values
|
|
|
|
// Optimize for ASCII values
|
|
|
|
if(ch < 128)
|
|
|
|
if (ch < 128)
|
|
|
|
{
|
|
|
|
{
|
|
|
|
*(out++) = ch;
|
|
|
|
*(out++) = ch;
|
|
|
|
return;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
const char *in = arr + ch*6;
|
|
|
|
const char *in = translationArray + ch*6;
|
|
|
|
int len = *(in++);
|
|
|
|
int len = *(in++);
|
|
|
|
for(int i=0; i<len; i++)
|
|
|
|
for (int i=0; i<len; i++)
|
|
|
|
*(out++) = *(in++);
|
|
|
|
*(out++) = *(in++);
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
std::string ToUTF8::getUtf8(ToUTF8::FromType from)
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
// Pick translation array
|
|
|
|
|
|
|
|
const char *arr;
|
|
|
|
|
|
|
|
switch (from)
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
case ToUTF8::WINDOWS_1252:
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
arr = ToUTF8::windows_1252;
|
|
|
|
|
|
|
|
break;
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
case ToUTF8::WINDOWS_1250:
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
arr = ToUTF8::windows_1250;
|
|
|
|
|
|
|
|
break;
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
case ToUTF8::WINDOWS_1251:
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
arr = ToUTF8::windows_1251;
|
|
|
|
|
|
|
|
break;
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
default:
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
assert(0);
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Double check that the input string stops at some point (it might
|
|
|
|
|
|
|
|
// contain zero terminators before this, inside its own data, which
|
|
|
|
|
|
|
|
// is also ok.)
|
|
|
|
|
|
|
|
const char* input = &buf[0];
|
|
|
|
|
|
|
|
assert(input[size] == 0);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// TODO: The rest of this function is designed for single-character
|
|
|
|
|
|
|
|
// input encodings only. It also assumes that the input the input
|
|
|
|
|
|
|
|
// encoding shares its first 128 values (0-127) with ASCII. These
|
|
|
|
|
|
|
|
// conditions must be checked again if you add more input encodings
|
|
|
|
|
|
|
|
// later.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Compute output length, and check for pure ascii input at the same
|
|
|
|
|
|
|
|
// time.
|
|
|
|
|
|
|
|
bool ascii;
|
|
|
|
|
|
|
|
size_t outlen = getLength(arr, input, ascii);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// If we're pure ascii, then don't bother converting anything.
|
|
|
|
|
|
|
|
if(ascii)
|
|
|
|
|
|
|
|
return std::string(input, outlen);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Make sure the output is large enough
|
|
|
|
|
|
|
|
resize(output, outlen);
|
|
|
|
|
|
|
|
char *out = &output[0];
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Translate
|
|
|
|
|
|
|
|
while(*input)
|
|
|
|
|
|
|
|
copyFromArray(arr, *(input++), out);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Make sure that we wrote the correct number of bytes
|
|
|
|
|
|
|
|
assert((out-&output[0]) == (int)outlen);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// And make extra sure the output is null terminated
|
|
|
|
|
|
|
|
assert(output.size() > outlen);
|
|
|
|
|
|
|
|
assert(output[outlen] == 0);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Return a string
|
|
|
|
|
|
|
|
return std::string(&output[0], outlen);
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
static size_t getLength2(const char *arr, const char* input, bool &ascii)
|
|
|
|
size_t Utf8Encoder::getLength2(const char* input, bool &ascii)
|
|
|
|
{
|
|
|
|
{
|
|
|
|
ascii = true;
|
|
|
|
ascii = true;
|
|
|
|
size_t len = 0;
|
|
|
|
size_t len = 0;
|
|
|
|
const char* ptr = input;
|
|
|
|
const char* ptr = input;
|
|
|
|
unsigned char inp = *ptr;
|
|
|
|
unsigned char inp = *ptr;
|
|
|
|
|
|
|
|
|
|
|
|
// Do away with the ascii part of the string first (this is almost
|
|
|
|
// Do away with the ascii part of the string first (this is almost
|
|
|
|
// always the entire string.)
|
|
|
|
// always the entire string.)
|
|
|
|
while(inp && inp < 128)
|
|
|
|
while (inp && inp < 128)
|
|
|
|
inp = *(++ptr);
|
|
|
|
inp = *(++ptr);
|
|
|
|
len += (ptr-input);
|
|
|
|
len += (ptr-input);
|
|
|
|
|
|
|
|
|
|
|
|
// If we're not at the null terminator at this point, then there
|
|
|
|
// If we're not at the null terminator at this point, then there
|
|
|
|
// were some non-ascii characters to deal with. Go to slow-mode for
|
|
|
|
// were some non-ascii characters to deal with. Go to slow-mode for
|
|
|
|
// the rest of the string.
|
|
|
|
// the rest of the string.
|
|
|
|
if(inp)
|
|
|
|
if (inp)
|
|
|
|
{
|
|
|
|
{
|
|
|
|
ascii = false;
|
|
|
|
ascii = false;
|
|
|
|
while(inp)
|
|
|
|
while(inp)
|
|
|
|
{
|
|
|
|
{
|
|
|
|
len += 1;
|
|
|
|
len += 1;
|
|
|
|
// Find the translated length of this character in the
|
|
|
|
// Find the translated length of this character in the
|
|
|
|
// lookup table.
|
|
|
|
// lookup table.
|
|
|
|
switch(inp)
|
|
|
|
switch(inp)
|
|
|
|
{
|
|
|
|
{
|
|
|
|
case 0xe2: len -= 2; break;
|
|
|
|
case 0xe2: len -= 2; break;
|
|
|
|
case 0xc2:
|
|
|
|
case 0xc2:
|
|
|
|
case 0xcb:
|
|
|
|
case 0xcb:
|
|
|
|
case 0xc4:
|
|
|
|
case 0xc4:
|
|
|
|
case 0xc6:
|
|
|
|
case 0xc6:
|
|
|
|
case 0xc3:
|
|
|
|
case 0xc3:
|
|
|
|
case 0xd0:
|
|
|
|
case 0xd0:
|
|
|
|
case 0xd1:
|
|
|
|
case 0xd1:
|
|
|
|
case 0xd2:
|
|
|
|
case 0xd2:
|
|
|
|
case 0xc5: len -= 1; break;
|
|
|
|
case 0xc5: len -= 1; break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
inp = *(++ptr);
|
|
|
|
inp = *(++ptr);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return len;
|
|
|
|
return len;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#include <iostream>
|
|
|
|
void Utf8Encoder::copyFromArray2(const char*& chp, char* &out)
|
|
|
|
#include <iomanip>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
static void copyFromArray2(const char *arr, char*& chp, char* &out)
|
|
|
|
|
|
|
|
{
|
|
|
|
{
|
|
|
|
unsigned char ch = *(chp++);
|
|
|
|
unsigned char ch = *(chp++);
|
|
|
|
// Optimize for ASCII values
|
|
|
|
// Optimize for ASCII values
|
|
|
|
if(ch < 128)
|
|
|
|
if (ch < 128)
|
|
|
|
{
|
|
|
|
{
|
|
|
|
*(out++) = ch;
|
|
|
|
*(out++) = ch;
|
|
|
|
return;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
int len = 1;
|
|
|
|
int len = 1;
|
|
|
|
switch (ch)
|
|
|
|
switch (ch)
|
|
|
|
{
|
|
|
|
|
|
|
|
case 0xe2: len = 3; break;
|
|
|
|
|
|
|
|
case 0xc2:
|
|
|
|
|
|
|
|
case 0xcb:
|
|
|
|
|
|
|
|
case 0xc4:
|
|
|
|
|
|
|
|
case 0xc6:
|
|
|
|
|
|
|
|
case 0xc3:
|
|
|
|
|
|
|
|
case 0xd0:
|
|
|
|
|
|
|
|
case 0xd1:
|
|
|
|
|
|
|
|
case 0xd2:
|
|
|
|
|
|
|
|
case 0xc5: len = 2; break;
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if (len == 1) // There is no 1 length utf-8 glyph that is not 0x20 (empty space)
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
*(out++) = ch;
|
|
|
|
|
|
|
|
return;
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
unsigned char ch2 = *(chp++);
|
|
|
|
|
|
|
|
unsigned char ch3 = '\0';
|
|
|
|
|
|
|
|
if (len == 3)
|
|
|
|
|
|
|
|
ch3 = *(chp++);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
for (int i = 128; i < 256; i++)
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
unsigned char b1 = arr[i*6 + 1], b2 = arr[i*6 + 2], b3 = arr[i*6 + 3];
|
|
|
|
|
|
|
|
if (b1 == ch && b2 == ch2 && (len != 3 || b3 == ch3))
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
*(out++) = (char)i;
|
|
|
|
|
|
|
|
return;
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
std::cout << "Could not find glyph " << std::hex << (int)ch << " " << (int)ch2 << " " << (int)ch3 << std::endl;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
*(out++) = ch; // Could not find glyph, just put whatever
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
std::string ToUTF8::getLegacyEnc(ToUTF8::FromType to)
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
// Pick translation array
|
|
|
|
|
|
|
|
const char *arr;
|
|
|
|
|
|
|
|
switch (to)
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
case ToUTF8::WINDOWS_1252:
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
arr = ToUTF8::windows_1252;
|
|
|
|
|
|
|
|
break;
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
case ToUTF8::WINDOWS_1250:
|
|
|
|
|
|
|
|
{
|
|
|
|
{
|
|
|
|
arr = ToUTF8::windows_1250;
|
|
|
|
case 0xe2: len = 3; break;
|
|
|
|
break;
|
|
|
|
case 0xc2:
|
|
|
|
|
|
|
|
case 0xcb:
|
|
|
|
|
|
|
|
case 0xc4:
|
|
|
|
|
|
|
|
case 0xc6:
|
|
|
|
|
|
|
|
case 0xc3:
|
|
|
|
|
|
|
|
case 0xd0:
|
|
|
|
|
|
|
|
case 0xd1:
|
|
|
|
|
|
|
|
case 0xd2:
|
|
|
|
|
|
|
|
case 0xc5: len = 2; break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
case ToUTF8::WINDOWS_1251:
|
|
|
|
|
|
|
|
|
|
|
|
if (len == 1) // There is no 1 length utf-8 glyph that is not 0x20 (empty space)
|
|
|
|
{
|
|
|
|
{
|
|
|
|
arr = ToUTF8::windows_1251;
|
|
|
|
*(out++) = ch;
|
|
|
|
break;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
default:
|
|
|
|
|
|
|
|
|
|
|
|
unsigned char ch2 = *(chp++);
|
|
|
|
|
|
|
|
unsigned char ch3 = '\0';
|
|
|
|
|
|
|
|
if (len == 3)
|
|
|
|
|
|
|
|
ch3 = *(chp++);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
for (int i = 128; i < 256; i++)
|
|
|
|
{
|
|
|
|
{
|
|
|
|
assert(0);
|
|
|
|
unsigned char b1 = translationArray[i*6 + 1], b2 = translationArray[i*6 + 2], b3 = translationArray[i*6 + 3];
|
|
|
|
|
|
|
|
if (b1 == ch && b2 == ch2 && (len != 3 || b3 == ch3))
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
*(out++) = (char)i;
|
|
|
|
|
|
|
|
return;
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
std::cout << "Could not find glyph " << std::hex << (int)ch << " " << (int)ch2 << " " << (int)ch3 << std::endl;
|
|
|
|
// Double check that the input string stops at some point (it might
|
|
|
|
|
|
|
|
// contain zero terminators before this, inside its own data, which
|
|
|
|
*(out++) = ch; // Could not find glyph, just put whatever
|
|
|
|
// is also ok.)
|
|
|
|
|
|
|
|
char* input = &buf[0];
|
|
|
|
|
|
|
|
assert(input[size] == 0);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// TODO: The rest of this function is designed for single-character
|
|
|
|
|
|
|
|
// input encodings only. It also assumes that the input the input
|
|
|
|
|
|
|
|
// encoding shares its first 128 values (0-127) with ASCII. These
|
|
|
|
|
|
|
|
// conditions must be checked again if you add more input encodings
|
|
|
|
|
|
|
|
// later.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Compute output length, and check for pure ascii input at the same
|
|
|
|
|
|
|
|
// time.
|
|
|
|
|
|
|
|
bool ascii;
|
|
|
|
|
|
|
|
size_t outlen = getLength2(arr, input, ascii);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// If we're pure ascii, then don't bother converting anything.
|
|
|
|
|
|
|
|
if(ascii)
|
|
|
|
|
|
|
|
return std::string(input, outlen);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Make sure the output is large enough
|
|
|
|
|
|
|
|
resize(output, outlen);
|
|
|
|
|
|
|
|
char *out = &output[0];
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Translate
|
|
|
|
|
|
|
|
while(*input)
|
|
|
|
|
|
|
|
copyFromArray2(arr, input, out);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Make sure that we wrote the correct number of bytes
|
|
|
|
|
|
|
|
assert((out-&output[0]) == (int)outlen);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// And make extra sure the output is null terminated
|
|
|
|
|
|
|
|
assert(output.size() > outlen);
|
|
|
|
|
|
|
|
assert(output[outlen] == 0);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Return a string
|
|
|
|
|
|
|
|
return std::string(&output[0], outlen);
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
ToUTF8::FromType ToUTF8::calculateEncoding(const std::string& encodingName)
|
|
|
|
ToUTF8::FromType ToUTF8::calculateEncoding(const std::string& encodingName)
|
|
|
|
{
|
|
|
|
{
|
|
|
|
if (encodingName == "win1250")
|
|
|
|
if (encodingName == "win1250")
|
|
|
|
return ToUTF8::WINDOWS_1250;
|
|
|
|
return ToUTF8::WINDOWS_1250;
|
|
|
|
else if (encodingName == "win1251")
|
|
|
|
else if (encodingName == "win1251")
|
|
|
|
return ToUTF8::WINDOWS_1251;
|
|
|
|
return ToUTF8::WINDOWS_1251;
|
|
|
|
else
|
|
|
|
else
|
|
|
|
return ToUTF8::WINDOWS_1252;
|
|
|
|
return ToUTF8::WINDOWS_1252;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
std::string ToUTF8::encodingUsingMessage(const std::string& encodingName)
|
|
|
|
std::string ToUTF8::encodingUsingMessage(const std::string& encodingName)
|
|
|
|
{
|
|
|
|
{
|
|
|
|
if (encodingName == "win1250")
|
|
|
|
if (encodingName == "win1250")
|
|
|
|
return "Using Central and Eastern European font encoding.";
|
|
|
|
return "Using Central and Eastern European font encoding.";
|
|
|
|
else if (encodingName == "win1251")
|
|
|
|
else if (encodingName == "win1251")
|
|
|
|
return "Using Cyrillic font encoding.";
|
|
|
|
return "Using Cyrillic font encoding.";
|
|
|
|
else
|
|
|
|
else
|
|
|
|
return "Using default (English) font encoding.";
|
|
|
|
return "Using default (English) font encoding.";
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
Marc Zinnschlag
12 years ago