Can now save strings with proper encoding, byte-perfect clones up until land records

apps/esmtool/esmtool.cpp

@@ -282,7 +282,9 @@ int load(Arguments& info)
         while(esm.hasMoreRecs())
         {
             NAME n = esm.getRecName();
-            esm.getRecHeader();
+            uint32_t flags;
+            esm.getRecHeader(flags);
+
             string id = esm.getHNOString("NAME");
             if(!quiet)
                 cout << "\nRecord: " << n.toString()
@@ -726,6 +728,8 @@ int load(Arguments& info)
             {
                 rec->setId(id);
 
+                rec->setFlags((int)flags);
+
                 if (save)
                     info.data.records.push_back(rec);
                 else
@@ -803,6 +807,7 @@ int clone(Arguments& info)
     cout << endl << "Saving records to: " << info.outname << "..." << endl;
 
     ESMWriter esm;
+    esm.setEncoding(info.encoding);
     esm.setAuthor(info.data.author);
     esm.setDescription(info.data.description);
     esm.setVersion(info.data.version);
@@ -822,13 +827,22 @@ int clone(Arguments& info)
         NAME n;
         n.val = rec->getName();
 
-        esm.startRecord(n.toString(), 0);
+        esm.startRecord(n.toString(), rec->getFlags());
         string id = rec->getId();
 
-        if (n.val == REC_GLOB || n.val == REC_CLAS || n.val == REC_FACT || n.val == REC_RACE)
+        if (n.val == REC_GLOB || n.val == REC_CLAS || n.val == REC_FACT || n.val == REC_RACE || n.val == REC_SOUN 
+            || n.val == REC_REGN || n.val == REC_BSGN || n.val == REC_LTEX || n.val == REC_STAT || n.val == REC_DOOR
+            || n.val == REC_MISC || n.val == REC_WEAP || n.val == REC_CONT || n.val == REC_SPEL || n.val == REC_CREA
+            || n.val == REC_BODY || n.val == REC_LIGH || n.val == REC_ENCH || n.val == REC_NPC_ || n.val == REC_ARMO
+            || n.val == REC_CLOT || n.val == REC_REPA || n.val == REC_ACTI || n.val == REC_APPA || n.val == REC_LOCK
+            || n.val == REC_PROB || n.val == REC_INGR || n.val == REC_BOOK || n.val == REC_ALCH || n.val == REC_LEVI
+            || n.val == REC_LEVC)
             esm.writeHNCString("NAME", id);
+        else if (n.val == REC_CELL)
+            esm.writeHNString("NAME", id);
         else
             esm.writeHNOString("NAME", id);
+
         rec->save(esm);
 
         if (n.val == REC_CELL && !info.data.cellRefs[rec].empty())

components/esm/defs.hpp

@@ -51,7 +51,7 @@ struct SpellList
     void save(ESMWriter &esm)
     {
         for (std::vector<std::string>::iterator it = list.begin(); it != list.end(); ++it)
-            esm.writeHNString("NPCS", *it);
+            esm.writeHNString("NPCS", *it, 32);
     }
 };
 
@@ -83,6 +83,139 @@ struct ENAMstruct
 
     // Struct size should be 24 bytes
 };
+
+struct AIDTstruct
+{
+    // These are probabilities
+    char hello, u1, fight, flee, alarm, u2, u3, u4;
+    // The last u's might be the skills that this NPC can train you
+    // in?
+    int services; // See the Services enum
+};  // 12 bytes
+
+struct DODTstruct
+{
+    float pos[3];
+    float rot[3];
+};
+
+struct AI_Package
+{
+    void load(ESMReader& esm)
+    {
+        getData(esm);
+        cndt = esm.getHNOString("CNDT");
+    }
+
+    void save(ESMWriter& esm)
+    {
+        esm.startSubRecord(getName());
+        saveData(esm);
+        esm.endRecord(getName());
+
+        esm.writeHNOCString("CNDT", cndt);
+    }
+
+    std::string cndt;
+
+    virtual void getData(ESMReader&) = 0;
+    virtual void saveData(ESMWriter&) = 0;
+
+    virtual std::string getName() const = 0;
+    virtual int size() const = 0;
+};
+
+struct AI_Wstruct : AI_Package
+{
+    struct Data
+    {
+        short distance, duration;
+        char timeofday;
+        char idle[8];
+        char unknown;
+    };
+
+    Data data;
+
+    void getData(ESMReader& esm) { esm.getHExact(&data, sizeof(data)); }
+    void saveData(ESMWriter& esm) { esm.writeT(data); }
+
+    std::string getName() const { return "AI_W"; }
+    int size() const { return sizeof(AI_Wstruct); }
+};
+
+struct AI_Tstruct : AI_Package
+{
+    struct Data
+    {
+        float pos[3];
+        int unknown;
+    };
+
+    Data data;
+
+    void getData(ESMReader& esm) { esm.getHExact(&data, sizeof(data)); }
+    void saveData(ESMWriter& esm) { esm.writeT(data); }
+
+    std::string getName() const { return "AI_T"; }
+    int size() const { return sizeof(AI_Tstruct); }
+};
+
+struct AI_Fstruct : AI_Package
+{
+    struct Data
+    {
+        float pos[3];
+        short duration;
+        NAME32 id;
+        short unknown;
+    };
+
+    Data data;
+    
+    void getData(ESMReader& esm) { esm.getHExact(&data, sizeof(data)); }
+    void saveData(ESMWriter& esm) { esm.writeT(data); }
+
+    std::string getName() const { return "AI_F"; }
+    int size() const { return sizeof(AI_Fstruct); }
+};
+
+struct AI_Estruct : AI_Package
+{
+    struct Data
+    {
+        float pos[3];
+        short duration;
+        NAME32 id;
+        short unknown;
+    };
+
+    Data data;
+
+    void getData(ESMReader& esm) { esm.getHExact(&data, sizeof(data)); }
+    void saveData(ESMWriter& esm) { esm.writeT(data); }
+
+    std::string getName() const { return "AI_E"; }
+    int size() const { return sizeof(AI_Estruct); }
+};
+
+struct AI_Astruct : AI_Package
+{
+    struct Data
+    {
+        NAME32 name;
+        char unknown;
+    };
+
+    Data data;
+
+    void getData(ESMReader& esm) { esm.getHExact(&data, sizeof(data)); }
+    void saveData(ESMWriter& esm) { esm.writeT(data); }
+
+    std::string getName() const { return "AI_A"; }
+    int size() const { return sizeof(AI_Astruct); }
+};
+
 #pragma pack(pop)
 
 struct EffectList
@@ -107,5 +240,86 @@ struct EffectList
     }
 };
 
+struct AIData
+{
+    struct Travelstruct
+    {
+        DODTstruct dodt;
+        std::string dnam;
+    };
+
+    AIDTstruct aidt;
+    bool hasAI;
+    std::vector<AI_Package*> packages;
+    std::vector<Travelstruct> travel;
+    
+    void load(ESMReader &esm)
+    {
+        if (esm.isNextSub("AIDT"))
+        {
+            esm.getHExact(&aidt, sizeof(aidt));
+            hasAI = true;
+        }
+        else
+            hasAI = false;
+
+#define LOAD_IF_FOUND(x)                        \
+        if (esm.isNextSub(#x))                  \
+        {                                       \
+            found = true;                       \
+            x##struct *t = new x##struct();     \
+            t->load(esm);                       \
+            packages.push_back(t);              \
+        }
+        
+        bool found = true;
+        while (esm.hasMoreSubs() && found)
+        {
+            found = false;
+
+            if (esm.isNextSub("DODT"))
+            {
+                found = true;
+                Travelstruct t;
+                esm.getHExact(&t.dodt, sizeof(t.dodt));
+                t.dnam = esm.getHNOString("DNAM");
+                travel.push_back(t);
+            }
+
+            LOAD_IF_FOUND(AI_W);
+            LOAD_IF_FOUND(AI_T);
+            LOAD_IF_FOUND(AI_F);
+            LOAD_IF_FOUND(AI_E);
+            LOAD_IF_FOUND(AI_A);
+        }
+    }
+
+    void save(ESMWriter &esm)
+    {
+        if (hasAI)
+            esm.writeHNT("AIDT", aidt);
+
+        for (std::vector<Travelstruct>::iterator it = travel.begin(); it != travel.end(); ++it)   
+        {
+            esm.writeHNT("DODT", it->dodt);
+            esm.writeHNOCString("DNAM", it->dnam);
+        }
+
+        for (std::vector<AI_Package*>::iterator it = packages.begin(); it != packages.end(); ++it)
+        {
+            (*it)->save(esm);
+        }
+    }
+
+    ~AIData()
+    {
+        for (std::vector<AI_Package*>::iterator it = packages.begin(); it != packages.end();)
+        {
+            delete *it;
+            packages.erase(it++);
+        }
+    }
+};
+
 }
 #endif

components/esm/esm_writer.cpp

@@ -136,7 +136,7 @@ void ESMWriter::writeHNString(const std::string& name, const std::string& data)
     endRecord(name);
 }
 
-    void ESMWriter::writeHNString(const std::string& name, const std::string& data, int size)
+void ESMWriter::writeHNString(const std::string& name, const std::string& data, int size)
 {
     assert(data.size() <= size);
     startSubRecord(name);
@@ -156,7 +156,23 @@ void ESMWriter::writeHString(const std::string& data)
     if (data.size() == 0)
         write("\0", 1);
     else
-        write(data.c_str(), data.size());
+    {
+        char *ptr = ToUTF8::getBuffer(data.size()+1);
+        strncpy(ptr, &data[0], data.size());
+        ptr[data.size()] = '\0';
+
+        // Convert to UTF8 and return
+        std::string ascii = ToUTF8::getASCII(m_encoding);
+
+        write(ascii.c_str(), ascii.size());
+    }
+}
+
+void ESMWriter::writeHCString(const std::string& data)
+{
+    writeHString(data);
+    if (data.size() > 0 && data[data.size()-1] != '\0')
+        write("\0", 1);
 }
 
 void ESMWriter::writeName(const std::string& name)
@@ -176,4 +192,21 @@ void ESMWriter::write(const char* data, int size)
     m_stream->write(data, size);
 }
 
+void ESMWriter::setEncoding(const std::string& encoding)
+{
+    if (encoding == "win1250")
+    {
+        m_encoding = ToUTF8::WINDOWS_1250;
+    }
+    else if (encoding == "win1251")
+    {
+        m_encoding = ToUTF8::WINDOWS_1251;
+    }
+    else
+    {
+        // Default Latin encoding
+        m_encoding = ToUTF8::WINDOWS_1252;
+    }
+}
+
 }

components/esm/esm_writer.hpp

@@ -6,6 +6,7 @@
 #include <assert.h>
 
 #include "esm_common.hpp"
+#include "../to_utf8/to_utf8.hpp"
 
 namespace ESM {
 
@@ -23,7 +24,7 @@ public:
     void setVersion(int ver);
     int getType();
     void setType(int type);
-//    void setEncoding(const std::string& encoding); // Write strings as UTF-8?
+    void setEncoding(const std::string& encoding); // Write strings as UTF-8?
     void setAuthor(const std::string& author);
     void setDescription(const std::string& desc);
 
@@ -38,9 +39,7 @@ public:
     void writeHNCString(const std::string& name, const std::string& data)
     {
         startSubRecord(name);
-        writeHString(data);
+        writeHCString(data);
-        if (data.size() > 0 && data[data.size()-1] != '\0')
-            write("\0", 1);
         endRecord(name);
     }
     void writeHNOString(const std::string& name, const std::string& data)
@@ -48,6 +47,11 @@ public:
         if (!data.empty())
             writeHNString(name, data);
     }
+    void writeHNOCString(const std::string& name, const std::string& data)
+    {
+        if (!data.empty())
+            writeHNCString(name, data);
+    }
 
     template<typename T>
     void writeHNT(const std::string& name, const T& data)
@@ -81,6 +85,7 @@ public:
     void startSubRecord(const std::string& name);
     void endRecord(const std::string& name);
     void writeHString(const std::string& data);
+    void writeHCString(const std::string& data);
     void writeName(const std::string& data);
     void write(const char* data, int size);
 
@@ -89,6 +94,7 @@ private:
     std::list<RecordData> m_records;
     std::ostream* m_stream;
     std::streampos m_headerPos;
+    ToUTF8::FromType m_encoding;
     int m_recordCount;
 
     HEDRstruct m_header;

components/esm/loadacti.cpp

@@ -10,8 +10,8 @@ void Activator::load(ESMReader &esm)
 }
 void Activator::save(ESMWriter &esm)
 {
-    esm.writeHNString("MODL", model);
+    esm.writeHNCString("MODL", model);
-    esm.writeHNString("FNAM", name);
+    esm.writeHNCString("FNAM", name);
-    esm.writeHNOString("SCRI", script);
+    esm.writeHNOCString("SCRI", script);
 }
 }

components/esm/loadalch.cpp

@@ -13,10 +13,10 @@ void Potion::load(ESMReader &esm)
 }
 void Potion::save(ESMWriter &esm)
 {
-    esm.writeHNString("MODL", model);
+    esm.writeHNCString("MODL", model);
-    esm.writeHNOString("TEXT", icon);
+    esm.writeHNOCString("TEXT", icon);
-    esm.writeHNOString("SCRI", script);
+    esm.writeHNOCString("SCRI", script);
-    esm.writeHNOString("FNAM", name);
+    esm.writeHNOCString("FNAM", name);
     esm.writeHNT("ALDT", data, 12);
     effects.save(esm);
 }

components/esm/loadappa.cpp

@@ -12,10 +12,10 @@ void Apparatus::load(ESMReader &esm)
 }
 void Apparatus::save(ESMWriter &esm)
 {
-    esm.writeHNString("MODL", model);
+    esm.writeHNCString("MODL", model);
-    esm.writeHNString("FNAM", name);
+    esm.writeHNCString("FNAM", name);
     esm.writeHNT("AADT", data, 16);
-    esm.writeHNOString("SCRI", script);
+    esm.writeHNOCString("SCRI", script);
-    esm.writeHNString("ITEX", icon);
+    esm.writeHNCString("ITEX", icon);
 }
 }

components/esm/loadarmo.cpp

@@ -38,13 +38,13 @@ void Armor::load(ESMReader &esm)
 
 void Armor::save(ESMWriter &esm)
 {
-    esm.writeHNString("MODL", model);
+    esm.writeHNCString("MODL", model);
-    esm.writeHNString("FNAM", name);
+    esm.writeHNCString("FNAM", name);
-    esm.writeHNOString("SCRI", script);
+    esm.writeHNOCString("SCRI", script);
     esm.writeHNT("AODT", data, 24);
-    esm.writeHNOString("ITEX", icon);
+    esm.writeHNOCString("ITEX", icon);
     parts.save(esm);
-    esm.writeHNOString("ENAM", enchant);
+    esm.writeHNOCString("ENAM", enchant);
 }
 
 }

components/esm/loadbody.cpp

@@ -11,8 +11,8 @@ void BodyPart::load(ESMReader &esm)
 }
 void BodyPart::save(ESMWriter &esm)
 {
-    esm.writeHNString("MODL", model);
+    esm.writeHNCString("MODL", model);
-    esm.writeHNString("FNAM", name);
+    esm.writeHNCString("FNAM", name);
     esm.writeHNT("BYDT", data, 4);
 }
 

components/esm/loadbook.cpp

@@ -15,13 +15,13 @@ void Book::load(ESMReader &esm)
 }
 void Book::save(ESMWriter &esm)
 {
-    esm.writeHNString("MODL", model);
+    esm.writeHNCString("MODL", model);
-    esm.writeHNOString("FNAM", name);
+    esm.writeHNOCString("FNAM", name);
     esm.writeHNT("BKDT", data, 20);
-    esm.writeHNOString("SCRI", script);
+    esm.writeHNOCString("SCRI", script);
-    esm.writeHNOString("ITEX", icon);
+    esm.writeHNOCString("ITEX", icon);
     esm.writeHNOString("TEXT", text);
-    esm.writeHNOString("ENAM", enchant);
+    esm.writeHNOCString("ENAM", enchant);
 }
 
 }

components/esm/loadbsgn.cpp

@@ -13,9 +13,9 @@ void BirthSign::load(ESMReader &esm)
 }
 void BirthSign::save(ESMWriter &esm)
 {
-    esm.writeHNString("FNAM", name);
+    esm.writeHNCString("FNAM", name);
-    esm.writeHNOString("TNAM", texture);
+    esm.writeHNOCString("TNAM", texture);
-    esm.writeHNOString("DESC", description);
+    esm.writeHNOCString("DESC", description);
 
     powers.save(esm);
 }

components/esm/loadcell.cpp

@@ -108,7 +108,7 @@ void Cell::save(ESMWriter &esm)
     }
     else
     {
-        esm.writeHNOString("RGNN", region);
+        esm.writeHNOCString("RGNN", region);
         if (mapColor != 0)
             esm.writeHNT("NAM5", mapColor);
     }

components/esm/loadclot.cpp

@@ -18,16 +18,16 @@ void Clothing::load(ESMReader &esm)
 }
 void Clothing::save(ESMWriter &esm)
 {
-    esm.writeHNString("MODL", model);
+    esm.writeHNCString("MODL", model);
-    esm.writeHNOString("FNAM", name);
+    esm.writeHNOCString("FNAM", name);
     esm.writeHNT("CTDT", data, 12);
 
-    esm.writeHNOString("SCRI", script);
+    esm.writeHNOCString("SCRI", script);
-    esm.writeHNOString("ITEX", icon);
+    esm.writeHNOCString("ITEX", icon);
     
     parts.save(esm);
     
-    esm.writeHNOString("ENAM", enchant);
+    esm.writeHNOCString("ENAM", enchant);
 }
 
 }

components/esm/loadcont.cpp

@@ -40,12 +40,12 @@ void Container::load(ESMReader &esm)
 
 void Container::save(ESMWriter &esm)
 {
-    esm.writeHNString("MODL", model);
+    esm.writeHNCString("MODL", model);
-    esm.writeHNOString("FNAM", name);
+    esm.writeHNOCString("FNAM", name);
     esm.writeHNT("CNDT", weight, 4);
     esm.writeHNT("FLAG", flags, 4);
 
-    esm.writeHNOString("SCRI", script);
+    esm.writeHNOCString("SCRI", script);
 
     inventory.save(esm);
 }

components/esm/loadcrea.cpp

@@ -16,6 +16,8 @@ void Creature::load(ESMReader &esm)
     esm.getHNOT(scale, "XSCL");
 
     inventory.load(esm);
+    spells.load(esm);
+    aiData.load(esm);
 
     // More subrecords:
 
@@ -37,15 +39,18 @@ void Creature::load(ESMReader &esm)
 
 void Creature::save(ESMWriter &esm)
 {
-    esm.writeHNString("MODL", model);
+    esm.writeHNCString("MODL", model);
-    esm.writeHNOString("CNAM", original);
+    esm.writeHNOCString("CNAM", original);
-    esm.writeHNOString("FNAM", name);
+    esm.writeHNOCString("FNAM", name);
-    esm.writeHNOString("SCRI", script);
+    esm.writeHNOCString("SCRI", script);
     esm.writeHNT("NPDT", data, 96);
     esm.writeHNT("FLAG", flags);
     if (scale != 1.0)
         esm.writeHNT("XSCL", scale);
+
     inventory.save(esm);
+    spells.save(esm);
+    aiData.save(esm);
 }
 
 }

components/esm/loadcrea.hpp

@@ -5,6 +5,7 @@
 #include "esm_reader.hpp"
 #include "esm_writer.hpp"
 #include "loadcont.hpp"
+#include "defs.hpp"
 
 namespace ESM
 {
@@ -54,6 +55,7 @@ struct Creature : public Record
     }; // 96 bytes
 
     NPDTstruct data;
+    AIData aiData;
 
     int flags;
     float scale;
@@ -62,6 +64,7 @@ struct Creature : public Record
 
     // Defined in loadcont.hpp
     InventoryList inventory;
+    SpellList spells;
 
     void load(ESMReader &esm);
     void save(ESMWriter &esm);

components/esm/loaddoor.cpp

@@ -13,11 +13,11 @@ void Door::load(ESMReader &esm)
 }
 void Door::save(ESMWriter &esm)
 {
-    esm.writeHNString("MODL", model);
+    esm.writeHNCString("MODL", model);
-    esm.writeHNOString("FNAM", name);
+    esm.writeHNOCString("FNAM", name);
-    esm.writeHNOString("SCRI", script);
+    esm.writeHNOCString("SCRI", script);
-    esm.writeHNOString("SNAM", openSound);
+    esm.writeHNOCString("SNAM", openSound);
-    esm.writeHNOString("ANAM", closeSound);
+    esm.writeHNOCString("ANAM", closeSound);
 }
 
 }

components/esm/loadingr.cpp

@@ -13,11 +13,11 @@ void Ingredient::load(ESMReader &esm)
 }
 void Ingredient::save(ESMWriter &esm)
 {
-    esm.writeHNString("MODL", model);
+    esm.writeHNCString("MODL", model);
-    esm.writeHNString("FNAM", name);
+    esm.writeHNCString("FNAM", name);
     esm.writeHNT("IRDT", data, 56);
-    esm.writeHNOString("SCRI", script);
+    esm.writeHNOCString("SCRI", script);
-    esm.writeHNOString("ITEX", script);
+    esm.writeHNOCString("ITEX", icon);
 }
 
 }

components/esm/loadlevlist.cpp

@@ -38,7 +38,7 @@ void LeveledListBase::save(ESMWriter &esm)
     
     for (std::vector<LevelItem>::iterator it = list.begin(); it != list.end(); ++it)
     {
-        esm.writeHNString(recName, it->id);
+        esm.writeHNCString(recName, it->id);
         esm.writeHNT("INTV", it->level);
     }
 }

components/esm/loadligh.cpp

@@ -15,12 +15,12 @@ void Light::load(ESMReader &esm)
 }
 void Light::save(ESMWriter &esm)
 {
-    esm.writeHNString("MODL", model);
+    esm.writeHNCString("MODL", model);
-    esm.writeHNOString("FNAM", name);
+    esm.writeHNOCString("FNAM", name);
-    esm.writeHNOString("ITEX", icon);
+    esm.writeHNOCString("ITEX", icon);
     esm.writeHNT("LHDT", data, 24);
-    esm.writeHNOString("SCRI", script);
+    esm.writeHNOCString("SCRI", script);
-    esm.writeHNOString("SNAM", sound);
+    esm.writeHNOCString("SNAM", sound);
 }
 
 }

components/esm/loadlocks.cpp

@@ -33,8 +33,8 @@ void Tool::load(ESMReader &esm)
 }
 void Tool::save(ESMWriter &esm)
 {
-    esm.writeHNString("MODL", model);
+    esm.writeHNCString("MODL", model);
-    esm.writeHNString("FNAM", name);
+    esm.writeHNCString("FNAM", name);
     
     std::string typeName;
     switch(type)
@@ -44,9 +44,17 @@ void Tool::save(ESMWriter &esm)
     case Type_Probe: typeName = "PBDT"; break;
     }
     
-    esm.writeHNT(typeName, data, 16);
+    Data write = data;
+    if (type == Type_Repair)
+    {
+        float tmp = *((float*) &write.uses);
+        write.uses = *((int*) &write.quality);
+        write.quality = tmp;
+    }
+
+    esm.writeHNT(typeName, write, 16);
     esm.writeHNOString("SCRI", script);
-    esm.writeHNOString("ITEX", icon);
+    esm.writeHNOCString("ITEX", icon);
 }
 
 

components/esm/loadltex.cpp

@@ -11,7 +11,7 @@ void LandTexture::load(ESMReader &esm)
 void LandTexture::save(ESMWriter &esm)
 {
     esm.writeHNT("INTV", index);
-    esm.writeHNString("DATA", texture);
+    esm.writeHNCString("DATA", texture);
 }
 
 }

components/esm/loadmgef.cpp

@@ -28,17 +28,17 @@ void MagicEffect::save(ESMWriter &esm)
     esm.writeHNT("INDX", index);
     esm.writeHNT("MEDT", data, 36);
 
-    esm.writeHNOString("ITEX", icon);
+    esm.writeHNOCString("ITEX", icon);
-    esm.writeHNOString("PTEX", particle);
+    esm.writeHNOCString("PTEX", particle);
-    esm.writeHNOString("BSND", boltSound);
+    esm.writeHNOCString("BSND", boltSound);
-    esm.writeHNOString("CSND", castSound);
+    esm.writeHNOCString("CSND", castSound);
-    esm.writeHNOString("HSND", hitSound);
+    esm.writeHNOCString("HSND", hitSound);
-    esm.writeHNOString("ASND", areaSound);
+    esm.writeHNOCString("ASND", areaSound);
     
-    esm.writeHNOString("CVFX", casting);
+    esm.writeHNOCString("CVFX", casting);
-    esm.writeHNOString("BVFX", bolt);
+    esm.writeHNOCString("BVFX", bolt);
-    esm.writeHNOString("HVFX", hit);
+    esm.writeHNOCString("HVFX", hit);
-    esm.writeHNOString("AVFX", area);
+    esm.writeHNOCString("AVFX", area);
     
     esm.writeHNOString("DESC", description);
 }

components/esm/loadmisc.cpp

@@ -13,11 +13,11 @@ void Miscellaneous::load(ESMReader &esm)
 }
 void Miscellaneous::save(ESMWriter &esm)
 {
-    esm.writeHNString("MODL", model);
+    esm.writeHNCString("MODL", model);
-    esm.writeHNOString("FNAM", name);
+    esm.writeHNOCString("FNAM", name);
     esm.writeHNT("MCDT", data, 12);
-    esm.writeHNOString("SCRI", script);
+    esm.writeHNOCString("SCRI", script);
-    esm.writeHNOString("ITEX", icon);
+    esm.writeHNOCString("ITEX", icon);
 }
 
 }

components/esm/loadnpc.cpp

@@ -37,27 +37,20 @@ void NPC::load(ESMReader &esm)
 
     inventory.load(esm);
     spells.load(esm);
-
+    aiData.load(esm);
-    if (esm.isNextSub("AIDT"))
-    {
-        esm.getHExact(&AI, sizeof(AI));
-        hasAI = true;
-    }
-    else
-        hasAI = false;
 
     esm.skipRecord();
 }
 void NPC::save(ESMWriter &esm)
 {
-    esm.writeHNOString("MODL", model);
+    esm.writeHNOCString("MODL", model);
-    esm.writeHNOString("FNAM", name);
+    esm.writeHNOCString("FNAM", name);
-    esm.writeHNString("RNAM", race);
+    esm.writeHNCString("RNAM", race);
-    esm.writeHNString("CNAM", cls);
+    esm.writeHNCString("CNAM", cls);
-    esm.writeHNString("ANAM", faction);
+    esm.writeHNCString("ANAM", faction);
-    esm.writeHNString("BNAM", head);
+    esm.writeHNCString("BNAM", head);
-    esm.writeHNString("KNAM", hair);
+    esm.writeHNCString("KNAM", hair);
-    esm.writeHNOString("SCRI", script);
+    esm.writeHNOCString("SCRI", script);
     
     if (npdtType == 52)
         esm.writeHNT("NPDT", npdt52, 52);
@@ -68,9 +61,7 @@ void NPC::save(ESMWriter &esm)
     
     inventory.save(esm);
     spells.save(esm);
-    
+    aiData.save(esm);
-    if (hasAI)
-        esm.writeHNT("AIDT", AI);
 }
 
 }

components/esm/loadnpc.hpp

@@ -74,15 +74,6 @@ struct NPC : public Record
     int gold; // ?? not certain
   }; // 12 bytes
 
-  struct AIDTstruct
-  {
-    // These are probabilities
-    char hello, u1, fight, flee, alarm, u2, u3, u4;
-    // The last u's might be the skills that this NPC can train you
-    // in?
-    int services; // See the Services enum
-  }; // 12 bytes
-
 #pragma pack(pop)
 
   char npdtType;
@@ -93,12 +84,10 @@ struct NPC : public Record
 
   InventoryList inventory;
   SpellList spells;
-
+    AIData aiData;
-  AIDTstruct AI;
-  bool hasAI;
 
   std::string name, model, race, cls, faction, script,
-    hair, head; // body parts
+      hair, head; // body parts
 
     //std::string mId;
 

components/esm/loadregn.cpp

@@ -27,14 +27,14 @@ void Region::load(ESMReader &esm)
 }
 void Region::save(ESMWriter &esm)
 {
-    esm.writeHNString("FNAM", name);
+    esm.writeHNCString("FNAM", name);
     
     if (esm.getVersion() == VER_12)
         esm.writeHNT("WEAT", data, sizeof(data) - 2);
     else
         esm.writeHNT("WEAT", data);
     
-    esm.writeHNOString("BNAM", sleepList);
+    esm.writeHNOCString("BNAM", sleepList);
     
     esm.writeHNT("CNAM", mapColor);
     for (std::vector<SoundRef>::iterator it = soundList.begin(); it != soundList.end(); ++it)

components/esm/loadscpt.cpp

@@ -42,21 +42,25 @@ void Script::save(ESMWriter &esm)
 {
     std::string varNameString;
     if (!varNames.empty())
-    {
         for (std::vector<std::string>::iterator it = varNames.begin(); it != varNames.end(); ++it)
-        {
             varNameString.append(*it);
-            //varNameString.append("\0");
-        }
-
-        data.stringTableSize = varNameString.size();
-    }
 
     esm.writeHNT("SCHD", data, 52);
     
-    esm.writeHNOString("SCVR", varNameString);
+    if (!varNames.empty())
+    {
+        esm.startSubRecord("SCVR");
+        for (std::vector<std::string>::iterator it = varNames.begin(); it != varNames.end(); ++it)
+        {
+            esm.writeHCString(*it);
+        }
+        esm.endRecord("SCVR");
+    }
+
+    esm.startSubRecord("SCDT");
+    esm.write(&scriptData[0], data.scriptDataSize);
+    esm.endRecord("SCDT");
 
-    esm.writeHNString("SCDT", std::string(&scriptData[0], scriptData.size()));
     esm.writeHNOString("SCTX", scriptText);
 }
 

components/esm/loadsoun.cpp

@@ -16,7 +16,7 @@ void Sound::load(ESMReader &esm)
 }
 void Sound::save(ESMWriter &esm)
 {
-    esm.writeHNString("FNAM", sound);
+    esm.writeHNCString("FNAM", sound);
     esm.writeHNT("DATA", data, 3);
 }
 

components/esm/loadspel.cpp

@@ -11,7 +11,7 @@ void Spell::load(ESMReader &esm)
 }
 void Spell::save(ESMWriter &esm)
 {
-    esm.writeHNOString("FNAM", name);
+    esm.writeHNOCString("FNAM", name);
     esm.writeHNT("SPDT", data, 12);
     effects.save(esm);
 }

components/esm/loadstat.cpp

@@ -9,7 +9,7 @@ void Static::load(ESMReader &esm)
 }
 void Static::save(ESMWriter &esm)
 {
-    esm.writeHNString("MODL", model);
+    esm.writeHNCString("MODL", model);
 }
 
 }

components/esm/loadweap.cpp

@@ -14,12 +14,12 @@ void Weapon::load(ESMReader &esm)
 }
 void Weapon::save(ESMWriter &esm)
 {
-    esm.writeHNString("MODL", model);
+    esm.writeHNCString("MODL", model);
-    esm.writeHNOString("FNAM", name);
+    esm.writeHNOCString("FNAM", name);
     esm.writeHNT("WPDT", data, 32);
-    esm.writeHNOString("SCRI", script);
+    esm.writeHNOCString("SCRI", script);
-    esm.writeHNOString("ITEX", icon);
+    esm.writeHNOCString("ITEX", icon);
-    esm.writeHNOString("ENAM", enchant);
+    esm.writeHNOCString("ENAM", enchant);
 }
 
 }

components/esm/record.hpp

@@ -67,10 +67,14 @@ public:
     std::string getId() const { return m_id; }
     void setId(const std::string& in) { m_id = in; } 
     
+    int getFlags() const { return (m_flags & 0x1 ? 0x00002000 : 0) | (m_flags & 0x2 ? 0x00000400 : 0); }
+    void setFlags(int in) { m_flags = (in & 0x00002000 ? 0x1 : 0) | (in & 0x00000400 ? 0x2 : 0); }
+
     virtual int getName() = 0;
 
 protected:
     std::string m_id;
+    char m_flags;
 };
 
 }

components/to_utf8/to_utf8.cpp

@@ -193,3 +193,167 @@ std::string ToUTF8::getUtf8(ToUTF8::FromType from)
   return std::string(&output[0], outlen);
 }
 
+static size_t getLength2(const char *arr, const char* input, bool &ascii)
+{
+  ascii = true;
+  size_t len = 0;
+  const char* ptr = input;
+  unsigned char inp = *ptr;
+
+  // Do away with the ascii part of the string first (this is almost
+  // always the entire string.)
+  while(inp && inp < 128)
+    inp = *(++ptr);
+  len += (ptr-input);
+
+  // If we're not at the null terminator at this point, then there
+  // were some non-ascii characters to deal with. Go to slow-mode for
+  // the rest of the string.
+  if(inp)
+    {
+      ascii = false;
+      while(inp)
+        {
+            len += 1;
+          // Find the translated length of this character in the
+          // lookup table.
+            switch(inp)
+            {
+            case 0xe2: len -= 2; break;
+            case 0xc2:
+            case 0xcb:
+            case 0xc4:
+            case 0xc6:
+            case 0xc3:
+            case 0xd0:
+            case 0xd1:
+            case 0xd2:
+            case 0xc5: len -= 1; break;
+            }
+
+          inp = *(++ptr);
+        }
+    }
+  return len;
+}
+
+#include <iostream>
+#include <iomanip>
+
+static void copyFromArray2(const char *arr, char*& chp, char* &out)
+{
+    unsigned char ch = *(chp++);
+  // Optimize for ASCII values
+  if(ch < 128)
+    {
+      *(out++) = ch;
+      return;
+    }
+
+  int len = 1;
+  switch (ch)
+  {
+  case 0xe2: len = 3; break;
+  case 0xc2:
+  case 0xcb:
+  case 0xc4:
+  case 0xc6:
+  case 0xc3:
+  case 0xd0:
+  case 0xd1:
+  case 0xd2:
+  case 0xc5: len = 2; break;
+  }
+
+  if (len == 1) // There is no 1 length utf-8 glyph that is not 0x20 (empty space)
+  {
+      *(out++) = ch;
+      return;
+  }
+
+  unsigned char ch2 = *(chp++);
+  unsigned char ch3 = '\0';
+  if (len == 3)
+      ch3 = *(chp++);
+
+  for (int i = 128; i < 256; i++)
+  {
+      unsigned char b1 = arr[i*6 + 1], b2 = arr[i*6 + 2], b3 = arr[i*6 + 3];
+      if (b1 == ch && b2 == ch2 && (len != 3 || b3 == ch3))
+      {
+          *(out++) = (char)i;
+          return;
+      }
+  }
+
+  std::cout << "Could not find glyph " << std::hex << (int)ch << " " << (int)ch2 << " " << (int)ch3 << std::endl;
+
+  *(out++) = ch; // Could not find glyph, just put whatever
+}
+
+std::string ToUTF8::getASCII(ToUTF8::FromType to)
+{
+  // Pick translation array
+  const char *arr;
+  switch (to)
+  {
+    case ToUTF8::WINDOWS_1252:
+    {
+      arr = ToUTF8::windows_1252;
+      break;
+    }
+    case ToUTF8::WINDOWS_1250:
+    {
+      arr = ToUTF8::windows_1250;
+      break;
+    }
+    case ToUTF8::WINDOWS_1251:
+    {
+      arr = ToUTF8::windows_1251;
+      break;
+    }
+    default:
+    {
+      assert(0);
+    }
+  }
+
+  // Double check that the input string stops at some point (it might
+  // contain zero terminators before this, inside its own data, which
+  // is also ok.)
+  char* input = &buf[0];
+  assert(input[size] == 0);
+
+  // TODO: The rest of this function is designed for single-character
+  // input encodings only. It also assumes that the input the input
+  // encoding shares its first 128 values (0-127) with ASCII. These
+  // conditions must be checked again if you add more input encodings
+  // later.
+
+  // Compute output length, and check for pure ascii input at the same
+  // time.
+  bool ascii;
+  size_t outlen = getLength2(arr, input, ascii);
+
+  // If we're pure ascii, then don't bother converting anything.
+  if(ascii)
+      return std::string(input, outlen);
+
+  // Make sure the output is large enough
+  resize(output, outlen);
+  char *out = &output[0];
+
+  // Translate
+  while(*input)
+    copyFromArray2(arr, input, out);
+
+  // Make sure that we wrote the correct number of bytes
+  assert((out-&output[0]) == (int)outlen);
+
+  // And make extra sure the output is null terminated
+  assert(output.size() > outlen);
+  assert(output[outlen] == 0);
+
+  // Return a string
+  return std::string(&output[0], outlen);
+}

components/to_utf8/to_utf8.hpp

@@ -21,6 +21,7 @@ namespace ToUTF8
   // Convert the previously written buffer to UTF8 from the given code
   // page.
   std::string getUtf8(FromType from);
+  std::string getASCII(FromType to);
 }
 
 #endif