mirror/openmw
1
0
Fork 0
mirror of https://github.com/OpenMW/openmw.git synced 2025-03-03 20:39:40 +00:00
Code Issues Releases Wiki Activity

Add components/lua/serilalization

Browse source
This commit is contained in:
Petr Mikheev 2021-01-10 14:41:50 +01:00
parent 4b068b27ca
commit 8dbaf6022c
5 changed files with 498 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
apps/openmw_test_suite/CMakeLists.txt
View file

@ -17,6 +17,7 @@ if (GTEST_FOUND AND GMOCK_FOUND)
lua/test_lua.cpp lua/test_lua.cpp
lua/test_utilpackage.cpp lua/test_utilpackage.cpp
lua/test_serialization.cpp
misc/test_stringops.cpp misc/test_stringops.cpp
misc/test_endianness.cpp misc/test_endianness.cpp

207
apps/openmw_test_suite/lua/test_serialization.cpp Normal file
View file

@ -0,0 +1,207 @@
#include "gmock/gmock.h"
#include <gtest/gtest.h>
#include <osg/Vec2f>
#include <osg/Vec3f>
#include <components/lua/serialization.hpp>
#include <components/misc/endianness.hpp>
#include "testing_util.hpp"
namespace
{
using namespace testing;
TEST(LuaSerializationTest, Nil)
{
sol::state lua;
EXPECT_EQ(LuaUtil::serialize(sol::nil), "");
EXPECT_EQ(LuaUtil::deserialize(lua, ""), sol::nil);
}
TEST(LuaSerializationTest, Number)
{
sol::state lua;
std::string serialized = LuaUtil::serialize(sol::make_object<double>(lua, 3.14));
EXPECT_EQ(serialized.size(), 10); // version, type, 8 bytes value
sol::object value = LuaUtil::deserialize(lua, serialized);
ASSERT_TRUE(value.is<double>());
EXPECT_FLOAT_EQ(value.as<double>(), 3.14);
}
TEST(LuaSerializationTest, Boolean)
{
sol::state lua;
{
std::string serialized = LuaUtil::serialize(sol::make_object<bool>(lua, true));
EXPECT_EQ(serialized.size(), 3); // version, type, 1 byte value
sol::object value = LuaUtil::deserialize(lua, serialized);
EXPECT_FALSE(value.is<double>());
ASSERT_TRUE(value.is<bool>());
EXPECT_TRUE(value.as<bool>());
}
{
std::string serialized = LuaUtil::serialize(sol::make_object<bool>(lua, false));
EXPECT_EQ(serialized.size(), 3); // version, type, 1 byte value
sol::object value = LuaUtil::deserialize(lua, serialized);
EXPECT_FALSE(value.is<double>());
ASSERT_TRUE(value.is<bool>());
EXPECT_FALSE(value.as<bool>());
}
}
TEST(LuaSerializationTest, String)
{
sol::state lua;
std::string_view emptyString = "";
std::string_view shortString = "abc";
std::string_view longString = "It is a string with more than 32 characters...........................";
{
std::string serialized = LuaUtil::serialize(sol::make_object<std::string_view>(lua, emptyString));
EXPECT_EQ(serialized.size(), 2); // version, type
sol::object value = LuaUtil::deserialize(lua, serialized);
ASSERT_TRUE(value.is<std::string>());
EXPECT_EQ(value.as<std::string>(), emptyString);
}
{
std::string serialized = LuaUtil::serialize(sol::make_object<std::string_view>(lua, shortString));
EXPECT_EQ(serialized.size(), 2 + shortString.size()); // version, type, str data
sol::object value = LuaUtil::deserialize(lua, serialized);
ASSERT_TRUE(value.is<std::string>());
EXPECT_EQ(value.as<std::string>(), shortString);
}
{
std::string serialized = LuaUtil::serialize(sol::make_object<std::string_view>(lua, longString));
EXPECT_EQ(serialized.size(), 6 + longString.size()); // version, type, size, str data
sol::object value = LuaUtil::deserialize(lua, serialized);
ASSERT_TRUE(value.is<std::string>());
EXPECT_EQ(value.as<std::string>(), longString);
}
}
TEST(LuaSerializationTest, Vector)
{
sol::state lua;
osg::Vec2f vec2(1, 2);
osg::Vec3f vec3(1, 2, 3);
{
std::string serialized = LuaUtil::serialize(sol::make_object(lua, vec2));
EXPECT_EQ(serialized.size(), 10); // version, type, 2x float
sol::object value = LuaUtil::deserialize(lua, serialized);
ASSERT_TRUE(value.is<osg::Vec2f>());
EXPECT_EQ(value.as<osg::Vec2f>(), vec2);
}
{
std::string serialized = LuaUtil::serialize(sol::make_object(lua, vec3));
EXPECT_EQ(serialized.size(), 14); // version, type, 3x float
sol::object value = LuaUtil::deserialize(lua, serialized);
ASSERT_TRUE(value.is<osg::Vec3f>());
EXPECT_EQ(value.as<osg::Vec3f>(), vec3);
}
}
TEST(LuaSerializationTest, Table)
{
sol::state lua;
sol::table table(lua, sol::create);
table["aa"] = 1;
table["ab"] = true;
table["nested"] = sol::table(lua, sol::create);
table["nested"]["aa"] = 2;
table["nested"]["bb"] = "something";
table["nested"][5] = -0.5;
table["nested_empty"] = sol::table(lua, sol::create);
table[1] = osg::Vec2f(1, 2);
table[2] = osg::Vec2f(2, 1);
std::string serialized = LuaUtil::serialize(table);
EXPECT_EQ(serialized.size(), 123);
sol::table res_table = LuaUtil::deserialize(lua, serialized);
EXPECT_EQ(res_table.get<int>("aa"), 1);
EXPECT_EQ(res_table.get<bool>("ab"), true);
EXPECT_EQ(res_table.get<sol::table>("nested").get<int>("aa"), 2);
EXPECT_EQ(res_table.get<sol::table>("nested").get<std::string>("bb"), "something");
EXPECT_FLOAT_EQ(res_table.get<sol::table>("nested").get<double>(5), -0.5);
EXPECT_EQ(res_table.get<osg::Vec2f>(1), osg::Vec2f(1, 2));
EXPECT_EQ(res_table.get<osg::Vec2f>(2), osg::Vec2f(2, 1));
}
struct TestStruct1 { double a, b; };
struct TestStruct2 { int a, b; };
class TestSerializer final : public LuaUtil::UserdataSerializer
{
bool serialize(LuaUtil::BinaryData& out, const sol::userdata& data) const override
{
if (data.is<TestStruct1>())
{
TestStruct1 t = data.as<TestStruct1>();
t.a = Misc::toLittleEndian(t.a);
t.b = Misc::toLittleEndian(t.b);
append(out, "ts1", &t, sizeof(t));
return true;
}
if (data.is<TestStruct2>())
{
TestStruct2 t = data.as<TestStruct2>();
t.a = Misc::toLittleEndian(t.a);
t.b = Misc::toLittleEndian(t.b);
append(out, "test_struct2", &t, sizeof(t));
return true;
}
return false;
}
bool deserialize(std::string_view typeName, std::string_view binaryData, sol::state& lua) const override
{
if (typeName == "ts1")
{
if (sizeof(TestStruct1) != binaryData.size())
throw std::runtime_error("Incorrect binaryData.size() for TestStruct1: " + std::to_string(binaryData.size()));
TestStruct1 t = *reinterpret_cast<const TestStruct1*>(binaryData.data());
t.a = Misc::fromLittleEndian(t.a);
t.b = Misc::fromLittleEndian(t.b);
sol::stack::push<TestStruct1>(lua, t);
return true;
}
if (typeName == "test_struct2")
{
if (sizeof(TestStruct2) != binaryData.size())
throw std::runtime_error("Incorrect binaryData.size() for TestStruct2: " + std::to_string(binaryData.size()));
TestStruct2 t = *reinterpret_cast<const TestStruct2*>(binaryData.data());
t.a = Misc::fromLittleEndian(t.a);
t.b = Misc::fromLittleEndian(t.b);
sol::stack::push<TestStruct2>(lua, t);
return true;
}
return false;
}
};
TEST(LuaSerializationTest, UserdataSerializer)
{
sol::state lua;
sol::table table(lua, sol::create);
table["x"] = TestStruct1{1.5, 2.5};
table["y"] = TestStruct2{4, 3};
TestSerializer serializer;
EXPECT_ERROR(LuaUtil::serialize(table), "Unknown userdata");
std::string serialized = LuaUtil::serialize(table, &serializer);
EXPECT_ERROR(LuaUtil::deserialize(lua, serialized), "Unknown type:");
sol::table res = LuaUtil::deserialize(lua, serialized, &serializer);
TestStruct1 rx = res.get<TestStruct1>("x");
TestStruct2 ry = res.get<TestStruct2>("y");
EXPECT_EQ(rx.a, 1.5);
EXPECT_EQ(rx.b, 2.5);
EXPECT_EQ(ry.a, 4);
EXPECT_EQ(ry.b, 3);
}
}

2
components/CMakeLists.txt
View file

@ -29,7 +29,7 @@ endif (GIT_CHECKOUT)
# source files # source files
add_component_dir (lua add_component_dir (lua
luastate utilpackage luastate utilpackage serialization
) )
add_component_dir (settings add_component_dir (settings

255
components/lua/serialization.cpp Normal file
View file

@ -0,0 +1,255 @@
#include "serialization.hpp"
#include <osg/Vec2f>
#include <osg/Vec3f>
#include <components/misc/endianness.hpp>
namespace LuaUtil
{
constexpr unsigned char FORMAT_VERSION = 0;
enum class SerializedType
{
NUMBER = 0x0,
LONG_STRING = 0x1,
BOOLEAN = 0x2,
TABLE_START = 0x3,
TABLE_END = 0x4,
VEC2 = 0x10,
VEC3 = 0x11,
};
constexpr unsigned char SHORT_STRING_FLAG = 0x20; // 0x001SSSSS. SSSSS = string length
constexpr unsigned char CUSTOM_FULL_FLAG = 0x40; // 0b01TTTTTT + 32bit dataSize
constexpr unsigned char CUSTOM_COMPACT_FLAG = 0x80; // 0b1SSSSTTT. SSSS = dataSize, TTT = (typeName size - 1)
static void appendType(BinaryData& out, SerializedType type)
{
out.push_back(static_cast<char>(type));
}
template <typename T>
static void appendValue(BinaryData& out, T v)
{
v = Misc::toLittleEndian(v);
out.append(reinterpret_cast<const char*>(&v), sizeof(v));
}
template <typename T>
static T getValue(std::string_view& binaryData)
{
if (binaryData.size() < sizeof(T))
throw std::runtime_error("Unexpected end");
T v;
std::memcpy(&v, binaryData.data(), sizeof(T));
binaryData = binaryData.substr(sizeof(T));
return Misc::fromLittleEndian(v);
}
static void appendString(BinaryData& out, std::string_view str)
{
if (str.size() < 32)
out.push_back(SHORT_STRING_FLAG | char(str.size()));
else
{
appendType(out, SerializedType::LONG_STRING);
appendValue<uint32_t>(out, str.size());
}
out.append(str.data(), str.size());
}
static void appendData(BinaryData& out, const void* data, size_t dataSize)
{
out.append(reinterpret_cast<const char*>(data), dataSize);
}
void UserdataSerializer::append(BinaryData& out, std::string_view typeName, const void* data, size_t dataSize)
{
assert(!typeName.empty() && typeName.size() <= 64);
if (typeName.size() <= 8 && dataSize < 16)
{ // Compact form: 0b1SSSSTTT. SSSS = dataSize, TTT = (typeName size - 1).
unsigned char t = CUSTOM_COMPACT_FLAG | (dataSize << 3) | (typeName.size() - 1);
out.push_back(t);
}
else
{ // Full form: 0b01TTTTTT + 32bit dataSize.
unsigned char t = CUSTOM_FULL_FLAG | (typeName.size() - 1);
out.push_back(t);
appendValue<uint32_t>(out, dataSize);
}
out.append(typeName.data(), typeName.size());
appendData(out, data, dataSize);
}
static void serializeUserdata(BinaryData& out, const sol::userdata& data, const UserdataSerializer* customSerializer)
{
if (data.is<osg::Vec2f>())
{
appendType(out, SerializedType::VEC2);
osg::Vec2f v = data.as<osg::Vec2f>();
appendValue<float>(out, v.x());
appendValue<float>(out, v.y());
return;
}
if (data.is<osg::Vec3f>())
{
appendType(out, SerializedType::VEC3);
osg::Vec3f v = data.as<osg::Vec3f>();
appendValue<float>(out, v.x());
appendValue<float>(out, v.y());
appendValue<float>(out, v.z());
return;
}
if (customSerializer && customSerializer->serialize(out, data))
return;
else
throw std::runtime_error("Unknown userdata");
}
static void serialize(BinaryData& out, const sol::object& obj, const UserdataSerializer* customSerializer, int recursionCounter)
{
if (obj.get_type() == sol::type::lightuserdata)
throw std::runtime_error("light userdata is not allowed to be serialized");
if (obj.is<sol::function>())
throw std::runtime_error("functions are not allowed to be serialized");
else if (obj.is<sol::userdata>())
serializeUserdata(out, obj, customSerializer);
else if (obj.is<sol::lua_table>())
{
if (recursionCounter >= 32)
throw std::runtime_error("Can not serialize more than 32 nested tables. Likely the table contains itself.");
sol::table table = obj;
appendType(out, SerializedType::TABLE_START);
for (auto& [key, value] : table)
{
serialize(out, key, customSerializer, recursionCounter + 1);
serialize(out, value, customSerializer, recursionCounter + 1);
}
appendType(out, SerializedType::TABLE_END);
}
else if (obj.is<double>())
{
appendType(out, SerializedType::NUMBER);
appendValue<double>(out, obj.as<double>());
}
else if (obj.is<std::string_view>())
appendString(out, obj.as<std::string_view>());
else if (obj.is<bool>())
{
char v = obj.as<bool>() ? 1 : 0;
appendType(out, SerializedType::BOOLEAN);
out.push_back(v);
} else
throw std::runtime_error("Unknown lua type");
}
static void deserializeImpl(sol::state& lua, std::string_view& binaryData, const UserdataSerializer* customSerializer)
{
if (binaryData.empty())
throw std::runtime_error("Unexpected end");
unsigned char type = binaryData[0];
binaryData = binaryData.substr(1);
if (type & (CUSTOM_COMPACT_FLAG | CUSTOM_FULL_FLAG))
{
size_t typeNameSize, dataSize;
if (type & CUSTOM_COMPACT_FLAG)
{ // Compact form: 0b1SSSSTTT. SSSS = dataSize, TTT = (typeName size - 1).
typeNameSize = (type & 7) + 1;
dataSize = (type >> 3) & 15;
}
else
{ // Full form: 0b01TTTTTT + 32bit dataSize.
typeNameSize = (type & 63) + 1;
dataSize = getValue<uint32_t>(binaryData);
}
std::string_view typeName = binaryData.substr(0, typeNameSize);
std::string_view data = binaryData.substr(typeNameSize, dataSize);
binaryData = binaryData.substr(typeNameSize + dataSize);
if (!customSerializer || !customSerializer->deserialize(typeName, data, lua))
throw std::runtime_error("Unknown type: " + std::string(typeName));
return;
}
if (type & SHORT_STRING_FLAG)
{
size_t size = type & 0x1f;
sol::stack::push<std::string_view>(lua.lua_state(), binaryData.substr(0, size));
binaryData = binaryData.substr(size);
return;
}
switch (static_cast<SerializedType>(type))
{
case SerializedType::NUMBER:
sol::stack::push<double>(lua.lua_state(), getValue<double>(binaryData));
return;
case SerializedType::BOOLEAN:
sol::stack::push<bool>(lua.lua_state(), getValue<char>(binaryData) != 0);
return;
case SerializedType::LONG_STRING:
{
uint32_t size = getValue<uint32_t>(binaryData);
sol::stack::push<std::string_view>(lua.lua_state(), binaryData.substr(0, size));
binaryData = binaryData.substr(size);
return;
}
case SerializedType::TABLE_START:
{
lua_createtable(lua, 0, 0);
while (!binaryData.empty() && binaryData[0] != char(SerializedType::TABLE_END))
{
deserializeImpl(lua, binaryData, customSerializer);
deserializeImpl(lua, binaryData, customSerializer);
lua_settable(lua, -3);
}
if (binaryData.empty())
throw std::runtime_error("Unexpected end");
binaryData = binaryData.substr(1);
return;
}
case SerializedType::TABLE_END:
throw std::runtime_error("Unexpected table end");
case SerializedType::VEC2:
{
float x = getValue<float>(binaryData);
float y = getValue<float>(binaryData);
sol::stack::push<osg::Vec2f>(lua.lua_state(), osg::Vec2f(x, y));
return;
}
case SerializedType::VEC3:
{
float x = getValue<float>(binaryData);
float y = getValue<float>(binaryData);
float z = getValue<float>(binaryData);
sol::stack::push<osg::Vec3f>(lua.lua_state(), osg::Vec3f(x, y, z));
return;
}
default: throw std::runtime_error("Unknown type: " + std::to_string(type));
}
}
BinaryData serialize(const sol::object& obj, const UserdataSerializer* customSerializer)
{
if (obj == sol::nil)
return "";
BinaryData res;
res.push_back(FORMAT_VERSION);
serialize(res, obj, customSerializer, 0);
return res;
}
sol::object deserialize(sol::state& lua, std::string_view binaryData, const UserdataSerializer* customSerializer)
{
if (binaryData.empty())
return sol::nil;
if (binaryData[0] != FORMAT_VERSION)
throw std::runtime_error("Incorrect version of Lua serialization format: " +
std::to_string(static_cast<unsigned>(binaryData[0])));
binaryData = binaryData.substr(1);
deserializeImpl(lua, binaryData, customSerializer);
if (!binaryData.empty())
throw std::runtime_error("Unexpected data after serialized object");
return sol::stack::pop<sol::object>(lua.lua_state());
}
}

34
components/lua/serialization.hpp Normal file
View file

@ -0,0 +1,34 @@
#ifndef COMPONENTS_LUA_SERIALIZATION_H
#define COMPONENTS_LUA_SERIALIZATION_H
#include <sol/sol.hpp>
namespace LuaUtil
{
// Note: it can contain \0
using BinaryData = std::string;
class UserdataSerializer
{
public:
virtual ~UserdataSerializer() {}
// Appends serialized sol::userdata to the end of BinaryData.
// Returns false if this type of userdata is not supported by this serializer.
virtual bool serialize(BinaryData&, const sol::userdata&) const = 0;
// Deserializes userdata of type "typeName" from binaryData. Should push the result on stack using sol::stack::push.
// Returns false if this type is not supported by this serializer.
virtual bool deserialize(std::string_view typeName, std::string_view binaryData, sol::state&) const = 0;
protected:
static void append(BinaryData&, std::string_view typeName, const void* data, size_t dataSize);
};
BinaryData serialize(const sol::object&, const UserdataSerializer* customSerializer = nullptr);
sol::object deserialize(sol::state& lua, std::string_view binaryData, const UserdataSerializer* customSerializer = nullptr);
}
#endif // COMPONENTS_LUA_SERIALIZATION_H