mirror of
https://github.com/OpenMW/openmw.git
synced 2025-10-30 16:56:38 +00:00
695 lines
18 KiB
C++
695 lines
18 KiB
C++
#include "scanner.hpp"
|
|
|
|
#include <cassert>
|
|
#include <sstream>
|
|
|
|
#include "errorhandler.hpp"
|
|
#include "exception.hpp"
|
|
#include "extensions.hpp"
|
|
#include "parser.hpp"
|
|
|
|
#include <components/misc/strings/lower.hpp>
|
|
|
|
namespace Compiler
|
|
{
|
|
bool Scanner::get(MultiChar& c)
|
|
{
|
|
if (!c.getFrom(mStream))
|
|
return false;
|
|
|
|
mPrevLoc = mLoc;
|
|
|
|
if (c == '\n')
|
|
{
|
|
mStrictKeywords = false;
|
|
mTolerantNames = false;
|
|
mExpectName = false;
|
|
mLoc.mColumn = 0;
|
|
++mLoc.mLine;
|
|
mLoc.mLiteral.clear();
|
|
mIgnoreSpecial = true;
|
|
}
|
|
else
|
|
{
|
|
++mLoc.mColumn;
|
|
c.appendTo(mLoc.mLiteral);
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
void Scanner::putback(MultiChar& c)
|
|
{
|
|
c.putback(mStream);
|
|
mLoc = mPrevLoc;
|
|
}
|
|
|
|
bool Scanner::scanToken(Parser& parser)
|
|
{
|
|
switch (mPutback)
|
|
{
|
|
case Putback_Special:
|
|
|
|
mPutback = Putback_None;
|
|
return parser.parseSpecial(mPutbackCode, mPutbackLoc, *this);
|
|
|
|
case Putback_Integer:
|
|
|
|
mPutback = Putback_None;
|
|
return parser.parseInt(mPutbackInteger, mPutbackLoc, *this);
|
|
|
|
case Putback_Float:
|
|
|
|
mPutback = Putback_None;
|
|
return parser.parseFloat(mPutbackFloat, mPutbackLoc, *this);
|
|
|
|
case Putback_Name:
|
|
|
|
mPutback = Putback_None;
|
|
return parser.parseName(mPutbackName, mPutbackLoc, *this);
|
|
|
|
case Putback_Keyword:
|
|
|
|
mPutback = Putback_None;
|
|
return parser.parseKeyword(mPutbackCode, mPutbackLoc, *this);
|
|
|
|
case Putback_None:
|
|
|
|
break;
|
|
}
|
|
|
|
MultiChar c;
|
|
|
|
if (!get(c))
|
|
{
|
|
parser.parseEOF(*this);
|
|
return false;
|
|
}
|
|
else if (c == ';')
|
|
{
|
|
std::string comment;
|
|
|
|
c.appendTo(comment);
|
|
|
|
while (get(c))
|
|
{
|
|
if (c == '\n')
|
|
{
|
|
putback(c);
|
|
break;
|
|
}
|
|
else
|
|
c.appendTo(comment);
|
|
}
|
|
|
|
TokenLoc loc(mLoc);
|
|
mLoc.mLiteral.clear();
|
|
|
|
return parser.parseComment(comment, loc, *this);
|
|
}
|
|
else if (c.isWhitespace())
|
|
{
|
|
mLoc.mLiteral.clear();
|
|
return true;
|
|
}
|
|
else if (c == ':')
|
|
{
|
|
// treat : as a whitespace :(
|
|
mLoc.mLiteral.clear();
|
|
return true;
|
|
}
|
|
else if (c.isAlpha() || c == '_' || c == '"')
|
|
{
|
|
mIgnoreSpecial = false;
|
|
bool cont = false;
|
|
|
|
if (scanName(c, parser, cont))
|
|
{
|
|
mLoc.mLiteral.clear();
|
|
return cont;
|
|
}
|
|
}
|
|
else if (c.isDigit())
|
|
{
|
|
mIgnoreSpecial = false;
|
|
bool cont = false;
|
|
|
|
bool scanned = mExpectName ? scanName(c, parser, cont) : scanInt(c, parser, cont);
|
|
if (scanned)
|
|
{
|
|
mLoc.mLiteral.clear();
|
|
return cont;
|
|
}
|
|
}
|
|
else if (c == 13) // linux compatibility hack
|
|
{
|
|
return true;
|
|
}
|
|
else
|
|
{
|
|
bool cont = false;
|
|
|
|
if (scanSpecial(c, parser, cont))
|
|
{
|
|
mLoc.mLiteral.clear();
|
|
return cont;
|
|
}
|
|
}
|
|
|
|
TokenLoc loc(mLoc);
|
|
mLoc.mLiteral.clear();
|
|
|
|
mErrorHandler.error("Syntax error", loc);
|
|
throw SourceException();
|
|
}
|
|
|
|
bool Scanner::scanInt(MultiChar& c, Parser& parser, bool& cont)
|
|
{
|
|
assert(c != '\0');
|
|
std::string value;
|
|
c.appendTo(value);
|
|
|
|
while (get(c))
|
|
{
|
|
if (c.isDigit())
|
|
{
|
|
c.appendTo(value);
|
|
}
|
|
else if (!c.isMinusSign() && isStringCharacter(c))
|
|
{
|
|
/// workaround that allows names to begin with digits
|
|
return scanName(c, parser, cont, value);
|
|
}
|
|
else if (c == '.')
|
|
{
|
|
return scanFloat(value, parser, cont);
|
|
}
|
|
else
|
|
{
|
|
putback(c);
|
|
break;
|
|
}
|
|
}
|
|
|
|
TokenLoc loc(mLoc);
|
|
mLoc.mLiteral.clear();
|
|
|
|
std::istringstream stream(value);
|
|
|
|
int intValue = 0;
|
|
stream >> intValue;
|
|
|
|
cont = parser.parseInt(intValue, loc, *this);
|
|
return true;
|
|
}
|
|
|
|
bool Scanner::scanFloat(const std::string& intValue, Parser& parser, bool& cont)
|
|
{
|
|
std::string value = intValue + ".";
|
|
|
|
MultiChar c;
|
|
|
|
bool empty = intValue.empty() || intValue == "-";
|
|
bool error = false;
|
|
|
|
while (get(c))
|
|
{
|
|
if (c.isDigit())
|
|
{
|
|
c.appendTo(value);
|
|
empty = false;
|
|
}
|
|
else if (c.isAlpha() || c == '_')
|
|
error = true;
|
|
else
|
|
{
|
|
putback(c);
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (empty || error)
|
|
return false;
|
|
|
|
TokenLoc loc(mLoc);
|
|
mLoc.mLiteral.clear();
|
|
|
|
std::istringstream stream(value);
|
|
|
|
float floatValue = 0;
|
|
stream >> floatValue;
|
|
|
|
cont = parser.parseFloat(floatValue, loc, *this);
|
|
return true;
|
|
}
|
|
|
|
static const char* sKeywords[] = {
|
|
"begin",
|
|
"end",
|
|
"short",
|
|
"long",
|
|
"float",
|
|
"if",
|
|
"endif",
|
|
"else",
|
|
"elseif",
|
|
"while",
|
|
"endwhile",
|
|
"return",
|
|
"messagebox",
|
|
"set",
|
|
"to",
|
|
nullptr,
|
|
};
|
|
|
|
bool Scanner::scanName(MultiChar& c, Parser& parser, bool& cont, std::string name)
|
|
{
|
|
c.appendTo(name);
|
|
|
|
if (!scanName(name))
|
|
return false;
|
|
else if (name.empty())
|
|
return true;
|
|
|
|
TokenLoc loc(mLoc);
|
|
mLoc.mLiteral.clear();
|
|
|
|
if (name.size() >= 2 && name[0] == '"' && name[name.size() - 1] == '"')
|
|
{
|
|
name = name.substr(1, name.size() - 2);
|
|
// allow keywords enclosed in ""
|
|
/// \todo optionally disable
|
|
if (mStrictKeywords)
|
|
{
|
|
cont = parser.parseName(name, loc, *this);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
int i = 0;
|
|
|
|
std::string lowerCase = Misc::StringUtils::lowerCase(name);
|
|
bool isKeyword = false;
|
|
for (; sKeywords[i]; ++i)
|
|
if (lowerCase == sKeywords[i])
|
|
{
|
|
isKeyword = true;
|
|
break;
|
|
}
|
|
|
|
// Russian localization and some mods use a quirk - add newline character directly
|
|
// to compiled bytecode via HEX-editor to implement multiline messageboxes.
|
|
// Of course, original editor can not compile such script.
|
|
// Allow messageboxes to bypass the "incomplete string or name" error.
|
|
if (lowerCase == "messagebox")
|
|
enableIgnoreNewlines();
|
|
else if (isKeyword)
|
|
mIgnoreNewline = false;
|
|
|
|
if (sKeywords[i])
|
|
{
|
|
cont = parser.parseKeyword(i, loc, *this);
|
|
return true;
|
|
}
|
|
|
|
if (mExtensions)
|
|
{
|
|
if (int keyword = mExtensions->searchKeyword(lowerCase))
|
|
{
|
|
cont = parser.parseKeyword(keyword, loc, *this);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
cont = parser.parseName(name, loc, *this);
|
|
|
|
return true;
|
|
}
|
|
|
|
bool Scanner::scanName(std::string& name)
|
|
{
|
|
MultiChar c;
|
|
bool error = false;
|
|
|
|
while (get(c))
|
|
{
|
|
if (!name.empty() && name[0] == '"')
|
|
{
|
|
if (c == '"')
|
|
{
|
|
c.appendTo(name);
|
|
break;
|
|
}
|
|
// ignoring escape sequences for now, because they are messing up stupid Windows path names.
|
|
// else if (c=='\\')
|
|
// {
|
|
// if (!get (c))
|
|
// {
|
|
// error = true;
|
|
// mErrorHandler.error ("incomplete escape sequence", mLoc);
|
|
// break;
|
|
// }
|
|
// }
|
|
else if (c == '\n')
|
|
{
|
|
if (mIgnoreNewline)
|
|
mErrorHandler.warning("string contains newline character, make sure that it is intended", mLoc);
|
|
else
|
|
{
|
|
bool allWhitespace = true;
|
|
for (size_t i = 1; i < name.size(); i++)
|
|
{
|
|
// ignore comments
|
|
if (name[i] == ';')
|
|
break;
|
|
else if (name[i] != '\t' && name[i] != ' ' && name[i] != '\r')
|
|
{
|
|
allWhitespace = false;
|
|
break;
|
|
}
|
|
}
|
|
if (allWhitespace)
|
|
{
|
|
name.clear();
|
|
mLoc.mLiteral.clear();
|
|
mErrorHandler.warning("unterminated empty string", mLoc);
|
|
return true;
|
|
}
|
|
|
|
error = true;
|
|
mErrorHandler.error("incomplete string or name", mLoc);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
else if (!(c == '"' && name.empty()))
|
|
{
|
|
if (!isStringCharacter(c) && !(mTolerantNames && (c == '.' || c == '-')))
|
|
{
|
|
putback(c);
|
|
break;
|
|
}
|
|
}
|
|
|
|
c.appendTo(name);
|
|
}
|
|
|
|
return !error;
|
|
}
|
|
|
|
bool Scanner::scanSpecial(MultiChar& c, Parser& parser, bool& cont)
|
|
{
|
|
bool expectName = mExpectName;
|
|
mExpectName = false;
|
|
int special = -1;
|
|
|
|
if (c == '\n')
|
|
special = S_newline;
|
|
else if (mIgnoreSpecial)
|
|
{
|
|
// Ignore junk special characters
|
|
TokenLoc loc = mLoc;
|
|
while (get(c))
|
|
{
|
|
if (c.isAlpha() || c == '_' || c == '"' || c.isDigit() || c == ';' || c == '\n')
|
|
{
|
|
putback(c);
|
|
break;
|
|
}
|
|
c.appendTo(loc.mLiteral);
|
|
}
|
|
mErrorHandler.warning("Stray special character at start of line", loc);
|
|
cont = true;
|
|
return true;
|
|
}
|
|
else if (c == '(' || c == '[') /// \todo option to disable the use of [ as alias for (
|
|
special = S_open;
|
|
else if (c == ')' || c == ']') /// \todo option to disable the use of ] as alias for )
|
|
special = S_close;
|
|
else if (c == '.')
|
|
{
|
|
MultiChar next;
|
|
// check, if this starts a float literal
|
|
if (get(next))
|
|
{
|
|
putback(next);
|
|
|
|
if (next.isDigit())
|
|
return scanFloat("", parser, cont);
|
|
}
|
|
|
|
special = S_member;
|
|
}
|
|
else if (c == '=')
|
|
{
|
|
if (get(c))
|
|
{
|
|
/// \todo hack to allow a space in comparison operators (add option to disable)
|
|
if (c == ' ' && !get(c))
|
|
special = S_cmpEQ;
|
|
else if (c == '=')
|
|
special = S_cmpEQ;
|
|
else if (c == '>' || c == '<') // Treat => and =< as ==
|
|
{
|
|
special = S_cmpEQ;
|
|
mErrorHandler.warning(std::string("invalid operator =") + c.data() + ", treating it as ==", mLoc);
|
|
}
|
|
else
|
|
{
|
|
special = S_cmpEQ;
|
|
putback(c);
|
|
// return false;
|
|
/// Allow = as synonym for ==. \todo optionally disable for post-1.0 scripting improvements.
|
|
}
|
|
}
|
|
else
|
|
{
|
|
putback(c);
|
|
return false;
|
|
}
|
|
}
|
|
else if (c == '!')
|
|
{
|
|
if (get(c))
|
|
{
|
|
/// \todo hack to allow a space in comparison operators (add option to disable)
|
|
if (c == ' ' && !get(c))
|
|
return false;
|
|
|
|
if (c == '=')
|
|
special = S_cmpNE;
|
|
else
|
|
{
|
|
putback(c);
|
|
return false;
|
|
}
|
|
}
|
|
else
|
|
return false;
|
|
}
|
|
else if (c.isMinusSign())
|
|
{
|
|
MultiChar next;
|
|
if (get(next))
|
|
{
|
|
if (next == '>')
|
|
special = S_ref;
|
|
else
|
|
{
|
|
putback(next);
|
|
special = S_minus;
|
|
}
|
|
}
|
|
else
|
|
special = S_minus;
|
|
}
|
|
else if (c == '<')
|
|
{
|
|
if (get(c))
|
|
{
|
|
/// \todo hack to allow a space in comparison operators (add option to disable)
|
|
if (c == ' ' && !get(c))
|
|
special = S_cmpLT;
|
|
else if (c == '=')
|
|
{
|
|
special = S_cmpLE;
|
|
|
|
if (get(c) && c != '=') // <== is a allowed as an alternative to <= :(
|
|
putback(c);
|
|
}
|
|
else if (c == '<' || c == '>') // Treat <> and << as <
|
|
{
|
|
special = S_cmpLT;
|
|
mErrorHandler.warning("Invalid operator, treating it as <", mLoc);
|
|
}
|
|
else
|
|
{
|
|
putback(c);
|
|
special = S_cmpLT;
|
|
}
|
|
}
|
|
else
|
|
special = S_cmpLT;
|
|
}
|
|
else if (c == '>')
|
|
{
|
|
if (get(c))
|
|
{
|
|
/// \todo hack to allow a space in comparison operators (add option to disable)
|
|
if (c == ' ' && !get(c))
|
|
special = S_cmpGT;
|
|
else if (c == '=')
|
|
{
|
|
special = S_cmpGE;
|
|
|
|
if (get(c) && c != '=') // >== is a allowed as an alternative to >= :(
|
|
putback(c);
|
|
}
|
|
else if (c == '<' || c == '>') // Treat >< and >> as >
|
|
{
|
|
special = S_cmpGT;
|
|
mErrorHandler.warning("Invalid operator, treating it as >", mLoc);
|
|
}
|
|
else
|
|
{
|
|
putback(c);
|
|
special = S_cmpGT;
|
|
}
|
|
}
|
|
else
|
|
special = S_cmpGT;
|
|
}
|
|
else if (c == '+')
|
|
special = S_plus;
|
|
else if (c == '*')
|
|
special = S_mult;
|
|
else if (c == '/')
|
|
special = S_div;
|
|
else
|
|
return false;
|
|
|
|
if (special == S_newline)
|
|
mLoc.mLiteral = "<newline>";
|
|
else if (expectName && (special == S_member || special == S_minus))
|
|
{
|
|
bool tolerant = mTolerantNames;
|
|
mTolerantNames = true;
|
|
bool out = scanName(c, parser, cont);
|
|
mTolerantNames = tolerant;
|
|
return out;
|
|
}
|
|
|
|
TokenLoc loc(mLoc);
|
|
mLoc.mLiteral.clear();
|
|
|
|
cont = parser.parseSpecial(special, loc, *this);
|
|
|
|
return true;
|
|
}
|
|
|
|
bool Scanner::isStringCharacter(MultiChar& c, bool lookAhead)
|
|
{
|
|
if (lookAhead && c.isMinusSign())
|
|
{
|
|
/// \todo disable this when doing more stricter compiling. Also, find out who is
|
|
/// responsible for allowing it in the first place and meet up with that person in
|
|
/// a dark alley.
|
|
MultiChar next;
|
|
if (next.peek(mStream) && isStringCharacter(next, false))
|
|
return true;
|
|
}
|
|
|
|
return c.isAlpha() || c.isDigit() || c == '_' ||
|
|
/// \todo disable this when doing more stricter compiling
|
|
c == '`' || c == '\'';
|
|
}
|
|
|
|
// constructor
|
|
|
|
Scanner::Scanner(ErrorHandler& errorHandler, std::istream& inputStream, const Extensions* extensions)
|
|
: mErrorHandler(errorHandler)
|
|
, mStream(inputStream)
|
|
, mExtensions(extensions)
|
|
, mPutback(Putback_None)
|
|
, mPutbackCode(0)
|
|
, mPutbackInteger(0)
|
|
, mPutbackFloat(0)
|
|
, mStrictKeywords(false)
|
|
, mTolerantNames(false)
|
|
, mIgnoreNewline(false)
|
|
, mExpectName(false)
|
|
, mIgnoreSpecial(true)
|
|
{
|
|
}
|
|
|
|
void Scanner::scan(Parser& parser)
|
|
{
|
|
while (scanToken(parser))
|
|
;
|
|
mExpectName = false;
|
|
}
|
|
|
|
void Scanner::putbackSpecial(int code, const TokenLoc& loc)
|
|
{
|
|
mPutback = Putback_Special;
|
|
mPutbackCode = code;
|
|
mPutbackLoc = loc;
|
|
}
|
|
|
|
void Scanner::putbackInt(int value, const TokenLoc& loc)
|
|
{
|
|
mPutback = Putback_Integer;
|
|
mPutbackInteger = value;
|
|
mPutbackLoc = loc;
|
|
}
|
|
|
|
void Scanner::putbackFloat(float value, const TokenLoc& loc)
|
|
{
|
|
mPutback = Putback_Float;
|
|
mPutbackFloat = value;
|
|
mPutbackLoc = loc;
|
|
}
|
|
|
|
void Scanner::putbackName(const std::string& name, const TokenLoc& loc)
|
|
{
|
|
mPutback = Putback_Name;
|
|
mPutbackName = name;
|
|
mPutbackLoc = loc;
|
|
}
|
|
|
|
void Scanner::putbackKeyword(int keyword, const TokenLoc& loc)
|
|
{
|
|
mPutback = Putback_Keyword;
|
|
mPutbackCode = keyword;
|
|
mPutbackLoc = loc;
|
|
}
|
|
|
|
void Scanner::listKeywords(std::vector<std::string>& keywords)
|
|
{
|
|
for (int i = 0; Compiler::sKeywords[i]; ++i)
|
|
keywords.emplace_back(Compiler::sKeywords[i]);
|
|
|
|
if (mExtensions)
|
|
mExtensions->listKeywords(keywords);
|
|
}
|
|
|
|
void Scanner::enableIgnoreNewlines()
|
|
{
|
|
mIgnoreNewline = true;
|
|
}
|
|
|
|
void Scanner::enableStrictKeywords()
|
|
{
|
|
mStrictKeywords = true;
|
|
}
|
|
|
|
void Scanner::enableTolerantNames()
|
|
{
|
|
mTolerantNames = true;
|
|
}
|
|
|
|
void Scanner::enableExpectName()
|
|
{
|
|
mExpectName = true;
|
|
}
|
|
}
|