summaryrefslogtreecommitdiff
path: root/tools
diff options
context:
space:
mode:
Diffstat (limited to 'tools')
-rw-r--r--tools/preproc/.gitignore1
-rw-r--r--tools/preproc/LICENSE19
-rw-r--r--tools/preproc/Makefile15
-rw-r--r--tools/preproc/asm_file.cpp563
-rw-r--r--tools/preproc/asm_file.h71
-rw-r--r--tools/preproc/char_util.h65
-rw-r--r--tools/preproc/charmap.cpp397
-rw-r--r--tools/preproc/charmap.h64
-rw-r--r--tools/preproc/preproc.cpp83
-rw-r--r--tools/preproc/preproc.h52
-rw-r--r--tools/preproc/utf8.cpp94
-rw-r--r--tools/preproc/utf8.h34
12 files changed, 1458 insertions, 0 deletions
diff --git a/tools/preproc/.gitignore b/tools/preproc/.gitignore
new file mode 100644
index 000000000..eb3470879
--- /dev/null
+++ b/tools/preproc/.gitignore
@@ -0,0 +1 @@
+preproc
diff --git a/tools/preproc/LICENSE b/tools/preproc/LICENSE
new file mode 100644
index 000000000..534d15349
--- /dev/null
+++ b/tools/preproc/LICENSE
@@ -0,0 +1,19 @@
+Copyright (c) 2016 YamaArashi
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
diff --git a/tools/preproc/Makefile b/tools/preproc/Makefile
new file mode 100644
index 000000000..10f930ebd
--- /dev/null
+++ b/tools/preproc/Makefile
@@ -0,0 +1,15 @@
+CXX := g++
+
+CXXFLAGS := -std=c++14 -O2
+
+SRCS := asm_file.cpp charmap.cpp preproc.cpp utf8.cpp
+
+HEADERS := asm_file.h char_util.h charmap.h preproc.h utf8.h
+
+.PHONY: clean
+
+preproc: $(SRCS) $(HEADERS)
+ $(CXX) $(CXXFLAGS) $(SRCS) -o $@
+
+clean:
+ $(RM) preproc preproc.exe
diff --git a/tools/preproc/asm_file.cpp b/tools/preproc/asm_file.cpp
new file mode 100644
index 000000000..0b42d9ab2
--- /dev/null
+++ b/tools/preproc/asm_file.cpp
@@ -0,0 +1,563 @@
+// Copyright(c) 2016 YamaArashi
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+#include <cstdio>
+#include <cstdarg>
+#include "preproc.h"
+#include "asm_file.h"
+#include "char_util.h"
+#include "utf8.h"
+
+AsmFile::AsmFile(std::string filename) : m_filename(filename)
+{
+ FILE *fp = std::fopen(filename.c_str(), "rb");
+
+ if (fp == NULL)
+ FATAL_ERROR("Failed to open \"%s\" for reading.\n", filename.c_str());
+
+ std::fseek(fp, 0, SEEK_END);
+
+ m_size = std::ftell(fp);
+
+ m_buffer = new char[m_size + 1];
+
+ std::rewind(fp);
+
+ if (std::fread(m_buffer, m_size, 1, fp) != 1)
+ FATAL_ERROR("Failed to read \"%s\".\n", filename.c_str());
+
+ m_buffer[m_size] = 0;
+
+ std::fclose(fp);
+
+ m_pos = 0;
+ m_lineNum = 1;
+ m_lineStart = 0;
+
+ RemoveComments();
+}
+
+AsmFile::AsmFile(AsmFile&& other) : m_filename(std::move(other.m_filename))
+{
+ m_buffer = other.m_buffer;
+ m_pos = other.m_pos;
+ m_size = other.m_size;
+ m_lineNum = other.m_lineNum;
+ m_lineStart = other.m_lineStart;
+
+ other.m_buffer = nullptr;
+}
+
+AsmFile::~AsmFile()
+{
+ delete[] m_buffer;
+}
+
+// Removes comments to simplify further processing.
+// It stops upon encountering a null character,
+// which may or may not be the end of file marker.
+// If it's not, the error will be caught later.
+void AsmFile::RemoveComments()
+{
+ long pos = 0;
+ bool inString = false;
+ char stringChar;
+
+ for (;;)
+ {
+ if (m_buffer[pos] == 0)
+ return;
+
+ if (inString)
+ {
+ if (m_buffer[pos] == '\\' && m_buffer[pos + 1] == stringChar)
+ {
+ pos += 2;
+ }
+ else
+ {
+ if (m_buffer[pos] == stringChar)
+ inString = false;
+ pos++;
+ }
+ }
+ else if (m_buffer[pos] == '@' && (pos == 0 || m_buffer[pos - 1] != '\\'))
+ {
+ while (m_buffer[pos] != '\n' && m_buffer[pos] != 0)
+ m_buffer[pos++] = ' ';
+ }
+ else if (m_buffer[pos] == '/' && m_buffer[pos + 1] == '*')
+ {
+ m_buffer[pos++] = ' ';
+ m_buffer[pos++] = ' ';
+
+ bool inCommentString = false;
+ char commentStringChar;
+
+ for (;;)
+ {
+ if (m_buffer[pos] == 0)
+ return;
+
+ if (inCommentString)
+ {
+ if (m_buffer[pos] == '\\' && m_buffer[pos + 1] == commentStringChar)
+ {
+ m_buffer[pos++] = ' ';
+ m_buffer[pos++] = ' ';
+ }
+ else
+ {
+ if (m_buffer[pos] == commentStringChar)
+ inCommentString = false;
+ if (m_buffer[pos] != '\n')
+ m_buffer[pos] = ' ';
+ pos++;
+ }
+ }
+ else
+ {
+ if (m_buffer[pos] == '*' && m_buffer[pos + 1] == '/')
+ {
+ m_buffer[pos++] = ' ';
+ m_buffer[pos++] = ' ';
+ break;
+ }
+ else
+ {
+ if (m_buffer[pos] == '"' || m_buffer[pos] == '\'')
+ {
+ commentStringChar = m_buffer[pos];
+ inCommentString = true;
+ }
+ if (m_buffer[pos] != '\n')
+ m_buffer[pos] = ' ';
+ pos++;
+ }
+ }
+ }
+ }
+ else
+ {
+ if (m_buffer[pos] == '"' || m_buffer[pos] == '\'')
+ {
+ stringChar = m_buffer[pos];
+ inString = true;
+ }
+ pos++;
+ }
+ }
+}
+
+// Checks if we're at a particular directive and if so, consumes it.
+// Returns whether the directive was found.
+bool AsmFile::CheckForDirective(std::string name)
+{
+ long i;
+ long length = static_cast<long>(name.length());
+
+ for (i = 0; i < length && m_pos + i < m_size; i++)
+ if (name[i] != m_buffer[m_pos + i])
+ return false;
+
+ if (i < length)
+ return false;
+
+ m_pos += length;
+
+ return true;
+}
+
+// Checks if we're at a known directive and if so, consumes it.
+// Returns which directive was found.
+Directive AsmFile::GetDirective()
+{
+ SkipWhitespace();
+
+ if (CheckForDirective(".include"))
+ return Directive::Include;
+ else if (CheckForDirective(".string"))
+ return Directive::String;
+ else
+ return Directive::Unknown;
+}
+
+// Skips tabs and spaces.
+void AsmFile::SkipWhitespace()
+{
+ while (m_buffer[m_pos] == '\t' || m_buffer[m_pos] == ' ')
+ m_pos++;
+}
+
+// Reads include path.
+std::string AsmFile::ReadPath()
+{
+ SkipWhitespace();
+
+ if (m_buffer[m_pos] != '"')
+ RaiseError("expected file path");
+
+ m_pos++;
+
+ int length = 0;
+ long startPos = m_pos;
+
+ while (m_buffer[m_pos] != '"')
+ {
+ unsigned char c = m_buffer[m_pos++];
+
+ if (c == 0)
+ {
+ if (m_pos >= m_size)
+ RaiseError("unexpected EOF in include string");
+ else
+ RaiseError("unexpected null character in include string");
+ }
+
+ if (!IsAsciiPrintable(c))
+ RaiseError("unexpected character '\\x%02X' in include string", c);
+
+ // Don't bother allowing any escape sequences.
+ if (c == '\\')
+ {
+ c = m_buffer[m_pos];
+ RaiseError("unexpected escape '\\%c' in include string", c);
+ }
+
+ length++;
+
+ if (length > kMaxPath)
+ RaiseError("path is too long");
+ }
+
+ m_pos++; // Go past the right quote.
+
+ ExpectEmptyRestOfLine();
+
+ return std::string(&m_buffer[startPos], length);
+}
+
+// Reads a charmap char or escape sequence.
+std::string AsmFile::ReadCharOrEscape()
+{
+ std::string sequence;
+
+ bool isEscape = (m_buffer[m_pos] == '\\');
+
+ if (isEscape)
+ {
+ m_pos++;
+
+ if (m_buffer[m_pos] == '"')
+ {
+ sequence = g_charmap->Char('"');
+
+ if (sequence.length() == 0)
+ RaiseError("no mapping exists for double quote");
+
+ return sequence;
+ }
+ else if (m_buffer[m_pos] == '\\')
+ {
+ sequence = g_charmap->Char('\\');
+
+ if (sequence.length() == 0)
+ RaiseError("no mapping exists for backslash");
+
+ return sequence;
+ }
+ }
+
+ unsigned char c = m_buffer[m_pos];
+
+ if (c == 0)
+ {
+ if (m_pos >= m_size)
+ RaiseError("unexpected EOF in UTF-8 string");
+ else
+ RaiseError("unexpected null character in UTF-8 string");
+ }
+
+ if (IsAscii(c) && !IsAsciiPrintable(c))
+ RaiseError("unexpected character U+%X in UTF-8 string", c);
+
+ UnicodeChar unicodeChar = DecodeUtf8(&m_buffer[m_pos]);
+ m_pos += unicodeChar.encodingLength;
+ std::int32_t code = unicodeChar.code;
+
+ if (code == -1)
+ RaiseError("invalid encoding in UTF-8 string");
+
+ if (isEscape && code >= 128)
+ RaiseError("escapes using non-ASCII characters are invalid");
+
+ sequence = isEscape ? g_charmap->Escape(code) : g_charmap->Char(code);
+
+ if (sequence.length() == 0)
+ {
+ if (isEscape)
+ RaiseError("unknown escape '\\%c'", code);
+ else
+ RaiseError("unknown character U+%X", code);
+ }
+
+ return sequence;
+}
+
+// Reads a charmap constant, i.e. "{FOO}".
+std::string AsmFile::ReadConstant()
+{
+ m_pos++; // Assume we're on the left curly bracket.
+
+ long startPos = m_pos;
+
+ while (IsIdentifierChar(m_buffer[m_pos]))
+ m_pos++;
+
+ if (m_buffer[m_pos] != '}')
+ {
+ if (m_buffer[m_pos] == 0)
+ {
+ if (m_pos >= m_size)
+ RaiseError("unexpected EOF in identifier");
+ else
+ RaiseError("unexpected null character in identifier");
+ }
+
+ RaiseError("unexpected character in identifier");
+ }
+
+ std::string sequence = g_charmap->Constant(std::string(&m_buffer[startPos], m_pos - startPos));
+
+ if (sequence.length() == 0)
+ {
+ m_buffer[m_pos] = 0;
+ RaiseError("unknown constant '%s'", &m_buffer[startPos]);
+ }
+
+ m_pos++; // Go past the right curly bracket.
+
+ return sequence;
+}
+
+// Reads a charmap string.
+int AsmFile::ReadString(unsigned char* s)
+{
+ SkipWhitespace();
+
+ if (m_buffer[m_pos] != '"')
+ RaiseError("expected UTF-8 string literal");
+
+ m_pos++;
+
+ int length = 0;
+
+ while (m_buffer[m_pos] != '"')
+ {
+ std::string sequence = (m_buffer[m_pos] == '{') ? ReadConstant() : ReadCharOrEscape();
+
+ for (const char& c : sequence)
+ {
+ if (length == kMaxStringLength)
+ RaiseError("mapped string longer than %d bytes", length);
+
+ s[length++] = c;
+ }
+ }
+
+ m_pos++; // Go past the right quote.
+
+ SkipWhitespace();
+
+ if (ConsumeComma())
+ {
+ SkipWhitespace();
+ int padLength = ReadPadLength();
+
+ while (length < padLength)
+ {
+ s[length++] = 0;
+ }
+ }
+
+ ExpectEmptyRestOfLine();
+
+ return length;
+}
+
+// If we're at a comma, consumes it.
+// Returns whether a comma was found.
+bool AsmFile::ConsumeComma()
+{
+ if (m_buffer[m_pos] == ',')
+ {
+ m_pos++;
+ return true;
+ }
+
+ return false;
+}
+
+// Converts digit character to numerical value.
+static int ConvertDigit(char c, int radix)
+{
+ int digit;
+
+ if (c >= '0' && c <= '9')
+ digit = c - '0';
+ else if (c >= 'A' && c <= 'F')
+ digit = 10 + c - 'A';
+ else if (c >= 'a' && c <= 'f')
+ digit = 10 + c - 'a';
+ else
+ return -1;
+
+ return (digit < radix) ? digit : -1;
+}
+
+// Reads the pad length for a charmap string.
+int AsmFile::ReadPadLength()
+{
+ if (!IsAsciiDigit(m_buffer[m_pos]))
+ RaiseError("expected integer");
+
+ int radix = 10;
+
+ if (m_buffer[m_pos] == '0' && m_buffer[m_pos + 1] == 'x')
+ {
+ radix = 16;
+ m_pos += 2;
+ }
+
+ int n = 0;
+ int digit;
+
+ while ((digit = ConvertDigit(m_buffer[m_pos], radix)) != -1)
+ {
+ n = n * radix + digit;
+
+ if (n > kMaxStringLength)
+ RaiseError("pad length greater than maximum length (%d)", kMaxStringLength);
+
+ m_pos++;
+ }
+
+ return n;
+}
+
+// Outputs the current line and moves to the next one.
+void AsmFile::OutputLine()
+{
+ while (m_buffer[m_pos] != '\n' && m_buffer[m_pos] != 0)
+ m_pos++;
+
+ if (m_buffer[m_pos] == 0)
+ {
+ if (m_pos >= m_size)
+ {
+ RaiseWarning("file doesn't end with newline");
+ puts(&m_buffer[m_lineStart]);
+ }
+ else
+ {
+ RaiseError("unexpected null character");
+ }
+ }
+ else
+ {
+ m_buffer[m_pos] = 0;
+ puts(&m_buffer[m_lineStart]);
+ m_buffer[m_pos] = '\n';
+ m_pos++;
+ m_lineStart = m_pos;
+ m_lineNum++;
+ }
+}
+
+// Asserts that the rest of the line is empty and moves to the next one.
+void AsmFile::ExpectEmptyRestOfLine()
+{
+ SkipWhitespace();
+
+ if (m_buffer[m_pos] == 0)
+ {
+ if (m_pos >= m_size)
+ RaiseWarning("file doesn't end with newline");
+ else
+ RaiseError("unexpected null character");
+ }
+ else if (m_buffer[m_pos] == '\n')
+ {
+ m_pos++;
+ m_lineStart = m_pos;
+ m_lineNum++;
+ }
+ else if (m_buffer[m_pos] == '\r')
+ {
+ RaiseError("only Unix-style LF newlines are supported");
+ }
+ else
+ {
+ RaiseError("junk at end of line");
+ }
+}
+
+// Checks if we're at the end of the file.
+bool AsmFile::IsAtEnd()
+{
+ return (m_pos >= m_size);
+}
+
+// Output the current location to set gas's logical file and line numbers.
+void AsmFile::OutputLocation()
+{
+ printf("# %ld \"%s\"\n", m_lineNum, m_filename.c_str());
+}
+
+// Reports a diagnostic message.
+void AsmFile::ReportDiagnostic(const char* type, const char* format, std::va_list args)
+{
+ const int bufferSize = 1024;
+ char buffer[bufferSize];
+ std::vsnprintf(buffer, bufferSize, format, args);
+ std::fprintf(stderr, "%s:%ld: %s: %s\n", m_filename.c_str(), m_lineNum, type, buffer);
+}
+
+#define DO_REPORT(type) \
+do \
+{ \
+ std::va_list args; \
+ va_start(args, format); \
+ ReportDiagnostic(type, format, args); \
+ va_end(args); \
+} while (0)
+
+// Reports an error diagnostic and terminates the program.
+void AsmFile::RaiseError(const char* format, ...)
+{
+ DO_REPORT("error");
+ exit(1);
+}
+
+// Reports a warning diagnostic.
+void AsmFile::RaiseWarning(const char* format, ...)
+{
+ DO_REPORT("warning");
+}
diff --git a/tools/preproc/asm_file.h b/tools/preproc/asm_file.h
new file mode 100644
index 000000000..1c137d7d3
--- /dev/null
+++ b/tools/preproc/asm_file.h
@@ -0,0 +1,71 @@
+// Copyright(c) 2016 YamaArashi
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+#ifndef ASM_FILE_H
+#define ASM_FILE_H
+
+#include <cstdarg>
+#include <cstdint>
+#include <string>
+#include "preproc.h"
+
+enum class Directive
+{
+ Include,
+ String,
+ Unknown
+};
+
+class AsmFile
+{
+public:
+ AsmFile(std::string filename);
+ AsmFile(AsmFile&& other);
+ AsmFile(const AsmFile&) = delete;
+ ~AsmFile();
+ Directive GetDirective();
+ std::string ReadPath();
+ int ReadString(unsigned char* s);
+ bool IsAtEnd();
+ void OutputLine();
+ void OutputLocation();
+
+private:
+ char* m_buffer;
+ long m_pos;
+ long m_size;
+ long m_lineNum;
+ long m_lineStart;
+ std::string m_filename;
+
+ bool ConsumeComma();
+ int ReadPadLength();
+ void RemoveComments();
+ bool CheckForDirective(std::string name);
+ std::string ReadCharOrEscape();
+ std::string ReadConstant();
+ void SkipWhitespace();
+ void ExpectEmptyRestOfLine();
+ void ReportDiagnostic(const char* type, const char* format, std::va_list args);
+ void RaiseError(const char* format, ...);
+ void RaiseWarning(const char *format, ...);
+};
+
+#endif // ASM_FILE_H
diff --git a/tools/preproc/char_util.h b/tools/preproc/char_util.h
new file mode 100644
index 000000000..05f9a1dd4
--- /dev/null
+++ b/tools/preproc/char_util.h
@@ -0,0 +1,65 @@
+// Copyright(c) 2016 YamaArashi
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+#ifndef CHAR_UTIL_H
+#define CHAR_UTIL_H
+
+#include <cstdint>
+#include <cassert>
+
+inline bool IsAscii(unsigned char c)
+{
+ return (c < 128);
+}
+
+inline bool IsAsciiAlpha(unsigned char c)
+{
+ return ((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z'));
+}
+
+inline bool IsAsciiDigit(unsigned char c)
+{
+ return (c >= '0' && c <= '9');
+}
+
+inline bool IsAsciiHexDigit(unsigned char c)
+{
+ return ((c >= '0' && c <= '9')
+ || (c >= 'a' && c <= 'f')
+ || (c >= 'A' && c <= 'F'));
+}
+
+inline bool IsAsciiAlphanum(unsigned char c)
+{
+ return (IsAsciiAlpha(c) || IsAsciiDigit(c));
+}
+
+inline bool IsAsciiPrintable(unsigned char c)
+{
+ return (c >= ' ' && c <= '~');
+}
+
+// Returns whether the character can be used in the identifier of a "{FOO}" constant in strings.
+inline bool IsIdentifierChar(unsigned char c)
+{
+ return IsAsciiAlphanum(c) || c == '_';
+}
+
+#endif // CHAR_UTIL_H
diff --git a/tools/preproc/charmap.cpp b/tools/preproc/charmap.cpp
new file mode 100644
index 000000000..de693eda5
--- /dev/null
+++ b/tools/preproc/charmap.cpp
@@ -0,0 +1,397 @@
+// Copyright(c) 2016 YamaArashi
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+#include <cstdio>
+#include <cstdint>
+#include <cstdarg>
+#include "preproc.h"
+#include "charmap.h"
+#include "char_util.h"
+#include "utf8.h"
+
+enum LhsType
+{
+ Char,
+ Escape,
+ Constant,
+ None
+};
+
+struct Lhs
+{
+ LhsType type;
+ std::string name;
+ std::int32_t code;
+};
+
+class CharmapReader
+{
+public:
+ CharmapReader(std::string filename);
+ CharmapReader(const CharmapReader&) = delete;
+ ~CharmapReader();
+ Lhs ReadLhs();
+ void ExpectEqualsSign();
+ std::string ReadSequence();
+ void ExpectEmptyRestOfLine();
+
+private:
+ char* m_buffer;
+ long m_pos;
+ long m_size;
+ long m_lineNum;
+ std::string m_filename;
+
+ void RaiseError(const char* format, ...);
+ void RemoveComments();
+ std::string ReadConstant();
+ void SkipWhitespace();
+};
+
+CharmapReader::CharmapReader(std::string filename) : m_filename(filename)
+{
+ FILE *fp = std::fopen(filename.c_str(), "rb");
+
+ if (fp == NULL)
+ FATAL_ERROR("Failed to open \"%s\" for reading.\n", filename.c_str());
+
+ std::fseek(fp, 0, SEEK_END);
+
+ m_size = std::ftell(fp);
+
+ m_buffer = new char[m_size + 1];
+
+ std::rewind(fp);
+
+ if (std::fread(m_buffer, m_size, 1, fp) != 1)
+ FATAL_ERROR("Failed to read \"%s\".\n", filename.c_str());
+
+ m_buffer[m_size] = 0;
+
+ std::fclose(fp);
+
+ m_pos = 0;
+ m_lineNum = 1;
+
+ RemoveComments();
+}
+
+CharmapReader::~CharmapReader()
+{
+ delete[] m_buffer;
+}
+
+Lhs CharmapReader::ReadLhs()
+{
+ Lhs lhs;
+
+ for (;;)
+ {
+ SkipWhitespace();
+
+ if (m_buffer[m_pos] == '\n')
+ {
+ m_pos++;
+ m_lineNum++;
+ }
+ else
+ {
+ break;
+ }
+ }
+
+ if (m_buffer[m_pos] == '\'')
+ {
+ m_pos++;
+
+ bool isEscape = (m_buffer[m_pos] == '\\');
+
+ if (isEscape)
+ {
+ m_pos++;
+ }
+
+ unsigned char c = m_buffer[m_pos];
+
+ if (c == 0)
+ {
+ if (m_pos >= m_size)
+ RaiseError("unexpected EOF in UTF-8 character literal");
+ else
+ RaiseError("unexpected null character in UTF-8 character literal");
+ }
+
+ if (IsAscii(c) && !IsAsciiPrintable(c))
+ RaiseError("unexpected character U+%X in UTF-8 character literal", c);
+
+ UnicodeChar unicodeChar = DecodeUtf8(&m_buffer[m_pos]);
+ std::int32_t code = unicodeChar.code;
+
+ if (code == -1)
+ RaiseError("invalid encoding in UTF-8 character literal");
+
+ m_pos += unicodeChar.encodingLength;
+
+ if (m_buffer[m_pos] != '\'')
+ RaiseError("unterminated character literal");
+
+ m_pos++;
+
+ lhs.code = code;
+
+ if (isEscape)
+ {
+ if (code >= 128)
+ RaiseError("escapes using non-ASCII characters are invalid");
+
+ switch (code)
+ {
+ case '\'':
+ lhs.type = LhsType::Char;
+ break;
+ case '\\':
+ lhs.type = LhsType::Char;
+ case '"':
+ RaiseError("cannot escape double quote");
+ break;
+ default:
+ lhs.type = LhsType::Escape;
+ }
+ }
+ else
+ {
+ if (code == '\'')
+ RaiseError("empty character literal");
+
+ lhs.type = LhsType::Char;
+ }
+ }
+ else if (IsIdentifierChar(m_buffer[m_pos]))
+ {
+ lhs.type = LhsType::Constant;
+ lhs.name = ReadConstant();
+ }
+ else if (m_buffer[m_pos] == '\r')
+ {
+ RaiseError("only Unix-style LF newlines are supported");
+ }
+ else if (m_buffer[m_pos] == 0)
+ {
+ if (m_pos < m_size)
+ RaiseError("unexpected null character");
+ lhs.type = LhsType::None;
+ }
+ else
+ {
+ RaiseError("junk at start of line");
+ }
+
+ return lhs;
+}
+
+void CharmapReader::ExpectEqualsSign()
+{
+ SkipWhitespace();
+
+ if (m_buffer[m_pos] != '=')
+ RaiseError("expected equals sign");
+
+ m_pos++;
+}
+
+static unsigned int ConvertHexDigit(char c)
+{
+ unsigned int digit = 0;
+
+ if (c >= '0' && c <= '9')
+ digit = c - '0';
+ else if (c >= 'A' && c <= 'F')
+ digit = 10 + c - 'A';
+ else if (c >= 'a' && c <= 'f')
+ digit = 10 + c - 'a';
+
+ return digit;
+}
+
+std::string CharmapReader::ReadSequence()
+{
+ SkipWhitespace();
+
+ long startPos = m_pos;
+
+ unsigned int length = 0;
+
+ while (IsAsciiHexDigit(m_buffer[m_pos]) && IsAsciiHexDigit(m_buffer[m_pos + 1]))
+ {
+ m_pos += 2;
+ length++;
+
+ if (length > kMaxCharmapSequenceLength)
+ RaiseError("byte sequence too long (max is %lu bytes)", kMaxCharmapSequenceLength);
+
+ SkipWhitespace();
+ }
+
+ if (IsAsciiHexDigit(m_buffer[m_pos]))
+ RaiseError("each byte must have 2 hex digits");
+
+ if (length == 0)
+ RaiseError("expected byte sequence");
+
+ std::string sequence;
+ sequence.reserve(length);
+
+ m_pos = startPos;
+
+ for (unsigned int i = 0; i < length; i++)
+ {
+ unsigned int digit1 = ConvertHexDigit(m_buffer[m_pos]);
+ unsigned int digit2 = ConvertHexDigit(m_buffer[m_pos + 1]);
+ unsigned char byte = digit1 * 16 + digit2;
+ sequence += byte;
+
+ m_pos += 2;
+ SkipWhitespace();
+ }
+
+ return sequence;
+}
+
+void CharmapReader::ExpectEmptyRestOfLine()
+{
+ SkipWhitespace();
+
+ if (m_buffer[m_pos] == 0)
+ {
+ if (m_pos < m_size)
+ RaiseError("unexpected null character");
+ }
+ else if (m_buffer[m_pos] == '\n')
+ {
+ m_pos++;
+ m_lineNum++;
+ }
+ else if (m_buffer[m_pos] == '\r')
+ {
+ RaiseError("only Unix-style LF newlines are supported");
+ }
+ else
+ {
+ RaiseError("junk at end of line");
+ }
+}
+
+void CharmapReader::RaiseError(const char* format, ...)
+{
+ const int bufferSize = 1024;
+ char buffer[bufferSize];
+
+ std::va_list args;
+ va_start(args, format);
+ std::vsnprintf(buffer, bufferSize, format, args);
+ va_end(args);
+
+ std::fprintf(stderr, "%s:%ld: error: %s\n", m_filename.c_str(), m_lineNum, buffer);
+}
+
+void CharmapReader::RemoveComments()
+{
+ long pos = 0;
+ bool inString = false;
+
+ for (;;)
+ {
+ if (m_buffer[pos] == 0)
+ return;
+
+ if (inString)
+ {
+ if (m_buffer[pos] == '\\' && m_buffer[pos + 1] == '\'')
+ {
+ pos += 2;
+ }
+ else
+ {
+ if (m_buffer[pos] == '\'')
+ inString = false;
+ pos++;
+ }
+ }
+ else if (m_buffer[pos] == '@')
+ {
+ while (m_buffer[pos] != '\n' && m_buffer[pos] != 0)
+ m_buffer[pos++] = ' ';
+ }
+ else
+ {
+ if (m_buffer[pos] == '\'')
+ inString = true;
+ pos++;
+ }
+ }
+}
+
+std::string CharmapReader::ReadConstant()
+{
+ long startPos = m_pos;
+
+ while (IsIdentifierChar(m_buffer[m_pos]))
+ m_pos++;
+
+ return std::string(&m_buffer[startPos], m_pos - startPos);
+}
+
+void CharmapReader::SkipWhitespace()
+{
+ while (m_buffer[m_pos] == '\t' || m_buffer[m_pos] == ' ')
+ m_pos++;
+}
+
+Charmap::Charmap(std::string filename)
+{
+ CharmapReader reader(filename);
+
+ for (;;)
+ {
+ Lhs lhs = reader.ReadLhs();
+
+ if (lhs.type == LhsType::None)
+ return;
+
+ reader.ExpectEqualsSign();
+
+ std::string sequence = reader.ReadSequence();
+
+ switch (lhs.type)
+ {
+ case LhsType::Char:
+ m_chars[lhs.code] = sequence;
+ break;
+ case LhsType::Escape:
+ m_escapes[lhs.code] = sequence;
+ break;
+ case LhsType::Constant:
+ m_constants[lhs.name] = sequence;
+ break;
+ }
+
+ reader.ExpectEmptyRestOfLine();
+ }
+}
diff --git a/tools/preproc/charmap.h b/tools/preproc/charmap.h
new file mode 100644
index 000000000..0d752ac99
--- /dev/null
+++ b/tools/preproc/charmap.h
@@ -0,0 +1,64 @@
+// Copyright(c) 2016 YamaArashi
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+#ifndef CHARMAP_H
+#define CHARMAP_H
+
+#include <cstdint>
+#include <string>
+#include <map>
+#include <vector>
+
+class Charmap
+{
+public:
+ Charmap(std::string filename);
+
+ std::string Char(std::int32_t code)
+ {
+ auto it = m_chars.find(code);
+
+ if (it == m_chars.end())
+ return std::string();
+
+ return it->second;
+ }
+
+ std::string Escape(unsigned char code)
+ {
+ return m_escapes[code];
+ }
+
+ std::string Constant(std::string identifier)
+ {
+ auto it = m_constants.find(identifier);
+
+ if (it == m_constants.end())
+ return std::string();
+
+ return it->second;
+ }
+private:
+ std::map<std::int32_t, std::string> m_chars;
+ std::string m_escapes[128];
+ std::map<std::string, std::string> m_constants;
+};
+
+#endif // CHARMAP_H
diff --git a/tools/preproc/preproc.cpp b/tools/preproc/preproc.cpp
new file mode 100644
index 000000000..7fc3f5c10
--- /dev/null
+++ b/tools/preproc/preproc.cpp
@@ -0,0 +1,83 @@
+// Copyright(c) 2016 YamaArashi
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+#include <stack>
+#include "preproc.h"
+#include "asm_file.h"
+#include "charmap.h"
+
+Charmap* g_charmap;
+
+int main(int argc, char **argv)
+{
+ if (argc != 3)
+ {
+ fprintf(stderr, "Usage: %s ASM_FILE CHARMAP_FILE", argv[0]);
+ return 1;
+ }
+
+ g_charmap = new Charmap(argv[2]);
+
+ std::stack<AsmFile> stack;
+
+ stack.push(AsmFile(argv[1]));
+
+ for (;;)
+ {
+ while (stack.top().IsAtEnd())
+ {
+ stack.pop();
+
+ if (stack.empty())
+ return 0;
+ else
+ stack.top().OutputLocation();
+ }
+
+ Directive directive = stack.top().GetDirective();
+
+ switch (directive)
+ {
+ case Directive::Include:
+ stack.push(AsmFile(stack.top().ReadPath()));
+ stack.top().OutputLocation();
+ break;
+ case Directive::String:
+ {
+ unsigned char s[kMaxStringLength];
+ int length = stack.top().ReadString(s);
+
+ printf("\t.byte ");
+ for (int i = 0; i < length; i++)
+ {
+ printf("0x%02X", s[i]);
+
+ if (i < length - 1)
+ printf(", ");
+ }
+ putchar('\n');
+ break;
+ }
+ case Directive::Unknown:
+ stack.top().OutputLine();
+ break;
+ }
+ }
+}
diff --git a/tools/preproc/preproc.h b/tools/preproc/preproc.h
new file mode 100644
index 000000000..c9e1a8414
--- /dev/null
+++ b/tools/preproc/preproc.h
@@ -0,0 +1,52 @@
+// Copyright(c) 2016 YamaArashi
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+#ifndef PREPROC_H
+#define PREPROC_H
+
+#include "charmap.h"
+
+#ifdef _MSC_VER
+
+#define FATAL_ERROR(format, ...) \
+do \
+{ \
+ fprintf(stderr, format, __VA_ARGS__); \
+ exit(1); \
+} while (0)
+
+#else
+
+#define FATAL_ERROR(format, ...) \
+do \
+{ \
+ fprintf(stderr, format, ##__VA_ARGS__); \
+ exit(1); \
+} while (0)
+
+#endif // _MSC_VER
+
+const int kMaxPath = 256;
+const int kMaxStringLength = 256;
+const unsigned long kMaxCharmapSequenceLength = 16;
+
+extern Charmap* g_charmap;
+
+#endif // PREPROC_H
diff --git a/tools/preproc/utf8.cpp b/tools/preproc/utf8.cpp
new file mode 100644
index 000000000..0aed83f4a
--- /dev/null
+++ b/tools/preproc/utf8.cpp
@@ -0,0 +1,94 @@
+// Copyright (c) 2008-2009 Bjoern Hoehrmann <bjoern@hoehrmann.de>
+// See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for details.
+//
+// Copyright(c) 2016 YamaArashi
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+#include <cstdint>
+#include "utf8.h"
+
+static const unsigned char s_byteTypeTable[] =
+{
+ 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 00..1f
+ 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 20..3f
+ 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 40..5f
+ 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 60..7f
+ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, // 80..9f
+ 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, // a0..bf
+ 8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, // c0..df
+ 0xa,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x4,0x3,0x3, // e0..ef
+ 0xb,0x6,0x6,0x6,0x5,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8, // f0..ff
+};
+
+const unsigned char s0 = 0 * 12;
+const unsigned char s1 = 1 * 12;
+const unsigned char s2 = 2 * 12;
+const unsigned char s3 = 3 * 12;
+const unsigned char s4 = 4 * 12;
+const unsigned char s5 = 5 * 12;
+const unsigned char s6 = 6 * 12;
+const unsigned char s7 = 7 * 12;
+const unsigned char s8 = 8 * 12;
+
+static const unsigned char s_transitionTable[] =
+{
+ s0,s1,s2,s3,s5,s8,s7,s1,s1,s1,s4,s6, // s0
+ s1,s1,s1,s1,s1,s1,s1,s1,s1,s1,s1,s1, // s1
+ s1,s0,s1,s1,s1,s1,s1,s0,s1,s0,s1,s1, // s2
+ s1,s2,s1,s1,s1,s1,s1,s2,s1,s2,s1,s1, // s3
+ s1,s1,s1,s1,s1,s1,s1,s2,s1,s1,s1,s1, // s4
+ s1,s2,s1,s1,s1,s1,s1,s1,s1,s2,s1,s1, // s5
+ s1,s1,s1,s1,s1,s1,s1,s3,s1,s3,s1,s1, // s6
+ s1,s3,s1,s1,s1,s1,s1,s3,s1,s3,s1,s1, // s7
+ s1,s3,s1,s1,s1,s1,s1,s1,s1,s1,s1,s1, // s8
+};
+
+// Decodes UTF-8 encoded Unicode code point at "s".
+// If the encoding is valid, it returns the code point and advances "s" past the byte sequence.
+// If the encoding is not valid, it returns -1 and doesn't advance "s".
+UnicodeChar DecodeUtf8(const char* s)
+{
+ UnicodeChar unicodeChar;
+ int state = s0;
+ auto start = s;
+
+ do
+ {
+ unsigned char byte = *s++;
+ int type = s_byteTypeTable[byte];
+
+ if (state == s0)
+ unicodeChar.code = (0xFF >> type) & byte;
+ else
+ unicodeChar.code = (unicodeChar.code << 6) | (byte & 0x3F);
+
+ state = s_transitionTable[state + type];
+
+ if (state == s1)
+ {
+ unicodeChar.code = -1;
+ return unicodeChar;
+ }
+ } while (state != s0);
+
+ unicodeChar.encodingLength = s - start;
+
+ return unicodeChar;
+}
diff --git a/tools/preproc/utf8.h b/tools/preproc/utf8.h
new file mode 100644
index 000000000..259de67c5
--- /dev/null
+++ b/tools/preproc/utf8.h
@@ -0,0 +1,34 @@
+// Copyright(c) 2016 YamaArashi
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+#ifndef UTF8_H
+#define UTF8_H
+
+#include <cstdint>
+
+struct UnicodeChar
+{
+ std::int32_t code;
+ int encodingLength;
+};
+
+UnicodeChar DecodeUtf8(const char* s);
+
+#endif // UTF8_H
generated by cgit v1.2.3 (git 2.25.1) at 2025-06-26 15:44:18 +0000