Merge branch 'master' into unk_02006D98

10 files changed, 764 insertions, 218 deletions

diff --git a/tools/msgenc/.gitignore b/tools/msgenc/.gitignore
index 15921071..760f8a2e 100644
--- a/tools/msgenc/.gitignore
+++ b/tools/msgenc/.gitignore
@@ -1 +1,2 @@
 msgenc
+.deps
diff --git a/tools/msgenc/Makefile b/tools/msgenc/Makefile
index 2fc4f3a9..b39d97cb 100644
--- a/tools/msgenc/Makefile
+++ b/tools/msgenc/Makefile
@@ -1,13 +1,40 @@
 CXXFLAGS := -std=c++17 -O2 -Wall -Wno-switch
 CFLAGS   := -O2 -Wall -Wno-switch
 
+ifeq ($(DEBUG),)
+CXXFLAGS += -DNDEBUG
+endif
+
+DEPDIR := .deps
+DEPFLAGS = -MT $@ -MMD -MP -MF $(DEPDIR)/$*.d
+
+define SRCS :=
+	msgenc.cpp
+	MessagesConverter.cpp
+	MessagesDecoder.cpp
+	MessagesEncoder.cpp
+endef
+
+OBJS := $(SRCS:%.cpp=%.o)
+
 .PHONY: all clean
 
 all: msgenc
 	@:
 
 clean:
-	$(RM) msgenc msgenc.exe
+	$(RM) -r msgenc msgenc.exe $(OBJS) $(DEPDIR)
+
+msgenc: $(OBJS)
+	$(CXX) $(LDFLAGS) -o $@ $^
+
+%.o: %.cpp
+%.o: %.cpp $(DEPDIR)/%.d | $(DEPDIR)
+	$(CXX) $(CXXFLAGS) $(DEPFLAGS) -c -o $@ $<
+
+$(DEPDIR): ; @mkdir -p $@
+
+DEPFILES := $(SRCS:%.cpp=$(DEPDIR)/%.d)
+$(DEPFILES):
 
-msgenc: msgenc.cpp
-	$(CXX) $(CXXFLAGS) -o $@ $^
+include $(wildcard $(DEPFILES))
diff --git a/tools/msgenc/MessagesConverter.cpp b/tools/msgenc/MessagesConverter.cpp
new file mode 100644
index 00000000..ff75f7d7
--- /dev/null
+++ b/tools/msgenc/MessagesConverter.cpp
@@ -0,0 +1,97 @@
+#include "MessagesConverter.h"
+
+void MessagesConverter::CharmapRegisterCharacter(string &code, uint16_t value)
+{
+    throw runtime_error("calling parent class method MessagesConverter::CharmapRegisterCharacter when child class method is required");
+}
+
+void MessagesConverter::CmdmapRegisterCommand(string &command, uint16_t value)
+{
+    throw runtime_error("calling parent class method MessagesConverter::CmdmapRegisterCommand when child class method is required");
+}
+
+string MessagesConverter::ReadTextFile(string& filename) {
+    ifstream file(filename);
+    if (!file.good()) {
+        throw ifstream::failure("unable to open file \"" + filename + "\" for reading");
+    }
+    stringstream ss;
+    ss << file.rdbuf();
+    file.close();
+    return ss.str();
+}
+
+void MessagesConverter::WriteTextFile(string& filename, string const& contents) {
+    ofstream file(filename);
+    if (!file.good()) {
+        throw ofstream::failure("unable to open file \"" + filename + "\" for writing");
+    }
+    file << contents;
+    file.close();
+}
+
+void MessagesConverter::ReadCharmap() {
+    string raw = ReadTextFile(charmapfilename);
+    string line;
+    size_t pos, eqpos, last_pos, lineno = 0;
+
+    for (
+        last_pos = 0;
+        last_pos != string::npos && (pos = raw.find_first_of("\r\n", last_pos)) != string::npos;
+        last_pos = raw.find_last_of("\r\n", pos + 1, 2) + 1, lineno++
+    ) {
+        line = raw.substr(last_pos, pos - last_pos);
+        if (line.find("//") != string::npos) {
+            line = line.substr(0, line.find("//"));
+        }
+        if (line.find_first_not_of(" \t") == string::npos)
+            continue;
+        line = line.substr(
+            line.find_first_not_of(" \t")
+        );
+        eqpos = line.find('=');
+        if (eqpos == string::npos) {
+            stringstream s;
+            s << "charmap syntax error at " << (lineno + 1);
+            throw runtime_error(s.str());
+        }
+        string value = line.substr(0, eqpos);
+        string code = line.substr(eqpos + 1);
+        uint16_t value_i = stoi(value, nullptr, 16);
+        if (code[0] == '{' && code[code.length() - 1] == '}') {
+            code = code.substr(1, code.length() - 2);
+            CmdmapRegisterCommand(code, value_i);
+        } else {
+            CharmapRegisterCharacter(code, value_i);
+        }
+    }
+}
+
+uint16_t MessagesConverter::CalcCRC()
+{
+    uint16_t crc = 0;
+    for (char d : binfilename) {
+        for (int i = 0; i < 8; i++) {
+            crc = (crc & 0x8000) ? ((crc << 1) ^ 0x8003) : (crc << 1);
+            if (d & 0x80) crc ^= 1;
+            d <<= 1;
+        }
+    }
+    return crc;
+}
+
+void MessagesConverter::WriteBinaryDecoded(string &filename)
+{
+    ofstream outfile(filename, ios::binary);
+    if (!outfile.good()) {
+        throw ios::failure("Unable to open file " + filename + " for writing");
+    }
+    outfile.write((char *)&header, sizeof(header));
+    outfile.write((char *)alloc_table.data(), alloc_table.size() * sizeof(MsgAlloc));
+    for (auto msg : vec_encoded) {
+        outfile.write((char *)msg.data(), msg.size() * 2);
+    }
+    outfile.close();
+}
+
+MessagesConverter::~MessagesConverter() = default;
diff --git a/tools/msgenc/MessagesConverter.h b/tools/msgenc/MessagesConverter.h
new file mode 100644
index 00000000..ad1ba5f6
--- /dev/null
+++ b/tools/msgenc/MessagesConverter.h
@@ -0,0 +1,103 @@
+#ifndef GUARD_MESSAGESCONVERTER_H
+#define GUARD_MESSAGESCONVERTER_H
+
+#include "util.h"
+#include <string>
+#include <fstream>
+#include <map>
+#include <sstream>
+#include <vector>
+#include <set>
+
+using namespace std;
+
+static inline uint16_t enc_short(uint16_t value, uint16_t & seed) {
+    value ^= seed;
+    seed += 18749;
+    return value;
+}
+
+enum ConvertMode : uint8_t {
+    CONV_ENCODE = 0,
+    CONV_DECODE,
+    CONV_INVALID = 0xFF,
+};
+
+struct MsgArcHeader
+{
+    uint16_t count;
+    uint16_t key;
+};
+
+struct MsgAlloc
+{
+    uint32_t offset;
+    uint32_t length;
+    void encrypt(uint16_t key, int i) {
+        uint32_t alloc_key = (765 * i * key) & 0xFFFF;
+        alloc_key |= alloc_key << 16;
+        offset ^= alloc_key;
+        length ^= alloc_key;
+    }
+    void decrypt(uint16_t key, int i) { encrypt(key, i); }
+};
+
+static inline void EncryptU16String(u16string & message, int & i) {
+    uint16_t key = i * 596947;
+    for (auto & code : message) {
+        code = enc_short(code, key);
+    }
+}
+
+static inline void DecryptU16String(u16string & message, int & i) {
+    EncryptU16String(message, i);
+}
+
+class MessagesConverter{
+    virtual void CharmapRegisterCharacter(string& code, uint16_t value);
+    virtual void CmdmapRegisterCommand(string& command, uint16_t value);
+
+protected:
+    ConvertMode mode;
+    string textfilename;
+    string charmapfilename;
+    string binfilename;
+
+    MsgArcHeader header = {};
+    vector<MsgAlloc> alloc_table;
+    vector<string> vec_decoded;
+    vector<u16string> vec_encoded;
+
+    template <typename key_type, typename mapped_type> void CreateInverseMap(map<key_type, mapped_type>const& _in, map<mapped_type, key_type>& _out) {
+        for (auto _pair : _in) {
+            _out[_pair.second] = _pair.first;
+        }
+    }
+    static string ReadTextFile(string& filename);
+    static void WriteTextFile(string& filename, string const & contents);
+
+public:
+    MessagesConverter(ConvertMode _mode, string &_textfilename, int _key, string &_charmapfilename, string &_binfilename) :
+        mode(_mode),
+        textfilename(_textfilename),
+        charmapfilename(_charmapfilename),
+        binfilename(_binfilename)
+    {
+        header.key = (_key == 0) ? CalcCRC() : static_cast<uint16_t>(_key);
+    }
+    void ReadCharmap();
+    virtual void ReadInput() = 0;
+    virtual void Convert() = 0;
+    virtual void WriteOutput() = 0;
+    virtual ~MessagesConverter() = 0;
+
+    uint16_t CalcCRC();
+
+    uint16_t GetKey() {
+        return header.key;
+    }
+
+    void WriteBinaryDecoded(string &filename);
+};
+
+#endif //GUARD_MESSAGESCONVERTER_H
diff --git a/tools/msgenc/MessagesDecoder.cpp b/tools/msgenc/MessagesDecoder.cpp
new file mode 100644
index 00000000..f98b28c7
--- /dev/null
+++ b/tools/msgenc/MessagesDecoder.cpp
@@ -0,0 +1,185 @@
+#include <algorithm>
+#include "MessagesDecoder.h"
+
+void MessagesDecoder::CmdmapRegisterCommand(string &command, uint16_t value)
+{
+    cmdmap[value] = command;
+    if (command.rfind("STRVAR_", 0) == 0)
+        strvar_codes.insert(value);
+}
+
+void MessagesDecoder::CharmapRegisterCharacter(string &code, uint16_t value)
+{
+    charmap[value] = code;
+}
+
+static string ConvertIntToHexStringN(unsigned value, StrConvMode mode, int n) {
+    string dest;
+    bool printing_zeroes = mode == STR_CONV_MODE_LEADING_ZEROS;
+    unsigned shift = 4 * (n - 1);
+
+    for (int i = 0; i < n; i++) {
+        unsigned nybble = (value >> shift) & 0xF;
+        if (nybble == 0 && !printing_zeroes) {
+            if (i == n - 1) printing_zeroes = true;
+            else if (mode == STR_CONV_MODE_RIGHT_ALIGN) {
+                dest += ' ';
+                continue;
+            }
+        }
+        if (nybble != 0 || printing_zeroes) {
+            if (nybble < 10) {
+                dest += (char)('0' + nybble);
+            } else {
+                dest += (char)('A' + nybble - 10);
+            }
+        }
+        shift -= 4;
+    }
+
+    return dest;
+}
+
+void MessagesDecoder::ReadMessagesFromBin(string& filename)
+{
+    ifstream infile(filename, ios_base::binary);
+    if (!infile.good()) {
+        throw ifstream::failure("Unable to open file \"" + filename + "\" for reading");
+    }
+    infile.read((char*)&header, sizeof(header));
+    debug_printf("%d lines\n", header.count);
+    alloc_table.resize(header.count);
+    infile.read((char*)alloc_table.data(), (streamsize)(sizeof(MsgAlloc) * header.count));
+    int i = 1;
+    for (auto & alloc : alloc_table) {
+        alloc.decrypt(header.key, i);
+        u16string str;
+        str.resize(alloc.length);
+        infile.read((char*)str.data(), (2 * alloc.length));
+        DecryptU16String(str, i);
+        vec_encoded.push_back(str);
+        i++;
+    }
+    infile.close();
+}
+
+u16string MessagesDecoder::DecodeTrainerNameMessage(u16string const &message)
+{
+    int bit = 0;
+    u16string out = u"\uf100";
+    auto src_p = message.cbegin() + 1;
+    char16_t cur_char;
+    do {
+        cur_char = ((*src_p >> bit) & 0x1FF);
+        bit += 9;
+        if (bit >= 15) {
+            src_p++;
+            bit -= 15;
+            if (bit != 0 && src_p < message.cend()) {
+                cur_char |= (*src_p << (9 - bit)) & 0x1FF;
+            }
+        }
+        out += cur_char;
+    } while (src_p < message.cend() && cur_char != 0x1FF);
+    return out;
+}
+
+string MessagesDecoder::DecodeMessage(u16string &message, int &i) {
+    string decoded;
+    bool is_trname = false;
+    for (size_t j = 0; j < message.size(); j++) {
+        uint16_t code = message[j];
+        debug_printf("%04X ", code);
+
+        if (charmap.find(code) != charmap.end()) {
+            decoded += charmap[code];
+        }
+        else if (code == (is_trname ? 0x01FF : 0xFFFF)) {
+            break;
+        }
+        else if (code == 0xFFFE) {
+            decoded += '{';
+            j++;
+            code = message[j++];
+            debug_printf("%04X ", code);
+            string command;
+            bool is_strvar = false;
+            if (find(strvar_codes.cbegin(), strvar_codes.cend(), code & 0xFF00) != strvar_codes.cend()) {
+                is_strvar = true;
+                command = "STRVAR_" + ConvertIntToHexStringN((code >> 8), STR_CONV_MODE_LEFT_ALIGN, 2);
+            }
+            else if (cmdmap.find(code) != cmdmap.end()) {
+                command = cmdmap[code];
+            } else {
+                throw runtime_error("Invalid control code in " + binfilename + ": " + ConvertIntToHexStringN(code, STR_CONV_MODE_LEADING_ZEROS, 4) + " at line " + to_string(i) + ":" + to_string(j));
+            }
+            decoded += command;
+            int nargs = message[j++];
+            debug_printf("%04X ", nargs);
+            if (is_strvar) {
+                decoded += ' ';
+                decoded += to_string(code & 0xFF);
+                if (nargs != 0)
+                    decoded += ',';
+            }
+            for (int k = 0; k < nargs; k++) {
+                decoded += ' ';
+                decoded += to_string(message[j + k]);
+                debug_printf("%04X ", message[j + k]);
+                if (k != nargs - 1)
+                    decoded += ',';
+            }
+            decoded += '}';
+            j += nargs - 1;
+        }
+        else if (code == 0xF100) {
+            decoded += "{TRNAME}";
+            message = DecodeTrainerNameMessage(message);
+            is_trname = true;
+        }
+        else {
+            throw runtime_error("invalid character in " + binfilename + ": " + ConvertIntToHexStringN(code, STR_CONV_MODE_LEADING_ZEROS, 4) + " at " + to_string(i) + ":" + to_string(j));
+        }
+    }
+    MsgAlloc & alloc = alloc_table[i];
+    debug_printf("msg %d: at 0x%08X, count %d\n", i + 1, alloc.offset, alloc.length);
+    return decoded;
+}
+
+template <typename T> void MessagesDecoder::WriteBinaryFile(string& filename, T& data) {
+    ofstream outfile(filename, ios_base::binary);
+    if (!outfile.good()) {
+        throw ofstream::failure("Unable to open file \"" + filename + "\" for writing");
+    }
+    outfile.write((const char *)&data, sizeof(data));
+    outfile.close();
+}
+
+void MessagesDecoder::WriteMessagesToText(string& filename) {
+    stringstream ss;
+    for (string& text : vec_decoded) {
+        ss << text << "\r\n";
+    }
+    WriteTextFile(filename, ss.str());
+}
+
+// Public virtual functions
+
+void MessagesDecoder::ReadInput()
+{
+    ReadMessagesFromBin(binfilename);
+}
+
+void MessagesDecoder::Convert()
+{
+    for (int i = 0; i < (int)vec_encoded.size(); i++) {
+        u16string message = vec_encoded[i];
+        string decoded = DecodeMessage(message, i);
+        vec_decoded.push_back(decoded);
+    }
+}
+
+void MessagesDecoder::WriteOutput()
+{
+    WriteMessagesToText(textfilename);
+}
diff --git a/tools/msgenc/MessagesDecoder.h b/tools/msgenc/MessagesDecoder.h
new file mode 100644
index 00000000..98c2383f
--- /dev/null
+++ b/tools/msgenc/MessagesDecoder.h
@@ -0,0 +1,36 @@
+#ifndef GUARD_MESSAGESDECODER_H
+#define GUARD_MESSAGESDECODER_H
+
+
+#include "MessagesConverter.h"
+
+enum StrConvMode {
+    STR_CONV_MODE_LEFT_ALIGN = 0,
+    STR_CONV_MODE_RIGHT_ALIGN,
+    STR_CONV_MODE_LEADING_ZEROS
+};
+
+class MessagesDecoder : public MessagesConverter
+{
+    map <uint16_t, string> cmdmap;
+    map <uint16_t, string> charmap;
+    set<uint16_t> strvar_codes;
+
+    void ReadMessagesFromBin(string& filename);
+    void WriteMessagesToText(string& filename);
+    template <typename T> void WriteBinaryFile(string& filename, T& data);
+    static u16string DecodeTrainerNameMessage(u16string const &message);
+    string DecodeMessage(u16string& message, int& i);
+    void CharmapRegisterCharacter(string& code, uint16_t value) override;
+    void CmdmapRegisterCommand(string& command, uint16_t value) override;
+
+public:
+    MessagesDecoder(string &_textfilename, int _key, string &_charmapfilename, string &_binfilename) : MessagesConverter(CONV_DECODE, _textfilename, _key, _charmapfilename, _binfilename) {}
+    void ReadInput() override;
+    void Convert() override;
+    void WriteOutput() override;
+    ~MessagesDecoder() override = default;
+};
+
+
+#endif //GUARD_MESSAGESDECODER_H
diff --git a/tools/msgenc/MessagesEncoder.cpp b/tools/msgenc/MessagesEncoder.cpp
new file mode 100644
index 00000000..8450fba0
--- /dev/null
+++ b/tools/msgenc/MessagesEncoder.cpp
@@ -0,0 +1,165 @@
+#include "MessagesEncoder.h"
+
+void MessagesEncoder::CmdmapRegisterCommand(string &command, uint16_t value)
+{
+    cmdmap[command] = value;
+}
+
+void MessagesEncoder::CharmapRegisterCharacter(string &code, uint16_t value)
+{
+    charmap[code] = value;
+}
+
+void MessagesEncoder::ReadMessagesFromText(string& fname) {
+    string text = ReadTextFile(fname);
+    size_t pos = 0;
+    do {
+        text = text.substr(pos);
+        if (text.empty())
+            break;
+        pos = text.find_first_of("\r\n");
+        vec_decoded.push_back(text.substr(0, pos));
+        pos = text.find_last_of("\r\n", pos + 1, 2);
+        if (pos == string::npos)
+            break;
+        pos++;
+    } while (pos != string::npos);
+    header.count = vec_decoded.size();
+    debug_printf("%d lines\n", header.count);
+}
+
+u16string MessagesEncoder::EncodeMessage(const string & message, int & i) {
+    u16string encoded;
+    bool is_trname = false;
+    uint32_t trnamebuf = 0;
+    int bit = 0;
+
+    for (size_t j = 0; j < message.size(); j++) {
+        if (message[j] == '{') {
+            size_t k = message.find('}', j);
+            string enclosed = message.substr(j + 1, k - j - 1);
+            j = k;
+            size_t pos = enclosed.find(' ');
+            string command = enclosed.substr(0, pos);
+            enclosed = enclosed.substr(pos + 1);
+            if (cmdmap.find(command) != cmdmap.end()) {
+                uint16_t command_i = cmdmap[command];
+                encoded += (char16_t)(0xFFFE);
+                debug_printf("%04X ", 0xFFFE);
+                vector<uint16_t> args;
+                if (pos != string::npos) {
+                    do {
+                        k = enclosed.find(',');
+                        string num = enclosed.substr(0, k);
+                        uint16_t num_i = stoi(num);
+                        args.push_back(num_i);
+                        enclosed = enclosed.substr(k + 1);
+                    } while (k != string::npos);
+                    if (command.rfind("STRVAR_", 0) == 0) {
+                        command_i |= args[0];
+                        args.erase(args.begin());
+                    }
+                }
+                encoded += (char16_t)(command_i);
+                debug_printf("%04X ", command_i);
+                encoded += (char16_t)(args.size());
+                debug_printf("%04X ", (unsigned)args.size());
+                for (auto num_i : args) {
+                    encoded += (char16_t)(num_i);
+                    debug_printf("%04X ", num_i);
+                }
+            } else if (command == "TRNAME") {
+                is_trname = true;
+                encoded += (char16_t)(0xF100);
+                debug_printf("%04X ", 0xF100);
+            } else {
+                encoded += (char16_t)(stoi(enclosed, nullptr, 16));
+                debug_printf("%04X ", stoi(enclosed, nullptr, 16));
+            }
+        } else {
+            uint16_t code = 0;
+            size_t k;
+            string substr;
+            for (k = 0; k < message.size() - j; k++) {
+                substr = message.substr(j, k + 1);
+                try {
+                    code = charmap.at(substr);
+                    break;
+                } catch (out_of_range& oor) { /* silently discard */}
+            }
+            if (code == 0 && substr != "\\x0000") {
+                stringstream ss;
+                ss << "unrecognized character in " << textfilename << ": line " << i << " pos " << (j + 1) << " value " << substr;
+                throw runtime_error(ss.str());
+            }
+            debug_printf("%04X ", code);
+            if (is_trname) {
+                if (code & ~0x1FF) {
+                    stringstream ss;
+                    ss << "invalid character for bitpacked string: " << substr;
+                    throw runtime_error(ss.str());
+                }
+                trnamebuf |= code << bit;
+                bit += 9;
+                if (bit >= 15) {
+                    bit -= 15;
+                    encoded += (char16_t)(trnamebuf & 0x7FFF);
+                    trnamebuf >>= 15;
+                }
+            } else {
+                encoded += (char16_t)(code);
+            }
+            j += k;
+        }
+    }
+    if (is_trname && bit > 1) {
+        trnamebuf |= 0xFFFF << bit;
+        encoded += (char16_t)(trnamebuf & 0x7FFF);
+        debug_printf("%04X ", trnamebuf & 0x7FFF);
+    }
+    encoded += (char16_t)(0xFFFF);
+    debug_printf("%04X ", 0xFFFF);
+    return encoded;
+}
+
+void MessagesEncoder::WriteMessagesToBin(string& filename) {
+    ofstream outfile(filename, ios_base::binary);
+    if (!outfile.good()) {
+        throw ofstream::failure("Unable to open file \"" + filename + "\" for writing");
+    }
+    outfile.write((char *)&header, sizeof(header));
+    for (int i = 1; i <= (int)alloc_table.size(); i++) {
+        alloc_table[i - 1].encrypt(header.key, i);
+        EncryptU16String(vec_encoded[i - 1], i);
+    }
+    outfile.write((char *)alloc_table.data(), (streamsize)(sizeof(MsgAlloc) * alloc_table.size()));
+    for (const u16string & m : vec_encoded) {
+        outfile.write((char *)m.c_str(), (streamsize)(m.size() * 2));
+    }
+    outfile.close();
+}
+
+// Public virtual functions
+
+void MessagesEncoder::ReadInput()
+{
+    ReadMessagesFromText(textfilename);
+}
+
+void MessagesEncoder::Convert() {
+    MsgAlloc alloc {(uint32_t)(sizeof(header) + sizeof(MsgAlloc) * header.count), 0};
+    int i = 1;
+    for (const auto& message : vec_decoded) {
+        u16string encoded = EncodeMessage(message, i);
+        alloc.length = encoded.size();
+        vec_encoded.push_back(encoded);
+        debug_printf("msg %d: at 0x%08X, count %d\n", i, alloc.offset, alloc.length);
+        alloc_table.push_back(alloc);
+        alloc.offset += alloc.length * 2;
+        i++;
+    }
+}
+
+void MessagesEncoder::WriteOutput() {
+    WriteMessagesToBin(binfilename);
+}
diff --git a/tools/msgenc/MessagesEncoder.h b/tools/msgenc/MessagesEncoder.h
new file mode 100644
index 00000000..a7b9111c
--- /dev/null
+++ b/tools/msgenc/MessagesEncoder.h
@@ -0,0 +1,26 @@
+#ifndef GUARD_MESSAGESENCODER_H
+#define GUARD_MESSAGESENCODER_H
+
+
+#include "MessagesConverter.h"
+
+class MessagesEncoder : public MessagesConverter
+{
+    map <string, uint16_t> cmdmap;
+    map <string, uint16_t> charmap;
+
+    void ReadMessagesFromText(string& filename);
+    void WriteMessagesToBin(string& filename);
+    u16string EncodeMessage(const string& message, int & i);
+    void CharmapRegisterCharacter(string& code, uint16_t value) override;
+    void CmdmapRegisterCommand(string& command, uint16_t value) override;
+public:
+    MessagesEncoder(string &_textfilename, int _key, string &_charmapfilename, string &_binfilename) : MessagesConverter(CONV_ENCODE, _textfilename, _key, _charmapfilename, _binfilename) {}
+    void ReadInput() override;
+    void Convert() override;
+    void WriteOutput() override;
+    ~MessagesEncoder() override = default;
+};
+
+
+#endif //GUARD_MESSAGESENCODER_H
diff --git a/tools/msgenc/msgenc.cpp b/tools/msgenc/msgenc.cpp
index 9ec21ed2..5caaf31c 100644
--- a/tools/msgenc/msgenc.cpp
+++ b/tools/msgenc/msgenc.cpp
@@ -6,231 +6,120 @@
  */
 
 #include <iostream>
-#include <string>
-#include <sstream>
-#include <fstream>
-#include <map>
-#include <vector>
-#include <algorithm>
-
-struct MsgArcHeader
-{
-    uint16_t count;
-    uint16_t key;
-};
-
-struct MsgAlloc
-{
-    uint32_t offset;
-    uint32_t length;
-};
-
-using namespace std;
-
-string ReadTextFile(string filename) {
-    fstream file(filename);
-    if (!file.good()) {
-        stringstream s;
-        s << "unable to open file \"" << filename << "\" for reading";
-        throw runtime_error(s.str());
-    }
-    stringstream ss;
-    ss << file.rdbuf();
-    file.close();
-    return ss.str();
+#include "MessagesDecoder.h"
+#include "MessagesEncoder.h"
+
+static const char* progname = "msgenc";
+static const char* version = "2021.08.27";
+
+
+static inline void usage() {
+    cout << progname << " v" << version << endl;
+    cout << "Usage: " << progname << " [-h] [-v] -d|-e [-k KEY] -c CHARMAP INFILE OUTFILE" << endl;
+    cout << endl;
+    cout << "INFILE        Required: Path to the input file to convert (-e: plaintext; -d: binary)." << endl;
+    cout << "OUTFILE       Required: Path to the output file (-e: binary; -d: plaintext)." << endl;
+    cout << "-c CHARMAP    Required: Path to a text file with a character mapping, for example pokeheartgold/charmap.txt." << endl;
+    cout << "-d            Decode from binary to text, also print the key" << endl;
+    cout << "-e            Encode from text to binary using the provided key" << endl;
+    cout << "-k KEY        The 16-bit encryption key for this message bank. Default: computes it from the binary file name" << endl;
+    cout << "-v            Print the program version and exit." << endl;
+    cout << "-h            Print this message and exit." << endl;
+    cout << "-D DUMPNAME   Dump the intermediate binary (after decryption or before encryption)." << endl;
 }
 
-static map<string, uint16_t> charmap;
-
-void read_charmap(string filename) {
-    string raw = ReadTextFile(filename);
-    size_t pos, eqpos, last_pos = 0;
-    while (last_pos != string::npos && (pos = raw.find_first_of("\r\n", last_pos)) != string::npos) {
-        eqpos = raw.find('=', last_pos);
-        if (eqpos == string::npos)
-        {
-            stringstream s;
-            s << "charmap syntax error at " << (charmap.size() + 1);
-            throw(runtime_error(s.str()));
+struct Options {
+    ConvertMode mode = CONV_INVALID;
+    int key = 0;
+    vector<string> posargs;
+    string failReason;
+    string charmap;
+    bool printUsage = false;
+    bool printVersion = false;
+    string dumpBinary;
+    Options(int argc, char ** argv) {
+        for (int i = 1; i < argc; i++) {
+            string arg(argv[i]);
+            if (arg == "-d") {
+                mode = CONV_DECODE;
+            } else if (arg == "-e") {
+                mode = CONV_ENCODE;
+            } else if (arg == "-h") {
+                printUsage = true;
+                return;
+            } else if (arg == "-v") {
+                printVersion = true;
+                return;
+            } else if (arg == "-k") {
+                key = stoi(argv[++i], nullptr, 0);
+                // If the key is 0, ensure that it is not overridden by the CRC.
+                key &= 0xFFFF;
+                key |= 0x10000;
+            } else if (arg == "-c") {
+                charmap = argv[++i];
+            } else if (arg == "-D") {
+                dumpBinary = argv[++i];
+            } else if (arg[0] != '-') {
+                posargs.push_back(arg);
+            } else {
+                failReason = "unrecognized option: " + arg;
+                break;
+            }
+        }
+        if (posargs.size() < 2) {
+            failReason = "missing required positional argument: " + (string[]){"INFILE", "OUTFILE"}[posargs.size()];
+        }
+        if (mode == CONV_INVALID) {
+            failReason = "missing mode flag: -d or -e is required";
+        }
+        if (charmap.empty()) {
+            failReason = "missing charmap file: -c CHARMAP is required";
         }
-        string value = raw.substr(last_pos, eqpos - last_pos);
-        string code = raw.substr(eqpos + 1, pos - eqpos - 1);
-        uint16_t value_i = stoi(value, nullptr, 16);
-        charmap[code] = value_i;
-        last_pos = raw.find_last_of("\r\n", pos + 1, 2) + 1;
-    }
-}
-
-static MsgArcHeader header;
-vector<MsgAlloc> alloc_table;
-static vector<string> files;
-static vector<u16string> outfiles;
-
-void read_key(string keyfname) {
-    fstream keyfile(keyfname, ios_base::in | ios_base::binary);
-    if (!keyfile.good()) {
-        stringstream s;
-        s << "unable to open file \"" << keyfname << "\" for reading";
-        throw runtime_error(s.str());
     }
-    keyfile.read((char *)&header.key, 2);
-}
-
-void read_msgs(string fname) {
-    string text = ReadTextFile(fname);
-    size_t pos = 0;
-    do {
-        text = text.substr(pos);
-        if (text.empty())
-            break;
-        pos = text.find_first_of("\r\n");
-        files.push_back(text.substr(0, pos));
-        pos = text.find_last_of("\r\n", pos + 1, 2);
-        if (pos == string::npos)
-            break;
-        pos++;
-    } while (pos != string::npos);
-    header.count = files.size();
-}
-
-uint16_t enc_short(uint16_t value, uint16_t & seed) {
-    value ^= seed;
-    seed += 18749;
-    return value;
-}
-
-static map<string, uint16_t> cmdmap = {
-    {"STRVAR", 0x0100},
-    {"YESNO", 0x200},
-    {"PAUSE", 0x201},
-    {"WAIT", 0x202},
-    {"CURSOR_X", 0x203},
-    {"CURSOR_Y", 0x204},
-    {"COLOR", 0xFF00},
-    {"SIZE", 0xFF01}
 };
 
-void encode_messages() {
-    int i = 1;
-    for (auto message : files) {
-        u16string encoded;
-        uint16_t seed = i * 596947;
-        bool is_trname = false;
-        uint32_t trnamebuf = 0;
-        int bit = 0;
-        for (size_t j = 0; j < message.size(); j++) {
-            if (message[j] == '{') {
-                size_t k = message.find('}', j);
-                string enclosed = message.substr(j + 1, k - j - 1);
-                j = k;
-                size_t pos = enclosed.find(' ');
-                string command = enclosed.substr(0, pos);
-                enclosed = enclosed.substr(pos + 1);
-                if (cmdmap.find(command) != cmdmap.end()) {
-                    uint16_t command_i = cmdmap[command];
-                    encoded += enc_short(0xFFFE, seed);
-                    vector<uint16_t> args;
-                    do {
-                        k = enclosed.find(',');
-                        string num = enclosed.substr(0, k);
-                        uint16_t num_i = stoi(num);
-                        args.push_back(num_i);
-                        enclosed = enclosed.substr(k + 1);
-                    } while (k++ != string::npos);
-
-                    if (command == "STRVAR") {
-                        command_i |= args[0];
-                        args.erase(args.begin());
-                    }
-                    encoded += enc_short(command_i, seed);
-                    encoded += enc_short(args.size(), seed);
-                    for (auto num_i : args) {
-                        encoded += enc_short(num_i, seed);
-                    }
-                } else if (command == "TRNAME") {
-                    is_trname = true;
-                    encoded += enc_short(0xF100, seed);
-                } else {
-                    encoded += enc_short(stoi(enclosed, nullptr, 16), seed);
-                }
-            } else {
-                uint16_t code = 0;
-                size_t k;
-                string substr;
-                for (k = 0; k < message.size() - j; k++) {
-                    substr = message.substr(j, k + 1);
-                    code = charmap[substr];
-                    if (code != 0 || substr == "\\x0000")
-                        break;
-                }
-                if (code == 0 && substr != "\\x0000") {
-                    stringstream ss;
-                    ss << "unrecognized character: file " << i << " pos " << (j + 1);
-                    throw runtime_error(ss.str());
-                }
-                if (is_trname) {
-                    if (code & ~0x1FF) {
-                        stringstream ss;
-                        ss << "invalid character for bitpacked string: " << substr;
-                        throw runtime_error(ss.str());
-                    }
-                    trnamebuf |= code << bit;
-                    bit += 9;
-                    if (bit >= 15) {
-                        bit -= 15;
-                        encoded += enc_short(trnamebuf & 0x7FFF, seed);
-                        trnamebuf >>= 15;
-                    }
-                } else {
-                    encoded += enc_short(code, seed);
-                }
-                j += k;
+int main(int argc, char ** argv) {
+    try {
+        Options options(argc, argv);
+        if (options.printUsage || !options.failReason.empty()) {
+            usage();
+            if (!options.failReason.empty()) {
+                throw invalid_argument(options.failReason);
             }
+            return 0;
+        } else if (options.printVersion) {
+            cout << progname << " v" << version << endl;
+            return 0;
         }
-        if (is_trname && bit > 1) {
-            trnamebuf |= 0xFFFF << bit;
-            encoded += enc_short(trnamebuf & 0x7FFF, seed);
+
+        MessagesConverter *converter;
+        if (options.mode == CONV_DECODE)
+        {
+            converter = new MessagesDecoder(options.posargs[1], options.key, options.charmap, options.posargs[0]);
         }
-        encoded += enc_short(0xFFFF, seed);
-        MsgAlloc alloc {0, 0};
-        if (i > 1) {
-            alloc.offset = alloc_table[i - 2].offset + alloc_table[i - 2].length * 2;
-        } else {
-            alloc.offset = sizeof(header) + sizeof(MsgAlloc) * header.count;
+        else
+        {
+            converter = new MessagesEncoder(options.posargs[0], options.key, options.charmap, options.posargs[1]);
         }
-        alloc.length = encoded.size();
-        outfiles.push_back(encoded);
-        alloc_table.push_back(alloc);
-        i++;
-    }
-    i = 1;
-    for (auto & x : alloc_table) {
-        uint32_t alloc_key = (765 * i * header.key) & 0xFFFF;
-        alloc_key |= alloc_key << 16;
-        x.offset ^= alloc_key;
-        x.length ^= alloc_key;
-        i++;
-    }
-}
-
-void write_messages(string filename) {
-    ofstream outfile(filename, ios_base::binary);
-    outfile.write((char *)&header, sizeof(header));
-    outfile.write((char *)alloc_table.data(), sizeof(MsgAlloc) * alloc_table.size());
-    for (auto m : outfiles) {
-        outfile.write((char *)m.c_str(), m.size() * 2);
+        converter->ReadInput();
+        converter->ReadCharmap();
+        converter->Convert();
+        if (!options.dumpBinary.empty())
+            converter->WriteBinaryDecoded(options.dumpBinary);
+        converter->WriteOutput();
+        if (options.mode == CONV_DECODE) {
+            cout << "Key: " << hex << converter->GetKey() << endl;
+        }
+        delete converter;
+    } catch (invalid_argument& ia) {
+        cerr << "Invalid Argument: " << ia.what() << endl;
+        return 1;
+    } catch (ios_base::failure& iof) {
+        cerr << "IO Failure: " << iof.what() << endl;
+        return 1;
+    } catch (runtime_error& exc) {
+        cerr << "Runtime Error: " << exc.what() << endl;
+        return 1;
     }
-    outfile.close();
-}
-
-int main(int argc, char ** argv) {
-    // msgenc TXTFILE KEYFILE CHARMAP OUTFILE
-    if (argc < 5)
-        throw invalid_argument("usage: msgenc TXTFILE KEYFILE CHARMAP OUTFILE");
-    read_msgs(argv[1]);
-    read_key(argv[2]);
-    read_charmap(argv[3]);
-    encode_messages();
-    write_messages(argv[4]);
     return 0;
 }
diff --git a/tools/msgenc/util.h b/tools/msgenc/util.h
new file mode 100644
index 00000000..5c93f4ca
--- /dev/null
+++ b/tools/msgenc/util.h
@@ -0,0 +1,17 @@
+#ifndef GUARD_UTIL_H
+#define GUARD_UTIL_H
+
+#include <cstdio>
+#include <cstdarg>
+
+static inline __attribute__((format(printf, 1, 2))) void debug_printf(const char * fmt, ...) {
+#ifndef NDEBUG
+    fputs("DEBUG: ", stderr);
+    va_list va_args;
+    va_start(va_args, fmt);
+    vfprintf(stderr, fmt, va_args);
+    va_end(va_args);
+#endif //NDEBUG
+}
+
+#endif //GUARD_UTIL_H