KNARC: Add handling of .knarcignore, .knarckeep; fix up cli

4 files changed, 219 insertions, 22 deletions

diff --git a/tools/csv2bin/Makefile b/tools/csv2bin/Makefile
new file mode 100644
index 00000000..949ec4e4
--- /dev/null
+++ b/tools/csv2bin/Makefile
@@ -0,0 +1,19 @@
+CC = gcc
+CFLAGS = -O3
+
+growth_rates_txt = $(wildcard *.txt)
+growth_rates_bin = $(growth_rates_txt:%.txt=%.bin)
+
+.PHONY: all clean
+
+all: csv2bin $(growth_rates_bin)
+	@:
+
+clean:
+	$(RM) csv2bin $(growth_rates_bin)
+
+csv2bin: csv2bin.c
+	$(CC) $(CFLAGS) -o $@ $^
+
+%.bin: %.txt
+	./csv2bin $<
diff --git a/tools/csv2bin/csv2bin.c b/tools/csv2bin/csv2bin.c
new file mode 100644
index 00000000..5c8b5111
--- /dev/null
+++ b/tools/csv2bin/csv2bin.c
@@ -0,0 +1,103 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+#include <ctype.h>
+
+char * ReadWholeFile(char * filename) {
+    FILE * infile = fopen(filename, "rb");
+    if (infile == NULL) return NULL;
+    fseek(infile, 0, SEEK_END);
+    size_t fsize = ftell(infile);
+    fseek(infile, 0, SEEK_SET);
+    char * ret = malloc(fsize);
+    if (ret != NULL) {
+        if (fread(ret, 1, fsize, infile) != fsize) {
+            free(ret);
+            ret = NULL;
+        }
+    }
+    fclose(infile);
+    return ret;
+}
+
+int main(int argc, char ** argv) {
+    int width = 4;
+    char * buffer = NULL;
+    char * endptr = NULL;
+    FILE * outfile = NULL;
+    char * infname = NULL;
+    for (int i = 1; i < argc; i++) {
+        if (strcmp(argv[i], "-w") == 0 || strcmp(argv[i], "--width") == 0) {
+            ++i;
+            width = strtol(argv[i], &endptr, 10);
+            if (width == 0) {
+                fprintf(stderr, "invalid integer value for %s: %s\n", argv[i - 1], argv[i]);
+                return EXIT_FAILURE;
+            }
+        } else if (strcmp(argv[i], "-o") == 0 || strcmp(argv[i], "--output") == 0) {
+            ++i;
+            outfile = fopen(argv[i], "wb");
+            if (outfile == NULL) {
+                fprintf(stderr, "unable to open file '%s' for writing\n", argv[i]);
+                return EXIT_FAILURE;
+            }
+        } else {
+            if (buffer == NULL)
+            {
+                buffer = ReadWholeFile(argv[i]);
+            }
+            if (buffer == NULL)
+            {
+                fprintf(stderr, "invalid argument: %s\n", argv[i]);
+                return EXIT_FAILURE;
+            }
+            infname = argv[i];
+        }
+    }
+    if (buffer == NULL) {
+        fprintf(stderr, "missing required argument: INFILE\n");
+        return EXIT_FAILURE;
+    }
+    if (outfile == NULL) {
+        char * infname_ext = strrchr(infname, '.');
+        char * outfname = malloc(infname_ext - infname + 5);
+        char * outfname_ext = stpncpy(outfname, argv[1], infname_ext - argv[1]);
+        strcpy(outfname_ext, ".bin");
+        outfile = fopen(outfname, "wb");
+        if (outfile == NULL) {
+            fprintf(stderr, "unable to open file '%s' for writing\n", outfname);
+            return EXIT_FAILURE;
+        }
+        free(outfname);
+    }
+    uint32_t value;
+    size_t size = 0;
+    char * ptr = buffer;
+    char * end = NULL;
+    int lineno = 1;
+    int colno = 1;
+    while (1) {
+        while (isspace(*ptr) || *ptr == ',') {
+            if (*ptr == 0) break;
+            if (*ptr == '\n')
+            { lineno++; colno = 1; }
+            if (*ptr == ',') colno++;
+            ptr++;
+        };
+        if (*ptr == 0) break;
+        value = strtoul(ptr, &end, 10);
+        if (value == 0 && ptr == end) {
+            fprintf(stderr, "syntax error: %d:%d\n", lineno, colno);
+            return EXIT_FAILURE;
+        }
+        if (fwrite(&value, 1, width, outfile) != width) {
+            fprintf(stderr, "write error\n");
+            return EXIT_FAILURE;
+        }
+        ptr = end;
+    }
+    fclose(outfile);
+    free(buffer);
+    return EXIT_SUCCESS;
+}
diff --git a/tools/knarc/Narc.cpp b/tools/knarc/Narc.cpp
index 506e050f..00a9d0ed 100644
--- a/tools/knarc/Narc.cpp
+++ b/tools/knarc/Narc.cpp
@@ -13,6 +13,8 @@
 #include <stack>
 #include <string>
 #include <vector>
+#include <iostream>
+#include <fnmatch.h>
 
 #if __GNUC__ <= 7
 #include <experimental/filesystem>
@@ -24,6 +26,9 @@ namespace fs = std::filesystem;
 
 using namespace std;
 
+extern bool debug;
+extern bool pack_no_fnt;
+
 void Narc::AlignDword(ofstream& ofs, uint8_t paddingChar)
 {
 	if ((ofs.tellp() % 4) != 0)
@@ -104,6 +109,26 @@ NarcError Narc::GetError() const
 	return error;
 }
 
+class WildcardVector : public vector<string> {
+public:
+    WildcardVector(fs::path fp) {
+        fstream infile;
+        if (!fs::exists(fp)) return;
+        infile.open(fp, ios_base::in);
+        string line;
+        while (getline(infile, line)) {
+            push_back(line);
+        }
+    }
+    bool matches(string fp) {
+        for (string& pattern : *this) {
+            if (fnmatch(pattern.c_str(), fp.c_str(), FNM_PERIOD) == 0)
+                return true;
+        }
+        return false;
+    }
+};
+
 bool Narc::Pack(const fs::path& fileName, const fs::path& directory)
 {
 	ofstream ofs(fileName, ios::binary);
@@ -113,14 +138,22 @@ bool Narc::Pack(const fs::path& fileName, const fs::path& directory)
 	vector<FileAllocationTableEntry> fatEntries;
 	uint16_t directoryCounter = 1;
 
+	WildcardVector ignore_patterns(directory / ".knarcignore");
+	ignore_patterns.push_back(".*ignore");
+	ignore_patterns.push_back(".*keep");
+	WildcardVector keep_patterns(directory / ".knarckeep");
+
 	for (const auto& de : OrderedDirectoryIterator(directory, true))
 	{
 		if (is_directory(de))
 		{
 			++directoryCounter;
 		}
-		else
+		else if (keep_patterns.matches(de.path().filename()) || !ignore_patterns.matches(de.path().filename()))
 		{
+		    if (debug) {
+		        cerr << "DEBUG: adding file " << de.path() << endl;
+		    }
 			fatEntries.push_back(FileAllocationTableEntry
 				{
 					.Start = 0x0,
@@ -143,7 +176,7 @@ bool Narc::Pack(const fs::path& fileName, const fs::path& directory)
 
 	FileAllocationTable fat
 	{
-		.Id = 0x46415442,
+		.Id = 0x46415442, // BTAF
 		.ChunkSize = sizeof(FileAllocationTable) + ((uint32_t)fatEntries.size() * sizeof(FileAllocationTableEntry)),
 		.FileCount = static_cast<uint16_t>(fatEntries.size()),
 		.Reserved = 0x0
@@ -156,7 +189,7 @@ bool Narc::Pack(const fs::path& fileName, const fs::path& directory)
 
 	for (const auto& de : OrderedDirectoryIterator(directory, true))
 	{
-		if (!subTables.count(de.path().parent_path()))
+		if (!subTables.count(de.path().parent_path()) && (keep_patterns.matches(de.path().filename()) || !ignore_patterns.matches(de.path().filename())))
 		{
 			subTables.insert({ de.path().parent_path(), "" });
 			paths.push_back(de.path().parent_path());
@@ -171,7 +204,7 @@ bool Narc::Pack(const fs::path& fileName, const fs::path& directory)
 			subTables[de.path().parent_path()] += (0xF000 + directoryCounter) & 0xFF;
 			subTables[de.path().parent_path()] += (0xF000 + directoryCounter) >> 8;
 		}
-		else
+		else if (keep_patterns.matches(de.path().filename()) || !ignore_patterns.matches(de.path().filename()))
 		{
 			subTables[de.path().parent_path()] += static_cast<uint8_t>(de.path().filename().string().size());
 			subTables[de.path().parent_path()] += de.path().filename().string();
@@ -185,7 +218,7 @@ bool Narc::Pack(const fs::path& fileName, const fs::path& directory)
 
 	vector<FileNameTableEntry> fntEntries;
 
-	if (!regex_match(fs::directory_iterator(directory)->path().string(), regex(".*_\\d{4,8}\\.bin")))
+	if (!pack_no_fnt)
 	{
 		fntEntries.push_back(
 			{
@@ -231,11 +264,11 @@ bool Narc::Pack(const fs::path& fileName, const fs::path& directory)
 
 	FileNameTable fnt
 	{
-		.Id = 0x464E5442,
+		.Id = 0x464E5442, // BTNF
 		.ChunkSize = sizeof(FileNameTable) + (fntEntries.size() * sizeof(FileNameTableEntry))
 	};
 
-	if (!regex_match(fs::directory_iterator(directory)->path().string(), regex(".*_\\d{4,8}\\.bin")))
+	if (!pack_no_fnt)
 	{
 		for (const auto& subTable : subTables)
 		{
@@ -250,7 +283,7 @@ bool Narc::Pack(const fs::path& fileName, const fs::path& directory)
 
 	FileImages fi
 	{
-		.Id = 0x46494D47,
+		.Id = 0x46494D47, // GMIF
 		.ChunkSize = sizeof(FileImages) + fatEntries.back().End
 	};
 
@@ -261,7 +294,7 @@ bool Narc::Pack(const fs::path& fileName, const fs::path& directory)
 
 	Header header
 	{
-		.Id = 0x4352414E,
+		.Id = 0x4352414E, // NARC
 		.ByteOrderMark = 0xFFFE,
 		.Version = 0x100,
 		.FileSize = sizeof(Header) + fat.ChunkSize + fnt.ChunkSize + fi.ChunkSize,
@@ -284,7 +317,7 @@ bool Narc::Pack(const fs::path& fileName, const fs::path& directory)
 		ofs.write(reinterpret_cast<char*>(&entry), sizeof(FileNameTableEntry));
 	}
 
-	if (!regex_match(fs::directory_iterator(directory)->path().string(), regex(".*_\\d{4,8}\\.bin")))
+	if (!pack_no_fnt)
 	{
 		for (const auto& path : paths)
 		{
@@ -303,6 +336,11 @@ bool Narc::Pack(const fs::path& fileName, const fs::path& directory)
 			continue;
 		}
 
+		if (!(keep_patterns.matches(de.path().filename()) || !ignore_patterns.matches(de.path().filename())))
+        {
+		    continue;
+        }
+
 		ifstream ifs(de.path(), ios::binary | ios::ate);
 
 		if (!ifs.good())
diff --git a/tools/knarc/Source.cpp b/tools/knarc/Source.cpp
index 51bb5320..32756c69 100644
--- a/tools/knarc/Source.cpp
+++ b/tools/knarc/Source.cpp
@@ -6,6 +6,9 @@
 
 using namespace std;
 
+bool debug = false;
+bool pack_no_fnt = true;
+
 void PrintError(NarcError error)
 {
 	switch (error)
@@ -27,20 +30,20 @@ void PrintError(NarcError error)
 	}
 }
 
+static inline void usage() {
+    cout << "OVERVIEW: Knarc" << endl << endl;
+    cout << "USAGE: knarc [options] -d DIRECTORY [-p TARGET | -u SOURCE]" << endl << endl;
+    cout << "OPTIONS:" << endl;
+    cout << "\t-d DIRECTORY\tDirectory to pack from/unpack to" << endl;
+    cout << "\t-p TARGET\tPack to the target NARC" << endl;
+    cout << "\t-u SOURCE\tUnpack from the source NARC" << endl;
+    cout << "\t-n\tSuppress building the filename table" << endl;
+    cout << "\t-D/--debug\tPrint additional debug messages" << endl;
+    cout << "\t-h/--help\tPrint this message and exit" << endl;
+}
+
 int main(int argc, char* argv[])
 {
-	if (argc != 5)
-	{
-		cout << "OVERVIEW: Knarc" << endl << endl;
-		cout << "USAGE: knarc [options] <inputs>" << endl << endl;
-		cout << "OPTIONS:" << endl;
-		cout << "\t-d\tDirectory to pack from/unpack to" << endl;
-		cout << "\t-p\tPack" << endl;
-		cout << "\t-u\tUnpack" << endl;
-
-		return 1;
-	}
-
 	string directory = "";
 	string fileName = "";
 	bool pack = false;
@@ -56,6 +59,10 @@ int main(int argc, char* argv[])
 				return 1;
 			}
 
+			if (!directory.empty()) {
+			    cerr << "ERROR: Multiple directories specified" << endl;
+			    return 1;
+			}
 			directory = argv[++i];
 		}
 		else if (!strcmp(argv[i], "-p"))
@@ -67,6 +74,10 @@ int main(int argc, char* argv[])
 				return 1;
 			}
 
+            if (!fileName.empty()) {
+                cerr << "ERROR: Multiple files specified" << endl;
+                return 1;
+            }
 			fileName = argv[++i];
 			pack = true;
 		}
@@ -79,10 +90,36 @@ int main(int argc, char* argv[])
 				return 1;
 			}
 
+			if (!fileName.empty()) {
+			    cerr << "ERROR: Multiple files specified" << endl;
+			    return 1;
+			}
 			fileName = argv[++i];
+		} else if (!strcmp(argv[i], "-D") || !strcmp(argv[i], "--debug")) {
+		    debug = true;
+		} else if (!strcmp(argv[i], "-h") || !strcmp(argv[i], "--help")) {
+		    usage();
+		    return 0;
+		}
+		else if (!strcmp(argv[i], "-n")) {
+		    pack_no_fnt = false;
+		}
+		else {
+		    usage();
+		    cerr << "ERROR: Unrecognized argument: " << argv[i] << endl;
+		    return 1;
 		}
 	}
 
+	if (fileName.empty()) {
+	    cerr << "ERROR: Missing -u or -p" << endl;
+	    return 1;
+	}
+	if (directory.empty()) {
+	    cerr << "ERROR: Missing -d" << endl;
+	    return 1;
+	}
+
 	Narc narc;
 
 	if (pack)