From 1877374a92c50cfbb410e7b183d7ce8770d281ba Mon Sep 17 00:00:00 2001 From: PikalaxALT Date: Tue, 16 Jun 2020 17:03:56 -0400 Subject: KNARC: Add handling of .knarcignore, .knarckeep; fix up cli --- CMakeLists.txt | 4 +- Makefile | 1 + files/poketool/personal/growtbl/.knarcignore | 3 + files/poketool/personal/growtbl/Makefile | 19 ----- files/poketool/personal/growtbl/grow2bin.c | 103 --------------------------- filesystem.mk | 4 +- tools/csv2bin/Makefile | 19 +++++ tools/csv2bin/csv2bin.c | 103 +++++++++++++++++++++++++++ tools/knarc/Narc.cpp | 58 ++++++++++++--- tools/knarc/Source.cpp | 61 ++++++++++++---- 10 files changed, 228 insertions(+), 147 deletions(-) create mode 100644 files/poketool/personal/growtbl/.knarcignore delete mode 100644 files/poketool/personal/growtbl/Makefile delete mode 100644 files/poketool/personal/growtbl/grow2bin.c create mode 100644 tools/csv2bin/Makefile create mode 100644 tools/csv2bin/csv2bin.c diff --git a/CMakeLists.txt b/CMakeLists.txt index abf430db..243b0cca 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required (VERSION 2.8.11) project(PokeDiamond) -set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD 17) # TODO: Add commands @@ -15,3 +15,5 @@ target_include_directories(PokeDiamond PRIVATE include include-mw arm9/lib/inclu add_executable(calcrom .travis/calcrom/calcrom.cpp) target_include_directories(calcrom PRIVATE /usr/local/include) + +add_executable(knarc tools/knarc/Source.cpp tools/knarc/Narc.cpp) diff --git a/Makefile b/Makefile index 77352f41..92ef1be5 100644 --- a/Makefile +++ b/Makefile @@ -199,6 +199,7 @@ LDFLAGS = -map -nodead -w off -proc v5te -interworking -map -symtab -m _start # DS TOOLS TOOLS_DIR = tools SHA1SUM = sha1sum +CSV2BIN = $(TOOLS_DIR)/csv2bin/csv2bin JSONPROC = $(TOOLS_DIR)/jsonproc/jsonproc GFX = $(TOOLS_DIR)/nitrogfx/nitrogfx MWASMARM_PATCHER = $(TOOLS_DIR)/mwasmarm_patcher/mwasmarm_patcher$(EXE) -q diff --git a/files/poketool/personal/growtbl/.knarcignore b/files/poketool/personal/growtbl/.knarcignore new file mode 100644 index 00000000..d633d071 --- /dev/null +++ b/files/poketool/personal/growtbl/.knarcignore @@ -0,0 +1,3 @@ +*.txt +*.exe +grow2bin diff --git a/files/poketool/personal/growtbl/Makefile b/files/poketool/personal/growtbl/Makefile deleted file mode 100644 index 4bbb579f..00000000 --- a/files/poketool/personal/growtbl/Makefile +++ /dev/null @@ -1,19 +0,0 @@ -CC = gcc -CFLAGS = -O3 - -growth_rates_txt = $(wildcard *.txt) -growth_rates_bin = $(growth_rates_txt:%.txt=%.bin) - -.PHONY: all clean - -all: grow2bin $(growth_rates_bin) - @: - -clean: - $(RM) grow2bin $(growth_rates_bin) - -grow2bin: grow2bin.c - $(CC) $(CFLAGS) -o $@ $^ - -%.bin: %.txt - ./grow2bin $< diff --git a/files/poketool/personal/growtbl/grow2bin.c b/files/poketool/personal/growtbl/grow2bin.c deleted file mode 100644 index 5c8b5111..00000000 --- a/files/poketool/personal/growtbl/grow2bin.c +++ /dev/null @@ -1,103 +0,0 @@ -#include -#include -#include -#include -#include - -char * ReadWholeFile(char * filename) { - FILE * infile = fopen(filename, "rb"); - if (infile == NULL) return NULL; - fseek(infile, 0, SEEK_END); - size_t fsize = ftell(infile); - fseek(infile, 0, SEEK_SET); - char * ret = malloc(fsize); - if (ret != NULL) { - if (fread(ret, 1, fsize, infile) != fsize) { - free(ret); - ret = NULL; - } - } - fclose(infile); - return ret; -} - -int main(int argc, char ** argv) { - int width = 4; - char * buffer = NULL; - char * endptr = NULL; - FILE * outfile = NULL; - char * infname = NULL; - for (int i = 1; i < argc; i++) { - if (strcmp(argv[i], "-w") == 0 || strcmp(argv[i], "--width") == 0) { - ++i; - width = strtol(argv[i], &endptr, 10); - if (width == 0) { - fprintf(stderr, "invalid integer value for %s: %s\n", argv[i - 1], argv[i]); - return EXIT_FAILURE; - } - } else if (strcmp(argv[i], "-o") == 0 || strcmp(argv[i], "--output") == 0) { - ++i; - outfile = fopen(argv[i], "wb"); - if (outfile == NULL) { - fprintf(stderr, "unable to open file '%s' for writing\n", argv[i]); - return EXIT_FAILURE; - } - } else { - if (buffer == NULL) - { - buffer = ReadWholeFile(argv[i]); - } - if (buffer == NULL) - { - fprintf(stderr, "invalid argument: %s\n", argv[i]); - return EXIT_FAILURE; - } - infname = argv[i]; - } - } - if (buffer == NULL) { - fprintf(stderr, "missing required argument: INFILE\n"); - return EXIT_FAILURE; - } - if (outfile == NULL) { - char * infname_ext = strrchr(infname, '.'); - char * outfname = malloc(infname_ext - infname + 5); - char * outfname_ext = stpncpy(outfname, argv[1], infname_ext - argv[1]); - strcpy(outfname_ext, ".bin"); - outfile = fopen(outfname, "wb"); - if (outfile == NULL) { - fprintf(stderr, "unable to open file '%s' for writing\n", outfname); - return EXIT_FAILURE; - } - free(outfname); - } - uint32_t value; - size_t size = 0; - char * ptr = buffer; - char * end = NULL; - int lineno = 1; - int colno = 1; - while (1) { - while (isspace(*ptr) || *ptr == ',') { - if (*ptr == 0) break; - if (*ptr == '\n') - { lineno++; colno = 1; } - if (*ptr == ',') colno++; - ptr++; - }; - if (*ptr == 0) break; - value = strtoul(ptr, &end, 10); - if (value == 0 && ptr == end) { - fprintf(stderr, "syntax error: %d:%d\n", lineno, colno); - return EXIT_FAILURE; - } - if (fwrite(&value, 1, width, outfile) != width) { - fprintf(stderr, "write error\n"); - return EXIT_FAILURE; - } - ptr = end; - } - fclose(outfile); - free(buffer); - return EXIT_SUCCESS; -} diff --git a/filesystem.mk b/filesystem.mk index b43877db..f557e59d 100644 --- a/filesystem.mk +++ b/filesystem.mk @@ -283,5 +283,5 @@ HOSTFS_FILES = $(NITROFS_FILES:%=files/%) files/poketool/personal/pms.narc: ; files/poketool/personal/growtbl.narc: $(wildcard files/poketool/personal/growtbl/*.txt) - $(MAKE) -C $( +#include +#include +#include +#include + +char * ReadWholeFile(char * filename) { + FILE * infile = fopen(filename, "rb"); + if (infile == NULL) return NULL; + fseek(infile, 0, SEEK_END); + size_t fsize = ftell(infile); + fseek(infile, 0, SEEK_SET); + char * ret = malloc(fsize); + if (ret != NULL) { + if (fread(ret, 1, fsize, infile) != fsize) { + free(ret); + ret = NULL; + } + } + fclose(infile); + return ret; +} + +int main(int argc, char ** argv) { + int width = 4; + char * buffer = NULL; + char * endptr = NULL; + FILE * outfile = NULL; + char * infname = NULL; + for (int i = 1; i < argc; i++) { + if (strcmp(argv[i], "-w") == 0 || strcmp(argv[i], "--width") == 0) { + ++i; + width = strtol(argv[i], &endptr, 10); + if (width == 0) { + fprintf(stderr, "invalid integer value for %s: %s\n", argv[i - 1], argv[i]); + return EXIT_FAILURE; + } + } else if (strcmp(argv[i], "-o") == 0 || strcmp(argv[i], "--output") == 0) { + ++i; + outfile = fopen(argv[i], "wb"); + if (outfile == NULL) { + fprintf(stderr, "unable to open file '%s' for writing\n", argv[i]); + return EXIT_FAILURE; + } + } else { + if (buffer == NULL) + { + buffer = ReadWholeFile(argv[i]); + } + if (buffer == NULL) + { + fprintf(stderr, "invalid argument: %s\n", argv[i]); + return EXIT_FAILURE; + } + infname = argv[i]; + } + } + if (buffer == NULL) { + fprintf(stderr, "missing required argument: INFILE\n"); + return EXIT_FAILURE; + } + if (outfile == NULL) { + char * infname_ext = strrchr(infname, '.'); + char * outfname = malloc(infname_ext - infname + 5); + char * outfname_ext = stpncpy(outfname, argv[1], infname_ext - argv[1]); + strcpy(outfname_ext, ".bin"); + outfile = fopen(outfname, "wb"); + if (outfile == NULL) { + fprintf(stderr, "unable to open file '%s' for writing\n", outfname); + return EXIT_FAILURE; + } + free(outfname); + } + uint32_t value; + size_t size = 0; + char * ptr = buffer; + char * end = NULL; + int lineno = 1; + int colno = 1; + while (1) { + while (isspace(*ptr) || *ptr == ',') { + if (*ptr == 0) break; + if (*ptr == '\n') + { lineno++; colno = 1; } + if (*ptr == ',') colno++; + ptr++; + }; + if (*ptr == 0) break; + value = strtoul(ptr, &end, 10); + if (value == 0 && ptr == end) { + fprintf(stderr, "syntax error: %d:%d\n", lineno, colno); + return EXIT_FAILURE; + } + if (fwrite(&value, 1, width, outfile) != width) { + fprintf(stderr, "write error\n"); + return EXIT_FAILURE; + } + ptr = end; + } + fclose(outfile); + free(buffer); + return EXIT_SUCCESS; +} diff --git a/tools/knarc/Narc.cpp b/tools/knarc/Narc.cpp index 506e050f..00a9d0ed 100644 --- a/tools/knarc/Narc.cpp +++ b/tools/knarc/Narc.cpp @@ -13,6 +13,8 @@ #include #include #include +#include +#include #if __GNUC__ <= 7 #include @@ -24,6 +26,9 @@ namespace fs = std::filesystem; using namespace std; +extern bool debug; +extern bool pack_no_fnt; + void Narc::AlignDword(ofstream& ofs, uint8_t paddingChar) { if ((ofs.tellp() % 4) != 0) @@ -104,6 +109,26 @@ NarcError Narc::GetError() const return error; } +class WildcardVector : public vector { +public: + WildcardVector(fs::path fp) { + fstream infile; + if (!fs::exists(fp)) return; + infile.open(fp, ios_base::in); + string line; + while (getline(infile, line)) { + push_back(line); + } + } + bool matches(string fp) { + for (string& pattern : *this) { + if (fnmatch(pattern.c_str(), fp.c_str(), FNM_PERIOD) == 0) + return true; + } + return false; + } +}; + bool Narc::Pack(const fs::path& fileName, const fs::path& directory) { ofstream ofs(fileName, ios::binary); @@ -113,14 +138,22 @@ bool Narc::Pack(const fs::path& fileName, const fs::path& directory) vector fatEntries; uint16_t directoryCounter = 1; + WildcardVector ignore_patterns(directory / ".knarcignore"); + ignore_patterns.push_back(".*ignore"); + ignore_patterns.push_back(".*keep"); + WildcardVector keep_patterns(directory / ".knarckeep"); + for (const auto& de : OrderedDirectoryIterator(directory, true)) { if (is_directory(de)) { ++directoryCounter; } - else + else if (keep_patterns.matches(de.path().filename()) || !ignore_patterns.matches(de.path().filename())) { + if (debug) { + cerr << "DEBUG: adding file " << de.path() << endl; + } fatEntries.push_back(FileAllocationTableEntry { .Start = 0x0, @@ -143,7 +176,7 @@ bool Narc::Pack(const fs::path& fileName, const fs::path& directory) FileAllocationTable fat { - .Id = 0x46415442, + .Id = 0x46415442, // BTAF .ChunkSize = sizeof(FileAllocationTable) + ((uint32_t)fatEntries.size() * sizeof(FileAllocationTableEntry)), .FileCount = static_cast(fatEntries.size()), .Reserved = 0x0 @@ -156,7 +189,7 @@ bool Narc::Pack(const fs::path& fileName, const fs::path& directory) for (const auto& de : OrderedDirectoryIterator(directory, true)) { - if (!subTables.count(de.path().parent_path())) + if (!subTables.count(de.path().parent_path()) && (keep_patterns.matches(de.path().filename()) || !ignore_patterns.matches(de.path().filename()))) { subTables.insert({ de.path().parent_path(), "" }); paths.push_back(de.path().parent_path()); @@ -171,7 +204,7 @@ bool Narc::Pack(const fs::path& fileName, const fs::path& directory) subTables[de.path().parent_path()] += (0xF000 + directoryCounter) & 0xFF; subTables[de.path().parent_path()] += (0xF000 + directoryCounter) >> 8; } - else + else if (keep_patterns.matches(de.path().filename()) || !ignore_patterns.matches(de.path().filename())) { subTables[de.path().parent_path()] += static_cast(de.path().filename().string().size()); subTables[de.path().parent_path()] += de.path().filename().string(); @@ -185,7 +218,7 @@ bool Narc::Pack(const fs::path& fileName, const fs::path& directory) vector fntEntries; - if (!regex_match(fs::directory_iterator(directory)->path().string(), regex(".*_\\d{4,8}\\.bin"))) + if (!pack_no_fnt) { fntEntries.push_back( { @@ -231,11 +264,11 @@ bool Narc::Pack(const fs::path& fileName, const fs::path& directory) FileNameTable fnt { - .Id = 0x464E5442, + .Id = 0x464E5442, // BTNF .ChunkSize = sizeof(FileNameTable) + (fntEntries.size() * sizeof(FileNameTableEntry)) }; - if (!regex_match(fs::directory_iterator(directory)->path().string(), regex(".*_\\d{4,8}\\.bin"))) + if (!pack_no_fnt) { for (const auto& subTable : subTables) { @@ -250,7 +283,7 @@ bool Narc::Pack(const fs::path& fileName, const fs::path& directory) FileImages fi { - .Id = 0x46494D47, + .Id = 0x46494D47, // GMIF .ChunkSize = sizeof(FileImages) + fatEntries.back().End }; @@ -261,7 +294,7 @@ bool Narc::Pack(const fs::path& fileName, const fs::path& directory) Header header { - .Id = 0x4352414E, + .Id = 0x4352414E, // NARC .ByteOrderMark = 0xFFFE, .Version = 0x100, .FileSize = sizeof(Header) + fat.ChunkSize + fnt.ChunkSize + fi.ChunkSize, @@ -284,7 +317,7 @@ bool Narc::Pack(const fs::path& fileName, const fs::path& directory) ofs.write(reinterpret_cast(&entry), sizeof(FileNameTableEntry)); } - if (!regex_match(fs::directory_iterator(directory)->path().string(), regex(".*_\\d{4,8}\\.bin"))) + if (!pack_no_fnt) { for (const auto& path : paths) { @@ -303,6 +336,11 @@ bool Narc::Pack(const fs::path& fileName, const fs::path& directory) continue; } + if (!(keep_patterns.matches(de.path().filename()) || !ignore_patterns.matches(de.path().filename()))) + { + continue; + } + ifstream ifs(de.path(), ios::binary | ios::ate); if (!ifs.good()) diff --git a/tools/knarc/Source.cpp b/tools/knarc/Source.cpp index 51bb5320..32756c69 100644 --- a/tools/knarc/Source.cpp +++ b/tools/knarc/Source.cpp @@ -6,6 +6,9 @@ using namespace std; +bool debug = false; +bool pack_no_fnt = true; + void PrintError(NarcError error) { switch (error) @@ -27,20 +30,20 @@ void PrintError(NarcError error) } } +static inline void usage() { + cout << "OVERVIEW: Knarc" << endl << endl; + cout << "USAGE: knarc [options] -d DIRECTORY [-p TARGET | -u SOURCE]" << endl << endl; + cout << "OPTIONS:" << endl; + cout << "\t-d DIRECTORY\tDirectory to pack from/unpack to" << endl; + cout << "\t-p TARGET\tPack to the target NARC" << endl; + cout << "\t-u SOURCE\tUnpack from the source NARC" << endl; + cout << "\t-n\tSuppress building the filename table" << endl; + cout << "\t-D/--debug\tPrint additional debug messages" << endl; + cout << "\t-h/--help\tPrint this message and exit" << endl; +} + int main(int argc, char* argv[]) { - if (argc != 5) - { - cout << "OVERVIEW: Knarc" << endl << endl; - cout << "USAGE: knarc [options] " << endl << endl; - cout << "OPTIONS:" << endl; - cout << "\t-d\tDirectory to pack from/unpack to" << endl; - cout << "\t-p\tPack" << endl; - cout << "\t-u\tUnpack" << endl; - - return 1; - } - string directory = ""; string fileName = ""; bool pack = false; @@ -56,6 +59,10 @@ int main(int argc, char* argv[]) return 1; } + if (!directory.empty()) { + cerr << "ERROR: Multiple directories specified" << endl; + return 1; + } directory = argv[++i]; } else if (!strcmp(argv[i], "-p")) @@ -67,6 +74,10 @@ int main(int argc, char* argv[]) return 1; } + if (!fileName.empty()) { + cerr << "ERROR: Multiple files specified" << endl; + return 1; + } fileName = argv[++i]; pack = true; } @@ -79,10 +90,36 @@ int main(int argc, char* argv[]) return 1; } + if (!fileName.empty()) { + cerr << "ERROR: Multiple files specified" << endl; + return 1; + } fileName = argv[++i]; + } else if (!strcmp(argv[i], "-D") || !strcmp(argv[i], "--debug")) { + debug = true; + } else if (!strcmp(argv[i], "-h") || !strcmp(argv[i], "--help")) { + usage(); + return 0; + } + else if (!strcmp(argv[i], "-n")) { + pack_no_fnt = false; + } + else { + usage(); + cerr << "ERROR: Unrecognized argument: " << argv[i] << endl; + return 1; } } + if (fileName.empty()) { + cerr << "ERROR: Missing -u or -p" << endl; + return 1; + } + if (directory.empty()) { + cerr << "ERROR: Missing -d" << endl; + return 1; + } + Narc narc; if (pack) -- cgit v1.2.3