Merge pull request #388 from yenatch/scaninc

[dont merge] scaninc: read C includes and add -I

4 files changed, 178 insertions, 97 deletions

diff --git a/Makefile b/Makefile
index 5ef851b1c..9aca31899 100644
--- a/Makefile
+++ b/Makefile
@@ -114,7 +114,7 @@ $1_DATA_ASM_OBJS := $$(DATA_ASM_SRCS:%.s=build/$1/%.o)
 
 ifeq ($$(NODEP),)
 build/$1/src/%.o: c_path = $$(*D)/$$(*F).c
-build/$1/src/%.o: c_dep = $$(shell $$(SCANINC) $$(wildcard $$(c_path:build/$1/=)))
+build/$1/src/%.o: c_dep = $$(shell $$(SCANINC) -I include $$(wildcard $$(c_path:build/$1/=)))
 build/$1/asm/%.o: asm_dep = $$(shell $$(SCANINC) asm/$$(*F).s)
 build/$1/data/%.o: asm_dep = $$(shell $$(SCANINC) data/$$(*F).s)
 endif
diff --git a/tools/scaninc/c_file.cpp b/tools/scaninc/c_file.cpp
index b82276dd6..c55ca9a8c 100644
--- a/tools/scaninc/c_file.cpp
+++ b/tools/scaninc/c_file.cpp
@@ -45,8 +45,6 @@ CFile::CFile(std::string path)
 
     m_pos = 0;
     m_lineNum = 1;
-
-    RemoveComments();
 }
 
 CFile::~CFile()
@@ -54,71 +52,6 @@ CFile::~CFile()
     delete[] m_buffer;
 }
 
-// Removes comments to simplify further processing.
-// It stops upon encountering a null character,
-// which may or may not be the end of file marker.
-// If it's not, the error will be caught later.
-void CFile::RemoveComments()
-{
-    long pos = 0;
-    char stringChar = 0;
-
-    for (;;)
-    {
-        if (m_buffer[pos] == 0)
-            return;
-
-        if (stringChar != 0)
-        {
-            if (m_buffer[pos] == '\\' && m_buffer[pos + 1] == stringChar)
-            {
-                pos += 2;
-            }
-            else
-            {
-                if (m_buffer[pos] == stringChar)
-                    stringChar = 0;
-                pos++;
-            }
-        }
-        else if (m_buffer[pos] == '/' && m_buffer[pos + 1] == '/')
-        {
-            while (m_buffer[pos] != '\n' && m_buffer[pos] != 0)
-                m_buffer[pos++] = ' ';
-        }
-        else if (m_buffer[pos] == '/' && m_buffer[pos + 1] == '*')
-        {
-            m_buffer[pos++] = ' ';
-            m_buffer[pos++] = ' ';
-
-            for (;;)
-            {
-                if (m_buffer[pos] == 0)
-                    return;
-
-                if (m_buffer[pos] == '*' && m_buffer[pos + 1] == '/')
-                {
-                    m_buffer[pos++] = ' ';
-                    m_buffer[pos++] = ' ';
-                    break;
-                }
-                else
-                {
-                    if (m_buffer[pos] != '\n')
-                        m_buffer[pos] = ' ';
-                    pos++;
-                }
-            }
-        }
-        else
-        {
-            if (m_buffer[pos] == '"' || m_buffer[pos] == '\'')
-                stringChar = m_buffer[pos];
-            pos++;
-        }
-    }
-}
-
 void CFile::FindIncbins()
 {
     char stringChar = 0;
@@ -145,6 +78,8 @@ void CFile::FindIncbins()
         }
         else
         {
+            SkipWhitespace();
+            CheckInclude();
             CheckIncbin();
 
             if (m_pos >= m_size)
@@ -177,6 +112,13 @@ bool CFile::ConsumeHorizontalWhitespace()
 
 bool CFile::ConsumeNewline()
 {
+    if (m_buffer[m_pos] == '\n')
+    {
+        m_pos++;
+        m_lineNum++;
+        return true;
+    }
+
     if (m_buffer[m_pos] == '\r' && m_buffer[m_pos + 1] == '\n')
     {
         m_pos += 2;
@@ -184,10 +126,31 @@ bool CFile::ConsumeNewline()
         return true;
     }
 
-    if (m_buffer[m_pos] == '\n')
+    return false;
+}
+
+bool CFile::ConsumeComment()
+{
+    if (m_buffer[m_pos] == '/' && m_buffer[m_pos + 1] == '*')
     {
-        m_pos++;
-        m_lineNum++;
+        m_pos += 2;
+        while (m_buffer[m_pos] != '*' && m_buffer[m_pos + 1] != '/')
+        {
+            if (!ConsumeNewline())
+            {
+                m_pos++;
+            }
+        }
+        m_pos += 2;
+        return true;
+    }
+    else if (m_buffer[m_pos] == '/' && m_buffer[m_pos + 1] == '/')
+    {
+        m_pos += 2;
+        while (!ConsumeNewline())
+        {
+            m_pos++;
+        }
         return true;
     }
 
@@ -196,7 +159,7 @@ bool CFile::ConsumeNewline()
 
 void CFile::SkipWhitespace()
 {
-    while (ConsumeHorizontalWhitespace() || ConsumeNewline())
+    while (ConsumeHorizontalWhitespace() || ConsumeNewline() || ConsumeComment())
         ;
 }
 
@@ -211,8 +174,43 @@ bool CFile::CheckIdentifier(const std::string& ident)
     return (i == ident.length());
 }
 
+void CFile::CheckInclude()
+{
+    if (m_buffer[m_pos] != '#')
+        return;
+
+    std::string ident = "#include";
+
+    if (!CheckIdentifier(ident))
+    {
+        return;
+    }
+
+    m_pos += ident.length();
+
+    ConsumeHorizontalWhitespace();
+
+    std::string path = ReadPath();
+
+    if (!path.empty()) {
+        m_includes.emplace(path);
+    }
+}
+
 void CFile::CheckIncbin()
 {
+    // Optimization: assume most lines are not incbins
+    if (!(m_buffer[m_pos+0] == 'I'
+       && m_buffer[m_pos+1] == 'N'
+       && m_buffer[m_pos+2] == 'C'
+       && m_buffer[m_pos+3] == 'B'
+       && m_buffer[m_pos+4] == 'I'
+       && m_buffer[m_pos+5] == 'N'
+       && m_buffer[m_pos+6] == '_'))
+    {
+            return;
+    }
+
     std::string idents[6] = { "INCBIN_S8", "INCBIN_U8", "INCBIN_S16", "INCBIN_U16", "INCBIN_S32", "INCBIN_U32" };
     int incbinType = -1;
 
@@ -246,8 +244,28 @@ void CFile::CheckIncbin()
 
     SkipWhitespace();
 
+    std::string path = ReadPath();
+
+    SkipWhitespace();
+
+    if (m_buffer[m_pos] != ')')
+        FATAL_INPUT_ERROR("expected ')'");
+
+    m_pos++;
+
+    m_incbins.emplace(path);
+}
+
+std::string CFile::ReadPath()
+{
     if (m_buffer[m_pos] != '"')
-        FATAL_INPUT_ERROR("expected double quote");
+    {
+        if (m_buffer[m_pos] == '<')
+        {
+            return std::string();
+        }
+        FATAL_INPUT_ERROR("expected '\"' or '<'");
+    }
 
     m_pos++;
 
@@ -272,16 +290,7 @@ void CFile::CheckIncbin()
         m_pos++;
     }
 
-    std::string path(&m_buffer[startPos], m_pos - startPos);
-
-    m_pos++;
-
-    SkipWhitespace();
-
-    if (m_buffer[m_pos] != ')')
-        FATAL_INPUT_ERROR("expected ')'");
-
     m_pos++;
 
-    m_incbins.emplace(path);
+    return std::string(m_buffer + startPos, m_pos - 1 - startPos);
 }
diff --git a/tools/scaninc/c_file.h b/tools/scaninc/c_file.h
index 922cb4639..618901b85 100644
--- a/tools/scaninc/c_file.h
+++ b/tools/scaninc/c_file.h
@@ -33,6 +33,7 @@ public:
     ~CFile();
     void FindIncbins();
     const std::set<std::string>& GetIncbins() { return m_incbins; }
+    const std::set<std::string>& GetIncludes() { return m_includes; }
 
 private:
     char *m_buffer;
@@ -41,13 +42,16 @@ private:
     int m_lineNum;
     std::string m_path;
     std::set<std::string> m_incbins;
+    std::set<std::string> m_includes;
 
-    void RemoveComments();
     bool ConsumeHorizontalWhitespace();
     bool ConsumeNewline();
+    bool ConsumeComment();
     void SkipWhitespace();
     bool CheckIdentifier(const std::string& ident);
+    void CheckInclude();
     void CheckIncbin();
+    std::string ReadPath();
 };
 
 #endif // C_FILE_H
diff --git a/tools/scaninc/scaninc.cpp b/tools/scaninc/scaninc.cpp
index b6f7ba767..3dc221479 100644
--- a/tools/scaninc/scaninc.cpp
+++ b/tools/scaninc/scaninc.cpp
@@ -20,7 +20,8 @@
 
 #include <cstdio>
 #include <cstdlib>
-#include <stack>
+#include <list>
+#include <queue>
 #include <set>
 #include <string>
 #include "scaninc.h"
@@ -38,15 +39,49 @@ bool CanOpenFile(std::string path)
     return true;
 }
 
+const char *const USAGE = "Usage: scaninc [-I INCLUDE_PATH] FILE_PATH\n";
+
 int main(int argc, char **argv)
 {
-    if (argc < 2)
-        FATAL_ERROR("Usage: scaninc FILE_PATH\n");
-
-    std::stack<std::string> filesToProcess;
+    std::queue<std::string> filesToProcess;
     std::set<std::string> dependencies;
 
-    std::string initialPath(argv[1]);
+    std::list<std::string> includeDirs;
+
+    argc--;
+    argv++;
+
+    while (argc > 1)
+    {
+        std::string arg(argv[0]);
+        if (arg.substr(0, 2) == "-I")
+        {
+            std::string includeDir = arg.substr(2);
+            if (includeDir.empty())
+            {
+                argc--;
+                argv++;
+                includeDir = std::string(argv[0]);
+            }
+            if (includeDir.back() != '/')
+            {
+                includeDir += '/';
+            }
+            includeDirs.push_back(includeDir);
+        }
+        else
+        {
+            FATAL_ERROR(USAGE);
+        }
+        argc--;
+        argv++;
+    }
+
+    if (argc != 1) {
+        FATAL_ERROR(USAGE);
+    }
+
+    std::string initialPath(argv[0]);
 
     std::size_t pos = initialPath.find_last_of('.');
 
@@ -55,20 +90,53 @@ int main(int argc, char **argv)
 
     std::string extension = initialPath.substr(pos + 1);
 
-    if (extension == "c")
+    std::string srcDir("");
+    std::size_t slash = initialPath.rfind('/');
+    if (slash != std::string::npos)
     {
-        CFile file(initialPath);
+        srcDir = initialPath.substr(0, slash + 1);
+    }
+    includeDirs.push_back(srcDir);
+
+    if (extension == "c" || extension == "h")
+    {
+        filesToProcess.push(initialPath);
+
+        while (!filesToProcess.empty())
+        {
+            CFile file(filesToProcess.front());
+            filesToProcess.pop();
 
-        file.FindIncbins();
-        dependencies = file.GetIncbins();
+            file.FindIncbins();
+            for (auto incbin : file.GetIncbins())
+            {
+                dependencies.insert(incbin);
+            }
+            for (auto include : file.GetIncludes())
+            {
+                for (auto includeDir : includeDirs)
+                {
+                    std::string path(includeDir + include);
+                    if (CanOpenFile(path))
+                    {
+                        bool inserted = dependencies.insert(path).second;
+                        if (inserted)
+                        {
+                            filesToProcess.push(path);
+                        }
+                        break;
+                    }
+                }
+            }
+        }
     }
-    else if (extension == "s")
+    else if (extension == "s" || extension == "inc")
     {
-        filesToProcess.push(std::string(argv[1]));
+        filesToProcess.push(initialPath);
 
         while (!filesToProcess.empty())
         {
-            AsmFile file(filesToProcess.top());
+            AsmFile file(filesToProcess.front());
 
             filesToProcess.pop();