Merge pull request #823 from PikalaxALT/push_symbols

Use github workflows instead of travis; make symfiles; fix debug modern builds

3 files changed, 264 insertions, 0 deletions

diff --git a/.github/calcrom/calcrom.pl b/.github/calcrom/calcrom.pl
new file mode 100755
index 000000000..1bc8bc74b
--- /dev/null
+++ b/.github/calcrom/calcrom.pl
@@ -0,0 +1,143 @@
+#!/usr/bin/perl
+
+use IPC::Cmd qw[ run ];
+
+(@ARGV == 1)
+    or die "ERROR: no map file specified.\n";
+open(my $file, $ARGV[0])
+    or die "ERROR: could not open file '$ARGV[0]'.\n";
+
+my $src = 0;
+my $asm = 0;
+my $srcdata = 0;
+my $data = 0;
+while (my $line = <$file>)
+{
+    if ($line =~ /^ \.(\w+)\s+0x[0-9a-f]+\s+(0x[0-9a-f]+) (\w+)\/.+\.o/)
+    {
+        my $section = $1;
+        my $size = hex($2);
+        my $dir = $3;
+
+        if ($section =~ /text/)
+        {
+            if ($dir eq 'src')
+            {
+                $src += $size;
+            }
+            elsif ($dir eq 'asm')
+            {
+                $asm += $size;
+            }
+        }
+        elsif ($section =~ /rodata/)
+        {
+            if ($dir eq 'src')
+            {
+                $srcdata += $size;
+            }
+            elsif ($dir eq 'data')
+            {
+                $data += $size;
+            }
+        }
+    }
+}
+
+(my $elffname = $ARGV[0]) =~ s/\.map/.elf/;
+
+# Note that the grep filters out all branch labels. It also requires a minimum
+# line length of 5, to filter out a ton of generated symbols (like AcCn). No
+# settings to nm seem to remove these symbols. Finally, nm prints out a separate
+# entry for whenever a name appears in a file, not just where it's defined. uniq
+# removes all the duplicate entries.
+#
+#
+# You'd expect this to take a while, because of uniq. It runs in under a second,
+# though. Uniq is pretty fast!
+my $base_cmd = "nm $elffname | awk '{print \$3}' | grep '^[^_].\\{4\\}' | uniq";
+
+# This looks for Unknown_, Unknown_, or sub_, followed by just numbers. Note that
+# it matches even if stuff precedes the unknown, like sUnknown/gUnknown.
+my $undoc_cmd = "grep '[Uu]nknown_[0-9a-fA-F]*\\|sub_[0-9a-fA-F]*'";
+
+# This looks for every symbol with an address at the end of it. Some things are
+# given a name based on their type / location, but still have an unknown purpose.
+# For example, FooMap_EventScript_FFFFFFF.
+my $partial_doc_cmd = "grep '_[0-28][0-9a-fA-F]\\{5,6\\}'";
+
+my $count_cmd = "wc -l";
+
+# It sucks that we have to run this three times, but I can't figure out how to get
+# stdin working for subcommands in perl while still having a timeout. It's decently
+# fast anyway.
+my $total_syms_as_string;
+(run (
+    command => "$base_cmd | $count_cmd",
+    buffer => \$total_syms_as_string,
+    timeout => 60
+))
+    or die "ERROR: Error while getting all symbols: $?";
+
+my $undocumented_as_string;
+(run (
+    command => "$base_cmd | $undoc_cmd | $count_cmd",
+    buffer => \$undocumented_as_string,
+    timeout => 60
+))
+    or die "ERROR: Error while filtering for undocumented symbols: $?";
+
+my $partial_documented_as_string;
+(run (
+    command => "$base_cmd | $partial_doc_cmd | $count_cmd",
+    buffer => \$partial_documented_as_string,
+    timeout => 60
+))
+    or die "ERROR: Error while filtering for partial symbols: $?";
+
+# Performing addition on a string converts it to a number. Any string that fails
+# to convert to a number becomes 0. So if our converted number is 0, but our string
+# is nonzero, then the conversion was an error.
+my $undocumented = $undocumented_as_string + 0;
+(($undocumented != 0) and ($undocumented_as_string ne "0"))
+    or die "ERROR: Cannot convert string to num: '$undocumented_as_string'";
+
+my $partial_documented = $partial_documented_as_string + 0;
+(($partial_documented != 0) and ($partial_documented_as_string ne "0"))
+	or die "ERROR: Cannot convert string to num: '$partial_documented_as_string'";
+
+my $total_syms = $total_syms_as_string + 0;
+(($total_syms != 0) and ($total_syms_as_string ne "0"))
+    or die "ERROR: Cannot convert string to num: '$total_syms_as_string'";
+
+($total_syms != 0)
+    or die "ERROR: No symbols found.";
+
+my $total = $src + $asm;
+my $srcPct = sprintf("%.4f", 100 * $src / $total);
+my $asmPct = sprintf("%.4f", 100 * $asm / $total);
+
+# partial_documented is double-counting the unknown_* and sub_* symbols.
+$partial_documented = $partial_documented - $undocumented;
+
+my $documented = $total_syms - ($undocumented + $partial_documented);
+my $docPct = sprintf("%.4f", 100 * $documented / $total_syms);
+my $partialPct = sprintf("%.4f", 100 * $partial_documented / $total_syms);
+my $undocPct = sprintf("%.4f", 100 * $undocumented / $total_syms);
+
+print "$total total bytes of code\n";
+print "$src bytes of code in src ($srcPct%)\n";
+print "$asm bytes of code in asm ($asmPct%)\n";
+print "\n";
+print "$total_syms total symbols\n";
+print "$documented symbols documented ($docPct%)\n";
+print "$partial_documented symbols partially documented ($partialPct%)\n";
+print "$undocumented symbols undocumented ($undocPct%)\n";
+
+print "\n";
+my $dataTotal = $srcdata + $data;
+my $srcDataPct = sprintf("%.4f", 100 * $srcdata / $dataTotal);
+my $dataPct = sprintf("%.4f", 100 * $data / $dataTotal);
+print "$dataTotal total bytes of data\n";
+print "$srcdata bytes of data in src ($srcDataPct%)\n";
+print "$data bytes of data in data ($dataPct%)\n";
diff --git a/.github/calcrom/webhook.sh b/.github/calcrom/webhook.sh
new file mode 100755
index 000000000..86da74c87
--- /dev/null
+++ b/.github/calcrom/webhook.sh
@@ -0,0 +1,16 @@
+#!/bin/bash -ex
+
+# Only run this script if it's the master branch build.
+if [[ "$TRAVIS_BRANCH" != "master" || "$TRAVIS_PULL_REQUEST" != "false" ]]; then
+  exit 0
+fi
+
+build_name=$1
+map_file=$build_name.map
+if [ ! -f $map_file ]; then
+  echo "$map_file does not exist!"
+  exit 1
+fi
+
+output=$(perl $(dirname "$0")/calcrom.pl $build_name.map | sed -E ':a;N;$!ba;s/\r{0,1}\n/\\n/g')
+curl -d "{\"username\": \"$CALCROM_DISCORD_WEBHOOK_USERNAME\", \"avatar_url\": \"$CALCROM_DISCORD_WEBHOOK_AVATAR_URL\", \"content\":\"\`\`\`$build_name progress:\\n$output\`\`\`\"}" -H "Content-Type: application/json" -X POST $CALCROM_DISCORD_WEBHOOK_URL
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
new file mode 100644
index 000000000..5ea79be17
--- /dev/null
+++ b/.github/workflows/build.yml
@@ -0,0 +1,105 @@
+name: CI
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+
+jobs:
+  build:
+    runs-on: ubuntu-18.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@master
+
+      - name: Checkout DE
+        uses: actions/checkout@master
+        with:
+          path: 'build_de'
+
+      - name: Checkout Symbols
+        uses: actions/checkout@master
+        with:
+          path: 'symbols'
+          ref: symbols
+
+      - name: Install binutils
+        run: sudo apt install gcc-arm-none-eabi binutils-arm-none-eabi
+        # build-essential, git, and libpng-dev are already installed
+        # gcc-arm-none-eabi is only needed for the modern build
+        # as an alternative to dkP
+
+      - name: Install agbcc
+        run: |
+          git clone https://github.com/pret/agbcc.git
+          cd agbcc 
+          ./build.sh
+          ./install.sh ../
+          ./install.sh ../build_de
+
+      - name: Compare
+        run: |
+          make -j${nproc} compare_ruby
+          make -j${nproc} compare_ruby_debug
+          make -j${nproc} compare_ruby_rev1
+          make -j${nproc} compare_ruby_rev2
+          make -j${nproc} compare_sapphire
+          make -j${nproc} compare_sapphire_debug
+          make -j${nproc} compare_sapphire_rev1
+          make -j${nproc} compare_sapphire_rev2
+          mv *.sym symbols
+
+      - name: Modern
+        run: |
+          make -j${nproc} ruby_modern
+          make -j${nproc} ruby_debug_modern
+          make -j${nproc} ruby_rev1_modern
+          make -j${nproc} ruby_rev2_modern
+          make -j${nproc} sapphire_modern
+          make -j${nproc} sapphire_debug_modern
+          make -j${nproc} sapphire_rev1_modern
+          make -j${nproc} sapphire_rev2_modern
+          mv *.sym symbols
+
+      - name: German Before
+        run: |
+          cd build_de
+          sh de_before.sh
+
+      - name: German
+        run: |
+          cd build_de
+          make -j${nproc} compare_ruby_de
+          make -j${nproc} compare_ruby_de_debug
+          make -j${nproc} compare_ruby_de_rev1
+          make -j${nproc} compare_sapphire_de
+          make -j${nproc} compare_sapphire_de_debug
+          make -j${nproc} compare_sapphire_de_rev1
+          mv *.sym ../symbols
+
+      - name: German Modern
+        run: |
+          cd build_de
+          make -j${nproc} ruby_de_modern
+          make -j${nproc} ruby_de_debug_modern
+          make -j${nproc} ruby_de_rev1_modern
+          make -j${nproc} sapphire_de_modern
+          make -j${nproc} sapphire_de_debug_modern
+          make -j${nproc} sapphire_de_rev1_modern
+          mv *.sym ../symbols
+
+      - name: Webhook
+        if: ${{ github.event_name == 'push' }}
+        env:
+          CALCROM_DISCORD_WEBHOOK_USERNAME: OK
+          CALCROM_DISCORD_WEBHOOK_AVATAR_URL: https://i.imgur.com/38BQHdd.png
+          CALCROM_DISCORD_WEBHOOK_URL: ${{ secrets.CALCROM_DISCORD_WEBHOOK_URL }}
+        run: sh .github/calcrom/webhook.sh pokeruby
+
+      - name: Update symfiles
+        if: ${{ github.event_name == 'push' }}
+        uses: EndBug/add-and-commit@v7
+        with:
+          branch: symbols
+          cwd: "./symbols"
+          add: "*.sym"