From e83467525bf1ffa4e16a44a8e8729380e77ec92a Mon Sep 17 00:00:00 2001 From: PikalaxALT Date: Sun, 13 Jun 2021 19:45:15 -0400 Subject: Migrate to github workflows and output symfiles --- .github/calcrom/calcrom.pl | 143 ++++++++++++++++++++++++++++++++++++++++++++ .github/calcrom/webhook.sh | 16 +++++ .github/workflows/build.yml | 105 ++++++++++++++++++++++++++++++++ .travis/calcrom/calcrom.pl | 143 -------------------------------------------- .travis/calcrom/webhook.sh | 16 ----- Makefile | 15 ++++- 6 files changed, 277 insertions(+), 161 deletions(-) create mode 100755 .github/calcrom/calcrom.pl create mode 100755 .github/calcrom/webhook.sh create mode 100644 .github/workflows/build.yml delete mode 100755 .travis/calcrom/calcrom.pl delete mode 100755 .travis/calcrom/webhook.sh diff --git a/.github/calcrom/calcrom.pl b/.github/calcrom/calcrom.pl new file mode 100755 index 000000000..1bc8bc74b --- /dev/null +++ b/.github/calcrom/calcrom.pl @@ -0,0 +1,143 @@ +#!/usr/bin/perl + +use IPC::Cmd qw[ run ]; + +(@ARGV == 1) + or die "ERROR: no map file specified.\n"; +open(my $file, $ARGV[0]) + or die "ERROR: could not open file '$ARGV[0]'.\n"; + +my $src = 0; +my $asm = 0; +my $srcdata = 0; +my $data = 0; +while (my $line = <$file>) +{ + if ($line =~ /^ \.(\w+)\s+0x[0-9a-f]+\s+(0x[0-9a-f]+) (\w+)\/.+\.o/) + { + my $section = $1; + my $size = hex($2); + my $dir = $3; + + if ($section =~ /text/) + { + if ($dir eq 'src') + { + $src += $size; + } + elsif ($dir eq 'asm') + { + $asm += $size; + } + } + elsif ($section =~ /rodata/) + { + if ($dir eq 'src') + { + $srcdata += $size; + } + elsif ($dir eq 'data') + { + $data += $size; + } + } + } +} + +(my $elffname = $ARGV[0]) =~ s/\.map/.elf/; + +# Note that the grep filters out all branch labels. It also requires a minimum +# line length of 5, to filter out a ton of generated symbols (like AcCn). No +# settings to nm seem to remove these symbols. Finally, nm prints out a separate +# entry for whenever a name appears in a file, not just where it's defined. uniq +# removes all the duplicate entries. +# +# +# You'd expect this to take a while, because of uniq. It runs in under a second, +# though. Uniq is pretty fast! +my $base_cmd = "nm $elffname | awk '{print \$3}' | grep '^[^_].\\{4\\}' | uniq"; + +# This looks for Unknown_, Unknown_, or sub_, followed by just numbers. Note that +# it matches even if stuff precedes the unknown, like sUnknown/gUnknown. +my $undoc_cmd = "grep '[Uu]nknown_[0-9a-fA-F]*\\|sub_[0-9a-fA-F]*'"; + +# This looks for every symbol with an address at the end of it. Some things are +# given a name based on their type / location, but still have an unknown purpose. +# For example, FooMap_EventScript_FFFFFFF. +my $partial_doc_cmd = "grep '_[0-28][0-9a-fA-F]\\{5,6\\}'"; + +my $count_cmd = "wc -l"; + +# It sucks that we have to run this three times, but I can't figure out how to get +# stdin working for subcommands in perl while still having a timeout. It's decently +# fast anyway. +my $total_syms_as_string; +(run ( + command => "$base_cmd | $count_cmd", + buffer => \$total_syms_as_string, + timeout => 60 +)) + or die "ERROR: Error while getting all symbols: $?"; + +my $undocumented_as_string; +(run ( + command => "$base_cmd | $undoc_cmd | $count_cmd", + buffer => \$undocumented_as_string, + timeout => 60 +)) + or die "ERROR: Error while filtering for undocumented symbols: $?"; + +my $partial_documented_as_string; +(run ( + command => "$base_cmd | $partial_doc_cmd | $count_cmd", + buffer => \$partial_documented_as_string, + timeout => 60 +)) + or die "ERROR: Error while filtering for partial symbols: $?"; + +# Performing addition on a string converts it to a number. Any string that fails +# to convert to a number becomes 0. So if our converted number is 0, but our string +# is nonzero, then the conversion was an error. +my $undocumented = $undocumented_as_string + 0; +(($undocumented != 0) and ($undocumented_as_string ne "0")) + or die "ERROR: Cannot convert string to num: '$undocumented_as_string'"; + +my $partial_documented = $partial_documented_as_string + 0; +(($partial_documented != 0) and ($partial_documented_as_string ne "0")) + or die "ERROR: Cannot convert string to num: '$partial_documented_as_string'"; + +my $total_syms = $total_syms_as_string + 0; +(($total_syms != 0) and ($total_syms_as_string ne "0")) + or die "ERROR: Cannot convert string to num: '$total_syms_as_string'"; + +($total_syms != 0) + or die "ERROR: No symbols found."; + +my $total = $src + $asm; +my $srcPct = sprintf("%.4f", 100 * $src / $total); +my $asmPct = sprintf("%.4f", 100 * $asm / $total); + +# partial_documented is double-counting the unknown_* and sub_* symbols. +$partial_documented = $partial_documented - $undocumented; + +my $documented = $total_syms - ($undocumented + $partial_documented); +my $docPct = sprintf("%.4f", 100 * $documented / $total_syms); +my $partialPct = sprintf("%.4f", 100 * $partial_documented / $total_syms); +my $undocPct = sprintf("%.4f", 100 * $undocumented / $total_syms); + +print "$total total bytes of code\n"; +print "$src bytes of code in src ($srcPct%)\n"; +print "$asm bytes of code in asm ($asmPct%)\n"; +print "\n"; +print "$total_syms total symbols\n"; +print "$documented symbols documented ($docPct%)\n"; +print "$partial_documented symbols partially documented ($partialPct%)\n"; +print "$undocumented symbols undocumented ($undocPct%)\n"; + +print "\n"; +my $dataTotal = $srcdata + $data; +my $srcDataPct = sprintf("%.4f", 100 * $srcdata / $dataTotal); +my $dataPct = sprintf("%.4f", 100 * $data / $dataTotal); +print "$dataTotal total bytes of data\n"; +print "$srcdata bytes of data in src ($srcDataPct%)\n"; +print "$data bytes of data in data ($dataPct%)\n"; diff --git a/.github/calcrom/webhook.sh b/.github/calcrom/webhook.sh new file mode 100755 index 000000000..86da74c87 --- /dev/null +++ b/.github/calcrom/webhook.sh @@ -0,0 +1,16 @@ +#!/bin/bash -ex + +# Only run this script if it's the master branch build. +if [[ "$TRAVIS_BRANCH" != "master" || "$TRAVIS_PULL_REQUEST" != "false" ]]; then + exit 0 +fi + +build_name=$1 +map_file=$build_name.map +if [ ! -f $map_file ]; then + echo "$map_file does not exist!" + exit 1 +fi + +output=$(perl $(dirname "$0")/calcrom.pl $build_name.map | sed -E ':a;N;$!ba;s/\r{0,1}\n/\\n/g') +curl -d "{\"username\": \"$CALCROM_DISCORD_WEBHOOK_USERNAME\", \"avatar_url\": \"$CALCROM_DISCORD_WEBHOOK_AVATAR_URL\", \"content\":\"\`\`\`$build_name progress:\\n$output\`\`\`\"}" -H "Content-Type: application/json" -X POST $CALCROM_DISCORD_WEBHOOK_URL diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 000000000..2dca9dd59 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,105 @@ +name: CI + +on: + push: + branches: [ master ] + pull_request: + +jobs: + build: + runs-on: ubuntu-18.04 + steps: + - name: Checkout + uses: actions/checkout@master + + - name: Checkout DE + uses: actions/checkout@master + with: + path: 'build_de' + + - name: Checkout Symbols + uses: actions/checkout@master + with: + path: 'symbols' + ref: symbols + + - name: Install binutils + run: sudo apt install gcc-arm-none-eabi binutils-arm-none-eabi + # build-essential, git, and libpng-dev are already installed + # gcc-arm-none-eabi is only needed for the modern build + # as an alternative to dkP + + - name: Install agbcc + run: | + git clone https://github.com/pret/agbcc.git + cd agbcc + ./build.sh + ./install.sh ../ + ./install.sh ../build_de + + - name: Compare + run: | + make -j${nproc} compare_ruby + make -j${nproc} compare_ruby_debug + make -j${nproc} compare_ruby_rev1 + make -j${nproc} compare_ruby_rev2 + make -j${nproc} compare_sapphire + make -j${nproc} compare_sapphire_debug + make -j${nproc} compare_sapphire_rev1 + make -j${nproc} compare_sapphire_rev2 + mv *.sym symbols + + - name: Modern + run: | + make -j${nproc} ruby_modern + make -j${nproc} ruby_debug_modern + make -j${nproc} ruby_rev1_modern + make -j${nproc} ruby_rev2_modern + make -j${nproc} sapphire_debug_modern + make -j${nproc} sapphire_modern + make -j${nproc} sapphire_rev1_modern + make -j${nproc} sapphire_rev2_modern + mv *.sym symbols + + - name: German Before + run: | + cd build_de + sh de_before.sh + + - name: German + run: | + cd build_de + make -j${nproc} compare_ruby_de + make -j${nproc} compare_ruby_de_debug + make -j${nproc} compare_ruby_de_rev1 + make -j${nproc} compare_sapphire_de + make -j${nproc} compare_sapphire_de_debug + make -j${nproc} compare_sapphire_de_rev1 + mv *.sym ../symbols + + - name: German Modern + run: | + cd build_de + make -j${nproc} ruby_de_modern + make -j${nproc} ruby_de_debug_modern + make -j${nproc} ruby_de_rev1_modern + make -j${nproc} sapphire_de_modern + make -j${nproc} sapphire_de_debug_modern + make -j${nproc} sapphire_de_rev1_modern + mv *.sym ../symbols + + - name: Webhook + if: ${{ github.event_name == 'push' }} + env: + CALCROM_DISCORD_WEBHOOK_USERNAME: OK + CALCROM_DISCORD_WEBHOOK_AVATAR_URL: https://i.imgur.com/38BQHdd.png + CALCROM_DISCORD_WEBHOOK_URL: ${{ secrets.CALCROM_DISCORD_WEBHOOK_URL }} + run: sh .github/calcrom/webhook.sh pokeruby + + - name: Update symfiles + if: ${{ github.event_name == 'push' }} + uses: EndBug/add-and-commit@v7 + with: + branch: symbols + cwd: "./symbols" + add: "*.sym" diff --git a/.travis/calcrom/calcrom.pl b/.travis/calcrom/calcrom.pl deleted file mode 100755 index 1bc8bc74b..000000000 --- a/.travis/calcrom/calcrom.pl +++ /dev/null @@ -1,143 +0,0 @@ -#!/usr/bin/perl - -use IPC::Cmd qw[ run ]; - -(@ARGV == 1) - or die "ERROR: no map file specified.\n"; -open(my $file, $ARGV[0]) - or die "ERROR: could not open file '$ARGV[0]'.\n"; - -my $src = 0; -my $asm = 0; -my $srcdata = 0; -my $data = 0; -while (my $line = <$file>) -{ - if ($line =~ /^ \.(\w+)\s+0x[0-9a-f]+\s+(0x[0-9a-f]+) (\w+)\/.+\.o/) - { - my $section = $1; - my $size = hex($2); - my $dir = $3; - - if ($section =~ /text/) - { - if ($dir eq 'src') - { - $src += $size; - } - elsif ($dir eq 'asm') - { - $asm += $size; - } - } - elsif ($section =~ /rodata/) - { - if ($dir eq 'src') - { - $srcdata += $size; - } - elsif ($dir eq 'data') - { - $data += $size; - } - } - } -} - -(my $elffname = $ARGV[0]) =~ s/\.map/.elf/; - -# Note that the grep filters out all branch labels. It also requires a minimum -# line length of 5, to filter out a ton of generated symbols (like AcCn). No -# settings to nm seem to remove these symbols. Finally, nm prints out a separate -# entry for whenever a name appears in a file, not just where it's defined. uniq -# removes all the duplicate entries. -# -# -# You'd expect this to take a while, because of uniq. It runs in under a second, -# though. Uniq is pretty fast! -my $base_cmd = "nm $elffname | awk '{print \$3}' | grep '^[^_].\\{4\\}' | uniq"; - -# This looks for Unknown_, Unknown_, or sub_, followed by just numbers. Note that -# it matches even if stuff precedes the unknown, like sUnknown/gUnknown. -my $undoc_cmd = "grep '[Uu]nknown_[0-9a-fA-F]*\\|sub_[0-9a-fA-F]*'"; - -# This looks for every symbol with an address at the end of it. Some things are -# given a name based on their type / location, but still have an unknown purpose. -# For example, FooMap_EventScript_FFFFFFF. -my $partial_doc_cmd = "grep '_[0-28][0-9a-fA-F]\\{5,6\\}'"; - -my $count_cmd = "wc -l"; - -# It sucks that we have to run this three times, but I can't figure out how to get -# stdin working for subcommands in perl while still having a timeout. It's decently -# fast anyway. -my $total_syms_as_string; -(run ( - command => "$base_cmd | $count_cmd", - buffer => \$total_syms_as_string, - timeout => 60 -)) - or die "ERROR: Error while getting all symbols: $?"; - -my $undocumented_as_string; -(run ( - command => "$base_cmd | $undoc_cmd | $count_cmd", - buffer => \$undocumented_as_string, - timeout => 60 -)) - or die "ERROR: Error while filtering for undocumented symbols: $?"; - -my $partial_documented_as_string; -(run ( - command => "$base_cmd | $partial_doc_cmd | $count_cmd", - buffer => \$partial_documented_as_string, - timeout => 60 -)) - or die "ERROR: Error while filtering for partial symbols: $?"; - -# Performing addition on a string converts it to a number. Any string that fails -# to convert to a number becomes 0. So if our converted number is 0, but our string -# is nonzero, then the conversion was an error. -my $undocumented = $undocumented_as_string + 0; -(($undocumented != 0) and ($undocumented_as_string ne "0")) - or die "ERROR: Cannot convert string to num: '$undocumented_as_string'"; - -my $partial_documented = $partial_documented_as_string + 0; -(($partial_documented != 0) and ($partial_documented_as_string ne "0")) - or die "ERROR: Cannot convert string to num: '$partial_documented_as_string'"; - -my $total_syms = $total_syms_as_string + 0; -(($total_syms != 0) and ($total_syms_as_string ne "0")) - or die "ERROR: Cannot convert string to num: '$total_syms_as_string'"; - -($total_syms != 0) - or die "ERROR: No symbols found."; - -my $total = $src + $asm; -my $srcPct = sprintf("%.4f", 100 * $src / $total); -my $asmPct = sprintf("%.4f", 100 * $asm / $total); - -# partial_documented is double-counting the unknown_* and sub_* symbols. -$partial_documented = $partial_documented - $undocumented; - -my $documented = $total_syms - ($undocumented + $partial_documented); -my $docPct = sprintf("%.4f", 100 * $documented / $total_syms); -my $partialPct = sprintf("%.4f", 100 * $partial_documented / $total_syms); -my $undocPct = sprintf("%.4f", 100 * $undocumented / $total_syms); - -print "$total total bytes of code\n"; -print "$src bytes of code in src ($srcPct%)\n"; -print "$asm bytes of code in asm ($asmPct%)\n"; -print "\n"; -print "$total_syms total symbols\n"; -print "$documented symbols documented ($docPct%)\n"; -print "$partial_documented symbols partially documented ($partialPct%)\n"; -print "$undocumented symbols undocumented ($undocPct%)\n"; - -print "\n"; -my $dataTotal = $srcdata + $data; -my $srcDataPct = sprintf("%.4f", 100 * $srcdata / $dataTotal); -my $dataPct = sprintf("%.4f", 100 * $data / $dataTotal); -print "$dataTotal total bytes of data\n"; -print "$srcdata bytes of data in src ($srcDataPct%)\n"; -print "$data bytes of data in data ($dataPct%)\n"; diff --git a/.travis/calcrom/webhook.sh b/.travis/calcrom/webhook.sh deleted file mode 100755 index 86da74c87..000000000 --- a/.travis/calcrom/webhook.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -ex - -# Only run this script if it's the master branch build. -if [[ "$TRAVIS_BRANCH" != "master" || "$TRAVIS_PULL_REQUEST" != "false" ]]; then - exit 0 -fi - -build_name=$1 -map_file=$build_name.map -if [ ! -f $map_file ]; then - echo "$map_file does not exist!" - exit 1 -fi - -output=$(perl $(dirname "$0")/calcrom.pl $build_name.map | sed -E ':a;N;$!ba;s/\r{0,1}\n/\\n/g') -curl -d "{\"username\": \"$CALCROM_DISCORD_WEBHOOK_USERNAME\", \"avatar_url\": \"$CALCROM_DISCORD_WEBHOOK_AVATAR_URL\", \"content\":\"\`\`\`$build_name progress:\\n$output\`\`\`\"}" -H "Content-Type: application/json" -X POST $CALCROM_DISCORD_WEBHOOK_URL diff --git a/Makefile b/Makefile index d4fcffb80..f2e6fdfcb 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,7 @@ OBJCOPY := $(PREFIX)objcopy CC := $(PREFIX)gcc AS := $(PREFIX)as endif +NM := $(PREFIX)nm include config.mk ifeq ($(OS),Windows_NT) @@ -60,6 +61,7 @@ endif ROM := poke$(BUILD_NAME).gba MAP := $(ROM:%.gba=%.map) +SYM := $(ROM:%.gba=%.sym) BUILD_DIR := build/$(BUILD_NAME) @@ -109,7 +111,7 @@ ALL_BUILDS := ruby ruby_debug ruby_rev1 ruby_rev2 sapphire sapphire_debug sapphi MODERN_BUILDS := $(ALL_BUILDS:%=%_modern) # Available targets -.PHONY: all clean mostlyclean tidy tools $(ALL_BUILDS) +.PHONY: all clean mostlyclean tidy tools syms $(ALL_BUILDS) infoshell = $(foreach line, $(shell $1 | sed "s/ /__SPACE__/g"), $(info $(subst __SPACE__, ,$(line)))) @@ -144,13 +146,15 @@ $(shell mkdir -p $(SUBDIRS)) AUTO_GEN_TARGETS := -all: $(ROM) +all: $(ROM) $(SYM) ifeq ($(COMPARE),1) @$(SHA1SUM) $(BUILD_NAME).sha1 endif compare: ; @$(MAKE) COMPARE=1 +syms: $(SYM) + mostlyclean: tidy find sound/direct_sound_samples \( -iname '*.bin' \) -exec rm {} + $(RM) $(ALL_OBJECTS) @@ -293,3 +297,10 @@ sound/%.bin: sound/%.aif sound/songs/%.s: sound/songs/%.mid cd $(@D) && ../../$(MID2AGB) $( $@ -- cgit v1.2.3