Release binaries for 68000, 68020 and 68040, geomean, command-line order

.github/workflows/binaries.yml

@@ -0,0 +1,51 @@
+name: Binaries
+
+# Builds benchwork-000, benchwork-020 and benchwork-040 with a pinned
+# m68k-amigaos-gcc release and uploads them as one artifact. The compiler
+# version is part of what a benchmark binary measures, so it is fixed here
+# rather than taken from whatever is current; bump it deliberately.
+on:
+  workflow_call:
+
+env:
+  TOOLCHAIN: 16.2-rc13
+
+jobs:
+  binaries:
+    runs-on: ubuntu-24.04
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v7
+        with:
+          # git describe needs the tags for the version banner
+          fetch-depth: 0
+
+      - name: Cache the toolchain
+        id: cache
+        uses: actions/cache@v6
+        with:
+          path: ~/m68k-amigaos-gcc-16.2
+          key: m68k-amigaos-gcc-${{ env.TOOLCHAIN }}-x86_64-linux
+
+      - name: Download the toolchain
+        if: steps.cache.outputs.cache-hit != 'true'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh release download "v$TOOLCHAIN" -R AmigaPorts/m68k-amigaos-gcc \
+            -p "m68k-amigaos-gcc-$TOOLCHAIN-x86_64-linux.tar.xz"
+          tar -xJf m68k-amigaos-gcc-*.tar.xz -C "$HOME"
+          rm m68k-amigaos-gcc-*.tar.xz
+
+      - name: Build the release binaries
+        run: |
+          export PATH="$HOME/m68k-amigaos-gcc-16.2/bin:$PATH"
+          m68k-amigaos-gcc --version | head -1
+          make -j "$(nproc)" release
+
+      - name: Upload the binaries
+        uses: actions/upload-artifact@v7
+        with:
+          name: benchwork
+          path: benchwork-0*

.github/workflows/build.yml

@@ -0,0 +1,14 @@
+name: Build
+
+# The release binaries on every push and pull request, so a tag push is never
+# the first build of them; release.yml publishes the same binaries for a tag.
+on:
+  push:
+    branches:
+      - '**'
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  build:
+    uses: ./.github/workflows/binaries.yml

.github/workflows/release.yml

@@ -0,0 +1,38 @@
+name: Release
+
+on:
+  push:
+    tags:
+      - 'v*'
+
+jobs:
+  build:
+    uses: ./.github/workflows/binaries.yml
+
+  release:
+    needs: build
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: write
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v7
+        with:
+          fetch-depth: 0
+
+      - name: Download the binaries
+        uses: actions/download-artifact@v8
+        with:
+          name: benchwork
+
+      # Bare executables, not archives: one download, copied to the Amiga
+      # as is. The tag message, subject and body, is the release text;
+      # write it with "git tag -a" or "git tag -s".
+      - name: Create release
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          TAG: ${{ github.ref_name }}
+        run: |
+          gh release create "$TAG" benchwork-0* \
+            --title "BenchWork ${TAG#v}" --notes-from-tag

Makefile

@@ -1,6 +1,11 @@
 # SPDX-License-Identifier: 0BSD
 
-CC ?= m68k-amigaos-gcc
+# make predefines CC as "cc", so "?=" would never apply; override only that
+# built-in default and keep a CC given on the command line or in the
+# environment.
+ifeq ($(origin CC),default)
+CC = m68k-amigaos-gcc
+endif
 BUILD ?= build
 TARGET = benchwork
 
@@ -11,6 +16,7 @@ CFLAGS ?= $(OPT) $(CPUFLAGS)
 # What the binary reports it was built with: taken now, before the
 # per-directory additions below.
 REPORTED_CFLAGS := $(CFLAGS)
+VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo unknown)
 
 # The harness is held to a stricter standard than the code it measures,
 # which is compiled the way its own build systems compile it.
@@ -19,7 +25,8 @@ WARNINGS = -Wall -Wextra -Wshadow -Wpointer-arith -Wwrite-strings \
 
 INCLUDES = -Isrc -Ithird_party/zlib -Ithird_party/libpng
 
-ALL_CFLAGS = $(CFLAGS) -noixemul $(INCLUDES) -DBENCH_CFLAGS='"$(REPORTED_CFLAGS)"'
+ALL_CFLAGS = $(CFLAGS) -noixemul $(INCLUDES) -DBENCH_CFLAGS='"$(REPORTED_CFLAGS)"' \
+	-DBENCH_VERSION='"$(VERSION)"'
 
 HARNESS_SRCS = \
 	src/main.c \
@@ -105,9 +112,22 @@ FONT ?= /usr/share/fonts/dejavu-sans-fonts/DejaVuSans.ttf
 glyphs: tools/dumpglyphs
 	tools/dumpglyphs $(FONT) > src/glyphs.c
 
+# The release binaries: one per CPU generation, named after it. The 68040 one
+# uses the FPU; the others are soft float, as a program shipped for those
+# machines would be.
+RELEASE_CPUS = 000 020 040
+CPUFLAGS_000 = -m68000
+CPUFLAGS_020 = -m68020
+CPUFLAGS_040 = -m68040 -mhard-float
+
+release: $(RELEASE_CPUS:%=release-%)
+
+$(RELEASE_CPUS:%=release-%): release-%:
+	$(MAKE) BUILD=build-$* TARGET=benchwork-$* CPUFLAGS="$(CPUFLAGS_$*)"
+
 clean:
-	rm -rf $(BUILD) $(TARGET)
+	rm -rf $(BUILD) $(TARGET) $(RELEASE_CPUS:%=build-%) $(RELEASE_CPUS:%=benchwork-%)
 
 -include $(OBJS:.o=.d)
 
-.PHONY: all clean glyphs
+.PHONY: all clean glyphs release $(RELEASE_CPUS:%=release-%)

README.md

@@ -40,9 +40,8 @@ The FreeType rasterizer keeps its cell pool on the stack, so the program
 refuses to start on the shell's default 4 KiB stack: `stack 65536` first.
 
 Each benchmark reports its fastest and mean iteration in milliseconds, and a
-checksum of its output that must be identical across compilers. Run it under
-a cycle-paced emulator or on real hardware with nothing else running; the
-timer is `ReadEClock()`.
+checksum of its output that must be identical across compilers. The timer is
+`ReadEClock()`.
 
 ## License
 

src/main.c

@@ -14,6 +14,7 @@
 #include <proto/exec.h>
 #include <exec/tasks.h>
 #include <devices/timer.h>
+#include <math.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -24,6 +25,9 @@
 #ifndef BENCH_CFLAGS
 #define BENCH_CFLAGS "?"
 #endif
+#ifndef BENCH_VERSION
+#define BENCH_VERSION "?"
+#endif
 
 // ftgrays keeps its 16 KiB cell pool on the stack and the Amiga shell's
 // default stack is 4 KiB. Swapping to a bigger one from C is not portable
@@ -73,8 +77,8 @@ static void print_ms(ULONG us) {
 }
 
 // Set up, time and tear down one benchmark. Returns false on failure, after
-// saying which step failed.
-static bool run_bench(const struct bench *b, int iters) {
+// saying which step failed; on success *BEST_US is the fastest iteration.
+static bool run_bench(const struct bench *b, int iters, ULONG *best_us) {
     ULONG best = ~0UL, total = 0, check = 0, first = 0;
     bool ok = true;
 
@@ -120,14 +124,18 @@ static bool run_bench(const struct bench *b, int iters) {
         print_ms(best);
         print_ms(total / (ULONG)iters);
         printf("   %08lx\n", (unsigned long)check);
+        *best_us = best;
     }
     if (b->teardown)
         b->teardown();
     return ok;
 }
 
 static int iters = 3;
-static bool selected[NBENCHES], any;
+// The benchmarks to run, in the order given on the command line, repeats
+// included; all of them, in table order, when none is named.
+static const struct bench **selected;
+static size_t nselected;
 static bool fastmem;
 
 // Allocate a benchmark buffer, released with bench_free(). WHAT names it if
@@ -167,15 +175,28 @@ void bench_zfree(void *opaque, void *address) {
     bench_free(address);
 }
 
-// Run the selected benchmarks. Returns the failure count.
+// Run the selected benchmarks, then the geometric mean of their fastest
+// iterations when more than one ran: the one number that weighs a 10%
+// change on a short workload the same as on a long one. Returns the failure
+// count.
 static int run_all(void) {
-    int failures = 0;
+    int failures = 0, ran = 0;
+    double logsum = 0;
 
-    for (size_t j = 0; j < NBENCHES; j++) {
-        if (any && !selected[j])
-            continue;
-        if (!run_bench(BENCHES[j], iters))
+    for (size_t j = 0; j < nselected; j++) {
+        ULONG best;
+
+        if (!run_bench(selected[j], iters, &best)) {
             failures++;
+            continue;
+        }
+        logsum += log((double)best);
+        ran++;
+    }
+    if (ran > 1) {
+        printf("%-12s %5s ", "geomean", "");
+        print_ms((ULONG)(exp(logsum / ran) + 0.5));
+        printf("\n");
     }
     return failures;
 }
@@ -192,6 +213,13 @@ int main(int argc, char **argv) {
         return 20;
     }
 
+    selected = malloc(((size_t)argc > NBENCHES ? (size_t)argc : NBENCHES)
+                      * sizeof *selected);
+    if (!selected) {
+        printf("out of memory\n");
+        return 20;
+    }
+
     for (i = 1; i < argc; i++) {
         if (!strcmp(argv[i], "-n") && i + 1 < argc) {
             iters = atoi(argv[++i]);
@@ -216,17 +244,21 @@ int main(int argc, char **argv) {
                 printf("unknown benchmark %s\n", argv[i]);
                 return 20;
             }
-            selected[j] = true;
-            any = true;
+            selected[nselected++] = BENCHES[j];
         }
     }
+    if (!nselected) {
+        for (size_t j = 0; j < NBENCHES; j++)
+            selected[j] = BENCHES[j];
+        nselected = NBENCHES;
+    }
 
     if (!timer_open()) {
         printf("cannot open timer.device\n");
         return 20;
     }
 
-    printf("benchwork: %s\n", __VERSION__);
+    printf("benchwork %s, built with gcc %s\n", BENCH_VERSION, __VERSION__);
     printf("CFLAGS: %s\n\n", BENCH_CFLAGS);
     printf("%-12s %5s %10s %10s   %s\n", "benchmark", "iters", "min ms",
            "mean ms", "check");