fuzz.yml

name: Fuzz and memory safety

# Coverage-blind crash fuzzing plus memory-safety tooling over the JavaScript
# suite. Scheduled rather than per-PR: a useful fuzz run takes far longer than
# a PR should wait, and Valgrind slows the suite by roughly an order of
# magnitude. PR-time coverage stays with the ordinary suite in pr.yml.
#
# Cadence: nightly (03:00 UTC). Manual trigger available via workflow_dispatch,
# where the fuzz duration can be raised for a longer campaign.
#
# Reproducing a finding locally is documented in
# docs/contributing/tooling.md#fuzzing-and-memory-safety.

on:
  schedule:
    - cron: '0 3 * * *'
  workflow_dispatch:
    inputs:
      fuzz-seconds:
        description: 'Seconds to fuzz per run'
        required: false
        default: '900'

permissions:
  contents: read

env:
  FUZZ_SECONDS: ${{ github.event.inputs.fuzz-seconds || '900' }}

jobs:
  # ── Coverage-blind fuzzing ───────────────────────────────────────────
  #
  # FPC emits no AFL instrumentation, so afl-fuzz runs in non-instrumented
  # (-n) mode. That trades coverage feedback for pure random mutation: it
  # still finds parser and lexer faults, but it will not discover deep
  # execution paths on its own. The seed corpus is what carries coverage
  # here, which is why it is derived from the real test suites.
  #
  # Instrumented builds were evaluated and rejected for now: afl-gcc-style
  # instrumentation requires either an FPC assembler pass rewrite or building
  # via -Cg + a GCC-compatible IR, neither of which FPC 3.2.2 exposes. The
  # tractable future path is afl-clang-lto over LLVM bitcode from an LLVM FPC
  # backend build; recorded as a follow-up rather than attempted here.
  fuzz:
    runs-on: ubuntu-latest
    timeout-minutes: 60
    steps:
      - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5
        with:
          persist-credentials: false

      - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2

      - name: Install FPC and AFL++
        run: |
          sudo apt-get update
          sudo apt-get install fpc afl++ -qq > /dev/null

      - name: Build fuzz harness
        run: ./build.pas fuzzharness

      - name: Build seed corpus
        run: bun run scripts/build-fuzz-corpus.ts --verbose

      - name: Verify the harness reports faults nonzero
        # Guards against the harness silently swallowing every outcome, which
        # would make a green fuzz run meaningless.
        run: |
          if ./build/GocciaFuzzHarness --self-test-fault; then
            echo "::error::Harness self-test returned 0; fault path is broken"
            exit 1
          fi

      - name: Sweep the seed corpus
        # Every seed is known-good input from our own suites, so any nonzero
        # exit here is a finding even before mutation begins.
        run: |
          status=0
          mkdir -p build/fuzz/findings
          for seed in build/fuzz/corpus/*.js; do
            if ! ./build/GocciaFuzzHarness "$seed" > "build/fuzz/findings/$(basename "$seed").log" 2>&1; then
              echo "::error::Seed sweep fault: $seed"
              cp "$seed" build/fuzz/findings/
              status=1
            else
              rm -f "build/fuzz/findings/$(basename "$seed").log"
            fi
          done
          exit $status

      - name: Fuzz
        run: |
          # AFL refuses to run when the host uses a core-dump handler pipe.
          echo core | sudo tee /proc/sys/kernel/core_pattern > /dev/null
          mkdir -p build/fuzz/out
          # -n: non-instrumented. AFL_SKIP_CPUFREQ/AFL_NO_AFFINITY are the
          # standard CI accommodations for shared runners.
          # -t 5000: the harness's own budget is 1000ms and a harness timeout
          # is a NORMAL exit-0 outcome, so AFL's exec threshold must sit safely
          # above it — otherwise process overhead around a legitimate slow input
          # lands it in hangs/ and false-fails the job.
          # -k 60: afl-fuzz does not reliably exit on the SIGTERM `timeout`
          # sends at the deadline — run 31771115536 sat 43 further minutes on a
          # delivered SIGTERM until the job's own 60-minute cap cancelled it,
          # leaving `timeout` and afl-fuzz behind as orphans. The kill
          # signal makes FUZZ_SECONDS an actual bound. Findings are written to
          # crashes/ and hangs/ as they are discovered, so a hard kill loses no
          # reproducers — only the final stats summary.
          status=0
          started=${SECONDS}
          AFL_SKIP_CPUFREQ=1 AFL_NO_AFFINITY=1 AFL_BENCH_UNTIL_CRASH=1 \
          timeout -k 60 "${FUZZ_SECONDS}" afl-fuzz -n -t 5000 \
            -i build/fuzz/corpus \
            -o build/fuzz/out \
            -- ./build/GocciaFuzzHarness @@ || status=$?
          elapsed=$(( SECONDS - started ))
          echo "afl-fuzz exit status: ${status} after ${elapsed}s"
          # 0 = afl-fuzz returned on its own; 124 = deadline reached and the
          # SIGTERM was honored; 137 = SIGKILL — expected only when it is
          # timeout's -k escalation at the deadline. A 137 long before the
          # deadline is an outside kill (OOM killer) and must not pass as a
          # completed fuzz run, so 137 is accepted only once the deadline
          # elapsed. Anything else is afl-fuzz never getting off the ground
          # (bad arguments, missing binary, 127 from a missing afl-fuzz) and
          # must fail the job rather than pass as a fuzz run that found
          # nothing. Crash triage still runs — "Report crashes" is
          # `!cancelled()`.
          case "${status}" in
            0 | 124) ;;
            137)
              if [ "${elapsed}" -lt "${FUZZ_SECONDS}" ]; then
                echo "::error::afl-fuzz was SIGKILLed after ${elapsed}s, before the ${FUZZ_SECONDS}s deadline (OOM killer?)"
                exit "${status}"
              fi
              ;;
            *)
              echo "::error::afl-fuzz did not run to a deadline (exit ${status})"
              exit "${status}"
              ;;
          esac

      - name: Report crashes
        # Runs even when the Fuzz step failed: findings are written to
        # crashes/ and hangs/ as they are discovered, so reproducers found
        # before the failure must still be triaged and uploaded. `!cancelled()`
        # rather than `always()` — a cancelled run should stop, not push on
        # through triage.
        if: ${{ !cancelled() }}
        run: |
          crashes=$(find build/fuzz/out -path '*/crashes/id:*' 2>/dev/null | wc -l)
          hangs=$(find build/fuzz/out -path '*/hangs/id:*' 2>/dev/null | wc -l)
          echo "AFL crashes: $crashes, hangs: $hangs"
          if [ "$crashes" -gt 0 ] || [ "$hangs" -gt 0 ]; then
            echo "::error::afl-fuzz recorded $crashes crashing input(s) and $hangs hanging input(s)"
            exit 1
          fi

      - name: Upload reproducers
        if: failure()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        with:
          name: fuzz-reproducers
          path: |
            build/fuzz/out/**/crashes/**
            build/fuzz/out/**/hangs/**
            build/fuzz/findings/**
          if-no-files-found: ignore
          retention-days: 30

  # ── Memory safety over the real suite ────────────────────────────────
  #
  # heaptrc and Valgrind answer different questions: heaptrc reports FPC-level
  # leaks and double-frees from inside the process, Valgrind catches invalid
  # reads and writes the allocator never sees. Both run the full JavaScript
  # suite because that is the only workload exercising every builtin.
  memory-safety:
    runs-on: ubuntu-latest
    timeout-minutes: 90
    strategy:
      fail-fast: false
      matrix:
        mode: [interpreter, bytecode]
    steps:
      - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5
        with:
          persist-credentials: false

      - name: Install FPC and Valgrind
        run: |
          sudo apt-get update
          sudo apt-get install fpc valgrind -qq > /dev/null

      - name: Build test runner with heaptrc
        # -gh links the heaptrc unit; the trailing summary goes to stderr on
        # exit and names every unfreed block with its allocation site. The
        # separate -FU dir follows the per-program unit-output rule (see
        # docs/contributing/tooling.md, "Shared -FU Directories Across
        # Programs" — Internal error 200611011) — but fpc does not create a
        # -FU directory, it only writes into one, so mkdir it first.
        run: |
          ./build.pas testrunner
          mkdir -p build/compiled/targets/testrunner-heaptrc
          fpc @config.cfg -gh -gl -FUbuild/compiled/targets/testrunner-heaptrc \
            -obuild/GocciaTestRunnerHeaptrc source/app/GocciaTestRunner.dpr

      - name: Run suite under heaptrc
        id: heaptrc
        run: |
          modeflag=""
          if [ "${{ matrix.mode }}" = "bytecode" ]; then modeflag="--mode=bytecode"; fi
          # HEAPTRC halt-on-error would abort mid-suite; capture and inspect
          # the summary instead so a leak reports alongside the test results.
          ./build/GocciaTestRunnerHeaptrc tests $modeflag 2> build/heaptrc-${{ matrix.mode }}.log
          tail -20 build/heaptrc-${{ matrix.mode }}.log
          if grep -qE '[1-9][0-9]* unfreed memory blocks' build/heaptrc-${{ matrix.mode }}.log; then
            echo "::error::heaptrc reported unfreed blocks in ${{ matrix.mode }} mode"
            exit 1
          fi

      - name: Run suite under Valgrind memcheck
        # A full suite run under memcheck is slow; --error-exitcode makes any
        # invalid access fail the job rather than scroll past.
        run: |
          modeflag=""
          if [ "${{ matrix.mode }}" = "bytecode" ]; then modeflag="--mode=bytecode"; fi
          valgrind \
            --tool=memcheck \
            --error-exitcode=42 \
            --errors-for-leak-kinds=none \
            --leak-check=no \
            --track-origins=yes \
            --log-file=build/valgrind-${{ matrix.mode }}.log \
            ./build/GocciaTestRunner tests $modeflag
          tail -40 build/valgrind-${{ matrix.mode }}.log

      - name: Upload memory-safety logs
        if: always()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        with:
          name: memory-safety-${{ matrix.mode }}
          path: build/*.log
          if-no-files-found: ignore
          retention-days: 30