From 53ec735cd97727e5ba6175551b07e7c5621a533c Mon Sep 17 00:00:00 2001 From: rldyourmnd Date: Wed, 2 Sep 2026 18:06:05 +0500 Subject: [PATCH 1/2] fix(provider): the runner pin constrains what runs, not what GitHub advertises Every create validated that the runner package GitHub advertises equals the version the fleet pins. The fleet never downloads that package: the warm claim and the direct-JIT cold create both run the runner baked into the image, and neither passes the cloud-config the metadata would have fed (.108). The version that runs is pinned where it is installed -- the image manifest names the archive and its digest, the build verifies both, the smoke asserts the installed runner. So the guard made every create depend on GitHub's release schedule. On 2026-09-02 GitHub moved all three organisations to 2.337.0 while the image baked 2.336.0, and within ten minutes 25 of 77 creates failed with "filename ... does not match pinned ..."; the share would have reached every create as the warm pools drained. The asset must still be an official actions/runner linux-x64 release tarball whose download URL is the github.com release URL for that same filename, so a substituted host, another architecture or a renamed asset is refused exactly as before. Claude-Session: https://claude.ai/code/session_0128syXKxAGCfJGRDxUUNQXp --- internal/garmproviderincus/provider/incus.go | 44 +++++++++++++++---- .../garmproviderincus/provider/incus_test.go | 37 +++++++++++++--- 2 files changed, 65 insertions(+), 16 deletions(-) diff --git a/internal/garmproviderincus/provider/incus.go b/internal/garmproviderincus/provider/incus.go index 9b5621c1..0c25a384 100644 --- a/internal/garmproviderincus/provider/incus.go +++ b/internal/garmproviderincus/provider/incus.go @@ -1387,22 +1387,48 @@ func (l *Incus) adoptColdDirectJIT( } } +// runnerToolFilename is the exact shape of an official linux-x64 runner +// release asset. Anything else -- another architecture, another project, a +// rewritten host -- is refused. +var runnerToolFilename = regexp.MustCompile(`^actions-runner-linux-x64-(\d+\.\d+\.\d+)\.tar\.gz$`) + +// validateRunnerTool checks the runner package GitHub advertises for this +// job. It does NOT require that package to be the version the fleet pins. +// +// The fleet does not download it. Every path the fleet uses -- the warm +// claim and the direct-JIT cold create -- runs the runner baked into the +// worker image, and neither passes the cloud-config this metadata would +// have fed (provider .108). The version that runs is pinned where it is +// installed: the image manifest names the archive and its digest, the build +// verifies both and the smoke asserts the installed runner. +// +// Requiring GitHub's advertised version to equal the pinned one made the +// fleet's create depend on GitHub's release schedule. On 2026-09-02 GitHub +// moved every organisation to 2.337.0 while the image baked 2.336.0, and +// within ten minutes 25 of 77 creates failed with "filename ... does not +// match pinned ...", a share that would have reached every create as the +// warm pools drained. A supply-chain guard must constrain what runs, not +// what an upstream advertises. +// +// What stays: the asset must be an official actions/runner linux-x64 +// release tarball whose download URL is the github.com release URL for that +// same filename, so a substituted host or a renamed asset is still refused. func (l *Incus) validateRunnerTool(tool commonParams.RunnerApplicationDownload) error { - version := strings.TrimPrefix(l.platform.ControlPlane.RunnerVersion, "v") - if version == "" || version == l.platform.ControlPlane.RunnerVersion { + if strings.TrimPrefix(l.platform.ControlPlane.RunnerVersion, "v") == l.platform.ControlPlane.RunnerVersion { return fmt.Errorf("platform runner version must have a v prefix") } - expectedFilename := fmt.Sprintf("actions-runner-linux-x64-%s.tar.gz", version) + filename := tool.GetFilename() + match := runnerToolFilename.FindStringSubmatch(filename) + if match == nil { + return fmt.Errorf("filename %q is not an official linux-x64 runner release asset", filename) + } expectedURL := fmt.Sprintf( "https://github.com/actions/runner/releases/download/v%s/%s", - version, - expectedFilename, + match[1], + filename, ) - if tool.GetFilename() != expectedFilename { - return fmt.Errorf("filename %q does not match pinned %q", tool.GetFilename(), expectedFilename) - } if tool.GetDownloadURL() != expectedURL { - return fmt.Errorf("download URL %q does not match pinned official URL", tool.GetDownloadURL()) + return fmt.Errorf("download URL %q does not match the official release URL for %q", tool.GetDownloadURL(), filename) } return nil } diff --git a/internal/garmproviderincus/provider/incus_test.go b/internal/garmproviderincus/provider/incus_test.go index 39dfb025..09494545 100644 --- a/internal/garmproviderincus/provider/incus_test.go +++ b/internal/garmproviderincus/provider/incus_test.go @@ -943,18 +943,41 @@ func TestGetCreateInstanceArgsInjectsOnlyTrustedRunnerCacheBootstrap(t *testing. require.Contains(t, string(specs.PreInstallScripts["01-nddev-runner-groups.sh"]), "usermod --groups sudo runner") } -func TestRunnerToolMetadataMustMatchPinnedImageVersion(t *testing.T) { +// The runner that runs is the one baked into the image, and the image +// manifest pins it. This guard constrains the package GitHub advertises to +// an official actions/runner linux-x64 release whose URL matches its own +// filename -- and deliberately not to the fleet's pinned version, because +// requiring that made every create depend on GitHub's release schedule: on +// 2026-09-02 GitHub moved every organisation to 2.337.0 and 25 of 77 creates +// failed in ten minutes while the image ran 2.336.0 perfectly well. +func TestRunnerToolMetadataMustBeAnOfficialReleaseAsset(t *testing.T) { provider := newTestProvider(new(MockIncusServer)) valid := testTools()[0] require.NoError(t, provider.validateRunnerTool(valid)) - wrongFilename := valid - wrongFilename.Filename = ptr("actions-runner-linux-x64-2.337.0.tar.gz") - require.ErrorContains(t, provider.validateRunnerTool(wrongFilename), "does not match pinned") + newerThanPinned := valid + newerThanPinned.Filename = ptr("actions-runner-linux-x64-2.337.0.tar.gz") + newerThanPinned.DownloadURL = ptr("https://github.com/actions/runner/releases/download/v2.337.0/actions-runner-linux-x64-2.337.0.tar.gz") + require.NoError(t, provider.validateRunnerTool(newerThanPinned), "a newer advertised runner is not a reason to refuse the job") - wrongURL := valid - wrongURL.DownloadURL = ptr("https://example.invalid/actions-runner-linux-x64-2.336.0.tar.gz") - require.ErrorContains(t, provider.validateRunnerTool(wrongURL), "does not match pinned official URL") + mismatched := valid + mismatched.Filename = ptr("actions-runner-linux-x64-2.337.0.tar.gz") + require.ErrorContains(t, provider.validateRunnerTool(mismatched), "does not match the official release URL") + + substitutedHost := valid + substitutedHost.DownloadURL = ptr("https://example.invalid/actions-runner-linux-x64-2.336.0.tar.gz") + require.ErrorContains(t, provider.validateRunnerTool(substitutedHost), "does not match the official release URL") + + for _, name := range []string{ + "actions-runner-linux-arm64-2.336.0.tar.gz", + "actions-runner-osx-x64-2.336.0.tar.gz", + "totally-not-a-runner-2.336.0.tar.gz", + "actions-runner-linux-x64-2.336.0.tar.gz.sig", + } { + wrong := valid + wrong.Filename = ptr(name) + require.ErrorContains(t, provider.validateRunnerTool(wrong), "not an official linux-x64 runner release asset", name) + } } func TestCanonicalRepositoryIdentity(t *testing.T) { From e2f145826df239cec33e700647db0925c8ab6c2f Mon Sep 17 00:00:00 2001 From: rldyourmnd Date: Wed, 2 Sep 2026 18:06:08 +0500 Subject: [PATCH 2/2] chore(provider): release v0.1.5-nddev.121, creates survive a runner release Source 53ec735cd97727e5ba6175551b07e7c5621a533c, built twice with CGO_ENABLED=0 go build -trimpath -buildvcs=false -ldflags "-buildid= -s -w -X main.version=v0.1.5-nddev.121 -X main.commit="; both builds agree on b0d1a6980bb7efdd112acaed141d01708eeed9bb211770edc965cc033ff322a7. Claude-Session: https://claude.ai/code/session_0128syXKxAGCfJGRDxUUNQXp --- config/example-runner-1.yaml | 2 +- config/example-runner-2.yaml | 2 +- config/example-runner-3.yaml | 2 +- config/example-runner-4.yaml | 2 +- config/example-services.yaml | 2 +- config/provider-derivative.yaml | 6 +++--- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/config/example-runner-1.yaml b/config/example-runner-1.yaml index 0a84522b..027ba39f 100644 --- a/config/example-runner-1.yaml +++ b/config/example-runner-1.yaml @@ -9,7 +9,7 @@ control_plane: manager_version: v0.2.1-nddev.88 scheduling_mode: scale-set provider: incus - provider_version: v0.1.5-nddev.120 + provider_version: v0.1.5-nddev.121 provider_interface: v0.1.0 worker_kind: incus-container runner: actions/runner diff --git a/config/example-runner-2.yaml b/config/example-runner-2.yaml index ccc022fe..7fbbb147 100644 --- a/config/example-runner-2.yaml +++ b/config/example-runner-2.yaml @@ -9,7 +9,7 @@ control_plane: manager_version: v0.2.1-nddev.88 scheduling_mode: scale-set provider: incus - provider_version: v0.1.5-nddev.120 + provider_version: v0.1.5-nddev.121 provider_interface: v0.1.0 worker_kind: incus-container runner: actions/runner diff --git a/config/example-runner-3.yaml b/config/example-runner-3.yaml index f60e5e79..9d4869f7 100644 --- a/config/example-runner-3.yaml +++ b/config/example-runner-3.yaml @@ -9,7 +9,7 @@ control_plane: manager_version: v0.2.1-nddev.88 scheduling_mode: scale-set provider: incus - provider_version: v0.1.5-nddev.120 + provider_version: v0.1.5-nddev.121 provider_interface: v0.1.0 worker_kind: incus-container runner: actions/runner diff --git a/config/example-runner-4.yaml b/config/example-runner-4.yaml index 1a9734be..d1a304ac 100644 --- a/config/example-runner-4.yaml +++ b/config/example-runner-4.yaml @@ -9,7 +9,7 @@ control_plane: manager_version: v0.2.1-nddev.88 scheduling_mode: scale-set provider: incus - provider_version: v0.1.5-nddev.120 + provider_version: v0.1.5-nddev.121 provider_interface: v0.1.0 worker_kind: incus-container runner: actions/runner diff --git a/config/example-services.yaml b/config/example-services.yaml index 0da65256..5d30e066 100644 --- a/config/example-services.yaml +++ b/config/example-services.yaml @@ -27,7 +27,7 @@ control_plane: manager_version: v0.2.1-nddev.88 scheduling_mode: scale-set provider: incus - provider_version: v0.1.5-nddev.120 + provider_version: v0.1.5-nddev.121 provider_interface: v0.1.0 worker_kind: incus-container runner: actions/runner diff --git a/config/provider-derivative.yaml b/config/provider-derivative.yaml index 4f10b5d1..8c4035a1 100644 --- a/config/provider-derivative.yaml +++ b/config/provider-derivative.yaml @@ -16,7 +16,7 @@ artifact: garm-provider-incus # state all move together, because all three derive from here. A provider change # that does not bump it ships under the previous version, which is exactly how # runner-1 and runner-2 diverged. -derivative_version: v0.1.5-nddev.120 +derivative_version: v0.1.5-nddev.121 # The external-provider protocol GARM speaks to this binary. It moves on its own # schedule -- a provider release does not imply an interface release -- so it is @@ -37,8 +37,8 @@ runtime: queue_intent_schema_version: 6 build: - source_commit: 3a77ae2562015ad006857deaf2823a84575e3e3c - binary_sha256: 3c07b99b8d11f1bcded6208147540522aaa37302bf2203da78937bdbd2c70368 + source_commit: 53ec735cd97727e5ba6175551b07e7c5621a533c + binary_sha256: b0d1a6980bb7efdd112acaed141d01708eeed9bb211770edc965cc033ff322a7 go_version: go1.26.7 cgo_enabled: false target_os: linux