diff --git a/.github/workflows/mirror.yml b/.github/workflows/mirror.yml new file mode 100644 index 0000000..3610963 --- /dev/null +++ b/.github/workflows/mirror.yml @@ -0,0 +1,61 @@ +name: Mirror + +# Copies a published release to https://dl.nodeq.tech/hbench, which install.sh and `hbench update` fall back to when +# GitHub's downloads fail: hbench// with the release's assets, then, only when the tag is the latest release, +# hbench/install.sh and hbench/latest (the tag). release.yml runs it after publishing; for an earlier release, run it +# by hand: gh workflow run mirror.yml -f tag=v0.3.13. The `mirror` environment (deployment refs: master and v* tags) +# holds an R2 token that can write that bucket alone; node-q/hpg-infra-cloud's docs/runbooks/hbench-release-mirror.md +# sets it up. +on: + workflow_call: + inputs: + tag: + type: string + required: true + workflow_dispatch: + inputs: + tag: + description: "The release's tag, such as v0.3.13" + type: string + required: true + +permissions: + contents: read + +jobs: + mirror: + name: Mirror ${{ inputs.tag }} to dl.nodeq.tech + runs-on: ubuntu-24.04 + environment: mirror + timeout-minutes: 15 + env: + TAG: ${{ inputs.tag }} + BUCKET: nodeq-hbench-releases + AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} + AWS_ENDPOINT_URL: ${{ secrets.R2_ENDPOINT }} + AWS_DEFAULT_REGION: auto + # Checksums only where S3 requires them, as Cloudflare's R2 docs advise for AWS CLI 2.23 and later. + AWS_REQUEST_CHECKSUM_CALCULATION: when_required + AWS_RESPONSE_CHECKSUM_VALIDATION: when_required + steps: + - name: Download the release + env: + GH_TOKEN: ${{ github.token }} + run: | + [[ "$TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+ ]] || { echo "::error::not a release tag: $TAG"; exit 1; } + gh release download "$TAG" -R "$GITHUB_REPOSITORY" -D dist + cd dist && sha256sum -c SHA256SUMS + - name: Upload hbench/${{ inputs.tag }}/ + run: aws s3 cp dist "s3://$BUCKET/hbench/$TAG/" --recursive --cache-control "public, max-age=31536000, immutable" + - name: Point hbench/latest at it if it is the latest release + env: + GH_TOKEN: ${{ github.token }} + run: | + latest=$(gh api "repos/$GITHUB_REPOSITORY/releases/latest" --jq .tag_name) + if [ "$latest" != "$TAG" ]; then + echo "The latest release is $latest: hbench/latest and hbench/install.sh stay." + exit 0 + fi + aws s3 cp dist/install.sh "s3://$BUCKET/hbench/install.sh" --cache-control "public, max-age=300" + printf '%s\n' "$TAG" | aws s3 cp - "s3://$BUCKET/hbench/latest" --cache-control "public, max-age=300" --content-type text/plain diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e4003b4..acb0ec2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,6 +7,7 @@ name: Release # tagged commit, so only a commit that passed CI on master is released. Only the publish job # can write: it attests install.sh and SHA256SUMS and creates this repo's release, with the tagged # commit's scripts/install.sh beside the archives, so `curl -fsSL https://github.com/node-q/hbench/releases/latest/download/install.sh | sh` installs the latest release. +# The mirror job then copies the release to dl.nodeq.tech (mirror.yml). on: push: tags: ["v[0-9]+.[0-9]+.[0-9]+*"] @@ -154,3 +155,10 @@ jobs: flags=(--verify-tag --title "hbench $GITHUB_REF_NAME" --generate-notes) [ "$PRERELEASE" = true ] && flags+=(--prerelease) gh release create "$GITHUB_REF_NAME" -R "$GITHUB_REPOSITORY" "${flags[@]}" dist/*.tar.gz dist/install.sh dist/SHA256SUMS + + mirror: + name: Mirror + needs: publish + uses: ./.github/workflows/mirror.yml + with: + tag: ${{ github.ref_name }} diff --git a/README.md b/README.md index 9d74805..9b30424 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,10 @@ curl -fsSL https://github.com/node-q/hbench/releases/latest/download/install.sh Later, `hbench update` installs the newest release. +When GitHub fails to serve a release, the script and `hbench update` download it from its copy at +`https://dl.nodeq.tech/hbench/` instead and check it the same way. If GitHub cannot serve the script either: +`curl -fsSL https://dl.nodeq.tech/hbench/install.sh | sh`. + The script checks the archive it downloads against the release's `SHA256SUMS`. To check the script itself before running it, download it, read it, and verify that this repository's release workflow built it (releases after v0.3.7; `gh attestation verify` needs the [GitHub CLI](https://cli.github.com)): diff --git a/docs/RELEASING.md b/docs/RELEASING.md index 1e6f810..6241ce1 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -18,3 +18,10 @@ job, in the `release` environment, creates the GitHub release with the archives, sums. The install script checks the archive it downloads against `SHA256SUMS` before unpacking it. Tags with a suffix (`v0.2.0-rc.1`) are published as prereleases, which the install script skips unless `HBENCH_VERSION` names one. + +The release workflow's last job, `mirror.yml`, copies the release to `https://dl.nodeq.tech/hbench//`, and when it +is the latest release also writes `hbench/install.sh` and `hbench/latest` (the tag). The install script and +`hbench update` fall back to that copy for whatever GitHub fails to serve; `HBENCH_MIRROR_ONLY=1` makes the script use +the copy alone, to check it. The `mirror` environment (deployment refs `master` and `v*` tags) holds an R2 token +that can write the bucket alone; node-q/hpg-infra-cloud's `docs/runbooks/hbench-release-mirror.md` creates both. To +copy an earlier release, or one whose mirror job failed: `gh workflow run mirror.yml -R node-q/hbench -f tag=v0.3.13`. diff --git a/scripts/install.sh b/scripts/install.sh index a389aae..1a888a1 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -2,8 +2,10 @@ # Install hbench into HBENCH_INSTALL_DIR (default ~/.local/bin). # curl -fsSL https://github.com/node-q/hbench/releases/latest/download/install.sh | sh # HBENCH_VERSION=vX.Y.Z pins a release; the default is the latest release of node-q/hbench. +# What GitHub fails to serve comes from the releases' copy at dl.nodeq.tech; HBENCH_MIRROR_ONLY=1 skips GitHub. set -eu REPO=node-q/hbench +MIRROR=https://dl.nodeq.tech/hbench # A site serving this script may fill in a default release here; HBENCH_VERSION still overrides it. DEFAULT_VERSION= DIR=${HBENCH_INSTALL_DIR:-$HOME/.local/bin} @@ -19,11 +21,25 @@ esac tmp=$(mktemp -d) trap 'rm -rf "$tmp"' EXIT +github() { [ "${HBENCH_MIRROR_ONLY:-0}" != 1 ]; } + # /releases/latest redirects to /releases/tag/; reading the redirect, without fetching that page, avoids the API rate limit. -tag=${HBENCH_VERSION:-${DEFAULT_VERSION:-$(curl -fsSI -o /dev/null -w '%{redirect_url}' "https://github.com/$REPO/releases/latest" | sed 's|.*/||')}} +latest() { + { github && curl -fsSI -o /dev/null -w '%{redirect_url}' "https://github.com/$REPO/releases/latest" | sed 's|.*/||' | grep '^v'; } || + curl -fsSL "$MIRROR/latest" || : +} + +# fetch : the release's file into $tmp, from GitHub or else from the mirror. +fetch() { + if github && curl -fsSL -o "$tmp/$1" "https://github.com/$REPO/releases/download/$tag/$1"; then return; fi + github && echo "downloading $1 from $MIRROR instead" >&2 + curl -fsSL -o "$tmp/$1" "$MIRROR/$tag/$1" +} + +tag=${HBENCH_VERSION:-${DEFAULT_VERSION:-$(latest)}} case "$tag" in v*) ;; *) echo "could not find the latest hbench release of $REPO" >&2; exit 1 ;; esac for f in "hbench-$tag-$target.tar.gz" SHA256SUMS; do - curl -fsSL -o "$tmp/$f" "https://github.com/$REPO/releases/download/$tag/$f" || { echo "could not download $f of hbench $tag from $REPO" >&2; exit 1; } + fetch "$f" || { echo "could not download $f of hbench $tag from $REPO or $MIRROR" >&2; exit 1; } done cd "$tmp" diff --git a/src/update.rs b/src/update.rs index a007957..7326f6b 100644 --- a/src/update.rs +++ b/src/update.rs @@ -8,7 +8,10 @@ use std::process::{Command, Stdio}; use std::time::{Duration, SystemTime}; const LATEST_URL: &str = "https://api.github.com/repos/node-q/hbench/releases/latest"; -const INSTALL: &str = "curl -fsSL https://github.com/node-q/hbench/releases/latest/download/install.sh | sh"; +/// The latest release's install script, from GitHub or else from its mirror (dl.nodeq.tech, filled by mirror.yml); +/// the script falls back to the mirror the same way for the release's files. +const INSTALL: &str = "s=$(curl -fsSL https://github.com/node-q/hbench/releases/latest/download/install.sh || \ + curl -fsSL https://dl.nodeq.tech/hbench/install.sh) && printf '%s\\n' \"$s\" | sh"; const MAX_AGE: Duration = Duration::from_secs(24 * 60 * 60); /// Prints a notice on stderr when the cached latest release is newer than this build, then refreshes a stale cache. @@ -38,7 +41,7 @@ pub fn notify() { } } -/// `hbench update`: runs the install script, which replaces this binary with the latest release. +/// `hbench update`: runs the latest release's install script, which replaces this binary with that release. pub fn run() -> anyhow::Result { let status = Command::new("sh").args(["-c", INSTALL]).status()?; Ok(if status.success() { std::process::ExitCode::SUCCESS } else { std::process::ExitCode::FAILURE })