ci: Install the monitors from monitors-build releases

.github/actions/find-monitors-release/action.yml

@@ -0,0 +1,109 @@
+name: Find the monitors-build release
+description: >
+  Find the release of https://github.com/urunc-dev/monitors-build that bundles
+  the given versions of the monitors and return the URL of its archive for the
+  given architecture. It also holds the default versions of the monitors for
+  the e2e tests.
+inputs:
+  firecracker_version:
+    description: 'Firecracker version (default: v1.7.0)'
+    required: false
+  cloud_hypervisor_version:
+    description: 'Cloud Hypervisor version (default: v50.0)'
+    required: false
+  hluk_version:
+    description: 'hyperlight-unikraft version (default: v0.18.0)'
+    required: false
+  solo5_version:
+    description: 'Solo5 version (default: v0.12.1)'
+    required: false
+  qemu_version:
+    description: 'QEMU version (default: v10.1.1)'
+    required: false
+  virtiofsd_version:
+    description: 'virtiofsd version (default: v1.13.0)'
+    required: false
+  arch:
+    description: 'Architecture of the archive (amd64 or arm64)'
+    required: true
+outputs:
+  url:
+    description: 'URL of the archive, which extracts to urunc/ (e.g. /opt/urunc)'
+    value: ${{ steps.find.outputs.url }}
+  sha256:
+    description: 'SHA-256 checksum of the archive'
+    value: ${{ steps.find.outputs.sha256 }}
+runs:
+  using: composite
+  steps:
+    - id: find
+      uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
+      env:
+        FIRECRACKER_VERSION: ${{ inputs.firecracker_version }}
+        CLOUD_HYPERVISOR_VERSION: ${{ inputs.cloud_hypervisor_version }}
+        HLUK_VERSION: ${{ inputs.hluk_version }}
+        SOLO5_VERSION: ${{ inputs.solo5_version }}
+        QEMU_VERSION: ${{ inputs.qemu_version }}
+        VIRTIOFSD_VERSION: ${{ inputs.virtiofsd_version }}
+        ARCH: ${{ inputs.arch }}
+      with:
+        script: |
+          // Each release of urunc-dev/monitors-build lists the versions it
+          // bundles in a fingerprint inside its description, e.g.
+          // FC-v1.7.0_CLH-v50.0_HLUK-v0.18.0_S5-v0.12.1_VFS-v1.13.0_QM-v10.1.1-9a44e,
+          // where the last part is the hash of the QEMU configuration. The archive
+          // with all the monitors of an architecture is named after it.
+          //
+          // The default versions of the monitors for the e2e tests. Callers
+          // forward unset versions as empty strings, which skip an input's
+          // default, hence the fallbacks.
+          const wanted = {
+            FC: process.env.FIRECRACKER_VERSION || 'v1.7.0',
+            CLH: process.env.CLOUD_HYPERVISOR_VERSION || 'v50.0',
+            HLUK: process.env.HLUK_VERSION || 'v0.18.0',
+            S5: process.env.SOLO5_VERSION || 'v0.12.1',
+            VFS: process.env.VIRTIOFSD_VERSION || 'v1.13.0',
+            QM: process.env.QEMU_VERSION || 'v10.1.1',
+          };
+          for (const [name, version] of Object.entries(wanted)) {
+            if (!/^v[0-9]+\.[0-9]+(\.[0-9]+)?$/.test(version)) {
+              core.setFailed(`Invalid ${name} version: ${version}`);
+              return;
+            }
+          }
+          const releases = await github.paginate(github.rest.repos.listReleases, {
+            owner: 'urunc-dev',
+            repo: 'monitors-build',
+          });
+          // Releases are listed from the newest one.
+          for (const release of releases) {
+            const match = /<!-- monitors-fingerprint: (\S+) -->/.exec(release.body || '');
+            if (release.draft || !match) {
+              continue;
+            }
+            const fingerprint = match[1];
+            // Split each part only on its first "-", so that a pre-release
+            // (e.g. v1.7.0-rc1) does not match its final version, and drop the
+            // hash after the QEMU version.
+            const versions = Object.fromEntries(fingerprint.split('_').map(part => {
+              const [, name, version] = /^([^-]+)-(.+)$/.exec(part) || [];
+              return [name, name === 'QM' ? version.replace(/-[0-9a-f]{5}$/, '') : version];
+            }));
+            if (!Object.entries(wanted).every(([name, version]) => versions[name] === version)) {
+              continue;
+            }
+            const asset = release.assets.find(a =>
+              a.name === `${fingerprint}-${process.env.ARCH}.tar.gz` && a.state === 'uploaded');
+            if (!asset) {
+              continue;
+            }
+            if (!/^sha256:[0-9a-f]{64}$/.test(asset.digest || '')) {
+              core.setFailed(`${asset.name} of monitors-build ${release.tag_name} has no SHA-256 digest`);
+              return;
+            }
+            core.info(`Using monitors-build ${release.tag_name}: ${fingerprint}`);
+            core.setOutput('url', asset.browser_download_url);
+            core.setOutput('sha256', asset.digest.slice('sha256:'.length));
+            return;
+          }
+          core.setFailed(`No monitors-build release provides ${JSON.stringify(wanted)} for ${process.env.ARCH}`);

.github/workflows/ci.yml

@@ -14,7 +14,8 @@ on:
         type: string
         default: "no"
       # Optional overrides of the e2e runners and environment versions. An
-      # empty value selects the default in vm_test.yml.
+      # empty value selects the default in vm_test.yml, or in the
+      # find-monitors-release action for the monitors.
       runners:
         type: string
         default: ''
@@ -42,6 +43,12 @@ on:
       solo5_version:
         type: string
         default: ''
+      qemu_version:
+        type: string
+        default: ''
+      virtiofsd_version:
+        type: string
+        default: ''
       hluk_version:
         type: string
         default: ''
@@ -98,7 +105,8 @@ jobs:
       # caller ref for non-PR runs (nightly/release), where base.sha is empty.
       ref: ${{ github.event.pull_request.base.sha || inputs.ref }}
       # Only set by manual dispatches. PR runs leave them empty, so the base
-      # branch's e2e code runs with vm_test.yml's default versions.
+      # branch's e2e code runs with the default versions of vm_test.yml and
+      # the find-monitors-release action.
       runners: ${{ inputs.runners }}
       runc_version: ${{ inputs.runc_version }}
       containerd_version: ${{ inputs.containerd_version }}
@@ -108,6 +116,8 @@ jobs:
       firecracker_version: ${{ inputs.firecracker_version }}
       cloud_hypervisor_version: ${{ inputs.cloud_hypervisor_version }}
       solo5_version: ${{ inputs.solo5_version }}
+      qemu_version: ${{ inputs.qemu_version }}
+      virtiofsd_version: ${{ inputs.virtiofsd_version }}
       hluk_version: ${{ inputs.hluk_version }}
     secrets: inherit
 
@@ -118,16 +128,21 @@ jobs:
     uses: ./.github/workflows/kind_test.yml
     with:
       ref: ${{ inputs.ref }}
-      firecracker_version: 'v1.7.0'
-      solo5_version: 'v0.12.1'
+      firecracker_version: ${{ inputs.firecracker_version }}
+      cloud_hypervisor_version: ${{ inputs.cloud_hypervisor_version }}
+      hluk_version: ${{ inputs.hluk_version }}
+      solo5_version: ${{ inputs.solo5_version }}
+      qemu_version: ${{ inputs.qemu_version }}
+      virtiofsd_version: ${{ inputs.virtiofsd_version }}
       runc_version: '1.3.0'
     secrets: inherit
 
   # The vm_test job above runs the base branch's e2e code. In order to also
   # test changes in the e2e testing code or environment, re-run the vm_test
   # with the PR's e2e code. However, do that only when a change in the files
   # that affect the e2e tests ("tests/*", "Makefile", "script/dm_create.sh",
-  # ".github/workflows/vm_test.yml") has taken place.
+  # ".github/workflows/vm_test.yml", ".github/actions/find-monitors-release/*")
+  # has taken place.
   detect-test-changes:
     if: ${{ github.event_name == 'pull_request' && inputs.skip-build != 'yes' }}
     name: Detect test changes
@@ -158,6 +173,7 @@ jobs:
               'Makefile',
               'script/dm_create.sh',
               '.github/workflows/vm_test.yml',
+              '.github/actions/find-monitors-release/',
             ];
             const changed = files.some(f => roots.some(r => f.filename.startsWith(r)));
             core.setOutput('tests_changed', changed ? 'yes' : 'no');

.github/workflows/ci_devel.yml

@@ -13,7 +13,8 @@ on:
         default: "no"
         type: string
       # The e2e runners and environment versions. Leave a field empty to use
-      # the default in vm_test.yml.
+      # the default in vm_test.yml, or in the find-monitors-release action for
+      # the monitors.
       runners:
         description: 'Runners for the e2e tests, as a JSON list (e.g. ["ubuntu-24.04"])'
         required: false
@@ -59,6 +60,16 @@ on:
         required: false
         default: ''
         type: string
+      qemu_version:
+        description: 'QEMU version for the e2e tests (e.g. v10.1.1)'
+        required: false
+        default: ''
+        type: string
+      virtiofsd_version:
+        description: 'virtiofsd version for the e2e tests (e.g. v1.13.0)'
+        required: false
+        default: ''
+        type: string
       hluk_version:
         description: 'hyperlight-unikraft version for the e2e tests (e.g. v0.18.0)'
         required: false
@@ -90,5 +101,7 @@ jobs:
       firecracker_version: ${{ inputs.firecracker_version }}
       cloud_hypervisor_version: ${{ inputs.cloud_hypervisor_version }}
       solo5_version: ${{ inputs.solo5_version }}
+      qemu_version: ${{ inputs.qemu_version }}
+      virtiofsd_version: ${{ inputs.virtiofsd_version }}
       hluk_version: ${{ inputs.hluk_version }}
     secrets: inherit

.github/workflows/ci_nightly.yml

@@ -39,8 +39,6 @@ jobs:
     name:  Kubernetes test
     uses: ./.github/workflows/kind_test.yml
     with:
-      firecracker_version: 'v1.7.0'
-      solo5_version: 'v0.12.1'
       runc_version: '1.3.0'
 
   urunc_deploy_test:

.github/workflows/kind_test.yml

@@ -5,12 +5,26 @@ on:
       ref:
         type: string
         default: ''
+      # Optional overrides of the monitors' versions. An empty value selects
+      # the default in the find-monitors-release action.
       firecracker_version:
         type: string
-        required: true
+        default: ''
+      cloud_hypervisor_version:
+        type: string
+        default: ''
+      hluk_version:
+        type: string
+        default: ''
       solo5_version:
         type: string
-        required: true
+        default: ''
+      qemu_version:
+        type: string
+        default: ''
+      virtiofsd_version:
+        type: string
+        default: ''
       runc_version:
         required: true
         type: string
@@ -40,6 +54,18 @@ jobs:
       with:
         ref: ${{ inputs.ref }}
 
+    # On pull requests, the checkout above is the head of the PR, while GitHub
+    # loaded this workflow from the merge commit (github.sha). Fetch the local
+    # actions from the latter, so that they match this workflow.
+    - name: Checkout the local actions
+      uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      with:
+        ref: ${{ github.sha }}
+        path: .workflow-actions
+        sparse-checkout: .github/actions/
+        sparse-checkout-cone-mode: false
+        persist-credentials: false
+
     - name: Install kind
       run: |
         ARCH=$(uname -m)
@@ -85,7 +111,7 @@ jobs:
     - name: Install dependencies inside kind node
       run: |
         docker exec urunc-test-control-plane apt-get update
-        docker exec urunc-test-control-plane apt-get install -y git wget build-essential libseccomp-dev pkg-config bc
+        docker exec urunc-test-control-plane apt-get install -y wget
 
     - name: Install runc inside kind node
       run: |
@@ -145,46 +171,69 @@ jobs:
           /usr/local/bin/urunc --version
         '
 
-    - name: Configure urunc with debug logging
+    - name: Configure urunc with debug logging and the monitors' paths
       run: |
         docker exec urunc-test-control-plane bash -c '
           mkdir -p /etc/urunc
           cat > /etc/urunc/config.toml <<EOF
         [log]
         level = "debug"
         syslog = true
-        EOF
-        '
 
-    - name: Install QEMU inside kind node (amd64 only)
-      if: matrix.arch == 'amd64'
-      run: |
-        docker exec urunc-test-control-plane apt-get install -y qemu-system
+        [monitors.qemu]
+        path = "/opt/urunc/bin/qemu-system-$(uname -m)"
+        data_path = "/opt/urunc/share/qemu"
 
+        [monitors.firecracker]
+        path = "/opt/urunc/bin/firecracker"
 
-    - name: Install Firecracker inside kind node
-      run: |      
-        FC_VERSION="${{ inputs.firecracker_version }}"
-        docker exec urunc-test-control-plane bash -c '
-          ARCH=$(uname -m)
-          echo "Downloading Firecracker version: '"$FC_VERSION"'"
-          curl -fsSL https://github.com/firecracker-microvm/firecracker/releases/download/'"$FC_VERSION"'/firecracker-'"$FC_VERSION"'-${ARCH}.tgz | tar -xz
-          install -m 755 release-'"$FC_VERSION"'-${ARCH}/firecracker-'"$FC_VERSION"'-${ARCH} /usr/local/bin/firecracker
-          firecracker --version
+        [monitors.cloud-hypervisor]
+        path = "/opt/urunc/bin/cloud-hypervisor"
+
+        [monitors.hyperlight-unikraft]
+        path = "/opt/urunc/bin/hluk"
+
+        [monitors.hvt]
+        path = "/opt/urunc/bin/solo5-hvt"
+
+        [monitors.spt]
+        path = "/opt/urunc/bin/solo5-spt"
+
+        [extra_binaries.virtiofsd]
+        path = "/opt/urunc/bin/virtiofsd"
+        EOF
         '
 
-    - name: Install Solo5 inside kind node
+    - name: Find the monitors-build release with these versions
+      id: monitors
+      uses: ./.workflow-actions/.github/actions/find-monitors-release
+      with:
+        firecracker_version: ${{ inputs.firecracker_version }}
+        cloud_hypervisor_version: ${{ inputs.cloud_hypervisor_version }}
+        hluk_version: ${{ inputs.hluk_version }}
+        solo5_version: ${{ inputs.solo5_version }}
+        qemu_version: ${{ inputs.qemu_version }}
+        virtiofsd_version: ${{ inputs.virtiofsd_version }}
+        arch: ${{ matrix.arch }}
+
+    - name: Install monitors inside kind node
+      env:
+        MONITORS_URL: ${{ steps.monitors.outputs.url }}
+        MONITORS_SHA256: ${{ steps.monitors.outputs.sha256 }}
       run: |
-        SOLO5_VERSION="${{ inputs.solo5_version }}"
+        curl -fsSL -o monitors.tar.gz "${MONITORS_URL}"
+        echo "${MONITORS_SHA256}  monitors.tar.gz" | sha256sum -c -
+        docker cp monitors.tar.gz urunc-test-control-plane:/monitors.tar.gz
+        rm -f monitors.tar.gz
+        # The archive extracts to /opt/urunc, which is where the urunc
+        # configuration points to. It records the files as owned by the user
+        # that built it, so let root own them instead.
         docker exec urunc-test-control-plane bash -c '
-          git clone -b "'"$SOLO5_VERSION"'" https://github.com/Solo5/solo5.git
-          cd solo5
-          ./configure.sh && make -j$(nproc)
-          install -m 755 tenders/hvt/solo5-hvt /usr/local/bin/
-          install -m 755 tenders/spt/solo5-spt /usr/local/bin/
-          cd ..
-          rm -rf solo5
-          solo5-hvt --version
+          set -ex
+          tar --no-same-owner --no-overwrite-dir -C /opt -xzf /monitors.tar.gz
+          rm -f /monitors.tar.gz
+          /opt/urunc/bin/firecracker --version
+          /opt/urunc/bin/solo5-hvt --version
         '
 
     - name: Add urunc to containerd config