diff --git a/.github/workflows/cloud-smoke-cleanup.yml b/.github/workflows/cloud-smoke-cleanup.yml
index 4d62288..6cdb48c 100644
--- a/.github/workflows/cloud-smoke-cleanup.yml
+++ b/.github/workflows/cloud-smoke-cleanup.yml
@@ -19,6 +19,12 @@ jobs:
fail-fast: false
matrix:
include:
+ - name: DigitalOcean ISLE
+ kind: app
+ provider: digitalocean
+ template: isle
+ environment: cloud-smoke-cleanup-digitalocean
+ concurrency_group: cloud-compose-smoke-digitalocean-isle
- name: DigitalOcean WordPress
kind: app
provider: digitalocean
diff --git a/.github/workflows/cloud-smoke.yml b/.github/workflows/cloud-smoke.yml
index 26a5cd6..e011fb3 100644
--- a/.github/workflows/cloud-smoke.yml
+++ b/.github/workflows/cloud-smoke.yml
@@ -126,17 +126,26 @@ jobs:
fail-fast: false
matrix:
include:
+ - name: DigitalOcean ISLE
+ provider: digitalocean
+ template: isle
+ # Exceed the 90-minute application service ceiling with enough
+ # room to observe the terminal unit state before cleanup.
+ boot_timeout: "6000"
- name: DigitalOcean WordPress
provider: digitalocean
template: wp
+ boot_timeout: "1200"
- name: Linode WordPress
provider: linode
template: wp
+ boot_timeout: "1200"
concurrency:
group: cloud-compose-smoke-${{ matrix.provider }}-${{ matrix.template }}
cancel-in-progress: false
env:
CLOUD_COMPOSE_SMOKE_AUTO_APPROVE: "true"
+ CLOUD_COMPOSE_SMOKE_BOOT_TIMEOUT: ${{ matrix.boot_timeout }}
CLOUD_COMPOSE_SMOKE_DESTROY_TIMEOUT: "1800"
CLOUD_COMPOSE_SMOKE_SWEEP_ORPHANS: "true"
CLOUD_COMPOSE_SMOKE_RUN_ID: ${{ github.run_id }}
diff --git a/Makefile b/Makefile
index cb7e0e5..ed0b7fb 100644
--- a/Makefile
+++ b/Makefile
@@ -1,4 +1,4 @@
-.PHONY: lint lint-check actionlint shell-lint runtime-config-contract application-env-contract compose-runtime-contract app-filesystem-convergence-contract backup-contract overlay-contract filesystem-prep-contract key-rotation-contract vault-runtime-contract managed-artifact-contract config-management-input-contract systemd-contract bootstrap-recovery-contract sitectl-version-contract go-fmt-check go-vet go-contracts template-version-contract rollout-parity-contract rootfs-package-contract host-runtime-security cos-jq-portability-contract source-trust-contract cloud-smoke-cleanup-contract hosted-cleanup-retry-contract gcp-upgrade-smoke-contract artifact-install-contract config-management-smoke cloud-compose-ci
+.PHONY: lint lint-check actionlint shell-lint runtime-config-contract application-env-contract compose-runtime-contract app-filesystem-convergence-contract backup-contract disaster-recovery-contract overlay-contract filesystem-prep-contract key-rotation-contract vault-runtime-contract managed-artifact-contract config-management-input-contract systemd-contract bootstrap-recovery-contract sitectl-version-contract go-fmt-check go-vet go-contracts template-version-contract rollout-parity-contract rootfs-package-contract host-runtime-security cos-jq-portability-contract source-trust-contract cloud-smoke-cleanup-contract hosted-cleanup-retry-contract gcp-upgrade-smoke-contract artifact-install-contract config-management-smoke cloud-compose-ci
.PHONY: terraform-fmt terraform-fmt-check terraform-validate terraform-validate-contract terraform-lint-check terraform-docs terraform-docs-check
.PHONY: config-management-cloud-smoke config-management-cloud-smoke-ansible-drupal config-management-cloud-smoke-salt-drupal
.PHONY: destroy-config-management-cloud-smoke destroy-config-management-cloud-smoke-ansible-drupal destroy-config-management-cloud-smoke-salt-drupal
@@ -16,9 +16,9 @@ export CLOUD_COMPOSE_CI_BIN
GO_MODULE_FILES := $(wildcard go.mod go.sum)
GO_SOURCES := $(shell find cmd internal -type f -name '*.go')
-lint: terraform-fmt actionlint shell-lint host-runtime-security cos-jq-portability-contract application-env-contract compose-runtime-contract app-filesystem-convergence-contract backup-contract overlay-contract filesystem-prep-contract key-rotation-contract vault-runtime-contract managed-artifact-contract config-management-input-contract systemd-contract bootstrap-recovery-contract source-trust-contract cloud-smoke-cleanup-contract hosted-cleanup-retry-contract gcp-upgrade-smoke-contract sitectl-version-contract template-version-contract rollout-parity-contract rootfs-package-contract artifact-install-contract terraform-validate terraform-docs-check
+lint: terraform-fmt actionlint shell-lint host-runtime-security cos-jq-portability-contract application-env-contract compose-runtime-contract app-filesystem-convergence-contract backup-contract disaster-recovery-contract overlay-contract filesystem-prep-contract key-rotation-contract vault-runtime-contract managed-artifact-contract config-management-input-contract systemd-contract bootstrap-recovery-contract source-trust-contract cloud-smoke-cleanup-contract hosted-cleanup-retry-contract gcp-upgrade-smoke-contract sitectl-version-contract template-version-contract rollout-parity-contract rootfs-package-contract artifact-install-contract terraform-validate terraform-docs-check
-lint-check: terraform-fmt-check actionlint shell-lint host-runtime-security cos-jq-portability-contract application-env-contract compose-runtime-contract app-filesystem-convergence-contract backup-contract overlay-contract filesystem-prep-contract key-rotation-contract vault-runtime-contract managed-artifact-contract config-management-input-contract systemd-contract bootstrap-recovery-contract source-trust-contract cloud-smoke-cleanup-contract hosted-cleanup-retry-contract gcp-upgrade-smoke-contract sitectl-version-contract template-version-contract rollout-parity-contract rootfs-package-contract artifact-install-contract terraform-validate terraform-docs-check
+lint-check: terraform-fmt-check actionlint shell-lint host-runtime-security cos-jq-portability-contract application-env-contract compose-runtime-contract app-filesystem-convergence-contract backup-contract disaster-recovery-contract overlay-contract filesystem-prep-contract key-rotation-contract vault-runtime-contract managed-artifact-contract config-management-input-contract systemd-contract bootstrap-recovery-contract source-trust-contract cloud-smoke-cleanup-contract hosted-cleanup-retry-contract gcp-upgrade-smoke-contract sitectl-version-contract template-version-contract rollout-parity-contract rootfs-package-contract artifact-install-contract terraform-validate terraform-docs-check
actionlint:
go run github.com/rhysd/actionlint/cmd/actionlint@$(ACTIONLINT_VERSION)
@@ -58,6 +58,9 @@ app-filesystem-convergence-contract:
backup-contract:
bash ci/backup-contract.sh
+disaster-recovery-contract:
+ bash ci/disaster-recovery-contract.sh
+
overlay-contract:
bash ci/overlay-contract.sh
diff --git a/README.md b/README.md
index 6e8962a..429bcd4 100644
--- a/README.md
+++ b/README.md
@@ -20,10 +20,15 @@ release set. Override individual selectors with
`runtime.sitectl.package_versions` when intentionally testing or promoting a
different compatible release set.
-All presets use their coordinated sitectl v1.0.0 core/plugin release set. The
-ISLE preset selects the `libops/isle` v1.1.0 template; every other application
-preset remains on its v1.0.0 template contract. Keep each preset's complete
-template and package set together when promoting an override.
+Renovate tracks both the Compose tags and sitectl release tags in the registry.
+Its pull requests are promotion candidates: keep the complete compatible set
+together and require the hosted application smoke before merging them.
+
+All presets pin sitectl v1.8.2 plus the latest reviewed application plugin.
+ISLE selects the `libops/isle` v1.3.0 template, Drupal and WordPress select
+their v1.1.0 templates, and the remaining application templates stay on
+v1.0.0. Keep each preset's complete template and package set together when
+promoting an override.
GCP deployments have two Terraform ownership layers. Apply
[`modules/gcp-foundation`](modules/gcp-foundation/README.md) once per service
@@ -36,6 +41,14 @@ They must not recreate or destroy the singleton foundation. The
cover the state boundary, Shared VPC setup, and Cloud Run Direct VPC egress
requirements.
+Production operators can require the provider-neutral
+[`runtime.disaster_recovery`](docs/disaster-recovery.md) contract. Cloud Compose
+then hands complete database, application-file, and volume-topology coverage to
+a root-owned operator driver and accepts only an encrypted off-host receipt plus
+scheduled disposable restore proof. Storage credentials remain outside
+Terraform and the runtime environment. Same-disk logical dumps and provider
+snapshots are never presented as disaster recovery.
+
## Requirements
@@ -68,7 +81,7 @@ No resources.
| [name](#input\_name) | Deployment name. | `string` | n/a | yes |
| [cloud\_provider](#input\_cloud\_provider) | Compatibility selector for the root GCP entrypoint. Use providers/do or providers/linode for other clouds. | `string` | `"gcp"` | no |
| [gcp](#input\_gcp) | Google Cloud infrastructure settings. | object({
project_id = optional(string, "")
project_number = optional(string, "")
region = optional(string, "us-east5")
zone = optional(string, "us-east5-b")
identity = optional(object({
vm_service_account_email = optional(string, "")
app_service_account_email = optional(string, "")
app_credentials_enabled = optional(bool, false)
}), {})
instance = optional(object({
machine_type = optional(string, "n4-standard-2")
os = optional(string, "cos-125-19216-220-185")
production = optional(bool, false)
}), {})
disks = optional(object({
type = optional(string, "hyperdisk-balanced")
data_size_gb = optional(number, 20)
docker_volumes_size_gb = optional(number, 50)
}), {})
network = optional(object({
create = optional(bool, true)
project_id = optional(string, "")
name = optional(string, "")
subnetwork = optional(string, "")
ip_cidr_range = optional(string, "10.42.0.0/24")
mtu = optional(number, 1460)
power_button_allowed_ips = optional(list(string), [])
power_button_ip_depth = optional(number)
ssh_ipv4 = optional(list(string), [])
ssh_ipv6 = optional(list(string), [])
}), {})
snapshots = optional(object({
enabled = optional(bool, true)
}), {})
overlay = optional(object({
source_instance = optional(string, "")
volume_names = optional(list(string), [])
}), {})
cloud_init = optional(object({
initcmd = optional(list(string), [])
runcmd = optional(list(string), [])
}), {})
artifact_registry = optional(object({
repository = optional(string, "")
location = optional(string, "us")
}), {})
power_management = optional(object({
enabled = optional(bool, false)
start_role = optional(string, "")
suspend_role = optional(string, "")
frontend = optional(object({
image = string
port = optional(number, 8080)
cpu = optional(string, "1000m")
memory = optional(string, "1Gi")
}), null)
}), {})
rollout = optional(object({
enabled = optional(bool, false)
release_url = optional(string, "")
release_sha256 = optional(string, "")
port = optional(number, 8081)
jwks_uri = optional(string, "")
jwt_audience = optional(string, "")
custom_claims = optional(string, "")
allowed_ipv4 = optional(list(string), ["10.0.0.0/8"])
}), {})
}) | `{}` | no |
-| [runtime](#input\_runtime) | Provider-neutral compose/runtime settings. | object({
rootfs = optional(string, "")
rootfs_archive_url = optional(string, "")
rootfs_archive_sha256 = optional(string, "")
users = optional(map(list(string)), {})
compose = optional(object({
primary = optional(string, "")
ingress_port = optional(number, 80)
ingress = optional(object({
letsencrypt = optional(bool, false)
bot_mitigation = optional(bool, false)
mode = optional(string, "")
domain = optional(string, "")
acme_email = optional(string, "")
trusted_ips = optional(list(string), [])
max_upload_size = optional(string, "")
upload_timeout = optional(string, "")
}), {})
repo = optional(string, "")
branch = optional(string, "")
projects = optional(map(object({
docker_compose_repo = string
docker_compose_branch = optional(string)
project_dir = optional(string)
compose_project_name = optional(string)
ingress_port = optional(number)
ingress = optional(object({
letsencrypt = optional(bool)
bot_mitigation = optional(bool)
mode = optional(string)
domain = optional(string)
acme_email = optional(string)
trusted_ips = optional(list(string))
max_upload_size = optional(string)
upload_timeout = optional(string)
}), {})
sitectl_context_name = optional(string)
sitectl_plugin = optional(string)
sitectl_environment = optional(string)
sitectl_packages = optional(list(string))
sitectl_verify_args = optional(list(string))
docker_compose_init = optional(list(string))
docker_compose_up = optional(list(string))
docker_compose_down = optional(list(string))
docker_compose_rollout = optional(list(string))
})), {})
init = optional(list(string))
up = optional(list(string))
down = optional(list(string))
rollout = optional(list(string))
}), {})
sitectl = optional(object({
packages = optional(list(string))
version = optional(string, "latest")
package_versions = optional(map(string), {})
context_name = optional(string, "")
plugin = optional(string, "core")
environment = optional(string, "production")
verify_args = optional(list(string), [])
}), {})
docker = optional(object({
# renovate: datasource=github-releases depName=docker-compose packageName=docker/compose versioning=semver
compose_version = optional(string, "v5.3.1")
# renovate: datasource=github-releases depName=docker-buildx packageName=docker/buildx versioning=semver
buildx_version = optional(string, "v0.35.0")
}), {})
managed_runtime = optional(object({
enabled = optional(bool, true)
internal_services_enabled = optional(bool, false)
internal_services_auto_update = optional(bool, false)
artifacts = optional(list(object({
name = string
url = string
sha256 = string
path = string
mode = optional(string, "0755")
owner = optional(string, "root")
group = optional(string, "root")
restart = optional(string, "")
})), [])
}), {})
vault = optional(object({
addr = optional(string, "")
namespace = optional(string, "")
role = optional(string, "")
agent_enabled = optional(bool, false)
auth_method = optional(string, "auto")
gcp_auth_mount_path = optional(string, "auth/gcp")
agent_token_path = optional(string, "/mnt/disks/data/vault/token")
agent_additional_config = optional(string, "")
agent_templates = optional(list(object({
destination = string
contents = string
perms = optional(string, "0640")
command = optional(string, "")
})), [])
}), {})
extra_env = optional(map(string), {})
}) | `{}` | no |
+| [runtime](#input\_runtime) | Provider-neutral compose/runtime settings. | object({
rootfs = optional(string, "")
rootfs_archive_url = optional(string, "")
rootfs_archive_sha256 = optional(string, "")
users = optional(map(list(string)), {})
disaster_recovery = optional(object({
required = optional(bool, false)
driver_path = optional(string, "/etc/cloud-compose/libexec/offhost-backup-driver")
}), {})
compose = optional(object({
primary = optional(string, "")
ingress_port = optional(number, 80)
ingress = optional(object({
letsencrypt = optional(bool, false)
bot_mitigation = optional(bool, false)
mode = optional(string, "")
domain = optional(string, "")
acme_email = optional(string, "")
trusted_ips = optional(list(string), [])
max_upload_size = optional(string, "")
upload_timeout = optional(string, "")
}), {})
repo = optional(string, "")
branch = optional(string, "")
projects = optional(map(object({
docker_compose_repo = string
docker_compose_branch = optional(string)
project_dir = optional(string)
compose_project_name = optional(string)
ingress_port = optional(number)
ingress = optional(object({
letsencrypt = optional(bool)
bot_mitigation = optional(bool)
mode = optional(string)
domain = optional(string)
acme_email = optional(string)
trusted_ips = optional(list(string))
max_upload_size = optional(string)
upload_timeout = optional(string)
}), {})
sitectl_context_name = optional(string)
sitectl_plugin = optional(string)
sitectl_environment = optional(string)
sitectl_packages = optional(list(string))
sitectl_verify_args = optional(list(string))
docker_compose_init = optional(list(string))
docker_compose_up = optional(list(string))
docker_compose_down = optional(list(string))
docker_compose_rollout = optional(list(string))
})), {})
init = optional(list(string))
up = optional(list(string))
down = optional(list(string))
rollout = optional(list(string))
}), {})
sitectl = optional(object({
packages = optional(list(string))
version = optional(string, "latest")
package_versions = optional(map(string), {})
context_name = optional(string, "")
plugin = optional(string, "core")
environment = optional(string, "production")
verify_args = optional(list(string), [])
}), {})
docker = optional(object({
# renovate: datasource=github-releases depName=docker-compose packageName=docker/compose versioning=semver
compose_version = optional(string, "v5.3.1")
# renovate: datasource=github-releases depName=docker-buildx packageName=docker/buildx versioning=semver
buildx_version = optional(string, "v0.35.0")
}), {})
managed_runtime = optional(object({
enabled = optional(bool, true)
internal_services_enabled = optional(bool, false)
internal_services_auto_update = optional(bool, false)
artifacts = optional(list(object({
name = string
url = string
sha256 = string
path = string
mode = optional(string, "0755")
owner = optional(string, "root")
group = optional(string, "root")
restart = optional(string, "")
})), [])
}), {})
vault = optional(object({
addr = optional(string, "")
namespace = optional(string, "")
role = optional(string, "")
agent_enabled = optional(bool, false)
auth_method = optional(string, "auto")
gcp_auth_mount_path = optional(string, "auth/gcp")
agent_token_path = optional(string, "/mnt/disks/data/vault/token")
agent_additional_config = optional(string, "")
agent_templates = optional(list(object({
destination = string
contents = string
perms = optional(string, "0640")
command = optional(string, "")
})), [])
}), {})
extra_env = optional(map(string), {})
}) | `{}` | no |
| [template](#input\_template) | Optional compose template preset. Supported values are archivesspace, ojs, isle, drupal, wp, omeka-s, and omeka-classic. Explicit runtime settings override preset defaults. | `string` | `""` | no |
## Outputs
diff --git a/ansible/README.md b/ansible/README.md
index b849020..f619bea 100644
--- a/ansible/README.md
+++ b/ansible/README.md
@@ -62,6 +62,21 @@ is meaningful: it disables that phase and is not replaced by the default.
Runtime feature switches must be YAML booleans, not quoted strings; ambiguous
values are rejected before host mutation.
+The provider-neutral disaster-recovery interface is shared with Terraform:
+
+```yaml
+cloud_compose_runtime:
+ disaster_recovery:
+ required: true
+ driver_path: /etc/cloud-compose/libexec/offhost-backup-driver
+```
+
+Install that executable and its credentials separately as root. The role
+renders only the boolean requirement and executable path; storage endpoints,
+credentials, encryption keys, and retention policy must not be placed in
+inventory. See [the driver contract](../docs/disaster-recovery.md) for receipt
+and scheduled restore-proof requirements.
+
Set `cloud_compose_runtime.rollout` to enable the same authenticated rollout
listener used by Terraform. Supply a pinned HTTPS `release_url`, its lowercase
`release_sha256`, an HTTPS `jwks_uri`, `jwt_audience`, and optional JSON-object
@@ -73,7 +88,8 @@ The role installs lifecycle dispatchers as `root:cloud-compose` mode `0750` and
the root-consumed `.env`, project/application JSON, and managed-artifact
manifest as `root:cloud-compose` mode `0640`. Reapplying the role restores that
ownership boundary while leaving app checkout directories writable by the
-`cloud-compose` account.
+`cloud-compose` account. Before a requested runtime bootstrap, the role invokes
+the same checked-in bootstrap path hardener used by the Terraform modules.
The normal on-prem shape is one app per machine. Put each machine in the
`cloud_compose` inventory group and set that host's template/runtime variables.
@@ -100,7 +116,7 @@ all:
sitectl:
environment: production
package_versions:
- sitectl-isle: v1.0.0
+ sitectl-isle: v1.5.0
wp-prod.example.edu:
ansible_user: debian
cloud_compose_name: wp-prod
diff --git a/ansible/roles/cloud_compose/defaults/main.yml b/ansible/roles/cloud_compose/defaults/main.yml
index 4dff754..34bd496 100644
--- a/ansible/roles/cloud_compose/defaults/main.yml
+++ b/ansible/roles/cloud_compose/defaults/main.yml
@@ -53,7 +53,7 @@ cloud_compose_default_ingress:
upload_timeout: ""
cloud_compose_default_init:
- - 'sitectl config set-context "${SITECTL_CONTEXT_NAME}" --type local --project-dir "${DOCKER_COMPOSE_DIR}" --site "${CLOUD_COMPOSE_INSTANCE_NAME}" --plugin "${SITECTL_PLUGIN}" --environment "${SITECTL_ENVIRONMENT}" --project-name "${CLOUD_COMPOSE_INSTANCE_NAME}" --compose-project-name "${COMPOSE_PROJECT_NAME}" --docker-socket /var/run/docker.sock --env-file .env --default'
+ - 'sitectl config set-context "${SITECTL_CONTEXT_NAME}" --type local --project-dir "${DOCKER_COMPOSE_DIR}" --site "${CLOUD_COMPOSE_INSTANCE_NAME}" --plugin "${SITECTL_PLUGIN}" --environment "${SITECTL_ENVIRONMENT}" --compose-project-name "${COMPOSE_PROJECT_NAME}" --docker-socket /var/run/docker.sock --env-file .env --yolo --default'
cloud_compose_default_up:
- 'sitectl compose --context "${SITECTL_CONTEXT_NAME}" up -d --remove-orphans'
- 'sitectl healthcheck --context "${SITECTL_CONTEXT_NAME}" --persist'
diff --git a/ansible/roles/cloud_compose/tasks/main.yml b/ansible/roles/cloud_compose/tasks/main.yml
index 7057042..9fe1bea 100644
--- a/ansible/roles/cloud_compose/tasks/main.yml
+++ b/ansible/roles/cloud_compose/tasks/main.yml
@@ -55,6 +55,7 @@
_cc_sitectl: "{{ cloud_compose_runtime.sitectl | default({}) }}"
_cc_managed: "{{ cloud_compose_runtime.managed_runtime | default({}) }}"
_cc_vault: "{{ cloud_compose_runtime.vault | default({}) }}"
+ _cc_disaster_recovery: "{{ cloud_compose_runtime.disaster_recovery | default({}) }}"
_cc_rollout_service: "{{ cloud_compose_default_rollout_service | combine(cloud_compose_runtime.rollout | default({}), recursive=True) }}"
_cc_extra_env: "{{ cloud_compose_runtime.extra_env | default(cloud_compose_extra_env) }}"
@@ -65,6 +66,7 @@
- _cc_sitectl is mapping
- _cc_managed is mapping
- _cc_vault is mapping
+ - _cc_disaster_recovery is mapping
- (cloud_compose_runtime.rollout | default({})) is mapping
- (_cc_compose.ingress | default({})) is mapping
- (_cc_compose.projects | default({})) is mapping
@@ -79,7 +81,12 @@
- (_cc_managed.internal_services_auto_update | default(cloud_compose_internal_services_auto_update)) is boolean
- (_cc_vault.agent_enabled | default(false)) is boolean
- not (_cc_vault.agent_enabled | default(false))
- fail_msg: "runtime sections must be maps, runtime feature switches must be booleans, template must name a supported app, and Vault Agent is currently supported only by Terraform providers (set vault.agent_enabled=false for Ansible)."
+ - (_cc_disaster_recovery.required | default(false)) is boolean
+ - (_cc_disaster_recovery.driver_path | default('/etc/cloud-compose/libexec/offhost-backup-driver')) is string
+ - (_cc_disaster_recovery.driver_path | default('/etc/cloud-compose/libexec/offhost-backup-driver')) is match('^/[A-Za-z0-9._/+:-]+$')
+ - "'//' not in (_cc_disaster_recovery.driver_path | default('/etc/cloud-compose/libexec/offhost-backup-driver'))"
+ - (_cc_disaster_recovery.driver_path | default('/etc/cloud-compose/libexec/offhost-backup-driver') | regex_search('(^|/)\\.\\.?(/|$)')) is none
+ fail_msg: "runtime sections must be maps, runtime feature switches must be booleans, template must name a supported app, Vault Agent is currently supported only by Terraform providers (set vault.agent_enabled=false for Ansible), and disaster_recovery.driver_path must be a safe absolute path without whitespace or dot segments."
- name: Require safe rollout service settings
ansible.builtin.assert:
@@ -136,6 +143,8 @@
_cc_down_commands: "{{ _cc_compose.down | default(cloud_compose_default_down) }}"
_cc_rollout_commands: "{{ _cc_compose.rollout | default(cloud_compose_default_rollout) }}"
_cc_internal_services_enabled: "{{ _cc_managed.internal_services_enabled | default(cloud_compose_internal_services_enabled) }}"
+ _cc_offhost_backup_required: "{{ _cc_disaster_recovery.required | default(false) }}"
+ _cc_offhost_backup_driver: "{{ _cc_disaster_recovery.driver_path | default('/etc/cloud-compose/libexec/offhost-backup-driver') }}"
- name: Reject GCP-only internal services on an on-prem host
ansible.builtin.assert:
@@ -398,6 +407,8 @@
CLOUD_COMPOSE_INSTANCE_NAME: "{{ cloud_compose_name }}"
CLOUD_COMPOSE_APPS: "{{ _cc_compose_projects.keys() | list | join(' ') }}"
CLOUD_COMPOSE_PRIMARY_APP: "{{ _cc_primary_key }}"
+ CLOUD_COMPOSE_OFFHOST_BACKUP_REQUIRED: "{{ _cc_offhost_backup_required | ternary('true', 'false') }}"
+ CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER: "{{ _cc_offhost_backup_driver }}"
COMPOSE_PROJECTS_FILE: "{{ cloud_compose_home }}/compose-projects.json"
COMPOSE_PROJECT_NAME: "{{ _cc_primary_project.compose_project_name }}"
COMPOSE_BIND_PORT: "{{ _cc_primary_project.ingress_port }}"
@@ -503,6 +514,19 @@
dest: /
mode: preserve
+- name: Secure Cloud Compose privileged program directories
+ ansible.builtin.file:
+ path: "{{ item }}"
+ state: directory
+ owner: root
+ group: root
+ mode: "0755"
+ loop:
+ - /etc/cloud-compose
+ - /etc/cloud-compose/bin
+ - /etc/cloud-compose/jq
+ - /etc/cloud-compose/libexec
+
- name: Inspect cloud-compose lifecycle lock directory
ansible.builtin.stat:
path: /run/lock/cloud-compose
@@ -532,20 +556,41 @@
(cloud_compose_lifecycle_lock_file.stat.mode | default('')) == '0660'
)
-- name: Find cloud-compose shell scripts
+- name: Find top-level cloud-compose control scripts
ansible.builtin.find:
- paths: "{{ cloud_compose_home }}"
+ paths:
+ - "{{ cloud_compose_home }}"
+ - /etc/cloud-compose/bin
+ - /etc/cloud-compose/libexec
patterns: "*.sh"
- recurse: true
+ recurse: false
file_type: file
register: cloud_compose_shell_scripts
-- name: Make cloud-compose shell scripts executable
+- name: Make cloud-compose control scripts root-owned and executable
ansible.builtin.file:
path: "{{ item.path }}"
+ owner: root
+ group: root
mode: "0755"
loop: "{{ cloud_compose_shell_scripts.files }}"
+- name: Find checked-in Cloud Compose jq programs
+ ansible.builtin.find:
+ paths: /etc/cloud-compose/jq
+ patterns: "*.jq"
+ recurse: false
+ file_type: file
+ register: cloud_compose_jq_programs
+
+- name: Make checked-in Cloud Compose jq programs root-controlled
+ ansible.builtin.file:
+ path: "{{ item.path }}"
+ owner: root
+ group: root
+ mode: "0644"
+ loop: "{{ cloud_compose_jq_programs.files }}"
+
- name: Write cloud-compose lifecycle dispatchers
ansible.builtin.copy:
dest: "{{ cloud_compose_home }}/{{ item }}"
@@ -599,6 +644,11 @@
mode: "0640"
no_log: true
+- name: Harden trusted cloud-compose bootstrap paths
+ ansible.builtin.command:
+ cmd: /etc/cloud-compose/libexec/harden-bootstrap-paths.sh
+ when: cloud_compose_run_bootstrap | bool
+
- name: Reload systemd units
ansible.builtin.systemd:
daemon_reload: true
@@ -606,19 +656,28 @@
- name: Install and start authenticated rollout service
ansible.builtin.command:
- cmd: bash "{{ cloud_compose_home }}/deploy-rollout.sh"
+ cmd: bash /etc/cloud-compose/libexec/run-root-program.sh deploy-rollout.sh
when: _cc_rollout_service.enabled | bool
- name: Clear cloud-compose bootstrap marker
ansible.builtin.file:
- path: "{{ cloud_compose_home }}/.cloud-compose-bootstrap-complete"
+ path: /var/lib/cloud-compose/bootstrap-complete
state: absent
when: cloud_compose_force_bootstrap | bool
+- name: Inspect validated cloud-compose bootstrap readiness
+ ansible.builtin.command:
+ cmd: bash /etc/cloud-compose/libexec/require-bootstrap-ready.sh
+ register: cloud_compose_bootstrap_readiness
+ changed_when: false
+ failed_when: false
+ when: cloud_compose_run_bootstrap | bool
+
- name: Bootstrap cloud-compose runtime
ansible.builtin.command:
- cmd: bash "{{ cloud_compose_home }}/start-cloud-compose-bootstrap.sh"
- creates: "{{ cloud_compose_home }}/.cloud-compose-bootstrap-complete"
+ cmd: bash /etc/cloud-compose/libexec/start-cloud-compose-bootstrap.sh
async: "{{ cloud_compose_bootstrap_timeout }}"
poll: "{{ cloud_compose_bootstrap_poll_interval }}"
- when: cloud_compose_run_bootstrap | bool
+ when:
+ - cloud_compose_run_bootstrap | bool
+ - (cloud_compose_bootstrap_readiness.rc | default(1)) != 0
diff --git a/ci/bootstrap-recovery-contract.sh b/ci/bootstrap-recovery-contract.sh
index e37e1ba..5486cf9 100644
--- a/ci/bootstrap-recovery-contract.sh
+++ b/ci/bootstrap-recovery-contract.sh
@@ -9,6 +9,11 @@ start_bootstrap="$repo_root/rootfs/home/cloud-compose/start-cloud-compose-bootst
run_script="$repo_root/rootfs/home/cloud-compose/run.sh"
app_init="$repo_root/rootfs/home/cloud-compose/app-init.sh"
run_bootstrap="$repo_root/rootfs/home/cloud-compose/run-bootstrap.sh"
+bootstrap_security="$repo_root/rootfs/etc/cloud-compose/libexec/bootstrap-security.sh"
+bootstrap_required="$repo_root/rootfs/etc/cloud-compose/libexec/bootstrap-required.sh"
+bootstrap_entrypoint="$repo_root/rootfs/etc/cloud-compose/libexec/run-bootstrap.sh"
+bootstrap_start_entrypoint="$repo_root/rootfs/etc/cloud-compose/libexec/start-cloud-compose-bootstrap.sh"
+bootstrap_readiness_gate="$repo_root/rootfs/etc/cloud-compose/libexec/require-bootstrap-ready.sh"
bootstrap_unit="$repo_root/rootfs/etc/systemd/system/cloud-compose-bootstrap.service"
tmp="$(mktemp -d)"
trap 'rm -rf "$tmp"' EXIT
@@ -182,10 +187,14 @@ fi
assert_contains "$app_init" 'acquire_cloud_compose_lifecycle_lock "init"'
assert_contains "$app_init" 'trap release_cloud_compose_lifecycle_lock EXIT'
assert_contains "$run_script" 'if cloud_compose_should_run_app_init'
+assert_contains "$run_script" 'durable_bootstrap_marker="/var/lib/cloud-compose/bootstrap-complete"'
assert_contains "$run_script" 'cloud_compose_publish_marker "$current_boot_app_init_marker"'
assert_contains "$run_script" 'cloud_compose_start_and_wait_for_oneshot cloud-compose.service "$app_wait_seconds"'
assert_contains "$run_script" '"$fresh_filesystem_marker" "$fresh_filesystem_identity"'
assert_contains "$run_script" 'cloud_compose_publish_marker "$durable_bootstrap_marker"'
+assert_contains "$helpers" 'if ((EUID == 0)) && ! chown 0:0 "$tmp_marker"; then'
+assert_contains "$helpers" '0:0:644:1:regular file'
+assert_contains "$helpers" '0:0:755:directory'
[[ "$(grep -Fc 'cloud_compose_consume_fresh_filesystem_marker \' "$run_script")" == "1" ]] ||
fail "bootstrap must have one fresh-filesystem authority consumption boundary"
rotation_line="$(grep -nF 'bash /home/cloud-compose/rotate-keys-daily.sh' "$run_script" | cut -d: -f1)"
@@ -209,6 +218,16 @@ durable_marker_line="$(grep -nF 'cloud_compose_publish_marker "$durable_bootstra
assert_contains "$run_bootstrap" 'if ((EUID != 0)); then'
assert_contains "$run_bootstrap" 'exec bash /home/cloud-compose/run.sh'
+assert_contains "$bootstrap_entrypoint" 'cloud_compose_secure_runtime_home'
+assert_contains "$bootstrap_entrypoint" 'exec /bin/bash /home/cloud-compose/run.sh'
+assert_contains "$bootstrap_start_entrypoint" 'cloud_compose_secure_runtime_home'
+assert_contains "$bootstrap_required" 'cloud_compose_bootstrap_marker_ready'
+assert_contains "$bootstrap_readiness_gate" 'cloud_compose_bootstrap_marker_ready'
+assert_contains "$bootstrap_security" '0:0:644:1:regular file'
+assert_contains "$bootstrap_security" '"$marker_size" == "6"'
+assert_contains "$bootstrap_security" '"$payload" == "ready"'
+assert_contains "$bootstrap_security" 'Cloud Compose control input is not root-controlled'
+assert_contains "$bootstrap_security" 'Cloud Compose lifecycle dispatcher is not root-controlled'
if rg -n 'bootstrap\\.log|exec (>>|>)[^[:space:]]' "$run_bootstrap" >/dev/null; then
fail "bootstrap wrapper writes an independently unbounded log file"
fi
@@ -245,6 +264,7 @@ done
fail "published sitectl is not executable after first-boot installation"
assert_contains "$start_bootstrap" 'CLOUD_COMPOSE_BOOTSTRAP_WAIT_SECONDS:-10800'
+assert_contains "$start_bootstrap" 'CLOUD_COMPOSE_BOOTSTRAP_COMPLETE_MARKER:-/var/lib/cloud-compose/bootstrap-complete'
assert_contains "$start_bootstrap" 'if cloud_compose_marker_exists "$durable_marker"; then'
assert_contains "$start_bootstrap" 'systemctl daemon-reload'
assert_contains "$start_bootstrap" 'systemctl stop -- "$bootstrap_unit"'
@@ -257,7 +277,7 @@ fi
for cloud_init_template in \
"$repo_root/templates/cloud-init.yml" \
"$repo_root/modules/linux-vm-runtime/templates/cloud-init.yml"; do
- assert_contains "$cloud_init_template" 'bash /home/cloud-compose/start-cloud-compose-bootstrap.sh'
+ assert_contains "$cloud_init_template" 'bash /etc/cloud-compose/libexec/start-cloud-compose-bootstrap.sh'
if grep -Fq 'bash /home/cloud-compose/run.sh > /home/cloud-compose/run.log 2>&1' \
"$cloud_init_template"; then
fail "cloud-init bypasses the retryable bootstrap unit"
@@ -304,6 +324,8 @@ rm -f -- "$durable_marker"
integration_root="$tmp/integration"
integration_home="$integration_root/home/cloud-compose"
integration_run="$integration_root/run"
+integration_state="$integration_root/var/lib/cloud-compose"
+integration_durable_marker="$integration_state/bootstrap-complete"
integration_bin="$integration_root/bin"
integration_log="$integration_root/systemctl.log"
app_init_count="$integration_root/app-init-count"
@@ -311,7 +333,7 @@ fresh_data_root="$integration_root/data"
fresh_marker="$fresh_data_root/.cloud-compose/fresh-filesystem"
first_output="$integration_root/first-attempt.log"
retry_output="$integration_root/retry-attempt.log"
-mkdir -p "$integration_home" "$integration_run" "$integration_bin" "$(dirname -- "$fresh_marker")"
+mkdir -p "$integration_home" "$integration_run" "$integration_state" "$integration_bin" "$(dirname -- "$fresh_marker")"
: >"$integration_log"
printf '0\n' >"$app_init_count"
printf 'fresh\n' >"$fresh_marker"
@@ -319,6 +341,7 @@ printf 'fresh\n' >"$fresh_marker"
sed \
-e "s#/home/cloud-compose#$integration_home#g" \
-e "s#/run/cloud-compose-app-init-complete#$integration_run/app-init-complete#g" \
+ -e "s#/var/lib/cloud-compose#$integration_state#g" \
"$run_script" >"$integration_home/run.sh"
cp "$helpers" "$integration_home/bootstrap-helpers.sh"
cat >"$integration_home/profile.sh" <<'EOF'
@@ -453,7 +476,7 @@ fi
fail "first full bootstrap attempt did not complete app-init exactly once"
[[ -f "$integration_run/app-init-complete" ]] ||
fail "failed first attempt did not retain current-boot app-init readiness"
-[[ ! -e "$integration_home/.cloud-compose-bootstrap-complete" ]] ||
+[[ ! -e "$integration_durable_marker" ]] ||
fail "failed first attempt published durable bootstrap readiness"
[[ ! -e "$fresh_marker" ]] ||
fail "failed first attempt retained fresh-filesystem authority past key convergence"
@@ -462,7 +485,7 @@ env "${integration_env[@]}" RUN_ATTEMPT=recover \
bash "$integration_home/run.sh" >"$retry_output" 2>&1
[[ "$(<"$app_init_count")" == "1" ]] ||
fail "bootstrap retry repeated successful app-init"
-[[ -f "$integration_home/.cloud-compose-bootstrap-complete" ]] ||
+[[ -f "$integration_durable_marker" ]] ||
fail "bootstrap retry did not publish durable readiness"
[[ ! -e "$fresh_marker" ]] ||
fail "successful bootstrap retry retained fresh-filesystem reconciliation authority"
@@ -474,7 +497,7 @@ grep -Fq 'recover enable -- cloud-compose.service' "$integration_log" ||
fail "bootstrap retry did not converge the application service"
# Marker removal must be flushed before durable readiness can be republished.
-rm -f -- "$integration_home/.cloud-compose-bootstrap-complete"
+rm -f -- "$integration_durable_marker"
printf 'fresh\n' >"$fresh_marker"
if env "${integration_env[@]}" RUN_ATTEMPT=sync-fail \
bash "$integration_home/run.sh" >/dev/null 2>&1; then
@@ -482,16 +505,16 @@ if env "${integration_env[@]}" RUN_ATTEMPT=sync-fail \
fi
[[ ! -e "$fresh_marker" ]] ||
fail "post-consume durability coverage did not remove the fresh marker"
-[[ ! -e "$integration_home/.cloud-compose-bootstrap-complete" ]] ||
+[[ ! -e "$integration_durable_marker" ]] ||
fail "failed post-consume durability barrier published readiness"
env "${integration_env[@]}" RUN_ATTEMPT=recover \
bash "$integration_home/run.sh" >/dev/null 2>&1
-[[ -f "$integration_home/.cloud-compose-bootstrap-complete" ]] ||
+[[ -f "$integration_durable_marker" ]] ||
fail "bootstrap retry did not flush an already-absent marker before readiness"
# GCP never falls back to the generic non-GCP marker identity. This check runs
# before key rotation, so missing disk identity cannot reach IAM.
-rm -f -- "$integration_home/.cloud-compose-bootstrap-complete"
+rm -f -- "$integration_durable_marker"
printf 'fresh\n' >"$fresh_marker"
if env "${integration_env[@]}" CLOUD_COMPOSE_PROVIDER=gcp RUN_ATTEMPT=recover \
bash "$integration_home/run.sh" >/dev/null 2>&1; then
@@ -499,12 +522,12 @@ if env "${integration_env[@]}" CLOUD_COMPOSE_PROVIDER=gcp RUN_ATTEMPT=recover \
fi
[[ -f "$fresh_marker" ]] ||
fail "GCP bootstrap consumed generic fresh-filesystem authority"
-[[ ! -e "$integration_home/.cloud-compose-bootstrap-complete" ]] ||
+[[ ! -e "$integration_durable_marker" ]] ||
fail "GCP bootstrap with generic authority published durable readiness"
# A marker payload for another incarnation must fail before application
# initialization or durable readiness.
-rm -f -- "$integration_home/.cloud-compose-bootstrap-complete"
+rm -f -- "$integration_durable_marker"
printf 'v1:gcp-disk-id:111111111111111111\n' >"$fresh_marker"
if env "${integration_env[@]}" RUN_ATTEMPT=recover \
bash "$integration_home/run.sh" >/dev/null 2>&1; then
@@ -512,13 +535,13 @@ if env "${integration_env[@]}" RUN_ATTEMPT=recover \
fi
[[ -f "$fresh_marker" ]] ||
fail "mismatched fresh-filesystem marker payload was consumed"
-[[ ! -e "$integration_home/.cloud-compose-bootstrap-complete" ]] ||
+[[ ! -e "$integration_durable_marker" ]] ||
fail "mismatched fresh-filesystem marker payload published durable readiness"
# Unsafe authority must fail closed before durable readiness. The current-boot
# app-init marker remains valid, so these attempts exercise only the early
# marker boundary rather than repeating application initialization.
-rm -f -- "$integration_home/.cloud-compose-bootstrap-complete"
+rm -f -- "$integration_durable_marker"
printf 'fresh\n' >"$fresh_marker"
if env "${integration_env[@]}" RUN_ATTEMPT=recover \
FRESH_MARKER_IDENTITY=1000:1000:600:1 \
@@ -527,7 +550,7 @@ if env "${integration_env[@]}" RUN_ATTEMPT=recover \
fi
[[ -f "$fresh_marker" ]] ||
fail "unsafe fresh-filesystem marker was consumed"
-[[ ! -e "$integration_home/.cloud-compose-bootstrap-complete" ]] ||
+[[ ! -e "$integration_durable_marker" ]] ||
fail "unsafe fresh-filesystem marker published durable readiness"
rm -f -- "$fresh_marker"
@@ -538,7 +561,7 @@ if env "${integration_env[@]}" RUN_ATTEMPT=recover \
fi
[[ -L "$fresh_marker" ]] ||
fail "symlink fresh-filesystem marker was consumed"
-[[ ! -e "$integration_home/.cloud-compose-bootstrap-complete" ]] ||
+[[ ! -e "$integration_durable_marker" ]] ||
fail "symlink fresh-filesystem marker published durable readiness"
echo "Bootstrap recovery contract passed"
diff --git a/ci/cloud-smoke.sh b/ci/cloud-smoke.sh
index 97a18fb..eb6ac11 100755
--- a/ci/cloud-smoke.sh
+++ b/ci/cloud-smoke.sh
@@ -3,6 +3,8 @@
set -euo pipefail
repo_root="$(cd "$(dirname "$0")/.." && pwd)"
+readonly diagnostics_program="/etc/cloud-compose/bin/cloud-compose-diagnostics.sh"
+readonly smoke_healthcheck_program="/home/cloud-compose/smoke-healthcheck.sh"
usage() {
cat <<'EOF'
@@ -377,42 +379,36 @@ wait_for_ssh() {
return 1
}
+remote_diagnostics_available() {
+ local home_dir="$1" key_path="$2" host="$3" port="$4" user="$5"
+
+ ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \
+ "test -x ${diagnostics_program}" >/dev/null 2>&1
+}
+
remote_bootstrap_state() {
local home_dir="$1" key_path="$2" host="$3" port="$4" user="$5"
- ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" "bash -lc 'set +e
-if [ -f /home/cloud-compose/.cloud-compose-bootstrap-complete ]; then
- echo complete
- exit 0
-fi
-bootstrap_load_state=\"\$(systemctl show --property=LoadState --value -- cloud-compose-bootstrap.service 2>/dev/null)\"
-if [ \"\$bootstrap_load_state\" = loaded ]; then
- bootstrap_active_state=\"\$(systemctl show --property=ActiveState --value -- cloud-compose-bootstrap.service 2>/dev/null)\"
- bootstrap_sub_state=\"\$(systemctl show --property=SubState --value -- cloud-compose-bootstrap.service 2>/dev/null)\"
- case \"\$bootstrap_active_state:\$bootstrap_sub_state\" in
- active:* | activating:* | *:auto-restart)
- echo active
- exit 0
- ;;
- esac
-elif [ \"\$bootstrap_load_state\" = not-found ] &&
- systemctl is-active --quiet cloud-compose; then
- # Releases before retryable bootstrap have no durable unit or marker; their
- # active application service remains the compatibility completion signal.
- echo complete
- exit 0
-fi
-if systemctl is-active --quiet cloud-final.service; then
- echo active
- exit 0
-fi
-if pgrep -f \"[/]home/cloud-compose/run[.]sh|[/]home/cloud-compose/[h]ost-conf[.]sh|[/]home/cloud-compose/[h]ost-init[.]sh|[/]home/cloud-compose/[a]pp-init[.]sh|[/]home/cloud-compose/[i]nstall-dependencies|[a]pt-get|[r]pm-ostree|[d]ocker run|[s]itectl|[g]it clone\" >/dev/null; then
- echo active
- exit 0
-fi
-echo idle
-exit 1
-'" 2>/dev/null || true
+ if remote_diagnostics_available "$home_dir" "$key_path" "$host" "$port" "$user"; then
+ ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \
+ "sudo -n ${diagnostics_program} state" 2>/dev/null || true
+ return
+ fi
+
+ # The pinned upgrade fixture predates the checked-in diagnostics program.
+ # Keep its compatibility probes simple and non-interactive.
+ if ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \
+ "test -f /home/cloud-compose/.cloud-compose-bootstrap-complete" >/dev/null 2>&1; then
+ echo complete
+ elif ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \
+ "systemctl is-active --quiet cloud-compose.service" >/dev/null 2>&1; then
+ echo complete
+ elif ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \
+ "systemctl is-active --quiet cloud-final.service" >/dev/null 2>&1; then
+ echo active
+ else
+ echo idle
+ fi
}
wait_for_cloud_init() {
@@ -421,25 +417,47 @@ wait_for_cloud_init() {
local deadline
local status_output
local bootstrap_state
- local last_dump=0
+ local diagnostics_available
timeout_seconds="$(boot_timeout_seconds)"
deadline=$((SECONDS + timeout_seconds))
echo "Waiting for cloud-init on ${host}"
while (( SECONDS < deadline )); do
+ diagnostics_available=false
+ if remote_diagnostics_available "$home_dir" "$key_path" "$host" "$port" "$user"; then
+ diagnostics_available=true
+ fi
bootstrap_state="$(remote_bootstrap_state "$home_dir" "$key_path" "$host" "$port" "$user")"
if [[ "$bootstrap_state" == "complete" ]]; then
return 0
fi
- status_output="$(
- ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \
- "if command -v cloud-init >/dev/null 2>&1; then sudo cloud-init status --long 2>&1; else echo 'cloud-init not installed'; fi" 2>&1 || true
- )"
+ if [[ "$diagnostics_available" == "true" ]]; then
+ status_output="$(
+ ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \
+ "sudo -n ${diagnostics_program} status" 2>&1 || true
+ )"
+ else
+ status_output="$(
+ ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \
+ "cloud-init status --long" 2>&1 || true
+ )"
+ fi
printf '%s\n' "$status_output"
if grep -q '^status: done' <<<"$status_output"; then
+ if [[ "$diagnostics_available" == "true" ]]; then
+ if [[ "$bootstrap_state" == "active" ]]; then
+ echo "cloud-init is done while cloud-compose bootstrap is still active; continuing"
+ sleep 30
+ continue
+ fi
+ echo "cloud-init completed without the Cloud Compose readiness marker" >&2
+ return 1
+ fi
+ # The pinned upgrade fixture predates the durable readiness marker.
+ # Retain cloud-init completion as its final compatibility signal.
return 0
fi
if grep -q '^status: error' <<<"$status_output"; then
@@ -451,24 +469,6 @@ wait_for_cloud_init() {
return 1
fi
- if (( SECONDS - last_dump >= 120 )); then
- last_dump=$SECONDS
- ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" "bash -lc 'set +e
-echo \"--- active bootstrap processes ---\"
-ps -eo pid,ppid,stat,etime,args | grep -E \"cloud-init|runcmd|run.sh|host-conf|host-init|app-init|install-dependencies|apt-get|docker|sitectl|git clone\" | grep -v grep
-echo \"--- cloud-compose bootstrap unit ---\"
-sudo journalctl -u cloud-compose-bootstrap --no-pager -n 160
-echo \"--- legacy cloud-compose bootstrap log (when present) ---\"
-if sudo test -f /home/cloud-compose/run.log && sudo test ! -L /home/cloud-compose/run.log; then
- sudo tail -n 160 /home/cloud-compose/run.log
-else
- echo \"Legacy bootstrap log is not present\"
-fi
-echo \"--- /var/log/cloud-init-output.log ---\"
-sudo tail -n 120 /var/log/cloud-init-output.log
-'" || true
- fi
-
sleep 30
done
@@ -482,40 +482,50 @@ dump_remote_logs() {
quoted_project_dir="$(shell_quote "$project_dir")"
echo "Dumping smoke-test diagnostics from ${host}" >&2
- ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" "bash -lc 'set +e
-echo \"--- cloud-init status ---\"
-sudo cloud-init status --long
-echo \"--- /var/log/cloud-init-output.log ---\"
-sudo tail -n 400 /var/log/cloud-init-output.log
-echo \"--- /var/log/cloud-init.log ---\"
-sudo tail -n 400 /var/log/cloud-init.log
-echo \"--- cloud-init runcmd ---\"
-sudo sed -n '1,240p' /var/lib/cloud/instance/scripts/runcmd
-echo \"--- cloud-compose bootstrap unit ---\"
-sudo journalctl -u cloud-compose-bootstrap --no-pager -n 400
-echo \"--- legacy cloud-compose bootstrap log (when present) ---\"
-if sudo test -f /home/cloud-compose/run.log && sudo test ! -L /home/cloud-compose/run.log; then
- sudo tail -n 400 /home/cloud-compose/run.log
-else
- echo \"Legacy bootstrap log is not present\"
-fi
-echo \"--- cloud-compose unit ---\"
-sudo journalctl -u cloud-compose --no-pager -n 300
-echo \"--- lifecycle lock permissions ---\"
-sudo stat -Lc '%A %a %U:%G %u:%g %n' /run/lock/cloud-compose /run/lock/cloud-compose/lifecycle.lock
-echo \"--- docker ps ---\"
-sudo docker ps -a
-echo \"--- docker compose ps ---\"
-if [ -d ${quoted_project_dir} ]; then
- if command -v runuser >/dev/null 2>&1; then
- runuser -u cloud-compose -- env HOME=/home/cloud-compose PROJECT_DIR=${quoted_project_dir} bash -lc \"source /home/cloud-compose/profile.sh && cd \\\"\$PROJECT_DIR\\\" && docker compose ps\"
- else
- sudo -u cloud-compose env HOME=/home/cloud-compose PROJECT_DIR=${quoted_project_dir} bash -lc \"source /home/cloud-compose/profile.sh && cd \\\"\$PROJECT_DIR\\\" && docker compose ps\"
+ if remote_diagnostics_available "$home_dir" "$key_path" "$host" "$port" "$user"; then
+ ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \
+ "sudo -n ${diagnostics_program} dump" || true
+ return
fi
-else
- echo \"Project directory ${project_dir} is not present yet\"
-fi
-'" || true
+
+ # Compatibility diagnostics for the pinned pre-program upgrade fixture.
+ echo "--- legacy cloud-init status ---"
+ ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \
+ "cloud-init status --long" || true
+ echo "--- legacy /var/log/cloud-init-output.log ---"
+ ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \
+ "tail -n 400 /var/log/cloud-init-output.log" || true
+ echo "--- legacy /var/log/cloud-init.log ---"
+ ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \
+ "tail -n 400 /var/log/cloud-init.log" || true
+ echo "--- legacy cloud-init runcmd ---"
+ ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \
+ "sed -n 1,240p /var/lib/cloud/instance/scripts/runcmd" || true
+ echo "--- legacy cloud-compose bootstrap unit ---"
+ ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \
+ "systemctl status cloud-compose-bootstrap.service --no-pager" || true
+ echo "--- legacy cloud-compose bootstrap log ---"
+ if ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \
+ "test -f /home/cloud-compose/run.log" >/dev/null 2>&1 &&
+ ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \
+ "test ! -L /home/cloud-compose/run.log" >/dev/null 2>&1; then
+ ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \
+ "tail -n 400 /home/cloud-compose/run.log" || true
+ else
+ echo "Legacy bootstrap log is not present"
+ fi
+ echo "--- legacy cloud-compose unit ---"
+ ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \
+ "sudo -n /usr/bin/systemctl status cloud-compose.service" || true
+ echo "--- legacy lifecycle lock permissions ---"
+ ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \
+ "stat -Lc '%A %a %U:%G %u:%g %n' /run/lock/cloud-compose /run/lock/cloud-compose/lifecycle.lock" || true
+ echo "--- legacy docker ps ---"
+ ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \
+ "docker ps -a" || true
+ echo "--- legacy docker compose ps ---"
+ ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \
+ "docker compose --project-directory ${quoted_project_dir} ps" || true
}
configure_sitectl_context() {
@@ -564,11 +574,17 @@ run_healthcheck() {
user="$(jq -r '.ssh_user' "$output_json")"
quoted_context="$(shell_quote "$context")"
- ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" "bash -lc 'set -euo pipefail
-export HOME=/home/cloud-compose
-source /home/cloud-compose/profile.sh
-exec sitectl healthcheck --context ${quoted_context} --persist --format table
-'"
+ if ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \
+ "test -x ${smoke_healthcheck_program}" >/dev/null 2>&1; then
+ ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \
+ "${smoke_healthcheck_program} ${quoted_context}"
+ else
+ # The pinned 0.10.2 upgrade fixture predates the checked-in wrapper.
+ # Invoke its sitectl binary directly with the environment profile's
+ # stable path settings, without sending an embedded shell program.
+ ssh_cmd "$home_dir" "$key_path" "$host" "$port" "$user" \
+ "env HOME=/home/cloud-compose DOCKER_CONFIG=/mnt/disks/data/docker-config PATH=/home/cloud-compose/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin sitectl healthcheck --context ${quoted_context} --persist --format table"
+ fi
return
fi
diff --git a/ci/compose-runtime-contract.sh b/ci/compose-runtime-contract.sh
index 31537ad..6a28199 100755
--- a/ci/compose-runtime-contract.sh
+++ b/ci/compose-runtime-contract.sh
@@ -27,6 +27,20 @@ mkdir -p "$CLOUD_COMPOSE_DATA_ROOT/project"
# shellcheck disable=SC1091
source "$repo_root/rootfs/home/cloud-compose/compose-apps.sh"
+cat >"$CLOUD_COMPOSE_DATA_ROOT/project/compose.yaml" <<'EOF'
+secrets:
+ DB_ROOT_PASSWORD:
+ file: ./secrets/DB_ROOT_PASSWORD
+ WORDPRESS_DB_PASSWORD:
+ file: ./secrets/WORDPRESS_DB_PASSWORD
+EOF
+mapfile -t compose_secrets < <(
+ cd "$CLOUD_COMPOSE_DATA_ROOT/project"
+ compose_secret_files
+)
+[[ "${compose_secrets[*]}" == "./secrets/DB_ROOT_PASSWORD ./secrets/WORDPRESS_DB_PASSWORD" ]] || \
+ fail "Compose Specification compose.yaml secret files were not discovered"
+
jq -n \
--arg project_dir "$CLOUD_COMPOSE_DATA_ROOT/project" \
--arg trailing_command $'printf "trailing newline preserved"\n' '{
diff --git a/ci/config-management-cloud-smoke-inner.sh b/ci/config-management-cloud-smoke-inner.sh
index f53111d..4a70bf9 100755
--- a/ci/config-management-cloud-smoke-inner.sh
+++ b/ci/config-management-cloud-smoke-inner.sh
@@ -197,6 +197,12 @@ test -x /home/cloud-compose/down
test -x /home/cloud-compose/rollout
test -x /home/cloud-compose/run.sh
test -x /home/cloud-compose/start-cloud-compose-bootstrap.sh
+test -x /etc/cloud-compose/libexec/bootstrap-required.sh
+test -x /etc/cloud-compose/libexec/bootstrap-security.sh
+test -x /etc/cloud-compose/libexec/run-bootstrap.sh
+test -x /etc/cloud-compose/libexec/require-bootstrap-ready.sh
+test -x /etc/cloud-compose/libexec/start-cloud-compose-bootstrap.sh
+test -x /etc/cloud-compose/libexec/run-root-program.sh
python3 -m json.tool /home/cloud-compose/compose-projects.json >/dev/null
python3 -m json.tool /home/cloud-compose/application-env.json >/dev/null
diff --git a/ci/config-management-input-contract.sh b/ci/config-management-input-contract.sh
index 8b3283b..f1fcae4 100755
--- a/ci/config-management-input-contract.sh
+++ b/ci/config-management-input-contract.sh
@@ -11,6 +11,52 @@ cmp -s "$validator" "$salt_validator" || {
exit 1
}
+ansible_tasks="$repo_root/ansible/roles/cloud_compose/tasks/main.yml"
+salt_state="$repo_root/salt/cloud-compose/init.sls"
+root_program_runner="$repo_root/rootfs/etc/cloud-compose/libexec/run-root-program.sh"
+rollout_runner="bash /etc/cloud-compose/libexec/run-root-program.sh deploy-rollout.sh"
+
+contract_fail() {
+ echo "config-management input contract: $1" >&2
+ exit 1
+}
+
+grep -Fq "cmd: $rollout_runner" "$ansible_tasks" || \
+ contract_fail "Ansible rollout does not use the trusted root-program runner"
+if grep -Fq 'cmd: bash "{{ cloud_compose_home }}/' "$ansible_tasks" || \
+ grep -Fq 'cmd: bash /home/cloud-compose/' "$ansible_tasks"; then
+ contract_fail "Ansible rollout executes a root program directly from the runtime home"
+fi
+grep -Fq 'cmd: /etc/cloud-compose/libexec/harden-bootstrap-paths.sh' "$ansible_tasks" || \
+ contract_fail "Ansible does not apply the shared bootstrap path hardener"
+grep -Fq -- '- name: /etc/cloud-compose/libexec/harden-bootstrap-paths.sh' "$salt_state" || \
+ contract_fail "Salt does not apply the shared bootstrap path hardener"
+
+grep -Fq 'configure-metadata-firewall.sh | deploy-rollout.sh | docker-prune.sh' "$root_program_runner" || \
+ contract_fail "the trusted root-program runner does not allow deploy-rollout.sh"
+
+salt_rollout_block="$(sed -n '/^cloud-compose-rollout-service:/,/^{% endif %}$/p' "$salt_state")"
+[[ -n "$salt_rollout_block" ]] || contract_fail "Salt rollout state is missing"
+if grep -Fq -- '- name: bash /home/cloud-compose/' "$salt_state"; then
+ contract_fail "Salt rollout executes a root program directly from the runtime home"
+fi
+for marker in \
+ "- name: $rollout_runner" \
+ '- file: cloud-compose-env' \
+ '- file: cloud-compose-application-env' \
+ '- file: cloud-compose-project-manifest' \
+ '- file: cloud-compose-managed-runtime-artifacts' \
+ '- file: cloud-compose-rootfs' \
+ '- cmd: cloud-compose-lifecycle-lock' \
+ '- cmd: cloud-compose-rootfs-script-modes' \
+ '- file: cloud-compose-lifecycle-init' \
+ '- file: cloud-compose-lifecycle-up' \
+ '- file: cloud-compose-lifecycle-down' \
+ '- file: cloud-compose-lifecycle-rollout'; do
+ grep -Fq -- "$marker" <<<"$salt_rollout_block" || \
+ contract_fail "Salt rollout state is missing $marker"
+done
+
python3 - "$repo_root" "$validator" <<'PY'
import base64
import copy
@@ -175,8 +221,10 @@ if "files/validate-runtime-inputs.py" not in ansible_tasks:
fail("Ansible does not execute the shared host-input validator")
if "--data-root" in ansible_tasks:
fail("Ansible makes the production project ownership boundary configurable")
-if 'cmd: bash "{{ cloud_compose_home }}/start-cloud-compose-bootstrap.sh"' not in ansible_tasks:
+if "cmd: bash /etc/cloud-compose/libexec/start-cloud-compose-bootstrap.sh" not in ansible_tasks:
fail("Ansible bypasses the retryable bootstrap service")
+if "cmd: bash /etc/cloud-compose/libexec/require-bootstrap-ready.sh" not in ansible_tasks:
+ fail("Ansible does not validate bootstrap readiness evidence")
if 'cmd: bash "{{ cloud_compose_home }}/run.sh"' in ansible_tasks:
fail("Ansible still invokes the one-shot bootstrap script directly")
if "cloud_compose_bootstrap_timeout: 14400" not in ansible_defaults:
@@ -189,8 +237,10 @@ salt_gate = salt_state.find("cloud-compose-host-inputs-valid:")
salt_first_mutation = salt_state.find("cloud-compose-packages:")
if salt_gate < 0 or salt_first_mutation < 0 or salt_gate > salt_first_mutation:
fail("Salt host-input validation does not precede its first host mutation")
-if "home ~ '/start-cloud-compose-bootstrap.sh'" not in salt_state:
+if "bash /etc/cloud-compose/libexec/start-cloud-compose-bootstrap.sh" not in salt_state:
fail("Salt bypasses the retryable bootstrap service")
+if "bash /etc/cloud-compose/libexec/require-bootstrap-ready.sh" not in salt_state:
+ fail("Salt does not validate bootstrap readiness evidence")
if "home ~ '/run.sh'" in salt_state:
fail("Salt still invokes the one-shot bootstrap script directly")
gate_block = salt_state[salt_gate:salt_first_mutation]
@@ -226,6 +276,9 @@ for marker in (
"item.value.sitectl_packages | default(_cc_sitectl_packages)",
"(_cc_managed.enabled | default(cloud_compose_managed_runtime_enabled)) is boolean",
"(_cc_vault.agent_enabled | default(false)) is boolean",
+ "(_cc_disaster_recovery.required | default(false)) is boolean",
+ "CLOUD_COMPOSE_OFFHOST_BACKUP_REQUIRED",
+ "CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER",
):
if marker not in ansible_tasks:
fail(f"Ansible template package-version parity marker is missing: {marker!r}")
@@ -247,6 +300,9 @@ for marker in (
"all_packages = sitectl_packages | list",
"'managed_runtime.enabled': managed_runtime_enabled",
"'vault.agent_enabled': vault.get('agent_enabled', False)",
+ "runtime_sections.disaster_recovery",
+ "CLOUD_COMPOSE_OFFHOST_BACKUP_REQUIRED",
+ "CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER",
"run_bootstrap is sameas true",
):
if marker not in salt_state:
@@ -359,6 +415,7 @@ adapter_fixture_markers = {
"invalid-lifecycle": "- 42",
"invalid-project-lifecycle": "docker_compose_up: not-a-list",
"invalid-internal-services": "internal_services_enabled: true",
+ "invalid-disaster-recovery": "driver_path: /etc/cloud-compose/libexec/../untrusted",
}
for fixture_name, marker in adapter_fixture_markers.items():
for relative_path in (
diff --git a/ci/config-management-smoke-assert.py b/ci/config-management-smoke-assert.py
new file mode 100644
index 0000000..fc777ab
--- /dev/null
+++ b/ci/config-management-smoke-assert.py
@@ -0,0 +1,338 @@
+#!/usr/bin/env python3
+
+"""Assertions for the containerized Ansible and Salt smoke tests."""
+
+import argparse
+import grp
+import json
+import os
+import stat
+import subprocess
+from pathlib import Path
+
+
+RUNTIME_HOME = Path("/home/cloud-compose")
+PRIVILEGED_PROGRAM_ROOT = Path("/etc/cloud-compose")
+DIAGNOSTICS_PROGRAM = PRIVILEGED_PROGRAM_ROOT / "bin/cloud-compose-diagnostics.sh"
+BOOTSTRAP_LIBEXEC = Path("/etc/cloud-compose/libexec")
+JQ_PROGRAM_DIR = PRIVILEGED_PROGRAM_ROOT / "jq"
+
+
+def load_runtime_env(path: Path) -> dict[str, str]:
+ result = subprocess.run(
+ [
+ "env",
+ "-i",
+ "PATH=/usr/bin:/bin",
+ f"CLOUD_COMPOSE_ENV_FILE={path}",
+ "bash",
+ "--noprofile",
+ "--norc",
+ "-c",
+ "source /home/cloud-compose/profile.sh; env -0",
+ ],
+ check=True,
+ stdout=subprocess.PIPE,
+ )
+ return {
+ entry.split(b"=", 1)[0].decode(): entry.split(b"=", 1)[1].decode()
+ for entry in result.stdout.split(b"\0")
+ if b"=" in entry
+ }
+
+
+def assert_runtime_files() -> None:
+ for path in [
+ RUNTIME_HOME / "init",
+ RUNTIME_HOME / "up",
+ RUNTIME_HOME / "down",
+ RUNTIME_HOME / "rollout",
+ RUNTIME_HOME / "run.sh",
+ RUNTIME_HOME / "start-cloud-compose-bootstrap.sh",
+ BOOTSTRAP_LIBEXEC / "bootstrap-required.sh",
+ BOOTSTRAP_LIBEXEC / "bootstrap-security.sh",
+ BOOTSTRAP_LIBEXEC / "run-bootstrap.sh",
+ BOOTSTRAP_LIBEXEC / "require-bootstrap-ready.sh",
+ BOOTSTRAP_LIBEXEC / "run-root-program.sh",
+ BOOTSTRAP_LIBEXEC / "start-cloud-compose-bootstrap.sh",
+ DIAGNOSTICS_PROGRAM,
+ ]:
+ assert path.exists(), path
+ assert os.access(path, os.X_OK), path
+
+ for path in [
+ JQ_PROGRAM_DIR / "diagnostics-validate-compose-projects.jq",
+ JQ_PROGRAM_DIR / "offhost-validate-manifest.jq",
+ ]:
+ assert path.exists(), path
+
+ cloud_compose_gid = grp.getgrnam("cloud-compose").gr_gid
+ for path, expected_mode in {
+ RUNTIME_HOME / "init": 0o750,
+ RUNTIME_HOME / "up": 0o750,
+ RUNTIME_HOME / "down": 0o750,
+ RUNTIME_HOME / "rollout": 0o750,
+ RUNTIME_HOME / ".env": 0o640,
+ RUNTIME_HOME / "application-env.json": 0o640,
+ RUNTIME_HOME / "compose-projects.json": 0o640,
+ RUNTIME_HOME / "managed-runtime-artifacts.tsv": 0o640,
+ }.items():
+ metadata = path.stat()
+ assert metadata.st_uid == 0, (path, metadata.st_uid)
+ assert metadata.st_gid == cloud_compose_gid, (path, metadata.st_gid)
+ assert stat.S_IMODE(metadata.st_mode) == expected_mode, (
+ path,
+ oct(stat.S_IMODE(metadata.st_mode)),
+ )
+
+ for path in BOOTSTRAP_LIBEXEC.glob("*.sh"):
+ metadata = path.stat()
+ assert metadata.st_uid == 0, (path, metadata.st_uid)
+ assert metadata.st_gid == 0, (path, metadata.st_gid)
+ assert stat.S_IMODE(metadata.st_mode) == 0o755, (
+ path,
+ oct(stat.S_IMODE(metadata.st_mode)),
+ )
+
+ for path in [
+ PRIVILEGED_PROGRAM_ROOT,
+ DIAGNOSTICS_PROGRAM.parent,
+ BOOTSTRAP_LIBEXEC,
+ JQ_PROGRAM_DIR,
+ ]:
+ metadata = path.stat()
+ assert metadata.st_uid == 0, (path, metadata.st_uid)
+ assert metadata.st_gid == 0, (path, metadata.st_gid)
+ assert stat.S_IMODE(metadata.st_mode) == 0o755, (
+ path,
+ oct(stat.S_IMODE(metadata.st_mode)),
+ )
+
+ diagnostics_metadata = DIAGNOSTICS_PROGRAM.stat()
+ assert diagnostics_metadata.st_uid == 0
+ assert diagnostics_metadata.st_gid == 0
+ assert stat.S_IMODE(diagnostics_metadata.st_mode) == 0o755
+
+ jq_programs = list(JQ_PROGRAM_DIR.glob("*.jq"))
+ assert jq_programs
+ for path in jq_programs:
+ metadata = path.stat()
+ assert metadata.st_uid == 0, (path, metadata.st_uid)
+ assert metadata.st_gid == 0, (path, metadata.st_gid)
+ assert stat.S_IMODE(metadata.st_mode) == 0o644, (
+ path,
+ oct(stat.S_IMODE(metadata.st_mode)),
+ )
+
+
+def assert_ansible_runtime() -> None:
+ env = load_runtime_env(RUNTIME_HOME / ".env")
+ application_env = json.loads(
+ (RUNTIME_HOME / "application-env.json").read_text()
+ )
+ projects = json.loads((RUNTIME_HOME / "compose-projects.json").read_text())
+ project = projects["isle-prod"]
+
+ assert env["CLOUD_COMPOSE_PROVIDER"] == "onprem"
+ assert env["CLOUD_COMPOSE_OFFHOST_BACKUP_REQUIRED"] == "false"
+ assert (
+ env["CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER"]
+ == "/etc/cloud-compose/libexec/ansible-offhost"
+ )
+ assert env["DOCKER_COMPOSE_DIR"] == "/mnt/disks/data/libops/isle/isle-prod"
+ assert env["DOCKER_COMPOSE_REPO"] == "https://github.com/libops/isle"
+ assert env["SITECTL_PLUGIN"] == "isle"
+ assert "sitectl-isle" in env["SITECTL_PACKAGES"].split()
+ assert json.loads(env["SITECTL_PACKAGE_VERSIONS"]) == {
+ "sitectl": "v1.8.2",
+ "sitectl-drupal": "v1.3.0",
+ "sitectl-isle": "v1.5.0",
+ }
+ assert project["docker_compose_repo"] == "https://github.com/libops/isle"
+ assert project["project_dir"] == "/mnt/disks/data/libops/isle/isle-prod"
+ assert project["compose_project_name"] == "libops-isle-v1-3-0"
+ assert project["ingress"]["domain"] == "isle.example.edu"
+ assert project["sitectl_plugin"] == "isle"
+ assert project["init_commands"] == []
+ assert project["up_commands"] == []
+ assert project["down_commands"] == ["app-down-command"]
+ assert project["rollout_commands"] == ["global-rollout-command"]
+ assert Path("/mnt/disks/data/libops/isle/isle-prod").is_dir()
+ assert "BASH_ENV" not in env
+ assert "LD_PRELOAD" not in env
+ assert "PORT" not in env
+ assert application_env["BASH_ENV"] == (
+ "/tmp/cloud-compose-ansible-untrusted-bash-env"
+ )
+ assert application_env["LD_PRELOAD"] == (
+ "/tmp/cloud-compose-ansible-untrusted-preload.so"
+ )
+ assert application_env["PORT"] == "9999"
+ assert application_env["CONTRACT_BACKTICKS"] == (
+ "`touch /tmp/cloud-compose-ansible-backtick-injection`"
+ )
+ assert application_env["CONTRACT_BACKSLASH"] == "a\\path\\ends\\"
+ assert application_env["CONTRACT_COMMAND_SUB"] == (
+ "$(touch /tmp/cloud-compose-ansible-command-injection)"
+ )
+ assert application_env["CONTRACT_DOLLARS"] == "$HOME ${HOME}"
+ assert application_env["CONTRACT_QUOTES"] == (
+ 'a "double" and a single quote: O\'Reilly'
+ )
+ assert application_env["CONTRACT_WHITESPACE"] == " leading and trailing "
+ assert application_env["CONTRACT_MULTILINE"] == "line one\nline two"
+ assert not Path("/tmp/cloud-compose-ansible-backtick-injection").exists()
+ assert not Path("/tmp/cloud-compose-ansible-command-injection").exists()
+
+ assert_runtime_files()
+
+
+def assert_salt_noop() -> None:
+ states = json.loads(Path("/tmp/cloud-compose-salt-noop.json").read_text())[
+ "local"
+ ]
+ lock_states = [
+ state
+ for state_id, state in states.items()
+ if "|-cloud-compose-lifecycle-lock_|-" in state_id
+ ]
+ assert len(lock_states) == 1, lock_states
+ assert lock_states[0]["result"] is True, lock_states[0]
+ assert lock_states[0]["changes"] == {}, lock_states[0]
+
+
+def assert_salt_runtime(
+ expected_name: str,
+ expected_repo: str,
+ expected_plugin: str,
+ expected_package: str,
+ expected_domain: str,
+ expected_project_dir: str,
+ expected_compose_project_name: str,
+) -> None:
+ env = load_runtime_env(RUNTIME_HOME / ".env")
+ application_env = json.loads(
+ (RUNTIME_HOME / "application-env.json").read_text()
+ )
+ projects = json.loads((RUNTIME_HOME / "compose-projects.json").read_text())
+ project = projects[expected_name]
+ artifact_manifest = (RUNTIME_HOME / "managed-runtime-artifacts.tsv").read_text()
+
+ assert env["CLOUD_COMPOSE_PROVIDER"] == "onprem"
+ assert env["CLOUD_COMPOSE_APPS"] == expected_name
+ assert env["CLOUD_COMPOSE_PRIMARY_APP"] == expected_name
+ assert env["DOCKER_COMPOSE_DIR"] == expected_project_dir
+ assert env["DOCKER_COMPOSE_REPO"] == expected_repo
+ assert env["COMPOSE_PROJECT_NAME"] == expected_compose_project_name, (
+ env["COMPOSE_PROJECT_NAME"],
+ expected_compose_project_name,
+ )
+ assert env["SITECTL_PLUGIN"] == expected_plugin
+ assert expected_package in env["SITECTL_PACKAGES"].split()
+ assert project["docker_compose_repo"] == expected_repo
+ assert project["project_dir"] == expected_project_dir
+ assert project["compose_project_name"] == expected_compose_project_name, (
+ project["compose_project_name"],
+ expected_compose_project_name,
+ )
+ assert project["sitectl_plugin"] == expected_plugin
+ assert project["ingress"]["domain"] == expected_domain
+ assert Path(expected_project_dir).is_dir()
+
+ if expected_name == "wp-prod":
+ assert env["CLOUD_COMPOSE_OFFHOST_BACKUP_REQUIRED"] == "false"
+ assert (
+ env["CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER"]
+ == "/etc/cloud-compose/libexec/salt-offhost"
+ )
+ assert json.loads(env["SITECTL_PACKAGE_VERSIONS"]) == {
+ "sitectl": "v1.8.2",
+ "sitectl-wp": "v2.0.0",
+ }
+ assert "BASH_ENV" not in env
+ assert "LD_PRELOAD" not in env
+ assert "PORT" not in env
+ assert application_env["BASH_ENV"] == (
+ "/tmp/cloud-compose-salt-untrusted-bash-env"
+ )
+ assert application_env["LD_PRELOAD"] == (
+ "/tmp/cloud-compose-salt-untrusted-preload.so"
+ )
+ assert application_env["PORT"] == "9999"
+ assert application_env["CONTRACT_BACKTICKS"] == (
+ "`touch /tmp/cloud-compose-salt-backtick-injection`"
+ )
+ assert application_env["CONTRACT_BACKSLASH"] == "a\\path\\ends\\"
+ assert application_env["CONTRACT_COMMAND_SUB"] == (
+ "$(touch /tmp/cloud-compose-salt-command-injection)"
+ )
+ assert application_env["CONTRACT_DOLLARS"] == "$HOME ${HOME}"
+ assert application_env["CONTRACT_QUOTES"] == (
+ 'a "double" and a single quote: O\'Reilly'
+ )
+ assert application_env["CONTRACT_WHITESPACE"] == " leading and trailing "
+ assert application_env["CONTRACT_MULTILINE"] == "line one\nline two"
+ assert env["LIBOPS_MANAGED_RUNTIME_ENABLED"] == "false"
+ assert env["LIBOPS_INTERNAL_SERVICES_ENABLED"] == "false"
+ assert env["LIBOPS_INTERNAL_SERVICES_AUTO_UPDATE"] == "false"
+ assert project["init_commands"] == []
+ assert project["up_commands"] == []
+ assert project["down_commands"] == ["app-down-command"]
+ assert project["rollout_commands"] == ["global-rollout-command"]
+ assert (
+ "contract-agent\thttps://example.invalid/contract-agent\t"
+ in artifact_manifest
+ )
+ assert (
+ "\t/usr/local/bin/contract-agent\t0750\troot\troot\t"
+ "cloud-compose.service"
+ in artifact_manifest
+ )
+ assert not Path("/tmp/cloud-compose-salt-backtick-injection").exists()
+ assert not Path("/tmp/cloud-compose-salt-command-injection").exists()
+ elif expected_name == "drupal-prod":
+ assert json.loads(env["SITECTL_PACKAGE_VERSIONS"]) == {
+ "sitectl": "v1.8.2",
+ "sitectl-drupal": "v1.3.0",
+ }
+
+ assert_runtime_files()
+
+
+def parse_args() -> argparse.Namespace:
+ parser = argparse.ArgumentParser()
+ subparsers = parser.add_subparsers(dest="command", required=True)
+ subparsers.add_parser("ansible-runtime")
+ subparsers.add_parser("salt-noop")
+
+ salt_runtime = subparsers.add_parser("salt-runtime")
+ salt_runtime.add_argument("expected_name")
+ salt_runtime.add_argument("expected_repo")
+ salt_runtime.add_argument("expected_plugin")
+ salt_runtime.add_argument("expected_package")
+ salt_runtime.add_argument("expected_domain")
+ salt_runtime.add_argument("expected_project_dir")
+ salt_runtime.add_argument("expected_compose_project_name")
+ return parser.parse_args()
+
+
+def main() -> None:
+ args = parse_args()
+ if args.command == "ansible-runtime":
+ assert_ansible_runtime()
+ elif args.command == "salt-noop":
+ assert_salt_noop()
+ else:
+ assert_salt_runtime(
+ args.expected_name,
+ args.expected_repo,
+ args.expected_plugin,
+ args.expected_package,
+ args.expected_domain,
+ args.expected_project_dir,
+ args.expected_compose_project_name,
+ )
+
+
+if __name__ == "__main__":
+ main()
diff --git a/ci/config-management-smoke-inner.sh b/ci/config-management-smoke-inner.sh
index 611e5db..fbd89e0 100755
--- a/ci/config-management-smoke-inner.sh
+++ b/ci/config-management-smoke-inner.sh
@@ -141,6 +141,7 @@ ansible_invalid_cases=(
invalid-primary
invalid-template
invalid-vault
+ invalid-disaster-recovery
invalid-package
invalid-host-ack
invalid-project-dir-root
@@ -206,6 +207,7 @@ run_invalid_ansible_case invalid-internal-services 'internal-services stack is G
run_invalid_ansible_case invalid-primary 'compose.primary must match'
run_invalid_ansible_case invalid-template 'template must name a supported app'
run_invalid_ansible_case invalid-vault 'Vault Agent is currently supported only by Terraform'
+run_invalid_ansible_case invalid-disaster-recovery 'driver_path must be a safe absolute path'
run_invalid_ansible_case invalid-package 'sitectl packages must use valid release package names'
run_invalid_ansible_case invalid-host-ack 'dedicated_host_acknowledged=true'
run_invalid_ansible_case invalid-project-dir-root 'project_dir must be a normalized non-root absolute path'
@@ -227,97 +229,7 @@ ansible-playbook \
verify_lifecycle_lock_contract
-python - <<'PY'
-import json
-import grp
-import os
-import stat
-import subprocess
-from pathlib import Path
-
-def load_runtime_env(path):
- result = subprocess.run(
- [
- "env", "-i", "PATH=/usr/bin:/bin", f"CLOUD_COMPOSE_ENV_FILE={path}",
- "bash", "--noprofile", "--norc", "-c",
- "source /home/cloud-compose/profile.sh; env -0",
- ],
- check=True,
- stdout=subprocess.PIPE,
- )
- return {
- entry.split(b"=", 1)[0].decode(): entry.split(b"=", 1)[1].decode()
- for entry in result.stdout.split(b"\0") if b"=" in entry
- }
-
-env = load_runtime_env(Path("/home/cloud-compose/.env"))
-application_env = json.loads(Path("/home/cloud-compose/application-env.json").read_text())
-
-projects = json.loads(Path("/home/cloud-compose/compose-projects.json").read_text())
-project = projects["isle-prod"]
-
-assert env["CLOUD_COMPOSE_PROVIDER"] == "onprem"
-assert env["DOCKER_COMPOSE_DIR"] == "/mnt/disks/data/libops/isle/isle-prod"
-assert env["DOCKER_COMPOSE_REPO"] == "https://github.com/libops/isle"
-assert env["SITECTL_PLUGIN"] == "isle"
-assert "sitectl-isle" in env["SITECTL_PACKAGES"].split()
-assert json.loads(env["SITECTL_PACKAGE_VERSIONS"]) == {
- "sitectl": "v1.0.0",
- "sitectl-drupal": "v1.0.0",
- "sitectl-isle": "v1.0.0",
-}
-assert project["docker_compose_repo"] == "https://github.com/libops/isle"
-assert project["project_dir"] == "/mnt/disks/data/libops/isle/isle-prod"
-assert project["ingress"]["domain"] == "isle.example.edu"
-assert project["sitectl_plugin"] == "isle"
-assert project["init_commands"] == []
-assert project["up_commands"] == []
-assert project["down_commands"] == ["app-down-command"]
-assert project["rollout_commands"] == ["global-rollout-command"]
-assert Path("/mnt/disks/data/libops/isle/isle-prod").is_dir()
-assert "BASH_ENV" not in env
-assert "LD_PRELOAD" not in env
-assert "PORT" not in env
-assert application_env["BASH_ENV"] == "/tmp/cloud-compose-ansible-untrusted-bash-env"
-assert application_env["LD_PRELOAD"] == "/tmp/cloud-compose-ansible-untrusted-preload.so"
-assert application_env["PORT"] == "9999"
-assert application_env["CONTRACT_BACKTICKS"] == "`touch /tmp/cloud-compose-ansible-backtick-injection`"
-assert application_env["CONTRACT_BACKSLASH"] == "a\\path\\ends\\"
-assert application_env["CONTRACT_COMMAND_SUB"] == "$(touch /tmp/cloud-compose-ansible-command-injection)"
-assert application_env["CONTRACT_DOLLARS"] == "$HOME ${HOME}"
-assert application_env["CONTRACT_QUOTES"] == 'a "double" and a single quote: O\'Reilly'
-assert application_env["CONTRACT_WHITESPACE"] == " leading and trailing "
-assert application_env["CONTRACT_MULTILINE"] == "line one\nline two"
-assert not Path("/tmp/cloud-compose-ansible-backtick-injection").exists()
-assert not Path("/tmp/cloud-compose-ansible-command-injection").exists()
-
-for path in [
- "/home/cloud-compose/init",
- "/home/cloud-compose/up",
- "/home/cloud-compose/down",
- "/home/cloud-compose/rollout",
- "/home/cloud-compose/run.sh",
- "/home/cloud-compose/start-cloud-compose-bootstrap.sh",
-]:
- assert Path(path).exists(), path
- assert os.access(path, os.X_OK), path
-
-cloud_compose_gid = grp.getgrnam("cloud-compose").gr_gid
-for path, expected_mode in {
- "/home/cloud-compose/init": 0o750,
- "/home/cloud-compose/up": 0o750,
- "/home/cloud-compose/down": 0o750,
- "/home/cloud-compose/rollout": 0o750,
- "/home/cloud-compose/.env": 0o640,
- "/home/cloud-compose/application-env.json": 0o640,
- "/home/cloud-compose/compose-projects.json": 0o640,
- "/home/cloud-compose/managed-runtime-artifacts.tsv": 0o640,
-}.items():
- metadata = Path(path).stat()
- assert metadata.st_uid == 0, (path, metadata.st_uid)
- assert metadata.st_gid == cloud_compose_gid, (path, metadata.st_gid)
- assert stat.S_IMODE(metadata.st_mode) == expected_mode, (path, oct(stat.S_IMODE(metadata.st_mode)))
-PY
+python /work/ci/config-management-smoke-assert.py ansible-runtime
rm -rf /home/cloud-compose /mnt/disks
@@ -379,135 +291,17 @@ run_salt_case() {
--config-dir=/tmp/salt/etc \
--out=json \
state.apply cloud-compose >/tmp/cloud-compose-salt-noop.json
- python - <<'PY'
-import json
-from pathlib import Path
-
-states = json.loads(Path("/tmp/cloud-compose-salt-noop.json").read_text())["local"]
-lock_states = [
- state for state_id, state in states.items()
- if "|-cloud-compose-lifecycle-lock_|-" in state_id
-]
-assert len(lock_states) == 1, lock_states
-assert lock_states[0]["result"] is True, lock_states[0]
-assert lock_states[0]["changes"] == {}, lock_states[0]
-PY
+ python /work/ci/config-management-smoke-assert.py salt-noop
python -m json.tool /home/cloud-compose/compose-projects.json >/dev/null
- python - "$expected_name" "$expected_repo" "$expected_plugin" "$expected_package" "$expected_domain" "$expected_project_dir" "$expected_compose_project_name" <<'PY'
-import json
-import grp
-import os
-import stat
-import subprocess
-import sys
-from pathlib import Path
-
-expected_name, expected_repo, expected_plugin, expected_package, expected_domain, expected_project_dir, expected_compose_project_name = sys.argv[1:]
-
-def load_runtime_env(path):
- result = subprocess.run(
- [
- "env", "-i", "PATH=/usr/bin:/bin", f"CLOUD_COMPOSE_ENV_FILE={path}",
- "bash", "--noprofile", "--norc", "-c",
- "source /home/cloud-compose/profile.sh; env -0",
- ],
- check=True,
- stdout=subprocess.PIPE,
- )
- return {
- entry.split(b"=", 1)[0].decode(): entry.split(b"=", 1)[1].decode()
- for entry in result.stdout.split(b"\0") if b"=" in entry
- }
-
-env = load_runtime_env(Path("/home/cloud-compose/.env"))
-application_env = json.loads(Path("/home/cloud-compose/application-env.json").read_text())
-
-projects = json.loads(Path("/home/cloud-compose/compose-projects.json").read_text())
-project = projects[expected_name]
-artifact_manifest = Path("/home/cloud-compose/managed-runtime-artifacts.tsv").read_text()
-
-assert env["CLOUD_COMPOSE_PROVIDER"] == "onprem"
-assert env["CLOUD_COMPOSE_APPS"] == expected_name
-assert env["CLOUD_COMPOSE_PRIMARY_APP"] == expected_name
-assert env["DOCKER_COMPOSE_DIR"] == expected_project_dir
-assert env["DOCKER_COMPOSE_REPO"] == expected_repo
-assert env["COMPOSE_PROJECT_NAME"] == expected_compose_project_name, (
- env["COMPOSE_PROJECT_NAME"], expected_compose_project_name
-)
-assert env["SITECTL_PLUGIN"] == expected_plugin
-assert expected_package in env["SITECTL_PACKAGES"].split()
-assert project["docker_compose_repo"] == expected_repo
-assert project["project_dir"] == expected_project_dir
-assert project["compose_project_name"] == expected_compose_project_name, (
- project["compose_project_name"], expected_compose_project_name
-)
-assert project["sitectl_plugin"] == expected_plugin
-assert project["ingress"]["domain"] == expected_domain
-assert Path(expected_project_dir).is_dir()
-
-if expected_name == "wp-prod":
- assert json.loads(env["SITECTL_PACKAGE_VERSIONS"]) == {
- "sitectl": "v1.0.0",
- "sitectl-wp": "v1.0.0",
- }
- assert "BASH_ENV" not in env
- assert "LD_PRELOAD" not in env
- assert "PORT" not in env
- assert application_env["BASH_ENV"] == "/tmp/cloud-compose-salt-untrusted-bash-env"
- assert application_env["LD_PRELOAD"] == "/tmp/cloud-compose-salt-untrusted-preload.so"
- assert application_env["PORT"] == "9999"
- assert application_env["CONTRACT_BACKTICKS"] == "`touch /tmp/cloud-compose-salt-backtick-injection`"
- assert application_env["CONTRACT_BACKSLASH"] == "a\\path\\ends\\"
- assert application_env["CONTRACT_COMMAND_SUB"] == "$(touch /tmp/cloud-compose-salt-command-injection)"
- assert application_env["CONTRACT_DOLLARS"] == "$HOME ${HOME}"
- assert application_env["CONTRACT_QUOTES"] == 'a "double" and a single quote: O\'Reilly'
- assert application_env["CONTRACT_WHITESPACE"] == " leading and trailing "
- assert application_env["CONTRACT_MULTILINE"] == "line one\nline two"
- assert env["LIBOPS_MANAGED_RUNTIME_ENABLED"] == "false"
- assert env["LIBOPS_INTERNAL_SERVICES_ENABLED"] == "false"
- assert env["LIBOPS_INTERNAL_SERVICES_AUTO_UPDATE"] == "false"
- assert project["init_commands"] == []
- assert project["up_commands"] == []
- assert project["down_commands"] == ["app-down-command"]
- assert project["rollout_commands"] == ["global-rollout-command"]
- assert "contract-agent\thttps://example.invalid/contract-agent\t" in artifact_manifest
- assert "\t/usr/local/bin/contract-agent\t0750\troot\troot\tcloud-compose.service" in artifact_manifest
- assert not Path("/tmp/cloud-compose-salt-backtick-injection").exists()
- assert not Path("/tmp/cloud-compose-salt-command-injection").exists()
-elif expected_name == "drupal-prod":
- assert json.loads(env["SITECTL_PACKAGE_VERSIONS"]) == {
- "sitectl": "v1.0.0",
- "sitectl-drupal": "v1.0.0",
- }
-
-for path in [
- "/home/cloud-compose/init",
- "/home/cloud-compose/up",
- "/home/cloud-compose/down",
- "/home/cloud-compose/rollout",
- "/home/cloud-compose/run.sh",
- "/home/cloud-compose/start-cloud-compose-bootstrap.sh",
-]:
- assert Path(path).exists(), path
- assert os.access(path, os.X_OK), path
-
-cloud_compose_gid = grp.getgrnam("cloud-compose").gr_gid
-for path, expected_mode in {
- "/home/cloud-compose/init": 0o750,
- "/home/cloud-compose/up": 0o750,
- "/home/cloud-compose/down": 0o750,
- "/home/cloud-compose/rollout": 0o750,
- "/home/cloud-compose/.env": 0o640,
- "/home/cloud-compose/application-env.json": 0o640,
- "/home/cloud-compose/compose-projects.json": 0o640,
- "/home/cloud-compose/managed-runtime-artifacts.tsv": 0o640,
-}.items():
- metadata = Path(path).stat()
- assert metadata.st_uid == 0, (path, metadata.st_uid)
- assert metadata.st_gid == cloud_compose_gid, (path, metadata.st_gid)
- assert stat.S_IMODE(metadata.st_mode) == expected_mode, (path, oct(stat.S_IMODE(metadata.st_mode)))
-PY
+ python /work/ci/config-management-smoke-assert.py salt-runtime \
+ "$expected_name" \
+ "$expected_repo" \
+ "$expected_plugin" \
+ "$expected_package" \
+ "$expected_domain" \
+ "$expected_project_dir" \
+ "$expected_compose_project_name"
}
run_salt_case \
@@ -518,7 +312,7 @@ run_salt_case \
sitectl-wp \
wp.example.edu \
/mnt/disks/data/libops/wp.git/wp-prod \
- libops-wp-v1-0-0
+ libops-wp-v1-1-0
run_salt_case \
drupal-prod \
@@ -528,7 +322,7 @@ run_salt_case \
sitectl-drupal \
drupal.example.edu \
/mnt/disks/data/libops/drupal.git/drupal-prod \
- libops-drupal-v1-0-0
+ libops-drupal-v1-1-0
run_invalid_salt_case() {
local invalid_case="$1" expected="$2"
@@ -581,6 +375,7 @@ run_invalid_salt_case invalid-internal-services 'internal-services stack is GCP-
run_invalid_salt_case invalid-primary 'compose.primary must match'
run_invalid_salt_case invalid-template 'template must name a supported cloud-compose app'
run_invalid_salt_case invalid-vault 'Vault Agent is currently supported only by Terraform'
+run_invalid_salt_case invalid-disaster-recovery 'driver_path must be a safe absolute path'
run_invalid_salt_case invalid-package 'invalid installed package'
run_invalid_salt_case invalid-host-ack 'dedicated_host_acknowledged=true'
run_invalid_salt_case invalid-project-dir-root 'project_dir must be a normalized non-root absolute path'
diff --git a/ci/cos-bootstrap-contract.sh b/ci/cos-bootstrap-contract.sh
index adf8219..43125b6 100755
--- a/ci/cos-bootstrap-contract.sh
+++ b/ci/cos-bootstrap-contract.sh
@@ -4,6 +4,7 @@ set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
INSTALLER="${ROOT_DIR}/rootfs/home/cloud-compose/install-dependencies-cos.sh"
+BUILD_PROGRAM="${ROOT_DIR}/rootfs/etc/cloud-compose/libexec/build-cos-make.sh"
TEST_ROOT="$(mktemp -d)"
trap 'rm -rf -- "$TEST_ROOT"' EXIT
@@ -21,22 +22,22 @@ if ! grep -Fqx 'ALPINE_BUILD_IMAGE="alpine:3.22@sha256:14358309a308569c32bdc37e2
echo "COS bootstrap Alpine image is not digest pinned" >&2
exit 1
fi
-grep -Fq '"${alpine_mirror}/v3.22/main"' "$INSTALLER" || {
+grep -Fq '"${alpine_mirror}/v3.22/main"' "$BUILD_PROGRAM" || {
echo "COS bootstrap Alpine main repository does not match the pinned image release" >&2
exit 1
}
-grep -Fq '"${alpine_mirror}/v3.22/community"' "$INSTALLER" || {
+grep -Fq '"${alpine_mirror}/v3.22/community"' "$BUILD_PROGRAM" || {
echo "COS bootstrap Alpine community repository does not match the pinned image release" >&2
exit 1
}
-# Assert the literal command in the bootstrap script.
+# Assert the literal command in the checked-in build program.
# shellcheck disable=SC2016
-if ! grep -Fq 'echo "${MAKE_SHA256} /tmp/make.tar.gz" | sha256sum -c -' "$INSTALLER"; then
+if ! grep -Fq 'echo "${MAKE_SHA256} /tmp/make.tar.gz" | sha256sum -c -' "$BUILD_PROGRAM"; then
echo "COS bootstrap does not verify the GNU Make source archive" >&2
exit 1
fi
-checksum_line="$(grep -nF 'sha256sum -c -' "$INSTALLER" | head -n1)"
-archive_line="$(grep -nF 'tar -xzf /tmp/make.tar.gz' "$INSTALLER" | head -n1)"
+checksum_line="$(grep -nF 'sha256sum -c -' "$BUILD_PROGRAM" | head -n1)"
+archive_line="$(grep -nF 'tar -xzf /tmp/make.tar.gz' "$BUILD_PROGRAM" | head -n1)"
if (( ${checksum_line%%:*} >= ${archive_line%%:*} )); then
echo "COS bootstrap extracts GNU Make before checksum verification" >&2
exit 1
@@ -45,11 +46,27 @@ if grep -Eq '^[[:space:]]*--network[[:space:]]+host([[:space:]\\]|$)' "$INSTALLE
echo "COS bootstrap exposes third-party build code to the host network" >&2
exit 1
fi
+[[ -x "$BUILD_PROGRAM" ]] || {
+ echo "COS checked-in Make build program is not executable" >&2
+ exit 1
+}
+grep -Fq -- '-v "${make_build_program}:/tmp/cloud-compose-build-cos-make.sh:ro"' "$INSTALLER" || {
+ echo "COS bootstrap does not mount its checked-in Make build program read-only" >&2
+ exit 1
+}
+grep -Fq '/bin/sh /tmp/cloud-compose-build-cos-make.sh; then' "$INSTALLER" || {
+ echo "COS bootstrap does not invoke its checked-in Make build program by path" >&2
+ exit 1
+}
+if grep -Eq '/bin/sh[[:space:]]+-[^[:space:]]*c[[:space:]]' "$INSTALLER"; then
+ echo "COS bootstrap still embeds a shell program in a command argument" >&2
+ exit 1
+fi
grep -Fq 'iptables -C DOCKER-USER -d 169.254.169.254/32 -j DROP' "$INSTALLER" || {
echo "COS bootstrap does not require the GCP metadata deny policy" >&2
exit 1
}
-grep -Fq 'COS_TOOL_STATE_DIR:-/mnt/disks/data/cloud-compose-tools' "$INSTALLER" || {
+grep -Fq 'COS_TOOL_STATE_DIR:-/mnt/disks/data/libops-managed/bin' "$INSTALLER" || {
echo "COS bootstrap stores an executable tool on a potentially noexec filesystem" >&2
exit 1
}
@@ -68,24 +85,24 @@ for mirror in \
https://dl-cdn.alpinelinux.org/alpine \
https://mirror.math.princeton.edu/pub/alpinelinux \
https://mirror.fel.cvut.cz/alpine; do
- grep -Fq "$mirror" "$INSTALLER" || {
+ grep -Fq "$mirror" "$BUILD_PROGRAM" || {
echo "COS bootstrap is missing Alpine package mirror: $mirror" >&2
exit 1
}
done
-grep -Fq 'for alpine_mirror in ${alpine_mirrors}; do' "$INSTALLER" || {
+grep -Fq 'for alpine_mirror in ${alpine_mirrors}; do' "$BUILD_PROGRAM" || {
echo "COS bootstrap does not fail over between Alpine package mirrors" >&2
exit 1
}
-grep -Fq 'if apk update && apk add build-base curl make tar; then' "$INSTALLER" || {
+grep -Fq 'if apk update && apk add build-base curl make tar; then' "$BUILD_PROGRAM" || {
echo "COS bootstrap does not validate an index before installing packages" >&2
exit 1
}
-grep -Fq 'All configured Alpine package mirrors failed' "$INSTALLER" || {
+grep -Fq 'All configured Alpine package mirrors failed' "$BUILD_PROGRAM" || {
echo "COS bootstrap accepts exhaustion of all Alpine package mirrors" >&2
exit 1
}
-if grep -Fq -- '--allow-untrusted' "$INSTALLER"; then
+if grep -Fq -- '--allow-untrusted' "$BUILD_PROGRAM"; then
echo "COS bootstrap disables Alpine package signature verification" >&2
exit 1
fi
@@ -97,7 +114,7 @@ retry_until_success() {
"$@"
}
export DOCKER_CONFIG="$MOCK_DOCKER_CONFIG"
-install_cos_dependencies "$TEST_CLOUD_HOME" "" "$TEST_DOCKER_BIN"
+install_cos_dependencies "$TEST_CLOUD_HOME" "" "$TEST_DOCKER_BIN" "$TEST_BUILD_PROGRAM"
EOF
cat >"${BIN_DIR}/bash" <<'EOF'
@@ -156,6 +173,7 @@ PATH="${BIN_DIR}:$PATH" \
TEST_CLOUD_HOME="$CLOUD_HOME" \
TEST_DATA_DIR="$DATA_DIR" \
TEST_DOCKER_BIN="${BIN_DIR}/docker" \
+ TEST_BUILD_PROGRAM="$BUILD_PROGRAM" \
MOCK_DOCKER_CONFIG="$DOCKER_CONFIG_DIR" \
MOCK_CALL_LOG="$CALL_LOG" \
/bin/bash "$HARNESS"
@@ -163,11 +181,16 @@ PATH="${BIN_DIR}:$PATH" \
grep -Fq "plugin:${DOCKER_CONFIG_DIR}/cli-plugins:${CLOUD_HOME}/install-docker-plugins.sh" "$CALL_LOG"
grep -Fq "plugin:${CLOUD_HOME}/.docker/cli-plugins:${CLOUD_HOME}/install-docker-plugins.sh" "$CALL_LOG"
grep -Fq 'alpine:3.22@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce' "$CALL_LOG"
+grep -Fq "${BUILD_PROGRAM}:/tmp/cloud-compose-build-cos-make.sh:ro" "$CALL_LOG"
[[ -x "${DATA_DIR}/make" ]]
"${DATA_DIR}/make" --version | grep -Fq 'GNU Make 4.4.1'
[[ -f "${DATA_DIR}/make.state" ]]
[[ -L "${CLOUD_HOME}/bin/make" ]]
[[ "$(readlink "${CLOUD_HOME}/bin/make")" == "${DATA_DIR}/make" ]]
+if grep -A3 -F 'chown -R cloud-compose:cloud-compose' "$INSTALLER" | grep -Fq '"${cloud_compose_home}/bin"'; then
+ echo "COS bootstrap gives the application account ownership of the privileged command directory" >&2
+ exit 1
+fi
# A failed replacement must not promote a partial binary or leave a pending
# artifact that a later boot could mistake for a completed build.
@@ -181,6 +204,7 @@ if PATH="${BIN_DIR}:$PATH" \
TEST_CLOUD_HOME="$CLOUD_HOME" \
TEST_DATA_DIR="$DATA_DIR" \
TEST_DOCKER_BIN="${BIN_DIR}/docker" \
+ TEST_BUILD_PROGRAM="$BUILD_PROGRAM" \
MOCK_DOCKER_CONFIG="$DOCKER_CONFIG_DIR" \
MOCK_CALL_LOG="$CALL_LOG" \
/bin/bash "$HARNESS" >/dev/null 2>&1; then
@@ -201,6 +225,7 @@ if PATH="${BIN_DIR}:$PATH" \
TEST_CLOUD_HOME="$CLOUD_HOME" \
TEST_DATA_DIR="$DATA_DIR" \
TEST_DOCKER_BIN="${BIN_DIR}/docker" \
+ TEST_BUILD_PROGRAM="$BUILD_PROGRAM" \
MOCK_DOCKER_CONFIG="$DOCKER_CONFIG_DIR" \
MOCK_CALL_LOG="$CALL_LOG" \
/bin/bash "$HARNESS" >/dev/null 2>&1; then
diff --git a/ci/cos-jq-portability-contract.sh b/ci/cos-jq-portability-contract.sh
index 9780f3c..48b07b7 100644
--- a/ci/cos-jq-portability-contract.sh
+++ b/ci/cos-jq-portability-contract.sh
@@ -3,13 +3,17 @@
set -euo pipefail
repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
-runtime_root="$repo_root/rootfs/home/cloud-compose"
+runtime_program_roots=(
+ "$repo_root/rootfs/home/cloud-compose"
+ "$repo_root/rootfs/etc/cloud-compose/jq"
+)
regex_call_pattern='(^|[^[:alnum:]_])(test|match|capture|scan|splits|sub|gsub)[[:space:]]*\('
nul_contains_pattern='contains[[:space:]]*\([[:space:]]*"\\u0000"[[:space:]]*\)'
failed=false
# Container-Optimized OS ships jq without Oniguruma. Its runtime scripts may
# use jq for JSON structure and types, but text validation belongs in Bash.
+# Check both shell call sites and the deployed jq program library.
# The remaining sub(/.../) and gsub(/.../) calls are awk regex literals, not
# jq filters; jq has no slash-delimited regex syntax.
while IFS= read -r call; do
@@ -18,7 +22,7 @@ while IFS= read -r call; do
fi
printf 'COS runtime contains a regex-dependent jq-style call: %s\n' "$call" >&2
failed=true
-done < <(grep -ERn --include='*.sh' "$regex_call_pattern" "$runtime_root" || true)
+done < <(grep -ERn --include='*.sh' --include='*.jq' "$regex_call_pattern" "${runtime_program_roots[@]}" || true)
# jq 1.6 treats every string as containing a NUL when contains("\u0000") is
# used. COS and supported configuration-management hosts can still run jq 1.6,
@@ -26,7 +30,7 @@ done < <(grep -ERn --include='*.sh' "$regex_call_pattern" "$runtime_root" || tru
while IFS= read -r call; do
printf 'Runtime contains jq 1.6-incompatible NUL validation: %s\n' "$call" >&2
failed=true
-done < <(grep -ERn --include='*.sh' "$nul_contains_pattern" "$runtime_root" || true)
+done < <(grep -ERn --include='*.sh' --include='*.jq' "$nul_contains_pattern" "${runtime_program_roots[@]}" || true)
if [[ "$failed" == "true" ]]; then
exit 1
diff --git a/ci/disaster-recovery-contract.sh b/ci/disaster-recovery-contract.sh
new file mode 100644
index 0000000..a6c7d5b
--- /dev/null
+++ b/ci/disaster-recovery-contract.sh
@@ -0,0 +1,90 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
+backup_script="$repo_root/rootfs/home/cloud-compose/offhost-backup.sh"
+restore_script="$repo_root/rootfs/home/cloud-compose/restore-test.sh"
+fixture_root="$repo_root/ci/testdata/disaster-recovery"
+tmp="$(mktemp -d "$repo_root/.disaster-recovery-contract.XXXXXX")"
+trap 'rm -rf -- "$tmp"' EXIT
+
+fail() {
+ echo "disaster recovery contract: $*" >&2
+ exit 1
+}
+
+mkdir -p "$tmp/bin" "$tmp/data/projects/alpha/files" "$tmp/data/backups/mariadb/alpha" "$tmp/drivers"
+chmod 0755 "$tmp" "$tmp/bin" "$tmp/data" "$tmp/data/projects" "$tmp/data/projects/alpha" "$tmp/data/projects/alpha/files" "$tmp/drivers"
+
+/usr/bin/install -m 0644 "$fixture_root/profile.sh" "$tmp/profile.sh"
+/usr/bin/install -m 0644 "$fixture_root/compose-apps.sh" "$tmp/compose-apps.sh"
+/usr/bin/install -m 0755 "$fixture_root/fake-stat.sh" "$tmp/bin/stat"
+/usr/bin/install -m 0755 "$fixture_root/fake-install.sh" "$tmp/bin/install"
+/usr/bin/install -m 0755 "$fixture_root/fake-docker.sh" "$tmp/bin/docker"
+/usr/bin/install -m 0644 "$fixture_root/fake-compose-config.jq" "$tmp/bin/fake-compose-config.jq"
+/usr/bin/install -m 0755 "$fixture_root/good-driver.sh" "$tmp/drivers/good-driver"
+/usr/bin/install -m 0755 "$fixture_root/incomplete-driver.sh" "$tmp/drivers/incomplete-driver"
+/usr/bin/install -m 0644 \
+ "$fixture_root/good-backup-receipt.jq" \
+ "$fixture_root/good-restore-proof.jq" \
+ "$fixture_root/incomplete-backup-receipt.jq" \
+ "$tmp/drivers/"
+printf 'logical database\n' | gzip -c >"$tmp/data/backups/mariadb/alpha/$(date -u +%Y%m%d)-alpha.sql.gz"
+
+export TEST_BIN="$tmp/bin"
+export TEST_DATA_ROOT="$tmp/data"
+export LOCK_LOG="$tmp/lock.log"
+export CLOUD_COMPOSE_PROFILE_PATH="$tmp/profile.sh"
+export CLOUD_COMPOSE_COMPOSE_APPS_PATH="$tmp/compose-apps.sh"
+export CLOUD_COMPOSE_DR_LIBRARY_PATH="$repo_root/rootfs/home/cloud-compose/disaster-recovery-lib.sh"
+export CLOUD_COMPOSE_DR_STATE_ROOT="$tmp/data/dr"
+export CLOUD_COMPOSE_JQ_PROGRAM_DIR="$repo_root/rootfs/etc/cloud-compose/jq"
+export CLOUD_COMPOSE_DATA_ROOT="$tmp/data"
+export CLOUD_COMPOSE_VOLUMES_ROOT="$tmp/data/volumes"
+export MARIADB_BACKUP_ROOT="$tmp/data/backups/mariadb"
+export CLOUD_COMPOSE_INSTANCE_NAME="contract-site"
+export CLOUD_COMPOSE_PROVIDER="contract"
+export CLOUD_COMPOSE_OFFHOST_BACKUP_REQUIRED="true"
+export CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER="$tmp/drivers/good-driver"
+export SHOULD_NOT_REACH_DRIVER="terraform-secret"
+: >"$LOCK_LOG"
+
+bash "$backup_script"
+receipt="$(find "$CLOUD_COMPOSE_DR_STATE_ROOT/backup-receipts" -maxdepth 1 -type f -name '*.json' -print -quit)"
+manifest="$(find "$CLOUD_COMPOSE_DR_STATE_ROOT/manifests" -maxdepth 1 -type f -name '*.json' -print -quit)"
+[[ -n "$receipt" && -n "$manifest" ]] || fail "successful driver did not publish its atomic manifest and receipt"
+jq -e -f "$fixture_root/assert-coverage-manifest.jq" "$manifest" >/dev/null || \
+ fail "coverage manifest omitted database, application files, or volume topology"
+
+# A second attempt sees the same already-existing daily dump but must still
+# invoke off-host transfer, allowing a failed first handoff to be retried.
+bash "$backup_script"
+[[ "$(grep -c '^backup$' "$tmp/drivers/good-driver.calls")" == "2" ]] || \
+ fail "nightly flow skipped off-host transfer when the daily dump already existed"
+
+published_receipt_sha="$(sha256sum "$receipt" | awk '{print $1}')"
+export CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER="$tmp/drivers/incomplete-driver"
+if bash "$backup_script" >/dev/null 2>&1; then
+ fail "driver receipt without complete volume coverage was accepted"
+fi
+[[ "$(sha256sum "$receipt" | awk '{print $1}')" == "$published_receipt_sha" ]] || \
+ fail "invalid driver output replaced the last validated receipt"
+
+export CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER="$tmp/drivers/good-driver"
+bash "$restore_script"
+proof="$(find "$CLOUD_COMPOSE_DR_STATE_ROOT/restore-proofs" -maxdepth 1 -type f -name '*.json' -print -quit)"
+[[ -n "$proof" ]] || fail "scheduled disposable restore test did not publish proof"
+jq -e -f "$fixture_root/assert-restore-proof.jq" "$proof" >/dev/null || \
+ fail "restore proof omitted required recovery evidence"
+
+rm -f -- "$tmp/data/backups/mariadb/alpha/$(date -u +%Y%m%d)-alpha.sql.gz"
+calls_before="$(wc -l <"$tmp/drivers/good-driver.calls")"
+if bash "$backup_script" >/dev/null 2>&1; then
+ fail "required DR coverage succeeded without a database artifact"
+fi
+[[ "$(wc -l <"$tmp/drivers/good-driver.calls")" == "$calls_before" ]] || \
+ fail "driver ran before required local coverage was validated"
+
+CLOUD_COMPOSE_OFFHOST_BACKUP_REQUIRED=false bash "$backup_script" >/dev/null
+echo "Disaster recovery contract passed"
diff --git a/ci/filesystem-prep-contract.sh b/ci/filesystem-prep-contract.sh
index 270e730..a6f49bd 100644
--- a/ci/filesystem-prep-contract.sh
+++ b/ci/filesystem-prep-contract.sh
@@ -643,11 +643,11 @@ for cloud_init_template in \
"$cloud_init_template" || fail "cloud-init does not verify every required mount before initialization"
grep -Fq 'Required cloud-compose mount is unavailable:' "$cloud_init_template" ||
fail "cloud-init mount gate does not report the unavailable path"
- grep -Fq ' bash /home/cloud-compose/start-cloud-compose-bootstrap.sh' \
+ grep -Fq ' bash /etc/cloud-compose/libexec/start-cloud-compose-bootstrap.sh' \
"$cloud_init_template" || fail "cloud-init retryable bootstrap startup is outside the fail-closed mount block"
- marker_reset_line="$(grep -nF ' rm -f /home/cloud-compose/.cloud-compose-bootstrap-complete' \
+ marker_reset_line="$(grep -nF ' rm -f /var/lib/cloud-compose/bootstrap-complete' \
"$cloud_init_template" | cut -d: -f1)"
- run_line="$(grep -nF ' bash /home/cloud-compose/start-cloud-compose-bootstrap.sh' \
+ run_line="$(grep -nF ' bash /etc/cloud-compose/libexec/start-cloud-compose-bootstrap.sh' \
"$cloud_init_template" | cut -d: -f1)"
[[ -n "$marker_reset_line" && -n "$run_line" && "$marker_reset_line" -lt "$run_line" ]] ||
fail "cloud-init does not clear stale bootstrap readiness before retryable startup"
diff --git a/ci/gcp-upgrade-smoke-contract.sh b/ci/gcp-upgrade-smoke-contract.sh
index 6d02628..1fcc795 100755
--- a/ci/gcp-upgrade-smoke-contract.sh
+++ b/ci/gcp-upgrade-smoke-contract.sh
@@ -47,16 +47,33 @@ grep -Fq "provider_tag_cleanup gcp-wp \"\$run_id\"" "$script" ||
fail "upgrade runner does not finish cleanup with the verified provider sweep"
grep -Fq 'target_env gcp-wp' "$script" ||
fail "upgrade cleanup does not load the concrete GCP WordPress target environment"
-[[ "$(grep -Fc 'sudo test -f /home/cloud-compose/run.log && sudo test ! -L /home/cloud-compose/run.log' "$shared_smoke")" -eq 2 ]] ||
- fail "shared smoke diagnostics do not retain guarded legacy bootstrap logs for the pinned baseline"
-grep -Fq 'bootstrap_load_state=\"\$(systemctl show --property=LoadState --value -- cloud-compose-bootstrap.service' "$shared_smoke" ||
- fail "shared smoke readiness does not distinguish retryable bootstrap from the legacy baseline"
-grep -Fq 'active:* | activating:* | *:auto-restart)' "$shared_smoke" ||
- fail "shared smoke readiness can abandon the bootstrap unit during its restart delay"
-grep -Fq 'elif [ \"\$bootstrap_load_state\" = not-found ] &&' "$shared_smoke" ||
- fail "shared smoke readiness does not restrict legacy fallback to an absent bootstrap unit"
-grep -Fq 'systemctl is-active --quiet cloud-compose; then' "$shared_smoke" ||
+grep -Fq 'readonly diagnostics_program="/etc/cloud-compose/bin/cloud-compose-diagnostics.sh"' "$shared_smoke" ||
+ fail "shared smoke diagnostics do not use the checked-in privileged program"
+for diagnostics_command in state status dump; do
+ grep -Fq "sudo -n \${diagnostics_program} ${diagnostics_command}" "$shared_smoke" ||
+ fail "shared smoke diagnostics do not invoke the ${diagnostics_command} command non-interactively"
+done
+grep -Fq 'test -f /home/cloud-compose/.cloud-compose-bootstrap-complete' "$shared_smoke" ||
+ fail "shared smoke readiness lost its pinned-baseline marker compatibility probe"
+grep -Fq 'systemctl is-active --quiet cloud-compose.service' "$shared_smoke" ||
fail "shared smoke readiness lost its pre-bootstrap-unit compatibility signal"
+grep -Fq 'cloud-init completed without the Cloud Compose readiness marker' "$shared_smoke" ||
+ fail "current smoke readiness can still accept cloud-init completion without durable application readiness"
+grep -Fq 'The pinned upgrade fixture predates the durable readiness marker.' "$shared_smoke" ||
+ fail "legacy cloud-init completion is not explicitly confined to the pinned upgrade fixture"
+grep -Fq 'test ! -L /home/cloud-compose/run.log' "$shared_smoke" ||
+ fail "shared smoke diagnostics do not retain a guarded legacy bootstrap log fallback"
+grep -Fq 'sudo -n /usr/bin/systemctl status cloud-compose.service' "$shared_smoke" ||
+ fail "shared smoke diagnostics do not limit pinned-baseline sudo to its existing exact command"
+grep -Fq 'readonly smoke_healthcheck_program="/home/cloud-compose/smoke-healthcheck.sh"' "$shared_smoke" ||
+ fail "shared smoke healthcheck does not use the checked-in host wrapper"
+grep -Fq '"${smoke_healthcheck_program} ${quoted_context}"' "$shared_smoke" ||
+ fail "shared smoke healthcheck does not invoke the checked-in host wrapper directly"
+grep -Fq 'env HOME=/home/cloud-compose DOCKER_CONFIG=/mnt/disks/data/docker-config PATH=/home/cloud-compose/bin:' "$shared_smoke" ||
+ fail "shared smoke healthcheck lost its direct pinned-baseline fallback"
+if sed -n '/^run_healthcheck()/,/^}/p' "$shared_smoke" | grep -Fq 'bash -lc'; then
+ fail "shared smoke healthcheck still sends an embedded Bash program over SSH"
+fi
grep -Fq 'CLOUD_COMPOSE_SMOKE_RUN_ID must match GITHUB_RUN_ID in GitHub Actions' "$script" ||
fail "hosted cleanup ownership is not bound to the actual GitHub run id"
grep -Fq 'CLOUD_COMPOSE_SMOKE_RUN_ID must be set explicitly outside GitHub Actions' "$script" ||
@@ -158,7 +175,7 @@ baseline_initcmd_line="$(grep -nF 'for CMD in ADDITIONAL_INITCMD' "$tmp/baseline
fail "baseline cloud-init does not execute fixture initcmd before run.sh"
current_bootstrap_line="$(
- grep -nF 'bash /home/cloud-compose/start-cloud-compose-bootstrap.sh' \
+ grep -nF 'bash /etc/cloud-compose/libexec/start-cloud-compose-bootstrap.sh' \
"$repo_root/templates/cloud-init.yml" |
cut -d: -f1 || true
)"
diff --git a/ci/host-runtime-security.sh b/ci/host-runtime-security.sh
index 6a76810..5b13cd1 100644
--- a/ci/host-runtime-security.sh
+++ b/ci/host-runtime-security.sh
@@ -165,6 +165,8 @@ assert_contains "$profile_script" 'export PATH="/usr/local/sbin:/usr/local/bin:/
tmpfiles_conf="$repo_root/rootfs/etc/tmpfiles.d/cloud-compose.conf"
assert_contains "$tmpfiles_conf" 'd /run/lock/cloud-compose 0750 root cloud-compose -'
assert_contains "$tmpfiles_conf" 'f /run/lock/cloud-compose/lifecycle.lock 0660 root cloud-compose -'
+assert_contains "$tmpfiles_conf" 'd /var/lib/cloud-compose 0755 root root -'
+assert_contains "$tmpfiles_conf" 'd /home/cloud-compose 0755 root root -'
host_init="$repo_root/rootfs/home/cloud-compose/host-init.sh"
if grep -Eq 'chown[[:space:]]+-R[[:space:]]+cloud-compose[^[:space:]]*[[:space:]]+/home/cloud-compose' "$host_init"; then
fail "host initialization gives the app account recursive ownership of root-executed code"
@@ -174,11 +176,25 @@ if grep -Eq 'chown[[:space:]]+-R[[:space:]]+cloud-compose[^[:space:]]*[[:space:]
fi
assert_contains "$host_init" 'chown root:root /home/cloud-compose'
assert_contains "$host_init" "-exec chown root:root {} +"
+assert_contains "$host_init" 'Unsafe Cloud Compose lifecycle dispatcher:'
+assert_contains "$repo_root/rootfs/etc/cloud-compose/libexec/bootstrap-security.sh" \
+ 'Cloud Compose control input is not root-controlled'
assert_contains "$host_init" '/home/cloud-compose/.sitectl \'
assert_contains "$host_init" 'install -d -m 0750 -o cloud-compose -g cloud-compose "$mutable_dir"'
+assert_contains "$host_init" 'Managed command directory was not secured by the runtime installer'
+if sed -n '/for mutable_dir in/,/done/p' "$host_init" | grep -Fq '/home/cloud-compose/bin'; then
+ fail "host initialization leaves the privileged published command directory app-writable"
+fi
assert_contains "$host_init" 'install -d -m 1775 -o root -g cloud-compose /mnt/disks/data'
assert_contains "$host_init" 'install -d -m 0775 -o cloud-compose -g cloud-compose /mnt/disks/volumes'
assert_contains "$host_init" 'install -d -m 0775 -o cloud-compose -g cloud-compose /mnt/disks/data/libops'
+managed_runtime="$repo_root/rootfs/home/cloud-compose/libops-managed-runtime.sh"
+assert_contains "$managed_runtime" 'prepare_managed_runtime_directory'
+assert_contains "$managed_runtime" 'validate_published_bin_directory'
+assert_contains "$managed_runtime" '$STATE_DIR:0755:false'
+assert_contains "$managed_runtime" '$PUBLISHED_BIN_DIR:0755:true'
+assert_contains "$managed_runtime" 'managed runtime updates must run as root'
+assert_contains "$managed_runtime" 'production managed runtime directories require a root updater'
if grep -A12 -F 'source_compose_app_env()' "$repo_root/rootfs/home/cloud-compose/compose-apps.sh" | \
grep -Eq 'source[[:space:]]+.*COMPOSE_APPS_ENV_DIR'; then
fail "privileged Compose manifest loading still sources an app-writable shell file"
@@ -227,11 +243,49 @@ if grep -Fq 'rotate-keys' "$repo_root/rootfs/home/cloud-compose/docker-prune.sh"
fail "provider-neutral Docker prune still invokes GCP key rotation"
fi
-archive_source="$repo_root/modules/linux-vm-runtime/main.tf"
-verify_line="$(grep -n 'sha256sum -c -' "$archive_source" | head -n 1 | cut -d: -f1)"
-extract_line="$(grep -n 'tar -xzf "\$tmp/rootfs.tar.gz"' "$archive_source" | head -n 1 | cut -d: -f1)"
-[[ -n "$verify_line" && -n "$extract_line" && "$verify_line" -lt "$extract_line" ]] || \
- fail "rootfs archive is not verified before extraction"
+for archive_source in \
+ "$repo_root/modules/linux-vm-runtime/main.tf" \
+ "$repo_root/modules/gcp/main.tf"; do
+ verify_line="$(grep -n 'sha256sum -c -' "$archive_source" | head -n 1 | cut -d: -f1)"
+ extract_line="$(grep -n 'tar --no-same-owner -xzf "\$tmp/rootfs.tar.gz"' "$archive_source" | head -n 1 | cut -d: -f1)"
+ normalize_line="$(grep -n 'chown -hR 0:0 -- "\$rootfs_dir"' "$archive_source" | head -n 1 | cut -d: -f1)"
+ copy_line="$(grep -n 'cp -a "\$rootfs_dir"/. /' "$archive_source" | head -n 1 | cut -d: -f1)"
+ [[ -n "$verify_line" && -n "$extract_line" && -n "$normalize_line" && -n "$copy_line" &&
+ "$verify_line" -lt "$extract_line" && "$extract_line" -lt "$normalize_line" &&
+ "$normalize_line" -lt "$copy_line" ]] || \
+ fail "$archive_source does not verify, ownership-safe extract, then normalize the rootfs to root"
+done
+
+if [[ -n "$(git -C "$repo_root" ls-files 'rootfs/usr/**')" ]]; then
+ fail "Cloud Compose-owned rootfs programs still target immutable /usr"
+fi
+for trusted_program in \
+ rootfs/etc/cloud-compose/bin/cloud-compose-diagnostics.sh \
+ rootfs/etc/cloud-compose/libexec/harden-bootstrap-paths.sh \
+ rootfs/etc/cloud-compose/libexec/build-cos-make.sh \
+ rootfs/etc/cloud-compose/libexec/bootstrap-security.sh \
+ rootfs/etc/cloud-compose/libexec/run-bootstrap.sh \
+ rootfs/etc/cloud-compose/libexec/run-root-program.sh \
+ rootfs/etc/cloud-compose/jq/offhost-validate-manifest.jq; do
+ [[ -f "$repo_root/$trusted_program" ]] || fail "COS-safe trusted program is missing: $trusted_program"
+done
+
+assert_contains "$repo_root/templates/cloud-init.yml" \
+ '/etc/cloud-compose/libexec/harden-bootstrap-paths.sh'
+assert_contains "$repo_root/modules/linux-vm-runtime/templates/cloud-init.yml" \
+ '/etc/cloud-compose/libexec/harden-bootstrap-paths.sh'
+assert_contains "$repo_root/rootfs/etc/cloud-compose/libexec/harden-bootstrap-paths.sh" \
+ 'chown root:root "$cloud_compose_home"'
+assert_contains "$repo_root/rootfs/etc/cloud-compose/libexec/harden-bootstrap-paths.sh" \
+ "0:1:regular file"
+for cloud_init_template in \
+ "$repo_root/templates/cloud-init.yml" \
+ "$repo_root/modules/linux-vm-runtime/templates/cloud-init.yml"; do
+ assert_contains "$cloud_init_template" 'install -d -m 0755 -o root -g root /etc/cloud-compose/bin'
+ if grep -Eq '^[[:space:]]*- path: /usr/|install -d[^#]* /usr/local' "$cloud_init_template"; then
+ fail "$cloud_init_template writes Cloud Compose-owned programs beneath immutable /usr"
+ fi
+done
for variables_file in \
"$repo_root/variables.tf" \
diff --git a/ci/rootfs-package-contract.sh b/ci/rootfs-package-contract.sh
index 0c00046..19a1a6d 100755
--- a/ci/rootfs-package-contract.sh
+++ b/ci/rootfs-package-contract.sh
@@ -20,5 +20,21 @@ tar -tzf "$tmp/one/cloud-compose-rootfs.tar.gz" | awk '
END { exit bad }
'
tar -tzf "$tmp/one/cloud-compose-rootfs.tar.gz" | grep -Fx 'rootfs/home/cloud-compose/run.sh' >/dev/null
+tar -tzf "$tmp/one/cloud-compose-rootfs.tar.gz" | \
+ grep -Fx 'rootfs/etc/cloud-compose/libexec/run-bootstrap.sh' >/dev/null
+tar -tzf "$tmp/one/cloud-compose-rootfs.tar.gz" | \
+ grep -Fx 'rootfs/etc/cloud-compose/libexec/run-root-program.sh' >/dev/null
+tar -tzf "$tmp/one/cloud-compose-rootfs.tar.gz" | \
+ grep -Fx 'rootfs/etc/cloud-compose/libexec/build-cos-make.sh' >/dev/null
+tar -tzf "$tmp/one/cloud-compose-rootfs.tar.gz" | \
+ grep -Fx 'rootfs/etc/cloud-compose/libexec/harden-bootstrap-paths.sh' >/dev/null
+tar -tzf "$tmp/one/cloud-compose-rootfs.tar.gz" | \
+ grep -Fx 'rootfs/etc/cloud-compose/jq/offhost-validate-manifest.jq' >/dev/null
+tar -tzf "$tmp/one/cloud-compose-rootfs.tar.gz" | \
+ grep -Fx 'rootfs/etc/cloud-compose/bin/cloud-compose-diagnostics.sh' >/dev/null
+if tar -tzf "$tmp/one/cloud-compose-rootfs.tar.gz" | grep -E '^rootfs/usr/' >/dev/null; then
+ echo "Rootfs package contains a Cloud Compose-owned immutable /usr path" >&2
+ exit 1
+fi
echo "Rootfs package contract passed"
diff --git a/ci/sitectl-version-contract.sh b/ci/sitectl-version-contract.sh
index 0ec2f60..f868d64 100644
--- a/ci/sitectl-version-contract.sh
+++ b/ci/sitectl-version-contract.sh
@@ -42,6 +42,53 @@ CLOUD_COMPOSE_PROFILE_PATH="$tmp/profile.sh" \
test "$(stat -c "%a" "$TMP_DIR")" = 700
test "$(stat -c "%a" "$PACKAGE_STATE_DIR")" = 700
test "$(stat -c "%a" "$ARTIFACT_STATE_DIR")" = 700
+ test "$(stat -c "%a" "$PUBLISHED_BIN_DIR")" = 755
+ test "$(stat -c "%u:%g" "$STATE_DIR")" = "$(id -u):$(id -g)"
+ test "$(stat -c "%u:%g" "$PUBLISHED_BIN_DIR")" = "$(id -u):$(id -g)"
+
+ touch "$PUBLISHED_BIN_DIR/docker"
+ if mkdirs; then
+ echo "managed runtime accepted an unmanaged published command" >&2
+ exit 1
+ fi
+ rm -f "$PUBLISHED_BIN_DIR/docker"
+
+ touch "$BIN_DIR/make"
+ ln -s "$BIN_DIR/make" "$PUBLISHED_BIN_DIR/make"
+ mkdirs
+ rm -f "$PUBLISHED_BIN_DIR/make"
+ ln -s "$2/unsafe-make" "$PUBLISHED_BIN_DIR/make"
+ if mkdirs; then
+ echo "managed runtime accepted an unsafe published Make target" >&2
+ exit 1
+ fi
+ rm -f "$PUBLISHED_BIN_DIR/make"
+
+ unsafe_target="$2/unsafe-target"
+ unsafe_state="$2/unsafe-state"
+ mkdir -p "$unsafe_target"
+ ln -s "$unsafe_target" "$unsafe_state"
+ STATE_DIR="$unsafe_state"
+ BIN_DIR="$STATE_DIR/bin"
+ TMP_DIR="$STATE_DIR/tmp"
+ PACKAGE_STATE_DIR="$STATE_DIR/packages"
+ ARTIFACT_STATE_DIR="$STATE_DIR/artifacts"
+ if mkdirs; then
+ echo "managed runtime accepted a redirected state directory" >&2
+ exit 1
+ fi
+
+ unsafe_state="$2/unsafe-writable-state"
+ mkdir -m 0775 "$unsafe_state"
+ STATE_DIR="$unsafe_state"
+ BIN_DIR="$STATE_DIR/bin"
+ TMP_DIR="$STATE_DIR/tmp"
+ PACKAGE_STATE_DIR="$STATE_DIR/packages"
+ ARTIFACT_STATE_DIR="$STATE_DIR/artifacts"
+ if mkdirs; then
+ echo "managed runtime accepted group-writable state" >&2
+ exit 1
+ fi
' cloud-compose-sitectl-modes "$runtime_script" "$tmp"
run_contract() {
diff --git a/ci/systemd-contract.sh b/ci/systemd-contract.sh
index 07e38a4..aa6f000 100755
--- a/ci/systemd-contract.sh
+++ b/ci/systemd-contract.sh
@@ -4,6 +4,10 @@ set -euo pipefail
repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
unit_dir="$repo_root/rootfs/etc/systemd/system"
+diagnostics_program="$repo_root/rootfs/etc/cloud-compose/bin/cloud-compose-diagnostics.sh"
+smoke_healthcheck_program="$repo_root/rootfs/home/cloud-compose/smoke-healthcheck.sh"
+bootstrap_security="$repo_root/rootfs/etc/cloud-compose/libexec/bootstrap-security.sh"
+root_program_runner="$repo_root/rootfs/etc/cloud-compose/libexec/run-root-program.sh"
fail() {
echo "systemd contract: $*" >&2
@@ -25,8 +29,15 @@ assert_contains() {
[[ -f "$unit_dir/cloud-compose-vault-agent.service" ]] || fail "namespaced Vault Agent unit is missing"
[[ -f "$unit_dir/cloud-compose-overlay.service" ]] || fail "Docker overlay mount unit is missing"
[[ -f "$unit_dir/cloud-compose-bootstrap.service" ]] || fail "retryable bootstrap unit is missing"
+[[ -x "$diagnostics_program" ]] || fail "checked-in Cloud Compose diagnostics program is missing or not executable"
+[[ -f "$bootstrap_security" ]] || fail "root-owned bootstrap security helper is missing"
+[[ -f "$root_program_runner" ]] || fail "root-owned service launcher is missing"
+[[ -x "$smoke_healthcheck_program" ]] || fail "checked-in smoke healthcheck wrapper is missing or not executable"
[[ -f "$unit_dir/cloud-compose-internal-services.service" && -f "$unit_dir/cloud-compose-internal-services.timer" ]] || \
fail "namespaced internal-services units are missing"
+[[ -f "$unit_dir/cloud-compose-offhost-backup.service" ]] || fail "off-host backup service is missing"
+[[ -f "$unit_dir/cloud-compose-restore-test.service" && -f "$unit_dir/cloud-compose-restore-test.timer" ]] || \
+ fail "scheduled restore-test units are missing"
assert_contains "$unit_dir/cloud-compose.service" 'Requires=docker.service cloud-compose-metadata-firewall.service'
assert_contains "$unit_dir/cloud-compose.service" 'RequiresMountsFor=/mnt/disks/data /mnt/disks/volumes /mnt/disks/data/docker/volumes'
@@ -34,8 +45,11 @@ assert_contains "$unit_dir/cloud-compose.service" 'After=network-online.target d
assert_contains "$unit_dir/cloud-compose.service" 'ExecStartPre=/bin/bash /home/cloud-compose/assert-app-initialized.sh'
assert_contains "$unit_dir/cloud-compose.service" 'Restart=on-failure'
assert_contains "$unit_dir/cloud-compose.service" 'RestartSec=30s'
-assert_contains "$unit_dir/cloud-compose-bootstrap.service" 'ConditionPathExists=!/home/cloud-compose/.cloud-compose-bootstrap-complete'
-assert_contains "$unit_dir/cloud-compose-bootstrap.service" 'ExecStart=/bin/bash /home/cloud-compose/run-bootstrap.sh'
+if grep -Fq 'ConditionPathExists=' "$unit_dir/cloud-compose-bootstrap.service"; then
+ fail "bootstrap still trusts an unvalidated marker path condition"
+fi
+assert_contains "$unit_dir/cloud-compose-bootstrap.service" 'ExecCondition=/bin/bash /etc/cloud-compose/libexec/bootstrap-required.sh'
+assert_contains "$unit_dir/cloud-compose-bootstrap.service" 'ExecStart=/bin/bash /etc/cloud-compose/libexec/run-bootstrap.sh'
assert_contains "$unit_dir/cloud-compose-bootstrap.service" 'Restart=on-failure'
assert_contains "$unit_dir/cloud-compose-bootstrap.service" 'RestartSec=30s'
assert_contains "$unit_dir/cloud-compose-bootstrap.service" 'TimeoutStartSec=2h'
@@ -48,8 +62,36 @@ assert_contains "$unit_dir/cloud-compose-bootstrap.service" 'SyslogLevelPrefix=n
assert_contains "$unit_dir/cloud-compose-bootstrap.service" 'LogRateLimitIntervalSec=30s'
assert_contains "$unit_dir/cloud-compose-bootstrap.service" 'LogRateLimitBurst=1000'
assert_contains "$unit_dir/cloud-compose-internal-services.service" 'Requires=cloud-compose.service cloud-compose-metadata-firewall.service'
-assert_contains "$unit_dir/cloud-compose.service" 'TimeoutStartSec=1h'
+assert_contains "$unit_dir/cloud-compose.service" 'TimeoutStartSec=90min'
+assert_contains "$diagnostics_program" 'usage: ${diagnostics_program} state|status|dump'
+assert_contains "$diagnostics_program" 'readonly PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"'
+assert_contains "$diagnostics_program" 'readonly bootstrap_marker="/var/lib/cloud-compose/bootstrap-complete"'
+assert_contains "$bootstrap_security" 'cloud_compose_bootstrap_marker_ready()'
+assert_contains "$bootstrap_security" '0:0:644:1:regular file'
+assert_contains "$bootstrap_security" '"$marker_size" == "6"'
+assert_contains "$bootstrap_security" '"$payload" == "ready"'
+assert_contains "$bootstrap_security" 'cloud_compose_secure_runtime_home()'
+assert_contains "$root_program_runner" 'cloud_compose_secure_runtime_home'
+assert_contains "$root_program_runner" 'Unsupported Cloud Compose root program:'
+[[ -x "$repo_root/rootfs/etc/cloud-compose/libexec/require-bootstrap-ready.sh" ]] || \
+ fail "validated bootstrap readiness gate is missing or not executable"
+assert_contains "$diagnostics_program" '--- Cloud Compose provisioning heartbeat ---'
+assert_contains "$diagnostics_program" 'ps -p "$main_pid" -o pid=,ppid=,stat=,etime=,comm='
+assert_contains "$smoke_healthcheck_program" 'source /home/cloud-compose/profile.sh'
+assert_contains "$smoke_healthcheck_program" 'exec sitectl healthcheck --context "$context" --persist --format table'
assert_contains "$unit_dir/cloud-compose-mariadb-backup.service" 'TimeoutStartSec=12h'
+assert_contains "$unit_dir/cloud-compose-mariadb-backup.service" 'User=cloud-compose'
+assert_contains "$unit_dir/cloud-compose-mariadb-backup.timer" 'Unit=cloud-compose-offhost-backup.service'
+assert_contains "$unit_dir/cloud-compose-offhost-backup.service" 'Requires=cloud-compose-mariadb-backup.service'
+assert_contains "$unit_dir/cloud-compose-offhost-backup.service" 'After=cloud-compose-mariadb-backup.service network-online.target'
+assert_contains "$unit_dir/cloud-compose-offhost-backup.service" 'User=root'
+assert_contains "$unit_dir/cloud-compose-offhost-backup.service" 'UMask=0077'
+assert_contains "$unit_dir/cloud-compose-offhost-backup.service" 'TimeoutStartSec=24h'
+assert_contains "$unit_dir/cloud-compose-restore-test.service" 'User=root'
+assert_contains "$unit_dir/cloud-compose-restore-test.service" 'UMask=0077'
+assert_contains "$unit_dir/cloud-compose-restore-test.service" 'TimeoutStartSec=24h'
+assert_contains "$unit_dir/cloud-compose-restore-test.timer" 'OnCalendar=Sun *-*-* 03:00:00'
+assert_contains "$unit_dir/cloud-compose-restore-test.timer" 'Persistent=true'
if grep -Fq 'Wants=cloud-compose.service' "$unit_dir/cloud-compose-mariadb-backup.service"; then
fail "backup service starts an intentionally inactive application"
fi
@@ -74,9 +116,24 @@ assert_contains "$docker_metadata_dropin" 'Requires=cloud-compose-metadata-firew
assert_contains "$docker_metadata_dropin" 'After=cloud-compose-metadata-firewall-pre.service'
assert_contains "$metadata_pre_unit" 'Before=docker.service'
assert_contains "$metadata_pre_unit" 'WantedBy=multi-user.target'
-assert_contains "$metadata_pre_unit" 'ExecStart=/bin/bash /home/cloud-compose/configure-metadata-firewall.sh pre-docker'
+assert_contains "$metadata_pre_unit" 'ExecStart=/bin/bash /etc/cloud-compose/libexec/run-root-program.sh configure-metadata-firewall.sh pre-docker'
assert_contains "$unit_dir/cloud-compose-overlay.service" 'Before=docker.service cloud-compose.service'
-assert_contains "$unit_dir/cloud-compose-overlay.service" 'ExecStart=/bin/bash /home/cloud-compose/mount-overlays.sh'
+assert_contains "$unit_dir/cloud-compose-overlay.service" 'ExecStart=/bin/bash /etc/cloud-compose/libexec/run-root-program.sh mount-overlays.sh'
+for root_home_unit in \
+ cloud-compose-docker-prune.service \
+ cloud-compose-key-rotation.service \
+ cloud-compose-metadata-firewall-pre.service \
+ cloud-compose-metadata-firewall.service \
+ cloud-compose-offhost-backup.service \
+ cloud-compose-overlay.service \
+ cloud-compose-restore-test.service \
+ cloud-compose-vault-agent.service \
+ libops-managed-runtime.service; do
+ if grep -Eq '^Exec(Start|StartPre|StartPost|Stop|StopPost)=/bin/bash /home/cloud-compose/' "$unit_dir/$root_home_unit"; then
+ fail "$root_home_unit executes historically writable home code without the root-owned launcher"
+ fi
+ assert_contains "$unit_dir/$root_home_unit" '/etc/cloud-compose/libexec/run-root-program.sh'
+done
if grep -Eq '^(After|Before|BindsTo|PartOf|Requires|Requisite|Wants)=.*cloud-compose-bootstrap\\.service' \
"$unit_dir/cloud-compose.service"; then
fail "application service has an ordering dependency on the bootstrap service"
@@ -98,6 +155,9 @@ assert_contains "$run_script" 'systemctl disable --now libops-managed-runtime.ti
assert_contains "$run_script" 'systemctl enable --now cloud-compose-docker-prune.timer'
assert_contains "$run_script" 'systemctl disable --now cloud-compose-docker-prune.timer cloud-compose-docker-prune.service'
assert_contains "$run_script" 'systemctl enable --now cloud-compose-mariadb-backup.timer'
+assert_contains "$run_script" 'runtime_enabled "${CLOUD_COMPOSE_OFFHOST_BACKUP_REQUIRED:-false}"'
+assert_contains "$run_script" 'systemctl enable --now cloud-compose-restore-test.timer'
+assert_contains "$run_script" 'systemctl disable --now cloud-compose-restore-test.timer cloud-compose-restore-test.service'
migration_script="$repo_root/rootfs/home/cloud-compose/migrate-legacy-systemd-units.sh"
migration_tmp="$(mktemp -d)"
diff --git a/ci/testdata/disaster-recovery/assert-coverage-manifest.jq b/ci/testdata/disaster-recovery/assert-coverage-manifest.jq
new file mode 100644
index 0000000..2aedb1c
--- /dev/null
+++ b/ci/testdata/disaster-recovery/assert-coverage-manifest.jq
@@ -0,0 +1,6 @@
+.required_coverage == ["database", "application_files", "volume_topology"] and
+(.applications | length == 1) and
+(.applications[0].databases | length == 1) and
+.applications[0].application_files.roots == [env.TEST_DATA_ROOT + "/projects/alpha"] and
+.applications[0].volume_topology.declared_named_volumes == ["alpha_data"] and
+(.applications[0].volume_topology.service_mounts | length == 3)
diff --git a/ci/testdata/disaster-recovery/assert-restore-proof.jq b/ci/testdata/disaster-recovery/assert-restore-proof.jq
new file mode 100644
index 0000000..c7752de
--- /dev/null
+++ b/ci/testdata/disaster-recovery/assert-restore-proof.jq
@@ -0,0 +1,4 @@
+.disposable_recovery == true and
+.recovery_destroyed == true and
+.integrity_verified == true and
+.coverage == {database: true, application_files: true, volume_topology: true}
diff --git a/ci/testdata/disaster-recovery/compose-apps.sh b/ci/testdata/disaster-recovery/compose-apps.sh
new file mode 100644
index 0000000..e220836
--- /dev/null
+++ b/ci/testdata/disaster-recovery/compose-apps.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+compose_app_names_array() {
+ local -n result="$1"
+ # shellcheck disable=SC2034 # The caller reads the array through this nameref.
+ result=(alpha)
+}
+
+source_compose_app_env() {
+ DOCKER_COMPOSE_DIR="${TEST_DATA_ROOT:?}/projects/$1"
+ export DOCKER_COMPOSE_DIR
+}
+
+validate_compose_project_dir() {
+ [[ "$1" == "${TEST_DATA_ROOT:?}/projects/"* ]]
+}
diff --git a/ci/testdata/disaster-recovery/fake-compose-config.jq b/ci/testdata/disaster-recovery/fake-compose-config.jq
new file mode 100644
index 0000000..919adbb
--- /dev/null
+++ b/ci/testdata/disaster-recovery/fake-compose-config.jq
@@ -0,0 +1,12 @@
+{
+ services: {
+ web: {
+ volumes: [
+ {type: "bind", source: ($root + "/projects/alpha/files"), target: "/srv/files", read_only: false},
+ {type: "volume", source: "alpha_data", target: "/var/lib/app", read_only: false},
+ {type: "tmpfs", source: "", target: "/run/app", read_only: false}
+ ]
+ }
+ },
+ volumes: {alpha_data: {name: "alpha_data"}}
+}
diff --git a/ci/testdata/disaster-recovery/fake-docker.sh b/ci/testdata/disaster-recovery/fake-docker.sh
new file mode 100644
index 0000000..9e04044
--- /dev/null
+++ b/ci/testdata/disaster-recovery/fake-docker.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+fixture_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+[[ "$*" == "compose config --format json" ]]
+jq -cn --arg root "${TEST_DATA_ROOT:?}" \
+ -f "$fixture_dir/fake-compose-config.jq"
diff --git a/ci/testdata/disaster-recovery/fake-install.sh b/ci/testdata/disaster-recovery/fake-install.sh
new file mode 100644
index 0000000..46c7700
--- /dev/null
+++ b/ci/testdata/disaster-recovery/fake-install.sh
@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+args=()
+while (($# > 0)); do
+ case "$1" in
+ -o | -g)
+ shift 2
+ ;;
+ *)
+ args+=("$1")
+ shift
+ ;;
+ esac
+done
+exec /usr/bin/install "${args[@]}"
diff --git a/ci/testdata/disaster-recovery/fake-stat.sh b/ci/testdata/disaster-recovery/fake-stat.sh
new file mode 100644
index 0000000..17c8e33
--- /dev/null
+++ b/ci/testdata/disaster-recovery/fake-stat.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+output="$(/usr/bin/stat "$@")"
+if [[ "$*" == *"%u:"* ]]; then
+ printf '0:%s\n' "${output#*:}"
+else
+ printf '%s\n' "$output"
+fi
diff --git a/ci/testdata/disaster-recovery/good-backup-receipt.jq b/ci/testdata/disaster-recovery/good-backup-receipt.jq
new file mode 100644
index 0000000..20e0d65
--- /dev/null
+++ b/ci/testdata/disaster-recovery/good-backup-receipt.jq
@@ -0,0 +1,12 @@
+{
+ schema_version: 1,
+ kind: "cloud-compose.offhost-backup-receipt",
+ operation_id: $operation_id,
+ completed_at: "2026-08-07T12:00:00Z",
+ manifest_sha256: $manifest_sha256,
+ encrypted: true,
+ off_host: true,
+ status: "succeeded",
+ remote_id: "contract/backup-1",
+ coverage: {database: true, application_files: true, volume_topology: true}
+}
diff --git a/ci/testdata/disaster-recovery/good-driver.sh b/ci/testdata/disaster-recovery/good-driver.sh
new file mode 100644
index 0000000..16a684d
--- /dev/null
+++ b/ci/testdata/disaster-recovery/good-driver.sh
@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+fixture_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+[[ -z "${SHOULD_NOT_REACH_DRIVER+x}" ]] || exit 90
+printf '%s\n' "$1" >>"${0}.calls"
+operation="$1"
+shift
+declare -A args=()
+while (($# > 0)); do
+ args["$1"]="$2"
+ shift 2
+done
+case "$operation" in
+ backup)
+ jq -cn \
+ --arg operation_id "${args[--operation-id]}" \
+ --arg manifest_sha256 "${args[--manifest-sha256]}" \
+ -f "$fixture_dir/good-backup-receipt.jq" >"${args[--receipt]}"
+ ;;
+ restore-test)
+ jq -cn \
+ --arg test_id "${args[--test-id]}" \
+ --arg manifest_sha256 "${args[--source-manifest-sha256]}" \
+ --arg receipt_sha256 "${args[--source-receipt-sha256]}" \
+ -f "$fixture_dir/good-restore-proof.jq" >"${args[--proof]}"
+ ;;
+ *) exit 64 ;;
+esac
diff --git a/ci/testdata/disaster-recovery/good-restore-proof.jq b/ci/testdata/disaster-recovery/good-restore-proof.jq
new file mode 100644
index 0000000..512892a
--- /dev/null
+++ b/ci/testdata/disaster-recovery/good-restore-proof.jq
@@ -0,0 +1,15 @@
+{
+ schema_version: 1,
+ kind: "cloud-compose.restore-test-proof",
+ test_id: $test_id,
+ completed_at: "2026-08-07T13:00:00Z",
+ source_manifest_sha256: $manifest_sha256,
+ source_receipt_sha256: $receipt_sha256,
+ source_encrypted: true,
+ status: "succeeded",
+ recovery_id: "contract/recovery-1",
+ disposable_recovery: true,
+ recovery_destroyed: true,
+ integrity_verified: true,
+ coverage: {database: true, application_files: true, volume_topology: true}
+}
diff --git a/ci/testdata/disaster-recovery/incomplete-backup-receipt.jq b/ci/testdata/disaster-recovery/incomplete-backup-receipt.jq
new file mode 100644
index 0000000..114cf7e
--- /dev/null
+++ b/ci/testdata/disaster-recovery/incomplete-backup-receipt.jq
@@ -0,0 +1,12 @@
+{
+ schema_version: 1,
+ kind: "cloud-compose.offhost-backup-receipt",
+ operation_id: $operation_id,
+ completed_at: "2026-08-07T12:00:00Z",
+ manifest_sha256: $manifest_sha256,
+ encrypted: true,
+ off_host: true,
+ status: "succeeded",
+ remote_id: "contract/incomplete",
+ coverage: {database: true, application_files: true, volume_topology: false}
+}
diff --git a/ci/testdata/disaster-recovery/incomplete-driver.sh b/ci/testdata/disaster-recovery/incomplete-driver.sh
new file mode 100644
index 0000000..4b5ed07
--- /dev/null
+++ b/ci/testdata/disaster-recovery/incomplete-driver.sh
@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+fixture_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+operation="$1"
+shift
+declare -A args=()
+while (($# > 0)); do
+ args["$1"]="$2"
+ shift 2
+done
+[[ "$operation" == "backup" ]]
+jq -cn \
+ --arg operation_id "${args[--operation-id]}" \
+ --arg manifest_sha256 "${args[--manifest-sha256]}" \
+ -f "$fixture_dir/incomplete-backup-receipt.jq" >"${args[--receipt]}"
diff --git a/ci/testdata/disaster-recovery/profile.sh b/ci/testdata/disaster-recovery/profile.sh
new file mode 100644
index 0000000..2c901e6
--- /dev/null
+++ b/ci/testdata/disaster-recovery/profile.sh
@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+export PATH="${TEST_BIN:?}:/usr/bin:/bin"
+
+acquire_cloud_compose_lifecycle_lock() {
+ printf '%s\n' "$1" >>"${LOCK_LOG:?}"
+}
diff --git a/ci/vault-runtime-contract.sh b/ci/vault-runtime-contract.sh
index 521413c..901d776 100644
--- a/ci/vault-runtime-contract.sh
+++ b/ci/vault-runtime-contract.sh
@@ -179,7 +179,13 @@ cmp -s "$tmp/expected-source-preparation.log" "$source_log" || \
fail "source preparation did not clone every app without executing lifecycle work"
grep -Fq 'ExecStartPre=/bin/bash /home/cloud-compose/assert-vault-ready.sh' \
"$repo_root/rootfs/etc/systemd/system/cloud-compose.service" || fail "app service lacks a Vault readiness gate"
-grep -Fq 'ExecStartPost=/bin/bash /home/cloud-compose/vault-agent-readiness.sh wait' \
- "$repo_root/rootfs/etc/systemd/system/cloud-compose-vault-agent.service" || fail "Vault unit does not publish token readiness"
+vault_unit="$repo_root/rootfs/etc/systemd/system/cloud-compose-vault-agent.service"
+for readiness_command in \
+ 'ExecStartPre=/bin/bash /etc/cloud-compose/libexec/run-root-program.sh vault-agent-readiness.sh prepare' \
+ 'ExecStartPost=/bin/bash /etc/cloud-compose/libexec/run-root-program.sh vault-agent-readiness.sh wait' \
+ 'ExecStopPost=/bin/bash /etc/cloud-compose/libexec/run-root-program.sh vault-agent-readiness.sh clear'; do
+ grep -Fq "$readiness_command" "$vault_unit" || \
+ fail "Vault unit does not route readiness through the trusted root launcher: $readiness_command"
+done
echo "Vault runtime contract passed"
diff --git a/docs/disaster-recovery.md b/docs/disaster-recovery.md
new file mode 100644
index 0000000..618201e
--- /dev/null
+++ b/docs/disaster-recovery.md
@@ -0,0 +1,183 @@
+# Disaster recovery driver
+
+Cloud Compose keeps a same-disk logical MariaDB dump for each application and
+can hand a complete recovery manifest to an operator-owned off-host backup
+driver. The interface is provider-neutral: cloud-compose does not choose an
+object store, account, encryption service, retention policy, or credential
+mechanism.
+
+Local dumps and provider boot-disk snapshots are recovery aids, not disaster
+recovery. A site has DR coverage only after the nightly unit publishes a valid
+receipt proving encrypted off-host coverage of every database, application-file
+root, bind mount, named volume, and service-mount topology in the manifest.
+
+## Enable the contract
+
+Install the reviewed driver and all of its configuration out of band. The file
+and every directory in its path must be root-owned, must not be a symlink or be
+group/world writable, and the executable must have exactly one hard link.
+`/etc/cloud-compose/libexec/offhost-backup-driver` is the portable default on
+COS and conventional Linux hosts. An explicit safe absolute path remains
+supported when an operator manages the driver elsewhere. Because COS rebuilds
+`/etc` at boot, provide the driver through the configured rootfs overlay or an
+equivalent startup provisioner there; a one-time manual copy will not survive a
+reboot. Keep credentials outside that overlay.
+Then enable the provider-neutral runtime input:
+
+```hcl
+runtime = {
+ disaster_recovery = {
+ required = true
+ driver_path = "/etc/cloud-compose/libexec/offhost-backup-driver"
+ }
+}
+```
+
+Terraform, cloud-init, `.env`, plans, state, and application lifecycle commands
+must not contain a storage endpoint, bucket credential, encryption key, or
+access token. The driver owns those details. A root-only configuration file,
+host workload identity, or an operator-managed Vault integration are suitable
+implementation choices. Do not use `runtime.extra_env` for driver credentials.
+
+Ansible and Salt accept the same nested `runtime.disaster_recovery` object.
+Changing `required` to `true` before the driver is installed intentionally
+makes the next nightly handoff and weekly restore test fail.
+
+## Backup invocation
+
+The nightly `cloud-compose-mariadb-backup.timer` starts
+`cloud-compose-offhost-backup.service`. Systemd first runs the existing
+unprivileged MariaDB dump service. The root service then creates a deterministic
+manifest and invokes:
+
+```text
+DRIVER backup \
+ --manifest PATH \
+ --manifest-sha256 SHA256 \
+ --operation-id YYYYMMDD-SITE \
+ --receipt PATH
+```
+
+The driver receives a clean environment containing only `HOME=/root` and a
+fixed system `PATH`. Its stdout and stderr are suppressed so an accidental SDK
+or credential diagnostic cannot enter the system journal. The driver must load
+its own operator-managed configuration and write its receipt to the requested
+path. It must not modify the manifest.
+
+For every app, the manifest includes:
+
+- a root-only staged copy of the validated `sql.gz` logical dump, including its
+ SHA-256 and byte count;
+- the application checkout root and persistent bind-mount sources;
+- every declared named volume; and
+- every resolved Compose service mount, including target, type, source, and
+ read-only state.
+
+Cloud Compose extracts only this topology from `docker compose config`; the
+rendered Compose model, which can contain application environment values, stays
+in a mode-0600 staging directory and is deleted. Persistent bind mounts outside
+`/mnt/disks/data` and `/mnt/disks/volumes` fail closed instead of extending the
+privileged backup boundary.
+
+The driver must finish encrypting and durably transferring every referenced
+component before it writes this exact receipt shape:
+
+```json
+{
+ "schema_version": 1,
+ "kind": "cloud-compose.offhost-backup-receipt",
+ "operation_id": "20260807-example-site",
+ "completed_at": "2026-08-07T12:00:00Z",
+ "manifest_sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
+ "encrypted": true,
+ "off_host": true,
+ "status": "succeeded",
+ "remote_id": "operator-safe-opaque-reference",
+ "coverage": {
+ "database": true,
+ "application_files": true,
+ "volume_topology": true
+ }
+}
+```
+
+`remote_id` is deliberately constrained to a short opaque identifier; it must
+not contain a signed URL, token, query string, or credential. Unknown fields,
+missing coverage, false encryption/off-host claims, a mismatched operation or
+manifest digest, unsafe ownership, links, oversized JSON, and malformed values
+are rejected. The validated manifest and receipt are atomically published under
+`/mnt/disks/data/.cloud-compose-disaster-recovery/`. Driver failure leaves the
+last valid receipt untouched.
+
+The off-host handoff runs even when the day's valid local dump already exists.
+That is what lets the timer retry an earlier transfer failure without rewriting
+the database artifact. The local dump remains subject to its independent
+14-day retention policy and must never be reported as DR coverage.
+
+## Scheduled restore proof
+
+When DR is required, bootstrap enables `cloud-compose-restore-test.timer`. It
+runs weekly on Sunday with a stable randomized delay of up to six hours. The
+service selects the newest validated backup receipt, creates a cryptographically
+random one-time test ID, and invokes:
+
+```text
+DRIVER restore-test \
+ --manifest PATH \
+ --backup-receipt PATH \
+ --source-manifest-sha256 SHA256 \
+ --source-receipt-sha256 SHA256 \
+ --test-id ONE_TIME_ID \
+ --proof PATH
+```
+
+The driver must restore from off-host encrypted storage into a disposable
+recovery environment, verify the database plus representative application-file
+and volume data, and destroy that environment. Only then may it emit:
+
+```json
+{
+ "schema_version": 1,
+ "kind": "cloud-compose.restore-test-proof",
+ "test_id": "20260807T130000Z-random-challenge",
+ "completed_at": "2026-08-07T13:00:00Z",
+ "source_manifest_sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
+ "source_receipt_sha256": "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789",
+ "source_encrypted": true,
+ "status": "succeeded",
+ "recovery_id": "operator-safe-opaque-reference",
+ "disposable_recovery": true,
+ "recovery_destroyed": true,
+ "integrity_verified": true,
+ "coverage": {
+ "database": true,
+ "application_files": true,
+ "volume_topology": true
+ }
+}
+```
+
+The one-time ID and both source digests prevent a stale proof from satisfying a
+new run. Proofs use the same root-owned, bounded, exact-schema, atomic
+publication rules as backup receipts.
+
+## Operator checks
+
+Treat either unit failure as loss of the managed recovery claim and alert on it:
+
+```bash
+systemctl status cloud-compose-mariadb-backup.service \
+ cloud-compose-offhost-backup.service \
+ cloud-compose-restore-test.service
+journalctl -u cloud-compose-offhost-backup.service \
+ -u cloud-compose-restore-test.service
+```
+
+The journal intentionally contains only cloud-compose's generic status and
+validation errors. Driver-specific diagnostics must go to an operator-owned
+root-only sink that applies its own secret redaction.
+
+Before approving a destructive Terraform plan, verify that the newest backup
+receipt covers the current topology and that a recent restore proof exists.
+Provider snapshots can shorten recovery time, but they do not replace this
+independent encrypted copy and disposable restore evidence.
diff --git a/docs/examples.md b/docs/examples.md
index e128c5c..dbad7ea 100644
--- a/docs/examples.md
+++ b/docs/examples.md
@@ -129,12 +129,12 @@ healthcheck settings:
runtime = {
sitectl = {
package_versions = {
- sitectl = "v1.0.0"
- sitectl-wp = "v1.0.0"
+ sitectl = "v1.8.2"
+ sitectl-wp = "v2.0.0"
}
}
compose = {
- branch = "v1.0.0"
+ branch = "v1.1.0"
ingress = {
letsencrypt = true
bot_mitigation = true
@@ -175,9 +175,9 @@ runtime = {
}
sitectl = {
package_versions = {
- sitectl = "v1.0.0"
- sitectl-wp = "v1.0.0"
- sitectl-drupal = "v1.0.0"
+ sitectl = "v1.8.2"
+ sitectl-wp = "v2.0.0"
+ sitectl-drupal = "v1.3.0"
}
}
}
diff --git a/docs/index.md b/docs/index.md
index 58bafa5..75f5439 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -21,7 +21,8 @@ the same lifecycle path used by later rollouts.
- Optional GCP power management through Cloud Run and lightsout
- Provider-neutral runtime contracts for DigitalOcean and Linode
- Existing-host deployment through Ansible or Salt
-- Nightly MariaDB backups through systemd timers
+- Nightly local MariaDB recovery dumps plus an optional provider-neutral,
+ encrypted off-host DR driver and scheduled restore proofs
## Who owns what
@@ -32,7 +33,7 @@ the same lifecycle path used by later rollouts.
| App source revision on an existing VM | Authenticated rollout endpoint or operator-run `/home/cloud-compose/rollout` | `sitectl deploy` against an explicit ref and manifest app key |
| App Compose behavior and health verification | sitectl plugin/component definitions | Versioned plugin release and normal lifecycle commands |
| Secrets and private forge credentials | Vault/operator secret delivery | Short-lived files rendered outside Terraform state |
-| Logical backup retention and off-host disaster recovery | cloud-compose timer plus operator-owned storage policy | Local dumps are pruned after 14 days; independent copies and restore tests remain an operator responsibility |
+| Local logical backup and off-host disaster recovery | cloud-compose timers plus operator-owned DR driver | Local dumps are pruned after 14 days; a strict receipt proves encrypted independent coverage and a scheduled disposable restore |
Any cloud-init byte can change the GCP boot-disk identity and replace the VM;
cloud-init is bootstrap configuration, not the day-2 app update channel. Keep
@@ -41,6 +42,8 @@ routine source deployments in rollout and application behavior in sitectl.
## Start here
- [Runtime contracts](runtime-contracts.md) explains the VM/app contract.
+- [Disaster recovery](disaster-recovery.md) defines the off-host driver,
+ receipts, restore proofs, and operator checks.
- [GCP foundation and application states](runtime-contracts.md#gcp-foundation-and-application-states) explains singleton API/IAM ownership, Shared VPC, and Direct VPC egress.
- [Managed runtime](managed-runtime.md) covers host tools and internal services.
- [Rollout API](rollout.md) covers authenticated deploy triggers.
diff --git a/docs/managed-runtime.md b/docs/managed-runtime.md
index 630426f..bc10f56 100644
--- a/docs/managed-runtime.md
+++ b/docs/managed-runtime.md
@@ -26,11 +26,11 @@ core and plugins independently. The provider-neutral
backward-compatible fallback only for an installed package with neither a
template selector nor an explicit per-package selector.
-All presets select coordinated sitectl v1 release packages. The ISLE preset
-uses the `libops/isle` v1.1.0 template with sitectl, sitectl-drupal, and
-sitectl-isle v1.0.0. Every other application preset remains on its v1.0.0
-template contract. Override the template or package selectors only as one
-reviewed, compatible release set.
+All presets pin sitectl v1.8.2 with a reviewed application plugin. ISLE uses
+the `libops/isle` v1.3.0 template with sitectl-drupal v1.3.0 and sitectl-isle
+v1.5.0. Drupal and WordPress use their v1.1.0 templates; the other application
+templates remain on v1.0.0. Override the template or package selectors only as
+one reviewed, compatible release set.
Omitting `runtime.sitectl.packages` selects the template's package set. An
explicit list replaces that set; `packages = []` or `packages = ["sitectl"]`
@@ -51,9 +51,9 @@ runtime = {
"sitectl-isle",
]
package_versions = {
- sitectl = "v1.0.0"
- sitectl-drupal = "v1.0.0"
- sitectl-isle = "v1.0.0"
+ sitectl = "v1.8.2"
+ sitectl-drupal = "v1.3.0"
+ sitectl-isle = "v1.5.0"
}
plugin = "isle"
}
diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml
index f879f50..f59c73b 100644
--- a/docs/mkdocs.yml
+++ b/docs/mkdocs.yml
@@ -63,6 +63,7 @@ extra_javascript:
nav:
- Home: index.md
- Runtime contracts: runtime-contracts.md
+ - Disaster recovery: disaster-recovery.md
- Managed runtime: managed-runtime.md
- Rollout API: rollout.md
- DigitalOcean and Linode: non-gcp-providers.md
diff --git a/docs/non-gcp-providers.md b/docs/non-gcp-providers.md
index 24321af..f22bce6 100644
--- a/docs/non-gcp-providers.md
+++ b/docs/non-gcp-providers.md
@@ -102,13 +102,14 @@ nightly MariaDB dumps under `/mnt/disks/data/backups` are useful for logical
recovery, but they live on the same data volume and are not disaster recovery.
Also, `terraform destroy` intentionally deletes both managed volumes.
-Before production, establish an independently owned offsite policy for both
-volumes (provider volume snapshots where available, or encrypted export to
-separate object storage/account), define retention separately from this
-application state, and test a restore into disposable new volumes. A restore is
-complete only after the Compose projects start, `sitectl healthcheck` passes,
-and representative files plus database records are verified. Do not enable a
-boot-disk backup toggle and record the application as protected.
+Before production, install an operator-owned driver and set
+`runtime.disaster_recovery.required = true`. The provider-neutral handoff
+requires encrypted off-host coverage for the logical database, application
+files, and complete Compose volume topology, then schedules restore tests into
+a disposable recovery environment. The exact driver and proof contract is in
+[Disaster recovery](disaster-recovery.md). Retention and credentials remain
+operator-owned. Do not enable a boot-disk backup toggle and record the
+application as protected.
Fedora CoreOS should use the CoreOS installer path. Debian and Ubuntu should use
the apt installer path. Both paths install the same minimum runtime surface:
diff --git a/docs/runtime-contracts.md b/docs/runtime-contracts.md
index a94d779..1febd43 100644
--- a/docs/runtime-contracts.md
+++ b/docs/runtime-contracts.md
@@ -156,6 +156,13 @@ with `systemctl status cloud-compose-bootstrap cloud-compose`. Bootstrap output
uses a fixed `info` priority and has no unit-specific Fluent Bit input, so raw
bootstrap output is not forwarded to Cloud Logging; systemd's own service
failures remain available to the existing warning-level collector.
+Root systemd jobs enter through `/etc/cloud-compose/libexec`, validate the
+root-owned home scripts and control inputs, and only then execute their
+allowlisted `/home/cloud-compose` program. The privileged entrypoints,
+diagnostics command, and checked-in `jq` programs live below
+`/etc/cloud-compose/{libexec,bin,jq}` because COS permits cloud-init to rebuild
+that stateless tree while its `/usr` filesystem is immutable. Application
+services retain their unprivileged execution model.
## Sitectl
@@ -176,6 +183,12 @@ package list before validation. Terraform, Ansible, and Salt serialize their
resolved map as `SITECTL_PACKAGE_VERSIONS` JSON and the privileged installer
validates it again before downloading anything. Per-project package versions
are not supported because projects on one host share the same binaries.
+`/home/cloud-compose/bin` is reserved for generated managed-tool symlinks. They
+target the root-owned managed binary directory and normally comprise `sitectl`,
+its plugins, and the verified static GNU Make build required by Container-
+Optimized OS. During an upgrade from the former application-owned directory,
+the installer closes the directory to root and rejects any other inherited
+command or target rather than carrying an untrusted PATH entry forward.
`sitectl_verify_args` remains a real argument list. The host stores it as JSON
and appends each value through an argv-aware wrapper when a lifecycle command
@@ -328,7 +341,9 @@ release versions and the Alpine tag/digest pair; review those changes as
supply-chain updates rather than accepting an unpinned replacement. The image
and its network-fetched package build scripts execute only after the metadata
firewall is installed and use the bridge network, so they cannot inherit the
-host network's root exemption.
+host network's root exemption. The build itself is a checked-in shell program
+mounted read-only into the container and invoked by path, keeping the audited
+program out of cloud-init and Docker command arguments.
The GCP COS VM image name is a reviewed manual pin. Renovate has no built-in
GCP Compute image-family datasource, and the shared LibOps preset does not add
@@ -654,16 +669,19 @@ used, it must be an HTTPS URL without whitespace and
`runtime.rootfs_archive_sha256` is mandatory with a 64-character SHA-256
digest. Boot restricts curl and redirects to HTTPS with TLS 1.2 or newer,
downloads to a temporary path, and verifies the complete file before extracting
-its `rootfs` directory. The GCP path stages
-the caller's packaged rootfs overlay and reapplies it after the archive, so
-consumer overrides still win. Use an immutable archive URL; a moving branch and
-a pinned checksum intentionally fail as soon as the branch content changes.
+its `rootfs` directory. Both archive installers normalize the verified tree to
+UID/GID `0:0` before loading a helper or copying it onto the host, so Git forge
+source-archive ownership cannot leak into the privileged runtime. The GCP path
+stages the caller's packaged rootfs overlay and reapplies it after the archive,
+so consumer overrides still win. Use an immutable archive URL; a moving branch
+and a pinned checksum intentionally fail as soon as the branch content changes.
## Backups
-`cloud-compose-mariadb-backup.timer` runs nightly between 9pm and 7am EST. It
-uses a fixed randomized delay so deployments spread out across that window while
-keeping a stable schedule on each VM. The timer executes:
+`cloud-compose-mariadb-backup.timer` runs the local-backup and off-host handoff
+flow nightly between 9pm and 7am EST. It uses a fixed randomized delay so
+deployments spread out across that window while keeping a stable schedule on
+each VM. The unprivileged local phase executes:
```bash
sitectl mariadb backup --context "$SITECTL_CONTEXT_NAME" --gzip --output "$path"
@@ -684,13 +702,24 @@ service exits non-zero after attempting all of them. Dumps older than
`MARIADB_BACKUP_RETENTION_DAYS` (14 by default) are pruned from each validated
app directory so they cannot fill the shared data disk indefinitely.
-Local dumps remain on the same failure-domain disk as application data.
-Downstream operators must still define reviewed encrypted off-host retention
-and restore tests. GCP production enables crash-consistent scheduled disk
-snapshots by default; `guest_flush = false` is deliberate because the logical
-dump supplies the application-consistent recovery artifact. DigitalOcean and
-Linode boot-disk backup toggles do not include attached volumes; see the
-provider guide before claiming disaster-recovery coverage.
+Local dumps remain on the same failure-domain disk as application data and are
+never disaster recovery. Set `runtime.disaster_recovery.required = true` only
+after installing the operator-owned root driver. The root handoff runs after
+the local service, including when the daily dump already exists, and requires a
+strict atomic receipt proving encrypted off-host coverage of each app's logical
+database, checkout/bind files, named volumes, and resolved service-mount
+topology. A weekly timer requires a challenge-bound proof that the driver
+restored all three coverage classes into a disposable recovery environment,
+verified integrity, and destroyed that environment. Storage-vendor settings
+and credentials stay behind the driver and never enter Terraform, cloud-init,
+the host environment, or logs. The complete interface and receipt schemas are
+in [Disaster recovery](disaster-recovery.md).
+
+GCP production enables crash-consistent scheduled disk snapshots by default;
+`guest_flush = false` is deliberate because the logical dump supplies the
+application-consistent database artifact. DigitalOcean and Linode boot-disk
+backup toggles do not include attached volumes. None of those provider-local
+copies replace the independent driver receipt and restore proof.
Terraform owns the attached data and Docker-volume disks. A normal
`terraform destroy` deletes them; GCP production snapshots are retained, but
@@ -839,10 +868,10 @@ deploy the same
`5058610fddc7267ace92d65a5c49713dce570ac3`; an early exact checkout bridges
the legacy runtime's branch-only clone behavior without following a moving
branch. Its `gcp.cloud_init.initcmd` disables both generations of the
-internal-service timer after cloud-init writes the units but before
-`/home/cloud-compose/run.sh` starts the potentially long bootstrap, so the
-disposable VM cannot suspend itself. The runner checks the units again after
-each boot.
+internal-service timer after cloud-init writes the units but before the
+root-owned `/etc/cloud-compose/libexec/run-bootstrap.sh` entrypoint
+starts the potentially long bootstrap, so the disposable VM cannot suspend
+itself. The runner checks the units again after each boot.
Only the ephemeral runner key is authorized, and SSH is limited to that
runner's public IPv4 `/32`.
diff --git a/examples/binpack/main.tf b/examples/binpack/main.tf
index 98f06b4..eefa061 100644
--- a/examples/binpack/main.tf
+++ b/examples/binpack/main.tf
@@ -29,9 +29,9 @@ module "apps" {
}
sitectl = {
package_versions = {
- sitectl = "v1.0.0"
- sitectl-wp = "v1.0.0"
- sitectl-drupal = "v1.0.0"
+ sitectl = "v1.8.2"
+ sitectl-wp = "v2.0.0"
+ sitectl-drupal = "v1.3.0"
}
}
}
diff --git a/internal/contracttest/cloud_smoke_cleanup_test.go b/internal/contracttest/cloud_smoke_cleanup_test.go
index 2c2ec3b..451e643 100644
--- a/internal/contracttest/cloud_smoke_cleanup_test.go
+++ b/internal/contracttest/cloud_smoke_cleanup_test.go
@@ -413,6 +413,7 @@ func TestNonGCPSmokeRunExitCleanupLifecycle(t *testing.T) {
command.Env = overriddenEnvironment(map[string]string{
"CLOUD_COMPOSE_CI_BIN": filepath.Join(binDirectory, "cloud-compose-ci"),
"CLOUD_COMPOSE_SMOKE_AUTO_APPROVE": "true",
+ "CLOUD_COMPOSE_SMOKE_BOOT_TIMEOUT": "5",
"CLOUD_COMPOSE_SMOKE_DESTROY_TIMEOUT": "10",
"CLOUD_COMPOSE_SMOKE_KEEP": "false",
"CLOUD_COMPOSE_SMOKE_RUN_ID": "123456789",
@@ -466,6 +467,13 @@ func TestNonGCPSmokeRunExitCleanupLifecycle(t *testing.T) {
func writeCloudSmokeLifecycleFakes(t testing.TB, directory string) {
t.Helper()
+ sshFixture, err := os.ReadFile(filepath.Join(
+ repositoryRoot(t),
+ "internal/contracttest/testdata/cloud-smoke-lifecycle/ssh.sh",
+ ))
+ if err != nil {
+ t.Fatalf("read lifecycle SSH fixture: %v", err)
+ }
executables := map[string]string{
"cloud-compose-ci": `#!/usr/bin/env bash
set -euo pipefail
@@ -534,13 +542,7 @@ printf 'ssh-ed25519 fake-public-key cloud-compose-smoke\n' >"${path}.pub"
set -euo pipefail
printf '127.0.0.1 ssh-ed25519 fake-host-key\n'
`,
- "ssh": `#!/usr/bin/env bash
-set -euo pipefail
-case "$*" in
- *cloud-compose-bootstrap-complete*) printf 'complete\n' ;;
- *cloud-init\ status*) printf 'cloud-init not installed\n' ;;
-esac
-`,
+ "ssh": string(sshFixture),
"sitectl": `#!/usr/bin/env bash
set -euo pipefail
exit 0
diff --git a/internal/contracttest/disaster_recovery_test.go b/internal/contracttest/disaster_recovery_test.go
new file mode 100644
index 0000000..61d6931
--- /dev/null
+++ b/internal/contracttest/disaster_recovery_test.go
@@ -0,0 +1,105 @@
+package contracttest
+
+import (
+ "strings"
+ "testing"
+)
+
+func TestDisasterRecoveryInputParity(t *testing.T) {
+ root := repositoryRoot(t)
+
+ for _, relativePath := range []string{
+ "variables.tf",
+ "providers/gcp/variables.tf",
+ "providers/do/variables.tf",
+ "providers/linode/variables.tf",
+ "modules/digitalocean/variables.tf",
+ "modules/linode/variables.tf",
+ } {
+ content := readRepositoryFile(t, root, relativePath)
+ requireContains(t, content, "disaster_recovery = optional(object({", relativePath+" disaster-recovery object")
+ requireContains(t, content, "required = optional(bool, false)", relativePath+" required switch")
+ requireContains(t, content, `driver_path = optional(string, "/etc/cloud-compose/libexec/offhost-backup-driver")`, relativePath+" driver default")
+ requireContains(t, content, "runtime.disaster_recovery.driver_path must be a safe absolute path", relativePath+" path validation")
+ }
+
+ for _, relativePath := range []string{
+ "main.tf",
+ "providers/gcp/main.tf",
+ "modules/digitalocean/main.tf",
+ "modules/linode/main.tf",
+ } {
+ content := readRepositoryFile(t, root, relativePath)
+ requireContains(t, content, "offhost_backup_required", relativePath+" required forwarding")
+ requireContains(t, content, "offhost_backup_driver_path", relativePath+" driver forwarding")
+ }
+
+ for _, relativePath := range []string{
+ "modules/gcp/main.tf",
+ "modules/linux-vm-runtime/main.tf",
+ "ansible/roles/cloud_compose/tasks/main.yml",
+ "salt/cloud-compose/init.sls",
+ } {
+ content := readRepositoryFile(t, root, relativePath)
+ requireContains(t, content, "CLOUD_COMPOSE_OFFHOST_BACKUP_REQUIRED", relativePath+" required host control")
+ requireContains(t, content, "CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER", relativePath+" driver host control")
+ }
+}
+
+func TestDisasterRecoveryReceiptAndRestoreContracts(t *testing.T) {
+ root := repositoryRoot(t)
+ library := readRepositoryFile(t, root, "rootfs/home/cloud-compose/disaster-recovery-lib.sh")
+ backup := readRepositoryFile(t, root, "rootfs/home/cloud-compose/offhost-backup.sh")
+ restore := readRepositoryFile(t, root, "rootfs/home/cloud-compose/restore-test.sh")
+ validationContract := strings.Join([]string{
+ library,
+ readRepositoryFile(t, root, "rootfs/etc/cloud-compose/jq/dr-validate-backup-receipt.jq"),
+ readRepositoryFile(t, root, "rootfs/etc/cloud-compose/jq/dr-validate-restore-proof.jq"),
+ }, "\n")
+ backupContract := strings.Join([]string{
+ backup,
+ readRepositoryFile(t, root, "rootfs/etc/cloud-compose/jq/offhost-build-application-coverage.jq"),
+ }, "\n")
+
+ for marker, label := range map[string]string{
+ `env -i HOME=/root`: "clean driver environment",
+ `>/dev/null 2>&1`: "suppressed driver output",
+ `cloud-compose.offhost-backup-receipt`: "backup receipt kind",
+ `.encrypted == true`: "encrypted coverage",
+ `.off_host == true`: "off-host coverage",
+ `.database == true`: "database coverage",
+ `.application_files == true`: "application-file coverage",
+ `.volume_topology == true`: "volume-topology coverage",
+ `cloud-compose.restore-test-proof`: "restore proof kind",
+ `.disposable_recovery == true`: "disposable recovery proof",
+ `.recovery_destroyed == true`: "recovery cleanup proof",
+ `.source_receipt_sha256 == $receipt_sha256`: "source receipt binding",
+ } {
+ requireContains(t, validationContract, marker, label)
+ }
+
+ for marker, label := range map[string]string{
+ `docker compose config --format json`: "resolved Compose topology",
+ `local_recovery_artifact`: "logical database artifact",
+ `application_files`: "application files",
+ `volume_topology`: "volume topology",
+ `cloud_compose_dr_run_driver`: "provider-neutral backup handoff",
+ } {
+ requireContains(t, backupContract, marker, label)
+ }
+
+ for marker, label := range map[string]string{
+ `/dev/urandom`: "one-time restore challenge",
+ `--backup-receipt`: "remote receipt input",
+ `cloud_compose_dr_validate_restore_proof`: "strict restore proof validation",
+ `mv -- "$staged_proof" "$proof_path"`: "atomic restore proof publication",
+ } {
+ requireContains(t, restore, marker, label)
+ }
+
+ for _, forbidden := range []string{"AWS_ACCESS_KEY", "GOOGLE_APPLICATION_CREDENTIALS", "AZURE_STORAGE_KEY"} {
+ if strings.Contains(library+backup+restore, forbidden) {
+ t.Errorf("disaster-recovery runtime hard-codes credential channel %q", forbidden)
+ }
+ }
+}
diff --git a/internal/contracttest/template_version_test.go b/internal/contracttest/template_version_test.go
index ba1b401..1b531bb 100644
--- a/internal/contracttest/template_version_test.go
+++ b/internal/contracttest/template_version_test.go
@@ -30,38 +30,47 @@ func TestTemplateVersionContract(t *testing.T) {
expectedVersions := map[string]map[string]string{
"default": {
- "sitectl": "v1.0.0",
+ "sitectl": "v1.8.2",
},
"archivesspace": {
- "sitectl": "v1.0.0",
- "sitectl-archivesspace": "v1.0.0",
+ "sitectl": "v1.8.2",
+ "sitectl-archivesspace": "v2.0.0",
},
"drupal": {
- "sitectl": "v1.0.0",
- "sitectl-drupal": "v1.0.0",
+ "sitectl": "v1.8.2",
+ "sitectl-drupal": "v1.3.0",
},
"isle": {
- "sitectl": "v1.0.0",
- "sitectl-drupal": "v1.0.0",
- "sitectl-isle": "v1.0.0",
+ "sitectl": "v1.8.2",
+ "sitectl-drupal": "v1.3.0",
+ "sitectl-isle": "v1.5.0",
},
"ojs": {
- "sitectl": "v1.0.0",
- "sitectl-ojs": "v1.0.0",
+ "sitectl": "v1.8.2",
+ "sitectl-ojs": "v1.2.0",
},
"omeka-classic": {
- "sitectl": "v1.0.0",
- "sitectl-omeka-classic": "v1.0.0",
+ "sitectl": "v1.8.2",
+ "sitectl-omeka-classic": "v1.2.0",
},
"omeka-s": {
- "sitectl": "v1.0.0",
- "sitectl-omeka-s": "v1.0.0",
+ "sitectl": "v1.8.2",
+ "sitectl-omeka-s": "v1.2.0",
},
"wp": {
- "sitectl": "v1.0.0",
- "sitectl-wp": "v1.0.0",
+ "sitectl": "v1.8.2",
+ "sitectl-wp": "v2.0.0",
},
}
+ expectedBranches := map[string]string{
+ "archivesspace": "v1.0.0",
+ "drupal": "v1.1.0",
+ "isle": "v1.3.0",
+ "ojs": "v1.0.0",
+ "omeka-classic": "v1.0.0",
+ "omeka-s": "v1.0.0",
+ "wp": "v1.1.0",
+ }
if !maps.Equal(registry.Default.PackageVersions, expectedVersions["default"]) {
t.Errorf("template %q package versions diverged:\nexpected %s\nactual %s", "default", prettyJSON(t, expectedVersions["default"]), prettyJSON(t, registry.Default.PackageVersions))
@@ -80,10 +89,7 @@ func TestTemplateVersionContract(t *testing.T) {
if !maps.Equal(definition.PackageVersions, expected) {
t.Errorf("template %q package versions diverged:\nexpected %s\nactual %s", name, prettyJSON(t, expected), prettyJSON(t, definition.PackageVersions))
}
- expectedBranch := "v1.0.0"
- if name == "isle" {
- expectedBranch = "v1.1.0"
- }
+ expectedBranch := expectedBranches[name]
if definition.Branch != expectedBranch {
t.Errorf("template %q branch = %q, want stable contract %s", name, definition.Branch, expectedBranch)
}
@@ -140,8 +146,8 @@ func TestTemplateVersionContract(t *testing.T) {
for _, packageName := range []string{"sitectl", "sitectl-wp", "sitectl-drupal"} {
requireContains(t, content, packageName, relativePath+" bin-pack package")
}
- requireContains(t, content, `sitectl = "v1.0.0"`, relativePath+" bin-pack core selector")
- requireContains(t, content, `sitectl-wp = "v1.0.0"`, relativePath+" bin-pack WordPress selector")
- requireContains(t, content, `sitectl-drupal = "v1.0.0"`, relativePath+" bin-pack Drupal selector")
+ requireContains(t, content, `sitectl = "v1.8.2"`, relativePath+" bin-pack core selector")
+ requireContains(t, content, `sitectl-wp = "v2.0.0"`, relativePath+" bin-pack WordPress selector")
+ requireContains(t, content, `sitectl-drupal = "v1.3.0"`, relativePath+" bin-pack Drupal selector")
}
}
diff --git a/internal/contracttest/testdata/cloud-smoke-lifecycle/ssh.sh b/internal/contracttest/testdata/cloud-smoke-lifecycle/ssh.sh
new file mode 100644
index 0000000..be68e10
--- /dev/null
+++ b/internal/contracttest/testdata/cloud-smoke-lifecycle/ssh.sh
@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+case "$*" in
+ *cloud-compose-diagnostics.sh\ state*) printf 'complete\n' ;;
+ *cloud-compose-bootstrap-complete*) printf 'complete\n' ;;
+ *cloud-init\ status*) printf 'cloud-init not installed\n' ;;
+esac
diff --git a/main.tf b/main.tf
index 49cfea3..d11f765 100644
--- a/main.tf
+++ b/main.tf
@@ -161,13 +161,15 @@ module "gcp" {
overlay_source_instance = local.gcp_overlay.source_instance
volume_names = local.gcp_overlay.volume_names
- users = local.runtime.users
- rootfs = local.runtime.rootfs
- rootfs_archive_url = local.runtime.rootfs_archive_url
- rootfs_archive_sha256 = local.runtime.rootfs_archive_sha256
- extra_env = local.runtime.extra_env
- runcmd = local.gcp_cloud_init.runcmd
- initcmd = local.gcp_cloud_init.initcmd
+ users = local.runtime.users
+ rootfs = local.runtime.rootfs
+ rootfs_archive_url = local.runtime.rootfs_archive_url
+ rootfs_archive_sha256 = local.runtime.rootfs_archive_sha256
+ offhost_backup_required = local.runtime.disaster_recovery.required
+ offhost_backup_driver_path = local.runtime.disaster_recovery.driver_path
+ extra_env = local.runtime.extra_env
+ runcmd = local.gcp_cloud_init.runcmd
+ initcmd = local.gcp_cloud_init.initcmd
artifact_registry_repository = local.gcp_artifact_registry.repository
artifact_registry_location = local.gcp_artifact_registry.location
diff --git a/modules/digitalocean/main.tf b/modules/digitalocean/main.tf
index 8cd72b9..bf78b55 100644
--- a/modules/digitalocean/main.tf
+++ b/modules/digitalocean/main.tf
@@ -13,33 +13,35 @@ locals {
module "runtime" {
source = "../linux-vm-runtime"
- name = var.name
- provider_name = "digitalocean"
- region = local.do.region
- data_device = "/dev/disk/by-id/scsi-0DO_Volume_${local.data_volume_name}"
- volumes_device = "/dev/disk/by-id/scsi-0DO_Volume_${local.docker_volumes_volume_name}"
- ssh_users = merge(local.runtime.users, local.do.ssh.users)
- cloud_compose_ssh_keys = local.do.ssh.cloud_compose_keys
- rootfs = local.runtime.rootfs
- rootfs_archive_url = local.runtime.rootfs_archive_url
- rootfs_archive_sha256 = local.runtime.rootfs_archive_sha256
- ingress_port = local.compose.ingress_port
- primary_compose_project = local.compose.primary
- sitectl_ingress = local.compose.ingress
- docker_compose_repo = local.compose.repo
- docker_compose_branch = local.compose.branch
- compose_projects = local.compose.projects
- docker_compose_init = local.compose.init
- docker_compose_up = local.compose.up
- docker_compose_down = local.compose.down
- docker_compose_rollout = local.compose.rollout
- rollout_enabled = local.do.rollout.enabled
- rollout_release_url = local.do.rollout.release_url
- rollout_release_sha256 = local.do.rollout.release_sha256
- rollout_port = local.do.rollout.port
- rollout_jwks_uri = local.do.rollout.jwks_uri
- rollout_jwt_audience = local.do.rollout.jwt_audience
- rollout_custom_claims = local.do.rollout.custom_claims
+ name = var.name
+ provider_name = "digitalocean"
+ region = local.do.region
+ data_device = "/dev/disk/by-id/scsi-0DO_Volume_${local.data_volume_name}"
+ volumes_device = "/dev/disk/by-id/scsi-0DO_Volume_${local.docker_volumes_volume_name}"
+ ssh_users = merge(local.runtime.users, local.do.ssh.users)
+ cloud_compose_ssh_keys = local.do.ssh.cloud_compose_keys
+ rootfs = local.runtime.rootfs
+ rootfs_archive_url = local.runtime.rootfs_archive_url
+ rootfs_archive_sha256 = local.runtime.rootfs_archive_sha256
+ offhost_backup_required = local.runtime.disaster_recovery.required
+ offhost_backup_driver_path = local.runtime.disaster_recovery.driver_path
+ ingress_port = local.compose.ingress_port
+ primary_compose_project = local.compose.primary
+ sitectl_ingress = local.compose.ingress
+ docker_compose_repo = local.compose.repo
+ docker_compose_branch = local.compose.branch
+ compose_projects = local.compose.projects
+ docker_compose_init = local.compose.init
+ docker_compose_up = local.compose.up
+ docker_compose_down = local.compose.down
+ docker_compose_rollout = local.compose.rollout
+ rollout_enabled = local.do.rollout.enabled
+ rollout_release_url = local.do.rollout.release_url
+ rollout_release_sha256 = local.do.rollout.release_sha256
+ rollout_port = local.do.rollout.port
+ rollout_jwks_uri = local.do.rollout.jwks_uri
+ rollout_jwt_audience = local.do.rollout.jwt_audience
+ rollout_custom_claims = local.do.rollout.custom_claims
sitectl_packages = local.sitectl.packages
sitectl_version = local.sitectl.version
diff --git a/modules/digitalocean/variables.tf b/modules/digitalocean/variables.tf
index 9f65b80..26a6104 100644
--- a/modules/digitalocean/variables.tf
+++ b/modules/digitalocean/variables.tf
@@ -76,6 +76,11 @@ variable "runtime" {
rootfs_archive_sha256 = optional(string, "")
users = optional(map(list(string)), {})
+ disaster_recovery = optional(object({
+ required = optional(bool, false)
+ driver_path = optional(string, "/etc/cloud-compose/libexec/offhost-backup-driver")
+ }), {})
+
compose = optional(object({
primary = optional(string, "")
ingress_port = optional(number, 80)
@@ -184,6 +189,15 @@ variable "runtime" {
error_message = "runtime.rootfs_archive_url and a 64-character runtime.rootfs_archive_sha256 must be supplied together."
}
+ validation {
+ condition = (
+ can(regex("^/[A-Za-z0-9._/+:-]+$", var.runtime.disaster_recovery.driver_path)) &&
+ !strcontains(var.runtime.disaster_recovery.driver_path, "//") &&
+ length(regexall("(^|/)\\.\\.?(/|$)", var.runtime.disaster_recovery.driver_path)) == 0
+ )
+ error_message = "runtime.disaster_recovery.driver_path must be a safe absolute path without whitespace or dot segments."
+ }
+
validation {
condition = alltrue([
for name in keys(var.runtime.extra_env) :
diff --git a/modules/gcp/main.tf b/modules/gcp/main.tf
index db483be..0833c86 100644
--- a/modules/gcp/main.tf
+++ b/modules/gcp/main.tf
@@ -232,11 +232,16 @@ locals {
}
}
)
+ rootfs_file_permissions = {
+ for file in setunion(local.base_files, local.additional_files) :
+ file => endswith(file, ".sh") || "/${file}" == var.offhost_backup_driver_path ? "0755" : "0644"
+ }
write_files_content = join("\n", [
for file, config in local.all_files : <<-EOT
- path: ${jsonencode(config.destination)}
- permissions: ${jsonencode(endswith(file, ".sh") ? "0755" : "0644")}
+ owner: "root:root"
+ permissions: ${jsonencode(local.rootfs_file_permissions[file])}
encoding: gzip+base64
content: ${jsonencode(base64gzip(file(config.source)))}
EOT
@@ -244,6 +249,7 @@ EOT
docker_compose_scripts = join("\n", [
for name in ["init", "up", "down", "rollout"] : <<-EOT
- path: "/home/cloud-compose/${name}"
+ owner: "root:root"
permissions: "0755"
encoding: gzip+base64
content: ${jsonencode(base64gzip(<<-EOS
@@ -259,6 +265,7 @@ EOT
])
compose_projects_file = <<-EOT
- path: "/home/cloud-compose/compose-projects.json"
+ owner: "root:root"
permissions: "0640"
encoding: gzip+base64
content: ${jsonencode(base64gzip(jsonencode(local.validated_compose_projects)))}
@@ -275,11 +282,13 @@ managed_runtime_artifact_lines = [
try(artifact.restart, ""),
])
]
-managed_runtime_artifacts_file = <<-EOT
+managed_runtime_artifacts_content = length(local.managed_runtime_artifact_lines) == 0 ? "\n" : "${join("\n", local.managed_runtime_artifact_lines)}\n"
+managed_runtime_artifacts_file = <<-EOT
- path: "/home/cloud-compose/managed-runtime-artifacts.tsv"
+ owner: "root:root"
permissions: "0640"
encoding: gzip+base64
- content: ${jsonencode(base64gzip(join("\n", local.managed_runtime_artifact_lines)))}
+ content: ${jsonencode(base64gzip(local.managed_runtime_artifacts_content))}
EOT
vault_agent_template_stanzas = join("\n", [
for template in var.vault_agent_templates : <<-EOT
@@ -362,56 +371,60 @@ fresh_filesystem_env = {
CLOUD_COMPOSE_FRESH_FILESYSTEM_IDENTITY = "v1:gcp-disk-id:${google_compute_disk.data.disk_id}"
}
host_env = merge({
- HOME = "/home/cloud-compose"
- GCP_PROJECT = var.project_id
- GCP_PROJECT_NUMBER = local.project_number
- GCP_INSTANCE_NAME = var.name
- CLOUD_COMPOSE_INSTANCE_NAME = var.name
- GCP_REGION = var.region
- GCP_ZONE = var.zone
- CLOUD_COMPOSE_PROVIDER = "gcp"
- CLOUD_COMPOSE_APPS = join(" ", keys(local.compose_projects))
- CLOUD_COMPOSE_PRIMARY_APP = local.primary_compose_project_key
- COMPOSE_PROJECTS_FILE = "/home/cloud-compose/compose-projects.json"
- COMPOSE_PROJECT_NAME = local.primary_compose_project.compose_project_name
- COMPOSE_BIND_PORT = tostring(local.primary_compose_project.ingress_port)
- DOCKER_COMPOSE_DIR = local.primary_compose_project.project_dir
- DOCKER_COMPOSE_REPO = local.primary_compose_project.docker_compose_repo
- DOCKER_COMPOSE_BRANCH = local.primary_compose_project.docker_compose_branch
- DOCKER_COMPOSE_VERSION = var.docker_compose_version
- DOCKER_BUILDX_VERSION = var.docker_buildx_version
- DOCKER_VOLUME_OVERLAYS = join(" ", var.volume_names)
- SITECTL_PACKAGES = join(" ", module.sitectl_runtime.packages)
- SITECTL_VERSION = var.sitectl_version
- SITECTL_PACKAGE_VERSIONS = jsonencode(module.sitectl_runtime.package_versions)
- SITECTL_CONTEXT_NAME = local.primary_compose_project.sitectl_context_name
- SITECTL_PLUGIN = local.primary_compose_project.sitectl_plugin
- SITECTL_ENVIRONMENT = local.primary_compose_project.sitectl_environment
- PRODUCTION = tostring(var.production)
- SITECTL_VERIFY_ARGS = join(" ", local.primary_compose_project.sitectl_verify_args)
- GCP_APP_SERVICE_ACCOUNT_EMAIL = local.app_service_account_email
- GCP_APP_SERVICE_ACCOUNT_MANAGED = tostring(local.app_service_account_managed)
- GCP_APP_CREDENTIALS_ENABLED = tostring(local.app_credentials_enabled)
- POWER_MANAGEMENT_ENABLED = tostring(var.power_management_enabled)
- COMPOSE_PROFILES = local.internal_services_compose_profiles
- VAULT_ADDR = trimspace(var.vault_addr)
- VAULT_NAMESPACE = trimspace(var.vault_namespace)
- VAULT_ROLE = trimspace(var.vault_role)
- VAULT_AGENT_ENABLED = var.vault_agent_enabled && trimspace(var.vault_addr) != "" ? "true" : "false"
- VAULT_AUTH_METHOD = var.vault_auth_method
- VAULT_AGENT_TOKEN_PATH = var.vault_agent_token_path
- LIBOPS_MANAGED_RUNTIME_ENABLED = tostring(var.libops_managed_runtime_enabled)
- LIBOPS_INTERNAL_SERVICES_ENABLED = tostring(local.internal_services_enabled)
- LIBOPS_INTERNAL_SERVICES_AUTO_UPDATE = tostring(local.internal_services_enabled && var.libops_internal_services_auto_update)
+ HOME = "/home/cloud-compose"
+ GCP_PROJECT = var.project_id
+ GCP_PROJECT_NUMBER = local.project_number
+ GCP_INSTANCE_NAME = var.name
+ CLOUD_COMPOSE_INSTANCE_NAME = var.name
+ GCP_REGION = var.region
+ GCP_ZONE = var.zone
+ CLOUD_COMPOSE_PROVIDER = "gcp"
+ CLOUD_COMPOSE_APPS = join(" ", keys(local.compose_projects))
+ CLOUD_COMPOSE_PRIMARY_APP = local.primary_compose_project_key
+ CLOUD_COMPOSE_OFFHOST_BACKUP_REQUIRED = tostring(var.offhost_backup_required)
+ CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER = var.offhost_backup_driver_path
+ COMPOSE_PROJECTS_FILE = "/home/cloud-compose/compose-projects.json"
+ COMPOSE_PROJECT_NAME = local.primary_compose_project.compose_project_name
+ COMPOSE_BIND_PORT = tostring(local.primary_compose_project.ingress_port)
+ DOCKER_COMPOSE_DIR = local.primary_compose_project.project_dir
+ DOCKER_COMPOSE_REPO = local.primary_compose_project.docker_compose_repo
+ DOCKER_COMPOSE_BRANCH = local.primary_compose_project.docker_compose_branch
+ DOCKER_COMPOSE_VERSION = var.docker_compose_version
+ DOCKER_BUILDX_VERSION = var.docker_buildx_version
+ DOCKER_VOLUME_OVERLAYS = join(" ", var.volume_names)
+ SITECTL_PACKAGES = join(" ", module.sitectl_runtime.packages)
+ SITECTL_VERSION = var.sitectl_version
+ SITECTL_PACKAGE_VERSIONS = jsonencode(module.sitectl_runtime.package_versions)
+ SITECTL_CONTEXT_NAME = local.primary_compose_project.sitectl_context_name
+ SITECTL_PLUGIN = local.primary_compose_project.sitectl_plugin
+ SITECTL_ENVIRONMENT = local.primary_compose_project.sitectl_environment
+ PRODUCTION = tostring(var.production)
+ SITECTL_VERIFY_ARGS = join(" ", local.primary_compose_project.sitectl_verify_args)
+ GCP_APP_SERVICE_ACCOUNT_EMAIL = local.app_service_account_email
+ GCP_APP_SERVICE_ACCOUNT_MANAGED = tostring(local.app_service_account_managed)
+ GCP_APP_CREDENTIALS_ENABLED = tostring(local.app_credentials_enabled)
+ POWER_MANAGEMENT_ENABLED = tostring(var.power_management_enabled)
+ COMPOSE_PROFILES = local.internal_services_compose_profiles
+ VAULT_ADDR = trimspace(var.vault_addr)
+ VAULT_NAMESPACE = trimspace(var.vault_namespace)
+ VAULT_ROLE = trimspace(var.vault_role)
+ VAULT_AGENT_ENABLED = var.vault_agent_enabled && trimspace(var.vault_addr) != "" ? "true" : "false"
+ VAULT_AUTH_METHOD = var.vault_auth_method
+ VAULT_AGENT_TOKEN_PATH = var.vault_agent_token_path
+ LIBOPS_MANAGED_RUNTIME_ENABLED = tostring(var.libops_managed_runtime_enabled)
+ LIBOPS_INTERNAL_SERVICES_ENABLED = tostring(local.internal_services_enabled)
+ LIBOPS_INTERNAL_SERVICES_AUTO_UPDATE = tostring(local.internal_services_enabled && var.libops_internal_services_auto_update)
}, local.fresh_filesystem_env, local.rollout_env)
env_file_content = <<-EOT
- path: "/home/cloud-compose/.env"
+ owner: "root:root"
permissions: "0640"
encoding: gzip+base64
content: ${jsonencode(base64gzip(module.runtime_env.content))}
EOT
application_env_file_content = <<-EOT
- path: "/home/cloud-compose/application-env.json"
+ owner: "root:root"
permissions: "0640"
encoding: gzip+base64
content: ${jsonencode(base64gzip(jsonencode(var.extra_env)))}
@@ -443,12 +456,13 @@ rootfs_archive_command_raw = <<-EOT
exit 1
fi
printf '%s %s\n' "$archive_sha256" "$tmp/rootfs.tar.gz" | sha256sum -c -
- tar -xzf "$tmp/rootfs.tar.gz" -C "$tmp"
+ tar --no-same-owner -xzf "$tmp/rootfs.tar.gz" -C "$tmp"
rootfs_dir="$(find "$tmp" -mindepth 1 -maxdepth 3 -type d -name rootfs -print -quit)"
- if [ -z "$rootfs_dir" ]; then
+ if [ -z "$rootfs_dir" ] || [ ! -d "$rootfs_dir" ]; then
echo "rootfs directory not found in $archive_url" >&2
exit 1
fi
+ chown -hR 0:0 -- "$rootfs_dir"
cp -a "$rootfs_dir"/. /
if [ -d "$overlay_dir" ]; then
cp -a "$overlay_dir"/. /
@@ -462,6 +476,7 @@ rollout_runcmd = var.rollout_enabled ? [
"bash /home/cloud-compose/deploy-rollout.sh >> /home/cloud-compose/run.log 2>&1",
] : []
cloud_init_yaml = templatefile("${path.module}/../../templates/cloud-init.yml", {
+ DIAGNOSTICS_SCRIPT_B64 = filebase64("${local.rootFs}/etc/cloud-compose/bin/cloud-compose-diagnostics.sh"),
FILESYSTEM_PREP_SCRIPT_B64 = filebase64("${local.rootFs}/home/cloud-compose/prepare-filesystem.sh"),
FILESYSTEM_PERSIST_SCRIPT_B64 = filebase64("${local.rootFs}/home/cloud-compose/persist-filesystems.sh"),
FRESH_FILESYSTEM_IDENTITY = "v1:gcp-disk-id:${google_compute_disk.data.disk_id}",
diff --git a/modules/gcp/runtime_contracts.tftest.hcl b/modules/gcp/runtime_contracts.tftest.hcl
index 38166be..50554ff 100644
--- a/modules/gcp/runtime_contracts.tftest.hcl
+++ b/modules/gcp/runtime_contracts.tftest.hcl
@@ -535,13 +535,14 @@ run "renders_verified_archive_before_downstream_overlay" {
command = plan
variables {
- name = "gcp-contract"
- project_id = "test-project"
- project_number = "123456789"
- docker_compose_repo = "https://github.com/libops/wp.git"
- rootfs = "testdata/rootfs"
- rootfs_archive_url = "https://example.invalid/cloud-compose.tar.gz?literal=$(id)"
- rootfs_archive_sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+ name = "gcp-contract"
+ project_id = "test-project"
+ project_number = "123456789"
+ docker_compose_repo = "https://github.com/libops/wp.git"
+ rootfs = "testdata/rootfs"
+ rootfs_archive_url = "https://example.invalid/cloud-compose.tar.gz?literal=$(id)"
+ rootfs_archive_sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+ offhost_backup_driver_path = "/etc/cloud-compose/libexec/custom-offhost-driver"
extra_env = {
NGINX_CLIENT_MAX_BODY_SIZE = "512m"
PHP_UPLOAD_MAX_FILESIZE = "512M"
@@ -561,10 +562,10 @@ run "renders_verified_archive_before_downstream_overlay" {
assert {
condition = can(regex(
- "(?s)cp -a \\\"\\$rootfs_dir\\\"/\\. /.*cp -a \\\"\\$overlay_dir\\\"/\\. /",
+ "(?s)sha256sum -c -.*tar --no-same-owner -xzf \\\"\\$tmp/rootfs\\.tar\\.gz\\\".*chown -hR 0:0 -- \\\"\\$rootfs_dir\\\".*cp -a \\\"\\$rootfs_dir\\\"/\\. /.*cp -a \\\"\\$overlay_dir\\\"/\\. /",
local.cloud_init_yaml,
))
- error_message = "The consumer rootfs overlay must be applied after the verified base archive."
+ error_message = "The verified base archive must be extracted ownership-safely, normalized to root, and installed before the consumer rootfs overlay."
}
assert {
@@ -581,6 +582,51 @@ run "renders_verified_archive_before_downstream_overlay" {
)
error_message = "The downstream overlay and isolated application environment data must be present in cloud-init."
}
+
+ assert {
+ condition = (
+ local.rootfs_file_permissions["etc/cloud-compose/libexec/custom-offhost-driver"] == "0755" &&
+ local.rootfs_file_permissions["etc/cloud-compose-overlay-marker"] == "0644" &&
+ strcontains(
+ local.write_files_content,
+ "- path: \"/var/lib/cloud-compose/rootfs-overlay/etc/cloud-compose/libexec/custom-offhost-driver\"\n owner: \"root:root\"\n permissions: \"0755\"",
+ ) &&
+ strcontains(
+ local.write_files_content,
+ "- path: \"/var/lib/cloud-compose/rootfs-overlay/etc/cloud-compose-overlay-marker\"\n owner: \"root:root\"\n permissions: \"0644\"",
+ )
+ )
+ error_message = "Archive-backed GCP overlays must make only the configured off-host backup driver executable."
+ }
+}
+
+run "renders_embedded_offhost_backup_driver_executable" {
+ command = plan
+
+ variables {
+ name = "gcp-embedded-driver"
+ project_id = "test-project"
+ project_number = "123456789"
+ docker_compose_repo = "https://github.com/libops/wp.git"
+ rootfs = "testdata/rootfs"
+ offhost_backup_driver_path = "/etc/cloud-compose/libexec/custom-offhost-driver"
+ }
+
+ assert {
+ condition = (
+ local.rootfs_file_permissions["etc/cloud-compose/libexec/custom-offhost-driver"] == "0755" &&
+ local.rootfs_file_permissions["etc/cloud-compose-overlay-marker"] == "0644" &&
+ strcontains(
+ local.write_files_content,
+ "- path: \"/etc/cloud-compose/libexec/custom-offhost-driver\"\n owner: \"root:root\"\n permissions: \"0755\"",
+ ) &&
+ strcontains(
+ local.write_files_content,
+ "- path: \"/etc/cloud-compose-overlay-marker\"\n owner: \"root:root\"\n permissions: \"0644\"",
+ )
+ )
+ error_message = "Embedded GCP overlays must make only the configured off-host backup driver executable."
+ }
}
run "namespaces_rollout_auth_away_from_application_environment" {
diff --git a/modules/gcp/testdata/rootfs/etc/cloud-compose/libexec/custom-offhost-driver b/modules/gcp/testdata/rootfs/etc/cloud-compose/libexec/custom-offhost-driver
new file mode 100644
index 0000000..df09e99
--- /dev/null
+++ b/modules/gcp/testdata/rootfs/etc/cloud-compose/libexec/custom-offhost-driver
@@ -0,0 +1 @@
+fixture driver payload
diff --git a/modules/gcp/variables.tf b/modules/gcp/variables.tf
index 9e1b969..df05258 100644
--- a/modules/gcp/variables.tf
+++ b/modules/gcp/variables.tf
@@ -262,7 +262,7 @@ variable "docker_compose_branch" {
variable "docker_compose_init" {
type = list(string)
default = [
- "sitectl config set-context \"$${SITECTL_CONTEXT_NAME}\" --type local --project-dir \"$${DOCKER_COMPOSE_DIR}\" --site \"$${CLOUD_COMPOSE_INSTANCE_NAME}\" --plugin \"$${SITECTL_PLUGIN}\" --environment \"$${SITECTL_ENVIRONMENT}\" --project-name \"$${CLOUD_COMPOSE_INSTANCE_NAME}\" --compose-project-name \"$${COMPOSE_PROJECT_NAME}\" --docker-socket /var/run/docker.sock --env-file .env --default"
+ "sitectl config set-context \"$${SITECTL_CONTEXT_NAME}\" --type local --project-dir \"$${DOCKER_COMPOSE_DIR}\" --site \"$${CLOUD_COMPOSE_INSTANCE_NAME}\" --plugin \"$${SITECTL_PLUGIN}\" --environment \"$${SITECTL_ENVIRONMENT}\" --compose-project-name \"$${COMPOSE_PROJECT_NAME}\" --docker-socket /var/run/docker.sock --env-file .env --yolo --default"
]
nullable = false
description = "After cloning the docker compose git repo, any initialization that needs to happen before the docker compose project can start. One command per list value"
@@ -608,6 +608,27 @@ variable "rootfs_archive_sha256" {
description = "Required 64-character SHA-256 checksum when rootfs_archive_url is set."
}
+variable "offhost_backup_required" {
+ type = bool
+ default = false
+ description = "Require nightly encrypted off-host coverage and scheduled disposable restore proofs from an operator-owned driver. Same-disk MariaDB dumps are retained but are not disaster recovery."
+}
+
+variable "offhost_backup_driver_path" {
+ type = string
+ default = "/etc/cloud-compose/libexec/offhost-backup-driver"
+ description = "Absolute path to the operator-supplied, root-owned provider-neutral DR driver. The driver owns its credentials; do not pass them through Terraform."
+
+ validation {
+ condition = (
+ can(regex("^/[A-Za-z0-9._/+:-]+$", var.offhost_backup_driver_path)) &&
+ !strcontains(var.offhost_backup_driver_path, "//") &&
+ length(regexall("(^|/)\\.\\.?(/|$)", var.offhost_backup_driver_path)) == 0
+ )
+ error_message = "offhost_backup_driver_path must be a safe absolute path without whitespace or dot segments."
+ }
+}
+
variable "extra_env" {
type = map(string)
default = {}
@@ -645,7 +666,7 @@ variable "runcmd" {
variable "initcmd" {
type = list(string)
default = []
- description = "Commands to run before /home/cloud-compose/run.sh"
+ description = "Commands to run before the root-owned Cloud Compose bootstrap entrypoint"
}
variable "artifact_registry_repository" {
diff --git a/modules/linode/main.tf b/modules/linode/main.tf
index 979dd4e..eed23d8 100644
--- a/modules/linode/main.tf
+++ b/modules/linode/main.tf
@@ -16,33 +16,35 @@ locals {
module "runtime" {
source = "../linux-vm-runtime"
- name = var.name
- provider_name = "linode"
- region = local.linode.region
- data_device = "/dev/disk/by-id/scsi-0Linode_Volume_${local.data_volume_label}"
- volumes_device = "/dev/disk/by-id/scsi-0Linode_Volume_${local.docker_volumes_volume_label}"
- ssh_users = merge(local.runtime.users, local.linode.ssh.users)
- cloud_compose_ssh_keys = local.linode.ssh.cloud_compose_keys
- rootfs = local.runtime.rootfs
- rootfs_archive_url = local.runtime.rootfs_archive_url
- rootfs_archive_sha256 = local.runtime.rootfs_archive_sha256
- ingress_port = local.compose.ingress_port
- primary_compose_project = local.compose.primary
- sitectl_ingress = local.compose.ingress
- docker_compose_repo = local.compose.repo
- docker_compose_branch = local.compose.branch
- compose_projects = local.compose.projects
- docker_compose_init = local.compose.init
- docker_compose_up = local.compose.up
- docker_compose_down = local.compose.down
- docker_compose_rollout = local.compose.rollout
- rollout_enabled = local.linode.rollout.enabled
- rollout_release_url = local.linode.rollout.release_url
- rollout_release_sha256 = local.linode.rollout.release_sha256
- rollout_port = local.linode.rollout.port
- rollout_jwks_uri = local.linode.rollout.jwks_uri
- rollout_jwt_audience = local.linode.rollout.jwt_audience
- rollout_custom_claims = local.linode.rollout.custom_claims
+ name = var.name
+ provider_name = "linode"
+ region = local.linode.region
+ data_device = "/dev/disk/by-id/scsi-0Linode_Volume_${local.data_volume_label}"
+ volumes_device = "/dev/disk/by-id/scsi-0Linode_Volume_${local.docker_volumes_volume_label}"
+ ssh_users = merge(local.runtime.users, local.linode.ssh.users)
+ cloud_compose_ssh_keys = local.linode.ssh.cloud_compose_keys
+ rootfs = local.runtime.rootfs
+ rootfs_archive_url = local.runtime.rootfs_archive_url
+ rootfs_archive_sha256 = local.runtime.rootfs_archive_sha256
+ offhost_backup_required = local.runtime.disaster_recovery.required
+ offhost_backup_driver_path = local.runtime.disaster_recovery.driver_path
+ ingress_port = local.compose.ingress_port
+ primary_compose_project = local.compose.primary
+ sitectl_ingress = local.compose.ingress
+ docker_compose_repo = local.compose.repo
+ docker_compose_branch = local.compose.branch
+ compose_projects = local.compose.projects
+ docker_compose_init = local.compose.init
+ docker_compose_up = local.compose.up
+ docker_compose_down = local.compose.down
+ docker_compose_rollout = local.compose.rollout
+ rollout_enabled = local.linode.rollout.enabled
+ rollout_release_url = local.linode.rollout.release_url
+ rollout_release_sha256 = local.linode.rollout.release_sha256
+ rollout_port = local.linode.rollout.port
+ rollout_jwks_uri = local.linode.rollout.jwks_uri
+ rollout_jwt_audience = local.linode.rollout.jwt_audience
+ rollout_custom_claims = local.linode.rollout.custom_claims
sitectl_packages = local.sitectl.packages
sitectl_version = local.sitectl.version
diff --git a/modules/linode/variables.tf b/modules/linode/variables.tf
index 6f3dae8..2ce975e 100644
--- a/modules/linode/variables.tf
+++ b/modules/linode/variables.tf
@@ -90,6 +90,11 @@ variable "runtime" {
rootfs_archive_sha256 = optional(string, "")
users = optional(map(list(string)), {})
+ disaster_recovery = optional(object({
+ required = optional(bool, false)
+ driver_path = optional(string, "/etc/cloud-compose/libexec/offhost-backup-driver")
+ }), {})
+
compose = optional(object({
primary = optional(string, "")
ingress_port = optional(number, 80)
@@ -198,6 +203,15 @@ variable "runtime" {
error_message = "runtime.rootfs_archive_url and a 64-character runtime.rootfs_archive_sha256 must be supplied together."
}
+ validation {
+ condition = (
+ can(regex("^/[A-Za-z0-9._/+:-]+$", var.runtime.disaster_recovery.driver_path)) &&
+ !strcontains(var.runtime.disaster_recovery.driver_path, "//") &&
+ length(regexall("(^|/)\\.\\.?(/|$)", var.runtime.disaster_recovery.driver_path)) == 0
+ )
+ error_message = "runtime.disaster_recovery.driver_path must be a safe absolute path without whitespace or dot segments."
+ }
+
validation {
condition = alltrue([
for name in keys(var.runtime.extra_env) :
diff --git a/modules/linux-vm-runtime/disaster_recovery.tftest.hcl b/modules/linux-vm-runtime/disaster_recovery.tftest.hcl
new file mode 100644
index 0000000..495c3fc
--- /dev/null
+++ b/modules/linux-vm-runtime/disaster_recovery.tftest.hcl
@@ -0,0 +1,48 @@
+run "renders_provider_neutral_disaster_recovery_controls" {
+ command = plan
+
+ variables {
+ name = "contract-test"
+ provider_name = "linode"
+ region = "us-east"
+ data_device = "/dev/test-data"
+ volumes_device = "/dev/test-volumes"
+ docker_compose_repo = "https://github.com/libops/wp.git"
+ offhost_backup_required = true
+ offhost_backup_driver_path = "/etc/cloud-compose/libexec/acme-offhost"
+ }
+
+ assert {
+ condition = (
+ local.host_env.CLOUD_COMPOSE_OFFHOST_BACKUP_REQUIRED == "true" &&
+ local.host_env.CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER == "/etc/cloud-compose/libexec/acme-offhost"
+ )
+ error_message = "The Linux VM runtime must render only the required switch and operator-owned driver path."
+ }
+
+ assert {
+ condition = (
+ !strcontains(module.runtime_env.content, "ACCESS_KEY") &&
+ !strcontains(module.runtime_env.content, "SECRET_KEY") &&
+ !strcontains(module.runtime_env.content, "BUCKET")
+ )
+ error_message = "The provider-neutral DR contract must not render storage credentials or a storage-vendor destination."
+ }
+}
+
+run "rejects_unsafe_disaster_recovery_driver_path" {
+ command = plan
+
+ variables {
+ name = "contract-test"
+ provider_name = "linode"
+ region = "us-east"
+ data_device = "/dev/test-data"
+ volumes_device = "/dev/test-volumes"
+ docker_compose_repo = "https://github.com/libops/wp.git"
+ offhost_backup_required = true
+ offhost_backup_driver_path = "/etc/cloud-compose/libexec/../untrusted"
+ }
+
+ expect_failures = [var.offhost_backup_driver_path]
+}
diff --git a/modules/linux-vm-runtime/main.tf b/modules/linux-vm-runtime/main.tf
index c2bb357..fc5e168 100644
--- a/modules/linux-vm-runtime/main.tf
+++ b/modules/linux-vm-runtime/main.tf
@@ -89,20 +89,25 @@ locals {
{ for file in local.base_files : file => "${local.rootfs}/${file}" },
{ for file in local.embedded_additional_files : file => "${local.additional_rootfs}/${file}" }
)
+ rootfs_file_permissions = {
+ for file in setunion(local.base_files, local.additional_files) :
+ file => endswith(file, ".sh") || "/${file}" == var.offhost_backup_driver_path ? "0755" : "0644"
+ }
archive_additional_rootfs_commands = join("\n", [
for file in local.additional_files : <<-EOT
destination="$(printf '%s' '${base64encode("/${file}")}' | base64 -d)"
install -d "$(dirname "$destination")"
printf '%s' '${filebase64("${local.additional_rootfs}/${file}")}' | base64 -d >"$destination"
- chmod ${endswith(file, ".sh") ? "0755" : "0644"} "$destination"
+ chmod ${local.rootfs_file_permissions[file]} "$destination"
EOT
])
write_files_content = join("\n", [
for file, fullpath in local.all_files : <<-EOT
- path: ${jsonencode(startswith(file, "mnt/disks/") ? "/var/lib/cloud-compose/mounted-rootfs/${file}" : "/${file}")}
- permissions: ${jsonencode(endswith(file, ".sh") ? "0755" : "0644")}
+ owner: "root:root"
+ permissions: ${jsonencode(local.rootfs_file_permissions[file])}
encoding: gzip+base64
content: ${jsonencode(base64gzip(file(fullpath)))}
EOT
@@ -111,6 +116,7 @@ EOT
docker_compose_scripts = join("\n", [
for name in ["init", "up", "down", "rollout"] : <<-EOT
- path: "/home/cloud-compose/${name}"
+ owner: "root:root"
permissions: "0755"
encoding: gzip+base64
content: ${jsonencode(base64gzip(<<-EOS
@@ -128,6 +134,7 @@ EOT
compose_projects_content = jsonencode(local.validated_compose_projects)
compose_projects_file = <<-EOT
- path: "/home/cloud-compose/compose-projects.json"
+ owner: "root:root"
permissions: "0640"
encoding: gzip+base64
content: ${jsonencode(base64gzip(local.compose_projects_content))}
@@ -145,9 +152,10 @@ managed_runtime_artifact_lines = [
try(artifact.restart, ""),
])
]
-managed_runtime_artifacts_content = join("\n", local.managed_runtime_artifact_lines)
+managed_runtime_artifacts_content = length(local.managed_runtime_artifact_lines) == 0 ? "\n" : "${join("\n", local.managed_runtime_artifact_lines)}\n"
managed_runtime_artifacts_file = <<-EOT
- path: "/home/cloud-compose/managed-runtime-artifacts.tsv"
+ owner: "root:root"
permissions: "0640"
encoding: gzip+base64
content: ${jsonencode(base64gzip(local.managed_runtime_artifacts_content))}
@@ -199,59 +207,62 @@ vault_agent_files_raw = <<-EOT
vault_agent_files = var.vault_agent_enabled && trimspace(var.vault_addr) != "" ? local.vault_agent_files_raw : ""
host_env = {
- HOME = "/home/cloud-compose"
- CLOUD_COMPOSE_PROVIDER = var.provider_name
- CLOUD_COMPOSE_INSTANCE_NAME = var.name
- CLOUD_COMPOSE_APPS = join(" ", keys(local.compose_projects))
- CLOUD_COMPOSE_PRIMARY_APP = local.primary_compose_project_key
- COMPOSE_PROJECTS_FILE = "/home/cloud-compose/compose-projects.json"
- COMPOSE_PROJECT_NAME = local.primary_compose_project.compose_project_name
- COMPOSE_BIND_PORT = tostring(local.primary_compose_project.ingress_port)
- DOCKER_COMPOSE_DIR = local.primary_compose_project.project_dir
- DOCKER_COMPOSE_REPO = local.primary_compose_project.docker_compose_repo
- DOCKER_COMPOSE_BRANCH = local.primary_compose_project.docker_compose_branch
- DOCKER_COMPOSE_VERSION = var.docker_compose_version
- DOCKER_BUILDX_VERSION = var.docker_buildx_version
- GCP_PROJECT = ""
- GCP_PROJECT_NUMBER = ""
- GCP_INSTANCE_NAME = var.name
- GCP_REGION = var.region
- GCP_ZONE = var.zone != "" ? var.zone : var.region
- GCP_APP_SERVICE_ACCOUNT_EMAIL = ""
- GCP_APP_CREDENTIALS_ENABLED = "false"
- SITECTL_PACKAGES = join(" ", module.sitectl_runtime.packages)
- SITECTL_VERSION = var.sitectl_version
- SITECTL_PACKAGE_VERSIONS = jsonencode(module.sitectl_runtime.package_versions)
- SITECTL_CONTEXT_NAME = local.primary_compose_project.sitectl_context_name
- SITECTL_PLUGIN = local.primary_compose_project.sitectl_plugin
- SITECTL_ENVIRONMENT = local.primary_compose_project.sitectl_environment
- SITECTL_VERIFY_ARGS = join(" ", local.primary_compose_project.sitectl_verify_args)
- POWER_MANAGEMENT_ENABLED = "false"
- COMPOSE_PROFILES = ""
- VAULT_ADDR = trimspace(var.vault_addr)
- VAULT_NAMESPACE = trimspace(var.vault_namespace)
- VAULT_ROLE = trimspace(var.vault_role)
- VAULT_AGENT_ENABLED = var.vault_agent_enabled && trimspace(var.vault_addr) != "" ? "true" : "false"
- VAULT_AUTH_METHOD = var.vault_auth_method
- ROLLOUT_ENABLED = tostring(var.rollout_enabled)
- ROLLOUT_DOWNLOAD_URL = trimspace(var.rollout_release_url)
- ROLLOUT_DOWNLOAD_SHA256 = trimspace(var.rollout_release_sha256)
- ROLLOUT_PORT = tostring(var.rollout_port)
- ROLLOUT_JWKS_URI = trimspace(var.rollout_jwks_uri)
- ROLLOUT_JWT_AUD = trimspace(var.rollout_jwt_audience)
- ROLLOUT_CUSTOM_CLAIMS = trimspace(var.rollout_custom_claims)
- ROLLOUT_CMD = "/bin/bash"
- ROLLOUT_ARGS = "/home/cloud-compose/rollout"
- ROLLOUT_LOCK_FILE = "/mnt/disks/data/rollout.lock"
- VAULT_AGENT_TOKEN_PATH = var.vault_agent_token_path
- LIBOPS_MANAGED_RUNTIME_ENABLED = tostring(var.libops_managed_runtime_enabled)
- LIBOPS_INTERNAL_SERVICES_ENABLED = tostring(var.libops_internal_services_enabled)
- LIBOPS_INTERNAL_SERVICES_AUTO_UPDATE = tostring(var.libops_internal_services_auto_update)
- INTERNAL_SERVICES_COMPOSE_PROFILES = ""
+ HOME = "/home/cloud-compose"
+ CLOUD_COMPOSE_PROVIDER = var.provider_name
+ CLOUD_COMPOSE_INSTANCE_NAME = var.name
+ CLOUD_COMPOSE_APPS = join(" ", keys(local.compose_projects))
+ CLOUD_COMPOSE_PRIMARY_APP = local.primary_compose_project_key
+ CLOUD_COMPOSE_OFFHOST_BACKUP_REQUIRED = tostring(var.offhost_backup_required)
+ CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER = var.offhost_backup_driver_path
+ COMPOSE_PROJECTS_FILE = "/home/cloud-compose/compose-projects.json"
+ COMPOSE_PROJECT_NAME = local.primary_compose_project.compose_project_name
+ COMPOSE_BIND_PORT = tostring(local.primary_compose_project.ingress_port)
+ DOCKER_COMPOSE_DIR = local.primary_compose_project.project_dir
+ DOCKER_COMPOSE_REPO = local.primary_compose_project.docker_compose_repo
+ DOCKER_COMPOSE_BRANCH = local.primary_compose_project.docker_compose_branch
+ DOCKER_COMPOSE_VERSION = var.docker_compose_version
+ DOCKER_BUILDX_VERSION = var.docker_buildx_version
+ GCP_PROJECT = ""
+ GCP_PROJECT_NUMBER = ""
+ GCP_INSTANCE_NAME = var.name
+ GCP_REGION = var.region
+ GCP_ZONE = var.zone != "" ? var.zone : var.region
+ GCP_APP_SERVICE_ACCOUNT_EMAIL = ""
+ GCP_APP_CREDENTIALS_ENABLED = "false"
+ SITECTL_PACKAGES = join(" ", module.sitectl_runtime.packages)
+ SITECTL_VERSION = var.sitectl_version
+ SITECTL_PACKAGE_VERSIONS = jsonencode(module.sitectl_runtime.package_versions)
+ SITECTL_CONTEXT_NAME = local.primary_compose_project.sitectl_context_name
+ SITECTL_PLUGIN = local.primary_compose_project.sitectl_plugin
+ SITECTL_ENVIRONMENT = local.primary_compose_project.sitectl_environment
+ SITECTL_VERIFY_ARGS = join(" ", local.primary_compose_project.sitectl_verify_args)
+ POWER_MANAGEMENT_ENABLED = "false"
+ COMPOSE_PROFILES = ""
+ VAULT_ADDR = trimspace(var.vault_addr)
+ VAULT_NAMESPACE = trimspace(var.vault_namespace)
+ VAULT_ROLE = trimspace(var.vault_role)
+ VAULT_AGENT_ENABLED = var.vault_agent_enabled && trimspace(var.vault_addr) != "" ? "true" : "false"
+ VAULT_AUTH_METHOD = var.vault_auth_method
+ ROLLOUT_ENABLED = tostring(var.rollout_enabled)
+ ROLLOUT_DOWNLOAD_URL = trimspace(var.rollout_release_url)
+ ROLLOUT_DOWNLOAD_SHA256 = trimspace(var.rollout_release_sha256)
+ ROLLOUT_PORT = tostring(var.rollout_port)
+ ROLLOUT_JWKS_URI = trimspace(var.rollout_jwks_uri)
+ ROLLOUT_JWT_AUD = trimspace(var.rollout_jwt_audience)
+ ROLLOUT_CUSTOM_CLAIMS = trimspace(var.rollout_custom_claims)
+ ROLLOUT_CMD = "/bin/bash"
+ ROLLOUT_ARGS = "/home/cloud-compose/rollout"
+ ROLLOUT_LOCK_FILE = "/mnt/disks/data/rollout.lock"
+ VAULT_AGENT_TOKEN_PATH = var.vault_agent_token_path
+ LIBOPS_MANAGED_RUNTIME_ENABLED = tostring(var.libops_managed_runtime_enabled)
+ LIBOPS_INTERNAL_SERVICES_ENABLED = tostring(var.libops_internal_services_enabled)
+ LIBOPS_INTERNAL_SERVICES_AUTO_UPDATE = tostring(var.libops_internal_services_auto_update)
+ INTERNAL_SERVICES_COMPOSE_PROFILES = ""
}
env_file_content = <<-EOT
- path: "/home/cloud-compose/.env"
+ owner: "root:root"
permissions: "0640"
encoding: gzip+base64
content: ${jsonencode(base64gzip(module.runtime_env.content))}
@@ -259,6 +270,7 @@ env_file_content = <<-EOT
application_env_file_content = <<-EOT
- path: "/home/cloud-compose/application-env.json"
+ owner: "root:root"
permissions: "0640"
encoding: gzip+base64
content: ${jsonencode(base64gzip(jsonencode(var.extra_env)))}
@@ -301,12 +313,13 @@ rootfs_archive_prepare_command_raw = <<-EOT
--retry 5 --retry-all-errors --retry-delay 2 --retry-max-time 900 \
--connect-timeout 10 --max-time 300 -o "$tmp/rootfs.tar.gz" -- "$archive_url"
printf '%s %s\n' "$archive_sha256" "$tmp/rootfs.tar.gz" | sha256sum -c -
- tar -xzf "$tmp/rootfs.tar.gz" -C "$tmp"
+ tar --no-same-owner -xzf "$tmp/rootfs.tar.gz" -C "$tmp"
rootfs_dir="$(find "$tmp" -mindepth 1 -maxdepth 3 -type d -name rootfs -print -quit)"
if [ -z "$rootfs_dir" ] || [ ! -d "$rootfs_dir" ]; then
echo "rootfs directory not found in verified archive $archive_url" >&2
exit 1
fi
+ chown -hR 0:0 -- "$rootfs_dir"
filesystem_prep_source="$rootfs_dir/home/cloud-compose/prepare-filesystem.sh"
filesystem_persist_source="$rootfs_dir/home/cloud-compose/persist-filesystems.sh"
if [ ! -f "$filesystem_prep_source" ] || [ ! -f "$filesystem_persist_source" ]; then
@@ -331,6 +344,7 @@ rootfs_archive_install_command = local.rootfs_archive_url != "" ? local.rootfs_a
cloud_init = templatefile("${path.module}/templates/cloud-init.yml", {
CLOUD_COMPOSE_SSH_KEYS = var.cloud_compose_ssh_keys
SSH_USERS = var.ssh_users
+ DIAGNOSTICS_SCRIPT_B64 = filebase64("${local.rootfs}/etc/cloud-compose/bin/cloud-compose-diagnostics.sh")
DATA_DEVICE = var.data_device
VOLUMES_DEVICE = var.volumes_device
WRITE_FILES_CONTENT = local.write_files_content
diff --git a/modules/linux-vm-runtime/runtime_inputs.tftest.hcl b/modules/linux-vm-runtime/runtime_inputs.tftest.hcl
index 26288d4..8aa38fb 100644
--- a/modules/linux-vm-runtime/runtime_inputs.tftest.hcl
+++ b/modules/linux-vm-runtime/runtime_inputs.tftest.hcl
@@ -25,6 +25,17 @@ run "renders_safe_ssh_values" {
condition = strcontains(output.cloud_init, "name: ${jsonencode("app_operator")}")
error_message = "SSH usernames must be YAML encoded as literal values."
}
+
+ assert {
+ condition = (
+ strcontains(output.cloud_init, "path: /var/lib/cloud-compose/bootstrap/cloud-compose-diagnostics.sh") &&
+ strcontains(output.cloud_init, "NOPASSWD:/etc/cloud-compose/bin/cloud-compose-diagnostics.sh state") &&
+ strcontains(output.cloud_init, "NOPASSWD:/etc/cloud-compose/bin/cloud-compose-diagnostics.sh status") &&
+ strcontains(output.cloud_init, "NOPASSWD:/etc/cloud-compose/bin/cloud-compose-diagnostics.sh dump") &&
+ strcontains(output.cloud_init, "install -m 0755 -o root -g root")
+ )
+ error_message = "Cloud-init must install one root-owned diagnostics program with exact passwordless sudo commands."
+ }
}
run "normalizes_minimal_compose_project" {
@@ -196,14 +207,16 @@ run "renders_verified_rootfs_archive" {
command = plan
variables {
- name = "contract-test"
- provider_name = "linode"
- region = "us-east"
- data_device = "/dev/test-data"
- volumes_device = "/dev/test-volumes"
- docker_compose_repo = "https://github.com/libops/wp.git"
- rootfs_archive_url = "https://example.invalid/cloud-compose-$(id).tar.gz"
- rootfs_archive_sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+ name = "contract-test"
+ provider_name = "linode"
+ region = "us-east"
+ data_device = "/dev/test-data"
+ volumes_device = "/dev/test-volumes"
+ docker_compose_repo = "https://github.com/libops/wp.git"
+ rootfs = "testdata/rootfs"
+ rootfs_archive_url = "https://example.invalid/cloud-compose-$(id).tar.gz"
+ rootfs_archive_sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+ offhost_backup_driver_path = "/etc/cloud-compose/libexec/custom-offhost-driver"
}
assert {
@@ -213,17 +226,19 @@ run "renders_verified_rootfs_archive" {
assert {
condition = (
- strcontains(output.cloud_init, "tar -xzf \"$tmp/rootfs.tar.gz\"") &&
+ strcontains(output.cloud_init, "tar --no-same-owner -xzf \"$tmp/rootfs.tar.gz\"") &&
+ strcontains(output.cloud_init, "chown -hR 0:0 -- \"$rootfs_dir\"") &&
strcontains(output.cloud_init, "install -m 0600 -- \"$filesystem_prep_source\" \"$filesystem_prep\"") &&
strcontains(output.cloud_init, "bash \"$filesystem_prep\"") &&
strcontains(output.cloud_init, "bash \"$filesystem_persist\"") &&
strcontains(output.cloud_init, "cp -a \"$rootfs_dir\"/. /") &&
- length(split("sha256sum -c -", output.cloud_init)[0]) < length(split("tar -xzf \"$tmp/rootfs.tar.gz\"", output.cloud_init)[0]) &&
- length(split("tar -xzf \"$tmp/rootfs.tar.gz\"", output.cloud_init)[0]) < length(split("install -m 0600 -- \"$filesystem_prep_source\" \"$filesystem_prep\"", output.cloud_init)[0]) &&
+ length(split("sha256sum -c -", output.cloud_init)[0]) < length(split("tar --no-same-owner -xzf \"$tmp/rootfs.tar.gz\"", output.cloud_init)[0]) &&
+ length(split("tar --no-same-owner -xzf \"$tmp/rootfs.tar.gz\"", output.cloud_init)[0]) < length(split("chown -hR 0:0 -- \"$rootfs_dir\"", output.cloud_init)[0]) &&
+ length(split("chown -hR 0:0 -- \"$rootfs_dir\"", output.cloud_init)[0]) < length(split("install -m 0600 -- \"$filesystem_prep_source\" \"$filesystem_prep\"", output.cloud_init)[0]) &&
length(split("install -m 0600 -- \"$filesystem_prep_source\" \"$filesystem_prep\"", output.cloud_init)[0]) < length(split("bash \"$filesystem_prep\"", output.cloud_init)[0]) &&
length(split("bash \"$filesystem_persist\"", output.cloud_init)[0]) < length(split("cp -a \"$rootfs_dir\"/. /", output.cloud_init)[0])
)
- error_message = "Archive cloud-init must verify and extract before loading its helpers, then install the rootfs only after filesystem preparation and persistence."
+ error_message = "Archive cloud-init must verify, extract, and normalize root ownership before loading its helpers, then install the rootfs only after filesystem preparation and persistence."
}
assert {
@@ -249,18 +264,36 @@ run "renders_verified_rootfs_archive" {
)
error_message = "Archive URLs must be rendered as base64 data rather than executable shell syntax."
}
+
+ assert {
+ condition = (
+ local.rootfs_file_permissions["etc/cloud-compose/libexec/custom-offhost-driver"] == "0755" &&
+ local.rootfs_file_permissions["etc/cloud-compose/unrelated-config"] == "0644" &&
+ strcontains(
+ split(base64encode("/etc/cloud-compose/libexec/custom-offhost-driver"), local.archive_additional_rootfs_commands)[1],
+ "chmod 0755 \"$destination\"",
+ ) &&
+ strcontains(
+ split(base64encode("/etc/cloud-compose/unrelated-config"), local.archive_additional_rootfs_commands)[1],
+ "chmod 0644 \"$destination\"",
+ )
+ )
+ error_message = "Archive-backed Linux VM overlays must make only the configured off-host backup driver executable."
+ }
}
run "embeds_filesystem_helpers_without_archive" {
command = plan
variables {
- name = "contract-test"
- provider_name = "digitalocean"
- region = "nyc3"
- data_device = "/dev/test-data"
- volumes_device = "/dev/test-volumes"
- docker_compose_repo = "https://github.com/libops/wp.git"
+ name = "contract-test"
+ provider_name = "digitalocean"
+ region = "nyc3"
+ data_device = "/dev/test-data"
+ volumes_device = "/dev/test-volumes"
+ docker_compose_repo = "https://github.com/libops/wp.git"
+ rootfs = "testdata/rootfs"
+ offhost_backup_driver_path = "/etc/cloud-compose/libexec/custom-offhost-driver"
}
assert {
@@ -271,6 +304,22 @@ run "embeds_filesystem_helpers_without_archive" {
)
error_message = "Inline cloud-init must retain the embedded filesystem-helper bootstrap when no archive is configured."
}
+
+ assert {
+ condition = (
+ local.rootfs_file_permissions["etc/cloud-compose/libexec/custom-offhost-driver"] == "0755" &&
+ local.rootfs_file_permissions["etc/cloud-compose/unrelated-config"] == "0644" &&
+ strcontains(
+ local.write_files_content,
+ "- path: \"/etc/cloud-compose/libexec/custom-offhost-driver\"\n owner: \"root:root\"\n permissions: \"0755\"",
+ ) &&
+ strcontains(
+ local.write_files_content,
+ "- path: \"/etc/cloud-compose/unrelated-config\"\n owner: \"root:root\"\n permissions: \"0644\"",
+ )
+ )
+ error_message = "Embedded Linux VM overlays must make only the configured off-host backup driver executable."
+ }
}
run "rejects_rootfs_archive_without_checksum" {
diff --git a/modules/linux-vm-runtime/templates/cloud-init.yml b/modules/linux-vm-runtime/templates/cloud-init.yml
index 1cea770..7cac7c0 100644
--- a/modules/linux-vm-runtime/templates/cloud-init.yml
+++ b/modules/linux-vm-runtime/templates/cloud-init.yml
@@ -10,6 +10,9 @@ users:
- docker
sudo:
- ALL=(root) NOPASSWD:/usr/bin/systemctl start cloud-compose.service,/usr/bin/systemctl stop cloud-compose.service,/usr/bin/systemctl restart cloud-compose.service,/usr/bin/systemctl status cloud-compose.service
+ - ALL=(root) NOPASSWD:/etc/cloud-compose/bin/cloud-compose-diagnostics.sh state
+ - ALL=(root) NOPASSWD:/etc/cloud-compose/bin/cloud-compose-diagnostics.sh status
+ - ALL=(root) NOPASSWD:/etc/cloud-compose/bin/cloud-compose-diagnostics.sh dump
%{ if length(CLOUD_COMPOSE_SSH_KEYS) > 0 ~}
ssh_authorized_keys:
%{ for key in CLOUD_COMPOSE_SSH_KEYS ~}
@@ -26,6 +29,11 @@ users:
%{ endfor ~}
write_files:
+- path: /var/lib/cloud-compose/bootstrap/cloud-compose-diagnostics.sh
+ owner: root:root
+ permissions: "0600"
+ encoding: b64
+ content: ${DIAGNOSTICS_SCRIPT_B64}
${WRITE_FILES_CONTENT}
${DOCKER_COMPOSE_SCRIPTS}
${COMPOSE_PROJECTS_FILE}
@@ -55,7 +63,12 @@ runcmd:
chown cloud-compose:cloud-compose /home/cloud-compose/.ssh/authorized_keys
chmod 0600 /home/cloud-compose/.ssh/authorized_keys
install -d -m 0755 /etc/sudoers.d
- printf 'cloud-compose ALL=(root) NOPASSWD:/usr/bin/systemctl start cloud-compose.service,/usr/bin/systemctl stop cloud-compose.service,/usr/bin/systemctl restart cloud-compose.service,/usr/bin/systemctl status cloud-compose.service\n' >/etc/sudoers.d/90-cloud-compose
+ printf '%s\n' \
+ 'cloud-compose ALL=(root) NOPASSWD:/usr/bin/systemctl start cloud-compose.service,/usr/bin/systemctl stop cloud-compose.service,/usr/bin/systemctl restart cloud-compose.service,/usr/bin/systemctl status cloud-compose.service' \
+ 'cloud-compose ALL=(root) NOPASSWD:/etc/cloud-compose/bin/cloud-compose-diagnostics.sh state' \
+ 'cloud-compose ALL=(root) NOPASSWD:/etc/cloud-compose/bin/cloud-compose-diagnostics.sh status' \
+ 'cloud-compose ALL=(root) NOPASSWD:/etc/cloud-compose/bin/cloud-compose-diagnostics.sh dump' \
+ >/etc/sudoers.d/90-cloud-compose
chmod 0440 /etc/sudoers.d/90-cloud-compose
}
@@ -90,6 +103,10 @@ runcmd:
cp -a /var/lib/cloud-compose/mounted-rootfs/mnt/disks/. /mnt/disks/
fi
%{ endif }
+ install -d -m 0755 -o root -g root /etc/cloud-compose/bin
+ install -m 0755 -o root -g root \
+ /var/lib/cloud-compose/bootstrap/cloud-compose-diagnostics.sh \
+ /etc/cloud-compose/bin/cloud-compose-diagnostics.sh
chown root:cloud-compose /mnt/disks/data
chmod 1775 /mnt/disks/data
chown cloud-compose:cloud-compose /mnt/disks/volumes
@@ -98,5 +115,6 @@ runcmd:
%{ if ROLLOUT_RUNCMD != "" ~}
${ROLLOUT_RUNCMD}
%{ endif ~}
- rm -f /home/cloud-compose/.cloud-compose-bootstrap-complete
- bash /home/cloud-compose/start-cloud-compose-bootstrap.sh
+ rm -f /var/lib/cloud-compose/bootstrap-complete
+ /etc/cloud-compose/libexec/harden-bootstrap-paths.sh
+ bash /etc/cloud-compose/libexec/start-cloud-compose-bootstrap.sh
diff --git a/modules/linux-vm-runtime/testdata/rootfs/etc/cloud-compose/libexec/custom-offhost-driver b/modules/linux-vm-runtime/testdata/rootfs/etc/cloud-compose/libexec/custom-offhost-driver
new file mode 100644
index 0000000..df09e99
--- /dev/null
+++ b/modules/linux-vm-runtime/testdata/rootfs/etc/cloud-compose/libexec/custom-offhost-driver
@@ -0,0 +1 @@
+fixture driver payload
diff --git a/modules/linux-vm-runtime/testdata/rootfs/etc/cloud-compose/unrelated-config b/modules/linux-vm-runtime/testdata/rootfs/etc/cloud-compose/unrelated-config
new file mode 100644
index 0000000..578c4da
--- /dev/null
+++ b/modules/linux-vm-runtime/testdata/rootfs/etc/cloud-compose/unrelated-config
@@ -0,0 +1 @@
+fixture config payload
diff --git a/modules/linux-vm-runtime/variables.tf b/modules/linux-vm-runtime/variables.tf
index 968cf42..2ec8c3c 100644
--- a/modules/linux-vm-runtime/variables.tf
+++ b/modules/linux-vm-runtime/variables.tf
@@ -96,6 +96,27 @@ variable "rootfs_archive_sha256" {
description = "Required 64-character SHA-256 checksum when rootfs_archive_url is set."
}
+variable "offhost_backup_required" {
+ type = bool
+ default = false
+ description = "Require nightly encrypted off-host coverage and scheduled disposable restore proofs from an operator-owned driver. Same-disk MariaDB dumps are retained but are not disaster recovery."
+}
+
+variable "offhost_backup_driver_path" {
+ type = string
+ default = "/etc/cloud-compose/libexec/offhost-backup-driver"
+ description = "Absolute path to the operator-supplied, root-owned provider-neutral DR driver. The driver owns its credentials; do not pass them through Terraform."
+
+ validation {
+ condition = (
+ can(regex("^/[A-Za-z0-9._/+:-]+$", var.offhost_backup_driver_path)) &&
+ !strcontains(var.offhost_backup_driver_path, "//") &&
+ length(regexall("(^|/)\\.\\.?(/|$)", var.offhost_backup_driver_path)) == 0
+ )
+ error_message = "offhost_backup_driver_path must be a safe absolute path without whitespace or dot segments."
+ }
+}
+
variable "ingress_port" {
type = number
default = 80
@@ -188,7 +209,7 @@ variable "compose_projects" {
variable "docker_compose_init" {
type = list(string)
default = [
- "sitectl config set-context \"$${SITECTL_CONTEXT_NAME}\" --type local --project-dir \"$${DOCKER_COMPOSE_DIR}\" --site \"$${CLOUD_COMPOSE_INSTANCE_NAME}\" --plugin \"$${SITECTL_PLUGIN}\" --environment \"$${SITECTL_ENVIRONMENT}\" --project-name \"$${CLOUD_COMPOSE_INSTANCE_NAME}\" --compose-project-name \"$${COMPOSE_PROJECT_NAME}\" --docker-socket /var/run/docker.sock --env-file .env --default"
+ "sitectl config set-context \"$${SITECTL_CONTEXT_NAME}\" --type local --project-dir \"$${DOCKER_COMPOSE_DIR}\" --site \"$${CLOUD_COMPOSE_INSTANCE_NAME}\" --plugin \"$${SITECTL_PLUGIN}\" --environment \"$${SITECTL_ENVIRONMENT}\" --compose-project-name \"$${COMPOSE_PROJECT_NAME}\" --docker-socket /var/run/docker.sock --env-file .env --yolo --default"
]
nullable = false
description = "Commands run after a compose repository is cloned."
diff --git a/providers/do/template_versions.tftest.hcl b/providers/do/template_versions.tftest.hcl
index 781c11c..3fc8276 100644
--- a/providers/do/template_versions.tftest.hcl
+++ b/providers/do/template_versions.tftest.hcl
@@ -28,8 +28,8 @@ run "custom_package_set_merges_only_applicable_template_versions" {
}
assert {
- condition = local.runtime.compose.branch == "v1.1.0"
- error_message = "The DigitalOcean entrypoint must inherit the ISLE v1.1.0 template branch when no override is supplied."
+ condition = local.runtime.compose.branch == "v1.3.0"
+ error_message = "The DigitalOcean entrypoint must inherit the ISLE v1.3.0 template branch when no override is supplied."
}
assert {
@@ -55,7 +55,7 @@ run "explicit_core_only_package_set_disables_template_plugins" {
assert {
condition = local.runtime.sitectl.packages == tolist(["sitectl"]) && local.runtime.sitectl.package_versions == {
- sitectl = "v1.0.0"
+ sitectl = "v1.8.2"
}
error_message = "The DigitalOcean entrypoint must preserve an explicit core-only package set."
}
diff --git a/providers/do/variables.tf b/providers/do/variables.tf
index 571ca4c..eab146b 100644
--- a/providers/do/variables.tf
+++ b/providers/do/variables.tf
@@ -87,6 +87,11 @@ variable "runtime" {
rootfs_archive_sha256 = optional(string, "")
users = optional(map(list(string)), {})
+ disaster_recovery = optional(object({
+ required = optional(bool, false)
+ driver_path = optional(string, "/etc/cloud-compose/libexec/offhost-backup-driver")
+ }), {})
+
compose = optional(object({
primary = optional(string, "")
ingress_port = optional(number, 80)
@@ -195,6 +200,15 @@ variable "runtime" {
error_message = "runtime.rootfs_archive_url and a 64-character runtime.rootfs_archive_sha256 must be supplied together."
}
+ validation {
+ condition = (
+ can(regex("^/[A-Za-z0-9._/+:-]+$", var.runtime.disaster_recovery.driver_path)) &&
+ !strcontains(var.runtime.disaster_recovery.driver_path, "//") &&
+ length(regexall("(^|/)\\.\\.?(/|$)", var.runtime.disaster_recovery.driver_path)) == 0
+ )
+ error_message = "runtime.disaster_recovery.driver_path must be a safe absolute path without whitespace or dot segments."
+ }
+
validation {
condition = alltrue([
for name in keys(var.runtime.extra_env) :
diff --git a/providers/gcp/main.tf b/providers/gcp/main.tf
index 80a74c0..0ff9757 100644
--- a/providers/gcp/main.tf
+++ b/providers/gcp/main.tf
@@ -145,13 +145,15 @@ module "gcp" {
overlay_source_instance = local.gcp_overlay.source_instance
volume_names = local.gcp_overlay.volume_names
- users = local.runtime.users
- rootfs = local.runtime.rootfs
- rootfs_archive_url = local.runtime.rootfs_archive_url
- rootfs_archive_sha256 = local.runtime.rootfs_archive_sha256
- extra_env = local.runtime.extra_env
- runcmd = local.gcp_cloud_init.runcmd
- initcmd = local.gcp_cloud_init.initcmd
+ users = local.runtime.users
+ rootfs = local.runtime.rootfs
+ rootfs_archive_url = local.runtime.rootfs_archive_url
+ rootfs_archive_sha256 = local.runtime.rootfs_archive_sha256
+ offhost_backup_required = local.runtime.disaster_recovery.required
+ offhost_backup_driver_path = local.runtime.disaster_recovery.driver_path
+ extra_env = local.runtime.extra_env
+ runcmd = local.gcp_cloud_init.runcmd
+ initcmd = local.gcp_cloud_init.initcmd
artifact_registry_repository = local.gcp_artifact_registry.repository
artifact_registry_location = local.gcp_artifact_registry.location
diff --git a/providers/gcp/template_versions.tftest.hcl b/providers/gcp/template_versions.tftest.hcl
index 958be67..4b7ff77 100644
--- a/providers/gcp/template_versions.tftest.hcl
+++ b/providers/gcp/template_versions.tftest.hcl
@@ -38,8 +38,8 @@ run "custom_package_set_merges_only_applicable_template_versions" {
}
assert {
- condition = local.runtime.compose.branch == "v1.1.0"
- error_message = "The GCP entrypoint must inherit the ISLE v1.1.0 template branch when no override is supplied."
+ condition = local.runtime.compose.branch == "v1.3.0"
+ error_message = "The GCP entrypoint must inherit the ISLE v1.3.0 template branch when no override is supplied."
}
assert {
@@ -67,7 +67,7 @@ run "explicit_core_only_package_set_disables_template_plugins" {
assert {
condition = local.runtime.sitectl.packages == tolist(["sitectl"]) && local.runtime.sitectl.package_versions == {
- sitectl = "v1.0.0"
+ sitectl = "v1.8.2"
}
error_message = "The GCP entrypoint must preserve an explicit core-only package set."
}
diff --git a/providers/gcp/variables.tf b/providers/gcp/variables.tf
index 1bfadee..df1ef48 100644
--- a/providers/gcp/variables.tf
+++ b/providers/gcp/variables.tf
@@ -199,6 +199,11 @@ variable "runtime" {
rootfs_archive_sha256 = optional(string, "")
users = optional(map(list(string)), {})
+ disaster_recovery = optional(object({
+ required = optional(bool, false)
+ driver_path = optional(string, "/etc/cloud-compose/libexec/offhost-backup-driver")
+ }), {})
+
compose = optional(object({
primary = optional(string, "")
ingress_port = optional(number, 80)
@@ -308,6 +313,15 @@ variable "runtime" {
error_message = "runtime.rootfs_archive_url and a 64-character runtime.rootfs_archive_sha256 must be supplied together."
}
+ validation {
+ condition = (
+ can(regex("^/[A-Za-z0-9._/+:-]+$", var.runtime.disaster_recovery.driver_path)) &&
+ !strcontains(var.runtime.disaster_recovery.driver_path, "//") &&
+ length(regexall("(^|/)\\.\\.?(/|$)", var.runtime.disaster_recovery.driver_path)) == 0
+ )
+ error_message = "runtime.disaster_recovery.driver_path must be a safe absolute path without whitespace or dot segments."
+ }
+
validation {
condition = alltrue([
for name in keys(var.runtime.extra_env) :
diff --git a/providers/linode/template_versions.tftest.hcl b/providers/linode/template_versions.tftest.hcl
index 460b3a2..14be098 100644
--- a/providers/linode/template_versions.tftest.hcl
+++ b/providers/linode/template_versions.tftest.hcl
@@ -33,8 +33,8 @@ run "custom_package_set_merges_only_applicable_template_versions" {
}
assert {
- condition = local.runtime.compose.branch == "v1.1.0"
- error_message = "The Linode entrypoint must inherit the ISLE v1.1.0 template branch when no override is supplied."
+ condition = local.runtime.compose.branch == "v1.3.0"
+ error_message = "The Linode entrypoint must inherit the ISLE v1.3.0 template branch when no override is supplied."
}
assert {
@@ -65,7 +65,7 @@ run "explicit_core_only_package_set_disables_template_plugins" {
assert {
condition = local.runtime.sitectl.packages == tolist(["sitectl"]) && local.runtime.sitectl.package_versions == {
- sitectl = "v1.0.0"
+ sitectl = "v1.8.2"
}
error_message = "The Linode entrypoint must preserve an explicit core-only package set."
}
diff --git a/providers/linode/variables.tf b/providers/linode/variables.tf
index 725ce0b..8ab7190 100644
--- a/providers/linode/variables.tf
+++ b/providers/linode/variables.tf
@@ -102,6 +102,11 @@ variable "runtime" {
rootfs_archive_sha256 = optional(string, "")
users = optional(map(list(string)), {})
+ disaster_recovery = optional(object({
+ required = optional(bool, false)
+ driver_path = optional(string, "/etc/cloud-compose/libexec/offhost-backup-driver")
+ }), {})
+
compose = optional(object({
primary = optional(string, "")
ingress_port = optional(number, 80)
@@ -210,6 +215,15 @@ variable "runtime" {
error_message = "runtime.rootfs_archive_url and a 64-character runtime.rootfs_archive_sha256 must be supplied together."
}
+ validation {
+ condition = (
+ can(regex("^/[A-Za-z0-9._/+:-]+$", var.runtime.disaster_recovery.driver_path)) &&
+ !strcontains(var.runtime.disaster_recovery.driver_path, "//") &&
+ length(regexall("(^|/)\\.\\.?(/|$)", var.runtime.disaster_recovery.driver_path)) == 0
+ )
+ error_message = "runtime.disaster_recovery.driver_path must be a safe absolute path without whitespace or dot segments."
+ }
+
validation {
condition = alltrue([
for name in keys(var.runtime.extra_env) :
diff --git a/renovate.json5 b/renovate.json5
index 169e5fc..eabf862 100644
--- a/renovate.json5
+++ b/renovate.json5
@@ -4,6 +4,32 @@
'github>libops/renovate-config:default.json5',
],
customManagers: [
+ {
+ customType: 'regex',
+ description: 'Propose reviewed sitectl package promotions for application presets',
+ managerFilePatterns: [
+ '/^templates/apps\.json$/',
+ ],
+ matchStrings: [
+ '"(?sitectl(?:-[a-z0-9-]+)?)"\\s*:\\s*"(?v[0-9]+\\.[0-9]+\\.[0-9]+)"',
+ ],
+ datasourceTemplate: 'github-releases',
+ packageNameTemplate: 'libops/{{{depName}}}',
+ versioningTemplate: 'semver',
+ },
+ {
+ customType: 'regex',
+ description: 'Propose reviewed Compose-template promotions for application presets',
+ managerFilePatterns: [
+ '/^templates/apps\.json$/',
+ ],
+ matchStrings: [
+ '"repo"\\s*:\\s*"https://github\\.com/libops/(?[a-z0-9-]+)(?:\\.git)?"\\s*,\\s*"branch"\\s*:\\s*"(?v[0-9]+\\.[0-9]+\\.[0-9]+)"',
+ ],
+ datasourceTemplate: 'github-tags',
+ packageNameTemplate: 'libops/{{{depName}}}',
+ versioningTemplate: 'semver',
+ },
{
customType: 'regex',
description: 'Update Docker Compose and Buildx Terraform defaults',
diff --git a/rootfs/etc/cloud-compose/bin/cloud-compose-diagnostics.sh b/rootfs/etc/cloud-compose/bin/cloud-compose-diagnostics.sh
new file mode 100755
index 0000000..185b24b
--- /dev/null
+++ b/rootfs/etc/cloud-compose/bin/cloud-compose-diagnostics.sh
@@ -0,0 +1,212 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# This program is reached through an exact sudoers command. Never resolve its
+# child commands from a caller-controlled tool directory.
+readonly PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+export PATH
+
+readonly bootstrap_marker="/var/lib/cloud-compose/bootstrap-complete"
+readonly diagnostics_program="/etc/cloud-compose/bin/cloud-compose-diagnostics.sh"
+readonly jq_program_dir="/etc/cloud-compose/jq"
+readonly process_pattern='[/]home/cloud-compose/run[.]sh|[/]home/cloud-compose/[h]ost-conf[.]sh|[/]home/cloud-compose/[h]ost-init[.]sh|[/]home/cloud-compose/[a]pp-init[.]sh|[/]home/cloud-compose/[i]nstall-dependencies|[a]pt-get|[r]pm-ostree|[d]ocker run|[s]itectl|[g]it clone'
+
+usage() {
+ echo "usage: ${diagnostics_program} state|status|dump" >&2
+}
+
+require_root() {
+ if ((EUID != 0)); then
+ echo "Cloud Compose diagnostics must run as root" >&2
+ exit 1
+ fi
+}
+
+unit_value() {
+ local unit="$1" property="$2" value
+
+ value="$(systemctl show --property="$property" --value -- "$unit" 2>/dev/null || true)"
+ printf '%s\n' "${value:-unknown}"
+}
+
+bootstrap_state() {
+ local bootstrap_load_state bootstrap_active_state bootstrap_sub_state
+ local marker_dir_metadata marker_metadata marker_size marker_payload
+
+ if [[ -f "$bootstrap_marker" && ! -L "$bootstrap_marker" ]]; then
+ marker_dir_metadata="$(stat -c '%u:%g:%a:%F' -- "$(dirname -- "$bootstrap_marker")" 2>/dev/null || true)"
+ marker_metadata="$(stat -c '%u:%g:%a:%h:%F' -- "$bootstrap_marker" 2>/dev/null || true)"
+ marker_size="$(stat -c '%s' -- "$bootstrap_marker" 2>/dev/null || true)"
+ marker_payload=""
+ IFS= read -r marker_payload <"$bootstrap_marker" || true
+ if [[ "$marker_dir_metadata" == "0:0:755:directory" &&
+ "$marker_metadata" == "0:0:644:1:regular file" &&
+ "$marker_size" == "6" && "$marker_payload" == "ready" ]]; then
+ echo complete
+ return 0
+ fi
+ fi
+
+ bootstrap_load_state="$(unit_value cloud-compose-bootstrap.service LoadState)"
+ if [[ "$bootstrap_load_state" == "loaded" ]]; then
+ bootstrap_active_state="$(unit_value cloud-compose-bootstrap.service ActiveState)"
+ bootstrap_sub_state="$(unit_value cloud-compose-bootstrap.service SubState)"
+ case "${bootstrap_active_state}:${bootstrap_sub_state}" in
+ active:* | activating:* | *:auto-restart)
+ echo active
+ return 0
+ ;;
+ esac
+ elif [[ "$bootstrap_load_state" == "not-found" ]] &&
+ systemctl is-active --quiet cloud-compose.service; then
+ # Compatibility with releases that predate the retryable bootstrap
+ # unit and durable readiness marker.
+ echo complete
+ return 0
+ fi
+
+ if systemctl is-active --quiet cloud-final.service; then
+ echo active
+ return 0
+ fi
+ if pgrep -f -- "$process_pattern" >/dev/null; then
+ echo active
+ return 0
+ fi
+
+ echo idle
+ return 1
+}
+
+unit_heartbeat() {
+ local unit main_pid
+
+ for unit in cloud-final.service cloud-compose-bootstrap.service cloud-compose.service; do
+ echo "--- ${unit} state ---"
+ systemctl show --no-pager \
+ --property=LoadState \
+ --property=ActiveState \
+ --property=SubState \
+ --property=Result \
+ --property=MainPID \
+ --property=ExecMainStatus \
+ -- "$unit" 2>&1 || true
+ main_pid="$(unit_value "$unit" MainPID)"
+ if [[ "$main_pid" =~ ^[1-9][0-9]*$ ]]; then
+ ps -p "$main_pid" -o pid=,ppid=,stat=,etime=,comm= 2>/dev/null || true
+ fi
+ done
+
+ echo "--- active bootstrap process state ---"
+ while IFS= read -r process_id; do
+ [[ "$process_id" =~ ^[1-9][0-9]*$ ]] || continue
+ ps -p "$process_id" -o pid=,ppid=,stat=,etime=,comm= 2>/dev/null || true
+ done < <(pgrep -f -- "$process_pattern" 2>/dev/null || true)
+}
+
+diagnostic_status() {
+ local cloud_init_status=0 state
+
+ echo "--- Cloud Compose provisioning heartbeat ---"
+ date -u '+%Y-%m-%dT%H:%M:%SZ'
+ echo "--- cloud-init status ---"
+ if command -v cloud-init >/dev/null 2>&1; then
+ cloud-init status --long || cloud_init_status=$?
+ else
+ echo "cloud-init not installed"
+ fi
+ unit_heartbeat
+ state="$(bootstrap_state 2>/dev/null || true)"
+ echo "bootstrap-state: ${state:-unknown}"
+ return "$cloud_init_status"
+}
+
+tail_regular_file() {
+ local label="$1" path="$2" lines="$3"
+
+ echo "--- ${label} ---"
+ if [[ -f "$path" && ! -L "$path" ]]; then
+ tail -n "$lines" -- "$path" 2>&1 || true
+ else
+ echo "${label} is not present as a regular file"
+ fi
+}
+
+dump_compose_state() {
+ local manifest="/home/cloud-compose/compose-projects.json"
+ local docker_path encoded row app project_dir
+
+ echo "--- docker ps ---"
+ docker_path="$(command -v docker || true)"
+ if [[ -z "$docker_path" ]]; then
+ echo "docker is not installed"
+ return 0
+ fi
+ "$docker_path" ps -a 2>&1 || true
+
+ echo "--- docker compose project state ---"
+ if [[ ! -f "$manifest" || -L "$manifest" ]] ||
+ ! jq -e -f "$jq_program_dir/diagnostics-validate-compose-projects.jq" \
+ "$manifest" >/dev/null 2>&1; then
+ echo "Compose project manifest is unavailable or invalid"
+ return 0
+ fi
+
+ while IFS= read -r encoded; do
+ [[ -n "$encoded" ]] || continue
+ row="$(printf '%s' "$encoded" | base64 -d)" || continue
+ app="$(jq -er -f "$jq_program_dir/diagnostics-entry-app.jq" <<<"$row" 2>/dev/null || true)"
+ project_dir="$(jq -er -f "$jq_program_dir/diagnostics-entry-project-dir.jq" <<<"$row" 2>/dev/null || true)"
+ if [[ -z "$app" || -z "$project_dir" || ! -d "$project_dir" || -L "$project_dir" ]]; then
+ echo "Skipping unavailable or unsafe Compose project: ${app:-unknown}"
+ continue
+ fi
+ echo "--- docker compose ps: ${app} ---"
+ runuser -u cloud-compose -- env HOME=/home/cloud-compose \
+ "$docker_path" compose --project-directory "$project_dir" ps 2>&1 || true
+ done < <(jq -r -f "$jq_program_dir/diagnostics-project-entries.jq" "$manifest")
+}
+
+diagnostic_dump() {
+ diagnostic_status || true
+ tail_regular_file "/var/log/cloud-init-output.log" "/var/log/cloud-init-output.log" 400
+ tail_regular_file "/var/log/cloud-init.log" "/var/log/cloud-init.log" 400
+ echo "--- cloud-init runcmd ---"
+ if [[ -f /var/lib/cloud/instance/scripts/runcmd &&
+ ! -L /var/lib/cloud/instance/scripts/runcmd ]]; then
+ sed -n '1,240p' /var/lib/cloud/instance/scripts/runcmd 2>&1 || true
+ else
+ echo "cloud-init runcmd is not present as a regular file"
+ fi
+ echo "--- cloud-compose bootstrap unit ---"
+ journalctl -u cloud-compose-bootstrap --no-pager -n 400 2>&1 || true
+ tail_regular_file "legacy cloud-compose bootstrap log" "/home/cloud-compose/run.log" 400
+ echo "--- cloud-compose unit ---"
+ journalctl -u cloud-compose --no-pager -n 300 2>&1 || true
+ echo "--- lifecycle lock permissions ---"
+ stat -Lc '%A %a %U:%G %u:%g %n' \
+ /run/lock/cloud-compose \
+ /run/lock/cloud-compose/lifecycle.lock 2>&1 || true
+ dump_compose_state
+}
+
+main() {
+ require_root
+ if [[ "$#" -ne 1 ]]; then
+ usage
+ return 2
+ fi
+
+ case "$1" in
+ state) bootstrap_state ;;
+ status) diagnostic_status ;;
+ dump) diagnostic_dump ;;
+ *)
+ usage
+ return 2
+ ;;
+ esac
+}
+
+main "$@"
diff --git a/rootfs/etc/cloud-compose/jq/diagnostics-entry-app.jq b/rootfs/etc/cloud-compose/jq/diagnostics-entry-app.jq
new file mode 100644
index 0000000..edb6cdb
--- /dev/null
+++ b/rootfs/etc/cloud-compose/jq/diagnostics-entry-app.jq
@@ -0,0 +1 @@
+.key | select(type == "string" and length > 0)
diff --git a/rootfs/etc/cloud-compose/jq/diagnostics-entry-project-dir.jq b/rootfs/etc/cloud-compose/jq/diagnostics-entry-project-dir.jq
new file mode 100644
index 0000000..f0b9664
--- /dev/null
+++ b/rootfs/etc/cloud-compose/jq/diagnostics-entry-project-dir.jq
@@ -0,0 +1 @@
+.value.project_dir | select(type == "string" and startswith("/mnt/disks/data/"))
diff --git a/rootfs/etc/cloud-compose/jq/diagnostics-project-entries.jq b/rootfs/etc/cloud-compose/jq/diagnostics-project-entries.jq
new file mode 100644
index 0000000..4785439
--- /dev/null
+++ b/rootfs/etc/cloud-compose/jq/diagnostics-project-entries.jq
@@ -0,0 +1 @@
+to_entries[] | @base64
diff --git a/rootfs/etc/cloud-compose/jq/diagnostics-validate-compose-projects.jq b/rootfs/etc/cloud-compose/jq/diagnostics-validate-compose-projects.jq
new file mode 100644
index 0000000..a25876a
--- /dev/null
+++ b/rootfs/etc/cloud-compose/jq/diagnostics-validate-compose-projects.jq
@@ -0,0 +1 @@
+type == "object"
diff --git a/rootfs/etc/cloud-compose/jq/dr-backup-completed-at.jq b/rootfs/etc/cloud-compose/jq/dr-backup-completed-at.jq
new file mode 100644
index 0000000..9822734
--- /dev/null
+++ b/rootfs/etc/cloud-compose/jq/dr-backup-completed-at.jq
@@ -0,0 +1 @@
+.completed_at
diff --git a/rootfs/etc/cloud-compose/jq/dr-backup-remote-id.jq b/rootfs/etc/cloud-compose/jq/dr-backup-remote-id.jq
new file mode 100644
index 0000000..f142fb1
--- /dev/null
+++ b/rootfs/etc/cloud-compose/jq/dr-backup-remote-id.jq
@@ -0,0 +1 @@
+.remote_id
diff --git a/rootfs/etc/cloud-compose/jq/dr-restore-completed-at.jq b/rootfs/etc/cloud-compose/jq/dr-restore-completed-at.jq
new file mode 100644
index 0000000..9822734
--- /dev/null
+++ b/rootfs/etc/cloud-compose/jq/dr-restore-completed-at.jq
@@ -0,0 +1 @@
+.completed_at
diff --git a/rootfs/etc/cloud-compose/jq/dr-restore-recovery-id.jq b/rootfs/etc/cloud-compose/jq/dr-restore-recovery-id.jq
new file mode 100644
index 0000000..fcaabb5
--- /dev/null
+++ b/rootfs/etc/cloud-compose/jq/dr-restore-recovery-id.jq
@@ -0,0 +1 @@
+.recovery_id
diff --git a/rootfs/etc/cloud-compose/jq/dr-validate-backup-receipt.jq b/rootfs/etc/cloud-compose/jq/dr-validate-backup-receipt.jq
new file mode 100644
index 0000000..628e5a3
--- /dev/null
+++ b/rootfs/etc/cloud-compose/jq/dr-validate-backup-receipt.jq
@@ -0,0 +1,16 @@
+type == "object" and length == 10 and
+.schema_version == 1 and
+.kind == "cloud-compose.offhost-backup-receipt" and
+.operation_id == $operation_id and
+.manifest_sha256 == $manifest_sha256 and
+.status == "succeeded" and
+.encrypted == true and
+.off_host == true and
+(.completed_at | type == "string" and length == 20 and
+ (explode | all(.[]; . >= 32 and . != 127))) and
+(.remote_id | type == "string" and length >= 1 and length <= 512 and
+ (explode | all(.[]; . >= 32 and . != 127))) and
+(.coverage | type == "object" and length == 3 and
+ .database == true and
+ .application_files == true and
+ .volume_topology == true)
diff --git a/rootfs/etc/cloud-compose/jq/dr-validate-restore-proof.jq b/rootfs/etc/cloud-compose/jq/dr-validate-restore-proof.jq
new file mode 100644
index 0000000..461bd52
--- /dev/null
+++ b/rootfs/etc/cloud-compose/jq/dr-validate-restore-proof.jq
@@ -0,0 +1,19 @@
+type == "object" and length == 13 and
+.schema_version == 1 and
+.kind == "cloud-compose.restore-test-proof" and
+.test_id == $test_id and
+.source_manifest_sha256 == $manifest_sha256 and
+.source_receipt_sha256 == $receipt_sha256 and
+.status == "succeeded" and
+.disposable_recovery == true and
+.recovery_destroyed == true and
+.integrity_verified == true and
+(.completed_at | type == "string" and length == 20 and
+ (explode | all(.[]; . >= 32 and . != 127))) and
+(.recovery_id | type == "string" and length >= 1 and length <= 512 and
+ (explode | all(.[]; . >= 32 and . != 127))) and
+(.coverage | type == "object" and length == 3 and
+ .database == true and
+ .application_files == true and
+ .volume_topology == true) and
+(.source_encrypted == true)
diff --git a/rootfs/etc/cloud-compose/jq/offhost-bind-sources.jq b/rootfs/etc/cloud-compose/jq/offhost-bind-sources.jq
new file mode 100644
index 0000000..880c92b
--- /dev/null
+++ b/rootfs/etc/cloud-compose/jq/offhost-bind-sources.jq
@@ -0,0 +1 @@
+.services[].volumes[]? | select(.type == "bind") | .source
diff --git a/rootfs/etc/cloud-compose/jq/offhost-build-application-coverage.jq b/rootfs/etc/cloud-compose/jq/offhost-build-application-coverage.jq
new file mode 100644
index 0000000..77633bd
--- /dev/null
+++ b/rootfs/etc/cloud-compose/jq/offhost-build-application-coverage.jq
@@ -0,0 +1,27 @@
+{
+ name: $app,
+ databases: [{
+ engine: "mariadb",
+ format: "sql.gz",
+ local_recovery_artifact: $dump_path,
+ sha256: $dump_sha256,
+ bytes: $dump_bytes
+ }],
+ application_files: {
+ roots: [$project_dir],
+ bind_mounts: [
+ .services | to_entries[] as $service |
+ ($service.value.volumes // [])[] |
+ select(.type == "bind") |
+ {service: $service.key, source: .source, target: .target, read_only: (.read_only // false)}
+ ] | sort_by(.service, .source, .target)
+ },
+ volume_topology: {
+ declared_named_volumes: ((.volumes // {}) | keys | sort),
+ service_mounts: [
+ .services | to_entries[] as $service |
+ ($service.value.volumes // [])[] |
+ {service: $service.key, type: .type, source: (.source // ""), target: .target, read_only: (.read_only // false)}
+ ] | sort_by(.service, .type, .source, .target)
+ }
+}
diff --git a/rootfs/etc/cloud-compose/jq/offhost-build-manifest.jq b/rootfs/etc/cloud-compose/jq/offhost-build-manifest.jq
new file mode 100644
index 0000000..a1d4451
--- /dev/null
+++ b/rootfs/etc/cloud-compose/jq/offhost-build-manifest.jq
@@ -0,0 +1,10 @@
+{
+ schema_version: 1,
+ kind: "cloud-compose.offhost-backup-manifest",
+ operation_id: $operation_id,
+ backup_date: $backup_date,
+ provider: $provider,
+ instance: $instance,
+ required_coverage: ["database", "application_files", "volume_topology"],
+ applications: (sort_by(.name))
+}
diff --git a/rootfs/etc/cloud-compose/jq/offhost-manifest-app-digests.jq b/rootfs/etc/cloud-compose/jq/offhost-manifest-app-digests.jq
new file mode 100644
index 0000000..c3cef5b
--- /dev/null
+++ b/rootfs/etc/cloud-compose/jq/offhost-manifest-app-digests.jq
@@ -0,0 +1 @@
+.applications[] | [.name, .databases[0].sha256] | @tsv
diff --git a/rootfs/etc/cloud-compose/jq/offhost-validate-bind-roots.jq b/rootfs/etc/cloud-compose/jq/offhost-validate-bind-roots.jq
new file mode 100644
index 0000000..3d8ac7b
--- /dev/null
+++ b/rootfs/etc/cloud-compose/jq/offhost-validate-bind-roots.jq
@@ -0,0 +1,8 @@
+all(.services[].volumes[]?;
+ .type != "bind" or
+ (.source | type == "string" and
+ (. == $data_root or startswith($data_root + "/") or
+ . == $volumes_root or startswith($volumes_root + "/")) and
+ (explode | all(.[]; . >= 32 and . != 127)) and
+ (contains("//") | not) and
+ length > 0))
diff --git a/rootfs/etc/cloud-compose/jq/offhost-validate-compose-config.jq b/rootfs/etc/cloud-compose/jq/offhost-validate-compose-config.jq
new file mode 100644
index 0000000..12b0534
--- /dev/null
+++ b/rootfs/etc/cloud-compose/jq/offhost-validate-compose-config.jq
@@ -0,0 +1,11 @@
+type == "object" and
+(.services | type == "object" and length > 0) and
+all(.services | to_entries[];
+ ((.value.volumes // []) | type == "array") and
+ all((.value.volumes // [])[];
+ type == "object" and
+ (.type | type == "string") and
+ (.type == "volume" or .type == "bind" or .type == "tmpfs") and
+ ((.source // "") | type == "string") and
+ ((.target // "") | type == "string" and length > 0))) and
+((.volumes // {}) | type == "object")
diff --git a/rootfs/etc/cloud-compose/jq/offhost-validate-manifest.jq b/rootfs/etc/cloud-compose/jq/offhost-validate-manifest.jq
new file mode 100644
index 0000000..f196ef2
--- /dev/null
+++ b/rootfs/etc/cloud-compose/jq/offhost-validate-manifest.jq
@@ -0,0 +1,14 @@
+.schema_version == 1 and
+.kind == "cloud-compose.offhost-backup-manifest" and
+(.applications | type == "array" and length == $app_count) and
+all(.applications[];
+ (.name | type == "string" and length >= 1 and length <= 63 and
+ (explode | all(.[]; . >= 32 and . != 127))) and
+ (.databases | length == 1) and
+ (.databases[0].sha256 | type == "string" and length == 64 and
+ (explode | all(.[]; . >= 32 and . != 127))) and
+ (.databases[0].bytes | type == "number" and . > 0) and
+ (.application_files.roots | type == "array" and length > 0) and
+ (.application_files.bind_mounts | type == "array") and
+ (.volume_topology.declared_named_volumes | type == "array") and
+ (.volume_topology.service_mounts | type == "array"))
diff --git a/rootfs/etc/cloud-compose/libexec/bootstrap-required.sh b/rootfs/etc/cloud-compose/libexec/bootstrap-required.sh
new file mode 100755
index 0000000..c0ba4fb
--- /dev/null
+++ b/rootfs/etc/cloud-compose/libexec/bootstrap-required.sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+set -uo pipefail
+
+# shellcheck disable=SC1091
+if ! source /etc/cloud-compose/libexec/bootstrap-security.sh; then
+ echo "Cloud Compose bootstrap security helper could not be loaded" >&2
+ exit 255
+fi
+
+if ! cloud_compose_bootstrap_require_root; then
+ exit 255
+fi
+if cloud_compose_bootstrap_marker_ready; then
+ # ExecCondition exit 1 skips an already-complete oneshot without marking it
+ # failed. Missing or invalid evidence returns zero below and must converge.
+ exit 1
+fi
+exit 0
diff --git a/rootfs/etc/cloud-compose/libexec/bootstrap-security.sh b/rootfs/etc/cloud-compose/libexec/bootstrap-security.sh
new file mode 100755
index 0000000..27e77c2
--- /dev/null
+++ b/rootfs/etc/cloud-compose/libexec/bootstrap-security.sh
@@ -0,0 +1,123 @@
+#!/usr/bin/env bash
+
+readonly CLOUD_COMPOSE_BOOTSTRAP_MARKER="/var/lib/cloud-compose/bootstrap-complete"
+readonly CLOUD_COMPOSE_RUNTIME_HOME="/home/cloud-compose"
+
+cloud_compose_bootstrap_require_root() {
+ if ((EUID != 0)); then
+ echo "Cloud Compose bootstrap control must run as root" >&2
+ return 1
+ fi
+}
+
+cloud_compose_bootstrap_marker_ready() {
+ local marker="${1:-$CLOUD_COMPOSE_BOOTSTRAP_MARKER}"
+ local marker_dir marker_dir_metadata marker_metadata marker_size payload
+
+ marker_dir="$(dirname -- "$marker")"
+ [[ -d "$marker_dir" && ! -L "$marker_dir" && -f "$marker" && ! -L "$marker" ]] || return 1
+ marker_dir_metadata="$(stat -c '%u:%g:%a:%F' -- "$marker_dir")" || return 1
+ marker_metadata="$(stat -c '%u:%g:%a:%h:%F' -- "$marker")" || return 1
+ marker_size="$(stat -c '%s' -- "$marker")" || return 1
+ [[ "$marker_dir_metadata" == "0:0:755:directory" &&
+ "$marker_metadata" == "0:0:644:1:regular file" &&
+ "$marker_size" == "6" ]] || return 1
+ IFS= read -r payload <"$marker" || return 1
+ [[ "$payload" == "ready" ]]
+}
+
+cloud_compose_secure_runtime_home() {
+ local home_metadata program dispatcher metadata owner_uid group_gid mode links kind
+ local -a required_programs programs dispatchers control_inputs
+
+ cloud_compose_bootstrap_require_root || return 1
+ if [[ -L "$CLOUD_COMPOSE_RUNTIME_HOME" || ! -d "$CLOUD_COMPOSE_RUNTIME_HOME" ]]; then
+ echo "Cloud Compose runtime home is missing or redirected" >&2
+ return 1
+ fi
+ home_metadata="$(stat -c '%a:%F' -- "$CLOUD_COMPOSE_RUNTIME_HOME")" || return 1
+ if [[ ! "$home_metadata" =~ ^[0-7]{3,4}:directory$ ]]; then
+ echo "Cloud Compose runtime home is not a real directory" >&2
+ return 1
+ fi
+
+ # Close the historical user-owned parent boundary before inspecting any
+ # program beneath it. A replacement file retains its non-root ownership and
+ # is rejected below; a now-unwritable parent prevents another replacement.
+ chown 0:0 "$CLOUD_COMPOSE_RUNTIME_HOME" || return 1
+ chmod 0755 "$CLOUD_COMPOSE_RUNTIME_HOME" || return 1
+
+ required_programs=(
+ "$CLOUD_COMPOSE_RUNTIME_HOME/run.sh"
+ "$CLOUD_COMPOSE_RUNTIME_HOME/profile.sh"
+ "$CLOUD_COMPOSE_RUNTIME_HOME/bootstrap-helpers.sh"
+ "$CLOUD_COMPOSE_RUNTIME_HOME/host-conf.sh"
+ "$CLOUD_COMPOSE_RUNTIME_HOME/host-init.sh"
+ "$CLOUD_COMPOSE_RUNTIME_HOME/converge-app-filesystems.sh"
+ "$CLOUD_COMPOSE_RUNTIME_HOME/prepare-app-sources.sh"
+ "$CLOUD_COMPOSE_RUNTIME_HOME/rotate-keys-daily.sh"
+ "$CLOUD_COMPOSE_RUNTIME_HOME/vault-agent-init.sh"
+ "$CLOUD_COMPOSE_RUNTIME_HOME/app-init.sh"
+ "$CLOUD_COMPOSE_RUNTIME_HOME/init"
+ "$CLOUD_COMPOSE_RUNTIME_HOME/up"
+ "$CLOUD_COMPOSE_RUNTIME_HOME/down"
+ "$CLOUD_COMPOSE_RUNTIME_HOME/rollout"
+ )
+ for program in "${required_programs[@]}"; do
+ if [[ -L "$program" || ! -f "$program" ]]; then
+ echo "Required Cloud Compose bootstrap program is missing or redirected: $program" >&2
+ return 1
+ fi
+ done
+
+ shopt -s nullglob
+ programs=("$CLOUD_COMPOSE_RUNTIME_HOME"/*.sh)
+ shopt -u nullglob
+ for program in "${programs[@]}"; do
+ if [[ -L "$program" || ! -f "$program" ]]; then
+ echo "Cloud Compose bootstrap program is not a regular file: $program" >&2
+ return 1
+ fi
+ metadata="$(stat -c '%u:%g:%a:%h:%F' -- "$program")" || return 1
+ IFS=: read -r owner_uid group_gid mode links kind <<<"$metadata"
+ if [[ "$owner_uid" != "0" || "$group_gid" != "0" || "$links" != "1" || "$kind" != "regular file" ||
+ ! "$mode" =~ ^[0-7]{3,4}$ || $((8#$mode & 0022)) -ne 0 ]]; then
+ echo "Cloud Compose bootstrap program is not root-controlled: $program" >&2
+ return 1
+ fi
+ chown 0:0 "$program" || return 1
+ chmod 0755 "$program" || return 1
+ done
+
+ control_inputs=(
+ "$CLOUD_COMPOSE_RUNTIME_HOME/.env"
+ "$CLOUD_COMPOSE_RUNTIME_HOME/compose-projects.json"
+ "$CLOUD_COMPOSE_RUNTIME_HOME/application-env.json"
+ "$CLOUD_COMPOSE_RUNTIME_HOME/managed-runtime-artifacts.tsv"
+ )
+ for program in "${control_inputs[@]}"; do
+ if [[ -L "$program" || ! -f "$program" ]]; then
+ echo "Required Cloud Compose control input is missing or redirected: $program" >&2
+ return 1
+ fi
+ metadata="$(stat -c '%u:%g:%a:%h:%F' -- "$program")" || return 1
+ IFS=: read -r owner_uid group_gid mode links kind <<<"$metadata"
+ if [[ "$owner_uid" != "0" || "$links" != "1" || "$kind" != "regular file" ||
+ ! "$mode" =~ ^[0-7]{3,4}$ || $((8#$mode & 0022)) -ne 0 ]]; then
+ echo "Cloud Compose control input is not root-controlled: $program" >&2
+ return 1
+ fi
+ done
+
+ dispatchers=(init up down rollout)
+ for dispatcher in "${dispatchers[@]}"; do
+ program="$CLOUD_COMPOSE_RUNTIME_HOME/$dispatcher"
+ metadata="$(stat -c '%u:%g:%a:%h:%F' -- "$program")" || return 1
+ IFS=: read -r owner_uid group_gid mode links kind <<<"$metadata"
+ if [[ "$owner_uid" != "0" || "$links" != "1" || "$kind" != "regular file" ||
+ ! "$mode" =~ ^[0-7]{3,4}$ || $((8#$mode & 0022)) -ne 0 ]]; then
+ echo "Cloud Compose lifecycle dispatcher is not root-controlled: $program" >&2
+ return 1
+ fi
+ done
+}
diff --git a/rootfs/etc/cloud-compose/libexec/build-cos-make.sh b/rootfs/etc/cloud-compose/libexec/build-cos-make.sh
new file mode 100755
index 0000000..f4f091a
--- /dev/null
+++ b/rootfs/etc/cloud-compose/libexec/build-cos-make.sh
@@ -0,0 +1,46 @@
+#!/bin/sh
+
+set -eux
+
+MAKE_VERSION="4.4.1"
+MAKE_SHA256="dd16fb1d67bfab79a72f5e8390735c49e3e8e70b4945a15ab1f81ddb78658fb3"
+
+# A single Alpine CDN outage must not make a healthy VM replacement fail.
+# These are HTTPS endpoints from the Alpine official mirror list; apk still
+# verifies the signed indexes and packages with the keys baked into the pinned
+# image.
+alpine_mirrors="
+ https://dl-cdn.alpinelinux.org/alpine
+ https://mirror.math.princeton.edu/pub/alpinelinux
+ https://mirror.fel.cvut.cz/alpine
+"
+packages_installed=false
+for alpine_mirror in ${alpine_mirrors}; do
+ printf "%s\n%s\n" \
+ "${alpine_mirror}/v3.22/main" \
+ "${alpine_mirror}/v3.22/community" \
+ >/etc/apk/repositories
+ rm -f /var/cache/apk/*
+ if apk update && apk add build-base curl make tar; then
+ packages_installed=true
+ break
+ fi
+ echo "Alpine package mirror failed: ${alpine_mirror}" >&2
+done
+if [ "${packages_installed}" != true ]; then
+ echo "All configured Alpine package mirrors failed" >&2
+ exit 1
+fi
+
+curl -fsSL --proto "=https" --proto-redir "=https" --tlsv1.2 \
+ --retry 5 --retry-all-errors --retry-delay 2 --retry-max-time 900 \
+ --connect-timeout 10 --max-time 300 \
+ "https://ftp.gnu.org/gnu/make/make-${MAKE_VERSION}.tar.gz" -o /tmp/make.tar.gz
+echo "${MAKE_SHA256} /tmp/make.tar.gz" | sha256sum -c -
+tar -xzf /tmp/make.tar.gz -C /tmp
+cd "/tmp/make-${MAKE_VERSION}"
+LDFLAGS="-static" ./configure --disable-nls
+make -j2
+install -m 0755 make /out/.cloud-compose-make.pending
+/out/.cloud-compose-make.pending --version | grep -Fqm 1 "GNU Make ${MAKE_VERSION}"
+mv -f /out/.cloud-compose-make.pending /out/make
diff --git a/rootfs/etc/cloud-compose/libexec/harden-bootstrap-paths.sh b/rootfs/etc/cloud-compose/libexec/harden-bootstrap-paths.sh
new file mode 100755
index 0000000..eccc403
--- /dev/null
+++ b/rootfs/etc/cloud-compose/libexec/harden-bootstrap-paths.sh
@@ -0,0 +1,106 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+cloud_compose_home="/home/cloud-compose"
+
+if [[ -L "$cloud_compose_home" || ! -d "$cloud_compose_home" ]]; then
+ echo "Cloud Compose home is unavailable or unsafe" >&2
+ exit 1
+fi
+
+# cloud-init creates the operator account before all providers finish placing
+# the checked-in runtime files. Close that initial writable-home window before
+# any root process sources or executes those files. Files must already be
+# root-owned, regular, and unlinked before this script normalizes archive modes;
+# an operator replacement therefore fails closed instead of being blessed.
+chown root:root "$cloud_compose_home"
+chmod 0755 "$cloud_compose_home"
+
+require_root_owned_regular_file() {
+ local path="$1"
+ local metadata
+
+ if [[ -L "$path" || ! -f "$path" ]]; then
+ echo "Unsafe Cloud Compose bootstrap file: $path" >&2
+ exit 1
+ fi
+ metadata="$(stat -c '%u:%h:%F' -- "$path")"
+ if [[ "$metadata" != "0:1:regular file" ]]; then
+ echo "Cloud Compose bootstrap file is not an unlinked root-owned regular file: $path" >&2
+ exit 1
+ fi
+}
+
+shopt -s nullglob
+programs=(
+ "$cloud_compose_home"/*.sh
+ "$cloud_compose_home"/*.jq
+ "$cloud_compose_home"/*.awk
+)
+shopt -u nullglob
+for path in "${programs[@]}"; do
+ require_root_owned_regular_file "$path"
+done
+
+for dispatcher in init up down rollout; do
+ path="${cloud_compose_home}/${dispatcher}"
+ if [[ -L "$path" || ( -e "$path" && ! -f "$path" ) ]]; then
+ echo "Unsafe Cloud Compose dispatcher: $path" >&2
+ exit 1
+ fi
+ if [[ -f "$path" ]]; then
+ require_root_owned_regular_file "$path"
+ fi
+done
+
+for input in .env compose-projects.json application-env.json managed-runtime-artifacts.tsv; do
+ path="${cloud_compose_home}/${input}"
+ if [[ -L "$path" || ( -e "$path" && ! -f "$path" ) ]]; then
+ echo "Unsafe Cloud Compose input: $path" >&2
+ exit 1
+ fi
+ if [[ -f "$path" ]]; then
+ require_root_owned_regular_file "$path"
+ fi
+done
+
+find "$cloud_compose_home" -mindepth 1 -maxdepth 1 -type f -name '*.sh' \
+ -exec chown root:root {} + \
+ -exec chmod 0755 {} +
+find "$cloud_compose_home" -mindepth 1 -maxdepth 1 -type f -name '*.jq' \
+ -exec chown root:root {} + \
+ -exec chmod 0644 {} +
+find "$cloud_compose_home" -mindepth 1 -maxdepth 1 -type f -name '*.awk' \
+ -exec chown root:root {} + \
+ -exec chmod 0644 {} +
+
+for dispatcher in init up down rollout; do
+ path="${cloud_compose_home}/${dispatcher}"
+ if [[ -f "$path" ]]; then
+ chown root:root "$path"
+ chmod 0755 "$path"
+ fi
+done
+
+for input in .env compose-projects.json application-env.json managed-runtime-artifacts.tsv; do
+ path="${cloud_compose_home}/${input}"
+ if [[ -f "$path" ]]; then
+ chown root:cloud-compose "$path"
+ chmod 0640 "$path"
+ fi
+done
+
+if [[ -L "${cloud_compose_home}/bin" ||
+ ( -e "${cloud_compose_home}/bin" && ! -d "${cloud_compose_home}/bin" ) ]]; then
+ echo "Unsafe Cloud Compose command directory" >&2
+ exit 1
+fi
+install -d -m 0755 -o root -g root "${cloud_compose_home}/bin"
+
+home_identity="$(stat -Lc '%U:%G:%a' -- "$cloud_compose_home")"
+bin_identity="$(stat -Lc '%U:%G:%a' -- "${cloud_compose_home}/bin")"
+if [[ "$home_identity" != "root:root:755" || "$bin_identity" != "root:root:755" ]]; then
+ echo "Cloud Compose privileged paths have unsafe ownership or modes" >&2
+ exit 1
+fi
diff --git a/rootfs/etc/cloud-compose/libexec/require-bootstrap-ready.sh b/rootfs/etc/cloud-compose/libexec/require-bootstrap-ready.sh
new file mode 100755
index 0000000..1526af1
--- /dev/null
+++ b/rootfs/etc/cloud-compose/libexec/require-bootstrap-ready.sh
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# shellcheck disable=SC1091
+source /etc/cloud-compose/libexec/bootstrap-security.sh
+
+cloud_compose_bootstrap_require_root
+if ! cloud_compose_bootstrap_marker_ready; then
+ echo "Cloud Compose bootstrap readiness evidence is missing or invalid" >&2
+ exit 1
+fi
diff --git a/rootfs/etc/cloud-compose/libexec/run-bootstrap.sh b/rootfs/etc/cloud-compose/libexec/run-bootstrap.sh
new file mode 100755
index 0000000..eda9d3b
--- /dev/null
+++ b/rootfs/etc/cloud-compose/libexec/run-bootstrap.sh
@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# shellcheck disable=SC1091
+source /etc/cloud-compose/libexec/bootstrap-security.sh
+
+cloud_compose_secure_runtime_home
+exec /bin/bash /home/cloud-compose/run.sh
diff --git a/rootfs/etc/cloud-compose/libexec/run-root-program.sh b/rootfs/etc/cloud-compose/libexec/run-root-program.sh
new file mode 100755
index 0000000..b5e37e3
--- /dev/null
+++ b/rootfs/etc/cloud-compose/libexec/run-root-program.sh
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# shellcheck disable=SC1091
+source /etc/cloud-compose/libexec/bootstrap-security.sh
+
+program="${1:-}"
+if [[ -z "$program" ]]; then
+ echo "A Cloud Compose root program is required" >&2
+ exit 2
+fi
+shift
+
+case "$program" in
+ configure-metadata-firewall.sh | deploy-rollout.sh | docker-prune.sh | \
+ libops-managed-runtime.sh | mount-overlays.sh | offhost-backup.sh | \
+ restore-test.sh | rotate-keys-daily.sh | \
+ vault-agent-readiness.sh) ;;
+ *)
+ echo "Unsupported Cloud Compose root program: $program" >&2
+ exit 2
+ ;;
+esac
+
+cloud_compose_secure_runtime_home
+exec /bin/bash "/home/cloud-compose/$program" "$@"
diff --git a/rootfs/etc/cloud-compose/libexec/start-cloud-compose-bootstrap.sh b/rootfs/etc/cloud-compose/libexec/start-cloud-compose-bootstrap.sh
new file mode 100755
index 0000000..b38af25
--- /dev/null
+++ b/rootfs/etc/cloud-compose/libexec/start-cloud-compose-bootstrap.sh
@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# shellcheck disable=SC1091
+source /etc/cloud-compose/libexec/bootstrap-security.sh
+
+cloud_compose_secure_runtime_home
+exec /bin/bash /home/cloud-compose/start-cloud-compose-bootstrap.sh "$@"
diff --git a/rootfs/etc/systemd/system/cloud-compose-bootstrap.service b/rootfs/etc/systemd/system/cloud-compose-bootstrap.service
index 48db515..1149e95 100644
--- a/rootfs/etc/systemd/system/cloud-compose-bootstrap.service
+++ b/rootfs/etc/systemd/system/cloud-compose-bootstrap.service
@@ -3,13 +3,13 @@ Description=Converge Cloud Compose application bootstrap
Wants=network-online.target
After=network-online.target
RequiresMountsFor=/mnt/disks/data /mnt/disks/volumes /mnt/disks/data/docker/volumes
-ConditionPathExists=!/home/cloud-compose/.cloud-compose-bootstrap-complete
StartLimitIntervalSec=0
[Service]
Type=oneshot
RemainAfterExit=yes
-ExecStart=/bin/bash /home/cloud-compose/run-bootstrap.sh
+ExecCondition=/bin/bash /etc/cloud-compose/libexec/bootstrap-required.sh
+ExecStart=/bin/bash /etc/cloud-compose/libexec/run-bootstrap.sh
Restart=on-failure
RestartSec=30s
TimeoutStartSec=2h
diff --git a/rootfs/etc/systemd/system/cloud-compose-docker-prune.service b/rootfs/etc/systemd/system/cloud-compose-docker-prune.service
index 03f8d29..701132d 100644
--- a/rootfs/etc/systemd/system/cloud-compose-docker-prune.service
+++ b/rootfs/etc/systemd/system/cloud-compose-docker-prune.service
@@ -8,5 +8,5 @@ After=docker.service
Type=oneshot
User=root
Group=root
-ExecStart=/bin/bash /home/cloud-compose/docker-prune.sh
+ExecStart=/bin/bash /etc/cloud-compose/libexec/run-root-program.sh docker-prune.sh
TimeoutStartSec=30min
diff --git a/rootfs/etc/systemd/system/cloud-compose-key-rotation.service b/rootfs/etc/systemd/system/cloud-compose-key-rotation.service
index b9b917d..4a39855 100644
--- a/rootfs/etc/systemd/system/cloud-compose-key-rotation.service
+++ b/rootfs/etc/systemd/system/cloud-compose-key-rotation.service
@@ -6,7 +6,7 @@ RequiresMountsFor=/mnt/disks/data
[Service]
Type=oneshot
-ExecStart=/bin/bash /home/cloud-compose/rotate-keys-daily.sh
+ExecStart=/bin/bash /etc/cloud-compose/libexec/run-root-program.sh rotate-keys-daily.sh
User=root
Group=root
TimeoutStartSec=1h
diff --git a/rootfs/etc/systemd/system/cloud-compose-mariadb-backup.timer b/rootfs/etc/systemd/system/cloud-compose-mariadb-backup.timer
index ef56532..09362b8 100644
--- a/rootfs/etc/systemd/system/cloud-compose-mariadb-backup.timer
+++ b/rootfs/etc/systemd/system/cloud-compose-mariadb-backup.timer
@@ -1,5 +1,5 @@
[Unit]
-Description=Run Cloud Compose MariaDB backups nightly
+Description=Run Cloud Compose local backups and off-host DR handoff nightly
[Timer]
# 9pm-7am EST, splayed per deployment by systemd's stable random delay.
@@ -8,7 +8,7 @@ Persistent=true
RandomizedDelaySec=10h
FixedRandomDelay=true
AccuracySec=1min
-Unit=cloud-compose-mariadb-backup.service
+Unit=cloud-compose-offhost-backup.service
[Install]
WantedBy=timers.target
diff --git a/rootfs/etc/systemd/system/cloud-compose-metadata-firewall-pre.service b/rootfs/etc/systemd/system/cloud-compose-metadata-firewall-pre.service
index fb04f77..e8742ab 100644
--- a/rootfs/etc/systemd/system/cloud-compose-metadata-firewall-pre.service
+++ b/rootfs/etc/systemd/system/cloud-compose-metadata-firewall-pre.service
@@ -6,7 +6,7 @@ Before=docker.service
[Service]
Type=oneshot
-ExecStart=/bin/bash /home/cloud-compose/configure-metadata-firewall.sh pre-docker
+ExecStart=/bin/bash /etc/cloud-compose/libexec/run-root-program.sh configure-metadata-firewall.sh pre-docker
RemainAfterExit=yes
TimeoutStartSec=2min
diff --git a/rootfs/etc/systemd/system/cloud-compose-metadata-firewall.service b/rootfs/etc/systemd/system/cloud-compose-metadata-firewall.service
index 40dec57..0333f5a 100644
--- a/rootfs/etc/systemd/system/cloud-compose-metadata-firewall.service
+++ b/rootfs/etc/systemd/system/cloud-compose-metadata-firewall.service
@@ -6,7 +6,7 @@ PartOf=docker.service
[Service]
Type=oneshot
-ExecStart=/bin/bash /home/cloud-compose/configure-metadata-firewall.sh
+ExecStart=/bin/bash /etc/cloud-compose/libexec/run-root-program.sh configure-metadata-firewall.sh
RemainAfterExit=yes
TimeoutStartSec=2min
diff --git a/rootfs/etc/systemd/system/cloud-compose-offhost-backup.service b/rootfs/etc/systemd/system/cloud-compose-offhost-backup.service
new file mode 100644
index 0000000..c1d4069
--- /dev/null
+++ b/rootfs/etc/systemd/system/cloud-compose-offhost-backup.service
@@ -0,0 +1,17 @@
+[Unit]
+Description=Cloud Compose encrypted off-host disaster-recovery handoff
+Requires=cloud-compose-mariadb-backup.service
+After=cloud-compose-mariadb-backup.service network-online.target
+Wants=network-online.target
+RequiresMountsFor=/mnt/disks/data /mnt/disks/volumes
+
+[Service]
+Type=oneshot
+User=root
+Group=root
+UMask=0077
+ExecStart=/bin/bash /etc/cloud-compose/libexec/run-root-program.sh offhost-backup.sh
+TimeoutStartSec=24h
+StandardOutput=journal
+StandardError=journal
+SyslogIdentifier=cloud-compose-offhost-backup
diff --git a/rootfs/etc/systemd/system/cloud-compose-overlay.service b/rootfs/etc/systemd/system/cloud-compose-overlay.service
index c280d3f..69f8544 100644
--- a/rootfs/etc/systemd/system/cloud-compose-overlay.service
+++ b/rootfs/etc/systemd/system/cloud-compose-overlay.service
@@ -8,7 +8,7 @@ Type=oneshot
User=root
Group=root
EnvironmentFile=-/home/cloud-compose/.env
-ExecStart=/bin/bash /home/cloud-compose/mount-overlays.sh
+ExecStart=/bin/bash /etc/cloud-compose/libexec/run-root-program.sh mount-overlays.sh
RemainAfterExit=yes
[Install]
diff --git a/rootfs/etc/systemd/system/cloud-compose-restore-test.service b/rootfs/etc/systemd/system/cloud-compose-restore-test.service
new file mode 100644
index 0000000..baa3b0a
--- /dev/null
+++ b/rootfs/etc/systemd/system/cloud-compose-restore-test.service
@@ -0,0 +1,16 @@
+[Unit]
+Description=Cloud Compose disposable disaster-recovery restore test
+After=network-online.target docker.service
+Wants=network-online.target
+RequiresMountsFor=/mnt/disks/data /mnt/disks/volumes
+
+[Service]
+Type=oneshot
+User=root
+Group=root
+UMask=0077
+ExecStart=/bin/bash /etc/cloud-compose/libexec/run-root-program.sh restore-test.sh
+TimeoutStartSec=24h
+StandardOutput=journal
+StandardError=journal
+SyslogIdentifier=cloud-compose-restore-test
diff --git a/rootfs/etc/systemd/system/cloud-compose-restore-test.timer b/rootfs/etc/systemd/system/cloud-compose-restore-test.timer
new file mode 100644
index 0000000..831f449
--- /dev/null
+++ b/rootfs/etc/systemd/system/cloud-compose-restore-test.timer
@@ -0,0 +1,13 @@
+[Unit]
+Description=Test Cloud Compose off-host recovery weekly
+
+[Timer]
+OnCalendar=Sun *-*-* 03:00:00
+Persistent=true
+RandomizedDelaySec=6h
+FixedRandomDelay=true
+AccuracySec=1min
+Unit=cloud-compose-restore-test.service
+
+[Install]
+WantedBy=timers.target
diff --git a/rootfs/etc/systemd/system/cloud-compose-vault-agent.service b/rootfs/etc/systemd/system/cloud-compose-vault-agent.service
index d3d1f63..c2e74f9 100644
--- a/rootfs/etc/systemd/system/cloud-compose-vault-agent.service
+++ b/rootfs/etc/systemd/system/cloud-compose-vault-agent.service
@@ -9,10 +9,10 @@ RequiresMountsFor=/mnt/disks/data
User=root
Group=root
EnvironmentFile=-/etc/default/vault-agent
-ExecStartPre=/bin/bash /home/cloud-compose/vault-agent-readiness.sh prepare
+ExecStartPre=/bin/bash /etc/cloud-compose/libexec/run-root-program.sh vault-agent-readiness.sh prepare
ExecStart=/usr/local/bin/vault agent -config=/etc/vault-agent.d/cloud-compose.hcl
-ExecStartPost=/bin/bash /home/cloud-compose/vault-agent-readiness.sh wait
-ExecStopPost=/bin/bash /home/cloud-compose/vault-agent-readiness.sh clear
+ExecStartPost=/bin/bash /etc/cloud-compose/libexec/run-root-program.sh vault-agent-readiness.sh wait
+ExecStopPost=/bin/bash /etc/cloud-compose/libexec/run-root-program.sh vault-agent-readiness.sh clear
Restart=on-failure
RestartSec=10s
TimeoutStartSec=2min
diff --git a/rootfs/etc/systemd/system/cloud-compose.service b/rootfs/etc/systemd/system/cloud-compose.service
index 2cd1381..83ead84 100644
--- a/rootfs/etc/systemd/system/cloud-compose.service
+++ b/rootfs/etc/systemd/system/cloud-compose.service
@@ -18,7 +18,10 @@ ExecStart=/bin/bash /home/cloud-compose/up
ExecStop=/bin/bash /home/cloud-compose/down
Restart=on-failure
RestartSec=30s
-TimeoutStartSec=1h
+# run.sh allows application convergence to take 75 minutes. Keep systemd's
+# ceiling above that application budget so the service manager cannot kill a
+# healthy first boot before the caller's bounded wait reports its own result.
+TimeoutStartSec=90min
TimeoutStopSec=15min
[Install]
diff --git a/rootfs/etc/systemd/system/libops-managed-runtime.service b/rootfs/etc/systemd/system/libops-managed-runtime.service
index 5ddf8ec..94d6781 100644
--- a/rootfs/etc/systemd/system/libops-managed-runtime.service
+++ b/rootfs/etc/systemd/system/libops-managed-runtime.service
@@ -8,7 +8,7 @@ RequiresMountsFor=/mnt/disks/data /mnt/disks/volumes /mnt/disks/data/docker/volu
Type=oneshot
User=root
Group=root
-ExecStart=/bin/bash /home/cloud-compose/libops-managed-runtime.sh update
+ExecStart=/bin/bash /etc/cloud-compose/libexec/run-root-program.sh libops-managed-runtime.sh update
TimeoutStartSec=1h
StandardOutput=journal
StandardError=journal
diff --git a/rootfs/etc/tmpfiles.d/cloud-compose.conf b/rootfs/etc/tmpfiles.d/cloud-compose.conf
index 9c5eb15..78617d7 100644
--- a/rootfs/etc/tmpfiles.d/cloud-compose.conf
+++ b/rootfs/etc/tmpfiles.d/cloud-compose.conf
@@ -1,2 +1,5 @@
+d /var/lib/cloud-compose 0755 root root -
+d /home/cloud-compose 0755 root root -
d /run/lock/cloud-compose 0750 root cloud-compose -
f /run/lock/cloud-compose/lifecycle.lock 0660 root cloud-compose -
+d /mnt/disks/data/.cloud-compose-disaster-recovery 0700 root root -
diff --git a/rootfs/home/cloud-compose/assert-app-initialized.sh b/rootfs/home/cloud-compose/assert-app-initialized.sh
index 6434942..276e4aa 100644
--- a/rootfs/home/cloud-compose/assert-app-initialized.sh
+++ b/rootfs/home/cloud-compose/assert-app-initialized.sh
@@ -5,7 +5,7 @@ set -euo pipefail
# shellcheck disable=SC1090
source "${CLOUD_COMPOSE_BOOTSTRAP_HELPERS_PATH:-/home/cloud-compose/bootstrap-helpers.sh}"
-durable_marker="${CLOUD_COMPOSE_BOOTSTRAP_COMPLETE_MARKER:-/home/cloud-compose/.cloud-compose-bootstrap-complete}"
+durable_marker="${CLOUD_COMPOSE_BOOTSTRAP_COMPLETE_MARKER:-/var/lib/cloud-compose/bootstrap-complete}"
boot_marker="${CLOUD_COMPOSE_APP_INIT_MARKER:-/run/cloud-compose-app-init-complete}"
if cloud_compose_marker_exists "$durable_marker" ||
diff --git a/rootfs/home/cloud-compose/bootstrap-helpers.sh b/rootfs/home/cloud-compose/bootstrap-helpers.sh
index 4a4c774..0e4cfa9 100644
--- a/rootfs/home/cloud-compose/bootstrap-helpers.sh
+++ b/rootfs/home/cloud-compose/bootstrap-helpers.sh
@@ -1,9 +1,17 @@
#!/usr/bin/env bash
cloud_compose_marker_exists() {
- local marker="$1"
-
- [[ -f "$marker" && ! -L "$marker" ]]
+ local marker="$1" marker_size payload
+
+ [[ -f "$marker" && ! -L "$marker" ]] || return 1
+ if [[ "$marker" == "/var/lib/cloud-compose/bootstrap-complete" ]]; then
+ [[ "$(stat -c '%u:%g:%a:%F' -- "$(dirname -- "$marker")")" == "0:0:755:directory" &&
+ "$(stat -c '%u:%g:%a:%h:%F' -- "$marker")" == "0:0:644:1:regular file" ]] || return 1
+ marker_size="$(stat -c '%s' -- "$marker")" || return 1
+ [[ "$marker_size" == "6" ]] || return 1
+ IFS= read -r payload <"$marker" || return 1
+ [[ "$payload" == "ready" ]]
+ fi
}
cloud_compose_should_run_app_init() {
@@ -23,12 +31,25 @@ cloud_compose_publish_marker() (
echo "Unsafe Cloud Compose marker directory: $marker_dir" >&2
return 1
fi
+ if [[ "$marker" == "/var/lib/cloud-compose/bootstrap-complete" &&
+ ( "$EUID" != "0" ||
+ "$(stat -c '%u:%g:%a:%F' -- "$marker_dir")" != "0:0:755:directory" ) ]]; then
+ echo "Durable Cloud Compose readiness requires a root-owned state directory" >&2
+ return 1
+ fi
umask 022
tmp_marker="$(mktemp "${marker}.tmp.XXXXXXXXXX")" || return 1
if ! printf 'ready\n' >"$tmp_marker" ||
- ! chmod 0644 "$tmp_marker" ||
- ! mv -fT -- "$tmp_marker" "$marker"; then
+ ! chmod 0644 "$tmp_marker"; then
+ rm -f -- "$tmp_marker"
+ return 1
+ fi
+ if ((EUID == 0)) && ! chown 0:0 "$tmp_marker"; then
+ rm -f -- "$tmp_marker"
+ return 1
+ fi
+ if ! mv -fT -- "$tmp_marker" "$marker"; then
rm -f -- "$tmp_marker"
return 1
fi
diff --git a/rootfs/home/cloud-compose/compose-apps.sh b/rootfs/home/cloud-compose/compose-apps.sh
index d1834b9..81438ef 100644
--- a/rootfs/home/cloud-compose/compose-apps.sh
+++ b/rootfs/home/cloud-compose/compose-apps.sh
@@ -6,6 +6,8 @@ COMPOSE_PROJECTS_FILE="${COMPOSE_PROJECTS_FILE:-/home/cloud-compose/compose-proj
COMPOSE_APPS_ENV_DIR="${COMPOSE_APPS_ENV_DIR:-/home/cloud-compose/apps}"
COMPOSE_APPS_STATE_DIR="${COMPOSE_APPS_STATE_DIR:-/home/cloud-compose/state}"
CLOUD_COMPOSE_DATA_ROOT="${CLOUD_COMPOSE_DATA_ROOT:-/mnt/disks/data}"
+compose_apps_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+COMPOSE_SECRET_FILES_PROGRAM="${CLOUD_COMPOSE_COMPOSE_SECRET_FILES_PROGRAM:-$compose_apps_dir/compose-secret-files.awk}"
shell_env_line() {
local name="$1"
@@ -570,27 +572,12 @@ EOF
compose_secret_files() {
local compose_file
- for compose_file in docker-compose.yaml docker-compose.yml; do
+ for compose_file in compose.yaml compose.yml docker-compose.yaml docker-compose.yml; do
if [ ! -f "$compose_file" ]; then
continue
fi
- awk '
- /^[[:space:]]*services:/ { in_secrets = 0 }
- /^[^[:space:]][^:]*:/ {
- if ($0 ~ /^secrets:/) {
- in_secrets = 1
- } else if (in_secrets) {
- in_secrets = 0
- }
- }
- in_secrets && /^[[:space:]]*file:[[:space:]]*/ {
- value = $0
- sub(/^[[:space:]]*file:[[:space:]]*/, "", value)
- gsub(/^["'\'']|["'\'']$/, "", value)
- print value
- }
- ' "$compose_file"
+ awk -f "$COMPOSE_SECRET_FILES_PROGRAM" "$compose_file"
done | sort -u
}
diff --git a/rootfs/home/cloud-compose/compose-secret-files.awk b/rootfs/home/cloud-compose/compose-secret-files.awk
new file mode 100644
index 0000000..1fc8827
--- /dev/null
+++ b/rootfs/home/cloud-compose/compose-secret-files.awk
@@ -0,0 +1,18 @@
+/^[[:space:]]*services:/ {
+ in_secrets = 0
+}
+
+/^[^[:space:]][^:]*:/ {
+ if ($0 ~ /^secrets:/) {
+ in_secrets = 1
+ } else if (in_secrets) {
+ in_secrets = 0
+ }
+}
+
+in_secrets && /^[[:space:]]*file:[[:space:]]*/ {
+ value = $0
+ sub(/^[[:space:]]*file:[[:space:]]*/, "", value)
+ gsub(/^["']|["']$/, "", value)
+ print value
+}
diff --git a/rootfs/home/cloud-compose/disaster-recovery-lib.sh b/rootfs/home/cloud-compose/disaster-recovery-lib.sh
new file mode 100644
index 0000000..8cba3df
--- /dev/null
+++ b/rootfs/home/cloud-compose/disaster-recovery-lib.sh
@@ -0,0 +1,179 @@
+#!/usr/bin/env bash
+
+# Shared validation for the provider-neutral disaster-recovery driver contract.
+# The caller must enable `set -euo pipefail` before sourcing this file.
+
+CLOUD_COMPOSE_DR_STATE_ROOT="${CLOUD_COMPOSE_DR_STATE_ROOT:-/mnt/disks/data/.cloud-compose-disaster-recovery}"
+CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER="${CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER:-/etc/cloud-compose/libexec/offhost-backup-driver}"
+CLOUD_COMPOSE_JQ_PROGRAM_DIR="${CLOUD_COMPOSE_JQ_PROGRAM_DIR:-/etc/cloud-compose/jq}"
+
+cloud_compose_dr_is_required() {
+ case "${CLOUD_COMPOSE_OFFHOST_BACKUP_REQUIRED:-false}" in
+ true) return 0 ;;
+ false) return 1 ;;
+ *)
+ echo "CLOUD_COMPOSE_OFFHOST_BACKUP_REQUIRED must be true or false" >&2
+ return 2
+ ;;
+ esac
+}
+
+cloud_compose_dr_validate_safe_absolute_path() {
+ local path="$1" label="$2"
+
+ if [[ ! "$path" =~ ^/[A-Za-z0-9._/+:-]+$ || "$path" == *"//"* ||
+ "$path" =~ (^|/)\.\.?(/|$) ]]; then
+ echo "$label must be a safe absolute path without whitespace or dot segments" >&2
+ return 1
+ fi
+}
+
+cloud_compose_dr_validate_utc_timestamp() {
+ local value="$1" label="$2"
+
+ if [[ ! "$value" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]]; then
+ echo "$label must be an RFC 3339 UTC timestamp with whole-second precision" >&2
+ return 1
+ fi
+}
+
+cloud_compose_dr_validate_remote_id() {
+ local value="$1" label="$2"
+
+ if [[ ! "$value" =~ ^[A-Za-z0-9][A-Za-z0-9._:/-]{0,511}$ ]]; then
+ echo "$label contains unsupported characters or exceeds 512 bytes" >&2
+ return 1
+ fi
+}
+
+cloud_compose_dr_validate_driver() {
+ local driver="${1:-$CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER}"
+ local current component metadata owner mode kind links resolved
+ local -a components
+
+ cloud_compose_dr_validate_safe_absolute_path "$driver" "Off-host backup driver path" || return 1
+ if [[ -L "$driver" || ! -f "$driver" || ! -x "$driver" ]]; then
+ echo "Off-host backup driver must be a non-symlink executable file: $driver" >&2
+ return 1
+ fi
+
+ resolved="$(realpath -e -- "$driver")" || return 1
+ if [[ "$resolved" != "$driver" ]]; then
+ echo "Off-host backup driver path must not traverse symbolic links: $driver" >&2
+ return 1
+ fi
+
+ IFS='/' read -r -a components <<<"${driver#/}"
+ current="/"
+ for component in "${components[@]:0:${#components[@]}-1}"; do
+ current="${current%/}/${component}"
+ if [[ -L "$current" || ! -d "$current" ]]; then
+ echo "Off-host backup driver parent must be a real directory: $current" >&2
+ return 1
+ fi
+ metadata="$(stat -c '%u:%a:%F' -- "$current")" || return 1
+ IFS=: read -r owner mode kind <<<"$metadata"
+ if [[ "$owner" != "0" || "$kind" != "directory" || ! "$mode" =~ ^[0-7]{3,4}$ ||
+ $((8#$mode & 0022)) -ne 0 ]]; then
+ echo "Off-host backup driver parents must be root-owned and not group/world writable: $current" >&2
+ return 1
+ fi
+ done
+
+ metadata="$(stat -c '%u:%a:%h:%F' -- "$driver")" || return 1
+ IFS=: read -r owner mode links kind <<<"$metadata"
+ if [[ "$owner" != "0" || "$links" != "1" || "$kind" != "regular file" ||
+ ! "$mode" =~ ^[0-7]{3,4}$ || $((8#$mode & 0022)) -ne 0 ]]; then
+ echo "Off-host backup driver must be a single-link, root-owned executable that is not group/world writable: $driver" >&2
+ return 1
+ fi
+}
+
+cloud_compose_dr_validate_json_file() {
+ local path="$1" label="$2" metadata owner mode links kind size
+
+ if [[ -L "$path" || ! -f "$path" ]]; then
+ echo "$label is missing or unsafe" >&2
+ return 1
+ fi
+ metadata="$(stat -c '%u:%a:%h:%F' -- "$path")" || return 1
+ IFS=: read -r owner mode links kind <<<"$metadata"
+ if [[ "$owner" != "0" || "$links" != "1" || "$kind" != "regular file" ||
+ ! "$mode" =~ ^[0-7]{3,4}$ || $((8#$mode & 0022)) -ne 0 ]]; then
+ echo "$label must be a single-link, root-owned regular file that is not group/world writable" >&2
+ return 1
+ fi
+ size="$(wc -c <"$path")" || return 1
+ if ((size < 2 || size > 65536)); then
+ echo "$label must contain between 2 and 65536 bytes" >&2
+ return 1
+ fi
+}
+
+cloud_compose_dr_prepare_state_directory() {
+ local path="$1" metadata owner mode kind
+
+ cloud_compose_dr_validate_safe_absolute_path "$path" "Disaster-recovery state path" || return 1
+ if [[ -L "$path" || ( -e "$path" && ! -d "$path" ) ]]; then
+ echo "Disaster-recovery state path is unsafe: $path" >&2
+ return 1
+ fi
+ install -d -m 0700 -o root -g root -- "$path" || return 1
+ metadata="$(stat -c '%u:%a:%F' -- "$path")" || return 1
+ IFS=: read -r owner mode kind <<<"$metadata"
+ if [[ "$owner" != "0" || "$mode" != "700" || "$kind" != "directory" || -L "$path" ]]; then
+ echo "Disaster-recovery state directories must be real root-owned mode-0700 directories: $path" >&2
+ return 1
+ fi
+}
+
+cloud_compose_dr_validate_backup_receipt() {
+ local path="$1" operation_id="$2" manifest_sha256="$3" completed_at remote_id
+
+ cloud_compose_dr_validate_json_file "$path" "Off-host backup receipt" || return 1
+ jq -e \
+ --arg operation_id "$operation_id" \
+ --arg manifest_sha256 "$manifest_sha256" \
+ -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/dr-validate-backup-receipt.jq" \
+ "$path" >/dev/null || {
+ echo "Off-host backup driver returned an invalid or incomplete coverage receipt" >&2
+ return 1
+ }
+ completed_at="$(jq -er -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/dr-backup-completed-at.jq" "$path")" || return 1
+ remote_id="$(jq -er -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/dr-backup-remote-id.jq" "$path")" || return 1
+ cloud_compose_dr_validate_utc_timestamp "$completed_at" "Off-host backup completion time" || return 1
+ cloud_compose_dr_validate_remote_id "$remote_id" "Off-host backup remote id" || return 1
+}
+
+cloud_compose_dr_validate_restore_proof() {
+ local path="$1" test_id="$2" manifest_sha256="$3" receipt_sha256="$4" completed_at recovery_id
+
+ cloud_compose_dr_validate_json_file "$path" "Restore-test proof" || return 1
+ jq -e \
+ --arg test_id "$test_id" \
+ --arg manifest_sha256 "$manifest_sha256" \
+ --arg receipt_sha256 "$receipt_sha256" \
+ -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/dr-validate-restore-proof.jq" \
+ "$path" >/dev/null || {
+ echo "Off-host backup driver returned an invalid restore-test proof" >&2
+ return 1
+ }
+ completed_at="$(jq -er -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/dr-restore-completed-at.jq" "$path")" || return 1
+ recovery_id="$(jq -er -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/dr-restore-recovery-id.jq" "$path")" || return 1
+ cloud_compose_dr_validate_utc_timestamp "$completed_at" "Restore-test completion time" || return 1
+ cloud_compose_dr_validate_remote_id "$recovery_id" "Restore-test recovery id" || return 1
+}
+
+cloud_compose_dr_run_driver() {
+ local driver="$1"
+ shift
+
+ # Driver credentials and configuration are installed and resolved by the
+ # operator-owned executable. Terraform-rendered host/application variables
+ # are deliberately absent, and driver output is never copied to the journal.
+ if ! env -i HOME=/root PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
+ "$driver" "$@" /dev/null 2>&1; then
+ echo "Off-host disaster-recovery driver failed; inspect its operator-owned diagnostics" >&2
+ return 1
+ fi
+}
diff --git a/rootfs/home/cloud-compose/host-init.sh b/rootfs/home/cloud-compose/host-init.sh
index 0c00eb4..673a93d 100644
--- a/rootfs/home/cloud-compose/host-init.sh
+++ b/rootfs/home/cloud-compose/host-init.sh
@@ -42,7 +42,6 @@ chown root:root /home/cloud-compose
chmod 0755 /home/cloud-compose
for mutable_dir in \
/home/cloud-compose/apps \
- /home/cloud-compose/bin \
/home/cloud-compose/state \
/home/cloud-compose/.sitectl \
/home/cloud-compose/.cache \
@@ -50,15 +49,32 @@ for mutable_dir in \
/home/cloud-compose/.local; do
install -d -m 0750 -o cloud-compose -g cloud-compose "$mutable_dir"
done
+if [[ -L /home/cloud-compose/bin || ! -d /home/cloud-compose/bin ||
+ "$(stat -c '%u:%g:%a:%F' -- /home/cloud-compose/bin)" != "0:0:755:directory" ]]; then
+ echo "Managed command directory was not secured by the runtime installer" >&2
+ exit 1
+fi
find /home/cloud-compose -maxdepth 1 -type f -name '*.sh' \
-exec chown root:root {} + \
-exec chmod 0755 {} +
for dispatcher in init up down rollout; do
- if [ -f "/home/cloud-compose/$dispatcher" ]; then
- chown root:root "/home/cloud-compose/$dispatcher"
- chmod 0755 "/home/cloud-compose/$dispatcher"
+ dispatcher_path="/home/cloud-compose/$dispatcher"
+ dispatcher_metadata="$(stat -c '%u:%g:%a:%h:%F' -- "$dispatcher_path")" || {
+ echo "Unable to inspect Cloud Compose lifecycle dispatcher: $dispatcher_path" >&2
+ exit 1
+ }
+ IFS=: read -r dispatcher_uid _ dispatcher_mode dispatcher_links dispatcher_kind \
+ <<<"$dispatcher_metadata"
+ if [[ -L "$dispatcher_path" || ! -f "$dispatcher_path" ||
+ "$dispatcher_uid" != "0" || "$dispatcher_links" != "1" ||
+ "$dispatcher_kind" != "regular file" || ! "$dispatcher_mode" =~ ^[0-7]{3,4}$ ||
+ $((8#$dispatcher_mode & 0022)) -ne 0 ]]; then
+ echo "Unsafe Cloud Compose lifecycle dispatcher: $dispatcher_path" >&2
+ exit 1
fi
+ chown root:root "$dispatcher_path"
+ chmod 0755 "$dispatcher_path"
done
for runtime_input in .env compose-projects.json application-env.json; do
if [ -f "/home/cloud-compose/$runtime_input" ]; then
diff --git a/rootfs/home/cloud-compose/install-dependencies-cos.sh b/rootfs/home/cloud-compose/install-dependencies-cos.sh
index 0f6320e..ba1da78 100644
--- a/rootfs/home/cloud-compose/install-dependencies-cos.sh
+++ b/rootfs/home/cloud-compose/install-dependencies-cos.sh
@@ -18,15 +18,16 @@ valid_make_binary() {
}
# This function is sourced by the COS bootstrap contract, which supplies all
-# three arguments even though the executable entrypoint below uses defaults.
+# four arguments even though the executable entrypoint below uses defaults.
# shellcheck disable=SC2120
install_cos_dependencies() {
local cloud_compose_home="${1:-/home/cloud-compose}"
# COS mounts both /home and /var with noexec. Keep the verified Make binary
# on the executable persistent data disk; only the unprivileged application
# PATH consumes the published symlink.
- local tool_state_dir="${2:-${COS_TOOL_STATE_DIR:-/mnt/disks/data/cloud-compose-tools}}"
+ local tool_state_dir="${2:-${COS_TOOL_STATE_DIR:-/mnt/disks/data/libops-managed/bin}}"
local docker_bin="${3:-/usr/bin/docker}"
+ local make_build_program="${4:-/etc/cloud-compose/libexec/build-cos-make.sh}"
local make_path pending_make_path make_state_path make_state_tmp make_sha
local tool_mount_options installer_uid installer_gid
@@ -61,14 +62,19 @@ install_cos_dependencies() {
installer_gid="$(id -g)"
chown "${installer_uid}:${installer_gid}" "$tool_state_dir"
chmod 0755 "$tool_state_dir"
+ # This directory is on the privileged host PATH. Close legacy
+ # application ownership before publishing any verified tool link. The
+ # installer is root in production; the numeric identity keeps the
+ # standalone contract harness unprivileged.
+ chown "${installer_uid}:${installer_gid}" "${cloud_compose_home}/bin"
+ chmod 0755 "${cloud_compose_home}/bin"
DOCKER_CLI_PLUGIN_DIR="${DOCKER_CONFIG}/cli-plugins" \
bash "${cloud_compose_home}/install-docker-plugins.sh"
DOCKER_CLI_PLUGIN_DIR="${cloud_compose_home}/.docker/cli-plugins" \
bash "${cloud_compose_home}/install-docker-plugins.sh"
chown -R cloud-compose:cloud-compose \
"$DOCKER_CONFIG" \
- "${cloud_compose_home}/.docker" \
- "${cloud_compose_home}/bin"
+ "${cloud_compose_home}/.docker"
make_path="${tool_state_dir}/make"
make_state_path="${tool_state_dir}/make.state"
@@ -81,6 +87,10 @@ install_cos_dependencies() {
return 1
fi
fi
+ if [[ -L "$make_build_program" || ! -f "$make_build_program" || ! -x "$make_build_program" ]]; then
+ echo "COS Make build program is missing or unsafe: $make_build_program" >&2
+ return 1
+ fi
rm -f -- "$pending_make_path"
# The GCP metadata deny policy is installed before this container runs.
# Keep the build on Docker's isolated bridge: `--network host` would
@@ -89,50 +99,9 @@ install_cos_dependencies() {
# shellcheck disable=SC2016
if ! retry_until_success "$docker_bin" run --rm \
-v "${tool_state_dir}:/out" \
+ -v "${make_build_program}:/tmp/cloud-compose-build-cos-make.sh:ro" \
"$ALPINE_BUILD_IMAGE" \
- /bin/sh -euxc '
- MAKE_VERSION="4.4.1"
- MAKE_SHA256="dd16fb1d67bfab79a72f5e8390735c49e3e8e70b4945a15ab1f81ddb78658fb3"
-
- # A single Alpine CDN outage must not make a healthy VM
- # replacement fail. These are HTTPS endpoints from the Alpine
- # official mirror list; apk still verifies the signed indexes
- # and packages with the keys baked into the pinned image.
- alpine_mirrors="
- https://dl-cdn.alpinelinux.org/alpine
- https://mirror.math.princeton.edu/pub/alpinelinux
- https://mirror.fel.cvut.cz/alpine
- "
- packages_installed=false
- for alpine_mirror in ${alpine_mirrors}; do
- printf "%s\n%s\n" \
- "${alpine_mirror}/v3.22/main" \
- "${alpine_mirror}/v3.22/community" \
- >/etc/apk/repositories
- rm -f /var/cache/apk/*
- if apk update && apk add build-base curl make tar; then
- packages_installed=true
- break
- fi
- echo "Alpine package mirror failed: ${alpine_mirror}" >&2
- done
- if [ "${packages_installed}" != true ]; then
- echo "All configured Alpine package mirrors failed" >&2
- exit 1
- fi
- curl -fsSL --proto "=https" --proto-redir "=https" --tlsv1.2 \
- --retry 5 --retry-all-errors --retry-delay 2 --retry-max-time 900 \
- --connect-timeout 10 --max-time 300 \
- "https://ftp.gnu.org/gnu/make/make-${MAKE_VERSION}.tar.gz" -o /tmp/make.tar.gz
- echo "${MAKE_SHA256} /tmp/make.tar.gz" | sha256sum -c -
- tar -xzf /tmp/make.tar.gz -C /tmp
- cd "/tmp/make-${MAKE_VERSION}"
- LDFLAGS="-static" ./configure --disable-nls
- make -j2
- install -m 0755 make /out/.cloud-compose-make.pending
- /out/.cloud-compose-make.pending --version | grep -Fqm 1 "GNU Make ${MAKE_VERSION}"
- mv -f /out/.cloud-compose-make.pending /out/make
- '; then
+ /bin/sh /tmp/cloud-compose-build-cos-make.sh; then
rm -f -- "$pending_make_path"
return 1
fi
diff --git a/rootfs/home/cloud-compose/libops-managed-runtime.sh b/rootfs/home/cloud-compose/libops-managed-runtime.sh
index 94bc722..9ef8b0a 100644
--- a/rootfs/home/cloud-compose/libops-managed-runtime.sh
+++ b/rootfs/home/cloud-compose/libops-managed-runtime.sh
@@ -27,20 +27,135 @@ enabled() {
}
mkdirs() {
+ local expected_uid expected_gid path mode allow_owner_migration spec
+ local -a directory_specs
+
+ expected_uid="$EUID"
+ expected_gid="$(id -g)"
+ if [[ "$STATE_DIR" == "/mnt/disks/data/libops-managed" ||
+ "$PUBLISHED_BIN_DIR" == "/home/cloud-compose/bin" ]]; then
+ if ((EUID != 0)); then
+ log "production managed runtime directories require a root updater"
+ return 1
+ fi
+ expected_uid=0
+ expected_gid=0
+ fi
+
# The managed binary is published through /home/cloud-compose/bin and must
# remain traversable after a root-owned bootstrap drops to cloud-compose.
- # Converge existing directories as well as new ones so a previously
- # restrictive service umask cannot leave the published symlink unusable.
- install -d -m 0755 "$STATE_DIR" "$BIN_DIR"
- install -d -m 0700 "$TMP_DIR" "$PACKAGE_STATE_DIR" "$ARTIFACT_STATE_DIR"
- mkdir -p "$PUBLISHED_BIN_DIR"
+ # Refuse redirected, non-directory, non-owner-controlled, or writable
+ # state before creating package staging files beneath the shared data mount.
+ directory_specs=(
+ "$STATE_DIR:0755:false"
+ "$BIN_DIR:0755:false"
+ "$TMP_DIR:0700:false"
+ "$PACKAGE_STATE_DIR:0700:false"
+ "$ARTIFACT_STATE_DIR:0700:false"
+ "$PUBLISHED_BIN_DIR:0755:true"
+ )
+ for spec in "${directory_specs[@]}"; do
+ IFS=: read -r path mode allow_owner_migration <<<"$spec"
+ prepare_managed_runtime_directory \
+ "$path" "$mode" "$expected_uid" "$expected_gid" "$allow_owner_migration" || return 1
+ done
+}
+
+prepare_managed_runtime_directory() {
+ local path="$1" mode="$2" expected_uid="$3" expected_gid="$4"
+ local allow_owner_migration="$5" metadata owner_uid group_gid actual_mode kind resolved desired_mode
+ local created=false
+
+ if [[ -L "$path" || ( -e "$path" && ! -d "$path" ) ]]; then
+ log "managed runtime path is not a real directory: ${path}"
+ return 1
+ fi
+ if [[ ! -e "$path" && ! -L "$path" ]]; then
+ # mkdir is the creation boundary: if an unprivileged process wins the
+ # name between inspection and creation, fail rather than adopting its
+ # pre-populated directory with install -d.
+ if ! mkdir -m "$mode" -- "$path"; then
+ log "managed runtime directory appeared during creation: ${path}"
+ return 1
+ fi
+ created=true
+ fi
+ if [[ "$created" != "true" ]]; then
+ if [[ -L "$path" || ! -d "$path" ]]; then
+ log "managed runtime path changed during validation: ${path}"
+ return 1
+ fi
+ metadata="$(stat -c '%u:%g:%a:%F' -- "$path")" || return 1
+ IFS=: read -r owner_uid group_gid actual_mode kind <<<"$metadata"
+ if [[ "$kind" != "directory" || ! "$actual_mode" =~ ^[0-7]{3,4}$ ||
+ $((8#$actual_mode & 0022)) -ne 0 ]]; then
+ log "managed runtime directory is writable by another account: ${path}"
+ return 1
+ fi
+ if [[ ( "$allow_owner_migration" != "true" || "$EUID" != "0" ) &&
+ ( "$owner_uid" != "$expected_uid" || "$group_gid" != "$expected_gid" ) ]]; then
+ log "managed runtime directory is not owned by the updater: ${path}"
+ return 1
+ fi
+ if [[ "$allow_owner_migration" == "true" ]]; then
+ # Close the legacy application-owned PATH directory before walking
+ # its entries. The bootstrap libexec boundary has already made its
+ # parent root-owned, so an old owner cannot race validation.
+ if ((EUID == 0)); then
+ install -d -m "$mode" -o "$expected_uid" -g "$expected_gid" -- "$path" || return 1
+ else
+ chmod "$mode" -- "$path" || return 1
+ fi
+ validate_published_bin_directory "$path" || return 1
+ fi
+ fi
+
+ if ((EUID == 0)); then
+ install -d -m "$mode" -o "$expected_uid" -g "$expected_gid" -- "$path" || return 1
+ else
+ chmod "$mode" -- "$path" || return 1
+ fi
+ resolved="$(readlink -f -- "$path")" || return 1
+ desired_mode="$(printf '%o' "$((8#$mode))")"
+ metadata="$(stat -c '%u:%g:%a:%F' -- "$path")" || return 1
+ if [[ "$resolved" != "$path" || "$metadata" != "${expected_uid}:${expected_gid}:${desired_mode}:directory" ||
+ -L "$path" ]]; then
+ log "managed runtime directory did not converge safely: ${path}"
+ return 1
+ fi
+}
+
+validate_published_bin_directory() {
+ local path="$1" entry name target
+ local -a entries
+
+ # /home/cloud-compose/bin was application-owned on older hosts. Preserve
+ # only generated managed-tool links whose targets remain under the
+ # validated root-owned package directory; reject every other inherited PATH
+ # entry. COS publishes its verified static Make build here because /home
+ # and /var are mounted noexec.
+ shopt -s nullglob dotglob
+ entries=("$path"/*)
+ shopt -u nullglob dotglob
+ for entry in "${entries[@]}"; do
+ name="${entry##*/}"
+ if [[ ! "$name" =~ ^(make|sitectl(-[a-z0-9]+)*)$ || ! -L "$entry" ]]; then
+ log "published command directory contains an unmanaged entry: ${entry}"
+ return 1
+ fi
+ target="$(readlink -- "$entry")" || return 1
+ if [[ "$target" != "${BIN_DIR}/${name}" ]]; then
+ log "published command has an unsafe target: ${entry}"
+ return 1
+ fi
+ done
}
with_lock() {
local action="$1"
shift
- mkdirs
+ mkdirs || return 1
if command -v flock >/dev/null 2>&1; then
exec 9>"${STATE_DIR}/runtime.lock"
if ! flock -n 9; then
@@ -406,7 +521,7 @@ install_sitectl_packages() {
local -A desired_packages=() stale_seen=()
validate_sitectl_configuration
- mkdirs
+ mkdirs || return 1
mapfile -t packages < <(sitectl_package_list)
for package in "${packages[@]}"; do
@@ -871,6 +986,11 @@ run_update() {
main() {
local command="${1:-update}"
+ if ((EUID != 0)); then
+ log "managed runtime updates must run as root"
+ return 1
+ fi
+
case "$command" in
install-tools)
with_lock run_install_tools
diff --git a/rootfs/home/cloud-compose/offhost-backup.sh b/rootfs/home/cloud-compose/offhost-backup.sh
new file mode 100644
index 0000000..e84ea8b
--- /dev/null
+++ b/rootfs/home/cloud-compose/offhost-backup.sh
@@ -0,0 +1,189 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+profile_path="${CLOUD_COMPOSE_PROFILE_PATH:-$script_dir/profile.sh}"
+compose_apps_path="${CLOUD_COMPOSE_COMPOSE_APPS_PATH:-$script_dir/compose-apps.sh}"
+dr_library_path="${CLOUD_COMPOSE_DR_LIBRARY_PATH:-$script_dir/disaster-recovery-lib.sh}"
+jq_program_dir="${CLOUD_COMPOSE_JQ_PROGRAM_DIR:-/etc/cloud-compose/jq}"
+# shellcheck disable=SC1090
+source "$profile_path"
+# shellcheck disable=SC1090
+source "$compose_apps_path"
+CLOUD_COMPOSE_JQ_PROGRAM_DIR="$jq_program_dir"
+# shellcheck disable=SC1090
+source "$dr_library_path"
+
+if cloud_compose_dr_is_required; then
+ :
+else
+ status=$?
+ if ((status == 1)); then
+ echo "Off-host disaster recovery is not required; local MariaDB dumps remain same-disk recovery artifacts only"
+ exit 0
+ fi
+ exit "$status"
+fi
+
+driver="$CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER"
+backup_root="${MARIADB_BACKUP_ROOT:-/mnt/disks/data/backups/mariadb}"
+data_root="${CLOUD_COMPOSE_DATA_ROOT:-/mnt/disks/data}"
+volumes_root="${CLOUD_COMPOSE_VOLUMES_ROOT:-/mnt/disks/volumes}"
+backup_date="$(date -u +%Y%m%d)"
+operation_id="${backup_date}-${CLOUD_COMPOSE_INSTANCE_NAME:-cloud-compose}"
+state_root="$CLOUD_COMPOSE_DR_STATE_ROOT"
+manifest_dir="$state_root/manifests"
+receipt_dir="$state_root/backup-receipts"
+staging_root="$state_root/staging"
+manifest_path="$manifest_dir/${operation_id}.json"
+receipt_path="$receipt_dir/${operation_id}.json"
+staging_dir=""
+
+cleanup() {
+ if [[ -n "$staging_dir" ]]; then
+ rm -rf -- "$staging_dir"
+ fi
+}
+trap cleanup EXIT
+
+cloud_compose_dr_validate_driver "$driver"
+acquire_cloud_compose_lifecycle_lock offhost-backup
+
+for path in "$state_root" "$manifest_dir" "$receipt_dir" "$staging_root"; do
+ cloud_compose_dr_prepare_state_directory "$path"
+done
+
+staging_dir="$(mktemp -d "$staging_root/.${operation_id}.XXXXXX")"
+chmod 0700 "$staging_dir"
+application_rows="$staging_dir/applications.jsonl"
+: >"$application_rows"
+chmod 0600 "$application_rows"
+
+apps=()
+compose_app_names_array apps
+if ((${#apps[@]} == 0)); then
+ echo "Off-host backup requires at least one compose application" >&2
+ exit 1
+fi
+
+for app in "${apps[@]}"; do
+ source_compose_app_env "$app"
+ project_dir="$DOCKER_COMPOSE_DIR"
+ dump_path="${backup_root}/${app}/${backup_date}-${app}.sql.gz"
+ staged_dump="$staging_dir/${app}.sql.gz"
+ compose_config="$staging_dir/${app}.compose-config.json"
+ application_row="$staging_dir/${app}.coverage.json"
+
+ validate_compose_project_dir "$project_dir"
+ if [[ -L "$project_dir" || ! -d "$project_dir" ]]; then
+ echo "Application checkout is missing or unsafe for ${app}: $project_dir" >&2
+ exit 1
+ fi
+ if [[ -L "$dump_path" || ! -f "$dump_path" || ! -s "$dump_path" ]]; then
+ echo "Required local MariaDB recovery artifact is missing or unsafe for ${app}" >&2
+ exit 1
+ fi
+ if [[ "$(stat -c '%h:%F' -- "$dump_path")" != "1:regular file" ]]; then
+ echo "Required local MariaDB recovery artifact must have one link for ${app}" >&2
+ exit 1
+ fi
+
+ # Copy through an already-open descriptor into the root-only handoff. This
+ # prevents the privileged driver from following a later path replacement in
+ # the application-owned local-backup directory.
+ exec {dump_fd}<"$dump_path"
+ dump_identity="$(stat -Lc '%d:%i' -- "/proc/${BASHPID}/fd/${dump_fd}")"
+ if [[ "$(stat -c '%d:%i' -- "$dump_path")" != "$dump_identity" ]]; then
+ echo "Local MariaDB recovery artifact changed while opening for ${app}" >&2
+ exit 1
+ fi
+ cat <&"$dump_fd" >"$staged_dump"
+ exec {dump_fd}<&-
+ chmod 0400 "$staged_dump"
+ if [[ ! -s "$staged_dump" ]] || ! gzip -t -- "$staged_dump"; then
+ echo "Required local MariaDB recovery artifact is invalid for ${app}" >&2
+ exit 1
+ fi
+ dump_sha256="$(sha256sum "$staged_dump" | awk '{print $1}')"
+ dump_bytes="$(wc -c <"$staged_dump")"
+
+ (
+ cd -- "$project_dir"
+ umask 077
+ docker compose config --format json >"$compose_config"
+ )
+ if ! jq -e \
+ -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/offhost-validate-compose-config.jq" \
+ "$compose_config" >/dev/null; then
+ echo "Docker Compose returned unsafe or unsupported volume topology for ${app}" >&2
+ exit 1
+ fi
+ if ! jq -e \
+ --arg data_root "$data_root" \
+ --arg volumes_root "$volumes_root" \
+ -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/offhost-validate-bind-roots.jq" \
+ "$compose_config" >/dev/null; then
+ echo "Persistent bind topology escapes managed data roots for ${app}" >&2
+ exit 1
+ fi
+ while IFS= read -r bind_source; do
+ if [[ "$bind_source" =~ (^|/)\.\.?(/|$) ]]; then
+ echo "Persistent bind topology contains a dot segment for ${app}" >&2
+ exit 1
+ fi
+ done < <(jq -r -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/offhost-bind-sources.jq" "$compose_config")
+
+ jq -cS \
+ --arg app "$app" \
+ --arg project_dir "$project_dir" \
+ --arg dump_path "$staged_dump" \
+ --arg dump_sha256 "$dump_sha256" \
+ --argjson dump_bytes "$dump_bytes" \
+ -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/offhost-build-application-coverage.jq" \
+ "$compose_config" >"$application_row"
+ cat "$application_row" >>"$application_rows"
+ rm -f -- "$compose_config"
+done
+
+staged_manifest="$staging_dir/manifest.json"
+jq -cS -s \
+ --arg operation_id "$operation_id" \
+ --arg backup_date "$backup_date" \
+ --arg provider "${CLOUD_COMPOSE_PROVIDER:-unknown}" \
+ --arg instance "${CLOUD_COMPOSE_INSTANCE_NAME:-cloud-compose}" \
+ -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/offhost-build-manifest.jq" \
+ "$application_rows" >"$staged_manifest"
+chmod 0400 "$staged_manifest"
+
+if ! jq -e \
+ --argjson app_count "${#apps[@]}" \
+ -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/offhost-validate-manifest.jq" \
+ "$staged_manifest" >/dev/null; then
+ echo "Generated off-host coverage manifest is incomplete" >&2
+ exit 1
+fi
+while IFS=$'\t' read -r manifest_app manifest_sha; do
+ if [[ ! "$manifest_app" =~ ^[a-z][a-z0-9-]*$ || ! "$manifest_sha" =~ ^[0-9a-f]{64}$ ]]; then
+ echo "Generated off-host coverage manifest contains an unsafe application name or digest" >&2
+ exit 1
+ fi
+done < <(jq -r -f "$CLOUD_COMPOSE_JQ_PROGRAM_DIR/offhost-manifest-app-digests.jq" "$staged_manifest")
+
+manifest_sha256="$(sha256sum "$staged_manifest" | awk '{print $1}')"
+staged_receipt="$staging_dir/receipt.json"
+cloud_compose_dr_run_driver "$driver" backup \
+ --manifest "$staged_manifest" \
+ --manifest-sha256 "$manifest_sha256" \
+ --operation-id "$operation_id" \
+ --receipt "$staged_receipt"
+if [[ "$(sha256sum "$staged_manifest" | awk '{print $1}')" != "$manifest_sha256" ]]; then
+ echo "Off-host backup driver modified the immutable coverage manifest" >&2
+ exit 1
+fi
+cloud_compose_dr_validate_backup_receipt "$staged_receipt" "$operation_id" "$manifest_sha256"
+
+chmod 0640 "$staged_manifest" "$staged_receipt"
+mv -f -- "$staged_manifest" "$manifest_path"
+mv -f -- "$staged_receipt" "$receipt_path"
+echo "Encrypted off-host disaster-recovery coverage proven for ${#apps[@]} application(s): $receipt_path"
diff --git a/rootfs/home/cloud-compose/restore-test.sh b/rootfs/home/cloud-compose/restore-test.sh
new file mode 100644
index 0000000..92c766f
--- /dev/null
+++ b/rootfs/home/cloud-compose/restore-test.sh
@@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+profile_path="${CLOUD_COMPOSE_PROFILE_PATH:-$script_dir/profile.sh}"
+dr_library_path="${CLOUD_COMPOSE_DR_LIBRARY_PATH:-$script_dir/disaster-recovery-lib.sh}"
+jq_program_dir="${CLOUD_COMPOSE_JQ_PROGRAM_DIR:-/etc/cloud-compose/jq}"
+# shellcheck disable=SC1090
+source "$profile_path"
+CLOUD_COMPOSE_JQ_PROGRAM_DIR="$jq_program_dir"
+# shellcheck disable=SC1090
+source "$dr_library_path"
+
+if cloud_compose_dr_is_required; then
+ :
+else
+ status=$?
+ if ((status == 1)); then
+ echo "Off-host disaster recovery is not required; skipping scheduled restore test"
+ exit 0
+ fi
+ exit "$status"
+fi
+
+driver="$CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER"
+state_root="$CLOUD_COMPOSE_DR_STATE_ROOT"
+manifest_dir="$state_root/manifests"
+receipt_dir="$state_root/backup-receipts"
+proof_dir="$state_root/restore-proofs"
+staging_root="$state_root/staging"
+staging_dir=""
+
+cleanup() {
+ if [[ -n "$staging_dir" ]]; then
+ rm -rf -- "$staging_dir"
+ fi
+}
+trap cleanup EXIT
+
+cloud_compose_dr_validate_driver "$driver"
+for path in "$state_root" "$manifest_dir" "$receipt_dir" "$proof_dir" "$staging_root"; do
+ cloud_compose_dr_prepare_state_directory "$path"
+done
+
+latest_receipt="$(find "$receipt_dir" -xdev -maxdepth 1 -type f -name '*.json' -printf '%f\n' | sort | tail -n 1)"
+if [[ -z "$latest_receipt" ]]; then
+ echo "Scheduled restore test requires a validated off-host backup receipt" >&2
+ exit 1
+fi
+operation_id="${latest_receipt%.json}"
+if [[ ! "$operation_id" =~ ^[0-9]{8}-[a-z][a-z0-9-]*$ ]]; then
+ echo "Latest off-host backup receipt has an unsafe operation id" >&2
+ exit 1
+fi
+receipt_path="$receipt_dir/$latest_receipt"
+manifest_path="$manifest_dir/$latest_receipt"
+if [[ -L "$manifest_path" || ! -f "$manifest_path" ]]; then
+ echo "Scheduled restore test is missing the source coverage manifest" >&2
+ exit 1
+fi
+cloud_compose_dr_validate_json_file "$manifest_path" "Off-host backup manifest"
+manifest_sha256="$(sha256sum "$manifest_path" | awk '{print $1}')"
+cloud_compose_dr_validate_backup_receipt "$receipt_path" "$operation_id" "$manifest_sha256"
+receipt_sha256="$(sha256sum "$receipt_path" | awk '{print $1}')"
+
+test_id="$(date -u +%Y%m%dT%H%M%SZ)-$(od -An -N16 -tx1 /dev/urandom | tr -d ' \n')"
+staging_dir="$(mktemp -d "$staging_root/.restore-${test_id}.XXXXXX")"
+chmod 0700 "$staging_dir"
+staged_proof="$staging_dir/proof.json"
+
+cloud_compose_dr_run_driver "$driver" restore-test \
+ --manifest "$manifest_path" \
+ --backup-receipt "$receipt_path" \
+ --source-manifest-sha256 "$manifest_sha256" \
+ --source-receipt-sha256 "$receipt_sha256" \
+ --test-id "$test_id" \
+ --proof "$staged_proof"
+cloud_compose_dr_validate_restore_proof \
+ "$staged_proof" "$test_id" "$manifest_sha256" "$receipt_sha256"
+
+proof_path="$proof_dir/${test_id}.json"
+chmod 0640 "$staged_proof"
+mv -- "$staged_proof" "$proof_path"
+echo "Disposable restore test proved database, application-file, and volume-topology recovery: $proof_path"
diff --git a/rootfs/home/cloud-compose/run.sh b/rootfs/home/cloud-compose/run.sh
index 470a72f..d4c6358 100644
--- a/rootfs/home/cloud-compose/run.sh
+++ b/rootfs/home/cloud-compose/run.sh
@@ -32,7 +32,7 @@ runtime_enabled() {
esac
}
-durable_bootstrap_marker="/home/cloud-compose/.cloud-compose-bootstrap-complete"
+durable_bootstrap_marker="/var/lib/cloud-compose/bootstrap-complete"
current_boot_app_init_marker="/run/cloud-compose-app-init-complete"
fresh_filesystem_marker="${CLOUD_COMPOSE_FRESH_FILESYSTEM_MARKER:-/mnt/disks/data/.cloud-compose/fresh-filesystem}"
fresh_filesystem_identity="${CLOUD_COMPOSE_FRESH_FILESYSTEM_IDENTITY:-fresh}"
@@ -113,4 +113,9 @@ else
systemctl disable --now cloud-compose-docker-prune.timer cloud-compose-docker-prune.service >/dev/null 2>&1 || true
fi
systemctl enable --now cloud-compose-mariadb-backup.timer
+if runtime_enabled "${CLOUD_COMPOSE_OFFHOST_BACKUP_REQUIRED:-false}"; then
+ systemctl enable --now cloud-compose-restore-test.timer
+else
+ systemctl disable --now cloud-compose-restore-test.timer cloud-compose-restore-test.service >/dev/null 2>&1 || true
+fi
cloud_compose_publish_marker "$durable_bootstrap_marker"
diff --git a/rootfs/home/cloud-compose/smoke-healthcheck.sh b/rootfs/home/cloud-compose/smoke-healthcheck.sh
new file mode 100755
index 0000000..2318fe4
--- /dev/null
+++ b/rootfs/home/cloud-compose/smoke-healthcheck.sh
@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+if (($# != 1)) || [[ -z "$1" ]]; then
+ echo "usage: smoke-healthcheck.sh CONTEXT" >&2
+ exit 2
+fi
+
+readonly context="$1"
+
+# Load the same validated runtime environment and tool path used by the
+# host-owned lifecycle scripts before handing control to sitectl.
+export HOME=/home/cloud-compose
+source /home/cloud-compose/profile.sh
+
+exec sitectl healthcheck --context "$context" --persist --format table
diff --git a/rootfs/home/cloud-compose/start-cloud-compose-bootstrap.sh b/rootfs/home/cloud-compose/start-cloud-compose-bootstrap.sh
index 0e55092..79f2480 100644
--- a/rootfs/home/cloud-compose/start-cloud-compose-bootstrap.sh
+++ b/rootfs/home/cloud-compose/start-cloud-compose-bootstrap.sh
@@ -5,7 +5,7 @@ set -euo pipefail
# shellcheck disable=SC1090
source "${CLOUD_COMPOSE_BOOTSTRAP_HELPERS_PATH:-/home/cloud-compose/bootstrap-helpers.sh}"
-durable_marker="${CLOUD_COMPOSE_BOOTSTRAP_COMPLETE_MARKER:-/home/cloud-compose/.cloud-compose-bootstrap-complete}"
+durable_marker="${CLOUD_COMPOSE_BOOTSTRAP_COMPLETE_MARKER:-/var/lib/cloud-compose/bootstrap-complete}"
wait_seconds="${CLOUD_COMPOSE_BOOTSTRAP_WAIT_SECONDS:-10800}"
bootstrap_unit="cloud-compose-bootstrap.service"
diff --git a/runtime_contracts.tftest.hcl b/runtime_contracts.tftest.hcl
index 1dfbf9d..d818bdd 100644
--- a/runtime_contracts.tftest.hcl
+++ b/runtime_contracts.tftest.hcl
@@ -147,6 +147,56 @@ run "public_entrypoint_rejects_reserved_extra_environment" {
expect_failures = [var.runtime]
}
+run "public_entrypoint_accepts_provider_neutral_disaster_recovery_driver" {
+ command = plan
+
+ variables {
+ name = "root-contract"
+ cloud_provider = "gcp"
+ template = "wp"
+ gcp = {
+ project_id = "test-project"
+ project_number = "123456789"
+ }
+ runtime = {
+ disaster_recovery = {
+ required = true
+ driver_path = "/etc/cloud-compose/libexec/acme-offhost"
+ }
+ }
+ }
+
+ assert {
+ condition = (
+ local.runtime.disaster_recovery.required &&
+ local.runtime.disaster_recovery.driver_path == "/etc/cloud-compose/libexec/acme-offhost"
+ )
+ error_message = "The public entrypoint must preserve provider-neutral DR controls."
+ }
+}
+
+run "public_entrypoint_rejects_unsafe_disaster_recovery_driver_path" {
+ command = plan
+
+ variables {
+ name = "root-contract"
+ cloud_provider = "gcp"
+ template = "wp"
+ gcp = {
+ project_id = "test-project"
+ project_number = "123456789"
+ }
+ runtime = {
+ disaster_recovery = {
+ required = true
+ driver_path = "/tmp/driver with spaces"
+ }
+ }
+ }
+
+ expect_failures = [var.runtime]
+}
+
run "public_entrypoint_accepts_direct_cloud_run_proxy_depth" {
command = plan
diff --git a/salt/cloud-compose/README.md b/salt/cloud-compose/README.md
index a9ebcbf..4829b17 100644
--- a/salt/cloud-compose/README.md
+++ b/salt/cloud-compose/README.md
@@ -65,6 +65,22 @@ empty list disables that phase and is preserved instead of restoring a default.
Runtime feature switches must be YAML booleans, not quoted strings; ambiguous
values are rejected before host mutation.
+The provider-neutral disaster-recovery interface is shared with Terraform:
+
+```yaml
+cloud_compose:
+ runtime:
+ disaster_recovery:
+ required: true
+ driver_path: /etc/cloud-compose/libexec/offhost-backup-driver
+```
+
+Install that executable and its credentials separately as root. The formula
+renders only the boolean requirement and executable path; storage endpoints,
+credentials, encryption keys, and retention policy must not be placed in
+pillar. See [the driver contract](../../docs/disaster-recovery.md) for receipt
+and scheduled restore-proof requirements.
+
Set `cloud_compose.runtime.rollout` to enable the same authenticated rollout
listener used by Terraform. Supply a pinned HTTPS `release_url`, its lowercase
`release_sha256`, an HTTPS `jwks_uri`, `jwt_audience`, and optional JSON-object
@@ -76,7 +92,9 @@ The formula installs lifecycle dispatchers as `root:cloud-compose` mode `0750`
and the root-consumed `.env`, project/application JSON, and managed-artifact
manifest as `root:cloud-compose` mode `0640`. Reapplying the state restores that
ownership boundary while leaving application checkout directories writable by
-the `cloud-compose` account.
+the `cloud-compose` account. Before a requested runtime bootstrap, the formula
+invokes the same checked-in bootstrap path hardener used by the Terraform
+modules.
The normal on-prem shape is one app per machine. Use pillar targeting to give
each minion its own `cloud_compose` values, then apply the same
@@ -121,7 +139,7 @@ cloud_compose:
sitectl:
environment: production
package_versions:
- sitectl-isle: v1.0.0
+ sitectl-isle: v1.5.0
```
Apply:
diff --git a/salt/cloud-compose/init.sls b/salt/cloud-compose/init.sls
index a6ac016..122b563 100644
--- a/salt/cloud-compose/init.sls
+++ b/salt/cloud-compose/init.sls
@@ -42,7 +42,8 @@
'sitectl': runtime.get('sitectl', {}),
'docker': runtime.get('docker', {}),
'managed_runtime': runtime.get('managed_runtime', {}),
- 'vault': runtime.get('vault', {})
+ 'vault': runtime.get('vault', {}),
+ 'disaster_recovery': runtime.get('disaster_recovery', {})
} %}
{% for section_name, section_value in runtime_sections.items() %}
{% if section_value is not mapping %}
@@ -54,6 +55,7 @@
{% set docker = runtime_sections.docker if runtime_sections.docker is mapping else {} %}
{% set managed = runtime_sections.managed_runtime if runtime_sections.managed_runtime is mapping else {} %}
{% set vault = runtime_sections.vault if runtime_sections.vault is mapping else {} %}
+{% set disaster_recovery = runtime_sections.disaster_recovery if runtime_sections.disaster_recovery is mapping else {} %}
{% set raw_rollout_service = runtime.get('rollout', {}) %}
{% if raw_rollout_service is mapping %}
{% set rollout_service = raw_rollout_service %}
@@ -141,6 +143,14 @@
{% if vault.get('agent_enabled', False) %}
{% set ignored = invalid_runtime_inputs.append('Vault Agent is currently supported only by Terraform providers; set vault.agent_enabled=false for Salt') %}
{% endif %}
+{% set offhost_backup_required = disaster_recovery.get('required', False) %}
+{% set offhost_backup_driver = disaster_recovery.get('driver_path', '/etc/cloud-compose/libexec/offhost-backup-driver') %}
+{% if offhost_backup_required is not boolean %}
+{% set ignored = invalid_runtime_inputs.append('runtime.disaster_recovery.required must be a boolean') %}
+{% endif %}
+{% if offhost_backup_driver is not string or not (offhost_backup_driver is match('^/[A-Za-z0-9._/+:-]+$')) or '//' in offhost_backup_driver or '/./' in offhost_backup_driver or '/../' in offhost_backup_driver or offhost_backup_driver.endswith('/.') or offhost_backup_driver.endswith('/..') %}
+{% set ignored = invalid_runtime_inputs.append('runtime.disaster_recovery.driver_path must be a safe absolute path without whitespace or dot segments') %}
+{% endif %}
{% set rollout_enabled = rollout_service.get('enabled', False) %}
{% set rollout_port = rollout_service.get('port', 8081) %}
{% if rollout_enabled is not boolean %}
@@ -199,7 +209,7 @@
'upload_timeout': ''
} %}
{% set default_init = [
- 'sitectl config set-context "${SITECTL_CONTEXT_NAME}" --type local --project-dir "${DOCKER_COMPOSE_DIR}" --site "${CLOUD_COMPOSE_INSTANCE_NAME}" --plugin "${SITECTL_PLUGIN}" --environment "${SITECTL_ENVIRONMENT}" --project-name "${CLOUD_COMPOSE_INSTANCE_NAME}" --compose-project-name "${COMPOSE_PROJECT_NAME}" --docker-socket /var/run/docker.sock --env-file .env --default'
+ 'sitectl config set-context "${SITECTL_CONTEXT_NAME}" --type local --project-dir "${DOCKER_COMPOSE_DIR}" --site "${CLOUD_COMPOSE_INSTANCE_NAME}" --plugin "${SITECTL_PLUGIN}" --environment "${SITECTL_ENVIRONMENT}" --compose-project-name "${COMPOSE_PROJECT_NAME}" --docker-socket /var/run/docker.sock --env-file .env --yolo --default'
] %}
{% set default_up = [
'sitectl compose --context "${SITECTL_CONTEXT_NAME}" up -d --remove-orphans',
@@ -464,6 +474,8 @@
'CLOUD_COMPOSE_INSTANCE_NAME': name,
'CLOUD_COMPOSE_APPS': compose_projects.keys() | list | join(' '),
'CLOUD_COMPOSE_PRIMARY_APP': primary_key,
+ 'CLOUD_COMPOSE_OFFHOST_BACKUP_REQUIRED': 'true' if offhost_backup_required is sameas true else 'false',
+ 'CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER': offhost_backup_driver,
'COMPOSE_PROJECTS_FILE': home ~ '/compose-projects.json',
'COMPOSE_PROJECT_NAME': primary_project.get('compose_project_name', compose_project_name),
'COMPOSE_BIND_PORT': primary_project.get('ingress_port', ingress_port),
@@ -646,6 +658,19 @@ cloud-compose-rootfs:
- require:
- user: cloud-compose-user
+cloud-compose-privileged-program-directories:
+ file.directory:
+ - names:
+ - /etc/cloud-compose
+ - /etc/cloud-compose/bin
+ - /etc/cloud-compose/jq
+ - /etc/cloud-compose/libexec
+ - user: root
+ - group: root
+ - mode: '0755'
+ - require:
+ - file: cloud-compose-rootfs
+
cloud-compose-lifecycle-lock:
cmd.run:
- name: systemd-tmpfiles --create /etc/tmpfiles.d/cloud-compose.conf
@@ -661,10 +686,19 @@ cloud-compose-lifecycle-lock:
cloud-compose-rootfs-script-modes:
cmd.run:
- - name: find /home/cloud-compose -type f -name '*.sh' -exec chmod 0755 {} +
- - unless: test -z "$(find /home/cloud-compose -type f -name '*.sh' ! -perm -u=x -print -quit)"
+ - name: find /home/cloud-compose /etc/cloud-compose/bin /etc/cloud-compose/libexec -maxdepth 1 -type f -name '*.sh' -exec chown root:root {} + -exec chmod 0755 {} +
+ - unless: test -z "$(find /home/cloud-compose /etc/cloud-compose/bin /etc/cloud-compose/libexec -maxdepth 1 -type f -name '*.sh' \( ! -user root -o ! -group root -o ! -perm 0755 \) -print -quit)"
- require:
- file: cloud-compose-rootfs
+ - file: cloud-compose-privileged-program-directories
+
+cloud-compose-rootfs-jq-modes:
+ cmd.run:
+ - name: find /etc/cloud-compose/jq -maxdepth 1 -type f -name '*.jq' -exec chown root:root {} + -exec chmod 0644 {} +
+ - unless: test -z "$(find /etc/cloud-compose/jq -maxdepth 1 -type f -name '*.jq' \( ! -user root -o ! -group root -o ! -perm 0644 \) -print -quit)"
+ - require:
+ - file: cloud-compose-rootfs
+ - file: cloud-compose-privileged-program-directories
{% for lifecycle in ['init', 'up', 'down', 'rollout'] %}
cloud-compose-lifecycle-{{ lifecycle }}:
@@ -741,6 +775,22 @@ cloud-compose-managed-runtime-artifacts:
- require:
- file: cloud-compose-rootfs
+{% if run_bootstrap is sameas true %}
+cloud-compose-bootstrap-paths-hardened:
+ cmd.run:
+ - name: /etc/cloud-compose/libexec/harden-bootstrap-paths.sh
+ - require:
+ - file: cloud-compose-env
+ - file: cloud-compose-application-env
+ - file: cloud-compose-project-manifest
+ - file: cloud-compose-managed-runtime-artifacts
+ - cmd: cloud-compose-rootfs-script-modes
+ - cmd: cloud-compose-rootfs-jq-modes
+{% for lifecycle in ['init', 'up', 'down', 'rollout'] %}
+ - file: cloud-compose-lifecycle-{{ lifecycle }}
+{% endfor %}
+{% endif %}
+
{% if reload_systemd %}
cloud-compose-systemd-reload:
module.run:
@@ -752,10 +802,19 @@ cloud-compose-systemd-reload:
{% if rollout_enabled is sameas true %}
cloud-compose-rollout-service:
cmd.run:
- - name: bash /home/cloud-compose/deploy-rollout.sh
+ - name: bash /etc/cloud-compose/libexec/run-root-program.sh deploy-rollout.sh
- require:
- file: cloud-compose-env
+ - file: cloud-compose-application-env
+ - file: cloud-compose-project-manifest
+ - file: cloud-compose-managed-runtime-artifacts
- file: cloud-compose-rootfs
+ - cmd: cloud-compose-lifecycle-lock
+ - cmd: cloud-compose-rootfs-script-modes
+ - file: cloud-compose-lifecycle-init
+ - file: cloud-compose-lifecycle-up
+ - file: cloud-compose-lifecycle-down
+ - file: cloud-compose-lifecycle-rollout
{% if reload_systemd %}
- module: cloud-compose-systemd-reload
{% endif %}
@@ -764,7 +823,7 @@ cloud-compose-rollout-service:
{% if force_bootstrap is sameas true %}
cloud-compose-clear-bootstrap-marker:
file.absent:
- - name: {{ (home ~ '/.cloud-compose-bootstrap-complete') | json }}
+ - name: /var/lib/cloud-compose/bootstrap-complete
- require:
- cmd: cloud-compose-host-inputs-valid
{% endif %}
@@ -772,8 +831,8 @@ cloud-compose-clear-bootstrap-marker:
{% if run_bootstrap is sameas true %}
cloud-compose-bootstrap:
cmd.run:
- - name: bash {{ (home ~ '/start-cloud-compose-bootstrap.sh') | json }}
- - creates: {{ (home ~ '/.cloud-compose-bootstrap-complete') | json }}
+ - name: bash /etc/cloud-compose/libexec/start-cloud-compose-bootstrap.sh
+ - unless: bash /etc/cloud-compose/libexec/require-bootstrap-ready.sh
- require:
{% if install_packages %}
- service: cloud-compose-docker
@@ -782,6 +841,15 @@ cloud-compose-bootstrap:
- file: cloud-compose-application-env
- file: cloud-compose-project-manifest
- file: cloud-compose-managed-runtime-artifacts
+ - cmd: cloud-compose-rootfs-script-modes
+ - cmd: cloud-compose-rootfs-jq-modes
+ - cmd: cloud-compose-bootstrap-paths-hardened
+{% for lifecycle in ['init', 'up', 'down', 'rollout'] %}
+ - file: cloud-compose-lifecycle-{{ lifecycle }}
+{% endfor %}
+{% if force_bootstrap is sameas true %}
+ - file: cloud-compose-clear-bootstrap-marker
+{% endif %}
{% if compose_projects %}
- file: cloud-compose-project-dirs
{% endif %}
diff --git a/template_versions.tftest.hcl b/template_versions.tftest.hcl
index b33564c..4b992a8 100644
--- a/template_versions.tftest.hcl
+++ b/template_versions.tftest.hcl
@@ -29,7 +29,7 @@ run "default_template_uses_v1_core" {
assert {
condition = local.sitectl.package_versions == {
- sitectl = "v1.0.0"
+ sitectl = "v1.8.2"
}
error_message = "The default template must select the released sitectl v1 core."
}
@@ -59,15 +59,15 @@ run "non_isle_template_uses_v1_release_set" {
assert {
condition = local.sitectl.package_versions == {
- sitectl = "v1.0.0"
- sitectl-wp = "v1.0.0"
+ sitectl = "v1.8.2"
+ sitectl-wp = "v2.0.0"
}
error_message = "Non-ISLE templates must select their coordinated sitectl v1 release set."
}
assert {
- condition = local.compose.branch == "v1.0.0"
- error_message = "Non-ISLE templates must retain their stable v1.0.0 template contract."
+ condition = local.compose.branch == "v1.1.0"
+ error_message = "The WordPress preset must select its stable v1.1.0 template contract."
}
}
@@ -90,16 +90,16 @@ run "isle_template_uses_v1_release_set" {
assert {
condition = local.sitectl.package_versions == {
- sitectl = "v1.0.0"
- sitectl-drupal = "v1.0.0"
- sitectl-isle = "v1.0.0"
+ sitectl = "v1.8.2"
+ sitectl-drupal = "v1.3.0"
+ sitectl-isle = "v1.5.0"
}
error_message = "The ISLE template must select its coordinated sitectl v1 release set by default."
}
assert {
- condition = local.compose.branch == "v1.1.0"
- error_message = "The ISLE preset must select the stable v1.1.0 template contract."
+ condition = local.compose.branch == "v1.3.0"
+ error_message = "The ISLE preset must select the stable v1.3.0 template contract."
}
assert {
@@ -168,7 +168,7 @@ run "explicit_package_versions_override_template_defaults" {
assert {
condition = local.sitectl.package_versions == {
sitectl = "v0.40.1"
- sitectl-drupal = "v1.0.0"
+ sitectl-drupal = "v1.3.0"
sitectl-isle = "v0.19.1"
}
error_message = "Explicit per-package selectors must override only their matching template defaults."
@@ -200,7 +200,7 @@ run "custom_package_set_filters_template_versions" {
assert {
condition = local.sitectl.package_versions == {
- sitectl = "v1.0.0"
+ sitectl = "v1.8.2"
sitectl-wp = "v0.6.1"
}
error_message = "Template selectors for packages omitted by a custom package set must not reach the runtime."
@@ -229,7 +229,7 @@ run "explicit_core_only_package_set_disables_template_plugins" {
assert {
condition = local.sitectl.packages == tolist(["sitectl"]) && local.sitectl.package_versions == {
- sitectl = "v1.0.0"
+ sitectl = "v1.8.2"
}
error_message = "An explicit core-only package set must not be mistaken for an omitted template package selection."
}
diff --git a/templates/apps.json b/templates/apps.json
index 5a8600d..ac347ac 100644
--- a/templates/apps.json
+++ b/templates/apps.json
@@ -6,7 +6,7 @@
"packages": ["sitectl"],
"extra_env": {},
"package_versions": {
- "sitectl": "v1.0.0"
+ "sitectl": "v1.8.2"
}
},
"templates": {
@@ -17,8 +17,8 @@
"packages": ["sitectl", "sitectl-archivesspace"],
"extra_env": {},
"package_versions": {
- "sitectl": "v1.0.0",
- "sitectl-archivesspace": "v1.0.0"
+ "sitectl": "v1.8.2",
+ "sitectl-archivesspace": "v2.0.0"
}
},
"ojs": {
@@ -28,44 +28,44 @@
"packages": ["sitectl", "sitectl-ojs"],
"extra_env": {},
"package_versions": {
- "sitectl": "v1.0.0",
- "sitectl-ojs": "v1.0.0"
+ "sitectl": "v1.8.2",
+ "sitectl-ojs": "v1.2.0"
}
},
"isle": {
"repo": "https://github.com/libops/isle",
- "branch": "v1.1.0",
+ "branch": "v1.3.0",
"plugin": "isle",
"packages": ["sitectl", "sitectl-drupal", "sitectl-isle"],
"extra_env": {
"ISLANDORA_TAG": "6.3.19"
},
"package_versions": {
- "sitectl": "v1.0.0",
- "sitectl-drupal": "v1.0.0",
- "sitectl-isle": "v1.0.0"
+ "sitectl": "v1.8.2",
+ "sitectl-drupal": "v1.3.0",
+ "sitectl-isle": "v1.5.0"
}
},
"drupal": {
"repo": "https://github.com/libops/drupal.git",
- "branch": "v1.0.0",
+ "branch": "v1.1.0",
"plugin": "drupal",
"packages": ["sitectl", "sitectl-drupal"],
"extra_env": {},
"package_versions": {
- "sitectl": "v1.0.0",
- "sitectl-drupal": "v1.0.0"
+ "sitectl": "v1.8.2",
+ "sitectl-drupal": "v1.3.0"
}
},
"wp": {
"repo": "https://github.com/libops/wp.git",
- "branch": "v1.0.0",
+ "branch": "v1.1.0",
"plugin": "wp",
"packages": ["sitectl", "sitectl-wp"],
"extra_env": {},
"package_versions": {
- "sitectl": "v1.0.0",
- "sitectl-wp": "v1.0.0"
+ "sitectl": "v1.8.2",
+ "sitectl-wp": "v2.0.0"
}
},
"omeka-s": {
@@ -75,8 +75,8 @@
"packages": ["sitectl", "sitectl-omeka-s"],
"extra_env": {},
"package_versions": {
- "sitectl": "v1.0.0",
- "sitectl-omeka-s": "v1.0.0"
+ "sitectl": "v1.8.2",
+ "sitectl-omeka-s": "v1.2.0"
}
},
"omeka-classic": {
@@ -86,8 +86,8 @@
"packages": ["sitectl", "sitectl-omeka-classic"],
"extra_env": {},
"package_versions": {
- "sitectl": "v1.0.0",
- "sitectl-omeka-classic": "v1.0.0"
+ "sitectl": "v1.8.2",
+ "sitectl-omeka-classic": "v1.2.0"
}
}
}
diff --git a/templates/cloud-init.yml b/templates/cloud-init.yml
index 4559bee..94d2d13 100644
--- a/templates/cloud-init.yml
+++ b/templates/cloud-init.yml
@@ -10,6 +10,9 @@ users:
- docker
sudo:
- ALL=(root) NOPASSWD:/usr/bin/systemctl start cloud-compose.service,/usr/bin/systemctl stop cloud-compose.service,/usr/bin/systemctl restart cloud-compose.service,/usr/bin/systemctl status cloud-compose.service
+ - ALL=(root) NOPASSWD:/etc/cloud-compose/bin/cloud-compose-diagnostics.sh state
+ - ALL=(root) NOPASSWD:/etc/cloud-compose/bin/cloud-compose-diagnostics.sh status
+ - ALL=(root) NOPASSWD:/etc/cloud-compose/bin/cloud-compose-diagnostics.sh dump
%{ if length(CLOUD_COMPOSE_SSH_KEYS) > 0 ~}
ssh_authorized_keys:
%{ for key in CLOUD_COMPOSE_SSH_KEYS ~}
@@ -70,6 +73,11 @@ bootcmd:
install -m 0600 /dev/null /run/cloud-compose-filesystems-ready
write_files:
+- path: /var/lib/cloud-compose/bootstrap/cloud-compose-diagnostics.sh
+ owner: root:root
+ permissions: "0600"
+ encoding: b64
+ content: ${DIAGNOSTICS_SCRIPT_B64}
${WRITE_FILES_CONTENT}
${DOCKER_COMPOSE_SCRIPTS}
${COMPOSE_PROJECTS_FILE}
@@ -82,6 +90,10 @@ runcmd:
${ROOTFS_ARCHIVE_COMMAND}
- |
set -eu
+ install -d -m 0755 -o root -g root /etc/cloud-compose/bin
+ install -m 0755 -o root -g root \
+ /var/lib/cloud-compose/bootstrap/cloud-compose-diagnostics.sh \
+ /etc/cloud-compose/bin/cloud-compose-diagnostics.sh
test -f /run/cloud-compose-filesystems-ready || {
echo "Cloud Compose filesystems were not prepared; refusing application initialization" >&2
exit 1
@@ -94,15 +106,13 @@ ${ROOTFS_ARCHIVE_COMMAND}
chown cloud-compose:cloud-compose /mnt/disks/volumes
chmod 0775 /mnt/disks/volumes
install -d -m 0775 -o cloud-compose -g cloud-compose /mnt/disks/data/libops
- rm -f /home/cloud-compose/.cloud-compose-bootstrap-complete
- bash /home/cloud-compose/start-cloud-compose-bootstrap.sh
+ rm -f /var/lib/cloud-compose/bootstrap-complete
+ /etc/cloud-compose/libexec/harden-bootstrap-paths.sh
+ bash /etc/cloud-compose/libexec/start-cloud-compose-bootstrap.sh
%{ if length(ADDITIONAL_RUNCMD) > 0 ~}
- |
set -eu
- test -f /home/cloud-compose/.cloud-compose-bootstrap-complete || {
- echo "Cloud Compose application initialization did not complete; refusing post-initialization commands" >&2
- exit 1
- }
+ bash /etc/cloud-compose/libexec/require-bootstrap-ready.sh
%{ for CMD in ADDITIONAL_RUNCMD ~}
${indent(2, CMD)}
%{ endfor ~}
diff --git a/tests/config-management/ansible/invalid-disaster-recovery.yml b/tests/config-management/ansible/invalid-disaster-recovery.yml
new file mode 100644
index 0000000..19aac33
--- /dev/null
+++ b/tests/config-management/ansible/invalid-disaster-recovery.yml
@@ -0,0 +1,16 @@
+- name: Reject unsafe disaster-recovery driver settings
+ hosts: localhost
+ gather_facts: true
+ vars:
+ cloud_compose_name: invalid-disaster-recovery
+ cloud_compose_template: wp
+ cloud_compose_dedicated_host_acknowledged: true
+ cloud_compose_install_packages: false
+ cloud_compose_reload_systemd: false
+ cloud_compose_run_bootstrap: false
+ cloud_compose_runtime:
+ disaster_recovery:
+ required: true
+ driver_path: /etc/cloud-compose/libexec/../untrusted
+ roles:
+ - role: cloud_compose
diff --git a/tests/config-management/ansible/smoke.yml b/tests/config-management/ansible/smoke.yml
index 57813e0..f03fb35 100644
--- a/tests/config-management/ansible/smoke.yml
+++ b/tests/config-management/ansible/smoke.yml
@@ -12,6 +12,9 @@
cloud_compose_internal_services_enabled: true
cloud_compose_internal_services_auto_update: true
cloud_compose_runtime:
+ disaster_recovery:
+ required: false
+ driver_path: /etc/cloud-compose/libexec/ansible-offhost
extra_env:
BASH_ENV: /tmp/cloud-compose-ansible-untrusted-bash-env
LD_PRELOAD: /tmp/cloud-compose-ansible-untrusted-preload.so
@@ -45,9 +48,9 @@
sitectl:
environment: production
package_versions:
- sitectl: v1.0.0
- sitectl-drupal: v1.0.0
- sitectl-isle: v1.0.0
+ sitectl: v1.8.2
+ sitectl-drupal: v1.3.0
+ sitectl-isle: v1.5.0
managed_runtime:
enabled: false
internal_services_enabled: false
@@ -94,6 +97,10 @@
that:
- >-
'CLOUD_COMPOSE_PROVIDER="onprem"' in (cloud_compose_env_file.content | b64decode)
+ - >-
+ 'CLOUD_COMPOSE_OFFHOST_BACKUP_REQUIRED="false"' in (cloud_compose_env_file.content | b64decode)
+ - >-
+ 'CLOUD_COMPOSE_OFFHOST_BACKUP_DRIVER="/etc/cloud-compose/libexec/ansible-offhost"' in (cloud_compose_env_file.content | b64decode)
- >-
'DOCKER_COMPOSE_DIR="/mnt/disks/data/libops/isle/isle-prod"' in (cloud_compose_env_file.content | b64decode)
- >-
@@ -117,7 +124,7 @@
- '(cloud_compose_application_env_file.content | b64decode | from_json).PORT == "9999"'
- '(cloud_compose_manifest_file.content | b64decode | from_json)["isle-prod"].docker_compose_repo == "https://github.com/libops/isle"'
- '(cloud_compose_manifest_file.content | b64decode | from_json)["isle-prod"].project_dir == "/mnt/disks/data/libops/isle/isle-prod"'
- - '(cloud_compose_manifest_file.content | b64decode | from_json)["isle-prod"].compose_project_name == "libops-isle-v1-1-0"'
+ - '(cloud_compose_manifest_file.content | b64decode | from_json)["isle-prod"].compose_project_name == "libops-isle-v1-3-0"'
- '(cloud_compose_manifest_file.content | b64decode | from_json)["isle-prod"].ingress.domain == "isle.example.edu"'
- '(cloud_compose_manifest_file.content | b64decode | from_json)["isle-prod"].sitectl_plugin == "isle"'
- '(cloud_compose_manifest_file.content | b64decode | from_json)["isle-prod"].init_commands == []'
diff --git a/tests/config-management/salt-pillar/invalid-disaster-recovery.sls b/tests/config-management/salt-pillar/invalid-disaster-recovery.sls
new file mode 100644
index 0000000..9454410
--- /dev/null
+++ b/tests/config-management/salt-pillar/invalid-disaster-recovery.sls
@@ -0,0 +1,12 @@
+cloud_compose:
+ name: invalid-disaster-recovery
+ provider: onprem
+ template: wp
+ dedicated_host_acknowledged: true
+ install_packages: false
+ reload_systemd: false
+ run_bootstrap: false
+ runtime:
+ disaster_recovery:
+ required: true
+ driver_path: /etc/cloud-compose/libexec/../untrusted
diff --git a/tests/config-management/salt-pillar/top.sls b/tests/config-management/salt-pillar/top.sls
index e652068..72b983b 100644
--- a/tests/config-management/salt-pillar/top.sls
+++ b/tests/config-management/salt-pillar/top.sls
@@ -29,6 +29,8 @@ base:
- invalid-template
'invalid-vault':
- invalid-vault
+ 'invalid-disaster-recovery':
+ - invalid-disaster-recovery
'invalid-package':
- invalid-package
'invalid-host-ack':
diff --git a/tests/config-management/salt-pillar/wp-prod.sls b/tests/config-management/salt-pillar/wp-prod.sls
index cc2526e..e02485e 100644
--- a/tests/config-management/salt-pillar/wp-prod.sls
+++ b/tests/config-management/salt-pillar/wp-prod.sls
@@ -10,6 +10,9 @@ cloud_compose:
internal_services_auto_update: true
template: wp
runtime:
+ disaster_recovery:
+ required: false
+ driver_path: /etc/cloud-compose/libexec/salt-offhost
extra_env:
BASH_ENV: /tmp/cloud-compose-salt-untrusted-bash-env
LD_PRELOAD: /tmp/cloud-compose-salt-untrusted-preload.so
@@ -43,8 +46,8 @@ cloud_compose:
sitectl:
environment: production
package_versions:
- sitectl: v1.0.0
- sitectl-wp: v1.0.0
+ sitectl: v1.8.2
+ sitectl-wp: v2.0.0
managed_runtime:
enabled: false
internal_services_enabled: false
diff --git a/variables.tf b/variables.tf
index 9e1c074..5809f27 100644
--- a/variables.tf
+++ b/variables.tf
@@ -210,6 +210,11 @@ variable "runtime" {
rootfs_archive_sha256 = optional(string, "")
users = optional(map(list(string)), {})
+ disaster_recovery = optional(object({
+ required = optional(bool, false)
+ driver_path = optional(string, "/etc/cloud-compose/libexec/offhost-backup-driver")
+ }), {})
+
compose = optional(object({
primary = optional(string, "")
ingress_port = optional(number, 80)
@@ -319,6 +324,15 @@ variable "runtime" {
error_message = "runtime.rootfs_archive_url and a 64-character runtime.rootfs_archive_sha256 must be supplied together."
}
+ validation {
+ condition = (
+ can(regex("^/[A-Za-z0-9._/+:-]+$", var.runtime.disaster_recovery.driver_path)) &&
+ !strcontains(var.runtime.disaster_recovery.driver_path, "//") &&
+ length(regexall("(^|/)\\.\\.?(/|$)", var.runtime.disaster_recovery.driver_path)) == 0
+ )
+ error_message = "runtime.disaster_recovery.driver_path must be a safe absolute path without whitespace or dot segments."
+ }
+
validation {
condition = alltrue([
for name in keys(var.runtime.extra_env) :