[experimental] CAS (Content-addressed storage) over shared object storage for antalya-26.6

.github/workflows/release_builds.yml

@@ -1334,9 +1334,162 @@ jobs:
           . ./ci/tmp/praktika_setup_env.sh
           PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (arm_binary, sequential)' --workflow "Release Builds" --ci --timestamp
 
+  stateless_tests_amd_binary_cas_s3_storage_parallel:
+    runs-on: [self-hosted, altinity-on-demand, altinity-builder, 16c]
+    needs: [build_amd_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest]
+    if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfYmluYXJ5LCBjYXMgczMgc3RvcmFnZSwgcGFyYWxsZWwp') }}
+    name: "Stateless tests (amd_binary, cas s3 storage, parallel)"
+    outputs:
+      data: ${{ steps.run.outputs.DATA }}
+      pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ env.CHECKOUT_REF }}
+
+      - name: Setup
+        uses: ./.github/actions/runner_setup
+      - name: Docker setup
+        uses: ./.github/actions/docker_setup
+        with:
+          test_name: "Stateless tests (amd_binary, cas s3 storage, parallel)"
+
+      - name: Prepare env script
+        run: |
+          rm -rf ./ci/tmp
+          mkdir -p ./ci/tmp
+          cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
+          export PYTHONPATH=./ci:.:
+          cat > ./ci/tmp/workflow_inputs.json << 'EOF'
+          ${{ toJson(github.event.inputs) }}
+          EOF
+          cat > ./ci/tmp/workflow_job.json << 'EOF'
+          ${{ toJson(job) }}
+          EOF
+          cat > ./ci/tmp/workflow_status.json << 'EOF'
+          ${{ toJson(needs) }}
+          EOF
+          ENV_SETUP_SCRIPT_EOF
+
+      - name: Download artifact CH_AMD_BINARY_GH
+        uses: actions/download-artifact@v8
+        continue-on-error: true
+        with:
+          name: CH_AMD_BINARY_GH
+          path: ./ci/tmp
+
+      - name: Run
+        id: run
+        run: |
+          . ./ci/tmp/praktika_setup_env.sh
+          PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_binary, cas s3 storage, parallel)' --workflow "Release Builds" --ci --timestamp
+
+  stateless_tests_arm_binary_cas_s3_storage_parallel:
+    runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64, 16c]
+    needs: [build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest]
+    if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhcm1fYmluYXJ5LCBjYXMgczMgc3RvcmFnZSwgcGFyYWxsZWwp') }}
+    name: "Stateless tests (arm_binary, cas s3 storage, parallel)"
+    outputs:
+      data: ${{ steps.run.outputs.DATA }}
+      pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ env.CHECKOUT_REF }}
+
+      - name: Setup
+        uses: ./.github/actions/runner_setup
+      - name: Docker setup
+        uses: ./.github/actions/docker_setup
+        with:
+          test_name: "Stateless tests (arm_binary, cas s3 storage, parallel)"
+
+      - name: Prepare env script
+        run: |
+          rm -rf ./ci/tmp
+          mkdir -p ./ci/tmp
+          cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
+          export PYTHONPATH=./ci:.:
+          cat > ./ci/tmp/workflow_inputs.json << 'EOF'
+          ${{ toJson(github.event.inputs) }}
+          EOF
+          cat > ./ci/tmp/workflow_job.json << 'EOF'
+          ${{ toJson(job) }}
+          EOF
+          cat > ./ci/tmp/workflow_status.json << 'EOF'
+          ${{ toJson(needs) }}
+          EOF
+          ENV_SETUP_SCRIPT_EOF
+
+      - name: Download artifact CH_ARM_BIN_GH
+        uses: actions/download-artifact@v8
+        continue-on-error: true
+        with:
+          name: CH_ARM_BIN_GH
+          path: ./ci/tmp
+
+      - name: Run
+        id: run
+        run: |
+          . ./ci/tmp/praktika_setup_env.sh
+          PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (arm_binary, cas s3 storage, parallel)' --workflow "Release Builds" --ci --timestamp
+
+  stateless_tests_amd_binary_cas_storage_parallel:
+    runs-on: [self-hosted, altinity-on-demand, altinity-builder, 16c]
+    needs: [build_amd_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest]
+    if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfYmluYXJ5LCBjYXMgc3RvcmFnZSwgcGFyYWxsZWwp') }}
+    name: "Stateless tests (amd_binary, cas storage, parallel)"
+    outputs:
+      data: ${{ steps.run.outputs.DATA }}
+      pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ env.CHECKOUT_REF }}
+
+      - name: Setup
+        uses: ./.github/actions/runner_setup
+      - name: Docker setup
+        uses: ./.github/actions/docker_setup
+        with:
+          test_name: "Stateless tests (amd_binary, cas storage, parallel)"
+
+      - name: Prepare env script
+        run: |
+          rm -rf ./ci/tmp
+          mkdir -p ./ci/tmp
+          cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
+          export PYTHONPATH=./ci:.:
+          cat > ./ci/tmp/workflow_inputs.json << 'EOF'
+          ${{ toJson(github.event.inputs) }}
+          EOF
+          cat > ./ci/tmp/workflow_job.json << 'EOF'
+          ${{ toJson(job) }}
+          EOF
+          cat > ./ci/tmp/workflow_status.json << 'EOF'
+          ${{ toJson(needs) }}
+          EOF
+          ENV_SETUP_SCRIPT_EOF
+
+      - name: Download artifact CH_AMD_BINARY_GH
+        uses: actions/download-artifact@v8
+        continue-on-error: true
+        with:
+          name: CH_AMD_BINARY_GH
+          path: ./ci/tmp
+
+      - name: Run
+        id: run
+        run: |
+          . ./ci/tmp/praktika_setup_env.sh
+          PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_binary, cas storage, parallel)' --workflow "Release Builds" --ci --timestamp
+
   finish_workflow:
     runs-on: [self-hosted, altinity-on-demand, altinity-style-checker]
-    needs: [build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_msan, build_amd_release, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_release, build_arm_tsan, build_arm_ubsan, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, install_packages_amd_release, install_packages_arm_release, sign_release_amd_release, sign_release_arm_release, source_upload, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential]
+    needs: [build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_msan, build_amd_release, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_release, build_arm_tsan, build_arm_ubsan, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, install_packages_amd_release, install_packages_arm_release, sign_release_amd_release, sign_release_arm_release, source_upload, stateless_tests_amd_binary_cas_s3_storage_parallel, stateless_tests_amd_binary_cas_storage_parallel, stateless_tests_arm_binary_cas_s3_storage_parallel, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential]
     if: ${{ !cancelled() && needs.config_workflow.outputs.pipeline_status != '' }}
     name: "Finish Workflow"
     outputs:
@@ -1433,6 +1586,9 @@ jobs:
       - source_upload
       - stateless_tests_arm_binary_parallel
       - stateless_tests_arm_binary_sequential
+      - stateless_tests_amd_binary_cas_s3_storage_parallel
+      - stateless_tests_arm_binary_cas_s3_storage_parallel
+      - stateless_tests_amd_binary_cas_storage_parallel
       - finish_workflow
       - GrypeScanServer
       - GrypeScanKeeper

ci/defs/altinity_jobs.py

@@ -1,6 +1,7 @@
 from praktika import Artifact, Job
 
 from ci.defs.defs import TEMP_DIR, ArtifactNames, RunnerLabels
+from ci.defs.job_configs import common_ft_job_config
 
 
 class AltinityArtifactNames:
@@ -69,3 +70,53 @@ class AltinityJobConfigs:
         command="python3 ./ci/jobs/source_upload.py",
         timeout=3600,
     )
+    # Stateless tests with a content-addressed disk as the default MergeTree storage.
+    cas_functional_tests_jobs = common_ft_job_config.parametrize(
+        # CAS over S3: RustFS, not MinIO OSS, because the incarnation pool needs
+        # enforced conditional deletes.
+        Job.ParamSet(
+            parameter="amd_binary, cas s3 storage, parallel",
+            runs_on=RunnerLabels.AMD_MEDIUM_CPU,
+            requires=[ArtifactNames.CH_AMD_BINARY_GH],
+        ),
+        # The sanitizer lanes are sharded because an unsharded one exceeds the 6h
+        # GitHub job timeout and is killed before it uploads any results.
+        *[
+            Job.ParamSet(
+                parameter=f"amd_asan_ubsan, cas s3 storage, parallel, {batch}/{total_batches}",
+                runs_on=RunnerLabels.AMD_MEDIUM_CPU,
+                requires=[ArtifactNames.CH_AMD_ASAN_UBSAN_GH],
+            )
+            for total_batches in (2,)
+            for batch in range(1, total_batches + 1)
+        ],
+        *[
+            Job.ParamSet(
+                parameter=f"amd_tsan, cas s3 storage, parallel, {batch}/{total_batches}",
+                runs_on=RunnerLabels.AMD_MEDIUM,
+                requires=[ArtifactNames.CH_AMD_TSAN_GH],
+            )
+            for total_batches in (2,)
+            for batch in range(1, total_batches + 1)
+        ],
+        *[
+            Job.ParamSet(
+                parameter=f"amd_msan, cas s3 storage, parallel, {batch}/{total_batches}",
+                runs_on=RunnerLabels.FUNC_TESTER_AMD,
+                requires=[ArtifactNames.CH_AMD_MSAN_GH],
+            )
+            for total_batches in (3,)
+            for batch in range(1, total_batches + 1)
+        ],
+        Job.ParamSet(
+            parameter="arm_binary, cas s3 storage, parallel",
+            runs_on=RunnerLabels.ARM_MEDIUM_CPU,
+            requires=[ArtifactNames.CH_ARM_BINARY_GH],
+        ),
+        # CAS over local object storage.
+        Job.ParamSet(
+            parameter="amd_binary, cas storage, parallel",
+            runs_on=RunnerLabels.AMD_MEDIUM_CPU,
+            requires=[ArtifactNames.CH_AMD_BINARY_GH],
+        ),
+    )

ci/jobs/functional_tests.py

@@ -142,6 +142,8 @@ def run_tests(
     "old analyzer": "--analyzer",
     "WasmEdge": "--wasm-engine wasmedge",
     "s3 storage": "--s3-storage",
+    "cas storage": "--cas-storage",
+    "cas s3 storage": "--cas-s3-storage",
     "DatabaseReplicated": "--db-replicated",
     "DatabaseOrdinary": "--db-ordinary",
     "wide parts enabled": "--wide-parts",
@@ -155,6 +157,8 @@ def run_tests(
 
 OPTIONS_TO_TEST_RUNNER_ARGUMENTS = {
     "s3 storage": "--s3-storage --no-stateful",
+    "cas storage": "--cas-storage",
+    "cas s3 storage": "--cas-s3-storage",
     "ParallelReplicas": "--no-zookeeper --no-shard --no-parallel-replicas",
     "AsyncInsert": " --no-async-insert",
     "DatabaseReplicated": " --no-stateful --replicated-database",
@@ -241,6 +245,7 @@ def main():
     is_targeted_check = False
     is_bugfix_validation = False
     is_s3_storage = False
+    is_cas_s3 = False
     is_azure_storage = False
     is_database_replicated = False
     is_shared_catalog = False
@@ -294,8 +299,13 @@ def main():
             is_excluded_from_llvm = True
         if "per_test_coverage" in to:
             is_per_test_coverage = True
-        if "s3 storage" in to:
+        if "s3 storage" in to and "cas" not in to:
+            # The CAS-over-s3 variant ("cas s3 storage") installs
+            # only its own default policy and must not pull in the s3 stateful-data / encrypted
+            # storage machinery, so it is deliberately excluded from is_s3_storage.
             is_s3_storage = True
+        if "cas s3 storage" in to:
+            is_cas_s3 = True
         if "azure" in to:
             is_azure_storage = True
         if "DatabaseReplicated" in to:
@@ -639,6 +649,14 @@ def main():
 
         def start():
             res = CH.start_minio(test_type="stateless") and CH.start_azurite()
+            if res and is_cas_s3:
+                # The CA-over-S3 pool lives on RustFS (M-W D-W8): the incarnation pool
+                # needs ENFORCED conditional deletes, which MinIO OSS lacks (the
+                # fail-closed capability probe rejects it). start_rustfs wipes its data
+                # dir per run, so no pool state bleeds between runs (the local-CA
+                # analogue is the per-run server-store wipe). MinIO keeps the non-CA
+                # s3 disks.
+                res = CH.start_rustfs()
             res = res and CH.start()
             res = res and CH.wait_ready()
             if res:

ci/jobs/scripts/check_style/various_checks.sh

@@ -230,6 +230,32 @@ done
 # CLICKHOUSE_URL already includes "?"
 git grep -P 'CLICKHOUSE_URL(|_HTTPS)(}|}/|/|)\?' $ROOT_PATH/tests/queries/0_stateless/*.sh && echo "CLICKHOUSE_URL already includes '?', use '&' to append query parameters"
 
+# A bare double quote inside a `-q """ ... """` block ends the string early.
+# Bash treats """ as an empty string followed by an open quote, so everything up to the next quote is
+# one argument. A quote anywhere inside -- including in a SQL comment -- closes it there, and the rest
+# of the block becomes shell words. The script stays syntactically valid, so `bash -n` and shellcheck
+# both pass; the only symptom is that the client receives a truncated query and reports a syntax error
+# pointing at whatever followed the quote. Escape it as \" or use single quotes.
+python3 - "$ROOT_PATH" <<'PYEOF'
+import glob, os, re, sys
+
+for path in sorted(glob.glob(os.path.join(sys.argv[1], "tests/queries/0_stateless/*.sh"))):
+    inside = False
+    with open(path, encoding="utf-8", errors="replace") as handle:
+        for number, line in enumerate(handle, 1):
+            if not inside:
+                if re.search(r'(-q|--query)\s+"""\s*$', line.strip()):
+                    inside = True
+                continue
+            # The block can close mid-line (`... LIMIT 10;"""`), so only what precedes the
+            # closing delimiter is still inside it.
+            body, closed, _ = line.partition('"""')
+            if '"' in re.sub(r'\\"', "", body):
+                print(f"{path}:{number}: bare double quote inside a -q \"\"\" block ends the SQL early")
+            if closed:
+                inside = False
+PYEOF
+
 # Large files checked into git.
 # Every byte committed is cloned by every contributor forever and cannot be removed without history rewriting.
 # Binary blobs (JARs, archives, .so, datasets) should be downloaded at test time or built from source.