From c16f0610d231fe937183e7f3a8e2da31b416384a Mon Sep 17 00:00:00 2001 From: James Rogers Date: Tue, 3 Nov 2026 11:13:31 +0000 Subject: [PATCH 1/8] Run the .NET consumer test against 51degrees-dotnet pipeline-dotnet has moved into 51degrees-dotnet, under pipeline/. The consumer test checks out the monorepo with its direct submodules, swaps the files under review into their new folder and runs the same test project from pipeline/, whose global.json pins the .NET 8 SDK. --- .github/workflows/consumer-tests.yml | 25 ++++++++++++++----------- README.md | 4 ++-- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/.github/workflows/consumer-tests.yml b/.github/workflows/consumer-tests.yml index 5783b8e..7a9ac93 100644 --- a/.github/workflows/consumer-tests.yml +++ b/.github/workflows/consumer-tests.yml @@ -25,7 +25,7 @@ on: workflow_dispatch: inputs: pipeline-dotnet-ref: - description: Branch, tag or SHA of pipeline-dotnet to test against + description: Branch, tag or SHA of 51degrees-dotnet to test against required: false default: main pipeline-java-ref: @@ -54,7 +54,7 @@ permissions: jobs: pipeline-dotnet: - name: pipeline-dotnet JavaScript builder + name: 51degrees-dotnet JavaScript builder runs-on: ubuntu-latest steps: - name: Check out the template under review @@ -62,28 +62,30 @@ jobs: with: path: template - - name: Check out pipeline-dotnet + - name: Check out 51degrees-dotnet uses: actions/checkout@v4 with: - repository: 51Degrees/pipeline-dotnet + repository: 51Degrees/51degrees-dotnet ref: ${{ inputs.pipeline-dotnet-ref || 'main' }} - submodules: recursive + # The Pipeline API packages are under pipeline/ and need only the + # direct submodules, not the data files under cxx/. + submodules: true path: consumer - name: Swap in the template under review run: | set -euo pipefail - dest=consumer/FiftyOne.Pipeline.Elements/FiftyOne.Pipeline.JavaScriptBuilderElement/Templates + dest=consumer/pipeline/JavaScriptBuilderElement/Templates if [ ! -d "$dest" ]; then echo "::error::$dest is missing - the submodule has moved and this workflow needs updating" exit 1 fi cp template/*.mustache "$dest/" # The templates are embedded resources. If the copy changed nothing - # then the run would exercise the revision pipeline-dotnet pins and + # then the run would exercise the revision 51degrees-dotnet pins and # report a pass that says nothing about this pull request. if git -C "$dest" diff --quiet; then - echo "::notice::identical to the revision pipeline-dotnet pins, nothing swapped in" + echo "::notice::identical to the revision 51degrees-dotnet pins, nothing swapped in" else git -C "$dest" --no-pager diff --stat fi @@ -98,10 +100,11 @@ jobs: chromedriver --version || echo "::warning::no chromedriver on the runner" - name: Run the JavaScript builder tests - working-directory: consumer + # pipeline/global.json pins the .NET 8 SDK installed above. + working-directory: consumer/pipeline run: > dotnet test -c Release - FiftyOne.Pipeline.Elements/FiftyOne.Pipeline.JavaScriptBuilderElementTests/FiftyOne.Pipeline.JavaScriptBuilderElementTests.csproj + Tests/JavaScriptBuilderElementTests/FiftyOne.Pipeline.JavaScriptBuilderElementTests.csproj --logger "console;verbosity=normal" --logger "trx;LogFileName=results.trx" @@ -109,7 +112,7 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: pipeline-dotnet-results + name: 51degrees-dotnet-results path: consumer/**/TestResults/*.trx if-no-files-found: warn diff --git a/README.md b/README.md index 374cc50..a11181b 100644 --- a/README.md +++ b/README.md @@ -159,13 +159,13 @@ Run it with Node 24 from the `tests` directory. Each check prints a line and the last line gives the totals. The `Template tests` workflow runs the same command on every pull request, and -the `Consumer tests` workflow builds pipeline-dotnet against the template and +the `Consumer tests` workflow builds 51degrees-dotnet against the template and runs the tests that drive it in a real browser. ## Shipping / Deployment This repo is not a stand-alone package, but is shipped as part of and used by each of the following repositories / packages: -- [pipeline-dotnet](https://github.com/51Degrees/pipeline-dotnet) as a submodule +- [51degrees-dotnet](https://github.com/51Degrees/51degrees-dotnet) as a submodule - [pipeline-java](https://github.com/51Degrees/pipeline-java) as a submodule - [pipeline-node](https://github.com/51Degrees/pipeline-node) as a submodule - [pipeline-python](https://github.com/51Degrees/pipeline-python) as a submodule From 0320d632cc153b10c924971d70dc584cd4e8a348 Mon Sep 17 00:00:00 2001 From: Eugene Dorfman Date: Tue, 3 Nov 2026 15:30:39 +0100 Subject: [PATCH 2/8] Build the Node consumer test from 51degrees-node --- .github/workflows/consumer-tests.yml | 18 ++++++++++-------- README.md | 2 +- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/.github/workflows/consumer-tests.yml b/.github/workflows/consumer-tests.yml index 5783b8e..ef437b2 100644 --- a/.github/workflows/consumer-tests.yml +++ b/.github/workflows/consumer-tests.yml @@ -33,7 +33,7 @@ on: required: false default: main pipeline-node-ref: - description: Branch, tag or SHA of pipeline-node to test against + description: Branch, tag or SHA of 51degrees-node to test against required: false default: main pipeline-python-ref: @@ -175,7 +175,7 @@ jobs: if-no-files-found: warn pipeline-node: - name: pipeline-node JavaScript builder + name: 51degrees-node JavaScript builder runs-on: ubuntu-latest steps: - name: Check out the template under review @@ -183,25 +183,27 @@ jobs: with: path: template - - name: Check out pipeline-node + - name: Check out 51degrees-node uses: actions/checkout@v4 with: - repository: 51Degrees/pipeline-node + repository: 51Degrees/51degrees-node ref: ${{ inputs.pipeline-node-ref || 'main' }} - submodules: recursive + # The Pipeline API packages are under pipeline/ and need only the + # direct submodules, not the data files under cxx/. + submodules: true path: consumer - name: Swap in the template under review run: | set -euo pipefail - dest=consumer/fiftyone.pipeline.core/javascript-templates + dest=consumer/pipeline/fiftyone.pipeline.core/javascript-templates if [ ! -d "$dest" ]; then echo "::error::$dest is missing - the submodule has moved and this workflow needs updating" exit 1 fi cp template/*.mustache "$dest/" if git -C "$dest" diff --quiet; then - echo "::notice::identical to the revision pipeline-node pins, nothing swapped in" + echo "::notice::identical to the revision 51degrees-node pins, nothing swapped in" else git -C "$dest" --no-pager diff --stat fi @@ -211,7 +213,7 @@ jobs: node-version: '22' - name: Run the JavaScript builder tests - working-directory: consumer/fiftyone.pipeline.core + working-directory: consumer/pipeline/fiftyone.pipeline.core run: | set -euo pipefail npm install diff --git a/README.md b/README.md index 374cc50..6a7621c 100644 --- a/README.md +++ b/README.md @@ -167,7 +167,7 @@ runs the tests that drive it in a real browser. This repo is not a stand-alone package, but is shipped as part of and used by each of the following repositories / packages: - [pipeline-dotnet](https://github.com/51Degrees/pipeline-dotnet) as a submodule - [pipeline-java](https://github.com/51Degrees/pipeline-java) as a submodule -- [pipeline-node](https://github.com/51Degrees/pipeline-node) as a submodule +- [51degrees-node](https://github.com/51Degrees/51degrees-node) as a submodule, under `pipeline/` - [pipeline-python](https://github.com/51Degrees/pipeline-python) as a submodule - [pipeline-php-core](https://github.com/51Degrees/pipeline-php-core) as a static dependency From bd453e5975ae9ba4751269cc8a749dbe712090fc Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 3 Nov 2026 16:32:27 +0000 Subject: [PATCH 3/8] Test the Python consumer from 51degrees-python pipeline-python has moved into 51degrees-python, where the Pipeline API packages are under pipeline/. Check out the monorepo with only its direct submodules, so the cxx data files are not fetched, and swap the template in at its new path. --- .github/workflows/consumer-tests.yml | 18 ++++++++++-------- README.md | 2 +- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/.github/workflows/consumer-tests.yml b/.github/workflows/consumer-tests.yml index 5783b8e..e013a10 100644 --- a/.github/workflows/consumer-tests.yml +++ b/.github/workflows/consumer-tests.yml @@ -37,7 +37,7 @@ on: required: false default: main pipeline-python-ref: - description: Branch, tag or SHA of pipeline-python to test against + description: Branch, tag or SHA of 51degrees-python to test against required: false default: main pipeline-php-core-ref: @@ -218,7 +218,7 @@ jobs: npx jest tests/javascriptbuilder.test.js pipeline-python: - name: pipeline-python JavaScript builder + name: 51degrees-python JavaScript builder runs-on: ubuntu-latest steps: - name: Check out the template under review @@ -226,25 +226,27 @@ jobs: with: path: template - - name: Check out pipeline-python + - name: Check out 51degrees-python uses: actions/checkout@v4 with: - repository: 51Degrees/pipeline-python + repository: 51Degrees/51degrees-python ref: ${{ inputs.pipeline-python-ref || 'main' }} - submodules: recursive + # The Pipeline API packages are under pipeline/ and need only the + # direct submodules, not the data files under cxx/. + submodules: true path: consumer - name: Swap in the template under review run: | set -euo pipefail - dest=consumer/fiftyone_pipeline_core/src/fiftyone_pipeline_core/js_templates + dest=consumer/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/js_templates if [ ! -d "$dest" ]; then echo "::error::$dest is missing - the submodule has moved and this workflow needs updating" exit 1 fi cp template/*.mustache "$dest/" if git -C "$dest" diff --quiet; then - echo "::notice::identical to the revision pipeline-python pins, nothing swapped in" + echo "::notice::identical to the revision 51degrees-python pins, nothing swapped in" else git -C "$dest" --no-pager diff --stat fi @@ -254,7 +256,7 @@ jobs: python-version: '3.13' - name: Run the JavaScript builder tests - working-directory: consumer/fiftyone_pipeline_core + working-directory: consumer/pipeline/fiftyone_pipeline_core run: | set -euo pipefail python -m pip install --upgrade pip diff --git a/README.md b/README.md index 374cc50..4d21087 100644 --- a/README.md +++ b/README.md @@ -168,7 +168,7 @@ This repo is not a stand-alone package, but is shipped as part of and used by ea - [pipeline-dotnet](https://github.com/51Degrees/pipeline-dotnet) as a submodule - [pipeline-java](https://github.com/51Degrees/pipeline-java) as a submodule - [pipeline-node](https://github.com/51Degrees/pipeline-node) as a submodule -- [pipeline-python](https://github.com/51Degrees/pipeline-python) as a submodule +- [51degrees-python](https://github.com/51Degrees/51degrees-python) as a submodule, under `pipeline/` - [pipeline-php-core](https://github.com/51Degrees/pipeline-php-core) as a static dependency Wherever it is a submodule it will be updated by `Nightly Submodule Update` action, wherever it is a static dependency it will be updated by the `Nightly Package Update` action within a target repository. From 2dcced0ac2bb16ac1125c5dafa86a6c8c69a1f0e Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 3 Nov 2026 18:44:50 +0000 Subject: [PATCH 4/8] Check out the Rust SDK as 51degrees-rust (#34) The rust repository is now 51degrees-rust (51Degrees/51degrees-rust#206). The Rust job checks it out by the new name, and its job name, step name, input description and notices say 51degrees-rust. The job id and the rust-ref input keep their names, because an id can't start with a digit. --- .github/workflows/consumer-tests.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/consumer-tests.yml b/.github/workflows/consumer-tests.yml index 5783b8e..a9740cc 100644 --- a/.github/workflows/consumer-tests.yml +++ b/.github/workflows/consumer-tests.yml @@ -45,7 +45,7 @@ on: required: false default: main rust-ref: - description: Branch, tag or SHA of rust to test against + description: Branch, tag or SHA of 51degrees-rust to test against required: false default: main @@ -311,7 +311,7 @@ jobs: vendor/bin/phpunit --testsuite Unit rust: - name: rust JavaScript builder + name: 51degrees-rust JavaScript builder runs-on: ubuntu-latest steps: - name: Check out the template under review @@ -319,10 +319,10 @@ jobs: with: path: template - - name: Check out rust + - name: Check out 51degrees-rust uses: actions/checkout@v4 with: - repository: 51Degrees/rust + repository: 51Degrees/51degrees-rust ref: ${{ inputs.rust-ref || 'main' }} submodules: recursive path: consumer @@ -338,9 +338,9 @@ jobs: fi cp template/JavaScriptResource.mustache "consumer/$file" if git -C consumer diff --quiet -- "$file"; then - echo "::notice::the copy in rust is identical to the revision under review" + echo "::notice::the copy in 51degrees-rust is identical to the revision under review" else - echo "::warning::the copy in rust differs from the revision under review" + echo "::warning::the copy in 51degrees-rust differs from the revision under review" git -C consumer --no-pager diff --stat -- "$file" fi From c301be03ceb34a4505ae0072ae42bca0dc06bb36 Mon Sep 17 00:00:00 2001 From: ben51degrees Date: Tue, 3 Nov 2026 18:45:04 +0000 Subject: [PATCH 5/8] Build the Java consumer test from 51degrees-java (#36) --- .github/workflows/consumer-tests.yml | 20 +++++++++++--------- README.md | 2 +- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/.github/workflows/consumer-tests.yml b/.github/workflows/consumer-tests.yml index a9740cc..cd2390d 100644 --- a/.github/workflows/consumer-tests.yml +++ b/.github/workflows/consumer-tests.yml @@ -29,7 +29,7 @@ on: required: false default: main pipeline-java-ref: - description: Branch, tag or SHA of pipeline-java to test against + description: Branch, tag or SHA of 51degrees-java to test against required: false default: main pipeline-node-ref: @@ -114,7 +114,7 @@ jobs: if-no-files-found: warn pipeline-java: - name: pipeline-java JavaScript builder + name: 51degrees-java JavaScript builder runs-on: ubuntu-latest steps: - name: Check out the template under review @@ -122,25 +122,27 @@ jobs: with: path: template - - name: Check out pipeline-java + - name: Check out 51degrees-java uses: actions/checkout@v4 with: - repository: 51Degrees/pipeline-java + repository: 51Degrees/51degrees-java ref: ${{ inputs.pipeline-java-ref || 'main' }} - submodules: recursive + # The Pipeline API packages are under pipeline/ and need only the + # direct submodules, not the data files under cxx/. + submodules: true path: consumer - name: Swap in the template under review run: | set -euo pipefail - dest=consumer/pipeline.javascriptbuilder/src/main/resources/fiftyone/pipeline/javascriptbuilder/templates + dest=consumer/pipeline/pipeline.javascriptbuilder/src/main/resources/fiftyone/pipeline/javascriptbuilder/templates if [ ! -d "$dest" ]; then echo "::error::$dest is missing - the submodule has moved and this workflow needs updating" exit 1 fi cp template/*.mustache "$dest/" if git -C "$dest" diff --quiet; then - echo "::notice::identical to the revision pipeline-java pins, nothing swapped in" + echo "::notice::identical to the revision 51degrees-java pins, nothing swapped in" else git -C "$dest" --no-pager diff --stat fi @@ -160,7 +162,7 @@ jobs: chromedriver --version || echo "::warning::no chromedriver on the runner" - name: Run the JavaScript builder tests - working-directory: consumer + working-directory: consumer/pipeline run: | set -euo pipefail mvn -B -q -pl pipeline.javascriptbuilder -am -DskipTests install @@ -170,7 +172,7 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: pipeline-java-results + name: 51degrees-java-results path: consumer/**/surefire-reports/*.xml if-no-files-found: warn diff --git a/README.md b/README.md index 374cc50..15ab7f9 100644 --- a/README.md +++ b/README.md @@ -166,7 +166,7 @@ runs the tests that drive it in a real browser. This repo is not a stand-alone package, but is shipped as part of and used by each of the following repositories / packages: - [pipeline-dotnet](https://github.com/51Degrees/pipeline-dotnet) as a submodule -- [pipeline-java](https://github.com/51Degrees/pipeline-java) as a submodule +- [51degrees-java](https://github.com/51Degrees/51degrees-java) as a submodule, under `pipeline/` - [pipeline-node](https://github.com/51Degrees/pipeline-node) as a submodule - [pipeline-python](https://github.com/51Degrees/pipeline-python) as a submodule - [pipeline-php-core](https://github.com/51Degrees/pipeline-php-core) as a static dependency From 78968615b828f805d08eae0197433c38b9b977d2 Mon Sep 17 00:00:00 2001 From: Yaroslav Date: Tue, 3 Nov 2026 20:45:22 +0200 Subject: [PATCH 6/8] ci: test the PHP consumer from 51degrees-php (#38) pipeline-php-core is now a read-only mirror. The PHP job checks out the monorepo, swaps the template into pipeline/core and runs that package's unit tests there. --- .github/workflows/consumer-tests.yml | 19 ++++++++++--------- README.md | 2 +- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/.github/workflows/consumer-tests.yml b/.github/workflows/consumer-tests.yml index cd2390d..6927fa0 100644 --- a/.github/workflows/consumer-tests.yml +++ b/.github/workflows/consumer-tests.yml @@ -41,7 +41,7 @@ on: required: false default: main pipeline-php-core-ref: - description: Branch, tag or SHA of pipeline-php-core to test against + description: Branch, tag or SHA of 51degrees-php to test against required: false default: main rust-ref: @@ -265,7 +265,7 @@ jobs: python -m pytest tests/test_javascriptbuilder.py -q pipeline-php-core: - name: pipeline-php-core JavaScript builder + name: 51degrees-php JavaScript builder runs-on: ubuntu-latest steps: - name: Check out the template under review @@ -273,12 +273,13 @@ jobs: with: path: template - - name: Check out pipeline-php-core + # pipeline/core of the monorepo carries the copy. Its unit tests need + # no submodule, so the C sources and data files are not checked out. + - name: Check out 51degrees-php uses: actions/checkout@v4 with: - repository: 51Degrees/pipeline-php-core + repository: 51Degrees/51degrees-php ref: ${{ inputs.pipeline-php-core-ref || 'main' }} - submodules: recursive path: consumer # A copy rather than a submodule, so it is compared with the file as @@ -287,16 +288,16 @@ jobs: - name: Swap in the template under review run: | set -euo pipefail - file=javascript-templates/JavaScriptResource.mustache + file=pipeline/core/javascript-templates/JavaScriptResource.mustache if [ ! -f "consumer/$file" ]; then echo "::error::consumer/$file is missing - the copy has moved and this workflow needs updating" exit 1 fi cp template/JavaScriptResource.mustache "consumer/$file" if git -C consumer diff --quiet -- "$file"; then - echo "::notice::the copy in pipeline-php-core is identical to the revision under review" + echo "::notice::the copy in 51degrees-php is identical to the revision under review" else - echo "::warning::the copy in pipeline-php-core differs from the revision under review" + echo "::warning::the copy in 51degrees-php differs from the revision under review" git -C consumer --no-pager diff --stat -- "$file" fi @@ -306,7 +307,7 @@ jobs: tools: composer - name: Run the unit tests - working-directory: consumer + working-directory: consumer/pipeline/core run: | set -euo pipefail composer install --no-interaction --no-progress diff --git a/README.md b/README.md index 15ab7f9..4782c0e 100644 --- a/README.md +++ b/README.md @@ -169,7 +169,7 @@ This repo is not a stand-alone package, but is shipped as part of and used by ea - [51degrees-java](https://github.com/51Degrees/51degrees-java) as a submodule, under `pipeline/` - [pipeline-node](https://github.com/51Degrees/pipeline-node) as a submodule - [pipeline-python](https://github.com/51Degrees/pipeline-python) as a submodule -- [pipeline-php-core](https://github.com/51Degrees/pipeline-php-core) as a static dependency +- [51degrees-php](https://github.com/51Degrees/51degrees-php/tree/main/pipeline/core) as a static dependency Wherever it is a submodule it will be updated by `Nightly Submodule Update` action, wherever it is a static dependency it will be updated by the `Nightly Package Update` action within a target repository. From afd9a1ae83ed68f2d33a8eebf4440dcdd50aea6b Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 3 Nov 2026 18:45:40 +0000 Subject: [PATCH 7/8] Check that only rc merges into the default branch (#40) --- .github/workflows/only-rc-into-main.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .github/workflows/only-rc-into-main.yml diff --git a/.github/workflows/only-rc-into-main.yml b/.github/workflows/only-rc-into-main.yml new file mode 100644 index 0000000..69e19ca --- /dev/null +++ b/.github/workflows/only-rc-into-main.yml @@ -0,0 +1,11 @@ +name: Only rc into main + +# A pull request into the default branch must come from rc. See +# 51Degrees/common-ci .github/workflows/only-rc-into-main.yml. +on: + pull_request: + +jobs: + rc: + name: Only rc into main + uses: 51Degrees/common-ci/.github/workflows/only-rc-into-main.yml@main From 4d181ff964d4489381d2a0d681b000fa7e0ee3f6 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 3 Nov 2026 21:43:01 +0000 Subject: [PATCH 8/8] Copy the OWID source into fodid before the Rust builder tests (#44) --- .github/workflows/consumer-tests.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/consumer-tests.yml b/.github/workflows/consumer-tests.yml index b7223fe..407312c 100644 --- a/.github/workflows/consumer-tests.yml +++ b/.github/workflows/consumer-tests.yml @@ -356,6 +356,13 @@ jobs: - uses: dtolnay/rust-toolchain@stable + # fodid compiles the OWID source in, and this script copies it from the + # owid-rust submodule into fodid/src/owid, which git ignores. + - name: Copy the OWID source into fodid + working-directory: consumer + shell: pwsh + run: ./ci/copy-owid-source.ps1 + - name: Run the JavaScript builder tests working-directory: consumer run: cargo test -p fiftyone-javascript-builder