diff --git a/.github/actions/build-recipes/action.yaml b/.github/actions/build-recipes/action.yaml new file mode 100644 index 000000000..ff6dd5cc4 --- /dev/null +++ b/.github/actions/build-recipes/action.yaml @@ -0,0 +1,76 @@ +# Build and upload some of a distribution's recipes: the steps of every job of the +# build workflows (.github/build-template.yaml). One action keeps each generated job +# a few lines long, so many more jobs fit GitHub's 500 KiB workflow file limit. +name: Build recipes +description: Build the given recipes of distros//work/recipes and upload them +inputs: + recipes: + description: Space-separated recipe names + required: true + platform: + description: Target platform (conda subdir) + required: true + token: + description: anaconda.org API token of the distribution's organization + required: false + keep-file: + description: File kept as an artifact (the build times the job recorded) + required: false +runs: + using: composite + steps: + # git auto-maintenance can leave a lock file in the git clones pixi caches, + # which breaks every later run restoring that cache + - name: Disable git auto-maintenance + shell: bash + run: git config --global maintenance.auto false && git config --global gc.auto 0 + - name: Setup pixi + uses: prefix-dev/setup-pixi@v0.11.0 + with: + pixi-version: v0.81.0 + cache: "true" + log-level: v + frozen: "true" + - if: runner.os == 'Windows' + uses: egor-tensin/cleanup-path@v5 + with: + dirs: C:\Program Files\Git\usr\bin;C:\Program Files\Git\bin;C:\Program Files\Git\cmd;C:\Program Files\Git\mingw64\bin + - if: runner.os == 'Windows' + name: conda-forge build setup + shell: cmd + run: | + :: hosted runners may have libcrypto/libssl DLLs in System32, found before + :: the environment's openssl + DEL C:\Windows\System32\libcrypto-1_1-x64.dll || (Echo Ignoring failure to delete libcrypto) + DEL C:\Windows\System32\libssl-1_1-x64.dll || (Echo Ignoring failure to delete libssl) + - if: runner.os == 'Windows' + name: Build + shell: cmd + env: + ANACONDA_API_TOKEN: ${{ inputs.token }} + CURRENT_RECIPES: ${{ inputs.recipes }} + CONDA_BLD_PATH: C:\\bld\\ + VINCA_CUSTOM_CMAKE_BUILD_DIR: C:\\x\\ + PYTHONUNBUFFERED: "1" + run: call .scripts\build_win.bat + - if: runner.os != 'Windows' + name: Build + shell: bash + env: + ANACONDA_API_TOKEN: ${{ inputs.token }} + CURRENT_RECIPES: ${{ inputs.recipes }} + BUILD_TARGET: ${{ inputs.platform }} + PYTHONUNBUFFERED: "1" + CI: github + run: | + export GIT_BRANCH=${GITHUB_REF_NAME} + export FEEDSTOCK_NAME=${GITHUB_REPOSITORY##*/} + .scripts/build_unix.sh --target "$BUILD_TARGET" + - name: Keep build times + if: always() && inputs.keep-file != '' + uses: actions/upload-artifact@v6 + with: + name: ${{ github.job }}-${{ github.run_attempt }} + path: ${{ inputs.keep-file }} + if-no-files-found: ignore + retention-days: 30 diff --git a/.github/build-template.yaml b/.github/build-template.yaml new file mode 100644 index 000000000..a9431aade --- /dev/null +++ b/.github/build-template.yaml @@ -0,0 +1,48 @@ +# Template of the build workflows (`pixi run rs gha`, vinca-rs workflow). +# Not a workflow itself: vinca-rs copies jobs.build once per job, sets each copy's +# name and needs (exactly the jobs building its recipes' dependencies) and replaces +# @recipes@ with its recipes; @distro@, @platform@, @branch@, @secret@ and +# @keep_file@ come from tools/robostack.py. Selectors (if/then/else) are resolved +# for the platform. The steps are .github/actions/build-recipes. +name: Build branch +run-name: "@distro@ @platform@" +on: + push: + branches: ["@branch@"] +# one run per distribution and platform at a time: a newer run (the latest main's +# recipes and workflow) cancels the older one, and skips what that one uploaded +# already. Re-run failed jobs only in the newest run. +concurrency: + group: build-@distro@-@platform@ + cancel-in-progress: true +env: + ROBOSTACK_DISTRO: "@distro@" + # optional repository variable: upload to this prefix.dev channel instead + # (e.g. a test channel), or 'none' to only build + ROBOSTACK_UPLOAD_CHANNEL: ${{ vars.ROBOSTACK_UPLOAD_CHANNEL }} +jobs: + build: + runs-on: + if: linux and x86_64 + then: ubuntu-latest + else: + if: linux + then: ubuntu-24.04-arm + else: + if: osx and x86_64 + then: macos-15-intel + else: + if: osx + then: macos-15 + else: windows-2022 + # jobs that declare permissions get none they don't list; prefix.dev uploads use + # trusted publishing (id-token), attestations are published too + permissions: {contents: read, id-token: write, attestations: write} + steps: + - uses: actions/checkout@v6 + - uses: ./.github/actions/build-recipes + with: + recipes: "@recipes@" + platform: "@platform@" + token: ${{ secrets.@secret@ }} + keep-file: "@keep_file@" diff --git a/.github/workflows/testpr.yaml b/.github/workflows/testpr.yaml index 71b7ee8f9..3d5bed1c9 100644 --- a/.github/workflows/testpr.yaml +++ b/.github/workflows/testpr.yaml @@ -1,6 +1,6 @@ # Pull-request build: only the distributions a PR touches are built # (a change outside distros/ affects all of them), on all five platforms. -name: Build +name: Pull request on: pull_request: @@ -10,8 +10,9 @@ on: description: Space-separated distributions to build (default all) required: false +# a fixed key, not github.workflow: renaming the workflow mustn't let two runs overlap concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + group: pr-build-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true permissions: @@ -24,6 +25,7 @@ env: jobs: plan: + name: "plan: distros and platforms to build" runs-on: ubuntu-latest permissions: contents: read @@ -67,6 +69,7 @@ jobs: echo "targets=$(pixi run python tools/robostack.py build-targets --labels "$labels")" | tee -a "$GITHUB_OUTPUT" check: + name: "checks: tooling tests, generated files" runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 @@ -97,11 +100,11 @@ jobs: fail-fast: false matrix: distro: ${{ fromJSON(needs.plan.outputs.distros) }} - name: ${{ matrix.distro }} patches + name: "${{ matrix.distro }}: patches apply" runs-on: ubuntu-latest env: DISTRO: ${{ matrix.distro }} - # generate-recipes -n leaves out what is on the channel, like the builds do + # generate-recipes leaves out what is on the channel (same build string), like the builds do ROBOSTACK_UPLOAD_CHANNEL: ${{ vars.ROBOSTACK_UPLOAD_CHANNEL }} steps: - uses: actions/checkout@v6 @@ -123,7 +126,7 @@ jobs: all=$(mktemp -d) for platform in linux-64 linux-aarch64 osx-64 osx-arm64 win-64; do for attempt in 1 2 3; do - if pixi run rs "$DISTRO" generate-recipes --platform "$platform" -n; then break; fi + if pixi run rs "$DISTRO" generate-recipes --platform "$platform"; then break; fi if [[ "$attempt" == 3 ]]; then echo "Recipe generation failed for $platform" >&2; exit 1; fi sleep 15 done @@ -147,7 +150,7 @@ jobs: matrix: distro: ${{ fromJSON(needs.plan.outputs.distros) }} target: ${{ fromJSON(needs.plan.outputs.targets) }} # tools/robostack.py BUILD_TARGETS - name: ${{ matrix.distro }} ${{ matrix.target.platform }} + name: "${{ matrix.distro }} ${{ matrix.target.platform }}: build" runs-on: ${{ matrix.target.os }} env: DISTRO: ${{ matrix.distro }} @@ -217,7 +220,7 @@ jobs: run: | set -e for attempt in 1 2 3; do - if pixi run -v rs "$DISTRO" generate-recipes --platform "$PLATFORM" -n; then break; fi + if pixi run -v rs "$DISTRO" generate-recipes --platform "$PLATFORM"; then break; fi if [[ "${attempt}" == "3" ]]; then echo "Recipe generation failed after 3 attempts" >&2; exit 1; fi echo "Recipe generation attempt ${attempt} failed; retrying..." >&2 sleep 15 diff --git a/.scripts/build_unix.sh b/.scripts/build_unix.sh index 12c548dac..4e0d24ff1 100755 --- a/.scripts/build_unix.sh +++ b/.scripts/build_unix.sh @@ -40,7 +40,9 @@ pixi run -v rs "${ROBOSTACK_DISTRO}" build-ci "${recipe_args[@]}" --output-dir $ # Check if it build something, this is a hotfix for the skips inside additional_recipes shopt -s nullglob -conda_packages=( "${CONDA_BLD_PATH}/${target}"*/*.conda ) +# noarch packages (the ros--* compatibility packages) are built and uploaded +# by every platform's jobs: the first upload wins, the others are skipped +conda_packages=( "${CONDA_BLD_PATH}/${target}"*/*.conda "${CONDA_BLD_PATH}/noarch/"*.conda ) if (( ${#conda_packages[@]} > 0 )); then # Upload packages one-by-one; the upload task (tools/robostack.py) uploads them to # prefix.dev and anaconda.org and skips packages that are already there. @@ -48,6 +50,6 @@ if (( ${#conda_packages[@]} > 0 )); then pixi run rs "${ROBOSTACK_DISTRO}" upload "${conda_package}" done else - echo "Warning: No .conda files found in ${CONDA_BLD_PATH}/${target}" + echo "Warning: No .conda files found in ${CONDA_BLD_PATH}/${target} or ${CONDA_BLD_PATH}/noarch" echo "This might be due to all the packages being skipped" fi diff --git a/.scripts/build_win.bat b/.scripts/build_win.bat index 97b29739d..eb9c5728a 100644 --- a/.scripts/build_win.bat +++ b/.scripts/build_win.bat @@ -37,17 +37,22 @@ for %%X in (%CURRENT_RECIPES%) do set "RECIPE_ARGS=!RECIPE_ARGS! --recipe %FEEDS pixi run -v rs %ROBOSTACK_DISTRO% build-ci !RECIPE_ARGS! --output-dir %CONDA_BLD_PATH% if errorlevel 1 exit 1 -:: Check if .conda files exist in the win-64 directory -if exist "%CONDA_BLD_PATH%\win-64\*.conda" ( - echo Found .conda files, starting upload... - rem Upload packages one-by-one; the upload task (tools/robostack.py) uploads them to - rem prefix.dev and anaconda.org and skips packages that are already there. - for %%F in ("%CONDA_BLD_PATH%\win-64\*.conda") do ( - echo Uploading %%~fF - pixi run rs %ROBOSTACK_DISTRO% upload "%%~fF" - if errorlevel 1 exit 1 +:: Upload the win-64 and noarch packages one-by-one; the upload task (tools/robostack.py) +:: uploads them to prefix.dev and anaconda.org and skips packages that are already there +:: (noarch ones, the ros--* compatibility packages, are built and uploaded by +:: every platform's jobs: the first upload wins). +set "UPLOADED=" +for %%D in (win-64 noarch) do ( + if exist "%CONDA_BLD_PATH%\%%D\*.conda" ( + for %%F in ("%CONDA_BLD_PATH%\%%D\*.conda") do ( + echo Uploading %%~fF + pixi run rs %ROBOSTACK_DISTRO% upload "%%~fF" + if errorlevel 1 exit 1 + set "UPLOADED=1" + ) ) -) else ( - echo Warning: No .conda files found in %CONDA_BLD_PATH%\win-64 +) +if not defined UPLOADED ( + echo Warning: No .conda files found in %CONDA_BLD_PATH%\win-64 or %CONDA_BLD_PATH%\noarch echo This might be due to all the packages being skipped ) diff --git a/README.md b/README.md index c3825a728..9792dcbd5 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,8 @@ distribution comes from the channel. The older names keep working: for every pac there is also a small `ros--` package that just depends on `ros2-` of the same version (vinca's `package_name_mode: both` in `shared/vinca.yaml`), so existing environments and instructions such as -`pixi add ros-jazzy-desktop` don't break. +`pixi add ros-jazzy-desktop` don't break. These are `noarch` packages, built once for +every platform. **A package is missing?** Open a *Package request* issue. robostack-bot opens a pull request that adds it to the build, and a maintainer reviews it. diff --git a/distros/humble/pkg_additional_info.yaml b/distros/humble/pkg_additional_info.yaml index 771647d4c..ac4a317a5 100644 --- a/distros/humble/pkg_additional_info.yaml +++ b/distros/humble/pkg_additional_info.yaml @@ -129,3 +129,5 @@ urdfdom_headers: # Remove build_number fields below on new full rebuild zenoh_cpp_vendor: additional_cmake_args: "-DAMENT_VENDOR_POLICY=NEVER_VENDOR_IGNORE_SATISFIED_CHECK -DUSE_SYSTEM_ZENOH=ON" +tracetools: # vinca-rs: remove_host pkg-config also removes the native-only host pkg-config + build_number: 26 diff --git a/distros/humble/rosdistro_additional_recipes.yaml b/distros/humble/rosdistro_additional_recipes.yaml index 639a7cd7e..1a8de2021 100644 --- a/distros/humble/rosdistro_additional_recipes.yaml +++ b/distros/humble/rosdistro_additional_recipes.yaml @@ -82,13 +82,6 @@ med7_moveit_config: rev: 2e932d4e062966afe216710a7cc2fe50a8099365 version: 0.3.0 additional_folder: lbr_moveit_config/med7_moveit_config -mujoco_vendor: - # jazzy's 0.1.0: unlike humble's 0.0.8 it honours AMENT_VENDOR_POLICY, so it uses - # conda-forge's libmujoco (shared pkg_additional_info) instead of downloading MuJoCo - # and symlinking it (rejected on Windows; macOS isn't supported by the download) - tag: release/jazzy/mujoco_vendor/0.1.0-1 - url: https://github.com/ros2-gbp/mujoco_vendor-release.git - version: 0.1.0 opennav_coverage: # Not released in rosdistro; humble-v2 branch (Fields2Cover v2) url: https://github.com/open-navigation/opennav_coverage.git diff --git a/distros/jazzy/rosdistro_additional_recipes.yaml b/distros/jazzy/rosdistro_additional_recipes.yaml index d8e00e703..2e7562197 100644 --- a/distros/jazzy/rosdistro_additional_recipes.yaml +++ b/distros/jazzy/rosdistro_additional_recipes.yaml @@ -145,7 +145,7 @@ ros2_easy_test: version: 0.3.0 additional_folder: ros2_easy_test rplidar_ros: - # 2.1.4 (as humble) for its win32 SDK arch; the released 2.1.0 is POSIX only + override: "2.1.4 (as humble) for its win32 SDK arch; the released 2.1.0 is POSIX only" tag: 2.1.4 url: https://github.com/Slamtec/rplidar_ros.git version: 2.1.4 diff --git a/distros/lyrical/rosdistro_additional_recipes.yaml b/distros/lyrical/rosdistro_additional_recipes.yaml index d65e6bd6b..2e8b508b2 100644 --- a/distros/lyrical/rosdistro_additional_recipes.yaml +++ b/distros/lyrical/rosdistro_additional_recipes.yaml @@ -22,10 +22,6 @@ autoware_behavior_velocity_stop_line_module: tag: release/humble/autoware_behavior_velocity_stop_line_module/1.7.0-1 url: https://github.com/ros2-gbp/autoware_core-release.git version: 1.7.0 -autoware_cmake: - tag: release/jazzy/autoware_cmake/1.1.0-1 - url: https://github.com/ros2-gbp/autoware_cmake-release.git - version: 1.1.0 autoware_component_interface_specs: tag: release/humble/autoware_component_interface_specs/1.7.0-1 url: https://github.com/ros2-gbp/autoware_core-release.git @@ -106,14 +102,6 @@ autoware_gyro_odometer: tag: release/humble/autoware_gyro_odometer/1.7.0-1 url: https://github.com/ros2-gbp/autoware_core-release.git version: 1.7.0 -autoware_internal_localization_msgs: - tag: release/jazzy/autoware_internal_localization_msgs/1.10.0-1 - url: https://github.com/ros2-gbp/autoware_internal_msgs-release.git - version: 1.10.0 -autoware_internal_planning_msgs: - tag: release/humble/autoware_internal_planning_msgs/1.12.0-1 - url: https://github.com/ros2-gbp/autoware_internal_msgs-release.git - version: 1.12.0 autoware_interpolation: tag: release/humble/autoware_interpolation/1.7.0-1 url: https://github.com/ros2-gbp/autoware_core-release.git @@ -123,10 +111,12 @@ autoware_kalman_filter: url: https://github.com/ros2-gbp/autoware_core-release.git version: 1.7.0 autoware_lanelet2_extension: + override: "0.8.0 is newer than lyrical's release 0.7.2" tag: release/humble/autoware_lanelet2_extension/0.8.0-1 url: https://github.com/ros2-gbp/autoware_lanelet2_extension-release.git version: 0.8.0 autoware_lanelet2_extension_python: + override: "0.8.0 is newer than lyrical's release 0.7.2" tag: release/humble/autoware_lanelet2_extension_python/0.8.0-1 url: https://github.com/ros2-gbp/autoware_lanelet2_extension-release.git version: 0.8.0 @@ -138,10 +128,6 @@ autoware_lanelet2_utils: tag: release/humble/autoware_lanelet2_utils/1.7.0-1 url: https://github.com/ros2-gbp/autoware_core-release.git version: 1.7.0 -autoware_localization_msgs: - tag: release/humble/autoware_localization_msgs/1.9.0-1 - url: https://github.com/ros2-gbp/autoware_msgs-release.git - version: 1.9.0 autoware_localization_util: tag: release/humble/autoware_localization_util/1.7.0-1 url: https://github.com/ros2-gbp/autoware_core-release.git @@ -214,10 +200,6 @@ autoware_planning_factor_interface: tag: release/humble/autoware_planning_factor_interface/1.7.0-1 url: https://github.com/ros2-gbp/autoware_core-release.git version: 1.7.0 -autoware_planning_msgs: - tag: release/humble/autoware_planning_msgs/1.9.0-1 - url: https://github.com/ros2-gbp/autoware_msgs-release.git - version: 1.9.0 autoware_planning_test_manager: tag: release/humble/autoware_planning_test_manager/1.7.0-1 url: https://github.com/ros2-gbp/autoware_core-release.git @@ -262,10 +244,6 @@ autoware_stop_filter: tag: release/humble/autoware_stop_filter/1.7.0-1 url: https://github.com/ros2-gbp/autoware_core-release.git version: 1.7.0 -autoware_system_msgs: - tag: release/humble/autoware_system_msgs/1.9.0-1 - url: https://github.com/ros2-gbp/autoware_msgs-release.git - version: 1.9.0 autoware_test_node: tag: release/humble/autoware_test_node/1.7.0-1 url: https://github.com/ros2-gbp/autoware_core-release.git @@ -286,54 +264,6 @@ autoware_twist2accel: tag: release/humble/autoware_twist2accel/1.7.0-1 url: https://github.com/ros2-gbp/autoware_core-release.git version: 1.7.0 -autoware_utils: - tag: release/jazzy/autoware_utils/1.4.2-1 - url: https://github.com/ros2-gbp/autoware_utils-release.git - version: 1.4.2 -autoware_utils_debug: - tag: release/jazzy/autoware_utils_debug/1.4.2-1 - url: https://github.com/ros2-gbp/autoware_utils-release.git - version: 1.4.2 -autoware_utils_diagnostics: - tag: release/jazzy/autoware_utils_diagnostics/1.4.2-1 - url: https://github.com/ros2-gbp/autoware_utils-release.git - version: 1.4.2 -autoware_utils_geometry: - tag: release/jazzy/autoware_utils_geometry/1.4.2-1 - url: https://github.com/ros2-gbp/autoware_utils-release.git - version: 1.4.2 -autoware_utils_logging: - tag: release/jazzy/autoware_utils_logging/1.4.2-1 - url: https://github.com/ros2-gbp/autoware_utils-release.git - version: 1.4.2 -autoware_utils_math: - tag: release/jazzy/autoware_utils_math/1.4.2-1 - url: https://github.com/ros2-gbp/autoware_utils-release.git - version: 1.4.2 -autoware_utils_pcl: - tag: release/jazzy/autoware_utils_pcl/1.4.2-1 - url: https://github.com/ros2-gbp/autoware_utils-release.git - version: 1.4.2 -autoware_utils_rclcpp: - tag: release/jazzy/autoware_utils_rclcpp/1.4.2-1 - url: https://github.com/ros2-gbp/autoware_utils-release.git - version: 1.4.2 -autoware_utils_system: - tag: release/jazzy/autoware_utils_system/1.4.2-1 - url: https://github.com/ros2-gbp/autoware_utils-release.git - version: 1.4.2 -autoware_utils_tf: - tag: release/jazzy/autoware_utils_tf/1.4.2-1 - url: https://github.com/ros2-gbp/autoware_utils-release.git - version: 1.4.2 -autoware_utils_uuid: - tag: release/jazzy/autoware_utils_uuid/1.4.2-1 - url: https://github.com/ros2-gbp/autoware_utils-release.git - version: 1.4.2 -autoware_utils_visualization: - tag: release/jazzy/autoware_utils_visualization/1.4.2-1 - url: https://github.com/ros2-gbp/autoware_utils-release.git - version: 1.4.2 autoware_vehicle_info_utils: tag: release/humble/autoware_vehicle_info_utils/1.7.0-1 url: https://github.com/ros2-gbp/autoware_core-release.git @@ -483,7 +413,7 @@ ros2_easy_test: # Until: https://github.com/felixdivo/ros2-easy-test/issues/53 version: 0.3.0 additional_folder: ros2_easy_test rplidar_ros: - # 2.1.4 (as humble) for its win32 SDK arch; the released 2.1.0 is POSIX only + override: "2.1.4 (as humble) for its win32 SDK arch; the released 2.1.0 is POSIX only" tag: 2.1.4 url: https://github.com/Slamtec/rplidar_ros.git version: 2.1.4 diff --git a/distros/rolling/patch/ros2-autoware-lanelet2-extension.patch b/distros/rolling/patch/ros2-autoware-lanelet2-extension.patch index 18e931851..bdc703333 100644 --- a/distros/rolling/patch/ros2-autoware-lanelet2-extension.patch +++ b/distros/rolling/patch/ros2-autoware-lanelet2-extension.patch @@ -1,8 +1,8 @@ diff --git a/CMakeLists.txt b/CMakeLists.txt -index a3ec429..6a64b82 100644 +index 6c1f16c..36ea672 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt -@@ -25,6 +25,14 @@ ament_auto_add_library(${PROJECT_NAME}_lib SHARED +@@ -29,6 +29,14 @@ ament_auto_add_library(${PROJECT_NAME}_lib SHARED lib/route_checker.cpp ) @@ -14,6 +14,30 @@ index a3ec429..6a64b82 100644 + ) +endif() + - ament_auto_add_executable(${PROJECT_NAME}_sample src/sample_code.cpp) - target_link_libraries(${PROJECT_NAME}_sample - ${PROJECT_NAME}_lib + # Suppress boost geometry uninitialized variable warnings + # This is a known issue in boost geometry library where internal template code + # may trigger maybe-uninitialized warnings that are false positives. +@@ -36,7 +44,9 @@ ament_auto_add_library(${PROJECT_NAME}_lib SHARED + # and do not affect the actual functionality of the code. + # Related: https://www.boost.org/doc/libs/latest/libs/utility/doc/html/utility/utilities/value_init.html + # and https://www.boost.org/doc/libs/1_89_0/libs/optional/doc/html/boost_optional/design/gotchas/false_positive_with__wmaybe_uninitialized.html +-target_compile_options(${PROJECT_NAME}_lib PRIVATE -Wno-error=maybe-uninitialized) ++if(NOT MSVC) ++ target_compile_options(${PROJECT_NAME}_lib PRIVATE -Wno-error=maybe-uninitialized) ++endif() + + if(BUILD_TESTING) + ament_add_ros_isolated_gtest(projector-test test/src/test_projector.cpp) +diff --git a/lib/deprecated.cpp b/lib/deprecated.cpp +index 7a78e45..1220fc2 100644 +--- a/lib/deprecated.cpp ++++ b/lib/deprecated.cpp +@@ -20,6 +20,8 @@ + + #include "deprecated.hpp" + ++#include ++#include + #include + + #include diff --git a/distros/rolling/patch/ros2-autoware-lanelet2-extension.win.patch b/distros/rolling/patch/ros2-autoware-lanelet2-extension.win.patch index febfdc433..3c76f9af1 100644 --- a/distros/rolling/patch/ros2-autoware-lanelet2-extension.win.patch +++ b/distros/rolling/patch/ros2-autoware-lanelet2-extension.win.patch @@ -1,5 +1,5 @@ diff --git a/src/check_right_of_way.cpp b/src/check_right_of_way.cpp -index d59ec14..e62be74 100644 +index ff48015..23af2e8 100644 --- a/src/check_right_of_way.cpp +++ b/src/check_right_of_way.cpp @@ -15,13 +15,6 @@ @@ -16,7 +16,7 @@ index d59ec14..e62be74 100644 #include #include -@@ -57,16 +50,17 @@ int main(int argc, char ** argv) +@@ -75,16 +68,17 @@ int main(int argc, char ** argv) lanelet::routing::RoutingGraphPtr routing_graph_ptr = lanelet::routing::RoutingGraph::build(*map, *traffic_rules); diff --git a/distros/rolling/rosdistro_additional_recipes.yaml b/distros/rolling/rosdistro_additional_recipes.yaml index ba8be1d8e..f151104a5 100644 --- a/distros/rolling/rosdistro_additional_recipes.yaml +++ b/distros/rolling/rosdistro_additional_recipes.yaml @@ -22,10 +22,6 @@ autoware_behavior_velocity_stop_line_module: tag: release/humble/autoware_behavior_velocity_stop_line_module/1.7.0-1 url: https://github.com/ros2-gbp/autoware_core-release.git version: 1.7.0 -autoware_cmake: - tag: release/jazzy/autoware_cmake/1.1.0-1 - url: https://github.com/ros2-gbp/autoware_cmake-release.git - version: 1.1.0 autoware_component_interface_specs: tag: release/humble/autoware_component_interface_specs/1.7.0-1 url: https://github.com/ros2-gbp/autoware_core-release.git @@ -106,14 +102,6 @@ autoware_gyro_odometer: tag: release/humble/autoware_gyro_odometer/1.7.0-1 url: https://github.com/ros2-gbp/autoware_core-release.git version: 1.7.0 -autoware_internal_localization_msgs: - tag: release/jazzy/autoware_internal_localization_msgs/1.10.0-1 - url: https://github.com/ros2-gbp/autoware_internal_msgs-release.git - version: 1.10.0 -autoware_internal_planning_msgs: - tag: release/humble/autoware_internal_planning_msgs/1.12.0-1 - url: https://github.com/ros2-gbp/autoware_internal_msgs-release.git - version: 1.12.0 autoware_interpolation: tag: release/humble/autoware_interpolation/1.7.0-1 url: https://github.com/ros2-gbp/autoware_core-release.git @@ -122,14 +110,6 @@ autoware_kalman_filter: tag: release/humble/autoware_kalman_filter/1.7.0-1 url: https://github.com/ros2-gbp/autoware_core-release.git version: 1.7.0 -autoware_lanelet2_extension: - tag: release/humble/autoware_lanelet2_extension/0.8.0-1 - url: https://github.com/ros2-gbp/autoware_lanelet2_extension-release.git - version: 0.8.0 -autoware_lanelet2_extension_python: - tag: release/humble/autoware_lanelet2_extension_python/0.8.0-1 - url: https://github.com/ros2-gbp/autoware_lanelet2_extension-release.git - version: 0.8.0 autoware_lanelet2_map_visualizer: tag: release/humble/autoware_lanelet2_map_visualizer/1.7.0-1 url: https://github.com/ros2-gbp/autoware_core-release.git @@ -138,10 +118,6 @@ autoware_lanelet2_utils: tag: release/humble/autoware_lanelet2_utils/1.7.0-1 url: https://github.com/ros2-gbp/autoware_core-release.git version: 1.7.0 -autoware_localization_msgs: - tag: release/humble/autoware_localization_msgs/1.9.0-1 - url: https://github.com/ros2-gbp/autoware_msgs-release.git - version: 1.9.0 autoware_localization_util: tag: release/humble/autoware_localization_util/1.7.0-1 url: https://github.com/ros2-gbp/autoware_core-release.git @@ -214,10 +190,6 @@ autoware_planning_factor_interface: tag: release/humble/autoware_planning_factor_interface/1.7.0-1 url: https://github.com/ros2-gbp/autoware_core-release.git version: 1.7.0 -autoware_planning_msgs: - tag: release/humble/autoware_planning_msgs/1.9.0-1 - url: https://github.com/ros2-gbp/autoware_msgs-release.git - version: 1.9.0 autoware_planning_test_manager: tag: release/humble/autoware_planning_test_manager/1.7.0-1 url: https://github.com/ros2-gbp/autoware_core-release.git @@ -262,10 +234,6 @@ autoware_stop_filter: tag: release/humble/autoware_stop_filter/1.7.0-1 url: https://github.com/ros2-gbp/autoware_core-release.git version: 1.7.0 -autoware_system_msgs: - tag: release/humble/autoware_system_msgs/1.9.0-1 - url: https://github.com/ros2-gbp/autoware_msgs-release.git - version: 1.9.0 autoware_test_node: tag: release/humble/autoware_test_node/1.7.0-1 url: https://github.com/ros2-gbp/autoware_core-release.git @@ -286,54 +254,6 @@ autoware_twist2accel: tag: release/humble/autoware_twist2accel/1.7.0-1 url: https://github.com/ros2-gbp/autoware_core-release.git version: 1.7.0 -autoware_utils: - tag: release/jazzy/autoware_utils/1.4.2-1 - url: https://github.com/ros2-gbp/autoware_utils-release.git - version: 1.4.2 -autoware_utils_debug: - tag: release/jazzy/autoware_utils_debug/1.4.2-1 - url: https://github.com/ros2-gbp/autoware_utils-release.git - version: 1.4.2 -autoware_utils_diagnostics: - tag: release/jazzy/autoware_utils_diagnostics/1.4.2-1 - url: https://github.com/ros2-gbp/autoware_utils-release.git - version: 1.4.2 -autoware_utils_geometry: - tag: release/jazzy/autoware_utils_geometry/1.4.2-1 - url: https://github.com/ros2-gbp/autoware_utils-release.git - version: 1.4.2 -autoware_utils_logging: - tag: release/jazzy/autoware_utils_logging/1.4.2-1 - url: https://github.com/ros2-gbp/autoware_utils-release.git - version: 1.4.2 -autoware_utils_math: - tag: release/jazzy/autoware_utils_math/1.4.2-1 - url: https://github.com/ros2-gbp/autoware_utils-release.git - version: 1.4.2 -autoware_utils_pcl: - tag: release/jazzy/autoware_utils_pcl/1.4.2-1 - url: https://github.com/ros2-gbp/autoware_utils-release.git - version: 1.4.2 -autoware_utils_rclcpp: - tag: release/jazzy/autoware_utils_rclcpp/1.4.2-1 - url: https://github.com/ros2-gbp/autoware_utils-release.git - version: 1.4.2 -autoware_utils_system: - tag: release/jazzy/autoware_utils_system/1.4.2-1 - url: https://github.com/ros2-gbp/autoware_utils-release.git - version: 1.4.2 -autoware_utils_tf: - tag: release/jazzy/autoware_utils_tf/1.4.2-1 - url: https://github.com/ros2-gbp/autoware_utils-release.git - version: 1.4.2 -autoware_utils_uuid: - tag: release/jazzy/autoware_utils_uuid/1.4.2-1 - url: https://github.com/ros2-gbp/autoware_utils-release.git - version: 1.4.2 -autoware_utils_visualization: - tag: release/jazzy/autoware_utils_visualization/1.4.2-1 - url: https://github.com/ros2-gbp/autoware_utils-release.git - version: 1.4.2 autoware_vehicle_info_utils: tag: release/humble/autoware_vehicle_info_utils/1.7.0-1 url: https://github.com/ros2-gbp/autoware_core-release.git @@ -694,13 +614,6 @@ rslidar_sdk: tag: v1.5.18 url: https://github.com/RoboSense-LiDAR/rslidar_sdk.git version: 1.5.18 -rtabmap: - # rolling's own release is 0.22.1, which no longer links on Windows against gtsam (LNK2005 - # on a std::vector member); jazzy's 0.23.13 does. rolling has - # no rtabmap_ros to keep in step. - tag: release/jazzy/rtabmap/0.23.13-1 - url: https://github.com/ros2-gbp/rtabmap-release.git - version: 0.23.13 yaets: tag: release/lyrical/yaets/1.2.0-1 url: https://github.com/fmrico/yaets-release.git diff --git a/pixi.lock b/pixi.lock index ab0edb4f3..5c63fd110 100644 --- a/pixi.lock +++ b/pixi.lock @@ -32,6 +32,7 @@ environments: default: channels: - url: https://prefix.dev/conda-forge/ + - url: https://prefix.dev/tobias-fischer/robostack-tools/ indexes: - https://pypi.org/simple packages: @@ -90,6 +91,7 @@ environments: - conda: https://prefix.dev/conda-forge/noarch/tomli-2.5.0-pyh5ded981_0.conda - conda: https://prefix.dev/conda-forge/noarch/typing_extensions-4.16.0-pyhcf101f3_0.conda - conda: https://prefix.dev/conda-forge/noarch/tzdata-2026c-h151e31d_0.conda + - conda: https://prefix.dev/tobias-fischer/robostack-tools/linux-64/vinca-rs-0.1.7-h41f06ac_0.conda - pypi: git+https://github.com/RoboStack/vinca.git?rev=74d1e856d5b00c0cb5ee0c1be41dac4afcae7bf5#74d1e856d5b00c0cb5ee0c1be41dac4afcae7bf5 - pypi: https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/74/ef303ec980ef962ffe548fa36cb138bb399d1d1315731ba7bb81da85f7e0/rosdistro-1.1.0-py3-none-any.whl @@ -170,6 +172,7 @@ environments: - conda: https://prefix.dev/conda-forge/noarch/tomli-2.5.0-pyh5ded981_0.conda - conda: https://prefix.dev/conda-forge/noarch/typing_extensions-4.16.0-pyhcf101f3_0.conda - conda: https://prefix.dev/conda-forge/noarch/tzdata-2026c-h151e31d_0.conda + - conda: https://prefix.dev/tobias-fischer/robostack-tools/linux-aarch64/vinca-rs-0.1.7-h11eb1b2_0.conda - pypi: git+https://github.com/RoboStack/vinca.git?rev=74d1e856d5b00c0cb5ee0c1be41dac4afcae7bf5#74d1e856d5b00c0cb5ee0c1be41dac4afcae7bf5 - pypi: https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/74/ef303ec980ef962ffe548fa36cb138bb399d1d1315731ba7bb81da85f7e0/rosdistro-1.1.0-py3-none-any.whl @@ -243,6 +246,7 @@ environments: - conda: https://prefix.dev/conda-forge/osx-64/tk-8.6.13-ha6c374e_4.conda - conda: https://prefix.dev/conda-forge/osx-64/yaml-0.2.5-had63097_3.conda - conda: https://prefix.dev/conda-forge/osx-64/zstd-1.5.7-hbc1a06c_7.conda + - conda: https://prefix.dev/tobias-fischer/robostack-tools/osx-64/vinca-rs-0.1.7-h4778241_0.conda - pypi: git+https://github.com/RoboStack/vinca.git?rev=74d1e856d5b00c0cb5ee0c1be41dac4afcae7bf5#74d1e856d5b00c0cb5ee0c1be41dac4afcae7bf5 - pypi: https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/74/ef303ec980ef962ffe548fa36cb138bb399d1d1315731ba7bb81da85f7e0/rosdistro-1.1.0-py3-none-any.whl @@ -316,6 +320,7 @@ environments: - conda: https://prefix.dev/conda-forge/osx-arm64/tk-8.6.13-hbeba79b_4.conda - conda: https://prefix.dev/conda-forge/osx-arm64/yaml-0.2.5-h74c22ad_3.conda - conda: https://prefix.dev/conda-forge/osx-arm64/zstd-1.5.7-hf451053_7.conda + - conda: https://prefix.dev/tobias-fischer/robostack-tools/osx-arm64/vinca-rs-0.1.7-hea27dcd_0.conda - pypi: git+https://github.com/RoboStack/vinca.git?rev=74d1e856d5b00c0cb5ee0c1be41dac4afcae7bf5#74d1e856d5b00c0cb5ee0c1be41dac4afcae7bf5 - pypi: https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/74/ef303ec980ef962ffe548fa36cb138bb399d1d1315731ba7bb81da85f7e0/rosdistro-1.1.0-py3-none-any.whl @@ -389,6 +394,7 @@ environments: - conda: https://prefix.dev/conda-forge/win-64/vcomp14-14.51.36247-habf1de7_41.conda - conda: https://prefix.dev/conda-forge/win-64/yaml-0.2.5-h6a83c73_3.conda - conda: https://prefix.dev/conda-forge/win-64/zstd-1.5.7-h534d264_7.conda + - conda: https://prefix.dev/tobias-fischer/robostack-tools/win-64/vinca-rs-0.1.7-he01afda_0.conda - pypi: git+https://github.com/RoboStack/vinca.git?rev=74d1e856d5b00c0cb5ee0c1be41dac4afcae7bf5#74d1e856d5b00c0cb5ee0c1be41dac4afcae7bf5 - pypi: https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/12/74/ef303ec980ef962ffe548fa36cb138bb399d1d1315731ba7bb81da85f7e0/rosdistro-1.1.0-py3-none-any.whl @@ -3289,6 +3295,62 @@ packages: - zstd >=1.5.7,<1.6.0a0 size: 387535 timestamp: 1786599623274 +- conda: https://prefix.dev/tobias-fischer/robostack-tools/linux-64/vinca-rs-0.1.7-h41f06ac_0.conda + sha256: 34557b8ae5d8499c3899aaa82a097cb746c93b6972bef0051a0288722558abb6 + md5: fb59bd254abd74c7934fe611d5a008bc + depends: + - libgcc >=16 + - __glibc >=2.17,<3.0.a0 + constrains: + - __glibc >=2.17 + license: MIT + run_exports: {} + size: 9566970 + timestamp: 1794572477863 +- conda: https://prefix.dev/tobias-fischer/robostack-tools/linux-aarch64/vinca-rs-0.1.7-h11eb1b2_0.conda + sha256: 6f1cd765bec384faa2cd2913d0d18dfb8cd52f92b770ae42ed69603df7b5d5ec + md5: 94e95c59cbfccc706d25eb6e64108ba6 + depends: + - libgcc >=16 + constrains: + - __glibc >=2.17 + license: MIT + run_exports: {} + size: 9570130 + timestamp: 1794572465783 +- conda: https://prefix.dev/tobias-fischer/robostack-tools/osx-64/vinca-rs-0.1.7-h4778241_0.conda + sha256: 9564f3cc2c748d89712162aded4ec55dfdabbed9332ef258770c22a249efa86b + md5: e6e39f7159190606b759d575216f8dba + depends: + - __osx >=13.0 + constrains: + - __osx >=11.0 + license: MIT + run_exports: {} + size: 9325029 + timestamp: 1794572496401 +- conda: https://prefix.dev/tobias-fischer/robostack-tools/osx-arm64/vinca-rs-0.1.7-hea27dcd_0.conda + sha256: 887aeee2f4655a588e6543497c2b815aaecf0a9ad73a0ce9384209f436b0a08b + md5: 17256a49286ca64d5c35a60e652f94ae + depends: + - __osx >=13.0 + constrains: + - __osx >=11.0 + license: MIT + run_exports: {} + size: 8745559 + timestamp: 1794572471712 +- conda: https://prefix.dev/tobias-fischer/robostack-tools/win-64/vinca-rs-0.1.7-he01afda_0.conda + sha256: 1a6ec7dd71f613ac273369443e2421239468ef033eeac262baa1776efd17dda1 + md5: b5971936b148667e4317a01dd8347348 + depends: + - vc >=14.3,<15 + - vc14_runtime >=14.44.35208 + - ucrt >=10.0.20348.0 + license: MIT + run_exports: {} + size: 8934936 + timestamp: 1794572471934 - pypi: git+https://github.com/RoboStack/vinca.git?rev=74d1e856d5b00c0cb5ee0c1be41dac4afcae7bf5#74d1e856d5b00c0cb5ee0c1be41dac4afcae7bf5 name: vinca version: 0.2.0 diff --git a/pixi.toml b/pixi.toml index c76cfff92..7b2eaaf99 100644 --- a/pixi.toml +++ b/pixi.toml @@ -2,7 +2,7 @@ name = "robostack" description = "RoboStack: conda packages for all ROS 2 distributions, from one repository" authors = ["Tobias Fischer ", "Wolf Vollprecht ", "Silvio Traversaro ", "Daisuke Nishimatsu "] -channels = ["https://prefix.dev/conda-forge"] +channels = ["https://prefix.dev/conda-forge", "https://prefix.dev/tobias-fischer/robostack-tools"] platforms = [ "osx-arm64", # 2.17 is the glibc version used in CentOS 7 @@ -27,6 +27,8 @@ git-lfs = "*" pytest = ">=9.1.1,<10" # gh for the workflows: one version everywhere (the runner's is not on PATH on Windows) gh = ">=2.102.0,<3" +# built and published by vinca-rs's release workflow +vinca-rs = { version = "==0.1.7", channel = "https://prefix.dev/tobias-fischer/robostack-tools" } [target.win-64.dependencies] # patch is required by rattler-build diff --git a/tools/check_dependency_compat.py b/tools/check_dependency_compat.py index ff145731f..30e707ca2 100644 --- a/tools/check_dependency_compat.py +++ b/tools/check_dependency_compat.py @@ -227,10 +227,11 @@ def collect_requirements_from_vinca( ) -> dict[tuple[Optional[str], str], set[str]]: """Like collect_requirements, for every selected package of the configuration (not only the recipes that happen to be generated, e.g. with skip_existing).""" - from vinca.rebuild import requirements_from_vinca + sys.path.insert(0, str(Path(__file__).resolve().parent)) + import robostack as rs requirements: dict[tuple[Optional[str], str], set[str]] = defaultdict(set) - packages, _ = requirements_from_vinca(vinca_path.resolve().parent, [platform]) + packages, _ = rs.vinca_requirements(vinca_path.resolve().parent, [platform]) for name, reqs in packages.items(): for section in sections: for condition, spec in walk_requirements(reqs.get(section)): @@ -419,13 +420,35 @@ def _by_name(requirements: Iterable[tuple[Optional[str], str]]) -> dict[str, lis class Solver: - """Thin wrapper that writes a fake recipe and solves it with rattler-build.""" + """Writes a fake recipe and solves it: with one `vinca-rs solve` process for the + whole run (the repodata loaded once), else rattler-build per solve.""" def __init__(self, args: argparse.Namespace, channels: list[str], workdir: Path) -> None: self.args = args self.channels = channels self.workdir = workdir self.calls = 0 + self._server: Optional[subprocess.Popen] = None + + def _vinca_rs(self) -> Optional[subprocess.Popen]: + if self._server is None: + sys.path.insert(0, str(Path(__file__).resolve().parent)) + import robostack as rs + + if not shutil.which(rs.VINCA_RS): + return None + env = dict(os.environ) + # the virtual packages run_solve() gives rattler-build + if self.args.platform.startswith("linux"): + env.setdefault("CONDA_OVERRIDE_GLIBC", glibc_floor(Path(self.args.variant_config), self.args.platform)) + elif self.args.platform.startswith("osx") and sys.platform != "darwin": + env.setdefault("CONDA_OVERRIDE_OSX", DEFAULT_OSX) + cmd = [rs.VINCA_RS, "solve", "--platform", self.args.platform, + "--variant-config", str(Path(self.args.variant_config).resolve())] + for channel in self.channels: + cmd += ["-c", channel] + self._server = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True, env=env) + return self._server def solve( self, @@ -437,6 +460,11 @@ def solve( recipe = self.workdir / label / "recipe.yaml" write_fake_recipe(recipe, pins, requirements) self.calls += 1 + if server := self._vinca_rs(): + server.stdin.write(json.dumps({"recipe": str(recipe.resolve()), "glibc": glibc}) + "\n") + server.stdin.flush() + response = json.loads(server.stdout.readline()) + return response["ok"], response["text"] return run_solve( recipe, Path(self.args.variant_config), diff --git a/tools/maintenance.py b/tools/maintenance.py index 3d24ae22b..9ff367178 100644 --- a/tools/maintenance.py +++ b/tools/maintenance.py @@ -208,7 +208,7 @@ def plan_rebuild(distro: str, old_cbc: str) -> tuple[dict[str, str], int, dict[s changed = {k: v for k, v in changed.items() if _pin_values(v[0]) != _pin_values(v[1])} if not changed: return {}, 0, {} - requirements, prefix = rebuild.requirements_from_vinca(rs.prepare(distro)) + requirements, prefix = rs.vinca_requirements(rs.prepare(distro)) return rebuild.plan(requirements, changed, prefix), len(requirements), changed @@ -412,7 +412,7 @@ def update_snapshot(distro: str) -> Result: old_text = snapshot.read_text() if snapshot.is_file() else "" old = _versions(snapshot) if rs.task(distro, "create-snapshot", ["--rosdistro-ref", ref] if ref else []): - return Result(f"{distro}: snapshot update failed", "`vinca-snapshot` failed, see the log.", ok=False) + return Result(f"{distro}: snapshot update failed", "`vinca-rs snapshot` failed, see the log.", ok=False) text = snapshot.read_text() data, ahead = keep_newer(yaml.safe_load(old_text) or {}, yaml.safe_load(text) or {}, (released_on_master(distro) or set()) if ref else set()) @@ -502,7 +502,7 @@ def update_pinning(distro: str) -> Result: before = (settings.get("conda_forge_pinning_version"), list(settings.get("conda_forge_migrations") or [])) print(f"Collecting dependencies of {distro}", flush=True) work = rs.prepare(distro) - dependencies = pinning.dependencies_from_vinca(work, pinning.DEFAULT_PLATFORMS) + dependencies = rs.vinca_dependencies(work) version, migrations, reports = pinning.update_pinning(work / "vinca_pinning.yaml", dependencies=dependencies) if str(before[0]) == str(version) and before[1] == list(migrations): return Result(f"{distro}: pinning is up to date", f"{distro} is on conda-forge-pinning {version}.") @@ -523,7 +523,7 @@ def update_shared_pinning() -> Result: for distro in distros: print(f"Collecting dependencies of {distro}", flush=True) works[distro] = rs.prepare(distro) - dependencies |= pinning.dependencies_from_vinca(works[distro], pinning.DEFAULT_PLATFORMS) + dependencies |= rs.vinca_dependencies(works[distro]) version, migrations, reports = pinning.update_pinning(works[distros[0]] / "vinca_pinning.yaml", dependencies=dependencies) if str(before[0]) == str(version) and before[1] == list(migrations): @@ -824,7 +824,7 @@ def rebuild_packages(packages: list[str], distros: list[str], with_dependents: b lines: list[str] = [] changed = False for distro in distros or rs.distros(): - requirements, prefix = rebuild.requirements_from_vinca(rs.prepare(distro)) + requirements, prefix = rs.vinca_requirements(rs.prepare(distro)) if missing := [p for p in wanted if p not in requirements]: lines.append(f"- **{distro}** doesn't build {', '.join(f'`{p}`' for p in missing)}") plan = rebuild_plan(requirements, prefix, wanted, with_dependents) diff --git a/tools/robostack.py b/tools/robostack.py index 96d2f4c67..f2e69e00f 100644 --- a/tools/robostack.py +++ b/tools/robostack.py @@ -78,20 +78,10 @@ def run(cmd: list[str], cwd: Path) -> int: return subprocess.run(cmd, cwd=cwd).returncode -def run_tee(cmd: list[str], cwd: Path) -> tuple[int, bytes]: - """run(), also returning the combined output.""" - print("+", " ".join(cmd), f" (in {cwd.relative_to(ROOT)})", flush=True) - output = bytearray() - with subprocess.Popen(cmd, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) as proc: - for line in proc.stdout: - sys.stdout.buffer.write(line) - sys.stdout.buffer.flush() - output += line - return proc.returncode, bytes(output) -WORKFLOW_SIZE_MESSAGE = b"exceeding the 500 KiB limit" -MAX_BATCH_SIZE = 100 +# the recipe and workflow generator (https://github.com/Tobias-Fischer/vinca-rs) +VINCA_RS = os.environ.get("VINCA_RS", "vinca-rs") # prefix.dev's sharded repodata (what rattler reads) lists a new upload a few minutes @@ -361,6 +351,50 @@ def prepare(distro: str) -> Path: return w +# the platforms RoboStack builds +PLATFORMS = ("linux-64", "linux-aarch64", "osx-64", "osx-arm64", "win-64") + + +def vinca_rs_requirements(work: Path, platform: str) -> dict: + """`vinca-rs requirements`: {"prefix", "packages": {ros name: {build, host, run}}, + "run_constraints"} for every package the configuration in `work` selects.""" + out = subprocess.run([VINCA_RS, "requirements", "--platform", platform], cwd=work, + capture_output=True, text=True) + if out.returncode: + raise SystemExit(f"vinca-rs requirements --platform {platform} failed:\n{out.stderr}") + return json.loads(out.stdout) + + +def vinca_requirements(work: Path, platforms=PLATFORMS) -> tuple[dict, str]: + """The requirements of every selected package, combined over the platforms + ({name: {"build"|"host"|"run": [each platform's list]}}), and the conda name prefix + of the ROS packages.""" + from concurrent.futures import ThreadPoolExecutor + + with ThreadPoolExecutor() as pool: + results = list(pool.map(lambda p: vinca_rs_requirements(work, p), platforms)) + combined: dict = {} + for result in results: + for name, reqs in result["packages"].items(): + target = combined.setdefault(name, {"build": [], "host": [], "run": []}) + for group in target: + target[group].append(reqs.get(group) or []) + return combined, (results[0]["prefix"] if results else "") + + +def vinca_dependencies(work: Path, platforms=PLATFORMS) -> set[str]: + """The non-ROS dependencies of every selected package (and the mutex's + run_constraints), on any of the platforms.""" + from vinca.pinning import _dependency_name, _walk_requirements + + names = set() + for platform in platforms: + result = vinca_rs_requirements(work, platform) + groups = [*result["packages"].values(), {"run_constraints": result["run_constraints"]}] + names |= {n for g in groups for r in _walk_requirements(g) if (n := _dependency_name(r))} + return names + + def read_vinca(distro: str, platform: str | None = None) -> dict: """The distribution's vinca configuration as vinca reads it (shared/ + distro, selectors resolved for the platform, default: this machine's).""" @@ -512,7 +546,7 @@ def task(distro: str, name: str, args: list[str]) -> int: shutil.copy2(w / "conda_build_config.yaml", d / "conda_build_config.yaml") return rc if name == "create-snapshot": - return run(["vinca-snapshot", "-d", distro, "-o", "rosdistro_snapshot.yaml", *args], d) + return run([VINCA_RS, "snapshot", "--distro", distro, "--out", "rosdistro_snapshot.yaml", *args], d) if name == "sort": files = [f for f in ("pkg_additional_info.yaml", "rosdistro_additional_recipes.yaml") if (d / f).is_file()] rc = run(["vinca-sort-vinca-lists", *args, "vinca.yaml"], d) @@ -537,7 +571,8 @@ def task(distro: str, name: str, args: list[str]) -> int: shutil.rmtree(w / "recipes", ignore_errors=True) shutil.rmtree(w / "recipes_only_patch", ignore_errors=True) (w / "recipes").mkdir() - return run(["vinca", "-m", *args], w) + # vinca-rs: the recipes, without those already on the skip_existing channels + return run([VINCA_RS, "generate", "--out", "recipes", *args], w) if not (w / "vinca.yaml").is_file(): prepare(distro) if name == "check-orphaned-patches": @@ -585,80 +620,38 @@ def task(distro: str, name: str, args: list[str]) -> int: def generate_gha(distro: str, args: list[str]) -> int: - """vinca-gha for one distribution; the workflow gets a distro-specific name and - trigger branch, the file name .github/workflows/build.yaml, and tells the build - scripts which distro to build.""" + """The staged build workflow of one distribution and platform, from + .github/build-template.yaml (vinca-rs workflow), as .github/workflows/build.yaml.""" parser = argparse.ArgumentParser(prog="rs gha") parser.add_argument("--platform", required=True) parser.add_argument("--batch-size", default="25") ns = parser.parse_args(args) d = work_dir(distro) branch = f"buildbranch_{distro}_{ns.platform.replace('-', '_')}" - before = set(d.glob("*.yml")) - # GitHub refuses workflow files over 500 KiB, which a full rebuild's many batches can - # reach (jazzy's win-64 at 25): then retry with larger, hence fewer, batches. - batch_size = int(ns.batch_size) - # each job keeps the build times it records (build-ci); vinca-gha packs each stage's - # recipes into jobs by those times (every recipe the same without them) + # each job keeps the build times it records (build-ci); vinca-rs packs the recipes + # into jobs by those times (every recipe the same without them) import build_timings - extra = ["--keep-file", f"distros/{distro}/work/{build_timings.JOURNAL}"] + extra = [] if build_timings.vinca_timings(distro, ns.platform, d / "build_timings.json"): extra += ["--timings", "build_timings.json"] - # vinca-gha keeps the workflow under GitHub's size limit itself (a job budget); only - # when even the fewest jobs the batch size allows are too many, retry with larger batches - while True: - rc, output = run_tee(["vinca-gha", "--platform", ns.platform, "--trigger-branch", branch, "-d", - "./recipes", "--batch_size", str(batch_size), *extra], d) - if not rc or WORKFLOW_SIZE_MESSAGE not in output: - break - if batch_size >= MAX_BATCH_SIZE: - break - batch_size = min(batch_size * 3 // 2, MAX_BATCH_SIZE) - print(f"The workflow is too large; retrying with --batch_size {batch_size}", flush=True) - if rc: - return rc - # Edit the text: a YAML round trip would turn the `on:` key into `true:` (YAML 1.1). - for wf in sorted(set(d.glob("*.yml")) - before): - lines = wf.read_text().splitlines() - # all build branches share .github/workflows/build.yaml, which GitHub lists as one - # workflow: run-name tells the runs apart - # one run per distribution and platform at a time: a newer run (the latest main's - # recipes and workflow) cancels the older one, and skips what that one uploaded - # already. Re-run failed jobs only in the newest run: re-running an older one - # would cancel the newer run. - lines = [f"name: build\nrun-name: {distro} {ns.platform}\n" - f"concurrency:\n group: build-{distro}-{ns.platform}\n cancel-in-progress: true" - if l.startswith("name: ") else l for l in lines] - # vinca-gha passes secrets.ANACONDA_API_TOKEN to the build jobs, for the anaconda.org - # upload (prefix.dev uses trusted publishing, id-token: write): each distribution - # has its own token (its organization's), the secret ANACONDA_API_TOKEN_ - secret = anaconda_token_secret(distro) - lines = [l.replace("secrets.ANACONDA_API_TOKEN }}", f"secrets.{secret} }}}}") for l in lines] - # jobs that declare permissions get none they don't list: checking out the - # repository needs contents: read (only works without it while it is public) - lines = [f"{l}\n{l[:len(l) - len(l.lstrip())]}contents: read" if l.strip() == "id-token: write" else l - for l in lines] - if sum("permissions:" in l for l in lines) != sum("contents: read" in l for l in lines): - raise SystemExit(f"{wf}: a job with permissions but without contents: read") - if ns.platform == "win-64" and not any("build_win.bat" in l for l in lines): - raise SystemExit(f"{wf}: the Windows workflow doesn't use .scripts/build_win.bat") - if any(l.startswith("env:") for l in lines): - raise SystemExit(f"{wf}: unexpected top-level env in the vinca-gha output") - lines += [ - "env:", - f" ROBOSTACK_DISTRO: {distro}", - " # optional repository variable: upload to this prefix.dev channel instead", - " # (e.g. a test channel), or 'none' to only build", - " ROBOSTACK_UPLOAD_CHANNEL: ${{ vars.ROBOSTACK_UPLOAD_CHANNEL }}", - ] - # one file name on every build branch (each branch carries only its own - # workflow), so a single prefix.dev Repository Access source matches them all - dest = ROOT / ".github" / "workflows" / "build.yaml" - dest.write_text("\n".join(lines) + "\n") - wf.unlink() + # one file name on every build branch (each branch carries only its own workflow), + # so a single prefix.dev Repository Access source matches them all + dest = ROOT / ".github" / "workflows" / "build.yaml" + variables = { + "distro": distro, + "branch": branch, + # each distribution has its own anaconda.org token (its organization's) + "secret": anaconda_token_secret(distro), + "keep_file": f"distros/{distro}/work/{build_timings.JOURNAL}", + } + rc = run([VINCA_RS, "workflow", "--recipes", "recipes", "--platform", ns.platform, + "--template", str(ROOT / ".github" / "build-template.yaml"), "--out", str(dest), + "--max-batch", ns.batch_size, *extra, + *[a for k, v in variables.items() for a in ("--var", f"{k}={v}")]], d) + if not rc and dest.is_file(): print(f"wrote {dest.relative_to(ROOT)} (trigger branch {branch})") - return 0 + return rc NO_BUILD_LABEL = "build:none" # only the checks (tooling, CI or docs changes) diff --git a/tools/tests/test_build_timings.py b/tools/tests/test_build_timings.py index 75e5b2b46..144ece18f 100644 --- a/tools/tests/test_build_timings.py +++ b/tools/tests/test_build_timings.py @@ -76,33 +76,6 @@ def collect(): assert not maintenance.update_build_timings().changed # nothing new: no PR -def test_gha_keeps_the_journal_and_uses_the_times(tmp_path, monkeypatch): - import robostack as rs - - work, root = tmp_path / "work", tmp_path / "root" - work.mkdir() - (root / ".github" / "workflows").mkdir(parents=True) - calls = [] - - def fake_run_tee(cmd, cwd): - calls.append(cmd) - (work / "linux.yml").write_text("name: build\njobs:\n stage_0_job_0:\n steps: []\n") - return 0, b"" - - monkeypatch.setattr(rs, "run_tee", fake_run_tee) - monkeypatch.setattr(rs, "work_dir", lambda distro: work) - monkeypatch.setattr(rs, "ROOT", root) - monkeypatch.setattr(build_timings, "TIMINGS", tmp_path / "timings") - assert rs.generate_gha("jazzy", ["--platform", "linux-64"]) == 0 - assert "--keep-file" in calls[-1] and "--timings" not in calls[-1] # no times yet - (tmp_path / "timings").mkdir() - (tmp_path / "timings" / "linux-64.json").write_text('{"ros2-a": {"jazzy": 3.0}}\n') - assert rs.generate_gha("jazzy", ["--platform", "linux-64"]) == 0 - assert calls[-1][calls[-1].index("--timings") + 1] == "build_timings.json" - assert json.loads((work / "build_timings.json").read_text()) == { - "recipes": {"ros2-a": 3.0, "ros-jazzy-a": build_timings.ALIAS_SECONDS}, "default": 3.0} - - def test_ci_estimate_uses_the_measured_times(tmp_path, monkeypatch, capsys): import ci_estimate diff --git a/tools/tests/test_generate_gha.py b/tools/tests/test_generate_gha.py index c9f359a66..2b7667a4c 100644 --- a/tools/tests/test_generate_gha.py +++ b/tools/tests/test_generate_gha.py @@ -1,55 +1,54 @@ -"""rs gha: larger batches when the generated workflow is too large for GitHub.""" +"""rs gha: the build workflow from .github/build-template.yaml (vinca-rs workflow).""" import build_timings import robostack as rs -def test_gha_retries_with_larger_batches_when_the_workflow_is_too_large(monkeypatch, tmp_path): - monkeypatch.setattr(build_timings, "TIMINGS", tmp_path / "no-timings") # batching by count - sizes = [] - - def fake_run_tee(cmd, cwd): - sizes.append(int(cmd[cmd.index("--batch_size") + 1])) - if sizes[-1] < 50: - return 1, b"RuntimeError: Generated workflow win.yml is 530.8 KiB, exceeding the 500 KiB limit." - return 0, b"" - - monkeypatch.setattr(rs, "run_tee", fake_run_tee) - monkeypatch.setattr(rs, "work_dir", lambda distro: tmp_path) - - assert rs.generate_gha("jazzy", ["--platform", "win-64", "--batch-size", "25"]) == 0 - assert sizes == [25, 37, 55] - - -def test_gha_does_not_retry_other_failures(monkeypatch, tmp_path): - sizes = [] - - def fake_run_tee(cmd, cwd): - sizes.append(int(cmd[cmd.index("--batch_size") + 1])) - return 1, b"some other error" - - monkeypatch.setattr(rs, "run_tee", fake_run_tee) - monkeypatch.setattr(rs, "work_dir", lambda distro: tmp_path) - - assert rs.generate_gha("jazzy", ["--platform", "win-64"]) == 1 - assert sizes == [25] - - -def test_gha_uses_the_distributions_anaconda_token(monkeypatch, tmp_path): +def gha(monkeypatch, tmp_path, distro, platform): + """Run generate_gha with vinca-rs stubbed out; return the vinca-rs command.""" + calls = [] work, root = tmp_path / "work", tmp_path / "root" - work.mkdir(); (root / ".github" / "workflows").mkdir(parents=True) + work.mkdir(exist_ok=True) + (root / ".github" / "workflows").mkdir(parents=True, exist_ok=True) - def fake_run_tee(cmd, cwd): - (work / "linux.yml").write_text( - "name: build\njobs:\n stage_0:\n permissions:\n id-token: write\n env:\n" - " ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }}\n") - return 0, b"" + def fake_run(cmd, cwd): + calls.append(cmd) + out = cmd[cmd.index("--out") + 1] + open(out, "w").write("name: build\n") + return 0 - monkeypatch.setattr(rs, "run_tee", fake_run_tee) + monkeypatch.setattr(rs, "run", fake_run) monkeypatch.setattr(rs, "work_dir", lambda distro: work) monkeypatch.setattr(rs, "ROOT", root) - - assert rs.generate_gha("lyrical", ["--platform", "linux-64"]) == 0 - text = (root / ".github" / "workflows" / "build.yaml").read_text() - assert "ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN_LYRICAL }}" in text - assert "secrets.ANACONDA_API_TOKEN }}" not in text + assert rs.generate_gha(distro, ["--platform", platform]) == 0 + return calls[-1], work, root + + +def variables(cmd): + return dict(cmd[i + 1].split("=", 1) for i, a in enumerate(cmd) if a == "--var") + + +def test_gha_renders_the_template_for_the_distribution(monkeypatch, tmp_path): + monkeypatch.setattr(build_timings, "TIMINGS", tmp_path / "no-timings") + cmd, _, root = gha(monkeypatch, tmp_path, "lyrical", "linux-64") + assert cmd[:2] == [rs.VINCA_RS, "workflow"] + assert cmd[cmd.index("--template") + 1] == str(root / ".github" / "build-template.yaml") + assert cmd[cmd.index("--out") + 1] == str(root / ".github" / "workflows" / "build.yaml") + assert variables(cmd) == { + "distro": "lyrical", + "branch": "buildbranch_lyrical_linux_64", + # each distribution has its own anaconda.org token + "secret": "ANACONDA_API_TOKEN_LYRICAL", + "keep_file": f"distros/lyrical/work/{build_timings.JOURNAL}", + } + + +def test_gha_uses_the_measured_times(monkeypatch, tmp_path): + monkeypatch.setattr(build_timings, "TIMINGS", tmp_path / "timings") + cmd, _, _ = gha(monkeypatch, tmp_path, "jazzy", "linux-64") + assert "--timings" not in cmd # no times yet + (tmp_path / "timings").mkdir() + (tmp_path / "timings" / "linux-64.json").write_text('{"ros2-a": {"jazzy": 3.0}}\n') + cmd, work, _ = gha(monkeypatch, tmp_path, "jazzy", "linux-64") + assert cmd[cmd.index("--timings") + 1] == "build_timings.json" + assert (work / "build_timings.json").is_file()