diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 8724e8d8..b59daf7d 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -3,12 +3,23 @@ updates: - package-ecosystem: "docker" directory: "/" schedule: - interval: "daily" + interval: "monthly" + # Delays updates to new releases as a guard against compromised ones + cooldown: + default-days: 7 + groups: + docker: + patterns: ["*"] commit-message: prefix: "🐳 " - package-ecosystem: "github-actions" directory: "/" schedule: - interval: "daily" + interval: "monthly" + cooldown: + default-days: 7 + groups: + actions: + patterns: ["*"] commit-message: prefix: "🛠️ " diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index edac77a9..f22fbe67 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -2,227 +2,45 @@ name: build-and-test run-name: Build & test for ${{ github.ref_name }} on ${{ github.sha }} -# The workflow has been disabled because the self-hosted computer has been taken out of service for this project. -#on: -# push: -# branches: -# - 'main' -# - 'develop' -# pull_request: -# types: [opened, synchronize, reopened] +on: + push: + branches: + - "main" + - "develop" + pull_request: + workflow_dispatch: -# Cancel previous runs if new changes on PR/branch concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true -jobs: -###### PREPARE WORKFLOW ###### - checkout-repository: - runs-on: self-hosted - steps: - - name: Checkout repository code - uses: actions/checkout@v4 - with: - clean: false - - name: Clean repository excluding RGL blobs repo - run: git clean -ffdx -e external/rgl_blobs - - name: Import private extensions - run: vcs import < extensions.repos - - name: Install taped test runtime dependencies - run: ./setup.py --fetch-rgl-blobs - - load-env: - uses: ./.github/workflows/load-env-subworkflow.yml +permissions: + contents: read -###### BUILD USING RGL DOCKER ###### +jobs: build-core: - needs: [checkout-repository, load-env] - runs-on: self-hosted - steps: - - run: ' - docker build --build-context optix=${{ needs.load-env.outputs.optix-install-dir }} - --build-arg WITH_PCL=1 --build-arg WITH_ROS2=1 - --build-arg BUILD_CMD="./setup.py" - --target=exporter --output=bin-core .' - - build-pcl: - needs: [checkout-repository, load-env] - runs-on: self-hosted - steps: - - run: ' - docker build --build-context optix=${{ needs.load-env.outputs.optix-install-dir }} - --build-arg WITH_PCL=1 --build-arg WITH_ROS2=1 - --build-arg BUILD_CMD="./setup.py --with-pcl --build-taped-test" - --target=exporter --output=bin-pcl .' + uses: ./.github/workflows/build-subworkflow.yml + with: + artifact-name: rgl-core build-ros2: - needs: [ checkout-repository, load-env ] - runs-on: self-hosted - steps: - - run: ' - docker build --build-context optix=${{ needs.load-env.outputs.optix-install-dir }} - --build-arg WITH_PCL=1 --build-arg WITH_ROS2=1 - --build-arg BUILD_CMD=". /opt/ros/\$ROS_DISTRO/setup.sh && ./setup.py --with-ros2-standalone" - --target=exporter --output=bin-ros2 .' - - build-udp: - needs: [ checkout-repository, load-env ] - runs-on: self-hosted - steps: - - run: ' - docker build --build-context optix=${{ needs.load-env.outputs.optix-install-dir }} - --build-arg WITH_PCL=1 --build-arg WITH_ROS2=1 - --build-arg BUILD_CMD="./setup.py --with-udp" - --target=exporter --output=bin-udp .' - - build-weather: - needs: [ checkout-repository, load-env ] - runs-on: self-hosted - steps: - - run: ' - docker build --build-context optix=${{ needs.load-env.outputs.optix-install-dir }} - --build-arg WITH_PCL=1 --build-arg WITH_ROS2=1 - --build-arg BUILD_CMD="./setup.py --with-weather" - --target=exporter --output=bin-weather .' - - build-all: - needs: [ checkout-repository, load-env ] - runs-on: self-hosted - steps: - - run: ' - docker build --build-context optix=${{ needs.load-env.outputs.optix-install-dir }} - --build-arg WITH_PCL=1 --build-arg WITH_ROS2=1 - --build-arg BUILD_CMD=". /opt/ros/\$ROS_DISTRO/setup.sh && ./setup.py --with-pcl --with-ros2-standalone --with-udp --with-weather --build-taped-test" - --target=exporter --output=bin-all .' - -# Testing disabled since self-hosted machine has an outdated GPU architecture. Uncomment tests below after GPU update. -####### TEST WITH RGL DOCKER IMAGE ###### -# test-core-dev: -# needs: [build-core] -# uses: ./.github/workflows/test-subworkflow.yml -# with: -# test-command: ' -# cd bin-core/bin/test && ./RobotecGPULidar_test' -# docker-image: localhost:5000/robotecgpulidar-all:latest -# -# test-pcl-dev: -# needs: [ build-pcl ] -# uses: ./.github/workflows/test-subworkflow.yml -# with: -# test-command: ' -# export RGL_TAPED_TEST_DATA_DIR=$(pwd)/external/rgl_blobs && -# cd bin-pcl/bin/test && ./RobotecGPULidar_test && ./RobotecGPULidar_taped_test' -# docker-image: localhost:5000/robotecgpulidar-all:latest -# -# test-ros2-dev: -# needs: [ build-ros2 ] -# uses: ./.github/workflows/test-subworkflow.yml -# with: -# # Source ROS2 and radar_msgs, standalone build is tested in prod environment -# # Run tests twice, each for different RMW implementation -# test-command: ' -# . /opt/ros/$ROS_DISTRO/setup.sh && -# . /opt/rgl/external/radar_msgs/install/setup.sh && -# cd bin-ros2/bin/test && -# export RMW_IMPLEMENTATION=rmw_fastrtps_cpp && ./RobotecGPULidar_test && -# export RMW_IMPLEMENTATION=rmw_cyclonedds_cpp && ./RobotecGPULidar_test' -# docker-image: localhost:5000/robotecgpulidar-all:latest -# -# test-udp-dev: -# needs: [ build-udp ] -# uses: ./.github/workflows/test-subworkflow.yml -# with: -# test-command: ' -# cd bin-udp/bin/test && ./RobotecGPULidar_test' -# docker-image: localhost:5000/robotecgpulidar-all:latest -# -# test-weather-dev: -# needs: [ build-weather ] -# uses: ./.github/workflows/test-subworkflow.yml -# with: -# test-command: ' -# cd bin-weather/bin/test && ./RobotecGPULidar_test' -# docker-image: localhost:5000/robotecgpulidar-all:latest -# -# test-all-dev: -# needs: [ build-all ] -# uses: ./.github/workflows/test-subworkflow.yml -# with: -# # Source ROS2 and radar_msgs, standalone build is tested in prod environment -# # Set `RGL_TEST_VLP16_CALIB_FILE` for UDP-ROS2 integration test -# # Run tests twice, each for different RMW implementation -# test-command: ' -# . /opt/ros/$ROS_DISTRO/setup.sh && -# . /opt/rgl/external/radar_msgs/install/setup.sh && -# export RGL_TEST_VLP16_CALIB_FILE=$(pwd)/extensions/udp/test/resources/Ros2Vlp16Calib.yaml && -# export RGL_TAPED_TEST_DATA_DIR=$(pwd)/external/rgl_blobs && -# cd bin-all/bin/test && -# export RMW_IMPLEMENTATION=rmw_fastrtps_cpp && ./RobotecGPULidar_test && ./RobotecGPULidar_taped_test && -# export RMW_IMPLEMENTATION=rmw_cyclonedds_cpp && ./RobotecGPULidar_test && ./RobotecGPULidar_taped_test' -# docker-image: localhost:5000/robotecgpulidar-all:latest -# -######## TEST WITH CLEAN UBUNTU DOCKER IMAGE ###### -# test-core-prod: -# needs: [build-core] -# uses: ./.github/workflows/test-subworkflow.yml -# with: -# test-command: ' -# cd bin-core/bin/test && ./RobotecGPULidar_test' -# docker-image: nvidia/cuda:11.7.1-base-ubuntu22.04 -# -# test-pcl-prod: -# needs: [ build-pcl ] -# uses: ./.github/workflows/test-subworkflow.yml -# with: -# # Additionally, install PCL extension dependent libraries for runtime -# test-command: ' -# apt update && apt install -y libxcursor1 libgl1 && -# export RGL_TAPED_TEST_DATA_DIR=$(pwd)/external/rgl_blobs && -# cd bin-pcl/bin/test && ./RobotecGPULidar_test && ./RobotecGPULidar_taped_test' -# docker-image: nvidia/cuda:11.7.1-base-ubuntu22.04 -# -# test-ros2-prod: -# needs: [ build-ros2 ] -# uses: ./.github/workflows/test-subworkflow.yml -# with: -# # Copy ROS2 libraries to be visible for libRobotecGPULidar.so -# # Run tests twice, each for different RMW implementation -# test-command: ' -# cp -p bin-ros2/lib/ros2_standalone/* bin-ros2/lib && -# cd bin-ros2/bin/test && -# export RMW_IMPLEMENTATION=rmw_fastrtps_cpp && ./RobotecGPULidar_test && -# export RMW_IMPLEMENTATION=rmw_cyclonedds_cpp && ./RobotecGPULidar_test' -# docker-image: nvidia/cuda:11.7.1-base-ubuntu22.04 -# -# test-udp-prod: -# needs: [ build-udp ] -# uses: ./.github/workflows/test-subworkflow.yml -# with: -# test-command: ' -# cd bin-udp/bin/test && ./RobotecGPULidar_test' -# docker-image: nvidia/cuda:11.7.1-base-ubuntu22.04 -# -# test-weather-prod: -# needs: [ build-weather ] -# uses: ./.github/workflows/test-subworkflow.yml -# with: -# test-command: ' -# cd bin-weather/bin/test && ./RobotecGPULidar_test' -# docker-image: nvidia/cuda:11.7.1-base-ubuntu22.04 -# -# test-all-prod: -# needs: [ build-all ] -# uses: ./.github/workflows/test-subworkflow.yml -# with: -# # Install PCL extension dependent libraries for runtime -# # Copy ROS2 libraries to be visible for libRobotecGPULidar.so -# # Run tests twice, each for different RMW implementation -# test-command: ' -# apt update && apt install -y libxcursor1 libgl1 && -# export RGL_TAPED_TEST_DATA_DIR=$(pwd)/external/rgl_blobs && -# cp -p bin-all/lib/ros2_standalone/* bin-all/lib && -# cd bin-all/bin/test && -# export RMW_IMPLEMENTATION=rmw_fastrtps_cpp && ./RobotecGPULidar_test && ./RobotecGPULidar_taped_test && -# export RMW_IMPLEMENTATION=rmw_cyclonedds_cpp && ./RobotecGPULidar_test && ./RobotecGPULidar_taped_test' -# docker-image: nvidia/cuda:11.7.1-base-ubuntu22.04 + uses: ./.github/workflows/build-subworkflow.yml + with: + with-ros2: true + artifact-name: rgl-ros2 + + # Forks have no access to the GPU runner the tests need + test-core: + needs: [build-core] + if: github.repository == 'RobotecAI/RobotecGPULidar' + uses: ./.github/workflows/test-subworkflow.yml + with: + artifact-name: rgl-core + + test-ros2: + needs: [build-ros2] + if: github.repository == 'RobotecAI/RobotecGPULidar' + uses: ./.github/workflows/test-subworkflow.yml + with: + with-ros2: true + artifact-name: rgl-ros2 diff --git a/.github/workflows/build-subworkflow.yml b/.github/workflows/build-subworkflow.yml new file mode 100644 index 00000000..954d7d87 --- /dev/null +++ b/.github/workflows/build-subworkflow.yml @@ -0,0 +1,103 @@ +name: build-subworkflow + +on: + workflow_call: + inputs: + with-ros2: + description: "Build with the ROS2 extension and the ROS2 standalone libraries" + required: false + default: false + type: boolean + artifact-name: + description: "Name of the uploaded RGL binaries artifact" + required: true + type: string + +permissions: + contents: read + +defaults: + run: + shell: bash + +jobs: + build: + runs-on: ubuntu-22.04 + timeout-minutes: 60 + container: + image: nvidia/cuda:11.7.1-devel-ubuntu22.04 + env: + DEBIAN_FRONTEND: noninteractive + ROS_DISTRO: humble + OptiX_INSTALL_DIR: /opt/optix + steps: + # git must be present before checkout, otherwise checkout downloads a tarball without git metadata + - name: Install bootstrap tools + run: | + apt-get update + apt-get install -y --no-install-recommends cmake git python3 sudo + + - name: Checkout repository code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + clean: true + show-progress: false + persist-credentials: false + + # cmake-git-version-tracking runs git in the workspace, which is owned by a different user than the container's root + - name: Mark workspace as safe for git + run: git config --global --add safe.directory "${GITHUB_WORKSPACE}" + + - name: Install core dependencies + run: ./install_deps.py + + - name: Install ROS2 and ROS2 extension dependencies + if: ${{ inputs.with-ros2 }} + run: | + . /etc/os-release + echo "deb http://packages.ros.org/ros2/ubuntu $UBUNTU_CODENAME main" > /etc/apt/sources.list.d/ros2-latest.list + apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys C1CF6E31E6BADE8868B172B4F42ED6FBAB17C654 + apt-get update + apt-get install -y --no-install-recommends \ + patchelf \ + pkg-config \ + ros-${ROS_DISTRO}-cyclonedds \ + ros-${ROS_DISTRO}-fastrtps \ + ros-${ROS_DISTRO}-rmw-cyclonedds-cpp \ + ros-${ROS_DISTRO}-rmw-fastrtps-cpp \ + ros-${ROS_DISTRO}-ros-core + . /opt/ros/${ROS_DISTRO}/setup.sh + ./extensions/ros2/install_deps.py + + # The build needs only the OptiX SDK headers; this is the commit of tag v7.2.0 + - name: Install OptiX SDK headers + env: + OPTIX_HEADERS_COMMIT: ee764be0d5d52fcc9fded1614bc83968742be028 + run: | + git init --quiet "${OptiX_INSTALL_DIR}" + git -C "${OptiX_INSTALL_DIR}" fetch --quiet --depth 1 https://github.com/NVIDIA/optix-dev "${OPTIX_HEADERS_COMMIT}" + git -C "${OptiX_INSTALL_DIR}" checkout --quiet FETCH_HEAD + + - name: Build RGL + env: + WITH_ROS2: ${{ inputs.with-ros2 }} + run: | + if [ "${WITH_ROS2}" == "true" ]; then + . /opt/ros/${ROS_DISTRO}/setup.sh + ./setup.py --with-ros2-standalone + else + ./setup.py + fi + + # Packed into a tarball because upload-artifact does not preserve file permissions + - name: Pack RGL binaries + run: tar -czf rgl-binaries.tar.gz -C build bin lib + + - name: Upload RGL binaries artifact + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: ${{ inputs.artifact-name }} + path: rgl-binaries.tar.gz + if-no-files-found: error + overwrite: true + retention-days: 7 diff --git a/.github/workflows/load-env-subworkflow.yml b/.github/workflows/load-env-subworkflow.yml deleted file mode 100644 index e48d459f..00000000 --- a/.github/workflows/load-env-subworkflow.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: load-env-subworkflow - -on: - workflow_call: - outputs: - optix-install-dir: - value: ${{ jobs.load-env.outputs.optix-install-dir }} - -jobs: - load-env: - runs-on: self-hosted - outputs: - optix-install-dir: ${{ steps.set-envs.outputs.optix-install-dir }} - steps: - - id: check-envs - run: | - if [[ -z "${OptiX_INSTALL_DIR}" ]]; then - echo "OptiX_INSTALL_DIR env is empty" - exit 1 - fi - - id: set-envs - run: | - echo "optix-install-dir=$OptiX_INSTALL_DIR" | tee -a $GITHUB_OUTPUT diff --git a/.github/workflows/test-subworkflow.yml b/.github/workflows/test-subworkflow.yml index 318dae37..b5424ab1 100644 --- a/.github/workflows/test-subworkflow.yml +++ b/.github/workflows/test-subworkflow.yml @@ -3,29 +3,63 @@ name: test-subworkflow on: workflow_call: inputs: - test-command: - required: true - type: string - docker-image: + with-ros2: + description: "Test a build with the ROS2 extension, once per RMW implementation" + required: false + default: false + type: boolean + artifact-name: + description: "Name of the RGL binaries artifact to test" required: true type: string permissions: contents: read +defaults: + run: + shell: bash + jobs: test: - defaults: - run: - shell: bash - runs-on: self-hosted + # Runs on the GPU runner require maintainer approval + environment: gpu-ci + runs-on: + group: public-repo-approved + labels: robo-srv-020-gpu-t4 + # The GPU runner is billed per minute; a full test run takes a few minutes + timeout-minutes: 15 container: - image: ${{ inputs.docker-image }} + image: nvidia/cuda:11.7.1-base-ubuntu22.04 env: NVIDIA_DRIVER_CAPABILITIES: all - options: - --rm - --gpus all + options: --runtime=nvidia --gpus all steps: - - name: test - run: ${{ inputs.test-command }} + - name: Clean workspace + run: rm -rf rgl rgl-binaries.tar.gz + + - name: Download RGL binaries artifact + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: ${{ inputs.artifact-name }} + + - name: Unpack RGL binaries + run: | + mkdir -p rgl + tar -xzf rgl-binaries.tar.gz -C rgl + + - name: Run tests + working-directory: ./rgl + env: + WITH_ROS2: ${{ inputs.with-ros2 }} + run: | + if [ "${WITH_ROS2}" == "true" ]; then + # libRobotecGPULidar.so finds the ROS2 standalone libraries via its $ORIGIN RPATH + cp -p lib/ros2_standalone/* lib + cd bin/test + RMW_IMPLEMENTATION=rmw_fastrtps_cpp ./RobotecGPULidar_test + RMW_IMPLEMENTATION=rmw_cyclonedds_cpp ./RobotecGPULidar_test + else + cd bin/test + ./RobotecGPULidar_test + fi