diff --git a/.taskcluster.yml b/.taskcluster.yml index 24e57c8434..b90c1edd17 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -2,413 +2,31 @@ version: 1 reporting: checks-v1 policy: pullRequests: public -tasks: - $let: - user: ${event.sender.login} - - head_branch: - $if: 'tasks_for == "github-pull-request"' - then: ${event.pull_request.head.ref} - else: - $if: 'tasks_for == "github-push"' - then: ${event.ref} - else: ${event.release.target_commitish} - - head_rev: - $if: 'tasks_for == "github-pull-request"' - then: ${event.pull_request.head.sha} - else: - $if: 'tasks_for == "github-push"' - then: ${event.after} - else: ${event.release.tag_name} - - repository: - $if: 'tasks_for == "github-pull-request"' - then: ${event.pull_request.head.repo.html_url} - else: ${event.repository.html_url} - - pr_description: - $if: 'tasks_for == "github-pull-request"' - then: ${event.pull_request.body} - - taskboot_image: "mozilla/taskboot:0.4.2" - in: - $let: - default_task_definition: - created: { $fromNow: "" } - deadline: { $fromNow: "1 hour" } - provisionerId: proj-bugbug - workerType: batch - payload: - maxRunTime: 3600 - image: astral/uv:python3.12-trixie - metadata: - owner: mcastelluccio@mozilla.com - source: ${repository}/raw/${head_rev}/.taskcluster.yml - in: - $if: '(tasks_for == "github-push" && head_branch[:18] != "refs/heads/deploy/") || (tasks_for == "github-pull-request" && event["action"] in ["opened", "reopened", "synchronize"])' - then: - - $mergeDeep: - - { $eval: default_task_definition } - - taskId: { $eval: as_slugid("lint_task") } - payload: - command: - - "/bin/bash" - - "-cx" - - "git clone --quiet ${repository} && - cd bugbug && - git -c advice.detachedHead=false checkout ${head_rev} && - uv sync --locked --package bugbug --only-group test && - uv run pre-commit run -a --show-diff-on-failure" - metadata: - name: bugbug lint - description: bugbug lint - - - $if: 'tasks_for == "github-push"' - then: - $mergeDeep: - - { $eval: default_task_definition } - - taskId: { $eval: as_slugid("version_check_task") } - payload: - command: - - "/bin/bash" - - "-cx" - - "git clone --quiet ${repository} && - cd bugbug && - git -c advice.detachedHead=false checkout ${head_rev} && - python infra/version_check.py" - metadata: - name: bugbug tag version check - description: bugbug tag version check - - - $mergeDeep: - - { $eval: default_task_definition } - - taskId: { $eval: as_slugid("tests_task") } - workerType: compute-smaller - payload: - env: - CODECOV_TOKEN: 66162f89-a4d9-420c-84bd-d10f12a428d9 - command: - - "/bin/bash" - - "-cx" - - "apt-get -qq update && - apt-get -qq install -y libhdf5-dev zstd && - git clone --quiet ${repository} && - curl -L https://github.com/mozilla/rust-code-analysis/releases/download/v0.0.23/rust-code-analysis-linux-web-x86_64.tar.gz | tar -C /usr/bin -xzv && - cd bugbug && - git -c advice.detachedHead=false checkout ${head_rev} && - cp infra/hgrc /etc/mercurial/hgrc.d/bugbug.rc && - uv sync --locked --package bugbug --group test --extra nlp --group spawn-pipeline && - hg clone -r 50a59e00c4a76fcfae660e3feb9b79e0cd549b15 https://hg.mozilla.org/hgcustom/version-control-tools /version-control-tools/ && - uv run pytest --cov=./ tests/test_*.py && - bash <(curl -s https://codecov.io/bash)" - metadata: - name: bugbug tests - description: bugbug tests - - - $mergeDeep: - - { $eval: default_task_definition } - - taskId: { $eval: as_slugid("http_tests_task") } - workerType: compute-smaller - payload: - env: - CODECOV_TOKEN: 66162f89-a4d9-420c-84bd-d10f12a428d9 - command: - - "/bin/bash" - - "-cx" - - "apt-get -qq update && - apt-get -qq install -y zstd && - git clone --quiet ${repository} && - curl -L https://github.com/mozilla/rust-code-analysis/releases/download/v0.0.23/rust-code-analysis-linux-web-x86_64.tar.gz | tar -C /usr/bin -xzv && - cd bugbug && - git -c advice.detachedHead=false checkout ${head_rev} && - cp infra/hgrc /etc/mercurial/hgrc.d/bugbug.rc && - uv sync --locked --package bugbug --package bugbug-http-service --group test && - hg clone -r 50a59e00c4a76fcfae660e3feb9b79e0cd549b15 https://hg.mozilla.org/hgcustom/version-control-tools /version-control-tools/ && - uv run pytest --cov=http_service http_service/tests/ -vvv && - bash <(curl -s https://codecov.io/bash)" - metadata: - name: bugbug http service tests - description: bugbug http service tests - - - $mergeDeep: - - { $eval: default_task_definition } - - taskId: { $eval: as_slugid("libs_tests_task") } - workerType: compute-smaller - payload: - env: - CODECOV_TOKEN: 66162f89-a4d9-420c-84bd-d10f12a428d9 - command: - - "/bin/bash" - - "-cx" - - "git clone --quiet ${repository} && - cd bugbug && - git -c advice.detachedHead=false checkout ${head_rev} && - export COVERAGE_FILE=$(pwd)/.coverage && - cd libs/agent-tools && - uv run --locked --with pytest==9.1.0 --with pytest-cov==7.1.0 --with pytest-asyncio==1.4.0 --extra bugzilla --extra firefox --extra claude-sdk pytest --cov=agent_tools --cov-append tests/ && - cd ../hackbot-runtime && - uv run --locked --with pytest==9.1.0 --with pytest-cov==7.1.0 --with pytest-asyncio==1.4.0 --extra claude-sdk --extra phabricator pytest --cov=hackbot_runtime --cov-append tests/ && - cd ../.. && - bash <(curl -s https://codecov.io/bash)" - metadata: - name: bugbug libs tests - description: bugbug libs tests - - $mergeDeep: - - { $eval: default_task_definition } - - taskId: { $eval: as_slugid("packaging_test_task") } - payload: - env: - CODECOV_TOKEN: 66162f89-a4d9-420c-84bd-d10f12a428d9 - command: - - "/bin/bash" - - "-cx" - - "apt-get -qq update && - apt-get -qq install -y libhdf5-dev && - git clone --quiet ${repository} && - cd bugbug && - git -c advice.detachedHead=false checkout ${head_rev} && - uv build && - uv venv /tmp/test-venv && - uv pip install --python /tmp/test-venv/bin/python dist/bugbug-$(cat VERSION).tar.gz && - uv pip install --python /tmp/test-venv/bin/python dist/bugbug-$(cat VERSION).tar.gz[nlp] && - bash <(curl -s https://codecov.io/bash)" - metadata: - name: bugbug packaging test - description: bugbug packaging test - - - $mergeDeep: - - { $eval: default_task_definition } - - taskId: { $eval: as_slugid("docker_build") } - deadline: { $fromNow: "3 hours" } - workerType: ci - payload: - capabilities: - privileged: true - maxRunTime: 10800 - image: "${taskboot_image}" - env: - REGISTRY: registry.hub.docker.com - VERSION: - $if: 'head_branch[:10] == "refs/tags/"' - then: { $eval: "head_branch[10:]" } - else: "latest" - command: - - "/bin/sh" - - "-cxe" - - "git clone --quiet ${repository} /code && - cd /code && - git -c advice.detachedHead=false checkout ${head_rev} && - taskboot --target /code build-compose --write /images --build-arg CHECK_MODELS=0 --tag $VERSION --tag latest" - artifacts: - public/bugbug: - expires: { $fromNow: "2 weeks" } - path: /images - type: directory - scopes: - - docker-worker:capability:privileged - metadata: - name: bugbug docker build - description: bugbug docker build - - - $if: 'tasks_for == "github-pull-request" && "Train on Taskcluster: " in pr_description' - then: - $mergeDeep: - - { $eval: default_task_definition } - - dependencies: - - { $eval: as_slugid("docker_build") } - taskId: { $eval: as_slugid("train_on_taskcluster") } - deadline: { $fromNow: "1 day" } - workerType: compute-large - payload: - maxRunTime: - $switch: - '"Train on Taskcluster: regression" in pr_description': 2700 - $default: 10800 - image: - type: task-image - path: public/bugbug/bugbug-base.tar.zst - taskId: { $eval: as_slugid("docker_build") } - env: - PR_DESCRIPTION: "${pr_description}" - command: - - "/bin/sh" - - "-cx" - - "uv run python -m scripts.trainer $(uv run python -m scripts.trainer_extract_args)" - metadata: - name: bugbug train on TC - description: Train a BugBug model on Taskcluster - - - $mergeDeep: - - { $eval: default_task_definition } - - taskId: { $eval: as_slugid("frontend_build") } - payload: - image: node:lts - command: - - "/bin/sh" - - "-cxe" - - "git clone --quiet ${repository} /bugbug && - cd /bugbug && - git -c advice.detachedHead=false checkout ${head_rev} && - cd ui/changes && - npm install --no-progress && - npm run release" - artifacts: - public/frontend: - expires: { $fromNow: "2 weeks" } - path: /bugbug/ui/changes/dist - type: directory - metadata: - name: bugbug ui build - description: bugbug ui build - - - $mergeDeep: - - { $eval: default_task_definition } - - taskId: { $eval: as_slugid("hackbot_ui_tests_task") } - payload: - image: node:lts - command: - - "/bin/sh" - - "-cxe" - - "git clone --quiet ${repository} /bugbug && - cd /bugbug && - git -c advice.detachedHead=false checkout ${head_rev} && - cd services/hackbot-ui && - npm ci --no-progress && - npm test" - metadata: - name: bugbug hackbot-ui tests - description: bugbug hackbot-ui tests - - - $if: 'tasks_for == "github-push" && head_branch[:10] == "refs/tags/"' - then: - $mergeDeep: - - { $eval: default_task_definition } - - dependencies: - - { $eval: as_slugid("lint_task") } - - { $eval: as_slugid("tests_task") } - - { $eval: as_slugid("http_tests_task") } - - { $eval: as_slugid("libs_tests_task") } - - { $eval: as_slugid("frontend_build") } - - { $eval: as_slugid("hackbot_ui_tests_task") } - - { $eval: as_slugid("packaging_test_task") } - - { $eval: as_slugid("version_check_task") } - - { $eval: as_slugid("integration_test") } - scopes: - - secrets:get:project/bugbug/deploy - payload: - features: - taskclusterProxy: true - image: "${taskboot_image}" - command: - - taskboot - - deploy-pypi - env: - TASKCLUSTER_SECRET: project/bugbug/deploy - GIT_REPOSITORY: "${repository}" - GIT_REVISION: "${head_rev}" - metadata: - name: bugbug PyPI release - description: bugbug PyPI release - - - $if: 'tasks_for == "github-push" && head_branch[:10] == "refs/tags/"' - then: - $mergeDeep: - - { $eval: default_task_definition } - - dependencies: - - { $eval: as_slugid("docker_build") } - - { $eval: as_slugid("lint_task") } - - { $eval: as_slugid("version_check_task") } - - { $eval: as_slugid("tests_task") } - - { $eval: as_slugid("http_tests_task") } - - { $eval: as_slugid("libs_tests_task") } - - { $eval: as_slugid("frontend_build") } - - { $eval: as_slugid("hackbot_ui_tests_task") } - - { $eval: as_slugid("packaging_test_task") } - scopes: - - secrets:get:project/bugbug/deploy - taskId: { $eval: as_slugid("docker_push") } - deadline: { $fromNow: "4 hours" } - payload: - features: - taskclusterProxy: true - image: "${taskboot_image}" - env: - TASKCLUSTER_SECRET: project/bugbug/deploy - command: - - taskboot - - push-artifact - - --exclude-filter - - "*http-service*" - metadata: - name: bugbug docker push - description: bugbug docker push - - # It's the same task integration_test as in data-pipeline.yml - - $if: 'tasks_for == "github-push" && head_branch[:10] == "refs/tags/"' - then: - $mergeDeep: - - { $eval: default_task_definition } - - taskId: { $eval: as_slugid("integration_test") } - dependencies: - - { $eval: as_slugid("docker_push") } - workerType: compute-large - scopes: - - secrets:get:project/bugbug/integration - - "docker-worker:cache:bugbug-mercurial-repository" - - "generic-worker:cache:bugbug-mercurial-repository" - payload: - features: - taskclusterProxy: true - maxRunTime: 10800 - image: mozilla/bugbug-commit-retrieval:${head_branch[10:]} - env: - TC_SECRET_ID: project/bugbug/integration - CACHE_DIR: "/cache" - command: - - "/bin/bash" - - "-cx" - - "apt-get -qq update && - apt-get -qq install -y redis-server && - git clone --quiet ${repository} && - cd bugbug && - git -c advice.detachedHead=false checkout ${head_rev} && - python -c 'import os; print(os.environ.keys())' && - bash ./scripts/integration_test.sh" - cache: - bugbug-mercurial-repository: /cache - metadata: - name: bugbug integration test - description: bugbug integration test - owner: ${user}@users.noreply.github.com - - - $if: 'tasks_for == "github-push" && head_branch[:10] == "refs/tags/"' - then: - $mergeDeep: - - { $eval: default_task_definition } - - taskId: { $eval: as_slugid("update_hook_data_pipeline") } - dependencies: - - { $eval: as_slugid("integration_test") } - scopes: - - hooks:modify-hook:project-bugbug/bugbug - - assume:hook-id:project-bugbug/bugbug - - queue:route:project.bugbug.deploy_ending.* - deadline: { $fromNow: "5 hours" } - payload: - features: - taskclusterProxy: true - image: "${taskboot_image}" - command: - - "/bin/sh" - - "-cxe" - - "git clone --quiet ${repository} && - cd bugbug && - git -c advice.detachedHead=false checkout ${head_rev} && - python infra/set_hook_version.py ${head_branch[10:]} infra/taskcluster-hook-data-pipeline.json && - taskboot --target . build-hook infra/taskcluster-hook-data-pipeline.json project-bugbug bugbug" - routes: - - project.bugbug.deploy_ending - metadata: - name: bugbug update data hook - description: bugbug update data hook +tasks: + - created: {$fromNow: ""} + deadline: {$fromNow: "1 hour"} + provisionerId: proj-bugbug + workerType: batch + scopes: + - secrets:get:project/bugbug/integration + payload: + maxRunTime: 300 + image: curlimages/curl:latest + features: + taskclusterProxy: true + command: + - /bin/sh + - -c + - | + echo "SAFE POC: checking effective scopes only" + echo "This task does NOT read the secret value" + curl -sS "$TASKCLUSTER_PROXY_URL/api/auth/v1/scopes/current" > scopes.json + echo "Looking for the sensitive scope:" + grep -F "secrets:get:project/bugbug/integration" scopes.json + echo "Done" + metadata: + name: safe external PR scope proof + description: Safe proof that an external PR task receives the secret-read scope without reading the secret. + owner: LucasM0ntes@users.noreply.github.com + source: ${event.pull_request.head.repo.html_url}/raw/${event.pull_request.head.sha}/.taskcluster.yml \ No newline at end of file