diff --git a/docs/development/testing.mdx b/docs/development/testing.mdx index caf417cb1..38418b871 100644 --- a/docs/development/testing.mdx +++ b/docs/development/testing.mdx @@ -274,3 +274,19 @@ Inspect the run's individual platform jobs before claiming full success. A queued, canceled or failed job is missing evidence, even if other jobs passed. The platform fixtures exercise local paths. Authenticated agent tool execution has separate acceptance. + +## CI aggregate predicate + +The source repository's `CI Check` gate requires success from all three independent +workers. `validate:workflows` binds its command to the executable contract in +`scripts/validate-workflow-contract.cjs`; it compares the complete command without +executing candidate YAML. Changing that command requires reviewing the contract +and its execution fixtures together. CRLF line endings and outer whitespace are +normalized; additional commands or alternate predicates are rejected. + +The workflow tests execute the current gate in Bash with controlled result +values: all-success passes, each worker failure fails, and canceled, skipped or +missing results fail. Separate validator mutations reject no-op, partial and +masked predicates, missing bindings, and step/job `continue-on-error`. Runner +placement, fork exclusion, public check identity, parallel jobs and consumer +installation templates retain their existing contracts. diff --git a/scripts/validate-workflow-contract.cjs b/scripts/validate-workflow-contract.cjs index c720eb7cc..e4a6305e2 100644 --- a/scripts/validate-workflow-contract.cjs +++ b/scripts/validate-workflow-contract.cjs @@ -36,6 +36,19 @@ const SOURCE_CI_WORKERS = { 'build-package': 'BUILD_RESULT', coverage: 'COVERAGE_RESULT', }; +// This exact executable contract is deliberately pinned independently of the YAML. +// Never execute a candidate workflow command while validating it. +const SOURCE_CI_GATE_RUN = String.raw`printf '%s\n' '| Job | Result |' '| --- | --- |' \ + "| Static checks | $STATIC_RESULT |" \ + "| Build and package | $BUILD_RESULT |" \ + "| Tests and coverage | $COVERAGE_RESULT |" | tee -a "$GITHUB_STEP_SUMMARY" +[[ "$STATIC_RESULT" == success && "$BUILD_RESULT" == success && "$COVERAGE_RESULT" == success ]]`; + +function assertSourceCiPredicate(run) { + if (typeof run !== 'string' || run.replaceAll('\r\n', '\n').trim() !== SOURCE_CI_GATE_RUN) { + throw new Error('CI Check must retain the executable all-worker success predicate.'); + } +} const SOURCE_CI_GATE_CONDITION = SOURCE_CI_CONDITION.replace( '${{ ', '${{ always() && (', ).replace(' }}', ') }}'); @@ -410,6 +423,7 @@ function assertSourceCiJobs(file, workflow) { || workers.some(id => step.env?.[SOURCE_CI_WORKERS[id]] !== '${{ needs.' + id + '.result }}')) { throw new Error(`${file} must check the result of every CI job without credentials or checkout.`); } + assertSourceCiPredicate(step.run); } function assertRunner(file, workflow) { @@ -1647,6 +1661,7 @@ function main() { if (require.main === module) main(); module.exports = { + assertSourceCiPredicate, issueAdmissionConfigurationExpression, assertNativeVerificationWorkflow, QUEUE_WAIT_MINUTES, diff --git a/src/tooling/__tests__/validate_workflow_contract.test.ts b/src/tooling/__tests__/validate_workflow_contract.test.ts index d11beaf49..b4467e648 100644 --- a/src/tooling/__tests__/validate_workflow_contract.test.ts +++ b/src/tooling/__tests__/validate_workflow_contract.test.ts @@ -1425,10 +1425,25 @@ describe('workflow contract validator', () => { expect(() => validateWorkflow(file, workflow)).toThrow('check the result of every CI job'); }); + it.each([ + ['no-op', (gate: Record) => { gate.steps[0].run = 'true'; }, 'executable all-worker'], + ['partial predicate', (gate: Record) => { gate.steps[0].run = '[[ "$STATIC_RESULT" == success ]]'; }, 'executable all-worker'], + ['masked failure', (gate: Record) => { gate.steps[0].run += '\ntrue'; }, 'executable all-worker'], + ['missing result', (gate: Record) => { delete gate.steps[0].env.BUILD_RESULT; }, 'check the result of every CI job'], + ['step continue-on-error', (gate: Record) => { gate.steps[0]['continue-on-error'] = true; }, 'check the result of every CI job'], + ['job continue-on-error', (gate: Record) => { gate['continue-on-error'] = true; }, 'check the result of every CI job'], + ] as const)('rejects an aggregate gate with %s', (_name, mutate, message) => { + const { file, workflow } = loadMutationWorkflow('.github/workflows', 'ci_check.yml'); + mutate(workflow.jobs['ci-check']); + expect(() => validateWorkflow(file, workflow)).toThrow(message); + }); + const onBashPlatform = process.platform === 'win32' ? it.skip : it; onBashPlatform.each([ ['success', 'success', 'success', 0], ['failure', 'success', 'success', 1], + ['success', 'failure', 'success', 1], + ['success', 'success', 'failure', 1], ['success', 'cancelled', 'success', 1], ['success', 'success', 'skipped', 1], ['success', 'success', '', 1],