From 4cff3640e5bbf38fffb336504e61a6e7d4d4bb5c Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 18:52:32 -0700 Subject: [PATCH 01/58] feat(orchestration): register the firefly/orchestration package in the library, deptrac and the installer --- composer.json | 6 + deptrac.yaml | 24 +++ docs/book.md | 2 +- docs/index.md | 2 +- docs/modules.md | 6 +- packages/firefly/composer.json | 1 + packages/installer/src/CapabilityCatalog.php | 1 + packages/orchestration/.gitattributes | 2 + packages/orchestration/LICENSE | 204 ++++++++++++++++++ packages/orchestration/README.md | 18 ++ .../firefly-orchestration-components.php | 8 + .../cache/firefly-orchestration-context.php | 8 + packages/orchestration/composer.json | 74 +++++++ .../src/OrchestrationServiceProvider.php | 27 +++ .../src/OrchestrationWiringProvider.php | 26 +++ .../tests/CompiledManifestFreshnessTest.php | 30 +++ .../orchestration/tests/PackageBootsTest.php | 21 ++ .../tests/ReflectionFreeOrchestrationTest.php | 31 +++ 18 files changed, 486 insertions(+), 5 deletions(-) create mode 100644 packages/orchestration/.gitattributes create mode 100644 packages/orchestration/LICENSE create mode 100644 packages/orchestration/README.md create mode 100644 packages/orchestration/cache/firefly-orchestration-components.php create mode 100644 packages/orchestration/cache/firefly-orchestration-context.php create mode 100644 packages/orchestration/composer.json create mode 100644 packages/orchestration/src/OrchestrationServiceProvider.php create mode 100644 packages/orchestration/src/OrchestrationWiringProvider.php create mode 100644 packages/orchestration/tests/CompiledManifestFreshnessTest.php create mode 100644 packages/orchestration/tests/PackageBootsTest.php create mode 100644 packages/orchestration/tests/ReflectionFreeOrchestrationTest.php diff --git a/composer.json b/composer.json index b0e106a3..34c32c1e 100644 --- a/composer.json +++ b/composer.json @@ -74,6 +74,7 @@ "Firefly\\Messaging\\Tests\\": "packages/messaging/tests/", "Firefly\\Observability\\Tests\\": "packages/observability/tests/", "Firefly\\OpenApi\\Tests\\": "packages/openapi/tests/", + "Firefly\\Orchestration\\Tests\\": "packages/orchestration/tests/", "Firefly\\Resilience\\Tests\\": "packages/resilience/tests/", "Firefly\\Scheduling\\Postgres\\Tests\\": "packages/scheduling-postgres/tests/", "Firefly\\Scheduling\\Tests\\": "packages/scheduling/tests/", @@ -108,6 +109,7 @@ "stan": "phpstan analyse --no-progress --memory-limit=1G", "test": "@php -d memory_limit=512M vendor/bin/pest", "test:browser": "@php -d memory_limit=512M vendor/bin/pest --testsuite=browser", + "test:integration": "@php -d memory_limit=512M vendor/bin/pest --group integration", "deptrac": "deptrac analyse --no-progress --config-file=deptrac.yaml", "mono-validate": "monorepo-builder validate", "check": [ @@ -140,6 +142,7 @@ "firefly/messaging": "self.version", "firefly/observability": "self.version", "firefly/openapi": "self.version", + "firefly/orchestration": "self.version", "firefly/resilience": "self.version", "firefly/scheduling": "self.version", "firefly/scheduling-postgres": "self.version", @@ -171,6 +174,7 @@ "Firefly\\Kernel\\": "packages/kernel/src/", "Firefly\\Messaging\\": "packages/messaging/src/", "Firefly\\Observability\\": "packages/observability/src/", + "Firefly\\Orchestration\\": "packages/orchestration/src/", "Firefly\\OpenApi\\": "packages/openapi/src/", "Firefly\\Resilience\\": "packages/resilience/src/", "Firefly\\Scheduling\\": "packages/scheduling/src/", @@ -259,6 +263,8 @@ "Firefly\\Security\\OAuth2\\Client\\SecurityOAuth2ClientWiringProvider", "Firefly\\FeatureFlags\\FeatureFlagsServiceProvider", "Firefly\\FeatureFlags\\FeatureFlagsWiringProvider", + "Firefly\\Orchestration\\OrchestrationServiceProvider", + "Firefly\\Orchestration\\OrchestrationWiringProvider", "Firefly\\Cli\\Boot\\FireflyCacheServiceProvider" ] } diff --git a/deptrac.yaml b/deptrac.yaml index e1eab59b..45311b79 100644 --- a/deptrac.yaml +++ b/deptrac.yaml @@ -102,6 +102,10 @@ deptrac: collectors: - type: directory value: packages/feature-flags/src/.* + - name: Orchestration + collectors: + - type: directory + value: packages/orchestration/src/.* - name: OpenApi collectors: - type: directory @@ -494,6 +498,26 @@ deptrac: - Actuator - Resilience + # Orchestration is the saga / TCC / durable-workflow capability. It reads Config, carries Container/Context + # attributes and boot passes, extends AutoConfigure's base, reuses Kernel exceptions, builds its step + # retries, timeouts and the CIRCUIT_BREAKER compensation policy on Resilience (Retry, TimeLimiter, + # CircuitBreaker, Duration), serializes writers through Scheduling's DistributedLock and contributes its tick + # to Scheduling's manifest (Cadence, ScheduledDescriptor), publishes lifecycle and step events and listens + # for trigger events through Eda's EventPublisher, and supplies an Actuator endpoint and HealthIndicator. + # NO Web edge (its REST routes mount on the Illuminate router, the FlagdSyncRouteRegistrar precedent) and NO + # Data edge (state is written through illuminate/database). Cli (firefly:cache compiles its manifest) and + # Observability (the metrics and tracing adapters) depend on it; it depends on neither. + Orchestration: + - Kernel + - Container + - Config + - Context + - AutoConfigure + - Resilience + - Scheduling + - Eda + - Actuator + # OpenApi is a top-of-stack capability (like Web/Cqrs/Security/Actuator): it generates an OpenAPI 3.1 # document from manifests that already exist, so it is nearly all READS. It reads Web's RouteManifest/ # RouteDescriptor (paths, verbs, statuses, binding plans) and Validation's ConstraintManifest + Rule diff --git a/docs/book.md b/docs/book.md index e289f3b5..b11d8e0a 100644 --- a/docs/book.md +++ b/docs/book.md @@ -27,7 +27,7 @@ composer global require fireflyframework/larafly firefly new my-app ``` -All 30 components are included in that package. An existing application starts with +All 31 components are included in that package. An existing application starts with `composer require fireflyframework/larafly`; compatible legacy `firefly/*` requirements are then satisfied by the framework's `replace` declarations. See [Installation](installation.md) for details. diff --git a/docs/index.md b/docs/index.md index 7a6ed7ba..da69695f 100644 --- a/docs/index.md +++ b/docs/index.md @@ -8,7 +8,7 @@ CQRS, event-driven architecture, first-party security including both halves of O directly onto Laravel's own runtime. Nothing is forked or wrapped: a LaraFly app is, in every respect a Laravel developer would recognize, still a Laravel app — it just boots like a Spring Boot one. -The whole framework ships as one Composer library, `fireflyframework/larafly`, containing all 30 components, +The whole framework ships as one Composer library, `fireflyframework/larafly`, containing all 31 components, wired together by a single zero-reflection boot pipeline: a component scan compiles to a cached manifest once, and every request after that runs against plain PHP arrays — no runtime reflection on the hot path. diff --git a/docs/modules.md b/docs/modules.md index 69241792..6629fec4 100644 --- a/docs/modules.md +++ b/docs/modules.md @@ -1,15 +1,15 @@ # Modules -LaraFly publishes one `fireflyframework/larafly` library containing the **30 component directories** under +LaraFly publishes one `fireflyframework/larafly` library containing the **31 component directories** under `packages/*`, each with its own tests or metadata. The **33 guides** below explain their public surfaces. Several components carry more than one guide: `firefly/data` answers for *Data & Repositories*, *Relational Data* and *Transactions*. Laravel discovers the root library's providers, and configuration selects optional features. A bean you declare wins over the framework default. Two idioms deliver that outcome, and -which one you are looking at matters the moment you go reading the source. **18 of the 30 packages** carry a +which one you are looking at matters the moment you go reading the source. **18 of the 31 packages** carry a `#[Configuration]` class whose `#[Bean]` methods sit behind `#[ConditionalOnMissingBean]`, so a bean of yours makes the framework's back off silently — `packages/data/src/DataAutoConfiguration.php` is the model. The -**other 12** have no such class to go looking for; `firefly/web`, the one nearly every reader installs, guards +**other 13** have no such class to go looking for; `firefly/web`, the one nearly every reader installs, guards each binding in `WebServiceProvider::register()` with a `bound()` check instead, first-one-wins by hand.
diff --git a/packages/firefly/composer.json b/packages/firefly/composer.json index faee7fec..876f2454 100644 --- a/packages/firefly/composer.json +++ b/packages/firefly/composer.json @@ -38,6 +38,7 @@ "firefly/messaging": "*@dev", "firefly/observability": "*@dev", "firefly/openapi": "*@dev", + "firefly/orchestration": "*@dev", "firefly/resilience": "*@dev", "firefly/scheduling": "*@dev", "firefly/scheduling-postgres": "*@dev", diff --git a/packages/installer/src/CapabilityCatalog.php b/packages/installer/src/CapabilityCatalog.php index 395f3719..2f8e07da 100644 --- a/packages/installer/src/CapabilityCatalog.php +++ b/packages/installer/src/CapabilityCatalog.php @@ -44,6 +44,7 @@ public static function all(): array new Capability('admin', 'firefly/admin', 'Server-rendered dashboard over the actuator (a Spring Boot Admin analog)'), new Capability('openapi', 'firefly/openapi', 'OpenAPI 3.1 document generated from the route and constraint manifests, plus a viewer'), new Capability('feature-flags', 'firefly/feature-flags', 'Feature flags on OpenFeature: flagd documents, targeting and percentage rollouts, a writable store, admin and actuator controls'), + new Capability('orchestration', 'firefly/orchestration', 'Sagas, TCC and durable workflows: compensation policies, signals and timers, recovery, a dead-letter queue, EDA triggers', requires: ['eda', 'scheduling', 'resilience', 'actuator']), new Capability('eda-kafka', 'firefly/eda-kafka', 'Kafka publisher/consumer over ext-rdkafka', requires: ['eda'], adapter: true), new Capability('eda-rabbitmq', 'firefly/eda-rabbitmq', 'RabbitMQ publisher/consumer over php-amqplib', requires: ['eda'], adapter: true), diff --git a/packages/orchestration/.gitattributes b/packages/orchestration/.gitattributes new file mode 100644 index 00000000..538b69a4 --- /dev/null +++ b/packages/orchestration/.gitattributes @@ -0,0 +1,2 @@ +/tests export-ignore +/.gitattributes export-ignore diff --git a/packages/orchestration/LICENSE b/packages/orchestration/LICENSE new file mode 100644 index 00000000..2240005d --- /dev/null +++ b/packages/orchestration/LICENSE @@ -0,0 +1,204 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +Copyright 2026 Firefly Software Solutions Inc. diff --git a/packages/orchestration/README.md b/packages/orchestration/README.md new file mode 100644 index 00000000..95e935a3 --- /dev/null +++ b/packages/orchestration/README.md @@ -0,0 +1,18 @@ +# firefly/orchestration + +LaraFly's distributed-transaction patterns, shaped after the Java `fireflyframework-orchestration` engine: +**sagas** (forward steps with compensations, five compensation policies, error handlers, a builder DSL and +saga composition), **TCC** (ordered try/confirm/cancel participants with per-phase timeouts and retries) and a +**durable workflow** engine (dependencies, ExpressionLanguage conditions, signals, timers, wait-for-all/any, +child workflows, continue-as-new, queries, suspend/resume and search attributes, triggered synchronously or +through the Laravel queue). + +Definitions are discovered from attributes (`#[Saga]`, `#[Tcc]`, `#[Workflow]`, …) and compiled by +`php artisan firefly:cache`; their topology is validated at boot. Execution state is checkpointed at every +step through the `ExecutionPersistenceProvider` port — `memory` (default), `cache` or `database` +(`firefly.orchestration.persistence.provider`) — so a recovery sweep can resume or compensate an execution a +crashed process left behind. Failed executions are parked in a dead-letter queue (`memory` or `database`), +lifecycle events publish through `firefly/eda`, the `orchestration` actuator endpoint and health component +report what is running, and `php artisan firefly:orchestration:*` lists, shows, signals, ticks and retries. + +Apache-2.0 © Firefly Software Solutions Inc. diff --git a/packages/orchestration/cache/firefly-orchestration-components.php b/packages/orchestration/cache/firefly-orchestration-components.php new file mode 100644 index 00000000..b2ca1444 --- /dev/null +++ b/packages/orchestration/cache/firefly-orchestration-components.php @@ -0,0 +1,8 @@ + + */ + public function passes(): array + { + return []; + } +} diff --git a/packages/orchestration/tests/CompiledManifestFreshnessTest.php b/packages/orchestration/tests/CompiledManifestFreshnessTest.php new file mode 100644 index 00000000..842416d0 --- /dev/null +++ b/packages/orchestration/tests/CompiledManifestFreshnessTest.php @@ -0,0 +1,30 @@ + dirname(__DIR__).'/src']; + $cache = dirname(__DIR__).'/cache'; + + $freshComponents = sys_get_temp_dir().'/firefly-orchestration-fresh-c-'.bin2hex(random_bytes(6)).'.php'; + $freshContext = sys_get_temp_dir().'/firefly-orchestration-fresh-x-'.bin2hex(random_bytes(6)).'.php'; + + try { + (new AutoConfigManifestCompiler)->write($src, $freshComponents, $freshContext); + + expect(ComponentManifest::load($freshComponents))->toEqual(ComponentManifest::load($cache.'/firefly-orchestration-components.php')) + ->and(ContextManifest::load($freshContext))->toEqual(ContextManifest::load($cache.'/firefly-orchestration-context.php')); + } finally { + @unlink($freshComponents); + @unlink($freshContext); + } +}); diff --git a/packages/orchestration/tests/PackageBootsTest.php b/packages/orchestration/tests/PackageBootsTest.php new file mode 100644 index 00000000..e33aa412 --- /dev/null +++ b/packages/orchestration/tests/PackageBootsTest.php @@ -0,0 +1,21 @@ + []], + [OrchestrationServiceProvider::class, OrchestrationWiringProvider::class], + needs: ['cache'], + ); + + expect($app->make(ApplicationContext::class))->toBeInstanceOf(ApplicationContext::class); +}); diff --git a/packages/orchestration/tests/ReflectionFreeOrchestrationTest.php b/packages/orchestration/tests/ReflectionFreeOrchestrationTest.php new file mode 100644 index 00000000..db34bf87 --- /dev/null +++ b/packages/orchestration/tests/ReflectionFreeOrchestrationTest.php @@ -0,0 +1,31 @@ + basenames under $dir whose contents reference boot-time reflection */ +function orchestrationReflectionHits(string $dir): array +{ + $hits = []; + $files = new RecursiveIteratorIterator(new RecursiveDirectoryIterator($dir, RecursiveDirectoryIterator::SKIP_DOTS)); + /** @var SplFileInfo $file */ + foreach ($files as $file) { + if (! $file->isFile() || $file->getExtension() !== 'php') { + continue; + } + $contents = (string) file_get_contents((string) $file->getRealPath()); + if (preg_match('/ReflectionClass|ReflectionMethod|getAttributes/', $contents) === 1) { + $hits[] = $file->getBasename(); + } + } + sort($hits); + + return $hits; +} + +/** + * Production loads the compiled OrchestrationManifest by require; reflection is allowed in exactly one file, + * the scanner that compiles it (Task 16 adds it and widens this list to that one name). + */ +it('confines reflection in firefly/orchestration to the scanner', function (): void { + expect(orchestrationReflectionHits(__DIR__.'/../src'))->toBe([]); +}); From eb3620bc14568f22118a9b1686fe9c93f7895b80 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 18:56:22 -0700 Subject: [PATCH 02/58] feat(orchestration): execution enums, topology layering, conditions and the exception family --- packages/orchestration/src/Core/Clock.php | 16 ++ .../src/Core/CompensationPolicy.php | 38 ++++ .../src/Core/Condition/ConditionEvaluator.php | 53 ++++++ .../Core/Exception/BackpressureException.php | 14 ++ .../Core/Exception/CompensationException.php | 16 ++ .../Exception/DeadLetterNotFoundException.php | 14 ++ .../Exception/DefinitionNotFoundException.php | 14 ++ .../Exception/ExecutionLockedException.php | 21 +++ .../Exception/ExecutionNotFoundException.php | 14 ++ .../InvalidExecutionStateException.php | 14 ++ .../Core/Exception/OrchestrationException.php | 24 +++ .../Exception/PatternDisabledException.php | 16 ++ .../Core/Exception/SignalTimeoutException.php | 8 + .../Core/Exception/StepTimeoutException.php | 21 +++ .../Exception/TopologyValidationException.php | 21 +++ .../src/Core/ExecutionPattern.php | 13 ++ .../src/Core/ExecutionStatus.php | 69 +++++++ packages/orchestration/src/Core/Rows.php | 169 ++++++++++++++++++ .../orchestration/src/Core/StepStatus.php | 36 ++++ .../orchestration/src/Core/SystemClock.php | 17 ++ packages/orchestration/src/Core/TccPhase.php | 13 ++ .../src/Core/Topology/TopologyBuilder.php | 100 +++++++++++ .../orchestration/src/Core/TriggerMode.php | 15 ++ .../tests/Core/ConditionEvaluatorTest.php | 24 +++ .../orchestration/tests/Core/EnumsTest.php | 45 +++++ .../orchestration/tests/Core/RowsTest.php | 23 +++ .../tests/Core/TopologyBuilderTest.php | 35 ++++ .../tests/Support/MutableClock.php | 35 ++++ 28 files changed, 898 insertions(+) create mode 100644 packages/orchestration/src/Core/Clock.php create mode 100644 packages/orchestration/src/Core/CompensationPolicy.php create mode 100644 packages/orchestration/src/Core/Condition/ConditionEvaluator.php create mode 100644 packages/orchestration/src/Core/Exception/BackpressureException.php create mode 100644 packages/orchestration/src/Core/Exception/CompensationException.php create mode 100644 packages/orchestration/src/Core/Exception/DeadLetterNotFoundException.php create mode 100644 packages/orchestration/src/Core/Exception/DefinitionNotFoundException.php create mode 100644 packages/orchestration/src/Core/Exception/ExecutionLockedException.php create mode 100644 packages/orchestration/src/Core/Exception/ExecutionNotFoundException.php create mode 100644 packages/orchestration/src/Core/Exception/InvalidExecutionStateException.php create mode 100644 packages/orchestration/src/Core/Exception/OrchestrationException.php create mode 100644 packages/orchestration/src/Core/Exception/PatternDisabledException.php create mode 100644 packages/orchestration/src/Core/Exception/SignalTimeoutException.php create mode 100644 packages/orchestration/src/Core/Exception/StepTimeoutException.php create mode 100644 packages/orchestration/src/Core/Exception/TopologyValidationException.php create mode 100644 packages/orchestration/src/Core/ExecutionPattern.php create mode 100644 packages/orchestration/src/Core/ExecutionStatus.php create mode 100644 packages/orchestration/src/Core/Rows.php create mode 100644 packages/orchestration/src/Core/StepStatus.php create mode 100644 packages/orchestration/src/Core/SystemClock.php create mode 100644 packages/orchestration/src/Core/TccPhase.php create mode 100644 packages/orchestration/src/Core/Topology/TopologyBuilder.php create mode 100644 packages/orchestration/src/Core/TriggerMode.php create mode 100644 packages/orchestration/tests/Core/ConditionEvaluatorTest.php create mode 100644 packages/orchestration/tests/Core/EnumsTest.php create mode 100644 packages/orchestration/tests/Core/RowsTest.php create mode 100644 packages/orchestration/tests/Core/TopologyBuilderTest.php create mode 100644 packages/orchestration/tests/Support/MutableClock.php diff --git a/packages/orchestration/src/Core/Clock.php b/packages/orchestration/src/Core/Clock.php new file mode 100644 index 00000000..a6cf4c30 --- /dev/null +++ b/packages/orchestration/src/Core/Clock.php @@ -0,0 +1,16 @@ + 100 and results["check"]["ok"]`. + * + * lint() runs at boot over every declared condition, with the four names, so a typo or an unknown variable + * refuses the application instead of failing the first execution that reaches the step. + */ +final class ConditionEvaluator +{ + /** @var list */ + public const array VARIABLES = ['input', 'results', 'variables', 'headers']; + + private readonly ExpressionLanguage $language; + + public function __construct(?ExpressionLanguage $language = null) + { + $this->language = $language ?? new ExpressionLanguage; + } + + public function lint(string $expression, string $where): void + { + try { + $this->language->lint($expression, self::VARIABLES); + } catch (SyntaxError $error) { + throw new TopologyValidationException("{$where}: condition [{$expression}] does not parse: {$error->getMessage()}", $error); + } + } + + /** + * @param array $values + */ + public function evaluate(string $expression, array $values): bool + { + try { + return (bool) $this->language->evaluate($expression, $values); + } catch (Throwable $error) { + throw new OrchestrationException("Condition [{$expression}] could not be evaluated: {$error->getMessage()}", 'ORCHESTRATION_CONDITION_FAILED', 500, $error); + } + } +} diff --git a/packages/orchestration/src/Core/Exception/BackpressureException.php b/packages/orchestration/src/Core/Exception/BackpressureException.php new file mode 100644 index 00000000..98568a29 --- /dev/null +++ b/packages/orchestration/src/Core/Exception/BackpressureException.php @@ -0,0 +1,14 @@ +value} executions are disabled ({$key} is false).", 'ORCHESTRATION_DISABLED', 503); + } +} diff --git a/packages/orchestration/src/Core/Exception/SignalTimeoutException.php b/packages/orchestration/src/Core/Exception/SignalTimeoutException.php new file mode 100644 index 00000000..9231fa30 --- /dev/null +++ b/packages/orchestration/src/Core/Exception/SignalTimeoutException.php @@ -0,0 +1,8 @@ + */ + public static function terminal(): array + { + return [self::Completed, self::Failed, self::Cancelled, self::TimedOut, self::Confirmed, self::Canceled]; + } + + /** @return list */ + public static function recoverable(): array + { + return [self::Pending, self::Running, self::Compensating, self::Trying, self::Confirming, self::Canceling]; + } +} diff --git a/packages/orchestration/src/Core/Rows.php b/packages/orchestration/src/Core/Rows.php new file mode 100644 index 00000000..955dcdb1 --- /dev/null +++ b/packages/orchestration/src/Core/Rows.php @@ -0,0 +1,169 @@ + $row */ + public static function string(array $row, string $key, string $default = ''): string + { + $value = $row[$key] ?? null; + + return match (true) { + is_string($value) => $value, + is_int($value), is_float($value) => (string) $value, + default => $default, + }; + } + + /** @param array $row */ + public static function nullableString(array $row, string $key): ?string + { + $value = $row[$key] ?? null; + + return is_string($value) ? $value : null; + } + + /** @param array $row */ + public static function int(array $row, string $key, int $default = 0): int + { + $value = $row[$key] ?? null; + + return match (true) { + is_int($value) => $value, + is_numeric($value) => (int) $value, + default => $default, + }; + } + + /** @param array $row */ + public static function float(array $row, string $key, float $default = 0.0): float + { + $value = $row[$key] ?? null; + + return is_numeric($value) ? (float) $value : $default; + } + + /** @param array $row */ + public static function bool(array $row, string $key, bool $default = false): bool + { + $value = $row[$key] ?? null; + + return is_bool($value) ? $value : $default; + } + + /** + * @param array $row + * @return list + */ + public static function list(array $row, string $key): array + { + $value = $row[$key] ?? null; + + return is_array($value) ? array_values($value) : []; + } + + /** + * @param array $row + * @return list + */ + public static function strings(array $row, string $key): array + { + $strings = []; + foreach (self::list($row, $key) as $value) { + if (is_string($value) || is_int($value)) { + $strings[] = (string) $value; + } + } + + return $strings; + } + + /** + * @param array $row + * @return array + */ + public static function map(array $row, string $key): array + { + $value = $row[$key] ?? null; + if (! is_array($value)) { + return []; + } + + $map = []; + foreach ($value as $k => $v) { + $map[(string) $k] = $v; + } + + return $map; + } + + /** + * @param array $row + * @return array|null + */ + public static function nullableMap(array $row, string $key): ?array + { + return is_array($row[$key] ?? null) ? self::map($row, $key) : null; + } + + /** + * @param array $row + * @return array + */ + public static function stringMap(array $row, string $key): array + { + $map = []; + foreach (self::map($row, $key) as $k => $v) { + if (is_bool($v)) { + $map[$k] = $v ? 'true' : 'false'; + } elseif (is_string($v) || is_int($v) || is_float($v)) { + $map[$k] = (string) $v; + } + } + + return $map; + } + + /** + * @param array $row + * @return array + */ + public static function intMap(array $row, string $key): array + { + $map = []; + foreach (self::map($row, $key) as $k => $v) { + if (is_numeric($v)) { + $map[$k] = (int) $v; + } + } + + return $map; + } + + /** + * @param array $row + * @return list> + */ + public static function rows(array $row, string $key): array + { + $rows = []; + foreach (self::list($row, $key) as $value) { + if (is_array($value)) { + $rows[] = self::map(['row' => $value], 'row'); + } + } + + return $rows; + } +} diff --git a/packages/orchestration/src/Core/StepStatus.php b/packages/orchestration/src/Core/StepStatus.php new file mode 100644 index 00000000..de16146c --- /dev/null +++ b/packages/orchestration/src/Core/StepStatus.php @@ -0,0 +1,36 @@ +> $dependencies step id => the ids it depends on, in declaration order; + * a numeric id arrives as an int key + * @return list> + */ + public static function layers(string $owner, array $dependencies): array + { + self::validate($owner, $dependencies); + + $remaining = $dependencies; + $placed = []; + $layers = []; + while ($remaining !== []) { + $layer = []; + foreach ($remaining as $id => $dependsOn) { + if (array_diff($dependsOn, array_map('strval', array_keys($placed))) === []) { + $layer[] = (string) $id; + } + } + foreach ($layer as $id) { + $placed[$id] = true; + unset($remaining[$id]); + } + $layers[] = $layer; + } + + return $layers; + } + + /** + * @param array> $dependencies + */ + public static function validate(string $owner, array $dependencies): void + { + if ($dependencies === []) { + throw new TopologyValidationException("{$owner} declares no steps."); + } + + foreach ($dependencies as $id => $dependsOn) { + $id = (string) $id; + foreach ($dependsOn as $dependency) { + if ($dependency === $id) { + throw new TopologyValidationException("{$owner}: step [{$id}] depends on itself."); + } + if (! array_key_exists($dependency, $dependencies)) { + throw new TopologyValidationException("{$owner}: step [{$id}] depends on [{$dependency}], which is not a step of it."); + } + } + } + + $state = []; + foreach (array_keys($dependencies) as $id) { + self::visit($owner, (string) $id, $dependencies, $state, []); + } + } + + /** + * @param array> $dependencies + * @param array $state 1 = on the current path, 2 = finished + * @param list $path + */ + private static function visit(string $owner, string $id, array $dependencies, array &$state, array $path): void + { + $mark = $state[$id] ?? 0; + if ($mark === 2) { + return; + } + if ($mark === 1) { + $start = array_search($id, $path, true); + $cycle = [...array_slice($path, $start === false ? 0 : $start), $id]; + + throw new TopologyValidationException("{$owner}: the steps form a cycle: ".implode(' -> ', $cycle).'.'); + } + + $state[$id] = 1; + foreach ($dependencies[$id] ?? [] as $dependency) { + self::visit($owner, $dependency, $dependencies, $state, [...$path, $id]); + } + $state[$id] = 2; + } +} diff --git a/packages/orchestration/src/Core/TriggerMode.php b/packages/orchestration/src/Core/TriggerMode.php new file mode 100644 index 00000000..e705372f --- /dev/null +++ b/packages/orchestration/src/Core/TriggerMode.php @@ -0,0 +1,15 @@ +evaluate('input["amount"] > 100 and results["check"]["ok"]', [ + 'input' => ['amount' => 150], 'results' => ['check' => ['ok' => true]], 'variables' => [], 'headers' => [], + ]))->toBeTrue() + ->and($evaluator->evaluate('variables["tier"] == "gold"', ['input' => [], 'results' => [], 'variables' => ['tier' => 'silver'], 'headers' => []]))->toBeFalse(); +}); + +it('refuses at lint time an expression that does not parse or names an unknown variable', function (string $expression): void { + (new ConditionEvaluator)->lint($expression, 'Workflow [w] step [s]'); +})->with(['input[', 'order.total > 1'])->throws(TopologyValidationException::class, 'Workflow [w] step [s]'); + +it('wraps a runtime evaluation error', function (): void { + (new ConditionEvaluator)->evaluate('input["a"]["b"] > 1', ['input' => ['a' => 3], 'results' => [], 'variables' => [], 'headers' => []]); +})->throws(OrchestrationException::class, 'could not be evaluated'); diff --git a/packages/orchestration/tests/Core/EnumsTest.php b/packages/orchestration/tests/Core/EnumsTest.php new file mode 100644 index 00000000..36cff972 --- /dev/null +++ b/packages/orchestration/tests/Core/EnumsTest.php @@ -0,0 +1,45 @@ +toBe([ + ExecutionStatus::Completed, ExecutionStatus::Failed, ExecutionStatus::Cancelled, + ExecutionStatus::TimedOut, ExecutionStatus::Confirmed, ExecutionStatus::Canceled, + ]) + ->and(ExecutionStatus::Waiting->isTerminal())->toBeFalse() + ->and(ExecutionStatus::Waiting->isRecoverable())->toBeFalse() + ->and(ExecutionStatus::Confirming->isRecoverable())->toBeTrue() + ->and(ExecutionStatus::Confirmed->isSuccess())->toBeTrue() + ->and(ExecutionStatus::Waiting->canSuspend())->toBeTrue() + ->and(ExecutionStatus::Completed->canSuspend())->toBeFalse() + ->and(ExecutionStatus::Suspended->canResume())->toBeTrue(); +}); + +it('knows which step statuses are settled and which are failures', function (): void { + expect(StepStatus::Done->isSettled())->toBeTrue() + ->and(StepStatus::Skipped->isSettled())->toBeTrue() + ->and(StepStatus::Running->isSettled())->toBeFalse() + ->and(StepStatus::TimedOut->isFailure())->toBeTrue() + ->and(StepStatus::Compensated->isFailure())->toBeFalse(); +}); + +it('reads compensation policies in kebab, snake and screaming spellings, with the jafly alias', function (string $value, CompensationPolicy $policy): void { + expect(CompensationPolicy::fromConfig($value, 'firefly.orchestration.saga.compensation-policy'))->toBe($policy); +})->with([ + ['strict-sequential', CompensationPolicy::StrictSequential], + ['GROUPED_PARALLEL', CompensationPolicy::GroupedParallel], + ['retry_with_backoff', CompensationPolicy::RetryWithBackoff], + ['circuit-breaker', CompensationPolicy::CircuitBreaker], + ['best-effort', CompensationPolicy::BestEffort], + ['BEST_EFFORT_PARALLEL', CompensationPolicy::BestEffort], +]); + +it('refuses an unknown compensation policy, naming the key', function (): void { + CompensationPolicy::fromConfig('optimistic', 'firefly.orchestration.saga.compensation-policy'); +})->throws(ConfigurationException::class, 'firefly.orchestration.saga.compensation-policy'); diff --git a/packages/orchestration/tests/Core/RowsTest.php b/packages/orchestration/tests/Core/RowsTest.php new file mode 100644 index 00000000..0cc25ac8 --- /dev/null +++ b/packages/orchestration/tests/Core/RowsTest.php @@ -0,0 +1,23 @@ + 'x', 'i' => '7', 'f' => 2, 'b' => true, 'l' => ['a', 1, null], 'm' => [1 => 'one'], 'r' => [['k' => 'v'], 'skip'], 'bad' => new stdClass]; + + expect(Rows::string($row, 's'))->toBe('x') + ->and(Rows::string($row, 'bad', 'd'))->toBe('d') + ->and(Rows::nullableString($row, 'missing'))->toBeNull() + ->and(Rows::int($row, 'i'))->toBe(7) + ->and(Rows::float($row, 'f'))->toBe(2.0) + ->and(Rows::bool($row, 'b'))->toBeTrue() + ->and(Rows::bool($row, 's'))->toBeFalse() + ->and(Rows::strings($row, 'l'))->toBe(['a', '1']) + ->and(Rows::map($row, 'm'))->toBe(['1' => 'one']) + ->and(Rows::stringMap(['m' => ['t' => true, 'n' => 3]], 'm'))->toBe(['t' => 'true', 'n' => '3']) + ->and(Rows::intMap(['m' => ['a' => '2', 'b' => 'x']], 'm'))->toBe(['a' => 2]) + ->and(Rows::rows($row, 'r'))->toBe([['k' => 'v']]) + ->and(Rows::nullableMap($row, 's'))->toBeNull(); +}); diff --git a/packages/orchestration/tests/Core/TopologyBuilderTest.php b/packages/orchestration/tests/Core/TopologyBuilderTest.php new file mode 100644 index 00000000..19566787 --- /dev/null +++ b/packages/orchestration/tests/Core/TopologyBuilderTest.php @@ -0,0 +1,35 @@ + [], + 'charge' => ['reserve'], + 'notify' => ['reserve'], + 'ship' => ['charge', 'notify'], + 'audit' => [], + ]); + + expect($layers)->toBe([['reserve', 'audit'], ['charge', 'notify'], ['ship']]); +}); + +it('keeps numeric step ids as strings', function (): void { + expect(TopologyBuilder::layers('Saga [n]', ['1' => [], '2' => ['1']]))->toBe([['1'], ['2']]); +}); + +it('refuses an empty definition, an unknown dependency and a self dependency', function (): void { + expect(fn () => TopologyBuilder::layers('Saga [s]', [])) + ->toThrow(TopologyValidationException::class, 'Saga [s] declares no steps.') + ->and(fn () => TopologyBuilder::layers('Saga [s]', ['a' => ['ghost']])) + ->toThrow(TopologyValidationException::class, 'step [a] depends on [ghost], which is not a step of it') + ->and(fn () => TopologyBuilder::layers('Saga [s]', ['a' => ['a']])) + ->toThrow(TopologyValidationException::class, 'step [a] depends on itself'); +}); + +it('refuses a cycle and names its path', function (): void { + TopologyBuilder::layers('Workflow [w]', ['a' => ['c'], 'b' => ['a'], 'c' => ['b']]); +})->throws(TopologyValidationException::class, 'Workflow [w]: the steps form a cycle: a -> c -> b -> a.'); diff --git a/packages/orchestration/tests/Support/MutableClock.php b/packages/orchestration/tests/Support/MutableClock.php new file mode 100644 index 00000000..a8119e82 --- /dev/null +++ b/packages/orchestration/tests/Support/MutableClock.php @@ -0,0 +1,35 @@ +set($at); + } + + public function now(): DateTimeImmutable + { + return $this->now; + } + + public function set(string $at): void + { + $this->now = (new DateTimeImmutable($at))->setTimezone(new DateTimeZone('UTC')); + } + + public function advance(float $seconds): void + { + $this->now = $this->now->modify(sprintf('%+d microseconds', (int) round($seconds * 1_000_000))); + } +} From aa4538839bda99a1a6ec91d1e10e54ee57091b4d Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 18:58:56 -0700 Subject: [PATCH 03/58] feat(orchestration): the execution context and its JSON-safe checkpoint state --- .../src/Core/ExecutionContext.php | 560 ++++++++++++++++++ .../src/Persistence/ExecutionCriteria.php | 26 + .../src/Persistence/ExecutionState.php | 230 +++++++ .../src/Persistence/StateSerializer.php | 99 ++++ .../tests/Core/ExecutionContextTest.php | 80 +++ .../Persistence/ExecutionCriteriaTest.php | 18 + .../tests/Persistence/ExecutionStateTest.php | 80 +++ .../tests/Persistence/StateSerializerTest.php | 61 ++ 8 files changed, 1154 insertions(+) create mode 100644 packages/orchestration/src/Core/ExecutionContext.php create mode 100644 packages/orchestration/src/Persistence/ExecutionCriteria.php create mode 100644 packages/orchestration/src/Persistence/ExecutionState.php create mode 100644 packages/orchestration/src/Persistence/StateSerializer.php create mode 100644 packages/orchestration/tests/Core/ExecutionContextTest.php create mode 100644 packages/orchestration/tests/Persistence/ExecutionCriteriaTest.php create mode 100644 packages/orchestration/tests/Persistence/ExecutionStateTest.php create mode 100644 packages/orchestration/tests/Persistence/StateSerializerTest.php diff --git a/packages/orchestration/src/Core/ExecutionContext.php b/packages/orchestration/src/Core/ExecutionContext.php new file mode 100644 index 00000000..2b221ee0 --- /dev/null +++ b/packages/orchestration/src/Core/ExecutionContext.php @@ -0,0 +1,560 @@ + */ + private array $variables = []; + + /** @var array */ + private array $stepResults = []; + + /** @var array */ + private array $stepStatuses = []; + + /** @var array */ + private array $stepAttempts = []; + + /** @var array */ + private array $stepLatenciesMs = []; + + /** @var array */ + private array $stepStartedAt = []; + + /** @var array */ + private array $stepInputs = []; + + /** @var list */ + private array $completionOrder = []; + + /** @var array */ + private array $compensationResults = []; + + /** @var array */ + private array $compensationErrors = []; + + /** @var array */ + private array $stepErrors = []; + + /** @var array */ + private array $tryResults = []; + + private ?TccPhase $currentPhase = null; + + /** @var array */ + private array $idempotencyKeys = []; + + /** @var list> */ + private array $topologyLayers = []; + + /** @var array */ + private array $searchAttributes = []; + + /** @var array */ + private array $signals = []; + + /** @var array */ + private array $timers = []; + + private ?string $waitingOn = null; + + private ?string $parentCorrelationId = null; + + /** @var array */ + private array $children = []; + + private ?DateTimeImmutable $deadline = null; + + /** + * @param array $input + * @param array $headers + */ + private function __construct( + public readonly string $correlationId, + public readonly string $executionName, + public readonly ExecutionPattern $pattern, + private readonly DateTimeImmutable $startedAt, + private readonly array $input, + private array $headers, + ) {} + + /** + * @param array $input + * @param array $headers + */ + public static function start(ExecutionPattern $pattern, string $name, array $input, array $headers, ?string $correlationId, DateTimeImmutable $now): self + { + return new self($correlationId ?? Str::uuid()->toString(), $name, $pattern, $now, $input, $headers); + } + + public static function fromState(ExecutionState $state): self + { + $context = new self($state->correlationId, $state->executionName, $state->pattern, $state->startedAt, $state->input, $state->headers); + $context->variables = $state->variables; + $context->stepResults = $state->stepResults; + $context->stepStatuses = $state->stepStatuses; + $context->stepAttempts = $state->stepAttempts; + $context->stepLatenciesMs = $state->stepLatenciesMs; + $context->stepInputs = $state->stepInputs; + $context->completionOrder = $state->completionOrder; + $context->compensationResults = $state->compensationResults; + $context->compensationErrors = array_map(static fn (string $message): Throwable => new RuntimeException($message), $state->compensationErrors); + $context->stepErrors = array_map(static fn (string $message): Throwable => new RuntimeException($message), $state->stepErrors); + $context->tryResults = $state->tryResults; + $context->currentPhase = $state->currentPhase; + $context->idempotencyKeys = array_fill_keys($state->idempotencyKeys, true); + $context->topologyLayers = $state->topologyLayers; + $context->searchAttributes = $state->searchAttributes; + $context->signals = $state->signals; + $context->timers = array_map(static fn (string $at): DateTimeImmutable => ExecutionState::parseInstant($at), $state->timers); + $context->waitingOn = $state->waitingOn; + $context->parentCorrelationId = $state->parentCorrelationId; + $context->children = $state->children; + + return $context; + } + + /** + * @param array|null $report + */ + public function toState(ExecutionStatus $status, DateTimeImmutable $now, ?string $failureReason = null, ?array $report = null): ExecutionState + { + /** @var array $input */ + $input = StateSerializer::normalize($this->input, 'input'); + /** @var array $variables */ + $variables = StateSerializer::normalize($this->variables, 'variables'); + /** @var array $results */ + $results = StateSerializer::normalize($this->stepResults, 'stepResults'); + /** @var array $stepInputs */ + $stepInputs = StateSerializer::normalize($this->stepInputs, 'stepInputs'); + /** @var array $compensationResults */ + $compensationResults = StateSerializer::normalize($this->compensationResults, 'compensationResults'); + /** @var array $tryResults */ + $tryResults = StateSerializer::normalize($this->tryResults, 'tryResults'); + /** @var array $signals */ + $signals = StateSerializer::normalize($this->signals, 'signals'); + /** @var array|null $normalizedReport */ + $normalizedReport = $report === null ? null : StateSerializer::normalize($report, 'report'); + + return new ExecutionState( + correlationId: $this->correlationId, + executionName: $this->executionName, + pattern: $this->pattern, + status: $status, + startedAt: $this->startedAt, + updatedAt: $now, + input: $input, + headers: $this->headers, + variables: $variables, + stepResults: $results, + stepStatuses: $this->stepStatuses, + stepAttempts: $this->stepAttempts, + stepLatenciesMs: $this->stepLatenciesMs, + stepInputs: $stepInputs, + completionOrder: $this->completionOrder, + compensationResults: $compensationResults, + compensationErrors: array_map(static fn (Throwable $error): string => $error->getMessage(), $this->compensationErrors), + stepErrors: array_map(static fn (Throwable $error): string => $error->getMessage(), $this->stepErrors), + tryResults: $tryResults, + currentPhase: $this->currentPhase, + idempotencyKeys: array_map('strval', array_keys($this->idempotencyKeys)), + topologyLayers: $this->topologyLayers, + failureReason: $failureReason, + completedAt: $status->isTerminal() ? $now : null, + wakeAt: $status === ExecutionStatus::Waiting ? $this->wakeAt($now) : null, + searchAttributes: $this->searchAttributes, + signals: $signals, + timers: array_map(static fn (DateTimeImmutable $at): string => ExecutionState::instant($at), $this->timers), + waitingOn: $this->waitingOn, + parentCorrelationId: $this->parentCorrelationId, + children: $this->children, + report: $normalizedReport, + ); + } + + public function startedAt(): DateTimeImmutable + { + return $this->startedAt; + } + + /** @return array */ + public function input(): array + { + return $this->input; + } + + public function stepInput(string $stepId): mixed + { + return $this->stepInputs[$stepId] ?? null; + } + + public function putStepInput(string $stepId, mixed $input): void + { + $this->stepInputs[$stepId] = $input; + } + + /** @return array */ + public function headers(): array + { + return $this->headers; + } + + public function header(string $name): ?string + { + return $this->headers[$name] ?? null; + } + + public function setHeader(string $name, string $value): void + { + $this->headers[$name] = $value; + } + + /** @return array */ + public function variables(): array + { + return $this->variables; + } + + public function variable(string $name): mixed + { + return $this->variables[$name] ?? null; + } + + public function hasVariable(string $name): bool + { + return array_key_exists($name, $this->variables); + } + + public function setVariable(string $name, mixed $value): void + { + $this->variables[$name] = $value; + } + + /** @return array */ + public function results(): array + { + return $this->stepResults; + } + + public function result(string $stepId): mixed + { + return $this->stepResults[$stepId] ?? null; + } + + public function hasResult(string $stepId): bool + { + return array_key_exists($stepId, $this->stepResults); + } + + public function putResult(string $stepId, mixed $result): void + { + $this->stepResults[$stepId] = $result; + } + + public function stepStatus(string $stepId): StepStatus + { + return $this->stepStatuses[$stepId] ?? StepStatus::Pending; + } + + public function setStepStatus(string $stepId, StepStatus $status): void + { + $this->stepStatuses[$stepId] = $status; + } + + /** @return array */ + public function stepStatuses(): array + { + return $this->stepStatuses; + } + + public function attempts(string $stepId): int + { + return $this->stepAttempts[$stepId] ?? 0; + } + + public function recordAttempt(string $stepId): int + { + return $this->stepAttempts[$stepId] = $this->attempts($stepId) + 1; + } + + public function latencyMs(string $stepId): int + { + return $this->stepLatenciesMs[$stepId] ?? 0; + } + + public function setLatencyMs(string $stepId, int $milliseconds): void + { + $this->stepLatenciesMs[$stepId] = $milliseconds; + } + + public function markStepStarted(string $stepId, DateTimeImmutable $at): void + { + $this->stepStartedAt[$stepId] = $at; + } + + public function stepStartedAt(string $stepId): ?DateTimeImmutable + { + return $this->stepStartedAt[$stepId] ?? null; + } + + /** @return list */ + public function completionOrder(): array + { + return $this->completionOrder; + } + + public function markCompleted(string $stepId): void + { + if (! in_array($stepId, $this->completionOrder, true)) { + $this->completionOrder[] = $stepId; + } + } + + public function compensationResult(string $stepId): mixed + { + return $this->compensationResults[$stepId] ?? null; + } + + /** @return array */ + public function compensationResults(): array + { + return $this->compensationResults; + } + + public function putCompensationResult(string $stepId, mixed $result): void + { + $this->compensationResults[$stepId] = $result; + } + + public function compensationError(string $stepId): ?Throwable + { + return $this->compensationErrors[$stepId] ?? null; + } + + /** @return array */ + public function compensationErrors(): array + { + return $this->compensationErrors; + } + + public function putCompensationError(string $stepId, Throwable $error): void + { + $this->compensationErrors[$stepId] = $error; + } + + public function stepError(string $stepId): ?Throwable + { + return $this->stepErrors[$stepId] ?? null; + } + + /** @return array */ + public function stepErrors(): array + { + return $this->stepErrors; + } + + public function putStepError(string $stepId, Throwable $error): void + { + $this->stepErrors[$stepId] = $error; + } + + public function clearStepError(string $stepId): void + { + unset($this->stepErrors[$stepId]); + } + + public function tryResult(string $participantId): mixed + { + return $this->tryResults[$participantId] ?? null; + } + + /** @return array */ + public function tryResults(): array + { + return $this->tryResults; + } + + public function putTryResult(string $participantId, mixed $result): void + { + $this->tryResults[$participantId] = $result; + } + + public function currentPhase(): ?TccPhase + { + return $this->currentPhase; + } + + public function setCurrentPhase(?TccPhase $phase): void + { + $this->currentPhase = $phase; + } + + public function hasIdempotencyKey(string $key): bool + { + return isset($this->idempotencyKeys[$key]); + } + + public function addIdempotencyKey(string $key): void + { + $this->idempotencyKeys[$key] = true; + } + + /** @return list> */ + public function topologyLayers(): array + { + return $this->topologyLayers; + } + + /** @param list> $layers */ + public function setTopologyLayers(array $layers): void + { + $this->topologyLayers = $layers; + } + + /** @return array */ + public function searchAttributes(): array + { + return $this->searchAttributes; + } + + public function setSearchAttribute(string $key, string|int|float|bool $value): void + { + $this->searchAttributes[$key] = $value; + } + + /** @return array */ + public function signals(): array + { + return $this->signals; + } + + public function hasSignal(string $name): bool + { + return array_key_exists($name, $this->signals); + } + + public function receiveSignal(string $name, mixed $payload): void + { + $this->signals[$name] = $payload; + } + + public function consumeSignal(string $name): mixed + { + $payload = $this->signals[$name] ?? null; + unset($this->signals[$name]); + + return $payload; + } + + /** @return array */ + public function timers(): array + { + return $this->timers; + } + + public function hasTimer(string $timerId): bool + { + return isset($this->timers[$timerId]); + } + + public function scheduleTimer(string $timerId, DateTimeImmutable $fireAt): void + { + $this->timers[$timerId] = $fireAt; + } + + public function timerFired(string $timerId, DateTimeImmutable $now): bool + { + return isset($this->timers[$timerId]) && $this->timers[$timerId] <= $now; + } + + public function removeTimer(string $timerId): void + { + unset($this->timers[$timerId]); + } + + public function waitingOn(): ?string + { + return $this->waitingOn; + } + + public function setWaitingOn(?string $stepId): void + { + $this->waitingOn = $stepId; + } + + public function parentCorrelationId(): ?string + { + return $this->parentCorrelationId; + } + + public function setParentCorrelationId(?string $correlationId): void + { + $this->parentCorrelationId = $correlationId; + } + + /** @return array */ + public function children(): array + { + return $this->children; + } + + public function child(string $stepId): ?string + { + return $this->children[$stepId] ?? null; + } + + public function setChild(string $stepId, string $childCorrelationId): void + { + $this->children[$stepId] = $childCorrelationId; + } + + public function deadline(): ?DateTimeImmutable + { + return $this->deadline; + } + + public function setDeadline(?DateTimeImmutable $deadline): void + { + $this->deadline = $deadline; + } + + /** + * The earliest PENDING instant: a timer already due is being handled by the run that checkpoints now (it + * fired, and the step is still waiting on something else — a signal of a WaitForAll), so counting it + * would make the tick wake this execution every minute for nothing. + */ + private function wakeAt(DateTimeImmutable $now): ?DateTimeImmutable + { + $candidates = array_values(array_filter($this->timers, static fn (DateTimeImmutable $at): bool => $at > $now)); + if ($this->deadline !== null) { + $candidates[] = $this->deadline; + } + + return $candidates === [] ? null : min($candidates); + } +} diff --git a/packages/orchestration/src/Persistence/ExecutionCriteria.php b/packages/orchestration/src/Persistence/ExecutionCriteria.php new file mode 100644 index 00000000..d95187c4 --- /dev/null +++ b/packages/orchestration/src/Persistence/ExecutionCriteria.php @@ -0,0 +1,26 @@ +pattern === null || $state->pattern === $this->pattern) + && ($this->status === null || $state->status === $this->status) + && ($this->name === null || $state->executionName === $this->name); + } +} diff --git a/packages/orchestration/src/Persistence/ExecutionState.php b/packages/orchestration/src/Persistence/ExecutionState.php new file mode 100644 index 00000000..906dffa0 --- /dev/null +++ b/packages/orchestration/src/Persistence/ExecutionState.php @@ -0,0 +1,230 @@ + $input + * @param array $headers + * @param array $variables + * @param array $stepResults + * @param array $stepStatuses + * @param array $stepAttempts + * @param array $stepLatenciesMs + * @param array $stepInputs + * @param list $completionOrder + * @param array $compensationResults + * @param array $compensationErrors + * @param array $stepErrors + * @param array $tryResults + * @param list $idempotencyKeys + * @param list> $topologyLayers + * @param array $searchAttributes + * @param array $signals + * @param array $timers timer id => fire-at instant (UTC, see instant()) + * @param array $children step id => child correlation id + * @param array|null $report + */ + public function __construct( + public string $correlationId, + public string $executionName, + public ExecutionPattern $pattern, + public ExecutionStatus $status, + public DateTimeImmutable $startedAt, + public DateTimeImmutable $updatedAt, + public array $input = [], + public array $headers = [], + public array $variables = [], + public array $stepResults = [], + public array $stepStatuses = [], + public array $stepAttempts = [], + public array $stepLatenciesMs = [], + public array $stepInputs = [], + public array $completionOrder = [], + public array $compensationResults = [], + public array $compensationErrors = [], + public array $stepErrors = [], + public array $tryResults = [], + public ?TccPhase $currentPhase = null, + public array $idempotencyKeys = [], + public array $topologyLayers = [], + public ?string $failureReason = null, + public ?DateTimeImmutable $completedAt = null, + public ?DateTimeImmutable $wakeAt = null, + public array $searchAttributes = [], + public array $signals = [], + public array $timers = [], + public ?string $waitingOn = null, + public ?string $parentCorrelationId = null, + public array $children = [], + public ?array $report = null, + ) {} + + public function isTerminal(): bool + { + return $this->status->isTerminal(); + } + + public function stepStatus(string $stepId): StepStatus + { + return $this->stepStatuses[$stepId] ?? StepStatus::Pending; + } + + public function withStatus(ExecutionStatus $status, DateTimeImmutable $now): self + { + $data = $this->toArray(); + $data['status'] = $status->value; + $data['updatedAt'] = self::instant($now); + $data['completedAt'] = $status->isTerminal() ? self::instant($now) : null; + if ($status !== ExecutionStatus::Waiting) { + $data['wakeAt'] = null; + } + + return self::fromArray($data); + } + + /** + * @return array + */ + public function toArray(): array + { + return [ + 'correlationId' => $this->correlationId, + 'executionName' => $this->executionName, + 'pattern' => $this->pattern->value, + 'status' => $this->status->value, + 'startedAt' => self::instant($this->startedAt), + 'updatedAt' => self::instant($this->updatedAt), + 'completedAt' => $this->completedAt === null ? null : self::instant($this->completedAt), + 'wakeAt' => $this->wakeAt === null ? null : self::instant($this->wakeAt), + 'input' => $this->input, + 'headers' => $this->headers, + 'variables' => $this->variables, + 'stepResults' => $this->stepResults, + 'stepStatuses' => array_map(static fn (StepStatus $status): string => $status->value, $this->stepStatuses), + 'stepAttempts' => $this->stepAttempts, + 'stepLatenciesMs' => $this->stepLatenciesMs, + 'stepInputs' => $this->stepInputs, + 'completionOrder' => $this->completionOrder, + 'compensationResults' => $this->compensationResults, + 'compensationErrors' => $this->compensationErrors, + 'stepErrors' => $this->stepErrors, + 'tryResults' => $this->tryResults, + 'currentPhase' => $this->currentPhase?->value, + 'idempotencyKeys' => $this->idempotencyKeys, + 'topologyLayers' => $this->topologyLayers, + 'failureReason' => $this->failureReason, + 'searchAttributes' => $this->searchAttributes, + 'signals' => $this->signals, + 'timers' => $this->timers, + 'waitingOn' => $this->waitingOn, + 'parentCorrelationId' => $this->parentCorrelationId, + 'children' => $this->children, + 'report' => $this->report, + ]; + } + + /** + * @param array $data + */ + public static function fromArray(array $data): self + { + $statuses = []; + foreach (Rows::stringMap($data, 'stepStatuses') as $step => $status) { + $statuses[$step] = StepStatus::tryFrom($status) ?? StepStatus::Pending; + } + + $layers = []; + foreach (Rows::list($data, 'topologyLayers') as $layer) { + if (is_array($layer)) { + $layers[] = Rows::strings(['layer' => $layer], 'layer'); + } + } + + $attributes = []; + foreach (Rows::map($data, 'searchAttributes') as $key => $value) { + if (is_string($value) || is_int($value) || is_float($value) || is_bool($value)) { + $attributes[$key] = $value; + } + } + + $completedAt = Rows::nullableString($data, 'completedAt'); + $wakeAt = Rows::nullableString($data, 'wakeAt'); + $phase = Rows::nullableString($data, 'currentPhase'); + + return new self( + correlationId: Rows::string($data, 'correlationId'), + executionName: Rows::string($data, 'executionName'), + pattern: ExecutionPattern::tryFrom(Rows::string($data, 'pattern')) ?? ExecutionPattern::Saga, + status: ExecutionStatus::tryFrom(Rows::string($data, 'status')) ?? ExecutionStatus::Pending, + startedAt: self::parseInstant(Rows::string($data, 'startedAt', 'now')), + updatedAt: self::parseInstant(Rows::string($data, 'updatedAt', 'now')), + input: Rows::map($data, 'input'), + headers: Rows::stringMap($data, 'headers'), + variables: Rows::map($data, 'variables'), + stepResults: Rows::map($data, 'stepResults'), + stepStatuses: $statuses, + stepAttempts: Rows::intMap($data, 'stepAttempts'), + stepLatenciesMs: Rows::intMap($data, 'stepLatenciesMs'), + stepInputs: Rows::map($data, 'stepInputs'), + completionOrder: Rows::strings($data, 'completionOrder'), + compensationResults: Rows::map($data, 'compensationResults'), + compensationErrors: Rows::stringMap($data, 'compensationErrors'), + stepErrors: Rows::stringMap($data, 'stepErrors'), + tryResults: Rows::map($data, 'tryResults'), + currentPhase: $phase === null ? null : TccPhase::tryFrom($phase), + idempotencyKeys: Rows::strings($data, 'idempotencyKeys'), + topologyLayers: $layers, + failureReason: Rows::nullableString($data, 'failureReason'), + completedAt: $completedAt === null ? null : self::parseInstant($completedAt), + wakeAt: $wakeAt === null ? null : self::parseInstant($wakeAt), + searchAttributes: $attributes, + signals: Rows::map($data, 'signals'), + timers: Rows::stringMap($data, 'timers'), + waitingOn: Rows::nullableString($data, 'waitingOn'), + parentCorrelationId: Rows::nullableString($data, 'parentCorrelationId'), + children: Rows::stringMap($data, 'children'), + report: Rows::nullableMap($data, 'report'), + ); + } + + public static function instant(DateTimeImmutable $at): string + { + return $at->setTimezone(new DateTimeZone('UTC'))->format('Y-m-d\TH:i:s.uP'); + } + + public static function parseInstant(string $value): DateTimeImmutable + { + return (new DateTimeImmutable($value))->setTimezone(new DateTimeZone('UTC')); + } + + /** + * The one string form a search attribute is compared in, by every provider: a database column holds a + * string, so the in-memory and cache providers compare the same string rather than PHP's loose ==. + */ + public static function attributeValue(string|int|float|bool $value): string + { + return is_bool($value) ? ($value ? 'true' : 'false') : (string) $value; + } +} diff --git a/packages/orchestration/src/Persistence/StateSerializer.php b/packages/orchestration/src/Persistence/StateSerializer.php new file mode 100644 index 00000000..c63b5f9a --- /dev/null +++ b/packages/orchestration/src/Persistence/StateSerializer.php @@ -0,0 +1,99 @@ + $item) { + $normalized[$key] = self::normalize($item, $path.'.'.$key); + } + + return $normalized; + } + + return match (true) { + $value instanceof JsonSerializable => self::normalize($value->jsonSerialize(), $path), + $value instanceof BackedEnum => $value->value, + $value instanceof UnitEnum => $value->name, + $value instanceof DateTimeInterface => DateTimeImmutable::createFromInterface($value)->setTimezone(new DateTimeZone('UTC'))->format('Y-m-d\TH:i:s.uP'), + $value instanceof Traversable => self::normalize(iterator_to_array($value), $path), + $value instanceof Closure => throw new OrchestrationException("{$path} holds a closure, which an orchestration checkpoint cannot persist. Return data (an array, a scalar, a JsonSerializable) from the step.", 'ORCHESTRATION_UNSERIALIZABLE'), + is_object($value) => self::normalize(get_object_vars($value), $path), + default => throw new OrchestrationException("{$path} holds a ".get_debug_type($value).', which an orchestration checkpoint cannot persist.', 'ORCHESTRATION_UNSERIALIZABLE'), + }; + } + + public static function encode(ExecutionState $state): string + { + return self::encodeArray($state->toArray()); + } + + public static function decode(string $payload): ExecutionState + { + return ExecutionState::fromArray(self::decodeArray($payload)); + } + + /** + * @param array $data + */ + public static function encodeArray(array $data): string + { + return json_encode($data, self::FLAGS); + } + + /** + * @return array + */ + public static function decodeArray(string $payload): array + { + try { + $data = json_decode($payload, true, 512, JSON_THROW_ON_ERROR); + } catch (JsonException $error) { + throw new OrchestrationException("A persisted orchestration payload is not valid JSON: {$error->getMessage()}", 'ORCHESTRATION_CORRUPT_STATE', 500, $error); + } + + if (! is_array($data)) { + throw new OrchestrationException('A persisted orchestration payload is not a JSON object.', 'ORCHESTRATION_CORRUPT_STATE'); + } + + $map = []; + foreach ($data as $key => $value) { + $map[(string) $key] = $value; + } + + return $map; + } +} diff --git a/packages/orchestration/tests/Core/ExecutionContextTest.php b/packages/orchestration/tests/Core/ExecutionContextTest.php new file mode 100644 index 00000000..7083afe4 --- /dev/null +++ b/packages/orchestration/tests/Core/ExecutionContextTest.php @@ -0,0 +1,80 @@ +correlationId)->toMatch('/^[0-9a-f-]{36}$/') + ->and(ExecutionContext::start(ExecutionPattern::Saga, 's', [], [], 'given', $now)->correlationId)->toBe('given'); +}); + +it('survives a checkpoint: everything an engine needs to resume comes back from the state', function (): void { + $now = new DateTimeImmutable('2026-10-12T10:00:00+00:00'); + $ctx = ExecutionContext::start(ExecutionPattern::Workflow, 'approval', ['orderId' => 9], ['tenant' => 'acme'], 'c-9', $now); + $ctx->putStepInput('reserve', ['orderId' => 9]); + $ctx->setStepStatus('reserve', StepStatus::Done); + $ctx->recordAttempt('reserve'); + $ctx->recordAttempt('reserve'); + $ctx->setLatencyMs('reserve', 12); + $ctx->putResult('reserve', new ArrayObject(['hold' => 'h-1'])); + $ctx->markCompleted('reserve'); + $ctx->markCompleted('reserve'); + $ctx->putStepError('charge', new RuntimeException('card declined')); + $ctx->putCompensationError('reserve', new RuntimeException('undo failed')); + $ctx->putTryResult('debit', 'hold'); + $ctx->setCurrentPhase(TccPhase::Try); + $ctx->addIdempotencyKey('order-9'); + $ctx->setTopologyLayers([['reserve'], ['charge']]); + $ctx->setSearchAttribute('customer', 'ada'); + $ctx->receiveSignal('approval', ['by' => 'grace']); + $ctx->scheduleTimer('t-1', $now->modify('+10 minutes')); + $ctx->scheduleTimer('t-2', $now->modify('+5 minutes')); + $ctx->setWaitingOn('approve'); + $ctx->setParentCorrelationId('p-1'); + $ctx->setChild('spawn', 'child-1'); + $ctx->setVariable('tier', 'gold'); + + $state = $ctx->toState(ExecutionStatus::Waiting, $now->modify('+1 second')); + $back = ExecutionContext::fromState($state); + + expect($state->stepResults)->toBe(['reserve' => ['hold' => 'h-1']]) + ->and($state->completionOrder)->toBe(['reserve']) + ->and($state->stepErrors)->toBe(['charge' => 'card declined']) + ->and(ExecutionState::instant($state->wakeAt ?? $now))->toBe('2026-10-12T10:05:00.000000+00:00') + ->and($back->correlationId)->toBe('c-9') + ->and($back->input())->toBe(['orderId' => 9]) + ->and($back->header('tenant'))->toBe('acme') + ->and($back->attempts('reserve'))->toBe(2) + ->and($back->stepStatus('reserve'))->toBe(StepStatus::Done) + ->and($back->stepError('charge')?->getMessage())->toBe('card declined') + ->and($back->compensationError('reserve')?->getMessage())->toBe('undo failed') + ->and($back->tryResult('debit'))->toBe('hold') + ->and($back->currentPhase())->toBe(TccPhase::Try) + ->and($back->hasIdempotencyKey('order-9'))->toBeTrue() + ->and($back->consumeSignal('approval'))->toBe(['by' => 'grace']) + ->and($back->hasSignal('approval'))->toBeFalse() + ->and($back->timerFired('t-2', $now->modify('+5 minutes')))->toBeTrue() + ->and($back->timerFired('t-1', $now->modify('+5 minutes')))->toBeFalse() + ->and($back->waitingOn())->toBe('approve') + ->and($back->parentCorrelationId())->toBe('p-1') + ->and($back->child('spawn'))->toBe('child-1') + ->and($back->variable('tier'))->toBe('gold'); +}); + +it('includes the workflow deadline in the wake-up instant while waiting, and clears it once running', function (): void { + $now = new DateTimeImmutable('2026-10-12T10:00:00+00:00'); + $ctx = ExecutionContext::start(ExecutionPattern::Workflow, 'w', [], [], 'c', $now); + $ctx->scheduleTimer('t', $now->modify('+1 hour')); + $ctx->setDeadline($now->modify('+30 minutes')); + + expect(ExecutionState::instant($ctx->toState(ExecutionStatus::Waiting, $now)->wakeAt ?? $now))->toBe('2026-10-12T10:30:00.000000+00:00') + ->and($ctx->toState(ExecutionStatus::Running, $now)->wakeAt)->toBeNull(); +}); diff --git a/packages/orchestration/tests/Persistence/ExecutionCriteriaTest.php b/packages/orchestration/tests/Persistence/ExecutionCriteriaTest.php new file mode 100644 index 00000000..3412b925 --- /dev/null +++ b/packages/orchestration/tests/Persistence/ExecutionCriteriaTest.php @@ -0,0 +1,18 @@ +matches($state))->toBeTrue() + ->and((new ExecutionCriteria(pattern: ExecutionPattern::Saga, status: ExecutionStatus::Failed, name: 'order'))->matches($state))->toBeTrue() + ->and((new ExecutionCriteria(pattern: ExecutionPattern::Tcc))->matches($state))->toBeFalse() + ->and((new ExecutionCriteria(name: 'other'))->matches($state))->toBeFalse(); +}); diff --git a/packages/orchestration/tests/Persistence/ExecutionStateTest.php b/packages/orchestration/tests/Persistence/ExecutionStateTest.php new file mode 100644 index 00000000..59cd197d --- /dev/null +++ b/packages/orchestration/tests/Persistence/ExecutionStateTest.php @@ -0,0 +1,80 @@ +modify('+1 second'), + input: ['orderId' => 7], + headers: ['tenant' => 'acme'], + variables: ['tier' => 'gold'], + stepResults: ['reserve' => ['ok' => true]], + stepStatuses: ['reserve' => StepStatus::Done, 'approve' => StepStatus::Waiting], + stepAttempts: ['reserve' => 2], + stepLatenciesMs: ['reserve' => 15], + stepInputs: ['reserve' => ['orderId' => 7]], + completionOrder: ['reserve'], + compensationResults: ['x' => 1], + compensationErrors: ['y' => 'boom'], + stepErrors: ['z' => 'bad'], + tryResults: ['debit' => 'hold-1'], + currentPhase: TccPhase::Confirm, + idempotencyKeys: ['k-1'], + topologyLayers: [['reserve'], ['approve']], + failureReason: null, + completedAt: null, + wakeAt: $at->modify('+1 hour'), + searchAttributes: ['customer' => 'ada', 'vip' => true], + signals: ['approval' => ['by' => 'grace']], + timers: ['approve:timer:0' => '2026-10-12T11:00:00.000000+00:00'], + waitingOn: 'approve', + parentCorrelationId: 'p-1', + children: ['spawn' => 'child-1'], + report: ['status' => 'WAITING'], + ); +} + +it('round-trips every field through toArray, JSON and fromArray', function (): void { + $state = orchestrationFullState(); + + $restored = StateSerializer::decode(StateSerializer::encode($state)); + + expect($restored->toArray())->toBe($state->toArray()) + ->and($restored->stepStatus('approve'))->toBe(StepStatus::Waiting) + ->and($restored->stepStatus('never'))->toBe(StepStatus::Pending) + ->and($restored->currentPhase)->toBe(TccPhase::Confirm) + ->and($restored->wakeAt?->format('H:i'))->toBe('11:00'); +}); + +it('changes status and the update instant without touching the rest', function (): void { + $state = orchestrationFullState(); + $done = $state->withStatus(ExecutionStatus::Completed, new DateTimeImmutable('2026-10-12T12:00:00+00:00')); + + expect($done->status)->toBe(ExecutionStatus::Completed) + ->and($done->isTerminal())->toBeTrue() + ->and(ExecutionState::instant($done->updatedAt))->toBe('2026-10-12T12:00:00.000000+00:00') + ->and($done->completedAt)->not->toBeNull() + ->and($done->stepResults)->toBe($state->stepResults); +}); + +it('compares search attribute values in one canonical string form', function (): void { + expect(ExecutionState::attributeValue(true))->toBe('true') + ->and(ExecutionState::attributeValue(3))->toBe('3') + ->and(ExecutionState::attributeValue(1.5))->toBe('1.5') + ->and(ExecutionState::attributeValue('x'))->toBe('x'); +}); diff --git a/packages/orchestration/tests/Persistence/StateSerializerTest.php b/packages/orchestration/tests/Persistence/StateSerializerTest.php new file mode 100644 index 00000000..55cb4d60 --- /dev/null +++ b/packages/orchestration/tests/Persistence/StateSerializerTest.php @@ -0,0 +1,61 @@ + $this->cents]; + } +} + +final class OrchestrationSerializerPlain +{ + public string $sku = 'A-1'; + + private string $secret = 'hidden'; + + public function secret(): string + { + return $this->secret; + } +} + +it('normalizes every value a step may return into its JSON-safe form', function (): void { + $normalized = StateSerializer::normalize([ + 'money' => new OrchestrationSerializerMoney(1250), + 'pattern' => ExecutionPattern::Saga, + 'at' => new DateTimeImmutable('2026-10-12T10:00:00.123456+02:00'), + 'plain' => new OrchestrationSerializerPlain, + 'list' => new ArrayIterator([1, 2]), + 'scalar' => 1.0, + ]); + + expect($normalized)->toBe([ + 'money' => ['cents' => 1250], + 'pattern' => 'SAGA', + 'at' => '2026-10-12T08:00:00.123456+00:00', + 'plain' => ['sku' => 'A-1'], + 'list' => [1, 2], + 'scalar' => 1.0, + ]); +}); + +it('refuses a closure and names where it sat', function (): void { + StateSerializer::normalize(['results' => ['charge' => static fn (): int => 1]], 'stepResults'); +})->throws(OrchestrationException::class, 'stepResults.results.charge'); + +it('round-trips an array payload through JSON keeping float zero fractions', function (): void { + $payload = StateSerializer::encodeArray(['amount' => 10.0, 'name' => 'é']); + + expect($payload)->toBe('{"amount":10.0,"name":"é"}') + ->and(StateSerializer::decodeArray($payload))->toBe(['amount' => 10.0, 'name' => 'é']); +}); From 1c93cbc0242bccc892d57fa99ed821116e0b09ba Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 19:02:10 -0700 Subject: [PATCH 04/58] fix(orchestration): refuse cyclic and over-deep step results instead of exhausting memory mid-checkpoint --- .../src/Persistence/StateSerializer.php | 41 +++++++++++++++--- .../tests/Persistence/StateSerializerTest.php | 42 +++++++++++++++++++ 2 files changed, 78 insertions(+), 5 deletions(-) diff --git a/packages/orchestration/src/Persistence/StateSerializer.php b/packages/orchestration/src/Persistence/StateSerializer.php index c63b5f9a..10d441a7 100644 --- a/packages/orchestration/src/Persistence/StateSerializer.php +++ b/packages/orchestration/src/Persistence/StateSerializer.php @@ -23,35 +23,66 @@ * become their jsonSerialize(), enums their value (or name), dates a UTC ISO instant, Traversables arrays, * and any other object its PUBLIC properties. A closure or a resource cannot be represented at all and is * refused with the path it sat at (`stepResults.charge.callback`) — a checkpoint that silently dropped it - * would resume into a different execution than the one that ran. + * would resume into a different execution than the one that ran. So is a cyclic object graph, or one nested + * deeper than json_decode() reads back, which would otherwise exhaust memory and kill the worker mid-checkpoint. */ final class StateSerializer { private const int FLAGS = JSON_THROW_ON_ERROR | JSON_PRESERVE_ZERO_FRACTION | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE; + /** + * The nesting depth a checkpoint may reach, matching the depth json_decode() reads back by default. + */ + private const int MAX_DEPTH = 512; + public static function normalize(mixed $value, string $path = 'value'): mixed + { + return self::normalizeAt($value, $path, [], 0); + } + + /** + * Normalizes one value, carrying the ids of the objects on the CURRENT path so a cycle (an entity + * holding its parent, a DTO with a back-reference) is refused with its path instead of recursing until + * PHP runs out of memory mid-checkpoint. An id leaves the set once its subtree is done, so the same + * object reached twice through sibling branches is a shared reference, not a cycle, and still works. + * + * @param array $ancestors + */ + private static function normalizeAt(mixed $value, string $path, array $ancestors, int $depth): mixed { if ($value === null || is_scalar($value)) { return $value; } + if ($depth >= self::MAX_DEPTH) { + throw new OrchestrationException("{$path} nests deeper than ".self::MAX_DEPTH.' levels, which an orchestration checkpoint cannot persist.', 'ORCHESTRATION_UNSERIALIZABLE'); + } + if (is_array($value)) { $normalized = []; foreach ($value as $key => $item) { - $normalized[$key] = self::normalize($item, $path.'.'.$key); + $normalized[$key] = self::normalizeAt($item, $path.'.'.$key, $ancestors, $depth + 1); } return $normalized; } + if (is_object($value) && ! $value instanceof UnitEnum && ! $value instanceof DateTimeInterface && ! $value instanceof Closure) { + $id = spl_object_id($value); + if (isset($ancestors[$id])) { + throw new OrchestrationException("{$path} holds a cyclic reference to a ".get_debug_type($value).' already on its own path, which an orchestration checkpoint cannot persist. Return acyclic data from the step.', 'ORCHESTRATION_UNSERIALIZABLE'); + } + $ancestors[$id] = true; + } + return match (true) { - $value instanceof JsonSerializable => self::normalize($value->jsonSerialize(), $path), + $value instanceof JsonSerializable => self::normalizeAt($value->jsonSerialize(), $path, $ancestors, $depth + 1), $value instanceof BackedEnum => $value->value, $value instanceof UnitEnum => $value->name, $value instanceof DateTimeInterface => DateTimeImmutable::createFromInterface($value)->setTimezone(new DateTimeZone('UTC'))->format('Y-m-d\TH:i:s.uP'), - $value instanceof Traversable => self::normalize(iterator_to_array($value), $path), + $value instanceof Traversable => self::normalizeAt(iterator_to_array($value), $path, $ancestors, $depth + 1), $value instanceof Closure => throw new OrchestrationException("{$path} holds a closure, which an orchestration checkpoint cannot persist. Return data (an array, a scalar, a JsonSerializable) from the step.", 'ORCHESTRATION_UNSERIALIZABLE'), - is_object($value) => self::normalize(get_object_vars($value), $path), + is_object($value) => self::normalizeAt(get_object_vars($value), $path, $ancestors, $depth + 1), default => throw new OrchestrationException("{$path} holds a ".get_debug_type($value).', which an orchestration checkpoint cannot persist.', 'ORCHESTRATION_UNSERIALIZABLE'), }; } diff --git a/packages/orchestration/tests/Persistence/StateSerializerTest.php b/packages/orchestration/tests/Persistence/StateSerializerTest.php index 55cb4d60..5d92001e 100644 --- a/packages/orchestration/tests/Persistence/StateSerializerTest.php +++ b/packages/orchestration/tests/Persistence/StateSerializerTest.php @@ -53,6 +53,48 @@ public function secret(): string StateSerializer::normalize(['results' => ['charge' => static fn (): int => 1]], 'stepResults'); })->throws(OrchestrationException::class, 'stepResults.results.charge'); +it('refuses a cyclic object graph and names where the cycle closed', function (): void { + $order = new stdClass; + $line = new stdClass; + $line->order = $order; + $order->lines = [$line]; + + StateSerializer::normalize(['reserve' => $order], 'stepResults'); +})->throws(OrchestrationException::class, 'stepResults.reserve.lines.0.order holds a cyclic reference'); + +it('refuses an object that references itself with the unserializable code', function (): void { + $self = new stdClass; + $self->self = $self; + + $error = null; + try { + StateSerializer::normalize($self); + } catch (OrchestrationException $caught) { + $error = $caught; + } + + expect($error)->toBeInstanceOf(OrchestrationException::class) + ->and($error?->getMessage())->toContain('value.self holds a cyclic reference') + ->and($error?->errorCode())->toBe('ORCHESTRATION_UNSERIALIZABLE'); +}); + +it('normalizes the same object reached through sibling branches as a shared reference', function (): void { + $customer = new stdClass; + $customer->id = 7; + + expect(StateSerializer::normalize(['billing' => $customer, 'shipping' => $customer, 'again' => [$customer]])) + ->toBe(['billing' => ['id' => 7], 'shipping' => ['id' => 7], 'again' => [['id' => 7]]]); +}); + +it('refuses a payload nested deeper than a checkpoint can read back', function (): void { + $value = 'leaf'; + for ($i = 0; $i < 600; $i++) { + $value = [$value]; + } + + StateSerializer::normalize($value); +})->throws(OrchestrationException::class, 'nests deeper than 512 levels'); + it('round-trips an array payload through JSON keeping float zero fractions', function (): void { $payload = StateSerializer::encodeArray(['amount' => 10.0, 'name' => 'é']); From 96959a22edf7e4d0597bd8ca1de4b67c719bb4f4 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 19:14:32 -0700 Subject: [PATCH 05/58] fix(orchestration): budget the checkpoint envelope in the depth guard and refuse non-finite floats and invalid UTF-8 --- .../src/Persistence/StateSerializer.php | 49 ++++++++- .../tests/Persistence/StateSerializerTest.php | 100 +++++++++++++++++- 2 files changed, 143 insertions(+), 6 deletions(-) diff --git a/packages/orchestration/src/Persistence/StateSerializer.php b/packages/orchestration/src/Persistence/StateSerializer.php index 10d441a7..41e4502f 100644 --- a/packages/orchestration/src/Persistence/StateSerializer.php +++ b/packages/orchestration/src/Persistence/StateSerializer.php @@ -24,16 +24,34 @@ * and any other object its PUBLIC properties. A closure or a resource cannot be represented at all and is * refused with the path it sat at (`stepResults.charge.callback`) — a checkpoint that silently dropped it * would resume into a different execution than the one that ran. So is a cyclic object graph, or one nested - * deeper than json_decode() reads back, which would otherwise exhaust memory and kill the worker mid-checkpoint. + * deeper than the checkpoint envelope leaves room for, which would otherwise exhaust memory mid-checkpoint or + * write a row json_decode() cannot read back. So, finally, is a scalar JSON has no form for: a NAN or INF + * float, or a string (or array key) that is not valid UTF-8 — binary data, a mis-encoded column. Refusing + * them here, on the normalized value every provider stores, is what keeps `memory` and `cache` from accepting + * a value the `database` provider would then fail to write. */ final class StateSerializer { private const int FLAGS = JSON_THROW_ON_ERROR | JSON_PRESERVE_ZERO_FRACTION | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE; /** - * The nesting depth a checkpoint may reach, matching the depth json_decode() reads back by default. + * The depth decodeArray() hands json_decode(): a payload of N nested arrays needs N + 1 levels, the + * innermost scalar counting as one, so at most JSON_DEPTH - 1 arrays read back. */ - private const int MAX_DEPTH = 512; + private const int JSON_DEPTH = 512; + + /** + * The arrays the checkpoint envelope wraps around every normalized field: the ExecutionState root + * object. A field (`stepResults`, `variables`, ...) is itself the next array down. + */ + private const int ENVELOPE_DEPTH = 1; + + /** + * How many arrays a normalized field may nest, counting the field's own map (`stepResults`) as the + * first: what json_decode() reads back once the envelope has taken its share, so a value that passes + * normalize() is a checkpoint that resumes. + */ + public const int MAX_DEPTH = self::JSON_DEPTH - 1 - self::ENVELOPE_DEPTH; public static function normalize(mixed $value, string $path = 'value'): mixed { @@ -50,6 +68,16 @@ public static function normalize(mixed $value, string $path = 'value'): mixed */ private static function normalizeAt(mixed $value, string $path, array $ancestors, int $depth): mixed { + if (is_float($value) && ! is_finite($value)) { + $kind = is_nan($value) ? 'NAN' : ($value > 0 ? 'INF' : '-INF'); + + throw new OrchestrationException("{$path} holds a non-finite float ({$kind}), which JSON cannot represent and an orchestration checkpoint cannot persist.", 'ORCHESTRATION_UNSERIALIZABLE'); + } + + if (is_string($value) && ! mb_check_encoding($value, 'UTF-8')) { + throw new OrchestrationException("{$path} holds a string that is not valid UTF-8, which an orchestration checkpoint cannot persist. Encode binary data (base64) before returning it from the step.", 'ORCHESTRATION_UNSERIALIZABLE'); + } + if ($value === null || is_scalar($value)) { return $value; } @@ -61,6 +89,9 @@ private static function normalizeAt(mixed $value, string $path, array $ancestors if (is_array($value)) { $normalized = []; foreach ($value as $key => $item) { + if (is_string($key) && ! mb_check_encoding($key, 'UTF-8')) { + throw new OrchestrationException("{$path} holds a key that is not valid UTF-8, which an orchestration checkpoint cannot persist.", 'ORCHESTRATION_UNSERIALIZABLE'); + } $normalized[$key] = self::normalizeAt($item, $path.'.'.$key, $ancestors, $depth + 1); } @@ -98,11 +129,19 @@ public static function decode(string $payload): ExecutionState } /** + * Encodes a checkpoint. A value normalize() let through always encodes; a field that never went through + * it (a header, a search attribute) and still holds what JSON cannot represent is refused with the same + * code rather than escaping as a bare JsonException. + * * @param array $data */ public static function encodeArray(array $data): string { - return json_encode($data, self::FLAGS); + try { + return json_encode($data, self::FLAGS, self::JSON_DEPTH); + } catch (JsonException $error) { + throw new OrchestrationException("An orchestration checkpoint cannot be encoded as JSON: {$error->getMessage()}", 'ORCHESTRATION_UNSERIALIZABLE', 500, $error); + } } /** @@ -111,7 +150,7 @@ public static function encodeArray(array $data): string public static function decodeArray(string $payload): array { try { - $data = json_decode($payload, true, 512, JSON_THROW_ON_ERROR); + $data = json_decode($payload, true, self::JSON_DEPTH, JSON_THROW_ON_ERROR); } catch (JsonException $error) { throw new OrchestrationException("A persisted orchestration payload is not valid JSON: {$error->getMessage()}", 'ORCHESTRATION_CORRUPT_STATE', 500, $error); } diff --git a/packages/orchestration/tests/Persistence/StateSerializerTest.php b/packages/orchestration/tests/Persistence/StateSerializerTest.php index 5d92001e..d053a2d4 100644 --- a/packages/orchestration/tests/Persistence/StateSerializerTest.php +++ b/packages/orchestration/tests/Persistence/StateSerializerTest.php @@ -3,7 +3,9 @@ declare(strict_types=1); use Firefly\Orchestration\Core\Exception\OrchestrationException; +use Firefly\Orchestration\Core\ExecutionContext; use Firefly\Orchestration\Core\ExecutionPattern; +use Firefly\Orchestration\Core\ExecutionStatus; use Firefly\Orchestration\Persistence\StateSerializer; final class OrchestrationSerializerMoney implements JsonSerializable @@ -93,7 +95,103 @@ public function secret(): string } StateSerializer::normalize($value); -})->throws(OrchestrationException::class, 'nests deeper than 512 levels'); +})->throws(OrchestrationException::class, 'nests deeper than 510 levels'); + +/** + * A step result of $arrays nested arrays, wrapped the way a checkpoint wraps it: the envelope root, the + * `stepResults` map, then the step id. + */ +function orchestrationSerializerDeepContext(int $arrays): ExecutionContext +{ + $value = 'leaf'; + for ($i = 0; $i < $arrays; $i++) { + $value = [$value]; + } + + $context = ExecutionContext::start(ExecutionPattern::Saga, 'deep', [], [], 'c-deep', new DateTimeImmutable('2026-10-12T10:00:00+00:00')); + $context->putResult('deep', $value); + + return $context; +} + +/** + * @return array{string, string}|null the error code and message, or null when nothing was thrown + */ +function orchestrationSerializerRefusal(callable $action): ?array +{ + try { + $action(); + } catch (OrchestrationException $caught) { + return [$caught->errorCode(), $caught->getMessage()]; + } + + return null; +} + +it('checkpoints and resumes a step result nested exactly as deep as the envelope leaves room for', function (): void { + // The stepResults map is the field's first array, so the step result itself may nest MAX_DEPTH - 1. + $state = orchestrationSerializerDeepContext(StateSerializer::MAX_DEPTH - 1)->toState(ExecutionStatus::Running, new DateTimeImmutable('2026-10-12T10:00:01+00:00')); + + $back = StateSerializer::decode(StateSerializer::encode($state)); + + expect($back->stepResults)->toBe($state->stepResults); +}); + +it('refuses a step result one level deeper than a checkpoint can read back, before it is written', function (): void { + $context = orchestrationSerializerDeepContext(StateSerializer::MAX_DEPTH); + + $refusal = orchestrationSerializerRefusal(static fn () => $context->toState(ExecutionStatus::Running, new DateTimeImmutable('2026-10-12T10:00:01+00:00'))); + + expect($refusal)->not->toBeNull() + ->and($refusal[0] ?? null)->toBe('ORCHESTRATION_UNSERIALIZABLE') + ->and($refusal[1] ?? '')->toContain('stepResults.deep.0.0') + ->and($refusal[1] ?? '')->toContain('nests deeper than 510 levels'); +}); + +it('refuses a non-finite float with its path and the unserializable code', function (float $float): void { + $refusal = orchestrationSerializerRefusal(static fn () => StateSerializer::normalize(['quote' => ['rate' => $float]], 'stepResults')); + + expect($refusal)->not->toBeNull() + ->and($refusal[0] ?? null)->toBe('ORCHESTRATION_UNSERIALIZABLE') + ->and($refusal[1] ?? '')->toContain('stepResults.quote.rate holds a non-finite float'); +})->with([ + 'NAN' => [NAN], + 'INF' => [INF], + '-INF' => [-INF], +]); + +it('refuses a string that is not valid UTF-8 with its path and the unserializable code', function (): void { + $refusal = orchestrationSerializerRefusal(static fn () => StateSerializer::normalize(['load' => ['blob' => "\xB1\x31"]], 'stepResults')); + + expect($refusal)->not->toBeNull() + ->and($refusal[0] ?? null)->toBe('ORCHESTRATION_UNSERIALIZABLE') + ->and($refusal[1] ?? '')->toContain('stepResults.load.blob holds a string that is not valid UTF-8'); +}); + +it('refuses an array key that is not valid UTF-8 with the unserializable code', function (): void { + $refusal = orchestrationSerializerRefusal(static fn () => StateSerializer::normalize(['load' => ["\xB1\x31" => 1]], 'stepResults')); + + expect($refusal)->not->toBeNull() + ->and($refusal[0] ?? null)->toBe('ORCHESTRATION_UNSERIALIZABLE') + ->and($refusal[1] ?? '')->toContain('stepResults.load holds a key that is not valid UTF-8'); +}); + +it('refuses through toState() too, so every provider rejects the value before a checkpoint exists', function (): void { + $context = ExecutionContext::start(ExecutionPattern::Saga, 's', [], [], 'c-nan', new DateTimeImmutable('2026-10-12T10:00:00+00:00')); + $context->putResult('price', NAN); + + $refusal = orchestrationSerializerRefusal(static fn () => $context->toState(ExecutionStatus::Running, new DateTimeImmutable('2026-10-12T10:00:01+00:00'))); + + expect($refusal[0] ?? null)->toBe('ORCHESTRATION_UNSERIALIZABLE'); +}); + +it('rethrows a JSON encoding failure in a field normalize() never saw with the unserializable code', function (): void { + $refusal = orchestrationSerializerRefusal(static fn () => StateSerializer::encodeArray(['headers' => ['x-raw' => "\xB1\x31"]])); + + expect($refusal)->not->toBeNull() + ->and($refusal[0] ?? null)->toBe('ORCHESTRATION_UNSERIALIZABLE') + ->and($refusal[1] ?? '')->toContain('Malformed UTF-8'); +}); it('round-trips an array payload through JSON keeping float zero fractions', function (): void { $payload = StateSerializer::encodeArray(['amount' => 10.0, 'name' => 'é']); From 90505e279bea881ea61234adee4e885320180fc5 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 19:17:24 -0700 Subject: [PATCH 06/58] feat(orchestration): compiled argument injection, a resilience-backed step invoker and lifecycle callbacks --- .../src/Core/Argument/ArgumentResolver.php | 58 ++++++++++ .../src/Core/Argument/CompensationError.php | 14 +++ .../src/Core/Argument/CorrelationId.php | 11 ++ .../Core/Argument/FromCompensationResult.php | 14 +++ .../src/Core/Argument/FromStep.php | 14 +++ .../src/Core/Argument/Header.php | 14 +++ .../src/Core/Argument/Headers.php | 11 ++ .../orchestration/src/Core/Argument/Input.php | 14 +++ .../src/Core/Argument/ParameterBinding.php | 64 +++++++++++ .../src/Core/Argument/Required.php | 11 ++ .../src/Core/Argument/SetVariable.php | 14 +++ .../src/Core/Argument/Variable.php | 14 +++ .../src/Core/Argument/Variables.php | 11 ++ .../src/Core/Callback/CallbackDefinition.php | 106 ++++++++++++++++++ .../src/Core/Callback/CallbackInvoker.php | 81 +++++++++++++ .../src/Core/ClosureInvocation.php | 37 ++++++ .../orchestration/src/Core/Invocation.php | 20 ++++ .../src/Core/InvocationRunner.php | 47 ++++++++ .../src/Core/MethodInvocation.php | 58 ++++++++++ .../orchestration/src/Core/StepInvoker.php | 49 ++++++++ .../orchestration/src/Core/StepOptions.php | 54 +++++++++ .../src/Event/NoOpOrchestrationTracing.php | 21 ++++ .../src/Event/OrchestrationTracing.php | 31 +++++ .../tests/Core/ArgumentResolverTest.php | 84 ++++++++++++++ .../tests/Core/CallbackInvokerTest.php | 65 +++++++++++ .../tests/Core/StepInvokerTest.php | 76 +++++++++++++ .../Fixtures/Arguments/ArgumentProbe.php | 50 +++++++++ .../tests/Support/RecordingLogger.php | 32 ++++++ 28 files changed, 1075 insertions(+) create mode 100644 packages/orchestration/src/Core/Argument/ArgumentResolver.php create mode 100644 packages/orchestration/src/Core/Argument/CompensationError.php create mode 100644 packages/orchestration/src/Core/Argument/CorrelationId.php create mode 100644 packages/orchestration/src/Core/Argument/FromCompensationResult.php create mode 100644 packages/orchestration/src/Core/Argument/FromStep.php create mode 100644 packages/orchestration/src/Core/Argument/Header.php create mode 100644 packages/orchestration/src/Core/Argument/Headers.php create mode 100644 packages/orchestration/src/Core/Argument/Input.php create mode 100644 packages/orchestration/src/Core/Argument/ParameterBinding.php create mode 100644 packages/orchestration/src/Core/Argument/Required.php create mode 100644 packages/orchestration/src/Core/Argument/SetVariable.php create mode 100644 packages/orchestration/src/Core/Argument/Variable.php create mode 100644 packages/orchestration/src/Core/Argument/Variables.php create mode 100644 packages/orchestration/src/Core/Callback/CallbackDefinition.php create mode 100644 packages/orchestration/src/Core/Callback/CallbackInvoker.php create mode 100644 packages/orchestration/src/Core/ClosureInvocation.php create mode 100644 packages/orchestration/src/Core/Invocation.php create mode 100644 packages/orchestration/src/Core/InvocationRunner.php create mode 100644 packages/orchestration/src/Core/MethodInvocation.php create mode 100644 packages/orchestration/src/Core/StepInvoker.php create mode 100644 packages/orchestration/src/Core/StepOptions.php create mode 100644 packages/orchestration/src/Event/NoOpOrchestrationTracing.php create mode 100644 packages/orchestration/src/Event/OrchestrationTracing.php create mode 100644 packages/orchestration/tests/Core/ArgumentResolverTest.php create mode 100644 packages/orchestration/tests/Core/CallbackInvokerTest.php create mode 100644 packages/orchestration/tests/Core/StepInvokerTest.php create mode 100644 packages/orchestration/tests/Fixtures/Arguments/ArgumentProbe.php create mode 100644 packages/orchestration/tests/Support/RecordingLogger.php diff --git a/packages/orchestration/src/Core/Argument/ArgumentResolver.php b/packages/orchestration/src/Core/Argument/ArgumentResolver.php new file mode 100644 index 00000000..0840e984 --- /dev/null +++ b/packages/orchestration/src/Core/Argument/ArgumentResolver.php @@ -0,0 +1,58 @@ + + */ + public function resolve(MethodInvocation $invocation, ExecutionContext $context, mixed $input, mixed $implicit): array + { + $arguments = []; + foreach ($invocation->parameters as $binding) { + if ($binding->kind === ParameterBinding::DEFAULT) { + continue; + } + + $value = $this->value($binding, $context, $input, $implicit); + if ($value === null && $binding->required) { + throw new OrchestrationException("Required parameter \${$binding->name} of {$invocation->describe()} resolved to null.", 'ORCHESTRATION_REQUIRED_ARGUMENT', 422); + } + + $arguments[$binding->name] = $value; + } + + return $arguments; + } + + private function value(ParameterBinding $binding, ExecutionContext $context, mixed $input, mixed $implicit): mixed + { + return match ($binding->kind) { + ParameterBinding::IMPLICIT => $implicit, + ParameterBinding::CONTEXT => $context, + ParameterBinding::INPUT => $input, + ParameterBinding::INPUT_KEY => is_array($input) ? ($input[$binding->ref] ?? null) : null, + ParameterBinding::FROM_STEP => $context->result($binding->ref), + ParameterBinding::FROM_COMPENSATION_RESULT => $context->compensationResult($binding->ref), + ParameterBinding::COMPENSATION_ERROR => $context->compensationError($binding->ref), + ParameterBinding::FROM_TRY => $binding->ref === '' ? $implicit : $context->tryResult($binding->ref), + ParameterBinding::HEADER => $context->header($binding->ref), + ParameterBinding::HEADERS => $context->headers(), + ParameterBinding::VARIABLE => $context->variable($binding->ref), + ParameterBinding::VARIABLES => $context->variables(), + ParameterBinding::CORRELATION_ID => $context->correlationId, + default => throw new OrchestrationException("Unknown parameter binding [{$binding->kind}] for \${$binding->name}; recompile with php artisan firefly:cache.", 'ORCHESTRATION_UNKNOWN_BINDING'), + }; + } +} diff --git a/packages/orchestration/src/Core/Argument/CompensationError.php b/packages/orchestration/src/Core/Argument/CompensationError.php new file mode 100644 index 00000000..cf13aecd --- /dev/null +++ b/packages/orchestration/src/Core/Argument/CompensationError.php @@ -0,0 +1,14 @@ +. */ +#[Attribute(Attribute::TARGET_PARAMETER)] +final readonly class Headers {} diff --git a/packages/orchestration/src/Core/Argument/Input.php b/packages/orchestration/src/Core/Argument/Input.php new file mode 100644 index 00000000..44677f59 --- /dev/null +++ b/packages/orchestration/src/Core/Argument/Input.php @@ -0,0 +1,14 @@ + $this->name, 'kind' => $this->kind, 'ref' => $this->ref, 'required' => $this->required]; + } + + /** @param array $row */ + public static function fromArray(array $row): self + { + return new self(Rows::string($row, 'name'), Rows::string($row, 'kind', self::IMPLICIT), Rows::string($row, 'ref'), Rows::bool($row, 'required')); + } +} diff --git a/packages/orchestration/src/Core/Argument/Required.php b/packages/orchestration/src/Core/Argument/Required.php new file mode 100644 index 00000000..30a7885d --- /dev/null +++ b/packages/orchestration/src/Core/Argument/Required.php @@ -0,0 +1,11 @@ +`. */ +#[Attribute(Attribute::TARGET_PARAMETER)] +final readonly class Variable +{ + public function __construct(public string $name) {} +} diff --git a/packages/orchestration/src/Core/Argument/Variables.php b/packages/orchestration/src/Core/Argument/Variables.php new file mode 100644 index 00000000..6e386752 --- /dev/null +++ b/packages/orchestration/src/Core/Argument/Variables.php @@ -0,0 +1,11 @@ +. */ +#[Attribute(Attribute::TARGET_PARAMETER)] +final readonly class Variables {} diff --git a/packages/orchestration/src/Core/Callback/CallbackDefinition.php b/packages/orchestration/src/Core/Callback/CallbackDefinition.php new file mode 100644 index 00000000..5e4b2fd7 --- /dev/null +++ b/packages/orchestration/src/Core/Callback/CallbackDefinition.php @@ -0,0 +1,106 @@ + $parameters + * @param list $errorTypes class names; empty matches every error + * @param list $stepIds step ids; empty matches every step + */ + public function __construct( + public ?string $class = null, + public ?string $method = null, + public array $parameters = [], + public bool $async = false, + public int $priority = 0, + public array $errorTypes = [], + public bool $suppressError = false, + public array $stepIds = [], + public ?Closure $closure = null, + ) {} + + /** + * @param list $errorTypes + * @param list $stepIds + */ + public static function closure(Closure $callback, bool $async = false, int $priority = 0, array $errorTypes = [], bool $suppressError = false, array $stepIds = []): self + { + return new self(async: $async, priority: $priority, errorTypes: $errorTypes, suppressError: $suppressError, stepIds: $stepIds, closure: $callback); + } + + public function matchesError(?Throwable $error): bool + { + if ($this->errorTypes === [] || $error === null) { + return true; + } + + foreach ($this->errorTypes as $type) { + if (is_a($error, $type)) { + return true; + } + } + + return false; + } + + public function matchesStep(?string $stepId): bool + { + return $this->stepIds === [] || $stepId === null || in_array($stepId, $this->stepIds, true); + } + + public function describe(): string + { + return $this->closure !== null ? 'closure' : $this->class.'::'.$this->method; + } + + /** @return array */ + public function toArray(): array + { + if ($this->closure !== null) { + throw new LogicException('A closure callback cannot be compiled into a manifest.'); + } + + return [ + 'class' => $this->class, 'method' => $this->method, 'parameters' => $this->parameters, 'async' => $this->async, + 'priority' => $this->priority, 'errorTypes' => $this->errorTypes, 'suppressError' => $this->suppressError, + 'stepIds' => $this->stepIds, + ]; + } + + /** @param array $row */ + public static function fromArray(array $row): self + { + return new self( + Rows::nullableString($row, 'class'), + Rows::nullableString($row, 'method'), + Rows::strings($row, 'parameters'), + Rows::bool($row, 'async'), + Rows::int($row, 'priority'), + Rows::strings($row, 'errorTypes'), + Rows::bool($row, 'suppressError'), + Rows::strings($row, 'stepIds'), + ); + } +} diff --git a/packages/orchestration/src/Core/Callback/CallbackInvoker.php b/packages/orchestration/src/Core/Callback/CallbackInvoker.php new file mode 100644 index 00000000..0fa108f3 --- /dev/null +++ b/packages/orchestration/src/Core/Callback/CallbackInvoker.php @@ -0,0 +1,81 @@ + $callbacks + */ + public function invoke(array $callbacks, ExecutionContext $context, mixed $result = null, ?Throwable $error = null, ?string $stepId = null): void + { + $eligible = array_values(array_filter( + $callbacks, + static fn (CallbackDefinition $callback): bool => $callback->matchesError($error) && $callback->matchesStep($stepId), + )); + usort($eligible, static fn (CallbackDefinition $a, CallbackDefinition $b): int => [$a->async, -$a->priority] <=> [$b->async, -$b->priority]); + + foreach ($eligible as $callback) { + try { + $this->call($callback, $context, $result, $error); + } catch (Throwable $failure) { + $this->logger->warning("Orchestration callback {$callback->describe()} for [{$context->executionName}] failed: {$failure->getMessage()}", [ + 'exception' => $failure, + 'correlationId' => $context->correlationId, + ]); + } + } + } + + /** + * @param list $callbacks + */ + public function suppresses(array $callbacks, Throwable $error): bool + { + foreach ($callbacks as $callback) { + if ($callback->suppressError && $callback->matchesError($error)) { + return true; + } + } + + return false; + } + + private function call(CallbackDefinition $callback, ExecutionContext $context, mixed $result, ?Throwable $error): void + { + if ($callback->closure !== null) { + ($callback->closure)($context, $error ?? $result); + + return; + } + + $bean = $this->container->make((string) $callback->class); + $method = (string) $callback->method; + $arguments = array_map(static fn (string $kind): mixed => match ($kind) { + CallbackDefinition::CONTEXT => $context, + CallbackDefinition::ERROR => $error, + default => $result, + }, $callback->parameters); + + if (is_object($bean)) { + $bean->{$method}(...$arguments); + } + } +} diff --git a/packages/orchestration/src/Core/ClosureInvocation.php b/packages/orchestration/src/Core/ClosureInvocation.php new file mode 100644 index 00000000..5902e785 --- /dev/null +++ b/packages/orchestration/src/Core/ClosureInvocation.php @@ -0,0 +1,37 @@ + …`. It lives only in the process that + * built it, so it can never be compiled into a manifest — and a builder-defined saga can be resumed after a + * crash only by a process that registered the same definition again. + */ +final readonly class ClosureInvocation implements Invocation +{ + public function __construct( + public Closure $closure, + public string $label = 'closure', + public ?string $variable = null, + ) {} + + public function describe(): string + { + return $this->label; + } + + public function setVariable(): ?string + { + return $this->variable; + } + + public function toArray(): array + { + throw new LogicException("Invocation [{$this->label}] was built from a closure and cannot be compiled into a manifest."); + } +} diff --git a/packages/orchestration/src/Core/Invocation.php b/packages/orchestration/src/Core/Invocation.php new file mode 100644 index 00000000..d783b49e --- /dev/null +++ b/packages/orchestration/src/Core/Invocation.php @@ -0,0 +1,20 @@ + */ + public function toArray(): array; +} diff --git a/packages/orchestration/src/Core/InvocationRunner.php b/packages/orchestration/src/Core/InvocationRunner.php new file mode 100644 index 00000000..dbc1a0bf --- /dev/null +++ b/packages/orchestration/src/Core/InvocationRunner.php @@ -0,0 +1,47 @@ + ($invocation->closure)($implicit, $context), + $invocation instanceof MethodInvocation => $this->call($invocation, $context, $input, $implicit), + default => throw new LogicException('Unsupported orchestration invocation '.$invocation::class.'.'), + }; + + $variable = $invocation->setVariable(); + if ($variable !== null) { + $context->setVariable($variable, $result); + } + + return $result; + } + + private function call(MethodInvocation $invocation, ExecutionContext $context, mixed $input, mixed $implicit): mixed + { + $bean = $this->container->make($invocation->class); + if (! is_object($bean) || ! method_exists($bean, $invocation->method)) { + throw new OrchestrationException("{$invocation->describe()} does not exist; recompile with php artisan firefly:cache.", 'ORCHESTRATION_UNKNOWN_METHOD'); + } + + $method = $invocation->method; + + return $bean->{$method}(...$this->arguments->resolve($invocation, $context, $input, $implicit)); + } +} diff --git a/packages/orchestration/src/Core/MethodInvocation.php b/packages/orchestration/src/Core/MethodInvocation.php new file mode 100644 index 00000000..ea233168 --- /dev/null +++ b/packages/orchestration/src/Core/MethodInvocation.php @@ -0,0 +1,58 @@ + $parameters + */ + public function __construct( + public string $class, + public string $method, + public array $parameters = [], + public ?string $variable = null, + ) {} + + public function describe(): string + { + return $this->class.'::'.$this->method; + } + + public function setVariable(): ?string + { + return $this->variable; + } + + /** @return array */ + public function toArray(): array + { + return [ + 'class' => $this->class, + 'method' => $this->method, + 'parameters' => array_map(static fn (ParameterBinding $binding): array => $binding->toArray(), $this->parameters), + 'setVariable' => $this->variable, + ]; + } + + /** @param array $row */ + public static function fromArray(array $row): self + { + return new self( + Rows::string($row, 'class'), + Rows::string($row, 'method'), + array_map(static fn (array $binding): ParameterBinding => ParameterBinding::fromArray($binding), Rows::rows($row, 'parameters')), + Rows::nullableString($row, 'setVariable'), + ); + } +} diff --git a/packages/orchestration/src/Core/StepInvoker.php b/packages/orchestration/src/Core/StepInvoker.php new file mode 100644 index 00000000..9314c1b5 --- /dev/null +++ b/packages/orchestration/src/Core/StepInvoker.php @@ -0,0 +1,49 @@ +recordAttempt($stepId); + + return $this->tracing->traceStep($executionName, $stepId, $context->correlationId, function () use ($invocation, $options, $context, $input, $implicit): mixed { + $call = fn (): mixed => $this->runner->run($invocation, $context, $input, $implicit); + + return $options->timeoutMs > 0 ? (new TimeLimiter($options->timeoutMs / 1000))->call($call) : $call(); + }); + }; + + if ($options->retry <= 0) { + return $attempt(); + } + + $wait = $options->backoffMs / 1000; + + return (new Retry( + maxAttempts: $options->retry + 1, + waitDuration: $wait, + jitter: $options->jitter ? $wait * $options->jitterFactor : 0.0, + ))->call($attempt); + } +} diff --git a/packages/orchestration/src/Core/StepOptions.php b/packages/orchestration/src/Core/StepOptions.php new file mode 100644 index 00000000..7df3c704 --- /dev/null +++ b/packages/orchestration/src/Core/StepOptions.php @@ -0,0 +1,54 @@ +retry, $this->backoffMs, $this->timeoutMs, $this->jitter, $this->jitterFactor); + } + + /** @return array{retry: int, backoffMs: int, timeoutMs: int, jitter: bool, jitterFactor: float, idempotencyKey: string} */ + public function toArray(): array + { + return [ + 'retry' => $this->retry, 'backoffMs' => $this->backoffMs, 'timeoutMs' => $this->timeoutMs, + 'jitter' => $this->jitter, 'jitterFactor' => $this->jitterFactor, 'idempotencyKey' => $this->idempotencyKey, + ]; + } + + /** @param array $row */ + public static function fromArray(array $row): self + { + return new self( + Rows::int($row, 'retry'), + Rows::int($row, 'backoffMs'), + Rows::int($row, 'timeoutMs'), + Rows::bool($row, 'jitter'), + Rows::float($row, 'jitterFactor', 0.5), + Rows::string($row, 'idempotencyKey'), + ); + } +} diff --git a/packages/orchestration/src/Event/NoOpOrchestrationTracing.php b/packages/orchestration/src/Event/NoOpOrchestrationTracing.php new file mode 100644 index 00000000..fa6cb126 --- /dev/null +++ b/packages/orchestration/src/Event/NoOpOrchestrationTracing.php @@ -0,0 +1,21 @@ + 40], ['tenant' => 'acme'], 'c-1', new DateTimeImmutable); + $context->putResult('reserve', ['hold' => 'h-1']); + $context->putCompensationResult('reserve', 'released'); + $context->putCompensationError('reserve', new RuntimeException('undo failed')); + $context->putTryResult('debit', 'try-1'); + $context->setVariable('tier', 'gold'); + + return $context; +} + +it('delivers every binding kind by name, and leaves a defaulted parameter to its default', function (): void { + $container = new Container; + $container->singleton(ArgumentProbe::class); + $runner = new InvocationRunner($container, new ArgumentResolver); + + $invocation = new MethodInvocation(ArgumentProbe::class, 'everything', [ + new ParameterBinding('implicit', ParameterBinding::IMPLICIT), + new ParameterBinding('context', ParameterBinding::CONTEXT), + new ParameterBinding('whole', ParameterBinding::INPUT), + new ParameterBinding('amount', ParameterBinding::INPUT_KEY, 'amount'), + new ParameterBinding('reserved', ParameterBinding::FROM_STEP, 'reserve'), + new ParameterBinding('undo', ParameterBinding::FROM_COMPENSATION_RESULT, 'reserve'), + new ParameterBinding('undoError', ParameterBinding::COMPENSATION_ERROR, 'reserve'), + new ParameterBinding('hold', ParameterBinding::FROM_TRY, 'debit'), + new ParameterBinding('tenant', ParameterBinding::HEADER, 'tenant'), + new ParameterBinding('headers', ParameterBinding::HEADERS), + new ParameterBinding('tier', ParameterBinding::VARIABLE, 'tier'), + new ParameterBinding('variables', ParameterBinding::VARIABLES), + new ParameterBinding('correlationId', ParameterBinding::CORRELATION_ID), + new ParameterBinding('optional', ParameterBinding::DEFAULT), + ], variable: 'probeResult'); + + $context = orchestrationArgumentContext(); + $result = $runner->run($invocation, $context, ['amount' => 40], 'the-implicit'); + $probe = $container->make(ArgumentProbe::class); + + // The probe records the variables BEFORE #[SetVariable] stores its result, so it saw only `tier`. + expect($result)->toBe('probed') + ->and($context->variable('probeResult'))->toBe('probed') + ->and($probe->seen)->toBe([ + 'implicit' => 'the-implicit', 'context' => 'c-1', 'whole' => ['amount' => 40], 'amount' => 40, + 'reserved' => ['hold' => 'h-1'], 'undo' => 'released', 'undoError' => 'undo failed', 'hold' => 'try-1', + 'tenant' => 'acme', 'headers' => ['tenant' => 'acme'], 'tier' => 'gold', 'variables' => ['tier' => 'gold'], + 'correlationId' => 'c-1', 'optional' => 'default', + ]); +}); + +it('refuses a #[Required] parameter that resolved to null, naming the method', function (): void { + $runner = new InvocationRunner(new Container, new ArgumentResolver); + $invocation = new MethodInvocation(ArgumentProbe::class, 'required', [new ParameterBinding('value', ParameterBinding::FROM_STEP, 'missing', required: true)]); + + $runner->run($invocation, orchestrationArgumentContext(), [], null); +})->throws(OrchestrationException::class, 'Required parameter $value of '.ArgumentProbe::class.'::required resolved to null.'); + +it('hands a closure the implicit value and the context', function (): void { + $runner = new InvocationRunner(new Container, new ArgumentResolver); + $invocation = new ClosureInvocation(static fn (string $implicit, ExecutionContext $context): string => $implicit.'@'.$context->executionName); + + expect($runner->run($invocation, orchestrationArgumentContext(), null, 'x'))->toBe('x@order'); +}); + +it('round-trips a method invocation and refuses to compile a closure', function (): void { + $invocation = new MethodInvocation(ArgumentProbe::class, 'everything', [new ParameterBinding('implicit', ParameterBinding::IMPLICIT, required: true)], 'v'); + + expect(MethodInvocation::fromArray($invocation->toArray()))->toEqual($invocation) + ->and(fn () => (new ClosureInvocation(static fn (): null => null))->toArray())->toThrow(LogicException::class, 'cannot be compiled'); +}); diff --git a/packages/orchestration/tests/Core/CallbackInvokerTest.php b/packages/orchestration/tests/Core/CallbackInvokerTest.php new file mode 100644 index 00000000..67f8b405 --- /dev/null +++ b/packages/orchestration/tests/Core/CallbackInvokerTest.php @@ -0,0 +1,65 @@ +invoke([ + CallbackDefinition::closure(static function () use (&$order): void { + $order[] = 'async'; + }, async: true, priority: 100), + CallbackDefinition::closure(static function () use (&$order): void { + $order[] = 'low'; + }, priority: 1), + CallbackDefinition::closure(static function (): never { + throw new RuntimeException('hook broke'); + }, priority: 5), + CallbackDefinition::closure(static function (ExecutionContext $ctx, mixed $result) use (&$order): void { + $order[] = 'high:'.(is_string($result) ? $result : ''); + }, priority: 10), + ], $context, 'done'); + + expect($order)->toBe(['high:done', 'low', 'async']) + ->and($logger->messages('warning')[0] ?? '')->toContain('hook broke'); +}); + +it('filters error callbacks by error type and step callbacks by step id', function (): void { + $seen = []; + $invoker = new CallbackInvoker(new Container, new RecordingLogger); + $context = ExecutionContext::start(ExecutionPattern::Workflow, 'w', [], [], 'c', new DateTimeImmutable); + $callbacks = [ + CallbackDefinition::closure(static function () use (&$seen): void { + $seen[] = 'logic'; + }, errorTypes: [LogicException::class]), + CallbackDefinition::closure(static function () use (&$seen): void { + $seen[] = 'any'; + }), + CallbackDefinition::closure(static function () use (&$seen): void { + $seen[] = 'ship'; + }, stepIds: ['ship']), + ]; + + $invoker->invoke($callbacks, $context, error: new RuntimeException('x'), stepId: 'pack'); + + expect($seen)->toBe(['any']) + ->and($invoker->suppresses([CallbackDefinition::closure(static fn (): null => null, errorTypes: [RuntimeException::class], suppressError: true)], new RuntimeException('x')))->toBeTrue() + ->and($invoker->suppresses([CallbackDefinition::closure(static fn (): null => null, errorTypes: [LogicException::class], suppressError: true)], new RuntimeException('x')))->toBeFalse(); +}); + +it('round-trips a method callback and refuses to compile a closure one', function (): void { + $callback = new CallbackDefinition(class: 'App\\Saga', method: 'done', parameters: [CallbackDefinition::CONTEXT, CallbackDefinition::RESULT], priority: 3, errorTypes: [RuntimeException::class], suppressError: true, stepIds: ['a']); + + expect(CallbackDefinition::fromArray($callback->toArray()))->toEqual($callback) + ->and(fn () => CallbackDefinition::closure(static fn (): null => null)->toArray())->toThrow(LogicException::class); +}); diff --git a/packages/orchestration/tests/Core/StepInvokerTest.php b/packages/orchestration/tests/Core/StepInvokerTest.php new file mode 100644 index 00000000..620e54db --- /dev/null +++ b/packages/orchestration/tests/Core/StepInvokerTest.php @@ -0,0 +1,76 @@ +invoke('s', 'charge', $flaky, StepOptions::of(2, 1, 0), $context, null, null))->toBe('ok') + ->and($context->attempts('charge'))->toBe(3); +}); + +it('gives up after retry + 1 attempts and rethrows the last error', function (): void { + $context = orchestrationInvokerContext(); + $failing = new ClosureInvocation(static fn (): never => throw new RuntimeException('down')); + + expect(fn () => orchestrationInvoker()->invoke('s', 'charge', $failing, StepOptions::of(1, 0, 0), $context, null, null)) + ->toThrow(RuntimeException::class, 'down') + ->and($context->attempts('charge'))->toBe(2); +}); + +it('backs off between attempts, with jitter bounded by the factor', function (): void { + $failing = new ClosureInvocation(static fn (): never => throw new RuntimeException('down')); + $started = microtime(true); + + try { + orchestrationInvoker()->invoke('s', 'charge', $failing, StepOptions::of(2, 20, 0, jitter: true, jitterFactor: 0.5), orchestrationInvokerContext(), null, null); + } catch (RuntimeException) { + } + + expect(microtime(true) - $started)->toBeGreaterThanOrEqual(0.040)->toBeLessThan(1.0); +}); + +it('times a step out after timeoutMs', function (): void { + $slow = new ClosureInvocation(static function (): string { + usleep(60_000); + + return 'late'; + }); + + orchestrationInvoker()->invoke('s', 'charge', $slow, StepOptions::of(0, 0, 10), orchestrationInvokerContext(), null, null); +})->throws(TimeoutException::class); + +it('normalizes negative annotation defaults to "none"', function (): void { + expect(StepOptions::of(-1, -1, -1))->toEqual(new StepOptions) + ->and(StepOptions::fromArray(StepOptions::of(2, 5, 7, true, 0.25, 'k')->toArray()))->toEqual(new StepOptions(2, 5, 7, true, 0.25, 'k')) + ->and(StepOptions::of(1, 1, 1, false, 0.5, 'k')->withoutIdempotency()->idempotencyKey)->toBe(''); +}); diff --git a/packages/orchestration/tests/Fixtures/Arguments/ArgumentProbe.php b/packages/orchestration/tests/Fixtures/Arguments/ArgumentProbe.php new file mode 100644 index 00000000..6bfc464a --- /dev/null +++ b/packages/orchestration/tests/Fixtures/Arguments/ArgumentProbe.php @@ -0,0 +1,50 @@ + */ + public array $seen = []; + + /** + * @param array $headers + * @param array $variables + */ + public function everything( + mixed $implicit, + ExecutionContext $context, + mixed $whole, + mixed $amount, + mixed $reserved, + mixed $undo, + ?Throwable $undoError, + mixed $hold, + ?string $tenant, + array $headers, + mixed $tier, + array $variables, + string $correlationId, + string $optional = 'default', + ): string { + $this->seen = [ + 'implicit' => $implicit, 'context' => $context->correlationId, 'whole' => $whole, 'amount' => $amount, + 'reserved' => $reserved, 'undo' => $undo, 'undoError' => $undoError?->getMessage(), 'hold' => $hold, + 'tenant' => $tenant, 'headers' => $headers, 'tier' => $tier, 'variables' => $variables, + 'correlationId' => $correlationId, 'optional' => $optional, + ]; + + return 'probed'; + } + + public function required(mixed $value): string + { + return 'never'; + } +} diff --git a/packages/orchestration/tests/Support/RecordingLogger.php b/packages/orchestration/tests/Support/RecordingLogger.php new file mode 100644 index 00000000..21998afd --- /dev/null +++ b/packages/orchestration/tests/Support/RecordingLogger.php @@ -0,0 +1,32 @@ + */ + public array $records = []; + + /** + * @param array $context + */ + public function log($level, string|Stringable $message, array $context = []): void + { + $this->records[] = ['level' => is_string($level) ? $level : 'unknown', 'message' => (string) $message]; + } + + /** @return list */ + public function messages(string $level): array + { + return array_values(array_map( + static fn (array $record): string => $record['message'], + array_filter($this->records, static fn (array $record): bool => $record['level'] === $level), + )); + } +} From 259c462a4958ab2e34751397601284d8547082f8 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 19:20:01 -0700 Subject: [PATCH 07/58] feat(orchestration): lifecycle hooks, a metrics seam, EDA lifecycle events and the trigger gateway --- .../Event/CompositeOrchestrationEvents.php | 251 ++++++++++++++++++ .../Event/EdaOrchestrationEventPublisher.php | 37 +++ .../orchestration/src/Event/EventGateway.php | 72 +++++ .../src/Event/LoggerOrchestrationEvents.php | 80 ++++++ .../src/Event/MetricsOrchestrationEvents.php | 44 +++ .../Event/NoOpOrchestrationEventPublisher.php | 11 + .../src/Event/NoOpOrchestrationMetrics.php | 19 ++ .../src/Event/OrchestrationEvent.php | 96 +++++++ .../src/Event/OrchestrationEventPublisher.php | 14 + .../src/Event/OrchestrationEvents.php | 70 +++++ .../src/Event/OrchestrationEventsAdapter.php | 68 +++++ .../src/Event/OrchestrationMetrics.php | 24 ++ .../CompositeOrchestrationEventsTest.php | 38 +++ .../EdaOrchestrationEventPublisherTest.php | 37 +++ .../tests/Event/EventGatewayTest.php | 25 ++ .../Event/LoggerOrchestrationEventsTest.php | 59 ++++ .../Event/MetricsOrchestrationEventsTest.php | 28 ++ .../Support/RecordingOrchestrationEvents.php | 164 ++++++++++++ .../Support/RecordingOrchestrationMetrics.php | 35 +++ 19 files changed, 1172 insertions(+) create mode 100644 packages/orchestration/src/Event/CompositeOrchestrationEvents.php create mode 100644 packages/orchestration/src/Event/EdaOrchestrationEventPublisher.php create mode 100644 packages/orchestration/src/Event/EventGateway.php create mode 100644 packages/orchestration/src/Event/LoggerOrchestrationEvents.php create mode 100644 packages/orchestration/src/Event/MetricsOrchestrationEvents.php create mode 100644 packages/orchestration/src/Event/NoOpOrchestrationEventPublisher.php create mode 100644 packages/orchestration/src/Event/NoOpOrchestrationMetrics.php create mode 100644 packages/orchestration/src/Event/OrchestrationEvent.php create mode 100644 packages/orchestration/src/Event/OrchestrationEventPublisher.php create mode 100644 packages/orchestration/src/Event/OrchestrationEvents.php create mode 100644 packages/orchestration/src/Event/OrchestrationEventsAdapter.php create mode 100644 packages/orchestration/src/Event/OrchestrationMetrics.php create mode 100644 packages/orchestration/tests/Event/CompositeOrchestrationEventsTest.php create mode 100644 packages/orchestration/tests/Event/EdaOrchestrationEventPublisherTest.php create mode 100644 packages/orchestration/tests/Event/EventGatewayTest.php create mode 100644 packages/orchestration/tests/Event/LoggerOrchestrationEventsTest.php create mode 100644 packages/orchestration/tests/Event/MetricsOrchestrationEventsTest.php create mode 100644 packages/orchestration/tests/Support/RecordingOrchestrationEvents.php create mode 100644 packages/orchestration/tests/Support/RecordingOrchestrationMetrics.php diff --git a/packages/orchestration/src/Event/CompositeOrchestrationEvents.php b/packages/orchestration/src/Event/CompositeOrchestrationEvents.php new file mode 100644 index 00000000..44951ae8 --- /dev/null +++ b/packages/orchestration/src/Event/CompositeOrchestrationEvents.php @@ -0,0 +1,251 @@ +|null */ + private ?array $resolved = null; + + private readonly LoggerInterface $logger; + + /** + * @param list $listeners + * @param (Closure(): iterable)|null $discover + */ + public function __construct( + private readonly array $listeners = [], + private readonly ?Closure $discover = null, + ?LoggerInterface $logger = null, + ) { + $this->logger = $logger ?? new NullLogger; + } + + /** @return list */ + public function listeners(): array + { + if ($this->resolved === null) { + $found = $this->listeners; + foreach ($this->discover === null ? [] : ($this->discover)() as $listener) { + if ($listener instanceof OrchestrationEvents && ! $listener instanceof self && ! in_array($listener, $found, true)) { + $found[] = $listener; + } + } + $this->resolved = $found; + } + + return $this->resolved; + } + + /** + * @param Closure(OrchestrationEvents): void $call + */ + private function each(string $hook, Closure $call): void + { + foreach ($this->listeners() as $listener) { + try { + $call($listener); + } catch (Throwable $error) { + $this->logger->warning("Orchestration hook {$hook} on ".$listener::class." failed: {$error->getMessage()}", ['exception' => $error]); + } + } + } + + public function onStart(string $name, string $correlationId, ExecutionPattern $pattern): void + { + $this->each(__FUNCTION__, static function (OrchestrationEvents $listener) use ($name, $correlationId, $pattern): void { + $listener->onStart($name, $correlationId, $pattern); + }); + } + + public function onStepStarted(string $name, string $correlationId, string $stepId): void + { + $this->each(__FUNCTION__, static function (OrchestrationEvents $listener) use ($name, $correlationId, $stepId): void { + $listener->onStepStarted($name, $correlationId, $stepId); + }); + } + + public function onStepSuccess(string $name, string $correlationId, string $stepId, int $attempts, int $latencyMs): void + { + $this->each(__FUNCTION__, static function (OrchestrationEvents $listener) use ($name, $correlationId, $stepId, $attempts, $latencyMs): void { + $listener->onStepSuccess($name, $correlationId, $stepId, $attempts, $latencyMs); + }); + } + + public function onStepFailed(string $name, string $correlationId, string $stepId, Throwable $error, int $attempts): void + { + $this->each(__FUNCTION__, static function (OrchestrationEvents $listener) use ($name, $correlationId, $stepId, $error, $attempts): void { + $listener->onStepFailed($name, $correlationId, $stepId, $error, $attempts); + }); + } + + public function onStepSkipped(string $name, string $correlationId, string $stepId): void + { + $this->each(__FUNCTION__, static function (OrchestrationEvents $listener) use ($name, $correlationId, $stepId): void { + $listener->onStepSkipped($name, $correlationId, $stepId); + }); + } + + public function onStepSkippedIdempotent(string $name, string $correlationId, string $stepId): void + { + $this->each(__FUNCTION__, static function (OrchestrationEvents $listener) use ($name, $correlationId, $stepId): void { + $listener->onStepSkippedIdempotent($name, $correlationId, $stepId); + }); + } + + public function onCompleted(string $name, string $correlationId, ExecutionPattern $pattern, bool $success, int $durationMs): void + { + $this->each(__FUNCTION__, static function (OrchestrationEvents $listener) use ($name, $correlationId, $pattern, $success, $durationMs): void { + $listener->onCompleted($name, $correlationId, $pattern, $success, $durationMs); + }); + } + + public function onCompensationStarted(string $name, string $correlationId): void + { + $this->each(__FUNCTION__, static function (OrchestrationEvents $listener) use ($name, $correlationId): void { + $listener->onCompensationStarted($name, $correlationId); + }); + } + + public function onStepCompensated(string $name, string $correlationId, string $stepId): void + { + $this->each(__FUNCTION__, static function (OrchestrationEvents $listener) use ($name, $correlationId, $stepId): void { + $listener->onStepCompensated($name, $correlationId, $stepId); + }); + } + + public function onStepCompensationFailed(string $name, string $correlationId, string $stepId, Throwable $error): void + { + $this->each(__FUNCTION__, static function (OrchestrationEvents $listener) use ($name, $correlationId, $stepId, $error): void { + $listener->onStepCompensationFailed($name, $correlationId, $stepId, $error); + }); + } + + public function onPhaseStarted(string $name, string $correlationId, TccPhase $phase): void + { + $this->each(__FUNCTION__, static function (OrchestrationEvents $listener) use ($name, $correlationId, $phase): void { + $listener->onPhaseStarted($name, $correlationId, $phase); + }); + } + + public function onPhaseCompleted(string $name, string $correlationId, TccPhase $phase, int $durationMs): void + { + $this->each(__FUNCTION__, static function (OrchestrationEvents $listener) use ($name, $correlationId, $phase, $durationMs): void { + $listener->onPhaseCompleted($name, $correlationId, $phase, $durationMs); + }); + } + + public function onPhaseFailed(string $name, string $correlationId, TccPhase $phase, ?Throwable $error): void + { + $this->each(__FUNCTION__, static function (OrchestrationEvents $listener) use ($name, $correlationId, $phase, $error): void { + $listener->onPhaseFailed($name, $correlationId, $phase, $error); + }); + } + + public function onParticipantStarted(string $name, string $correlationId, string $participantId, TccPhase $phase): void + { + $this->each(__FUNCTION__, static function (OrchestrationEvents $listener) use ($name, $correlationId, $participantId, $phase): void { + $listener->onParticipantStarted($name, $correlationId, $participantId, $phase); + }); + } + + public function onParticipantSuccess(string $name, string $correlationId, string $participantId, TccPhase $phase): void + { + $this->each(__FUNCTION__, static function (OrchestrationEvents $listener) use ($name, $correlationId, $participantId, $phase): void { + $listener->onParticipantSuccess($name, $correlationId, $participantId, $phase); + }); + } + + public function onParticipantFailed(string $name, string $correlationId, string $participantId, TccPhase $phase, Throwable $error): void + { + $this->each(__FUNCTION__, static function (OrchestrationEvents $listener) use ($name, $correlationId, $participantId, $phase, $error): void { + $listener->onParticipantFailed($name, $correlationId, $participantId, $phase, $error); + }); + } + + public function onWorkflowSuspended(string $name, string $correlationId, string $reason): void + { + $this->each(__FUNCTION__, static function (OrchestrationEvents $listener) use ($name, $correlationId, $reason): void { + $listener->onWorkflowSuspended($name, $correlationId, $reason); + }); + } + + public function onWorkflowResumed(string $name, string $correlationId): void + { + $this->each(__FUNCTION__, static function (OrchestrationEvents $listener) use ($name, $correlationId): void { + $listener->onWorkflowResumed($name, $correlationId); + }); + } + + public function onCompositionStarted(string $compositionName, string $correlationId): void + { + $this->each(__FUNCTION__, static function (OrchestrationEvents $listener) use ($compositionName, $correlationId): void { + $listener->onCompositionStarted($compositionName, $correlationId); + }); + } + + public function onCompositionCompleted(string $compositionName, string $correlationId, bool $success): void + { + $this->each(__FUNCTION__, static function (OrchestrationEvents $listener) use ($compositionName, $correlationId, $success): void { + $listener->onCompositionCompleted($compositionName, $correlationId, $success); + }); + } + + public function onSignalDelivered(string $name, string $correlationId, string $signalName): void + { + $this->each(__FUNCTION__, static function (OrchestrationEvents $listener) use ($name, $correlationId, $signalName): void { + $listener->onSignalDelivered($name, $correlationId, $signalName); + }); + } + + public function onTimerFired(string $name, string $correlationId, string $timerId): void + { + $this->each(__FUNCTION__, static function (OrchestrationEvents $listener) use ($name, $correlationId, $timerId): void { + $listener->onTimerFired($name, $correlationId, $timerId); + }); + } + + public function onChildWorkflowStarted(string $parentName, string $parentCorrelationId, string $childWorkflowId, string $childCorrelationId): void + { + $this->each(__FUNCTION__, static function (OrchestrationEvents $listener) use ($parentName, $parentCorrelationId, $childWorkflowId, $childCorrelationId): void { + $listener->onChildWorkflowStarted($parentName, $parentCorrelationId, $childWorkflowId, $childCorrelationId); + }); + } + + public function onChildWorkflowCompleted(string $parentName, string $parentCorrelationId, string $childCorrelationId, bool $success): void + { + $this->each(__FUNCTION__, static function (OrchestrationEvents $listener) use ($parentName, $parentCorrelationId, $childCorrelationId, $success): void { + $listener->onChildWorkflowCompleted($parentName, $parentCorrelationId, $childCorrelationId, $success); + }); + } + + public function onContinueAsNew(string $name, string $oldCorrelationId, string $newCorrelationId): void + { + $this->each(__FUNCTION__, static function (OrchestrationEvents $listener) use ($name, $oldCorrelationId, $newCorrelationId): void { + $listener->onContinueAsNew($name, $oldCorrelationId, $newCorrelationId); + }); + } + + public function onDeadLettered(string $name, string $correlationId, ?string $stepId, string $error): void + { + $this->each(__FUNCTION__, static function (OrchestrationEvents $listener) use ($name, $correlationId, $stepId, $error): void { + $listener->onDeadLettered($name, $correlationId, $stepId, $error); + }); + } +} diff --git a/packages/orchestration/src/Event/EdaOrchestrationEventPublisher.php b/packages/orchestration/src/Event/EdaOrchestrationEventPublisher.php new file mode 100644 index 00000000..650a067c --- /dev/null +++ b/packages/orchestration/src/Event/EdaOrchestrationEventPublisher.php @@ -0,0 +1,37 @@ +` (`orchestration.execution_completed`); headers carry the correlation id, + * the pattern and, for keyed step events, the key a partitioned broker routes by. + */ +final class EdaOrchestrationEventPublisher implements OrchestrationEventPublisher +{ + public function __construct( + private readonly EventPublisher $publisher, + private readonly string $defaultTopic = '', + ) {} + + public function publish(OrchestrationEvent $event): void + { + $headers = ['correlationId' => $event->correlationId, 'pattern' => $event->pattern->value]; + if ($event->key !== null) { + $headers['key'] = $event->key; + } + + $this->publisher->publish( + $event->topic ?? ($this->defaultTopic !== '' ? $this->defaultTopic : 'orchestration'), + $event->type ?? 'orchestration.'.strtolower($event->eventType), + $event->toArray(), + $headers, + ); + } +} diff --git a/packages/orchestration/src/Event/EventGateway.php b/packages/orchestration/src/Event/EventGateway.php new file mode 100644 index 00000000..76ec18eb --- /dev/null +++ b/packages/orchestration/src/Event/EventGateway.php @@ -0,0 +1,72 @@ +): mixed}> */ + private array $triggers = []; + + /** + * @param Closure(array): mixed $executor + */ + public function register(string $eventType, string $target, ExecutionPattern $pattern, Closure $executor): void + { + if (isset($this->triggers[$eventType])) { + $existing = $this->triggers[$eventType]; + + throw new ConfigurationException("Trigger event type [{$eventType}] is declared by {$existing['pattern']->value} [{$existing['target']}] and by {$pattern->value} [{$target}]; one event type may start one definition."); + } + + $this->triggers[$eventType] = ['target' => $target, 'pattern' => $pattern, 'executor' => $executor]; + } + + /** + * @param array $payload + */ + public function route(string $eventType, array $payload): mixed + { + $trigger = $this->triggers[$eventType] ?? null; + + return $trigger === null ? null : ($trigger['executor'])($payload); + } + + public function has(string $eventType): bool + { + return isset($this->triggers[$eventType]); + } + + public function count(): int + { + return count($this->triggers); + } + + /** @return list */ + public function eventTypes(): array + { + return array_map('strval', array_keys($this->triggers)); + } + + /** @return list */ + public function describe(): array + { + $rows = []; + foreach ($this->triggers as $eventType => $trigger) { + $rows[] = ['eventType' => (string) $eventType, 'target' => $trigger['target'], 'pattern' => $trigger['pattern']->value]; + } + + return $rows; + } +} diff --git a/packages/orchestration/src/Event/LoggerOrchestrationEvents.php b/packages/orchestration/src/Event/LoggerOrchestrationEvents.php new file mode 100644 index 00000000..321571bd --- /dev/null +++ b/packages/orchestration/src/Event/LoggerOrchestrationEvents.php @@ -0,0 +1,80 @@ +logger->info("Orchestration {$pattern->value} [{$name}] started.", ['correlationId' => $correlationId]); + } + + public function onStepFailed(string $name, string $correlationId, string $stepId, Throwable $error, int $attempts): void + { + $this->logger->warning("Orchestration [{$name}] step [{$stepId}] failed after {$attempts} attempt(s): {$error->getMessage()}", ['correlationId' => $correlationId, 'exception' => $error]); + } + + public function onCompleted(string $name, string $correlationId, ExecutionPattern $pattern, bool $success, int $durationMs): void + { + $message = "Orchestration {$pattern->value} [{$name}] ".($success ? 'completed' : 'failed')." in {$durationMs} ms."; + if ($success) { + $this->logger->info($message, ['correlationId' => $correlationId]); + } else { + $this->logger->warning($message, ['correlationId' => $correlationId]); + } + } + + public function onCompensationStarted(string $name, string $correlationId): void + { + $this->logger->info("Orchestration [{$name}] is compensating.", ['correlationId' => $correlationId]); + } + + public function onStepCompensationFailed(string $name, string $correlationId, string $stepId, Throwable $error): void + { + $this->logger->error("Orchestration [{$name}] could not compensate step [{$stepId}]: {$error->getMessage()}", ['correlationId' => $correlationId, 'exception' => $error]); + } + + public function onPhaseFailed(string $name, string $correlationId, TccPhase $phase, ?Throwable $error): void + { + $this->logger->warning("Orchestration TCC [{$name}] phase {$phase->value} failed".($error === null ? '.' : ": {$error->getMessage()}"), ['correlationId' => $correlationId]); + } + + public function onWorkflowSuspended(string $name, string $correlationId, string $reason): void + { + $this->logger->info("Orchestration workflow [{$name}] suspended".($reason === '' ? '.' : ": {$reason}"), ['correlationId' => $correlationId]); + } + + public function onWorkflowResumed(string $name, string $correlationId): void + { + $this->logger->info("Orchestration workflow [{$name}] resumed.", ['correlationId' => $correlationId]); + } + + public function onSignalDelivered(string $name, string $correlationId, string $signalName): void + { + $this->logger->info("Orchestration workflow [{$name}] received signal [{$signalName}].", ['correlationId' => $correlationId]); + } + + public function onContinueAsNew(string $name, string $oldCorrelationId, string $newCorrelationId): void + { + $this->logger->info("Orchestration workflow [{$name}] continued as new execution [{$newCorrelationId}].", ['correlationId' => $oldCorrelationId]); + } + + public function onDeadLettered(string $name, string $correlationId, ?string $stepId, string $error): void + { + $this->logger->warning("Orchestration [{$name}]".($stepId === null ? '' : " step [{$stepId}]")." was dead-lettered: {$error}", ['correlationId' => $correlationId]); + } +} diff --git a/packages/orchestration/src/Event/MetricsOrchestrationEvents.php b/packages/orchestration/src/Event/MetricsOrchestrationEvents.php new file mode 100644 index 00000000..54b8b013 --- /dev/null +++ b/packages/orchestration/src/Event/MetricsOrchestrationEvents.php @@ -0,0 +1,44 @@ +metrics->recordStep($name, $stepId, 'success', $attempts, $latencyMs / 1000); + } + + public function onStepFailed(string $name, string $correlationId, string $stepId, Throwable $error, int $attempts): void + { + $this->metrics->recordStep($name, $stepId, 'failure', $attempts, 0.0); + } + + public function onStepCompensated(string $name, string $correlationId, string $stepId): void + { + $this->metrics->recordCompensation($name, $stepId, 'compensated'); + } + + public function onStepCompensationFailed(string $name, string $correlationId, string $stepId, Throwable $error): void + { + $this->metrics->recordCompensation($name, $stepId, 'failed'); + } + + public function onCompleted(string $name, string $correlationId, ExecutionPattern $pattern, bool $success, int $durationMs): void + { + $this->metrics->recordExecution($name, $pattern, $success ? 'success' : 'failure', $durationMs / 1000); + } + + public function onDeadLettered(string $name, string $correlationId, ?string $stepId, string $error): void + { + $this->metrics->recordDeadLetter($name); + } +} diff --git a/packages/orchestration/src/Event/NoOpOrchestrationEventPublisher.php b/packages/orchestration/src/Event/NoOpOrchestrationEventPublisher.php new file mode 100644 index 00000000..8a51afb0 --- /dev/null +++ b/packages/orchestration/src/Event/NoOpOrchestrationEventPublisher.php @@ -0,0 +1,11 @@ + $payload + */ + public function __construct( + public string $eventType, + public string $executionName, + public string $correlationId, + public ExecutionPattern $pattern, + public ?string $stepId, + public ?ExecutionStatus $status, + public array $payload, + public DateTimeImmutable $timestamp, + public ?string $topic = null, + public ?string $type = null, + public ?string $key = null, + ) {} + + public static function executionStarted(string $name, string $correlationId, ExecutionPattern $pattern, DateTimeImmutable $now): self + { + return new self(self::EXECUTION_STARTED, $name, $correlationId, $pattern, null, ExecutionStatus::Running, [], $now); + } + + public static function executionCompleted(string $name, string $correlationId, ExecutionPattern $pattern, ExecutionStatus $status, DateTimeImmutable $now): self + { + return new self(self::EXECUTION_COMPLETED, $name, $correlationId, $pattern, null, $status, [], $now); + } + + public static function stepCompleted(string $name, string $correlationId, ExecutionPattern $pattern, string $stepId, mixed $result, DateTimeImmutable $now): self + { + return new self(self::STEP_COMPLETED, $name, $correlationId, $pattern, $stepId, null, $result === null ? [] : ['result' => StateSerializer::normalize($result, "step {$stepId} result")], $now); + } + + public static function stepFailed(string $name, string $correlationId, ExecutionPattern $pattern, string $stepId, Throwable $error, DateTimeImmutable $now): self + { + return new self(self::STEP_FAILED, $name, $correlationId, $pattern, $stepId, null, ['error' => $error->getMessage(), 'errorType' => $error::class], $now); + } + + public function withTopic(string $topic): self + { + return new self($this->eventType, $this->executionName, $this->correlationId, $this->pattern, $this->stepId, $this->status, $this->payload, $this->timestamp, $topic === '' ? null : $topic, $this->type, $this->key); + } + + public function withType(string $type): self + { + return new self($this->eventType, $this->executionName, $this->correlationId, $this->pattern, $this->stepId, $this->status, $this->payload, $this->timestamp, $this->topic, $type === '' ? null : $type, $this->key); + } + + public function withKey(string $key): self + { + return new self($this->eventType, $this->executionName, $this->correlationId, $this->pattern, $this->stepId, $this->status, $this->payload, $this->timestamp, $this->topic, $this->type, $key === '' ? null : $key); + } + + /** @return array */ + public function toArray(): array + { + return [ + 'eventType' => $this->eventType, + 'executionName' => $this->executionName, + 'correlationId' => $this->correlationId, + 'pattern' => $this->pattern->value, + 'stepId' => $this->stepId, + 'status' => $this->status?->value, + 'payload' => $this->payload, + 'timestamp' => ExecutionState::instant($this->timestamp), + 'key' => $this->key, + ]; + } +} diff --git a/packages/orchestration/src/Event/OrchestrationEventPublisher.php b/packages/orchestration/src/Event/OrchestrationEventPublisher.php new file mode 100644 index 00000000..430a0be7 --- /dev/null +++ b/packages/orchestration/src/Event/OrchestrationEventPublisher.php @@ -0,0 +1,14 @@ +onStart('order', 'c-1', ExecutionPattern::Saga); + $composite->onStepCompensated('order', 'c-1', 'reserve'); + + expect($first->hooks())->toBe(['onStart', 'onStepCompensated']) + ->and($discovered->hooks())->toBe(['onStart', 'onStepCompensated']) + ->and($lookups)->toBe(1) + ->and($composite->listeners())->toHaveCount(3) + ->and($logger->messages('warning')[0] ?? '')->toContain('listener down'); +}); diff --git a/packages/orchestration/tests/Event/EdaOrchestrationEventPublisherTest.php b/packages/orchestration/tests/Event/EdaOrchestrationEventPublisherTest.php new file mode 100644 index 00000000..2b2b0746 --- /dev/null +++ b/packages/orchestration/tests/Event/EdaOrchestrationEventPublisherTest.php @@ -0,0 +1,37 @@ +publish(OrchestrationEvent::executionCompleted('order', 'c-1', ExecutionPattern::Saga, ExecutionStatus::Completed, $now)); + + expect($bus->published)->toHaveCount(1) + ->and($bus->published[0]['destination'])->toBe('orchestration') + ->and($bus->published[0]['eventType'])->toBe('orchestration.execution_completed') + ->and($bus->published[0]['headers'])->toBe(['correlationId' => 'c-1', 'pattern' => 'SAGA']) + ->and($bus->published[0]['payload'])->toMatchArray(['eventType' => 'EXECUTION_COMPLETED', 'executionName' => 'order', 'status' => 'COMPLETED']); +}); + +it('honours a step event topic, type and key, and the configured default topic', function (): void { + $bus = new RecordingEventPublisher; + $now = new DateTimeImmutable; + $publisher = new EdaOrchestrationEventPublisher($bus, 'sagas'); + + $publisher->publish(OrchestrationEvent::stepCompleted('order', 'c-1', ExecutionPattern::Saga, 'charge', ['amount' => 10], $now)->withTopic('payments')->withType('payment.captured')->withKey('order-7')); + $publisher->publish(OrchestrationEvent::executionStarted('order', 'c-2', ExecutionPattern::Saga, $now)); + + expect($bus->published[0]['destination'])->toBe('payments') + ->and($bus->published[0]['eventType'])->toBe('payment.captured') + ->and($bus->published[0]['headers']['key'] ?? null)->toBe('order-7') + ->and($bus->published[0]['payload']['payload'] ?? null)->toBe(['result' => ['amount' => 10]]) + ->and($bus->published[1]['destination'])->toBe('sagas'); +}); diff --git a/packages/orchestration/tests/Event/EventGatewayTest.php b/packages/orchestration/tests/Event/EventGatewayTest.php new file mode 100644 index 00000000..e515245b --- /dev/null +++ b/packages/orchestration/tests/Event/EventGatewayTest.php @@ -0,0 +1,25 @@ +register('orders.placed', 'order-saga', ExecutionPattern::Saga, static fn (array $payload): string => 'started '.json_encode($payload)); + + expect($gateway->route('orders.placed', ['id' => 7]))->toBe('started {"id":7}') + ->and($gateway->route('orders.cancelled', []))->toBeNull() + ->and($gateway->has('orders.placed'))->toBeTrue() + ->and($gateway->count())->toBe(1) + ->and($gateway->eventTypes())->toBe(['orders.placed']) + ->and($gateway->describe())->toBe([['eventType' => 'orders.placed', 'target' => 'order-saga', 'pattern' => 'SAGA']]); +}); + +it('refuses two definitions triggered by the same event type', function (): void { + $gateway = new EventGateway; + $gateway->register('orders.placed', 'a', ExecutionPattern::Saga, static fn (array $payload): null => null); + $gateway->register('orders.placed', 'b', ExecutionPattern::Workflow, static fn (array $payload): null => null); +})->throws(ConfigurationException::class, 'orders.placed'); diff --git a/packages/orchestration/tests/Event/LoggerOrchestrationEventsTest.php b/packages/orchestration/tests/Event/LoggerOrchestrationEventsTest.php new file mode 100644 index 00000000..e2ee26b3 --- /dev/null +++ b/packages/orchestration/tests/Event/LoggerOrchestrationEventsTest.php @@ -0,0 +1,59 @@ +onStart('order', 'c-1', ExecutionPattern::Saga); + $events->onStepFailed('order', 'c-1', 'charge', new RuntimeException('card declined'), 3); + $events->onCompensationStarted('order', 'c-1'); + $events->onStepCompensationFailed('order', 'c-1', 'reserve', new RuntimeException('stock service down')); + $events->onPhaseFailed('transfer', 'c-2', TccPhase::Confirm, null); + $events->onCompleted('order', 'c-1', ExecutionPattern::Saga, false, 42); + $events->onCompleted('ship', 'c-3', ExecutionPattern::Workflow, true, 7); + $events->onDeadLettered('order', 'c-1', null, 'card declined'); + $events->onStepSuccess('order', 'c-1', 'reserve', 1, 5); + + expect($logger->messages('info'))->toBe([ + 'Orchestration SAGA [order] started.', + 'Orchestration [order] is compensating.', + 'Orchestration WORKFLOW [ship] completed in 7 ms.', + ])->and($logger->messages('warning'))->toBe([ + 'Orchestration [order] step [charge] failed after 3 attempt(s): card declined', + 'Orchestration TCC [transfer] phase CONFIRM failed.', + 'Orchestration SAGA [order] failed in 42 ms.', + 'Orchestration [order] was dead-lettered: card declined', + ])->and($logger->messages('error'))->toBe([ + 'Orchestration [order] could not compensate step [reserve]: stock service down', + ]); +}); + +it('ships no-op defaults for metrics and lifecycle publishing', function (): void { + expect(static function (): void { + (new NoOpOrchestrationMetrics)->recordExecution('order', ExecutionPattern::Saga, 'success', 0.1); + (new NoOpOrchestrationEventPublisher)->publish(OrchestrationEvent::executionStarted('order', 'c-1', ExecutionPattern::Saga, new DateTimeImmutable)); + })->not->toThrow(Throwable::class); +}); + +it('carries a normalized step failure payload and the UTC instant in its array form', function (): void { + $event = OrchestrationEvent::stepFailed('order', 'c-1', ExecutionPattern::Saga, 'charge', new LogicException('nope'), new DateTimeImmutable('2026-10-12T12:00:00+02:00')); + + expect($event->toArray())->toMatchArray([ + 'eventType' => 'STEP_FAILED', + 'stepId' => 'charge', + 'status' => null, + 'payload' => ['error' => 'nope', 'errorType' => LogicException::class], + 'timestamp' => '2026-10-12T10:00:00.000000+00:00', + 'key' => null, + ])->and($event->withTopic('')->topic)->toBeNull(); +}); diff --git a/packages/orchestration/tests/Event/MetricsOrchestrationEventsTest.php b/packages/orchestration/tests/Event/MetricsOrchestrationEventsTest.php new file mode 100644 index 00000000..e6e3d28f --- /dev/null +++ b/packages/orchestration/tests/Event/MetricsOrchestrationEventsTest.php @@ -0,0 +1,28 @@ +onStepSuccess('order', 'c', 'reserve', 2, 15); + $events->onStepFailed('order', 'c', 'charge', new RuntimeException('x'), 3); + $events->onStepCompensated('order', 'c', 'reserve'); + $events->onStepCompensationFailed('order', 'c', 'reserve', new RuntimeException('y')); + $events->onCompleted('order', 'c', ExecutionPattern::Saga, false, 120); + $events->onDeadLettered('order', 'c', 'charge', 'x'); + + expect($metrics->records)->toBe([ + 'step order.reserve success x2', + 'step order.charge failure x3', + 'compensation order.reserve compensated', + 'compensation order.reserve failed', + 'execution order SAGA failure', + 'dead-letter order', + ]); +}); diff --git a/packages/orchestration/tests/Support/RecordingOrchestrationEvents.php b/packages/orchestration/tests/Support/RecordingOrchestrationEvents.php new file mode 100644 index 00000000..cdba40cf --- /dev/null +++ b/packages/orchestration/tests/Support/RecordingOrchestrationEvents.php @@ -0,0 +1,164 @@ +}> */ + public array $calls = []; + + /** @return list */ + public function hooks(): array + { + return array_map(static fn (array $call): string => $call['hook'], $this->calls); + } + + public function count(string $hook): int + { + return count(array_filter($this->calls, static fn (array $call): bool => $call['hook'] === $hook)); + } + + /** @param list $args */ + private function record(string $hook, array $args): void + { + $this->calls[] = ['hook' => $hook, 'args' => $args]; + } + + public function onStart(string $name, string $correlationId, ExecutionPattern $pattern): void + { + $this->record(__FUNCTION__, [$name, $correlationId, $pattern]); + } + + public function onStepStarted(string $name, string $correlationId, string $stepId): void + { + $this->record(__FUNCTION__, [$name, $correlationId, $stepId]); + } + + public function onStepSuccess(string $name, string $correlationId, string $stepId, int $attempts, int $latencyMs): void + { + $this->record(__FUNCTION__, [$name, $correlationId, $stepId, $attempts, $latencyMs]); + } + + public function onStepFailed(string $name, string $correlationId, string $stepId, Throwable $error, int $attempts): void + { + $this->record(__FUNCTION__, [$name, $correlationId, $stepId, $error->getMessage(), $attempts]); + } + + public function onStepSkipped(string $name, string $correlationId, string $stepId): void + { + $this->record(__FUNCTION__, [$name, $correlationId, $stepId]); + } + + public function onStepSkippedIdempotent(string $name, string $correlationId, string $stepId): void + { + $this->record(__FUNCTION__, [$name, $correlationId, $stepId]); + } + + public function onCompleted(string $name, string $correlationId, ExecutionPattern $pattern, bool $success, int $durationMs): void + { + $this->record(__FUNCTION__, [$name, $correlationId, $pattern, $success]); + } + + public function onCompensationStarted(string $name, string $correlationId): void + { + $this->record(__FUNCTION__, [$name, $correlationId]); + } + + public function onStepCompensated(string $name, string $correlationId, string $stepId): void + { + $this->record(__FUNCTION__, [$name, $correlationId, $stepId]); + } + + public function onStepCompensationFailed(string $name, string $correlationId, string $stepId, Throwable $error): void + { + $this->record(__FUNCTION__, [$name, $correlationId, $stepId, $error->getMessage()]); + } + + public function onPhaseStarted(string $name, string $correlationId, TccPhase $phase): void + { + $this->record(__FUNCTION__, [$name, $correlationId, $phase]); + } + + public function onPhaseCompleted(string $name, string $correlationId, TccPhase $phase, int $durationMs): void + { + $this->record(__FUNCTION__, [$name, $correlationId, $phase]); + } + + public function onPhaseFailed(string $name, string $correlationId, TccPhase $phase, ?Throwable $error): void + { + $this->record(__FUNCTION__, [$name, $correlationId, $phase, $error?->getMessage()]); + } + + public function onParticipantStarted(string $name, string $correlationId, string $participantId, TccPhase $phase): void + { + $this->record(__FUNCTION__, [$name, $correlationId, $participantId, $phase]); + } + + public function onParticipantSuccess(string $name, string $correlationId, string $participantId, TccPhase $phase): void + { + $this->record(__FUNCTION__, [$name, $correlationId, $participantId, $phase]); + } + + public function onParticipantFailed(string $name, string $correlationId, string $participantId, TccPhase $phase, Throwable $error): void + { + $this->record(__FUNCTION__, [$name, $correlationId, $participantId, $phase, $error->getMessage()]); + } + + public function onWorkflowSuspended(string $name, string $correlationId, string $reason): void + { + $this->record(__FUNCTION__, [$name, $correlationId, $reason]); + } + + public function onWorkflowResumed(string $name, string $correlationId): void + { + $this->record(__FUNCTION__, [$name, $correlationId]); + } + + public function onCompositionStarted(string $compositionName, string $correlationId): void + { + $this->record(__FUNCTION__, [$compositionName, $correlationId]); + } + + public function onCompositionCompleted(string $compositionName, string $correlationId, bool $success): void + { + $this->record(__FUNCTION__, [$compositionName, $correlationId, $success]); + } + + public function onSignalDelivered(string $name, string $correlationId, string $signalName): void + { + $this->record(__FUNCTION__, [$name, $correlationId, $signalName]); + } + + public function onTimerFired(string $name, string $correlationId, string $timerId): void + { + $this->record(__FUNCTION__, [$name, $correlationId, $timerId]); + } + + public function onChildWorkflowStarted(string $parentName, string $parentCorrelationId, string $childWorkflowId, string $childCorrelationId): void + { + $this->record(__FUNCTION__, [$parentName, $parentCorrelationId, $childWorkflowId, $childCorrelationId]); + } + + public function onChildWorkflowCompleted(string $parentName, string $parentCorrelationId, string $childCorrelationId, bool $success): void + { + $this->record(__FUNCTION__, [$parentName, $parentCorrelationId, $childCorrelationId, $success]); + } + + public function onContinueAsNew(string $name, string $oldCorrelationId, string $newCorrelationId): void + { + $this->record(__FUNCTION__, [$name, $oldCorrelationId, $newCorrelationId]); + } + + public function onDeadLettered(string $name, string $correlationId, ?string $stepId, string $error): void + { + $this->record(__FUNCTION__, [$name, $correlationId, $stepId, $error]); + } +} diff --git a/packages/orchestration/tests/Support/RecordingOrchestrationMetrics.php b/packages/orchestration/tests/Support/RecordingOrchestrationMetrics.php new file mode 100644 index 00000000..6e075e16 --- /dev/null +++ b/packages/orchestration/tests/Support/RecordingOrchestrationMetrics.php @@ -0,0 +1,35 @@ + */ + public array $records = []; + + public function recordExecution(string $name, ExecutionPattern $pattern, string $outcome, float $seconds): void + { + $this->records[] = "execution {$name} {$pattern->value} {$outcome}"; + } + + public function recordStep(string $name, string $stepId, string $outcome, int $attempts, float $seconds): void + { + $this->records[] = "step {$name}.{$stepId} {$outcome} x{$attempts}"; + } + + public function recordCompensation(string $name, string $stepId, string $outcome): void + { + $this->records[] = "compensation {$name}.{$stepId} {$outcome}"; + } + + public function recordDeadLetter(string $name): void + { + $this->records[] = "dead-letter {$name}"; + } +} From 1136e9e5fbe7d06908e33ab6776125b8c60cde02 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 19:22:14 -0700 Subject: [PATCH 08/58] fix(orchestration): stamp the EDA x-correlation-id and partition_key headers on lifecycle events --- .../Event/EdaOrchestrationEventPublisher.php | 16 +++++++++++++--- .../EdaOrchestrationEventPublisherTest.php | 17 ++++++++++++++++- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/packages/orchestration/src/Event/EdaOrchestrationEventPublisher.php b/packages/orchestration/src/Event/EdaOrchestrationEventPublisher.php index 650a067c..5855a33a 100644 --- a/packages/orchestration/src/Event/EdaOrchestrationEventPublisher.php +++ b/packages/orchestration/src/Event/EdaOrchestrationEventPublisher.php @@ -10,8 +10,13 @@ * Publishes OrchestrationEvents through firefly/eda, so they reach whatever broker the application configured * (memory, queue, the database outbox, RabbitMQ, Kafka). The destination is the event's topic, else * `firefly.orchestration.events.default-topic`, else `orchestration`; the event type is the event's own type, - * else `orchestration.` (`orchestration.execution_completed`); headers carry the correlation id, - * the pattern and, for keyed step events, the key a partitioned broker routes by. + * else `orchestration.` (`orchestration.execution_completed`). + * + * Headers follow the framework's EDA convention, the one EdaCommandEventPublisher stamps and the transports + * read: `x-correlation-id` carries the correlation id (the Postgres outbox takes its transaction id from it, + * and Kafka partitions by it when no key is set), and a keyed step event adds `partition_key`, which Kafka + * routes by first, so every event for one key lands on one partition in order. The `correlationId`, `key` + * and `pattern` headers ride alongside (jafly parity). */ final class EdaOrchestrationEventPublisher implements OrchestrationEventPublisher { @@ -22,8 +27,13 @@ public function __construct( public function publish(OrchestrationEvent $event): void { - $headers = ['correlationId' => $event->correlationId, 'pattern' => $event->pattern->value]; + $headers = [ + 'x-correlation-id' => $event->correlationId, + 'correlationId' => $event->correlationId, + 'pattern' => $event->pattern->value, + ]; if ($event->key !== null) { + $headers['partition_key'] = $event->key; $headers['key'] = $event->key; } diff --git a/packages/orchestration/tests/Event/EdaOrchestrationEventPublisherTest.php b/packages/orchestration/tests/Event/EdaOrchestrationEventPublisherTest.php index 2b2b0746..687af682 100644 --- a/packages/orchestration/tests/Event/EdaOrchestrationEventPublisherTest.php +++ b/packages/orchestration/tests/Event/EdaOrchestrationEventPublisherTest.php @@ -2,6 +2,7 @@ declare(strict_types=1); +use Firefly\Eda\Kafka\KafkaEventPublisher; use Firefly\Orchestration\Core\ExecutionPattern; use Firefly\Orchestration\Core\ExecutionStatus; use Firefly\Orchestration\Event\EdaOrchestrationEventPublisher; @@ -17,7 +18,7 @@ expect($bus->published)->toHaveCount(1) ->and($bus->published[0]['destination'])->toBe('orchestration') ->and($bus->published[0]['eventType'])->toBe('orchestration.execution_completed') - ->and($bus->published[0]['headers'])->toBe(['correlationId' => 'c-1', 'pattern' => 'SAGA']) + ->and($bus->published[0]['headers'])->toBe(['x-correlation-id' => 'c-1', 'correlationId' => 'c-1', 'pattern' => 'SAGA']) ->and($bus->published[0]['payload'])->toMatchArray(['eventType' => 'EXECUTION_COMPLETED', 'executionName' => 'order', 'status' => 'COMPLETED']); }); @@ -31,7 +32,21 @@ expect($bus->published[0]['destination'])->toBe('payments') ->and($bus->published[0]['eventType'])->toBe('payment.captured') + ->and($bus->published[0]['headers']['partition_key'] ?? null)->toBe('order-7') + ->and($bus->published[0]['headers']['x-correlation-id'] ?? null)->toBe('c-1') ->and($bus->published[0]['headers']['key'] ?? null)->toBe('order-7') ->and($bus->published[0]['payload']['payload'] ?? null)->toBe(['result' => ['amount' => 10]]) ->and($bus->published[1]['destination'])->toBe('sagas'); }); + +it('stamps the headers the EDA transports route by, so Kafka partitions a keyed step event by its key', function (): void { + $bus = new RecordingEventPublisher; + $now = new DateTimeImmutable; + $publisher = new EdaOrchestrationEventPublisher($bus); + + $publisher->publish(OrchestrationEvent::stepCompleted('order', 'c-1', ExecutionPattern::Saga, 'charge', null, $now)->withKey('order-7')); + $publisher->publish(OrchestrationEvent::executionStarted('order', 'c-2', ExecutionPattern::Saga, $now)); + + expect(KafkaEventPublisher::partitionKey($bus->published[0]['eventType'], $bus->published[0]['headers']))->toBe('order-7') + ->and(KafkaEventPublisher::partitionKey($bus->published[1]['eventType'], $bus->published[1]['headers']))->toBe('c-2'); +}); From d2a80ad5262505ef48d535dd7574a71498fefd6d Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 19:25:40 -0700 Subject: [PATCH 09/58] feat(orchestration): the execution persistence port with memory and cache providers, and a re-entrant execution lock --- .../orchestration/src/Core/ExecutionLock.php | 73 +++++++ .../CacheExecutionPersistenceProvider.php | 196 ++++++++++++++++++ .../ExecutionPersistenceProvider.php | 56 +++++ .../MemoryExecutionPersistenceProvider.php | 142 +++++++++++++ .../tests/Core/ExecutionLockTest.php | 31 +++ .../CacheExecutionPersistenceProviderTest.php | 26 +++ ...MemoryExecutionPersistenceProviderTest.php | 12 ++ .../Support/ExecutionPersistenceContract.php | 143 +++++++++++++ 8 files changed, 679 insertions(+) create mode 100644 packages/orchestration/src/Core/ExecutionLock.php create mode 100644 packages/orchestration/src/Persistence/CacheExecutionPersistenceProvider.php create mode 100644 packages/orchestration/src/Persistence/ExecutionPersistenceProvider.php create mode 100644 packages/orchestration/src/Persistence/MemoryExecutionPersistenceProvider.php create mode 100644 packages/orchestration/tests/Core/ExecutionLockTest.php create mode 100644 packages/orchestration/tests/Persistence/CacheExecutionPersistenceProviderTest.php create mode 100644 packages/orchestration/tests/Persistence/MemoryExecutionPersistenceProviderTest.php create mode 100644 packages/orchestration/tests/Support/ExecutionPersistenceContract.php diff --git a/packages/orchestration/src/Core/ExecutionLock.php b/packages/orchestration/src/Core/ExecutionLock.php new file mode 100644 index 00000000..a468e84b --- /dev/null +++ b/packages/orchestration/src/Core/ExecutionLock.php @@ -0,0 +1,73 @@ + */ + private array $held = []; + + public function __construct( + private readonly DistributedLock $lock, + private readonly float $ttlSeconds = 300.0, + private readonly float $waitSeconds = 5.0, + ) {} + + public static function name(string $correlationId): string + { + return 'firefly:orchestration:execution:'.$correlationId; + } + + /** + * @template T + * + * @param Closure(): T $callback + * @return T + */ + public function synchronized(string $correlationId, Closure $callback): mixed + { + $name = self::name($correlationId); + + if (isset($this->held[$name])) { + $this->held[$name]++; + try { + return $callback(); + } finally { + $this->held[$name]--; + } + } + + $deadline = microtime(true) + $this->waitSeconds; + while (! $this->lock->tryAcquire($name, $this->ttlSeconds)) { + if (microtime(true) >= $deadline) { + throw new ExecutionLockedException($correlationId, $this->waitSeconds); + } + usleep(50_000); + } + + $this->held[$name] = 1; + try { + return $callback(); + } finally { + unset($this->held[$name]); + $this->lock->release($name); + } + } +} diff --git a/packages/orchestration/src/Persistence/CacheExecutionPersistenceProvider.php b/packages/orchestration/src/Persistence/CacheExecutionPersistenceProvider.php new file mode 100644 index 00000000..c9bfc763 --- /dev/null +++ b/packages/orchestration/src/Persistence/CacheExecutionPersistenceProvider.php @@ -0,0 +1,196 @@ +key($state->correlationId); + if ($this->ttlSeconds === null) { + $this->cache->forever($key, $state->toArray()); + } else { + $this->cache->put($key, $state->toArray(), $this->ttlSeconds); + } + + $id = $state->correlationId; + $this->rewriteIndex(static fn (array $ids): array => in_array($id, $ids, true) ? $ids : [...$ids, $id]); + } + + public function findById(string $correlationId): ?ExecutionState + { + $row = $this->cache->get($this->key($correlationId)); + + return is_array($row) ? ExecutionState::fromArray($row) : null; + } + + public function find(ExecutionCriteria $criteria): array + { + return MemoryExecutionPersistenceProvider::newestFirst(array_filter($this->all(), static fn (ExecutionState $state): bool => $criteria->matches($state)), $criteria->limit); + } + + public function findInFlight(int $limit = 100): array + { + return MemoryExecutionPersistenceProvider::newestFirst(array_filter($this->all(), static fn (ExecutionState $state): bool => ! $state->isTerminal()), $limit); + } + + public function findStale(DateTimeImmutable $before, int $limit = 100): array + { + return MemoryExecutionPersistenceProvider::newestFirst(array_filter( + $this->all(), + static fn (ExecutionState $state): bool => $state->status->isRecoverable() && $state->updatedAt < $before, + ), $limit); + } + + public function findDue(DateTimeImmutable $now, int $limit = 100): array + { + $memory = new MemoryExecutionPersistenceProvider; + foreach ($this->all() as $state) { + $memory->save($state); + } + + return $memory->findDue($now, $limit); + } + + public function findBySearchAttributes(array $criteria, int $limit = 100): array + { + return MemoryExecutionPersistenceProvider::newestFirst(array_filter( + $this->all(), + static fn (ExecutionState $state): bool => MemoryExecutionPersistenceProvider::hasAttributes($state, $criteria), + ), $limit); + } + + public function updateStatus(string $correlationId, ExecutionStatus $status, DateTimeImmutable $now): void + { + $state = $this->findById($correlationId); + if ($state !== null) { + $this->save($state->withStatus($status, $now)); + } + } + + public function delete(string $correlationId): bool + { + $existed = $this->cache->has($this->key($correlationId)); + $this->cache->forget($this->key($correlationId)); + $this->rewriteIndex(static fn (array $ids): array => array_values(array_filter($ids, static fn (string $id): bool => $id !== $correlationId))); + + return $existed; + } + + public function cleanup(DateTimeImmutable $before): int + { + $removed = 0; + foreach ($this->ids() as $id) { + $state = $this->findById($id); + if ($state === null) { + $this->delete($id); + + continue; + } + if ($state->isTerminal() && $state->updatedAt < $before) { + $this->delete($id); + $removed++; + } + } + + return $removed; + } + + public function countInFlight(): int + { + return count(array_filter($this->all(), static fn (ExecutionState $state): bool => ! $state->isTerminal())); + } + + public function countByStatus(): array + { + $counts = []; + foreach ($this->all() as $state) { + $counts[$state->status->value] = ($counts[$state->status->value] ?? 0) + 1; + } + + return $counts; + } + + public function isHealthy(): bool + { + try { + $probe = $this->prefix.'health'; + $this->cache->put($probe, 'ok', 10); + + return $this->cache->get($probe) === 'ok'; + } catch (Throwable) { + return false; + } + } + + private function key(string $correlationId): string + { + return $this->prefix.'execution:'.$correlationId; + } + + /** @return list */ + private function ids(): array + { + $ids = $this->cache->get($this->prefix.'index', []); + + return is_array($ids) ? array_values(array_filter($ids, 'is_string')) : []; + } + + /** @return list */ + private function all(): array + { + $states = []; + foreach ($this->ids() as $id) { + $state = $this->findById($id); + if ($state !== null) { + $states[] = $state; + } + } + + return $states; + } + + /** + * @param Closure(list): list $change + */ + private function rewriteIndex(Closure $change): void + { + $apply = function () use ($change): void { + $this->cache->forever($this->prefix.'index', $change($this->ids())); + }; + + $store = $this->cache->getStore(); + if ($store instanceof LockProvider) { + $store->lock($this->prefix.'index-lock', 10)->block(5, $apply); + + return; + } + + $apply(); + } +} diff --git a/packages/orchestration/src/Persistence/ExecutionPersistenceProvider.php b/packages/orchestration/src/Persistence/ExecutionPersistenceProvider.php new file mode 100644 index 00000000..165683d3 --- /dev/null +++ b/packages/orchestration/src/Persistence/ExecutionPersistenceProvider.php @@ -0,0 +1,56 @@ + newest update first, then by correlation id */ + public function find(ExecutionCriteria $criteria): array; + + /** @return list every execution whose status is not terminal */ + public function findInFlight(int $limit = 100): array; + + /** @return list recoverable-status executions last updated before $before */ + public function findStale(DateTimeImmutable $before, int $limit = 100): array; + + /** @return list WAITING executions whose wake-up instant is at or before $now, earliest first */ + public function findDue(DateTimeImmutable $now, int $limit = 100): array; + + /** + * @param array $criteria every attribute must equal (ExecutionState::attributeValue form) + * @return list + */ + public function findBySearchAttributes(array $criteria, int $limit = 100): array; + + public function updateStatus(string $correlationId, ExecutionStatus $status, DateTimeImmutable $now): void; + + public function delete(string $correlationId): bool; + + /** Deletes terminal executions last updated before $before; returns how many. */ + public function cleanup(DateTimeImmutable $before): int; + + public function countInFlight(): int; + + /** @return array status value => executions */ + public function countByStatus(): array; + + public function isHealthy(): bool; +} diff --git a/packages/orchestration/src/Persistence/MemoryExecutionPersistenceProvider.php b/packages/orchestration/src/Persistence/MemoryExecutionPersistenceProvider.php new file mode 100644 index 00000000..f31d581d --- /dev/null +++ b/packages/orchestration/src/Persistence/MemoryExecutionPersistenceProvider.php @@ -0,0 +1,142 @@ + */ + private array $states = []; + + public function save(ExecutionState $state): void + { + $this->states[$state->correlationId] = $state; + } + + public function findById(string $correlationId): ?ExecutionState + { + return $this->states[$correlationId] ?? null; + } + + public function find(ExecutionCriteria $criteria): array + { + return self::newestFirst(array_filter($this->states, static fn (ExecutionState $state): bool => $criteria->matches($state)), $criteria->limit); + } + + public function findInFlight(int $limit = 100): array + { + return self::newestFirst(array_filter($this->states, static fn (ExecutionState $state): bool => ! $state->isTerminal()), $limit); + } + + public function findStale(DateTimeImmutable $before, int $limit = 100): array + { + return self::newestFirst(array_filter( + $this->states, + static fn (ExecutionState $state): bool => $state->status->isRecoverable() && $state->updatedAt < $before, + ), $limit); + } + + public function findDue(DateTimeImmutable $now, int $limit = 100): array + { + $due = array_values(array_filter( + $this->states, + static fn (ExecutionState $state): bool => $state->status === ExecutionStatus::Waiting && $state->wakeAt !== null && $state->wakeAt <= $now, + )); + usort($due, static fn (ExecutionState $a, ExecutionState $b): int => [$a->wakeAt, $a->correlationId] <=> [$b->wakeAt, $b->correlationId]); + + return array_slice($due, 0, $limit); + } + + public function findBySearchAttributes(array $criteria, int $limit = 100): array + { + return self::newestFirst(array_filter($this->states, static fn (ExecutionState $state): bool => self::hasAttributes($state, $criteria)), $limit); + } + + public function updateStatus(string $correlationId, ExecutionStatus $status, DateTimeImmutable $now): void + { + $state = $this->states[$correlationId] ?? null; + if ($state !== null) { + $this->states[$correlationId] = $state->withStatus($status, $now); + } + } + + public function delete(string $correlationId): bool + { + if (! isset($this->states[$correlationId])) { + return false; + } + + unset($this->states[$correlationId]); + + return true; + } + + public function cleanup(DateTimeImmutable $before): int + { + $removed = 0; + foreach ($this->states as $id => $state) { + if ($state->isTerminal() && $state->updatedAt < $before) { + unset($this->states[$id]); + $removed++; + } + } + + return $removed; + } + + public function countInFlight(): int + { + return count(array_filter($this->states, static fn (ExecutionState $state): bool => ! $state->isTerminal())); + } + + public function countByStatus(): array + { + $counts = []; + foreach ($this->states as $state) { + $counts[$state->status->value] = ($counts[$state->status->value] ?? 0) + 1; + } + + return $counts; + } + + public function isHealthy(): bool + { + return true; + } + + /** + * @param array $criteria + */ + public static function hasAttributes(ExecutionState $state, array $criteria): bool + { + foreach ($criteria as $key => $value) { + $actual = $state->searchAttributes[$key] ?? null; + if ($actual === null || ExecutionState::attributeValue($actual) !== ExecutionState::attributeValue($value)) { + return false; + } + } + + return true; + } + + /** + * @param array $states + * @return list + */ + public static function newestFirst(array $states, int $limit): array + { + $states = array_values($states); + usort($states, static fn (ExecutionState $a, ExecutionState $b): int => [$b->updatedAt, $a->correlationId] <=> [$a->updatedAt, $b->correlationId]); + + return array_slice($states, 0, max(0, $limit)); + } +} diff --git a/packages/orchestration/tests/Core/ExecutionLockTest.php b/packages/orchestration/tests/Core/ExecutionLockTest.php new file mode 100644 index 00000000..7601b1ee --- /dev/null +++ b/packages/orchestration/tests/Core/ExecutionLockTest.php @@ -0,0 +1,31 @@ +synchronized('c-1', static fn (): string => $lock->synchronized('c-1', static fn (): string => 'nested')); + + expect($result)->toBe('nested') + ->and((new CacheLock($cache))->tryAcquire(ExecutionLock::name('c-1'), 1.0))->toBeTrue(); +}); + +it('refuses a second process while the first holds the execution', function (): void { + $cache = new Repository(new ArrayStore); + $first = new ExecutionLock(new CacheLock($cache), 30.0, 0.1); + $second = new ExecutionLock(new CacheLock($cache), 30.0, 0.1); + + $first->synchronized('c-1', static function () use ($second): void { + expect(fn () => $second->synchronized('c-1', static fn (): null => null))->toThrow(ExecutionLockedException::class, 'c-1'); + }); + + expect($second->synchronized('c-1', static fn (): string => 'free again'))->toBe('free again'); +}); diff --git a/packages/orchestration/tests/Persistence/CacheExecutionPersistenceProviderTest.php b/packages/orchestration/tests/Persistence/CacheExecutionPersistenceProviderTest.php new file mode 100644 index 00000000..b1da8044 --- /dev/null +++ b/packages/orchestration/tests/Persistence/CacheExecutionPersistenceProviderTest.php @@ -0,0 +1,26 @@ + $assert) { + it("cache persistence {$case}", function () use ($assert): void { + $assert(new CacheExecutionPersistenceProvider(new Repository(new ArrayStore), 'test:')); + }); +} + +it('keeps two prefixes apart in one store', function (): void { + $store = new Repository(new ArrayStore); + $first = new CacheExecutionPersistenceProvider($store, 'one:'); + $second = new CacheExecutionPersistenceProvider($store, 'two:'); + + $first->save(ExecutionPersistenceContract::state('c-1')); + + expect($second->findById('c-1'))->toBeNull() + ->and($second->countInFlight())->toBe(0) + ->and($first->countInFlight())->toBe(1); +}); diff --git a/packages/orchestration/tests/Persistence/MemoryExecutionPersistenceProviderTest.php b/packages/orchestration/tests/Persistence/MemoryExecutionPersistenceProviderTest.php new file mode 100644 index 00000000..84dd91a2 --- /dev/null +++ b/packages/orchestration/tests/Persistence/MemoryExecutionPersistenceProviderTest.php @@ -0,0 +1,12 @@ + $assert) { + it("memory persistence {$case}", function () use ($assert): void { + $assert(new MemoryExecutionPersistenceProvider); + }); +} diff --git a/packages/orchestration/tests/Support/ExecutionPersistenceContract.php b/packages/orchestration/tests/Support/ExecutionPersistenceContract.php new file mode 100644 index 00000000..38417312 --- /dev/null +++ b/packages/orchestration/tests/Support/ExecutionPersistenceContract.php @@ -0,0 +1,143 @@ + $overrides + */ + public static function state(string $id, ExecutionPattern $pattern = ExecutionPattern::Saga, ExecutionStatus $status = ExecutionStatus::Running, string $updatedAt = '2026-10-12T10:00:00+00:00', array $overrides = []): ExecutionState + { + return ExecutionState::fromArray(array_merge([ + 'correlationId' => $id, + 'executionName' => 'order', + 'pattern' => $pattern->value, + 'status' => $status->value, + 'startedAt' => '2026-10-12T07:00:00.000000+00:00', + 'updatedAt' => ExecutionState::instant(new DateTimeImmutable($updatedAt)), + 'completedAt' => $status->isTerminal() ? ExecutionState::instant(new DateTimeImmutable($updatedAt)) : null, + 'input' => ['orderId' => $id, 'amount' => 10.0, 'note' => 'café'], + 'stepResults' => ['reserve' => ['hold' => 'h-'.$id]], + 'stepStatuses' => ['reserve' => 'DONE'], + ], $overrides)); + } + + /** + * @param list $states + * @return list + */ + public static function ids(array $states, bool $sorted = false): array + { + $ids = array_map(static fn (ExecutionState $state): string => $state->correlationId, $states); + if ($sorted) { + sort($ids); + } + + return $ids; + } + + /** + * @return array + */ + public static function cases(): array + { + return [ + 'saves, finds and replaces an execution' => static function (ExecutionPersistenceProvider $provider): void { + $state = self::state('c-1', overrides: [ + 'searchAttributes' => ['customer' => 'ada'], + 'timers' => ['t' => '2026-10-12T11:00:00.000000+00:00'], + 'report' => ['ok' => true, 'steps' => ['reserve' => ['attempts' => 1]]], + ]); + $provider->save($state); + + expect($provider->findById('c-1')?->toArray())->toBe($state->toArray()) + ->and($provider->findById('absent'))->toBeNull(); + + $provider->save($state->withStatus(ExecutionStatus::Completed, new DateTimeImmutable('2026-10-12T10:05:00+00:00'))); + + expect($provider->findById('c-1')?->status)->toBe(ExecutionStatus::Completed) + ->and($provider->find(new ExecutionCriteria))->toHaveCount(1); + }, + 'lists by criteria, newest update first, limited' => static function (ExecutionPersistenceProvider $provider): void { + $provider->save(self::state('c-a', ExecutionPattern::Saga, ExecutionStatus::Running, '2026-10-12T10:00:00+00:00')); + $provider->save(self::state('c-b', ExecutionPattern::Saga, ExecutionStatus::Failed, '2026-10-12T10:02:00+00:00', ['executionName' => 'other'])); + $provider->save(self::state('c-c', ExecutionPattern::Tcc, ExecutionStatus::Confirmed, '2026-10-12T10:01:00+00:00')); + $provider->save(self::state('c-d', ExecutionPattern::Workflow, ExecutionStatus::Waiting, '2026-10-12T10:03:00+00:00')); + + expect(self::ids($provider->find(new ExecutionCriteria)))->toBe(['c-d', 'c-b', 'c-c', 'c-a']) + ->and(self::ids($provider->find(new ExecutionCriteria(pattern: ExecutionPattern::Saga))))->toBe(['c-b', 'c-a']) + ->and(self::ids($provider->find(new ExecutionCriteria(status: ExecutionStatus::Failed))))->toBe(['c-b']) + ->and(self::ids($provider->find(new ExecutionCriteria(name: 'other'))))->toBe(['c-b']) + ->and(self::ids($provider->find(new ExecutionCriteria(limit: 2))))->toBe(['c-d', 'c-b']); + }, + 'tells in-flight, stale and due executions apart' => static function (ExecutionPersistenceProvider $provider): void { + $provider->save(self::state('a', ExecutionPattern::Saga, ExecutionStatus::Running, '2026-10-12T09:00:00+00:00')); + $provider->save(self::state('b', ExecutionPattern::Workflow, ExecutionStatus::Waiting, '2026-10-12T09:00:00+00:00', ['wakeAt' => '2026-10-12T10:30:00.000000+00:00'])); + $provider->save(self::state('c', ExecutionPattern::Saga, ExecutionStatus::Completed, '2026-10-12T08:00:00+00:00')); + $provider->save(self::state('d', ExecutionPattern::Saga, ExecutionStatus::Compensating, '2026-10-12T10:00:00+00:00')); + + expect(self::ids($provider->findInFlight(), sorted: true))->toBe(['a', 'b', 'd']) + ->and(self::ids($provider->findStale(new DateTimeImmutable('2026-10-12T09:30:00+00:00'))))->toBe(['a']) + ->and($provider->findDue(new DateTimeImmutable('2026-10-12T10:29:59+00:00')))->toBe([]) + ->and(self::ids($provider->findDue(new DateTimeImmutable('2026-10-12T10:30:00+00:00'))))->toBe(['b']) + ->and($provider->countInFlight())->toBe(3); + + $counts = $provider->countByStatus(); + ksort($counts); + expect($counts)->toBe(['COMPENSATING' => 1, 'COMPLETED' => 1, 'RUNNING' => 1, 'WAITING' => 1]); + }, + 'finds by search attributes in their canonical string form' => static function (ExecutionPersistenceProvider $provider): void { + $provider->save(self::state('x', ExecutionPattern::Workflow, overrides: ['searchAttributes' => ['customer' => 'ada', 'vip' => true]])); + $provider->save(self::state('y', ExecutionPattern::Workflow, overrides: ['searchAttributes' => ['customer' => 'grace', 'vip' => true]])); + $provider->save(self::state('z', ExecutionPattern::Workflow)); + + expect(self::ids($provider->findBySearchAttributes(['customer' => 'ada'])))->toBe(['x']) + ->and($provider->findBySearchAttributes(['customer' => 'ADA']))->toBe([]) + ->and(self::ids($provider->findBySearchAttributes(['vip' => true]), sorted: true))->toBe(['x', 'y']) + ->and(self::ids($provider->findBySearchAttributes(['vip' => 'true', 'customer' => 'grace'])))->toBe(['y']); + + $provider->save(self::state('x', ExecutionPattern::Workflow, overrides: ['searchAttributes' => ['customer' => 'alan']])); + + expect($provider->findBySearchAttributes(['customer' => 'ada']))->toBe([]) + ->and(self::ids($provider->findBySearchAttributes(['customer' => 'alan'])))->toBe(['x']); + }, + 'updates status, deletes and cleans up terminal executions' => static function (ExecutionPersistenceProvider $provider): void { + $provider->save(self::state('a', ExecutionPattern::Saga, ExecutionStatus::Completed, '2026-10-12T08:00:00+00:00')); + $provider->save(self::state('b', ExecutionPattern::Saga, ExecutionStatus::Failed, '2026-10-12T09:00:00+00:00')); + $provider->save(self::state('c', ExecutionPattern::Workflow, ExecutionStatus::Running, '2026-10-12T07:00:00+00:00', ['searchAttributes' => ['k' => 'v']])); + + $provider->updateStatus('c', ExecutionStatus::Suspended, new DateTimeImmutable('2026-10-12T10:00:00+00:00')); + + $suspended = $provider->findById('c'); + + expect($suspended?->status)->toBe(ExecutionStatus::Suspended) + ->and($suspended === null ? null : ExecutionState::instant($suspended->updatedAt))->toBe('2026-10-12T10:00:00.000000+00:00') + ->and($provider->cleanup(new DateTimeImmutable('2026-10-12T08:30:00+00:00')))->toBe(1) + ->and($provider->findById('a'))->toBeNull() + ->and($provider->findById('b'))->not->toBeNull() + ->and($provider->delete('c'))->toBeTrue() + ->and($provider->delete('c'))->toBeFalse() + ->and($provider->findBySearchAttributes(['k' => 'v']))->toBe([]); + }, + 'reports its health' => static function (ExecutionPersistenceProvider $provider): void { + expect($provider->isHealthy())->toBeTrue(); + }, + ]; + } +} From 0f7cfa1d7a7fa374c1bd386d9123fcf592efa5dc Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 19:30:54 -0700 Subject: [PATCH 10/58] feat(orchestration): settings and their reference, execution reports, the dead-letter queue and back-pressure admission --- .../orchestration/src/Core/EngineSupport.php | 90 ++++++++ .../src/Core/ExecutionAdmission.php | 60 ++++++ .../src/Core/FailedExecution.php | 21 ++ .../orchestration/src/Dlq/DeadLetterEntry.php | 105 +++++++++ .../src/Dlq/DeadLetterService.php | 84 ++++++++ .../orchestration/src/Dlq/DeadLetterStore.php | 29 +++ .../src/Dlq/MemoryDeadLetterStore.php | 65 ++++++ .../src/OrchestrationSettings.php | 200 ++++++++++++++++++ .../src/Report/ExecutionReport.php | 66 ++++++ .../src/Report/ExecutionReportBuilder.php | 49 +++++ .../orchestration/src/Report/StepReport.php | 31 +++ .../src/Settings/BackpressureSettings.php | 22 ++ .../src/Settings/DlqSettings.php | 14 ++ .../src/Settings/DurationReader.php | 55 +++++ .../src/Settings/EventsSettings.php | 15 ++ .../src/Settings/LockSettings.php | 14 ++ .../src/Settings/PersistenceSettings.php | 19 ++ .../src/Settings/RecoverySettings.php | 25 +++ .../src/Settings/RestSettings.php | 14 ++ .../src/Settings/SagaSettings.php | 18 ++ .../src/Settings/TccSettings.php | 18 ++ .../src/Settings/WorkflowSettings.php | 17 ++ .../tests/Core/EngineSupportTest.php | 73 +++++++ .../tests/Core/ExecutionAdmissionTest.php | 31 +++ .../tests/Dlq/DeadLetterServiceTest.php | 40 ++++ .../tests/Dlq/MemoryDeadLetterStoreTest.php | 12 ++ .../tests/OrchestrationSettingsTest.php | 81 +++++++ .../Report/ExecutionReportBuilderTest.php | 37 ++++ .../tests/Support/DeadLetterStoreContract.php | 62 ++++++ .../Support/FailingExecutionPersistence.php | 81 +++++++ skeleton/config/firefly.php | 147 +++++++++++++ 31 files changed, 1595 insertions(+) create mode 100644 packages/orchestration/src/Core/EngineSupport.php create mode 100644 packages/orchestration/src/Core/ExecutionAdmission.php create mode 100644 packages/orchestration/src/Core/FailedExecution.php create mode 100644 packages/orchestration/src/Dlq/DeadLetterEntry.php create mode 100644 packages/orchestration/src/Dlq/DeadLetterService.php create mode 100644 packages/orchestration/src/Dlq/DeadLetterStore.php create mode 100644 packages/orchestration/src/Dlq/MemoryDeadLetterStore.php create mode 100644 packages/orchestration/src/OrchestrationSettings.php create mode 100644 packages/orchestration/src/Report/ExecutionReport.php create mode 100644 packages/orchestration/src/Report/ExecutionReportBuilder.php create mode 100644 packages/orchestration/src/Report/StepReport.php create mode 100644 packages/orchestration/src/Settings/BackpressureSettings.php create mode 100644 packages/orchestration/src/Settings/DlqSettings.php create mode 100644 packages/orchestration/src/Settings/DurationReader.php create mode 100644 packages/orchestration/src/Settings/EventsSettings.php create mode 100644 packages/orchestration/src/Settings/LockSettings.php create mode 100644 packages/orchestration/src/Settings/PersistenceSettings.php create mode 100644 packages/orchestration/src/Settings/RecoverySettings.php create mode 100644 packages/orchestration/src/Settings/RestSettings.php create mode 100644 packages/orchestration/src/Settings/SagaSettings.php create mode 100644 packages/orchestration/src/Settings/TccSettings.php create mode 100644 packages/orchestration/src/Settings/WorkflowSettings.php create mode 100644 packages/orchestration/tests/Core/EngineSupportTest.php create mode 100644 packages/orchestration/tests/Core/ExecutionAdmissionTest.php create mode 100644 packages/orchestration/tests/Dlq/DeadLetterServiceTest.php create mode 100644 packages/orchestration/tests/Dlq/MemoryDeadLetterStoreTest.php create mode 100644 packages/orchestration/tests/OrchestrationSettingsTest.php create mode 100644 packages/orchestration/tests/Report/ExecutionReportBuilderTest.php create mode 100644 packages/orchestration/tests/Support/DeadLetterStoreContract.php create mode 100644 packages/orchestration/tests/Support/FailingExecutionPersistence.php diff --git a/packages/orchestration/src/Core/EngineSupport.php b/packages/orchestration/src/Core/EngineSupport.php new file mode 100644 index 00000000..31b31db2 --- /dev/null +++ b/packages/orchestration/src/Core/EngineSupport.php @@ -0,0 +1,90 @@ +clock->now(); + } + + public function checkpoint(ExecutionContext $context, ExecutionStatus $status, ?string $failureReason = null, ?ExecutionReport $report = null): ExecutionState + { + $state = $context->toState($status, $this->now(), $failureReason, $report?->toArray()); + + try { + $this->persistence->save($state); + } catch (Throwable $error) { + $this->logger->warning("Orchestration [{$context->executionName}] could not be checkpointed as {$status->value}: {$error->getMessage()}", [ + 'exception' => $error, + 'correlationId' => $context->correlationId, + ]); + } + + return $state; + } + + public function deadLetter(ExecutionContext $context, ?string $stepId, ExecutionStatus $status, Throwable $error): void + { + $headers = $context->headers(); + $retries = isset($headers[DeadLetterEntry::RETRY_COUNT_HEADER]) ? (int) $headers[DeadLetterEntry::RETRY_COUNT_HEADER] + 1 : 0; + unset($headers[DeadLetterEntry::RETRY_COUNT_HEADER], $headers[DeadLetterEntry::RETRY_OF_HEADER]); + + try { + $this->deadLetters->deadLetter(DeadLetterEntry::create( + $context->executionName, $context->correlationId, $context->pattern, $stepId, $status, $error, + $context->input(), $headers, $this->now(), $retries, + )); + } catch (Throwable $failure) { + $this->logger->warning("Orchestration [{$context->executionName}] could not be dead-lettered: {$failure->getMessage()}", [ + 'exception' => $failure, + 'correlationId' => $context->correlationId, + ]); + } + } + + public function elapsedMs(ExecutionContext $context): int + { + return max(0, (int) round(((float) $this->now()->format('U.u') - (float) $context->startedAt()->format('U.u')) * 1000)); + } +} diff --git a/packages/orchestration/src/Core/ExecutionAdmission.php b/packages/orchestration/src/Core/ExecutionAdmission.php new file mode 100644 index 00000000..dc88b30d --- /dev/null +++ b/packages/orchestration/src/Core/ExecutionAdmission.php @@ -0,0 +1,60 @@ +settings->maxInFlight > 0 && $this->persistence->countInFlight() >= $this->settings->maxInFlight) { + throw new BackpressureException("Orchestration [{$name}] refused: {$this->settings->maxInFlight} executions are already in flight (firefly.orchestration.backpressure.max-in-flight)."); + } + + if ($this->breaker === null) { + return $execution(); + } + + try { + return $this->breaker->call(static function () use ($execution, $succeeded): object { + $result = $execution(); + if (! $succeeded($result)) { + throw new FailedExecution($result); + } + + return $result; + }); + } catch (FailedExecution $failed) { + /** @var T */ + return $failed->result; + } + } +} diff --git a/packages/orchestration/src/Core/FailedExecution.php b/packages/orchestration/src/Core/FailedExecution.php new file mode 100644 index 00000000..ebd33335 --- /dev/null +++ b/packages/orchestration/src/Core/FailedExecution.php @@ -0,0 +1,21 @@ + $input + * @param array $headers + */ + public function __construct( + public string $id, + public string $executionName, + public string $correlationId, + public ExecutionPattern $pattern, + public ?string $stepId, + public ExecutionStatus $statusAtFailure, + public string $errorMessage, + public string $errorType, + public array $input, + public array $headers, + public int $retryCount, + public DateTimeImmutable $createdAt, + public ?DateTimeImmutable $lastRetriedAt = null, + ) {} + + /** + * @param array $input + * @param array $headers + */ + public static function create(string $executionName, string $correlationId, ExecutionPattern $pattern, ?string $stepId, ExecutionStatus $statusAtFailure, Throwable $error, array $input, array $headers, DateTimeImmutable $now, int $retryCount = 0): self + { + /** @var array $normalized */ + $normalized = StateSerializer::normalize($input, 'input'); + + return new self(Str::uuid()->toString(), $executionName, $correlationId, $pattern, $stepId, $statusAtFailure, $error->getMessage(), $error::class, $normalized, $headers, $retryCount, $now); + } + + public function withRetry(DateTimeImmutable $now): self + { + return new self($this->id, $this->executionName, $this->correlationId, $this->pattern, $this->stepId, $this->statusAtFailure, $this->errorMessage, $this->errorType, $this->input, $this->headers, $this->retryCount + 1, $this->createdAt, $now); + } + + /** @return array */ + public function toArray(): array + { + return [ + 'id' => $this->id, + 'executionName' => $this->executionName, + 'correlationId' => $this->correlationId, + 'pattern' => $this->pattern->value, + 'stepId' => $this->stepId, + 'statusAtFailure' => $this->statusAtFailure->value, + 'errorMessage' => $this->errorMessage, + 'errorType' => $this->errorType, + 'input' => $this->input, + 'headers' => $this->headers, + 'retryCount' => $this->retryCount, + 'createdAt' => ExecutionState::instant($this->createdAt), + 'lastRetriedAt' => $this->lastRetriedAt === null ? null : ExecutionState::instant($this->lastRetriedAt), + ]; + } + + /** @param array $row */ + public static function fromArray(array $row): self + { + $retried = Rows::nullableString($row, 'lastRetriedAt'); + + return new self( + id: Rows::string($row, 'id'), + executionName: Rows::string($row, 'executionName'), + correlationId: Rows::string($row, 'correlationId'), + pattern: ExecutionPattern::tryFrom(Rows::string($row, 'pattern')) ?? ExecutionPattern::Saga, + stepId: Rows::nullableString($row, 'stepId'), + statusAtFailure: ExecutionStatus::tryFrom(Rows::string($row, 'statusAtFailure')) ?? ExecutionStatus::Failed, + errorMessage: Rows::string($row, 'errorMessage'), + errorType: Rows::string($row, 'errorType'), + input: Rows::map($row, 'input'), + headers: Rows::stringMap($row, 'headers'), + retryCount: Rows::int($row, 'retryCount'), + createdAt: ExecutionState::parseInstant(Rows::string($row, 'createdAt', 'now')), + lastRetriedAt: $retried === null ? null : ExecutionState::parseInstant($retried), + ); + } +} diff --git a/packages/orchestration/src/Dlq/DeadLetterService.php b/packages/orchestration/src/Dlq/DeadLetterService.php new file mode 100644 index 00000000..3232eb04 --- /dev/null +++ b/packages/orchestration/src/Dlq/DeadLetterService.php @@ -0,0 +1,84 @@ +enabled; + } + + public function deadLetter(DeadLetterEntry $entry): void + { + if (! $this->enabled) { + return; + } + + $this->store->save($entry); + $this->events->onDeadLettered($entry->executionName, $entry->correlationId, $entry->stepId, $entry->errorMessage); + } + + /** @return list */ + public function all(int $limit = 100): array + { + return $this->store->findAll($limit); + } + + public function find(string $id): ?DeadLetterEntry + { + return $this->store->findById($id); + } + + public function get(string $id): DeadLetterEntry + { + return $this->store->findById($id) ?? throw new DeadLetterNotFoundException($id); + } + + /** @return list */ + public function byExecutionName(string $executionName, int $limit = 100): array + { + return $this->store->findByExecutionName($executionName, $limit); + } + + /** @return list */ + public function byCorrelationId(string $correlationId): array + { + return $this->store->findByCorrelationId($correlationId); + } + + public function delete(string $id): bool + { + return $this->store->delete($id); + } + + public function count(): int + { + return $this->store->count(); + } + + public function markRetried(string $id, DateTimeImmutable $now): DeadLetterEntry + { + $retried = $this->get($id)->withRetry($now); + $this->store->save($retried); + + return $retried; + } +} diff --git a/packages/orchestration/src/Dlq/DeadLetterStore.php b/packages/orchestration/src/Dlq/DeadLetterStore.php new file mode 100644 index 00000000..246af18f --- /dev/null +++ b/packages/orchestration/src/Dlq/DeadLetterStore.php @@ -0,0 +1,29 @@ + */ + public function findAll(int $limit = 100): array; + + /** @return list */ + public function findByExecutionName(string $executionName, int $limit = 100): array; + + /** @return list */ + public function findByCorrelationId(string $correlationId): array; + + public function delete(string $id): bool; + + public function count(): int; +} diff --git a/packages/orchestration/src/Dlq/MemoryDeadLetterStore.php b/packages/orchestration/src/Dlq/MemoryDeadLetterStore.php new file mode 100644 index 00000000..23e72fcd --- /dev/null +++ b/packages/orchestration/src/Dlq/MemoryDeadLetterStore.php @@ -0,0 +1,65 @@ + */ + private array $entries = []; + + public function save(DeadLetterEntry $entry): void + { + $this->entries[$entry->id] = $entry; + } + + public function findById(string $id): ?DeadLetterEntry + { + return $this->entries[$id] ?? null; + } + + public function findAll(int $limit = 100): array + { + return self::newestFirst($this->entries, $limit); + } + + public function findByExecutionName(string $executionName, int $limit = 100): array + { + return self::newestFirst(array_filter($this->entries, static fn (DeadLetterEntry $entry): bool => $entry->executionName === $executionName), $limit); + } + + public function findByCorrelationId(string $correlationId): array + { + return self::newestFirst(array_filter($this->entries, static fn (DeadLetterEntry $entry): bool => $entry->correlationId === $correlationId), PHP_INT_MAX); + } + + public function delete(string $id): bool + { + if (! isset($this->entries[$id])) { + return false; + } + + unset($this->entries[$id]); + + return true; + } + + public function count(): int + { + return count($this->entries); + } + + /** + * @param array $entries + * @return list + */ + private static function newestFirst(array $entries, int $limit): array + { + $entries = array_values($entries); + usort($entries, static fn (DeadLetterEntry $a, DeadLetterEntry $b): int => [$b->createdAt, $a->id] <=> [$a->createdAt, $b->id]); + + return array_slice($entries, 0, $limit); + } +} diff --git a/packages/orchestration/src/OrchestrationSettings.php b/packages/orchestration/src/OrchestrationSettings.php new file mode 100644 index 00000000..0693a7b9 --- /dev/null +++ b/packages/orchestration/src/OrchestrationSettings.php @@ -0,0 +1,200 @@ + */ + public const array PERSISTENCE_PROVIDERS = ['memory', 'cache', 'database', 'event-sourced']; + + /** @var list */ + public const array DLQ_STORES = ['memory', 'database']; + + /** @var list */ + public const array RECOVERY_ACTIONS = [RecoverySettings::REPORT, RecoverySettings::RESUME, RecoverySettings::COMPENSATE]; + + /** @var list */ + public const array BACKPRESSURE_STRATEGIES = ['none', 'circuit-breaker']; + + public function __construct( + public SagaSettings $saga, + public TccSettings $tcc, + public WorkflowSettings $workflow, + public PersistenceSettings $persistence, + public RecoverySettings $recovery, + public DlqSettings $dlq, + public EventsSettings $events, + public RestSettings $rest, + public BackpressureSettings $backpressure, + public LockSettings $lock, + public bool $healthEnabled = true, + public bool $metricsEnabled = true, + public bool $tracingEnabled = true, + public bool $validationEnabled = true, + public bool $failOnWarning = false, + ) {} + + public static function defaults(): self + { + return self::fromConfig(new Config(new Repository([]))); + } + + /** A duration in seconds as the string a ScheduledDescriptor takes: 3600.0 → '3600s', 0.5 → '0.5s'. */ + public static function interval(float $seconds): string + { + return $seconds.'s'; + } + + public static function fromConfig(Config $config): self + { + $durations = new DurationReader($config); + + $saga = new SagaSettings( + enabled: $config->bool('firefly.orchestration.saga.enabled', true), + compensationPolicy: CompensationPolicy::fromConfig($config->string('firefly.orchestration.saga.compensation-policy', 'strict-sequential'), 'firefly.orchestration.saga.compensation-policy'), + defaultTimeout: $durations->get('firefly.orchestration.saga.default-timeout', 300.0), + compensationErrorHandler: self::nonEmpty($config->string('firefly.orchestration.saga.compensation-error-handler', 'default'), 'firefly.orchestration.saga.compensation-error-handler'), + ); + + $tcc = new TccSettings( + enabled: $config->bool('firefly.orchestration.tcc.enabled', true), + defaultTimeout: $durations->get('firefly.orchestration.tcc.default-timeout', 30.0), + compositionEnabled: $config->bool('firefly.orchestration.tcc.composition.enabled', true), + compositionCompensationPolicy: CompensationPolicy::fromConfig($config->string('firefly.orchestration.tcc.composition.compensation-policy', 'strict-sequential'), 'firefly.orchestration.tcc.composition.compensation-policy'), + ); + + $workflow = new WorkflowSettings( + enabled: $config->bool('firefly.orchestration.workflow.enabled', true), + queueConnection: self::nullableName($config->get('firefly.orchestration.workflow.queue.connection'), 'firefly.orchestration.workflow.queue.connection'), + queueName: self::nullableName($config->get('firefly.orchestration.workflow.queue.name'), 'firefly.orchestration.workflow.queue.name'), + tickEnabled: $config->bool('firefly.orchestration.workflow.tick.enabled', true), + tickInterval: $durations->get('firefly.orchestration.workflow.tick.interval', 60.0, 'a zero tick interval would schedule the timer sweep continuously'), + ); + + $provider = $config->string('firefly.orchestration.persistence.provider', 'memory'); + if (! in_array($provider, self::PERSISTENCE_PROVIDERS, true)) { + throw new ConfigurationException("Configuration key [firefly.orchestration.persistence.provider] must be one of memory, cache, database or event-sourced, got [{$provider}]."); + } + $keyTtl = $durations->get('firefly.orchestration.persistence.key-ttl', 0.0); + $persistence = new PersistenceSettings( + provider: $provider, + connection: self::nullableName($config->get('firefly.orchestration.persistence.connection'), 'firefly.orchestration.persistence.connection'), + cacheStore: self::nullableName($config->get('firefly.orchestration.persistence.cache-store'), 'firefly.orchestration.persistence.cache-store'), + keyPrefix: $config->string('firefly.orchestration.persistence.key-prefix', 'orchestration:'), + keyTtl: $keyTtl > 0.0 ? (int) ceil($keyTtl) : null, + retentionPeriod: $durations->get('firefly.orchestration.persistence.retention-period', 604800.0, 'a zero retention period would delete every execution the moment it ends'), + cleanupInterval: $durations->get('firefly.orchestration.persistence.cleanup-interval', 3600.0, 'a zero cleanup interval would run the recovery sweep continuously'), + ); + + $action = $config->string('firefly.orchestration.recovery.action', RecoverySettings::REPORT); + if (! in_array($action, self::RECOVERY_ACTIONS, true)) { + throw new ConfigurationException("Configuration key [firefly.orchestration.recovery.action] must be report, resume or compensate, got [{$action}]."); + } + $recovery = new RecoverySettings( + enabled: $config->bool('firefly.orchestration.recovery.enabled', true), + staleThreshold: $durations->get('firefly.orchestration.recovery.stale-threshold', 3600.0), + action: $action, + ); + + $store = $config->string('firefly.orchestration.dlq.store', 'memory'); + if (! in_array($store, self::DLQ_STORES, true)) { + throw new ConfigurationException("Configuration key [firefly.orchestration.dlq.store] must be memory or database, got [{$store}]."); + } + + $path = '/'.trim($config->string('firefly.orchestration.rest.path', '/orchestration'), '/'); + if ($path === '/') { + throw new ConfigurationException('Configuration key [firefly.orchestration.rest.path] must name a path such as /orchestration, not the application root.'); + } + + $strategy = $config->string('firefly.orchestration.backpressure.strategy', 'none'); + if (! in_array($strategy, self::BACKPRESSURE_STRATEGIES, true)) { + throw new ConfigurationException("Configuration key [firefly.orchestration.backpressure.strategy] must be none or circuit-breaker, got [{$strategy}]. jafly's adaptive and batched strategies schedule reactive streams and have no meaning in a synchronous PHP process; use max-in-flight and batch-size instead."); + } + + return new self( + saga: $saga, + tcc: $tcc, + workflow: $workflow, + persistence: $persistence, + recovery: $recovery, + dlq: new DlqSettings($config->bool('firefly.orchestration.dlq.enabled', true), $store), + events: new EventsSettings( + enabled: $config->bool('firefly.orchestration.events.enabled', false), + defaultTopic: $config->string('firefly.orchestration.events.default-topic', ''), + gatewayEnabled: $config->bool('firefly.orchestration.event-gateway.enabled', true), + ), + rest: new RestSettings($config->bool('firefly.orchestration.rest.enabled', false), $path), + backpressure: new BackpressureSettings( + strategy: $strategy, + batchSize: self::atLeast($config->int('firefly.orchestration.backpressure.batch-size', 10), 1, 'firefly.orchestration.backpressure.batch-size'), + maxInFlight: self::atLeast($config->int('firefly.orchestration.backpressure.max-in-flight', 0), 0, 'firefly.orchestration.backpressure.max-in-flight'), + failureThreshold: self::atLeast($config->int('firefly.orchestration.backpressure.circuit-breaker.failure-threshold', 5), 1, 'firefly.orchestration.backpressure.circuit-breaker.failure-threshold'), + recoveryTimeout: $durations->get('firefly.orchestration.backpressure.circuit-breaker.recovery-timeout', 30.0), + halfOpenMaxCalls: self::atLeast($config->int('firefly.orchestration.backpressure.circuit-breaker.half-open-max-calls', 3), 1, 'firefly.orchestration.backpressure.circuit-breaker.half-open-max-calls'), + ), + lock: new LockSettings( + ttl: $durations->get('firefly.orchestration.lock.ttl', 300.0, 'a zero lock lifetime would let a second process in immediately'), + wait: $durations->get('firefly.orchestration.lock.wait', 5.0), + ), + healthEnabled: $config->bool('firefly.orchestration.health.enabled', true), + metricsEnabled: $config->bool('firefly.orchestration.metrics.enabled', true), + tracingEnabled: $config->bool('firefly.orchestration.tracing.enabled', true), + validationEnabled: $config->bool('firefly.orchestration.validation.enabled', true), + failOnWarning: $config->bool('firefly.orchestration.validation.fail-on-warning', false), + ); + } + + private static function nonEmpty(string $value, string $key): string + { + if (trim($value) === '') { + throw new ConfigurationException("Configuration key [{$key}] must not be empty."); + } + + return $value; + } + + private static function nullableName(mixed $value, string $key): ?string + { + if ($value === null || $value === '') { + return null; + } + if (! is_string($value)) { + throw new ConfigurationException("Configuration key [{$key}] must be a name (or null for the default), got ".get_debug_type($value).'.'); + } + + return $value; + } + + private static function atLeast(int $value, int $minimum, string $key): int + { + if ($value < $minimum) { + throw new ConfigurationException("Configuration key [{$key}] must be at least {$minimum}, got [{$value}]."); + } + + return $value; + } +} diff --git a/packages/orchestration/src/Report/ExecutionReport.php b/packages/orchestration/src/Report/ExecutionReport.php new file mode 100644 index 00000000..4d324e41 --- /dev/null +++ b/packages/orchestration/src/Report/ExecutionReport.php @@ -0,0 +1,66 @@ + $steps + * @param list> $layers + */ + public function __construct( + public string $name, + public string $correlationId, + public ExecutionPattern $pattern, + public ExecutionStatus $status, + public DateTimeImmutable $startedAt, + public DateTimeImmutable $completedAt, + public int $durationMs, + public array $steps, + public ?string $failureReason = null, + public array $layers = [], + ) {} + + /** @return list */ + public function failedSteps(): array + { + return array_map('strval', array_keys(array_filter($this->steps, static fn (StepReport $step): bool => $step->status->isFailure()))); + } + + /** @return list */ + public function compensatedSteps(): array + { + return array_map('strval', array_keys(array_filter($this->steps, static fn (StepReport $step): bool => $step->compensated || $step->status === StepStatus::Compensated))); + } + + /** + * @return array{name: string, correlationId: string, pattern: string, status: string, startedAt: string, completedAt: string, durationMs: int, failureReason: ?string, layers: list>, steps: array} + */ + public function toArray(): array + { + return [ + 'name' => $this->name, + 'correlationId' => $this->correlationId, + 'pattern' => $this->pattern->value, + 'status' => $this->status->value, + 'startedAt' => ExecutionState::instant($this->startedAt), + 'completedAt' => ExecutionState::instant($this->completedAt), + 'durationMs' => $this->durationMs, + 'failureReason' => $this->failureReason, + 'layers' => $this->layers, + 'steps' => array_map(static fn (StepReport $step): array => $step->toArray(), $this->steps), + ]; + } +} diff --git a/packages/orchestration/src/Report/ExecutionReportBuilder.php b/packages/orchestration/src/Report/ExecutionReportBuilder.php new file mode 100644 index 00000000..a6b0fd60 --- /dev/null +++ b/packages/orchestration/src/Report/ExecutionReportBuilder.php @@ -0,0 +1,49 @@ + $stepIds + */ + public static function fromContext(ExecutionContext $context, ExecutionStatus $status, ?string $failureReason, DateTimeImmutable $now, array $stepIds): ExecutionReport + { + $steps = []; + foreach ($stepIds as $stepId) { + $stepStatus = $context->stepStatus($stepId); + $steps[$stepId] = new StepReport( + stepId: $stepId, + status: $stepStatus, + attempts: $context->attempts($stepId), + latencyMs: $context->latencyMs($stepId), + error: $context->stepError($stepId)?->getMessage(), + compensated: $stepStatus === StepStatus::Compensated, + compensationError: $context->compensationError($stepId)?->getMessage(), + ); + } + + $duration = (int) round(((float) $now->format('U.u') - (float) $context->startedAt()->format('U.u')) * 1000); + + return new ExecutionReport( + name: $context->executionName, + correlationId: $context->correlationId, + pattern: $context->pattern, + status: $status, + startedAt: $context->startedAt(), + completedAt: $now, + durationMs: max(0, $duration), + steps: $steps, + failureReason: $failureReason, + layers: $context->topologyLayers(), + ); + } +} diff --git a/packages/orchestration/src/Report/StepReport.php b/packages/orchestration/src/Report/StepReport.php new file mode 100644 index 00000000..ac53b2d2 --- /dev/null +++ b/packages/orchestration/src/Report/StepReport.php @@ -0,0 +1,31 @@ + $this->stepId, 'status' => $this->status->value, 'attempts' => $this->attempts, + 'latencyMs' => $this->latencyMs, 'error' => $this->error, 'compensated' => $this->compensated, + 'compensationError' => $this->compensationError, + ]; + } +} diff --git a/packages/orchestration/src/Settings/BackpressureSettings.php b/packages/orchestration/src/Settings/BackpressureSettings.php new file mode 100644 index 00000000..2441768b --- /dev/null +++ b/packages/orchestration/src/Settings/BackpressureSettings.php @@ -0,0 +1,22 @@ +get('firefly.…')`, so every duration key stays + * visible to the reference guard. Accepted: the framework grammar ('250ms', '30s', '5m', '1h', 'P7D') or a + * number of seconds. Null is unset. Negative is refused; zero is refused when the caller says why. + * + * @internal + */ +final readonly class DurationReader +{ + public function __construct(private Config $config) {} + + public function get(string $key, float $default, ?string $refuseZeroBecause = null): float + { + $value = $this->config->get($key); + if ($value === null || $value === '') { + return $default; + } + + if (is_int($value) || is_float($value)) { + $seconds = (float) $value; + } elseif (is_string($value)) { + try { + $seconds = Duration::parse(ltrim($value, '-')); + } catch (ConfigurationException $exception) { + throw new ConfigurationException("Configuration key [{$key}] is not a duration this framework can parse. {$exception->getMessage()}", previous: $exception); + } + if (str_starts_with(trim($value), '-')) { + $seconds = -$seconds; + } + } else { + throw new ConfigurationException("Configuration key [{$key}] must be a duration such as '30s', '5m' or 'PT1H', or a number of seconds, got ".get_debug_type($value).'.'); + } + + if (! is_finite($seconds) || $seconds < 0.0) { + throw new ConfigurationException("Configuration key [{$key}] must not be negative, got [".var_export($value, true).'].'); + } + if ($refuseZeroBecause !== null && $seconds <= 0.0) { + throw new ConfigurationException("Configuration key [{$key}] must be greater than zero: {$refuseZeroBecause}."); + } + + return $seconds; + } +} diff --git a/packages/orchestration/src/Settings/EventsSettings.php b/packages/orchestration/src/Settings/EventsSettings.php new file mode 100644 index 00000000..bd80c385 --- /dev/null +++ b/packages/orchestration/src/Settings/EventsSettings.php @@ -0,0 +1,15 @@ +now()); + + $support->checkpoint($context, ExecutionStatus::Running); + expect($persistence->findById('c-1')?->status)->toBe(ExecutionStatus::Running); + + $broken = orchestrationSupport(new FailingExecutionPersistence, $logger, new MemoryDeadLetterStore); + $state = $broken->checkpoint($context, ExecutionStatus::Completed); + + expect($state->status)->toBe(ExecutionStatus::Completed) + ->and($logger->messages('warning')[0] ?? '')->toContain('could not be checkpointed'); +}); + +it('dead-letters with the retry count carried by the execution headers', function (): void { + $dlq = new MemoryDeadLetterStore; + $support = orchestrationSupport(new MemoryExecutionPersistenceProvider, new RecordingLogger, $dlq); + $context = ExecutionContext::start(ExecutionPattern::Saga, 'order', ['orderId' => 7], [DeadLetterEntry::RETRY_COUNT_HEADER => '2', DeadLetterEntry::RETRY_OF_HEADER => 'd-0', 'tenant' => 'acme'], 'c-1', $support->now()); + + $support->deadLetter($context, 'charge', ExecutionStatus::Failed, new RuntimeException('card declined')); + + $entry = $dlq->findAll()[0]; + expect($entry->retryCount)->toBe(3) + ->and($entry->input)->toBe(['orderId' => 7]) + ->and($entry->headers)->toBe(['tenant' => 'acme']); +}); diff --git a/packages/orchestration/tests/Core/ExecutionAdmissionTest.php b/packages/orchestration/tests/Core/ExecutionAdmissionTest.php new file mode 100644 index 00000000..5385d81a --- /dev/null +++ b/packages/orchestration/tests/Core/ExecutionAdmissionTest.php @@ -0,0 +1,31 @@ +save(ExecutionPersistenceContract::state('a')); + $admission = new ExecutionAdmission($persistence, new BackpressureSettings('none', 10, 1, 5, 30.0, 3)); + + $admission->admit('order', static fn (): stdClass => new stdClass, static fn (stdClass $r): bool => true); +})->throws(BackpressureException::class, 'max-in-flight'); + +it('opens the breaker after failed executions and returns each failed result while closed', function (): void { + $breaker = new CircuitBreaker(key: 'test:backpressure', store: new InMemoryResilienceStore, failureThreshold: 2, waitDurationInOpen: 60.0, halfOpenMaxCalls: 1); + $admission = new ExecutionAdmission(new MemoryExecutionPersistenceProvider, new BackpressureSettings('circuit-breaker', 10, 0, 2, 60.0, 1), $breaker); + $failed = static fn (): ArrayObject => new ArrayObject(['success' => false]); + $succeeded = static fn (ArrayObject $result): bool => $result['success'] === true; + + expect($admission->admit('order', $failed, $succeeded)['success'])->toBeFalse() + ->and($admission->admit('order', $failed, $succeeded)['success'])->toBeFalse() + ->and(fn () => $admission->admit('order', $failed, $succeeded))->toThrow(CircuitBreakerOpenException::class); +}); diff --git a/packages/orchestration/tests/Dlq/DeadLetterServiceTest.php b/packages/orchestration/tests/Dlq/DeadLetterServiceTest.php new file mode 100644 index 00000000..86a3e7a4 --- /dev/null +++ b/packages/orchestration/tests/Dlq/DeadLetterServiceTest.php @@ -0,0 +1,40 @@ +deadLetter(DeadLetterStoreContract::entry('d-1')); + + expect($service->count())->toBe(1) + ->and($service->get('d-1')->errorMessage)->toBe('card declined') + ->and($events->calls[0] ?? null)->toBe(['hook' => 'onDeadLettered', 'args' => ['order', 'c-1', 'charge', 'card declined']]); +}); + +it('drops entries while firefly.orchestration.dlq.enabled is false', function (): void { + $events = new RecordingOrchestrationEvents; + $service = new DeadLetterService(new MemoryDeadLetterStore, $events, enabled: false); + + $service->deadLetter(DeadLetterStoreContract::entry('d-1')); + + expect($service->enabled())->toBeFalse()->and($service->count())->toBe(0)->and($events->calls)->toBe([]); +}); + +it('counts a retry on the entry and refuses an unknown id', function (): void { + $service = new DeadLetterService(new MemoryDeadLetterStore, new RecordingOrchestrationEvents); + $service->deadLetter(DeadLetterStoreContract::entry('d-1')); + + $retried = $service->markRetried('d-1', new DateTimeImmutable('2026-10-12T12:00:00+00:00')); + + expect($retried->retryCount)->toBe(1) + ->and($retried->lastRetriedAt?->format('H:i'))->toBe('12:00') + ->and(fn () => $service->get('nope'))->toThrow(DeadLetterNotFoundException::class); +}); diff --git a/packages/orchestration/tests/Dlq/MemoryDeadLetterStoreTest.php b/packages/orchestration/tests/Dlq/MemoryDeadLetterStoreTest.php new file mode 100644 index 00000000..0c2138a0 --- /dev/null +++ b/packages/orchestration/tests/Dlq/MemoryDeadLetterStoreTest.php @@ -0,0 +1,12 @@ + $assert) { + it("memory dead-letter store {$case}", function () use ($assert): void { + $assert(new MemoryDeadLetterStore); + }); +} diff --git a/packages/orchestration/tests/OrchestrationSettingsTest.php b/packages/orchestration/tests/OrchestrationSettingsTest.php new file mode 100644 index 00000000..e0e3a9ff --- /dev/null +++ b/packages/orchestration/tests/OrchestrationSettingsTest.php @@ -0,0 +1,81 @@ + $orchestration the raw firefly.orchestration block, as a dataset row passes it */ +function orchestrationSettings(array $orchestration): OrchestrationSettings +{ + return OrchestrationSettings::fromConfig(new Config(new Repository(['firefly' => ['orchestration' => $orchestration]]))); +} + +it('defaults to the jafly OrchestrationProperties values a PHP application can honour', function (): void { + $settings = OrchestrationSettings::defaults(); + + expect($settings->saga->enabled)->toBeTrue() + ->and($settings->saga->compensationPolicy)->toBe(CompensationPolicy::StrictSequential) + ->and($settings->saga->defaultTimeout)->toBe(300.0) + ->and($settings->saga->compensationErrorHandler)->toBe('default') + ->and($settings->tcc->defaultTimeout)->toBe(30.0) + ->and($settings->workflow->tickInterval)->toBe(60.0) + ->and($settings->persistence->provider)->toBe('memory') + ->and($settings->persistence->keyPrefix)->toBe('orchestration:') + ->and($settings->persistence->keyTtl)->toBeNull() + ->and($settings->persistence->retentionPeriod)->toBe(604800.0) + ->and($settings->persistence->cleanupInterval)->toBe(3600.0) + ->and($settings->recovery->action)->toBe('report') + ->and($settings->recovery->staleThreshold)->toBe(3600.0) + ->and($settings->dlq->store)->toBe('memory') + ->and($settings->events->enabled)->toBeFalse() + ->and($settings->events->gatewayEnabled)->toBeTrue() + ->and($settings->rest->enabled)->toBeFalse() + ->and($settings->rest->path)->toBe('/orchestration') + ->and($settings->backpressure->strategy)->toBe('none') + ->and($settings->backpressure->batchSize)->toBe(10) + ->and($settings->lock->wait)->toBe(5.0) + ->and(OrchestrationSettings::interval(3600.0))->toBe('3600s'); +}); + +it('reads every override', function (): void { + $settings = orchestrationSettings([ + 'saga' => ['enabled' => false, 'compensation-policy' => 'BEST_EFFORT_PARALLEL', 'default-timeout' => '0', 'compensation-error-handler' => 'robust'], + 'persistence' => ['provider' => 'database', 'connection' => 'pgsql', 'key-ttl' => '1h', 'retention-period' => 'P1D'], + 'recovery' => ['action' => 'compensate', 'stale-threshold' => '15m'], + 'rest' => ['enabled' => true, 'path' => 'ops/orchestration'], + 'backpressure' => ['strategy' => 'circuit-breaker', 'max-in-flight' => 50, 'circuit-breaker' => ['failure-threshold' => 2, 'recovery-timeout' => '5s']], + 'workflow' => ['queue' => ['connection' => 'redis', 'name' => 'workflows']], + ]); + + expect($settings->saga->enabled)->toBeFalse() + ->and($settings->saga->compensationPolicy)->toBe(CompensationPolicy::BestEffort) + ->and($settings->saga->defaultTimeout)->toBe(0.0) + ->and($settings->persistence->connection)->toBe('pgsql') + ->and($settings->persistence->keyTtl)->toBe(3600) + ->and($settings->persistence->retentionPeriod)->toBe(86400.0) + ->and($settings->recovery->action)->toBe('compensate') + ->and($settings->rest->path)->toBe('/ops/orchestration') + ->and($settings->backpressure->maxInFlight)->toBe(50) + ->and($settings->backpressure->failureThreshold)->toBe(2) + ->and($settings->backpressure->recoveryTimeout)->toBe(5.0) + ->and($settings->workflow->queueConnection)->toBe('redis') + ->and($settings->workflow->queueName)->toBe('workflows'); +}); + +it('refuses a value it cannot honour, naming the key', function (array $orchestration, string $key): void { + expect(fn () => orchestrationSettings($orchestration))->toThrow(ConfigurationException::class, $key); +})->with([ + [['persistence' => ['provider' => 'redis']], 'firefly.orchestration.persistence.provider'], + [['dlq' => ['store' => 'cache']], 'firefly.orchestration.dlq.store'], + [['recovery' => ['action' => 'retry']], 'firefly.orchestration.recovery.action'], + [['backpressure' => ['strategy' => 'adaptive']], 'firefly.orchestration.backpressure.strategy'], + [['backpressure' => ['batch-size' => 0]], 'firefly.orchestration.backpressure.batch-size'], + [['saga' => ['default-timeout' => '-5s']], 'firefly.orchestration.saga.default-timeout'], + [['workflow' => ['tick' => ['interval' => '0']]], 'firefly.orchestration.workflow.tick.interval'], + [['rest' => ['path' => '/']], 'firefly.orchestration.rest.path'], + [['saga' => ['compensation-policy' => 'optimistic']], 'firefly.orchestration.saga.compensation-policy'], +]); diff --git a/packages/orchestration/tests/Report/ExecutionReportBuilderTest.php b/packages/orchestration/tests/Report/ExecutionReportBuilderTest.php new file mode 100644 index 00000000..6dcb0d69 --- /dev/null +++ b/packages/orchestration/tests/Report/ExecutionReportBuilderTest.php @@ -0,0 +1,37 @@ +setTopologyLayers([['reserve'], ['charge', 'notify']]); + $context->setStepStatus('reserve', StepStatus::Compensated); + $context->recordAttempt('reserve'); + $context->setLatencyMs('reserve', 12); + $context->setStepStatus('charge', StepStatus::Failed); + $context->recordAttempt('charge'); + $context->recordAttempt('charge'); + $context->putStepError('charge', new RuntimeException('card declined')); + + $report = ExecutionReportBuilder::fromContext($context, ExecutionStatus::Failed, 'card declined', $start->modify('+250 milliseconds'), ['reserve', 'charge', 'notify']); + + expect($report->durationMs)->toBe(250) + ->and($report->failedSteps())->toBe(['charge']) + ->and($report->compensatedSteps())->toBe(['reserve']) + ->and($report->steps['notify']->status)->toBe(StepStatus::Pending) + ->and($report->toArray())->toMatchArray([ + 'name' => 'order', 'correlationId' => 'c-1', 'pattern' => 'SAGA', 'status' => 'FAILED', + 'durationMs' => 250, 'failureReason' => 'card declined', 'layers' => [['reserve'], ['charge', 'notify']], + ]) + ->and($report->toArray()['steps']['charge'])->toBe([ + 'stepId' => 'charge', 'status' => 'FAILED', 'attempts' => 2, 'latencyMs' => 0, 'error' => 'card declined', + 'compensated' => false, 'compensationError' => null, + ]); +}); diff --git a/packages/orchestration/tests/Support/DeadLetterStoreContract.php b/packages/orchestration/tests/Support/DeadLetterStoreContract.php new file mode 100644 index 00000000..4aea319f --- /dev/null +++ b/packages/orchestration/tests/Support/DeadLetterStoreContract.php @@ -0,0 +1,62 @@ + 7, 'note' => 'café'], ['tenant' => 'acme'], new DateTimeImmutable($createdAt)); + + return DeadLetterEntry::fromArray(['id' => $id] + $entry->toArray()); + } + + /** + * @return array + */ + public static function cases(): array + { + return [ + 'saves, finds and replaces an entry' => static function (DeadLetterStore $store): void { + $entry = self::entry('d-1'); + $store->save($entry); + + expect($store->findById('d-1')?->toArray())->toBe($entry->toArray()) + ->and($store->findById('absent'))->toBeNull(); + + $store->save($entry->withRetry(new DateTimeImmutable('2026-10-12T11:00:00+00:00'))); + + expect($store->findById('d-1')?->retryCount)->toBe(1) + ->and($store->count())->toBe(1); + }, + 'lists newest first and filters by execution name and correlation id' => static function (DeadLetterStore $store): void { + $store->save(self::entry('d-1', 'order', 'c-1', '2026-10-12T10:00:00+00:00')); + $store->save(self::entry('d-2', 'refund', 'c-2', '2026-10-12T10:02:00+00:00')); + $store->save(self::entry('d-3', 'order', 'c-3', '2026-10-12T10:01:00+00:00')); + + expect(array_map(static fn (DeadLetterEntry $e): string => $e->id, $store->findAll()))->toBe(['d-2', 'd-3', 'd-1']) + ->and(array_map(static fn (DeadLetterEntry $e): string => $e->id, $store->findAll(2)))->toBe(['d-2', 'd-3']) + ->and(array_map(static fn (DeadLetterEntry $e): string => $e->id, $store->findByExecutionName('order')))->toBe(['d-3', 'd-1']) + ->and(array_map(static fn (DeadLetterEntry $e): string => $e->id, $store->findByCorrelationId('c-2')))->toBe(['d-2']); + }, + 'deletes an entry once' => static function (DeadLetterStore $store): void { + $store->save(self::entry('d-1')); + + expect($store->delete('d-1'))->toBeTrue() + ->and($store->delete('d-1'))->toBeFalse() + ->and($store->count())->toBe(0); + }, + ]; + } +} diff --git a/packages/orchestration/tests/Support/FailingExecutionPersistence.php b/packages/orchestration/tests/Support/FailingExecutionPersistence.php new file mode 100644 index 00000000..db92f5a1 --- /dev/null +++ b/packages/orchestration/tests/Support/FailingExecutionPersistence.php @@ -0,0 +1,81 @@ + ['enabled' => false, 'path' => '/feature-flags/flagd.json', 'token' => env('FIREFLY_FEATURE_FLAGS_SERVER_TOKEN', ''), 'allow-anonymous' => false], ], + /* + |-------------------------------------------------------------------------- + | Orchestration — firefly/orchestration + |-------------------------------------------------------------------------- + | + | Sagas, TCC transactions and durable workflows discovered from #[Saga], #[Tcc] and #[Workflow] classes + | (compiled by php artisan firefly:cache, validated at boot). Steps of one topology layer run one after + | another — PHP is synchronous — so what an application can rely on is the LAYER guarantee: every + | dependency finished before a dependent starts. Durations: '250ms', '30s', '5m', '1h', 'P7D' or seconds. + | See docs/modules/orchestration.md. + | + */ + + 'orchestration' => [ + + /* + | Each pattern can be switched off; a disabled pattern registers nothing and answers every start with + | a 503 (PatternDisabledException). `default-timeout` is the saga-wide deadline ('0' = none); + | `compensation-policy` is strict-sequential, grouped-parallel (layer by layer, sequential inside a + | layer), retry-with-backoff, circuit-breaker (a firefly/resilience breaker per saga, configured under + | firefly.resilience.circuit-breaker.orchestration-compensation-) or best-effort. + | `compensation-error-handler` is default, fail-fast, log-and-continue, retry, robust, or the class + | name of your own CompensationErrorHandler bean. The TCC `default-timeout` applies to a phase that + | sets no timeoutMs. Defaults: enabled true, strict-sequential, 5m, default, 30s. + */ + 'saga' => [ + 'enabled' => env('FIREFLY_ORCHESTRATION_SAGA_ENABLED', true), + 'compensation-policy' => env('FIREFLY_ORCHESTRATION_SAGA_COMPENSATION_POLICY', 'strict-sequential'), + 'default-timeout' => env('FIREFLY_ORCHESTRATION_SAGA_DEFAULT_TIMEOUT', '5m'), + 'compensation-error-handler' => env('FIREFLY_ORCHESTRATION_SAGA_COMPENSATION_ERROR_HANDLER', 'default'), + ], + 'tcc' => [ + 'enabled' => env('FIREFLY_ORCHESTRATION_TCC_ENABLED', true), + 'default-timeout' => env('FIREFLY_ORCHESTRATION_TCC_DEFAULT_TIMEOUT', '30s'), + // 'composition' => ['enabled' => true, 'compensation-policy' => 'strict-sequential'], + ], + + /* + | Workflows triggered ASYNC are persisted PENDING and run by a RunWorkflowJob on this queue + | (null = the default connection / queue); they need `cache` or `database` persistence, because the + | worker is another process. The `tick` is a scheduled task (firefly:schedule lists it) that resumes + | WAITING workflows whose timers or deadlines are due, under the DistributedLock; without a scheduler + | run php artisan firefly:orchestration:tick from cron. Defaults: enabled true, tick every 1m. + */ + 'workflow' => [ + 'enabled' => env('FIREFLY_ORCHESTRATION_WORKFLOW_ENABLED', true), + 'queue' => [ + 'connection' => env('FIREFLY_ORCHESTRATION_WORKFLOW_QUEUE_CONNECTION'), + 'name' => env('FIREFLY_ORCHESTRATION_WORKFLOW_QUEUE'), + ], + 'tick' => [ + 'enabled' => env('FIREFLY_ORCHESTRATION_TICK_ENABLED', true), + 'interval' => env('FIREFLY_ORCHESTRATION_TICK_INTERVAL', '1m'), + ], + ], + + /* + | Where execution checkpoints live: `memory` (this process only — the default), `cache` (the Laravel + | cache store `cache-store` names, null = the default store; `key-prefix`, optional `key-ttl`) or + | `database` (three portable tables on `connection`, null = the default; run the migration, + | --tag=firefly-orchestration-migrations to publish it). Terminal executions older than + | `retention-period` are deleted by the recovery sweep, which runs every `cleanup-interval`. + | Defaults: memory, orchestration:, forever, P7D, 1h. + */ + 'persistence' => [ + 'provider' => env('FIREFLY_ORCHESTRATION_PERSISTENCE', 'memory'), + 'connection' => env('FIREFLY_ORCHESTRATION_DB_CONNECTION'), + 'cache-store' => env('FIREFLY_ORCHESTRATION_CACHE_STORE'), + 'key-prefix' => 'orchestration:', + 'key-ttl' => null, + 'retention-period' => env('FIREFLY_ORCHESTRATION_RETENTION', 'P7D'), + 'cleanup-interval' => '1h', + ], + + /* + | The recovery sweep finds executions a crashed process left RUNNING, COMPENSATING, TRYING, + | CONFIRMING, CANCELING or PENDING for longer than `stale-threshold`, and `action`s them: report (log + | only), resume (continue from the last checkpoint — make your steps idempotent) or compensate (undo + | what completed). WAITING and SUSPENDED workflows are idle by design and never touched. + | Defaults: enabled true, 1h, report. + */ + 'recovery' => [ + 'enabled' => env('FIREFLY_ORCHESTRATION_RECOVERY_ENABLED', true), + 'stale-threshold' => env('FIREFLY_ORCHESTRATION_STALE_THRESHOLD', '1h'), + 'action' => env('FIREFLY_ORCHESTRATION_RECOVERY_ACTION', 'report'), + ], + + // Failed executions, one entry per failed step: `memory` or `database`. php artisan firefly:orchestration:dlq lists + // them, firefly:orchestration:dlq:retry starts the same definition again with the same input. + 'dlq' => [ + 'enabled' => true, + 'store' => env('FIREFLY_ORCHESTRATION_DLQ_STORE', 'memory'), + ], + + // Lifecycle events (EXECUTION_STARTED/COMPLETED, STEP_COMPLETED/FAILED) through firefly/eda, on `default-topic` + // ('' = orchestration). #[StepEvent], #[TccEvent] and outputEventType publish whatever this says. + 'events' => [ + 'enabled' => env('FIREFLY_ORCHESTRATION_EVENTS_ENABLED', false), + 'default-topic' => env('FIREFLY_ORCHESTRATION_EVENTS_TOPIC', ''), + ], + + // Start the definition whose triggerEventType matches an EDA event, with the event payload as input. + 'event-gateway' => ['enabled' => true], + + // The REST surface (start, list, show, signal, suspend/resume/cancel, query, DLQ). Off by default; protect `path` + // with your firefly.security URL rules before switching it on. + 'rest' => [ + 'enabled' => env('FIREFLY_ORCHESTRATION_REST_ENABLED', false), + 'path' => '/orchestration', + ], + + // The `orchestration` health component, the metrics hook and the step/execution spans. + 'health' => ['enabled' => true], + 'metrics' => ['enabled' => true], + 'tracing' => ['enabled' => true], + + // Topology validation at boot (cycles, unknown dependsOn, missing compensations, bad conditions). Warnings — a + // compensationCritical step without a compensation — refuse the boot too when `fail-on-warning` is true. + 'validation' => [ + 'enabled' => true, + 'fail-on-warning' => false, + ], + + // The per-execution lock over firefly.scheduling.lock.provider: held at most `ttl`, waited for at most `wait`. + 'lock' => [ + 'ttl' => '5m', + 'wait' => '5s', + ], + + /* + | Back-pressure where it means something in a synchronous process: `max-in-flight` (0 = unlimited) + | refuses new starts with a 503 while that many executions are open; `strategy: circuit-breaker` sheds + | starts once `failure-threshold` executions in a row ended FAILED, for `recovery-timeout`, then + | admits `half-open-max-calls` probes; `batch-size` caps one tick or recovery sweep. + */ + 'backpressure' => [ + 'strategy' => 'none', + 'batch-size' => 10, + 'max-in-flight' => 0, + 'circuit-breaker' => [ + 'failure-threshold' => 5, + 'recovery-timeout' => '30s', + 'half-open-max-calls' => 3, + ], + ], + ], + ]; From 8f4d3cf8c86d32942c8aa19d92ee8d603af8a2c9 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 19:35:48 -0700 Subject: [PATCH 11/58] feat(orchestration): saga attributes, validated definitions, the registry, the builder DSL and saga results --- .../Saga/Attributes/CompensationSagaStep.php | 20 ++ .../src/Saga/Attributes/ExternalSagaStep.php | 37 ++++ .../src/Saga/Attributes/OnSagaComplete.php | 21 ++ .../src/Saga/Attributes/OnSagaError.php | 26 +++ .../src/Saga/Attributes/Saga.php | 23 +++ .../src/Saga/Attributes/SagaStep.php | 41 ++++ .../src/Saga/Attributes/ScheduledSaga.php | 28 +++ .../src/Saga/Attributes/StepEvent.php | 23 +++ .../src/Saga/Builder/SagaBuilder.php | 101 ++++++++++ .../src/Saga/Builder/SagaStepBuilder.php | 186 ++++++++++++++++++ .../src/Saga/Engine/SagaResult.php | 121 ++++++++++++ .../src/Saga/Engine/StepInputs.php | 80 ++++++++ .../src/Saga/Engine/StepOutcome.php | 39 ++++ .../src/Saga/Registry/SagaDefinition.php | 90 +++++++++ .../src/Saga/Registry/SagaRegistry.php | 50 +++++ .../src/Saga/Registry/SagaStepDefinition.php | 85 ++++++++ .../src/Saga/Registry/StepEventDefinition.php | 29 +++ .../tests/Saga/SagaBuilderTest.php | 47 +++++ .../tests/Saga/SagaDefinitionTest.php | 81 ++++++++ .../tests/Saga/SagaResultTest.php | 37 ++++ .../tests/Saga/StepInputsTest.php | 28 +++ 21 files changed, 1193 insertions(+) create mode 100644 packages/orchestration/src/Saga/Attributes/CompensationSagaStep.php create mode 100644 packages/orchestration/src/Saga/Attributes/ExternalSagaStep.php create mode 100644 packages/orchestration/src/Saga/Attributes/OnSagaComplete.php create mode 100644 packages/orchestration/src/Saga/Attributes/OnSagaError.php create mode 100644 packages/orchestration/src/Saga/Attributes/Saga.php create mode 100644 packages/orchestration/src/Saga/Attributes/SagaStep.php create mode 100644 packages/orchestration/src/Saga/Attributes/ScheduledSaga.php create mode 100644 packages/orchestration/src/Saga/Attributes/StepEvent.php create mode 100644 packages/orchestration/src/Saga/Builder/SagaBuilder.php create mode 100644 packages/orchestration/src/Saga/Builder/SagaStepBuilder.php create mode 100644 packages/orchestration/src/Saga/Engine/SagaResult.php create mode 100644 packages/orchestration/src/Saga/Engine/StepInputs.php create mode 100644 packages/orchestration/src/Saga/Engine/StepOutcome.php create mode 100644 packages/orchestration/src/Saga/Registry/SagaDefinition.php create mode 100644 packages/orchestration/src/Saga/Registry/SagaRegistry.php create mode 100644 packages/orchestration/src/Saga/Registry/SagaStepDefinition.php create mode 100644 packages/orchestration/src/Saga/Registry/StepEventDefinition.php create mode 100644 packages/orchestration/tests/Saga/SagaBuilderTest.php create mode 100644 packages/orchestration/tests/Saga/SagaDefinitionTest.php create mode 100644 packages/orchestration/tests/Saga/SagaResultTest.php create mode 100644 packages/orchestration/tests/Saga/StepInputsTest.php diff --git a/packages/orchestration/src/Saga/Attributes/CompensationSagaStep.php b/packages/orchestration/src/Saga/Attributes/CompensationSagaStep.php new file mode 100644 index 00000000..75a14b53 --- /dev/null +++ b/packages/orchestration/src/Saga/Attributes/CompensationSagaStep.php @@ -0,0 +1,20 @@ + $dependsOn + */ + public function __construct( + public string $saga, + public string $id, + public string $compensate = '', + public array $dependsOn = [], + public int $retry = 0, + public int $backoffMs = -1, + public int $timeoutMs = -1, + public bool $jitter = false, + public float $jitterFactor = 0.5, + public string $idempotencyKey = '', + public bool $cpuBound = false, + public int $compensationRetry = -1, + public int $compensationTimeoutMs = -1, + public int $compensationBackoffMs = -1, + public bool $compensationCritical = false, + ) {} +} diff --git a/packages/orchestration/src/Saga/Attributes/OnSagaComplete.php b/packages/orchestration/src/Saga/Attributes/OnSagaComplete.php new file mode 100644 index 00000000..8e79b960 --- /dev/null +++ b/packages/orchestration/src/Saga/Attributes/OnSagaComplete.php @@ -0,0 +1,21 @@ + $errorTypes exception class names; empty matches every error + */ + public function __construct( + public array $errorTypes = [], + public bool $suppressError = false, + public bool $async = false, + public int $priority = 0, + ) {} +} diff --git a/packages/orchestration/src/Saga/Attributes/Saga.php b/packages/orchestration/src/Saga/Attributes/Saga.php new file mode 100644 index 00000000..a5b11764 --- /dev/null +++ b/packages/orchestration/src/Saga/Attributes/Saga.php @@ -0,0 +1,23 @@ +}, {header.}) is + * recorded once the step succeeds, and a step whose key is already recorded is skipped. The `compensation*` + * members override retry/backoff/timeout for the compensation; `compensationCritical` makes a failed + * compensation stop the CIRCUIT_BREAKER policy. `cpuBound` is accepted for parity and has no effect. -1 means + * "not set". + */ +#[Attribute(Attribute::TARGET_METHOD)] +final readonly class SagaStep +{ + /** + * @param list $dependsOn + */ + public function __construct( + public string $id, + public string $compensate = '', + public array $dependsOn = [], + public int $retry = 0, + public int $backoffMs = -1, + public int $timeoutMs = -1, + public bool $jitter = false, + public float $jitterFactor = 0.5, + public string $idempotencyKey = '', + public bool $cpuBound = false, + public int $compensationRetry = -1, + public int $compensationTimeoutMs = -1, + public int $compensationBackoffMs = -1, + public bool $compensationCritical = false, + ) {} +} diff --git a/packages/orchestration/src/Saga/Attributes/ScheduledSaga.php b/packages/orchestration/src/Saga/Attributes/ScheduledSaga.php new file mode 100644 index 00000000..895b2a1c --- /dev/null +++ b/packages/orchestration/src/Saga/Attributes/ScheduledSaga.php @@ -0,0 +1,28 @@ +step('debit')->handler(fn (array $in) => $ledger->debit($in))->compensation(fn ($ref) => $ledger->credit($ref))->add() + * ->step('credit')->dependsOn('debit')->handler(fn (array $in) => $ledger->credit($in))->add() + * ->build(); + * + * A built saga runs with SagaEngine::execute($definition, …); register it in SagaRegistry (in every process) + * for it to be startable by name, triggerable from EDA and recoverable after a crash. + */ +final class SagaBuilder +{ + /** @var array */ + private array $steps = []; + + private string $triggerEventType = ''; + + /** @var list */ + private array $onComplete = []; + + /** @var list */ + private array $onError = []; + + private function __construct( + private readonly string $name, + private readonly int $layerConcurrency, + ) {} + + public static function saga(string $name, int $layerConcurrency = 0): self + { + return new self($name, $layerConcurrency); + } + + public function triggerEventType(string $eventType): self + { + $this->triggerEventType = $eventType; + + return $this; + } + + public function step(string $id): SagaStepBuilder + { + return new SagaStepBuilder($this, $this->name, $id); + } + + /** + * @param Closure(ExecutionContext, mixed): void $callback + */ + public function onComplete(Closure $callback, int $priority = 0, bool $async = false): self + { + $this->onComplete[] = CallbackDefinition::closure($callback, $async, $priority); + + return $this; + } + + /** + * @param Closure(ExecutionContext, mixed): void $callback + * @param list $errorTypes + */ + public function onError(Closure $callback, bool $suppressError = false, array $errorTypes = [], int $priority = 0, bool $async = false): self + { + $this->onError[] = CallbackDefinition::closure($callback, $async, $priority, $errorTypes, $suppressError); + + return $this; + } + + /** + * A second step with an id already added refuses, rather than silently replacing the first. + * + * @internal called by SagaStepBuilder::add() + */ + public function addStep(SagaStepDefinition $step): self + { + if (isset($this->steps[$step->id])) { + throw new TopologyValidationException("Saga [{$this->name}] declares step [{$step->id}] twice."); + } + + $this->steps[$step->id] = $step; + + return $this; + } + + public function build(): SagaDefinition + { + return new SagaDefinition($this->name, $this->steps, $this->layerConcurrency, $this->triggerEventType, $this->onComplete, $this->onError); + } +} diff --git a/packages/orchestration/src/Saga/Builder/SagaStepBuilder.php b/packages/orchestration/src/Saga/Builder/SagaStepBuilder.php new file mode 100644 index 00000000..04ade0ad --- /dev/null +++ b/packages/orchestration/src/Saga/Builder/SagaStepBuilder.php @@ -0,0 +1,186 @@ +stepInput($id)`. + */ +final class SagaStepBuilder +{ + /** @var list */ + private array $dependsOn = []; + + private int $retry = 0; + + private int $backoffMs = 0; + + private int $timeoutMs = 0; + + private bool $jitter = false; + + private float $jitterFactor = 0.5; + + private string $idempotencyKey = ''; + + private int $compensationRetry = -1; + + private int $compensationBackoffMs = -1; + + private int $compensationTimeoutMs = -1; + + private bool $compensationCritical = false; + + private bool $cpuBound = false; + + private ?StepEventDefinition $event = null; + + private ?string $variable = null; + + private ?Closure $handler = null; + + private ?Closure $compensation = null; + + public function __construct( + private readonly SagaBuilder $saga, + private readonly string $sagaName, + private readonly string $id, + ) {} + + public function dependsOn(string ...$ids): self + { + $this->dependsOn = array_values([...$this->dependsOn, ...$ids]); + + return $this; + } + + public function retry(int $retry): self + { + $this->retry = $retry; + + return $this; + } + + public function backoffMs(int $milliseconds): self + { + $this->backoffMs = $milliseconds; + + return $this; + } + + public function timeoutMs(int $milliseconds): self + { + $this->timeoutMs = $milliseconds; + + return $this; + } + + public function jitter(bool $enabled = true, float $factor = 0.5): self + { + $this->jitter = $enabled; + $this->jitterFactor = $factor; + + return $this; + } + + public function idempotencyKey(string $key): self + { + $this->idempotencyKey = $key; + + return $this; + } + + public function compensationRetry(int $retry): self + { + $this->compensationRetry = $retry; + + return $this; + } + + public function compensationBackoffMs(int $milliseconds): self + { + $this->compensationBackoffMs = $milliseconds; + + return $this; + } + + public function compensationTimeoutMs(int $milliseconds): self + { + $this->compensationTimeoutMs = $milliseconds; + + return $this; + } + + public function compensationCritical(bool $critical = true): self + { + $this->compensationCritical = $critical; + + return $this; + } + + public function cpuBound(bool $cpuBound = true): self + { + $this->cpuBound = $cpuBound; + + return $this; + } + + public function stepEvent(string $topic, string $type = '', string $key = ''): self + { + $this->event = new StepEventDefinition($topic, $type, $key); + + return $this; + } + + public function setVariable(string $name): self + { + $this->variable = $name; + + return $this; + } + + public function handler(Closure $handler): self + { + $this->handler = $handler; + + return $this; + } + + public function compensation(Closure $compensation): self + { + $this->compensation = $compensation; + + return $this; + } + + public function add(): SagaBuilder + { + if ($this->handler === null) { + throw new TopologyValidationException("Saga [{$this->sagaName}] step [{$this->id}] has no handler."); + } + + return $this->saga->addStep(new SagaStepDefinition( + id: $this->id, + invocation: new ClosureInvocation($this->handler, "{$this->sagaName}.{$this->id}", $this->variable), + dependsOn: $this->dependsOn, + options: StepOptions::of($this->retry, $this->backoffMs, $this->timeoutMs, $this->jitter, $this->jitterFactor, $this->idempotencyKey), + compensation: $this->compensation === null ? null : new ClosureInvocation($this->compensation, "{$this->sagaName}.{$this->id}:compensate"), + compensationRetry: $this->compensationRetry, + compensationBackoffMs: $this->compensationBackoffMs, + compensationTimeoutMs: $this->compensationTimeoutMs, + compensationCritical: $this->compensationCritical, + cpuBound: $this->cpuBound, + event: $this->event, + )); + } +} diff --git a/packages/orchestration/src/Saga/Engine/SagaResult.php b/packages/orchestration/src/Saga/Engine/SagaResult.php new file mode 100644 index 00000000..523fe71b --- /dev/null +++ b/packages/orchestration/src/Saga/Engine/SagaResult.php @@ -0,0 +1,121 @@ + $steps + * @param array $headers + */ + public function __construct( + public string $sagaName, + public string $correlationId, + public ExecutionStatus $status, + public bool $success, + public ?Throwable $error, + public array $steps, + public array $headers, + public ?ExecutionReport $report, + public DateTimeImmutable $startedAt, + public DateTimeImmutable $completedAt, + ) {} + + public static function from(SagaDefinition $saga, ExecutionContext $context, ExecutionStatus $status, ?Throwable $error, ?ExecutionReport $report, DateTimeImmutable $now): self + { + $steps = []; + foreach ($saga->stepIds() as $stepId) { + $steps[$stepId] = new StepOutcome( + status: $context->stepStatus($stepId), + attempts: $context->attempts($stepId), + latencyMs: $context->latencyMs($stepId), + result: $context->result($stepId), + error: $context->stepError($stepId), + compensated: $context->stepStatus($stepId) === StepStatus::Compensated, + compensationResult: $context->compensationResult($stepId), + compensationError: $context->compensationError($stepId), + ); + } + + return new self($saga->name, $context->correlationId, $status, $status->isSuccess(), $error, $steps, $context->headers(), $report, $context->startedAt(), $now); + } + + public function isSuccess(): bool + { + return $this->success; + } + + /** @return list */ + public function failedSteps(): array + { + return array_map('strval', array_keys(array_filter($this->steps, static fn (StepOutcome $step): bool => $step->status->isFailure() || $step->error !== null))); + } + + /** @return list */ + public function compensatedSteps(): array + { + return array_map('strval', array_keys(array_filter($this->steps, static fn (StepOutcome $step): bool => $step->compensated))); + } + + public function firstErrorStepId(): ?string + { + return $this->failedSteps()[0] ?? null; + } + + public function resultOf(string $stepId): mixed + { + return ($this->steps[$stepId] ?? null)?->result; + } + + /** @return array */ + public function stepResults(): array + { + $results = []; + foreach ($this->steps as $stepId => $step) { + if ($step->result !== null) { + $results[$stepId] = $step->result; + } + } + + return $results; + } + + public function durationMs(): int + { + return max(0, (int) round(((float) $this->completedAt->format('U.u') - (float) $this->startedAt->format('U.u')) * 1000)); + } + + /** @return array */ + public function toArray(): array + { + return [ + 'sagaName' => $this->sagaName, + 'correlationId' => $this->correlationId, + 'status' => $this->status->value, + 'success' => $this->success, + 'error' => $this->error?->getMessage(), + 'startedAt' => ExecutionState::instant($this->startedAt), + 'completedAt' => ExecutionState::instant($this->completedAt), + 'durationMs' => $this->durationMs(), + 'headers' => $this->headers, + 'steps' => array_map(static fn (StepOutcome $step): array => $step->toArray(), $this->steps), + 'report' => $this->report?->toArray(), + ]; + } +} diff --git a/packages/orchestration/src/Saga/Engine/StepInputs.php b/packages/orchestration/src/Saga/Engine/StepInputs.php new file mode 100644 index 00000000..6613ed11 --- /dev/null +++ b/packages/orchestration/src/Saga/Engine/StepInputs.php @@ -0,0 +1,80 @@ + */ + private array $resolved = []; + + /** + * @param array $global + * @param array $values + * @param array $resolvers + */ + private function __construct( + private readonly array $global, + private readonly array $values, + private readonly array $resolvers, + ) {} + + /** + * @param array $global + */ + public static function of(array $global = []): self + { + return new self($global, [], []); + } + + public function forStep(string $stepId, mixed $input): self + { + return new self($this->global, [...$this->values, $stepId => $input], $this->resolvers); + } + + /** + * @param Closure(ExecutionContext): mixed $resolver + */ + public function forStepResolver(string $stepId, Closure $resolver): self + { + return new self($this->global, $this->values, [...$this->resolvers, $stepId => $resolver]); + } + + public function resolveFor(string $stepId, ExecutionContext $context): mixed + { + if (array_key_exists($stepId, $this->values)) { + return $this->values[$stepId]; + } + + if (isset($this->resolvers[$stepId])) { + if (! array_key_exists($stepId, $this->resolved)) { + $this->resolved[$stepId] = ($this->resolvers[$stepId])($context); + } + + return $this->resolved[$stepId]; + } + + return $this->global; + } + + /** @return array */ + public function global(): array + { + return $this->global; + } + + /** @return array */ + public function explicit(): array + { + return $this->values; + } +} diff --git a/packages/orchestration/src/Saga/Engine/StepOutcome.php b/packages/orchestration/src/Saga/Engine/StepOutcome.php new file mode 100644 index 00000000..47d19e23 --- /dev/null +++ b/packages/orchestration/src/Saga/Engine/StepOutcome.php @@ -0,0 +1,39 @@ + */ + public function toArray(): array + { + return [ + 'status' => $this->status->value, + 'attempts' => $this->attempts, + 'latencyMs' => $this->latencyMs, + 'result' => StateSerializer::normalize($this->result, 'result'), + 'error' => $this->error?->getMessage(), + 'compensated' => $this->compensated, + 'compensationResult' => StateSerializer::normalize($this->compensationResult, 'compensationResult'), + 'compensationError' => $this->compensationError?->getMessage(), + ]; + } +} diff --git a/packages/orchestration/src/Saga/Registry/SagaDefinition.php b/packages/orchestration/src/Saga/Registry/SagaDefinition.php new file mode 100644 index 00000000..54904307 --- /dev/null +++ b/packages/orchestration/src/Saga/Registry/SagaDefinition.php @@ -0,0 +1,90 @@ +> */ + public array $layers; + + /** + * @param array $steps + * @param list $onComplete + * @param list $onError + */ + public function __construct( + public string $name, + public array $steps, + public int $layerConcurrency = 0, + public string $triggerEventType = '', + public array $onComplete = [], + public array $onError = [], + ) { + $dependencies = []; + foreach ($steps as $id => $step) { + if ((string) $id !== $step->id) { + throw new TopologyValidationException("Saga [{$name}]: step [{$step->id}] is filed under key [{$id}]; a step must be keyed by its own id."); + } + $dependencies[(string) $id] = $step->dependsOn; + } + + $this->layers = TopologyBuilder::layers("Saga [{$name}]", $dependencies); + } + + public function step(string $id): SagaStepDefinition + { + return $this->steps[$id] ?? throw new DefinitionNotFoundException("Saga [{$this->name}] has no step [{$id}]."); + } + + /** @return list */ + public function stepIds(): array + { + return array_map('strval', array_keys($this->steps)); + } + + /** @return array */ + public function toArray(): array + { + return [ + 'name' => $this->name, + 'layerConcurrency' => $this->layerConcurrency, + 'triggerEventType' => $this->triggerEventType, + 'steps' => array_values(array_map(static fn (SagaStepDefinition $step): array => $step->toArray(), $this->steps)), + 'onComplete' => array_map(static fn (CallbackDefinition $callback): array => $callback->toArray(), $this->onComplete), + 'onError' => array_map(static fn (CallbackDefinition $callback): array => $callback->toArray(), $this->onError), + ]; + } + + /** @param array $row */ + public static function fromArray(array $row): self + { + $steps = []; + foreach (Rows::rows($row, 'steps') as $stepRow) { + $step = SagaStepDefinition::fromArray($stepRow); + $steps[$step->id] = $step; + } + + return new self( + name: Rows::string($row, 'name'), + steps: $steps, + layerConcurrency: Rows::int($row, 'layerConcurrency'), + triggerEventType: Rows::string($row, 'triggerEventType'), + onComplete: array_map(static fn (array $callback): CallbackDefinition => CallbackDefinition::fromArray($callback), Rows::rows($row, 'onComplete')), + onError: array_map(static fn (array $callback): CallbackDefinition => CallbackDefinition::fromArray($callback), Rows::rows($row, 'onError')), + ); + } +} diff --git a/packages/orchestration/src/Saga/Registry/SagaRegistry.php b/packages/orchestration/src/Saga/Registry/SagaRegistry.php new file mode 100644 index 00000000..1d8587a9 --- /dev/null +++ b/packages/orchestration/src/Saga/Registry/SagaRegistry.php @@ -0,0 +1,50 @@ + */ + private array $sagas = []; + + public function register(SagaDefinition $saga): void + { + if (isset($this->sagas[$saga->name])) { + throw new TopologyValidationException("Saga [{$saga->name}] is defined twice."); + } + + $this->sagas[$saga->name] = $saga; + } + + public function get(string $name): SagaDefinition + { + return $this->sagas[$name] ?? throw new DefinitionNotFoundException("No saga [{$name}] is registered."); + } + + public function has(string $name): bool + { + return isset($this->sagas[$name]); + } + + /** @return array */ + public function all(): array + { + return $this->sagas; + } + + /** @return list */ + public function names(): array + { + return array_map('strval', array_keys($this->sagas)); + } +} diff --git a/packages/orchestration/src/Saga/Registry/SagaStepDefinition.php b/packages/orchestration/src/Saga/Registry/SagaStepDefinition.php new file mode 100644 index 00000000..340baeef --- /dev/null +++ b/packages/orchestration/src/Saga/Registry/SagaStepDefinition.php @@ -0,0 +1,85 @@ + $dependsOn + */ + public function __construct( + public string $id, + public Invocation $invocation, + public array $dependsOn = [], + public StepOptions $options = new StepOptions, + public ?Invocation $compensation = null, + public int $compensationRetry = -1, + public int $compensationBackoffMs = -1, + public int $compensationTimeoutMs = -1, + public bool $compensationCritical = false, + public bool $cpuBound = false, + public ?StepEventDefinition $event = null, + ) {} + + public function compensationOptions(): StepOptions + { + return new StepOptions( + $this->compensationRetry >= 0 ? $this->compensationRetry : $this->options->retry, + $this->compensationBackoffMs >= 0 ? $this->compensationBackoffMs : $this->options->backoffMs, + $this->compensationTimeoutMs >= 0 ? $this->compensationTimeoutMs : $this->options->timeoutMs, + $this->options->jitter, + $this->options->jitterFactor, + ); + } + + /** @return array */ + public function toArray(): array + { + return [ + 'id' => $this->id, + 'invocation' => $this->invocation->toArray(), + 'dependsOn' => $this->dependsOn, + 'options' => $this->options->toArray(), + 'compensation' => $this->compensation?->toArray(), + 'compensationRetry' => $this->compensationRetry, + 'compensationBackoffMs' => $this->compensationBackoffMs, + 'compensationTimeoutMs' => $this->compensationTimeoutMs, + 'compensationCritical' => $this->compensationCritical, + 'cpuBound' => $this->cpuBound, + 'event' => $this->event?->toArray(), + ]; + } + + /** @param array $row */ + public static function fromArray(array $row): self + { + $compensation = Rows::nullableMap($row, 'compensation'); + $event = Rows::nullableMap($row, 'event'); + + return new self( + id: Rows::string($row, 'id'), + invocation: MethodInvocation::fromArray(Rows::map($row, 'invocation')), + dependsOn: Rows::strings($row, 'dependsOn'), + options: StepOptions::fromArray(Rows::map($row, 'options')), + compensation: $compensation === null ? null : MethodInvocation::fromArray($compensation), + compensationRetry: Rows::int($row, 'compensationRetry', -1), + compensationBackoffMs: Rows::int($row, 'compensationBackoffMs', -1), + compensationTimeoutMs: Rows::int($row, 'compensationTimeoutMs', -1), + compensationCritical: Rows::bool($row, 'compensationCritical'), + cpuBound: Rows::bool($row, 'cpuBound'), + event: $event === null ? null : StepEventDefinition::fromArray($event), + ); + } +} diff --git a/packages/orchestration/src/Saga/Registry/StepEventDefinition.php b/packages/orchestration/src/Saga/Registry/StepEventDefinition.php new file mode 100644 index 00000000..f5000075 --- /dev/null +++ b/packages/orchestration/src/Saga/Registry/StepEventDefinition.php @@ -0,0 +1,29 @@ + $this->topic, 'type' => $this->type, 'key' => $this->key]; + } + + /** @param array $row */ + public static function fromArray(array $row): self + { + return new self(Rows::string($row, 'topic'), Rows::string($row, 'type'), Rows::string($row, 'key')); + } +} diff --git a/packages/orchestration/tests/Saga/SagaBuilderTest.php b/packages/orchestration/tests/Saga/SagaBuilderTest.php new file mode 100644 index 00000000..3eea69e8 --- /dev/null +++ b/packages/orchestration/tests/Saga/SagaBuilderTest.php @@ -0,0 +1,47 @@ +triggerEventType('transfers.requested') + ->step('debit') + ->handler(static fn (mixed $input): string => 'debited') + ->compensation(static fn (mixed $result): string => 'credited back') + ->retry(2)->backoffMs(5)->timeoutMs(100)->jitter(true, 0.2)->idempotencyKey('debit-{correlationId}') + ->compensationRetry(1)->compensationBackoffMs(3)->compensationTimeoutMs(50)->compensationCritical() + ->stepEvent('ledger', 'ledger.debited', 'accountId')->setVariable('debitRef') + ->add() + ->step('credit')->dependsOn('debit')->handler(static fn (): string => 'credited')->add() + ->onComplete(static function (): void {}) + ->onError(static function (): void {}, suppressError: true, errorTypes: [RuntimeException::class]) + ->build(); + + $debit = $saga->step('debit'); + + expect($saga->layers)->toBe([['debit'], ['credit']]) + ->and($saga->triggerEventType)->toBe('transfers.requested') + ->and($debit->options)->toEqual(new StepOptions(2, 5, 100, true, 0.2, 'debit-{correlationId}')) + ->and($debit->compensationOptions())->toEqual(new StepOptions(1, 3, 50, true, 0.2)) + ->and($debit->compensationCritical)->toBeTrue() + ->and($debit->event?->type)->toBe('ledger.debited') + ->and($debit->invocation)->toBeInstanceOf(ClosureInvocation::class) + ->and($debit->invocation->setVariable())->toBe('debitRef') + ->and($saga->onComplete)->toHaveCount(1) + ->and($saga->onError[0]->suppressError)->toBeTrue(); +}); + +it('refuses a step without a handler', function (): void { + SagaBuilder::saga('broken')->step('a')->add(); +})->throws(TopologyValidationException::class, 'Saga [broken] step [a] has no handler'); + +it('refuses a step id declared twice instead of silently replacing the first', function (): void { + SagaBuilder::saga('twice') + ->step('a')->handler(static fn (): string => 'first')->add() + ->step('a')->handler(static fn (): string => 'second')->add(); +})->throws(TopologyValidationException::class, 'Saga [twice] declares step [a] twice'); diff --git a/packages/orchestration/tests/Saga/SagaDefinitionTest.php b/packages/orchestration/tests/Saga/SagaDefinitionTest.php new file mode 100644 index 00000000..c69579a8 --- /dev/null +++ b/packages/orchestration/tests/Saga/SagaDefinitionTest.php @@ -0,0 +1,81 @@ + $dependsOn */ +function orchestrationMethodSagaStep(string $method, array $dependsOn = [], ?string $compensate = null): SagaStepDefinition +{ + return new SagaStepDefinition( + id: $method, + invocation: new MethodInvocation('App\\OrderSaga', $method, [new ParameterBinding('input', ParameterBinding::IMPLICIT)]), + dependsOn: $dependsOn, + options: StepOptions::of(1, 10, 500, true, 0.25, 'order-{correlationId}'), + compensation: $compensate === null ? null : new MethodInvocation('App\\OrderSaga', $compensate), + compensationRetry: 2, + compensationCritical: $compensate !== null, + event: $method === 'charge' ? new StepEventDefinition('payments', 'payment.captured', 'orderId') : null, + ); +} + +function orchestrationMethodSaga(): SagaDefinition +{ + $step = orchestrationMethodSagaStep(...); + + return new SagaDefinition( + name: 'order', + steps: ['reserve' => $step('reserve', [], 'release'), 'charge' => $step('charge', ['reserve'], 'refund'), 'notify' => $step('notify', ['reserve'])], + triggerEventType: 'orders.placed', + onComplete: [new CallbackDefinition(class: 'App\\OrderSaga', method: 'done', parameters: [CallbackDefinition::RESULT])], + ); +} + +it('computes its layers at construction and keeps step order', function (): void { + $saga = orchestrationMethodSaga(); + + expect($saga->layers)->toBe([['reserve'], ['charge', 'notify']]) + ->and($saga->stepIds())->toBe(['reserve', 'charge', 'notify']) + ->and($saga->step('charge')->compensationOptions())->toEqual(new StepOptions(2, 10, 500, true, 0.25)) + ->and(fn () => $saga->step('ghost'))->toThrow(DefinitionNotFoundException::class, 'Saga [order] has no step [ghost]'); +}); + +it('round-trips a compiled definition through its array form', function (): void { + $saga = orchestrationMethodSaga(); + + expect(SagaDefinition::fromArray($saga->toArray()))->toEqual($saga); +}); + +it('refuses an invalid topology the moment it is built', function (): void { + new SagaDefinition('loop', [ + 'a' => new SagaStepDefinition('a', new ClosureInvocation(static fn (): null => null), ['b']), + 'b' => new SagaStepDefinition('b', new ClosureInvocation(static fn (): null => null), ['a']), + ]); +})->throws(TopologyValidationException::class, 'Saga [loop]: the steps form a cycle'); + +it('registers each saga once and finds it by name', function (): void { + $registry = new SagaRegistry; + $registry->register(orchestrationMethodSaga()); + + expect($registry->get('order')->name)->toBe('order') + ->and($registry->has('order'))->toBeTrue() + ->and($registry->names())->toBe(['order']) + ->and(fn () => $registry->register(orchestrationMethodSaga()))->toThrow(TopologyValidationException::class, 'Saga [order] is defined twice') + ->and(fn () => $registry->get('refund'))->toThrow(DefinitionNotFoundException::class, 'No saga [refund] is registered'); +}); + +it('refuses a step filed under a key that is not its id', function (): void { + new SagaDefinition('mismatch', [ + 'a' => new SagaStepDefinition('b', new ClosureInvocation(static fn (): null => null)), + ]); +})->throws(TopologyValidationException::class, 'Saga [mismatch]: step [b] is filed under key [a]'); diff --git a/packages/orchestration/tests/Saga/SagaResultTest.php b/packages/orchestration/tests/Saga/SagaResultTest.php new file mode 100644 index 00000000..78fc3201 --- /dev/null +++ b/packages/orchestration/tests/Saga/SagaResultTest.php @@ -0,0 +1,37 @@ +step('reserve')->handler(static fn (): string => 'r')->add() + ->step('charge')->dependsOn('reserve')->handler(static fn (): string => 'c')->add() + ->build(); + $start = new DateTimeImmutable('2026-10-12T10:00:00+00:00'); + $context = ExecutionContext::start(ExecutionPattern::Saga, 'order', [], ['tenant' => 'acme'], 'c-1', $start); + $context->putResult('reserve', ['hold' => 'h-1']); + $context->setStepStatus('reserve', StepStatus::Compensated); + $context->setStepStatus('charge', StepStatus::Failed); + $error = new RuntimeException('card declined'); + $context->putStepError('charge', $error); + $now = $start->modify('+40 milliseconds'); + + $result = SagaResult::from($saga, $context, ExecutionStatus::Failed, $error, ExecutionReportBuilder::fromContext($context, ExecutionStatus::Failed, 'card declined', $now, $saga->stepIds()), $now); + + expect($result->isSuccess())->toBeFalse() + ->and($result->failedSteps())->toBe(['charge']) + ->and($result->compensatedSteps())->toBe(['reserve']) + ->and($result->firstErrorStepId())->toBe('charge') + ->and($result->resultOf('reserve'))->toBe(['hold' => 'h-1']) + ->and($result->stepResults())->toBe(['reserve' => ['hold' => 'h-1']]) + ->and($result->durationMs())->toBe(40) + ->and($result->toArray())->toMatchArray(['sagaName' => 'order', 'correlationId' => 'c-1', 'status' => 'FAILED', 'success' => false, 'error' => 'card declined']); +}); diff --git a/packages/orchestration/tests/Saga/StepInputsTest.php b/packages/orchestration/tests/Saga/StepInputsTest.php new file mode 100644 index 00000000..5e4d380f --- /dev/null +++ b/packages/orchestration/tests/Saga/StepInputsTest.php @@ -0,0 +1,28 @@ +putResult('reserve', ['hold' => 'h-1']); + $inputs = StepInputs::of(['orderId' => 7]) + ->forStep('reserve', ['sku' => 'A-1']) + ->forStepResolver('charge', static function (ExecutionContext $ctx) use (&$calls): mixed { + $calls++; + + return $ctx->result('reserve'); + }); + + expect($inputs->resolveFor('reserve', $context))->toBe(['sku' => 'A-1']) + ->and($inputs->resolveFor('charge', $context))->toBe(['hold' => 'h-1']) + ->and($inputs->resolveFor('charge', $context))->toBe(['hold' => 'h-1']) + ->and($calls)->toBe(1) + ->and($inputs->resolveFor('notify', $context))->toBe(['orderId' => 7]) + ->and($inputs->global())->toBe(['orderId' => 7]) + ->and($inputs->explicit())->toBe(['reserve' => ['sku' => 'A-1']]); +}); From 50cbe939906a8ecfcd603dd464dd410131123f50 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 19:39:03 -0700 Subject: [PATCH 12/58] feat(orchestration): the five saga compensation policies and the compensation error handlers --- .../Compensation/CompensationErrorHandler.php | 17 ++ .../CompensationErrorHandlerFactory.php | 65 +++++ .../Compensation/CompensationErrorResult.php | 19 ++ .../CompositeCompensationErrorHandler.php | 28 ++ .../DefaultCompensationErrorHandler.php | 16 + .../FailFastCompensationErrorHandler.php | 16 + ...LogAndContinueCompensationErrorHandler.php | 21 ++ ...tryWithBackoffCompensationErrorHandler.php | 46 +++ .../RobustCompensationErrorHandler.php | 28 ++ .../src/Saga/Compensation/SagaCompensator.php | 270 +++++++++++++++++ .../CompensationErrorHandlerTest.php | 51 ++++ .../Saga/Compensation/SagaCompensatorTest.php | 273 ++++++++++++++++++ .../tests/Support/TestSupport.php | 63 ++++ 13 files changed, 913 insertions(+) create mode 100644 packages/orchestration/src/Saga/Compensation/CompensationErrorHandler.php create mode 100644 packages/orchestration/src/Saga/Compensation/CompensationErrorHandlerFactory.php create mode 100644 packages/orchestration/src/Saga/Compensation/CompensationErrorResult.php create mode 100644 packages/orchestration/src/Saga/Compensation/CompositeCompensationErrorHandler.php create mode 100644 packages/orchestration/src/Saga/Compensation/DefaultCompensationErrorHandler.php create mode 100644 packages/orchestration/src/Saga/Compensation/FailFastCompensationErrorHandler.php create mode 100644 packages/orchestration/src/Saga/Compensation/LogAndContinueCompensationErrorHandler.php create mode 100644 packages/orchestration/src/Saga/Compensation/RetryWithBackoffCompensationErrorHandler.php create mode 100644 packages/orchestration/src/Saga/Compensation/RobustCompensationErrorHandler.php create mode 100644 packages/orchestration/src/Saga/Compensation/SagaCompensator.php create mode 100644 packages/orchestration/tests/Saga/Compensation/CompensationErrorHandlerTest.php create mode 100644 packages/orchestration/tests/Saga/Compensation/SagaCompensatorTest.php create mode 100644 packages/orchestration/tests/Support/TestSupport.php diff --git a/packages/orchestration/src/Saga/Compensation/CompensationErrorHandler.php b/packages/orchestration/src/Saga/Compensation/CompensationErrorHandler.php new file mode 100644 index 00000000..e13c28c9 --- /dev/null +++ b/packages/orchestration/src/Saga/Compensation/CompensationErrorHandler.php @@ -0,0 +1,17 @@ + */ + private array $factories; + + public function __construct( + private readonly Container $container, + LoggerInterface $logger, + ) { + $this->factories = [ + 'default' => static fn (): CompensationErrorHandler => new DefaultCompensationErrorHandler, + 'fail-fast' => static fn (): CompensationErrorHandler => new FailFastCompensationErrorHandler, + 'log-and-continue' => static fn (): CompensationErrorHandler => new LogAndContinueCompensationErrorHandler($logger), + 'retry' => static fn (): CompensationErrorHandler => new RetryWithBackoffCompensationErrorHandler(3), + 'robust' => static fn (): CompensationErrorHandler => new RobustCompensationErrorHandler($logger, 3), + ]; + } + + /** + * @param Closure(): CompensationErrorHandler $factory + */ + public function register(string $name, Closure $factory): void + { + $this->factories[$name] = $factory; + } + + /** @return list */ + public function names(): array + { + return array_map('strval', array_keys($this->factories)); + } + + public function get(string $name): CompensationErrorHandler + { + if (isset($this->factories[$name])) { + return ($this->factories[$name])(); + } + + if (class_exists($name) && is_a($name, CompensationErrorHandler::class, true)) { + $handler = $this->container->make($name); + if ($handler instanceof CompensationErrorHandler) { + return $handler; + } + } + + throw new ConfigurationException("Configuration key [firefly.orchestration.saga.compensation-error-handler] names [{$name}], which is neither one of ".implode(', ', $this->names()).' nor a class implementing '.CompensationErrorHandler::class.'.'); + } +} diff --git a/packages/orchestration/src/Saga/Compensation/CompensationErrorResult.php b/packages/orchestration/src/Saga/Compensation/CompensationErrorResult.php new file mode 100644 index 00000000..fd23f594 --- /dev/null +++ b/packages/orchestration/src/Saga/Compensation/CompensationErrorResult.php @@ -0,0 +1,19 @@ + $handlers + */ + public function __construct(private readonly array $handlers) {} + + public function handle(string $sagaName, string $stepId, Throwable $error, int $attempt): CompensationErrorResult + { + foreach ($this->handlers as $handler) { + $result = $handler->handle($sagaName, $stepId, $error, $attempt); + if ($result !== CompensationErrorResult::Continue) { + return $result; + } + } + + return CompensationErrorResult::Continue; + } +} diff --git a/packages/orchestration/src/Saga/Compensation/DefaultCompensationErrorHandler.php b/packages/orchestration/src/Saga/Compensation/DefaultCompensationErrorHandler.php new file mode 100644 index 00000000..8d01f594 --- /dev/null +++ b/packages/orchestration/src/Saga/Compensation/DefaultCompensationErrorHandler.php @@ -0,0 +1,16 @@ +logger->warning("Saga [{$sagaName}] could not compensate step [{$stepId}] (attempt {$attempt}); continuing: {$error->getMessage()}"); + + return CompensationErrorResult::Continue; + } +} diff --git a/packages/orchestration/src/Saga/Compensation/RetryWithBackoffCompensationErrorHandler.php b/packages/orchestration/src/Saga/Compensation/RetryWithBackoffCompensationErrorHandler.php new file mode 100644 index 00000000..56c326b8 --- /dev/null +++ b/packages/orchestration/src/Saga/Compensation/RetryWithBackoffCompensationErrorHandler.php @@ -0,0 +1,46 @@ + $retryable + */ + public function __construct( + private readonly int $maxRetries = 3, + private readonly array $retryable = [], + ) {} + + public function handle(string $sagaName, string $stepId, Throwable $error, int $attempt): CompensationErrorResult + { + if (! $this->isRetryable($error)) { + return CompensationErrorResult::FailSaga; + } + + return $attempt < $this->maxRetries ? CompensationErrorResult::Retry : CompensationErrorResult::FailSaga; + } + + private function isRetryable(Throwable $error): bool + { + if ($this->retryable === []) { + return true; + } + + foreach ($this->retryable as $type) { + if (is_a($error, $type)) { + return true; + } + } + + return false; + } +} diff --git a/packages/orchestration/src/Saga/Compensation/RobustCompensationErrorHandler.php b/packages/orchestration/src/Saga/Compensation/RobustCompensationErrorHandler.php new file mode 100644 index 00000000..6a7240ae --- /dev/null +++ b/packages/orchestration/src/Saga/Compensation/RobustCompensationErrorHandler.php @@ -0,0 +1,28 @@ +maxRetries) { + return CompensationErrorResult::Retry; + } + + $this->logger->warning("Saga [{$sagaName}] gave up compensating step [{$stepId}] after {$attempt} retries; continuing: {$error->getMessage()}"); + + return CompensationErrorResult::Continue; + } +} diff --git a/packages/orchestration/src/Saga/Compensation/SagaCompensator.php b/packages/orchestration/src/Saga/Compensation/SagaCompensator.php new file mode 100644 index 00000000..79e5ae71 --- /dev/null +++ b/packages/orchestration/src/Saga/Compensation/SagaCompensator.php @@ -0,0 +1,270 @@ +policy; + } + + public function compensate(SagaDefinition $saga, ExecutionContext $context): void + { + $completed = array_values(array_filter( + $context->completionOrder(), + static fn (string $stepId): bool => isset($saga->steps[$stepId]) && $context->stepStatus($stepId) === StepStatus::Done, + )); + if ($completed === []) { + return; + } + + $this->support->events->onCompensationStarted($saga->name, $context->correlationId); + $reversed = array_reverse($completed); + + match ($this->policy) { + CompensationPolicy::StrictSequential => $this->sequentially($saga, $context, $reversed, withRetry: false, honourFailSaga: true), + CompensationPolicy::RetryWithBackoff => $this->sequentially($saga, $context, $reversed, withRetry: true, honourFailSaga: true), + CompensationPolicy::BestEffort => $this->sequentially($saga, $context, $reversed, withRetry: false, honourFailSaga: false), + CompensationPolicy::GroupedParallel => $this->sequentially($saga, $context, $this->byLayer($saga, $completed), withRetry: false, honourFailSaga: true), + CompensationPolicy::CircuitBreaker => $this->throughBreaker($saga, $context, $reversed), + }; + } + + /** + * @param list $order + */ + private function sequentially(SagaDefinition $saga, ExecutionContext $context, array $order, bool $withRetry, bool $honourFailSaga): void + { + foreach ($order as $stepId) { + $step = $saga->steps[$stepId]; + if ($step->compensation === null) { + continue; + } + + $options = $withRetry ? $step->compensationOptions() : new StepOptions(timeoutMs: $step->compensationOptions()->timeoutMs); + $error = $this->attempt($saga, $context, $stepId, $options, null); + if ($error !== null) { + $this->onFailure($saga, $context, $stepId, $error, $honourFailSaga); + } + } + } + + /** + * @param list $reversed + */ + private function throughBreaker(SagaDefinition $saga, ExecutionContext $context, array $reversed): void + { + $breaker = $this->breaker($saga->name); + $left = []; + $open = false; + + foreach ($reversed as $stepId) { + $step = $saga->steps[$stepId]; + if ($step->compensation === null) { + continue; + } + if ($open) { + $left[] = $stepId; + + continue; + } + + try { + $error = $this->attempt($saga, $context, $stepId, new StepOptions(timeoutMs: $step->compensationOptions()->timeoutMs), $breaker); + } catch (CircuitBreakerOpenException) { + $open = true; + $left[] = $stepId; + + continue; + } + + if ($error === null) { + continue; + } + if ($step->compensationCritical || ($breaker !== null && $breaker->state() === 'open')) { + $open = true; + + continue; + } + + $this->onFailure($saga, $context, $stepId, $error, honourFailSaga: true); + } + + if ($left !== []) { + $this->support->logger->warning("Saga [{$saga->name}] opened its compensation circuit and left [".implode(', ', $left).'] uncompensated; they stay DONE for recovery or an operator.', [ + 'correlationId' => $context->correlationId, + ]); + } + } + + /** Runs one compensation; returns its error, or null when it succeeded. Throws only when `$breaker` refused admission. */ + private function attempt(SagaDefinition $saga, ExecutionContext $context, string $stepId, StepOptions $options, ?CircuitBreaker $breaker): ?Throwable + { + $step = $saga->steps[$stepId]; + $compensation = $step->compensation; + if ($compensation === null) { + return null; + } + + // Only OUR breaker refusing admission stops the sweep: a compensation that itself calls a downstream + // service behind an open breaker failed like any other compensation and goes to the error handler. + $ran = false; + $run = function () use ($saga, $stepId, $compensation, $options, $context, &$ran): mixed { + $ran = true; + + return $this->invoker->invoke($saga->name, $stepId.':compensate', $compensation, $options, $context, $context->stepInput($stepId), $context->result($stepId)); + }; + + try { + $result = $breaker === null ? $run() : $breaker->call($run); + } catch (Throwable $error) { + if ($error instanceof CircuitBreakerOpenException && $breaker !== null && ! $ran) { + throw $error; + } + + $context->putCompensationError($stepId, $error); + $context->setStepStatus($stepId, StepStatus::CompensationFailed); + $this->support->events->onStepCompensationFailed($saga->name, $context->correlationId, $stepId, $error); + $this->support->checkpoint($context, ExecutionStatus::Compensating); + + return $error; + } + + $context->putCompensationResult($stepId, $result); + $this->markCompensated($saga, $context, $stepId); + + return null; + } + + private function onFailure(SagaDefinition $saga, ExecutionContext $context, string $stepId, Throwable $error, bool $honourFailSaga): void + { + $step = $saga->steps[$stepId]; + + for ($attempt = 0; $attempt < self::MAX_HANDLER_RETRIES; $attempt++) { + $decision = $this->handler->handle($saga->name, $stepId, $error, $attempt); + + if ($decision === CompensationErrorResult::Continue || $decision === CompensationErrorResult::SkipStep) { + return; + } + + if ($decision === CompensationErrorResult::MarkCompensated) { + $this->markCompensated($saga, $context, $stepId); + + return; + } + + if ($decision === CompensationErrorResult::FailSaga) { + if ($honourFailSaga) { + throw new CompensationException("Saga [{$saga->name}] stopped compensating at step [{$stepId}]: {$error->getMessage()}", $error); + } + + return; + } + + $backoff = $step->compensationOptions()->backoffMs; + if ($backoff > 0) { + usleep($backoff * 1000); + } + + $retryError = $this->attempt($saga, $context, $stepId, new StepOptions(timeoutMs: $step->compensationOptions()->timeoutMs), null); + if ($retryError === null) { + return; + } + $error = $retryError; + } + + $this->support->logger->warning("Saga [{$saga->name}] stopped retrying the compensation of step [{$stepId}] after ".self::MAX_HANDLER_RETRIES.' handler retries.', [ + 'correlationId' => $context->correlationId, + ]); + } + + private function markCompensated(SagaDefinition $saga, ExecutionContext $context, string $stepId): void + { + $context->setStepStatus($stepId, StepStatus::Compensated); + $this->support->events->onStepCompensated($saga->name, $context->correlationId, $stepId); + $this->support->checkpoint($context, ExecutionStatus::Compensating); + } + + /** + * @param list $completed + * @return list + */ + private function byLayer(SagaDefinition $saga, array $completed): array + { + $order = []; + foreach (array_reverse($saga->layers) as $layer) { + foreach ($layer as $stepId) { + if (in_array($stepId, $completed, true)) { + $order[] = $stepId; + } + } + } + + return $order; + } + + private function breaker(string $sagaName): ?CircuitBreaker + { + if ($this->resilience !== null) { + try { + return $this->resilience->circuitBreaker('orchestration-compensation-'.$sagaName); + } catch (ConfigurationException) { + // Not configured under firefly.resilience.circuit-breaker: fall through to the default breaker. + } + } + + return $this->store === null ? null : new CircuitBreaker( + key: 'firefly:orchestration:compensation:'.$sagaName, + store: $this->store, + failureThreshold: 5, + waitDurationInOpen: 30.0, + ); + } +} diff --git a/packages/orchestration/tests/Saga/Compensation/CompensationErrorHandlerTest.php b/packages/orchestration/tests/Saga/Compensation/CompensationErrorHandlerTest.php new file mode 100644 index 00000000..3549a93f --- /dev/null +++ b/packages/orchestration/tests/Saga/Compensation/CompensationErrorHandlerTest.php @@ -0,0 +1,51 @@ +handle('s', 'a', $error, 0))->toBe(CompensationErrorResult::Continue) + ->and((new FailFastCompensationErrorHandler)->handle('s', 'a', $error, 0))->toBe(CompensationErrorResult::FailSaga) + ->and($retry->handle('s', 'a', $error, 0))->toBe(CompensationErrorResult::Retry) + ->and($retry->handle('s', 'a', $error, 2))->toBe(CompensationErrorResult::FailSaga) + ->and($retry->handle('s', 'a', new LogicException('y'), 0))->toBe(CompensationErrorResult::FailSaga) + ->and((new CompositeCompensationErrorHandler([new DefaultCompensationErrorHandler, new FailFastCompensationErrorHandler]))->handle('s', 'a', $error, 0))->toBe(CompensationErrorResult::FailSaga); +}); + +it('resolves the built-in names, a robust chain, and an application class by name', function (): void { + $logger = new RecordingLogger; + $factory = new CompensationErrorHandlerFactory(new Container, $logger); + $robust = $factory->get('robust'); + + expect($factory->names())->toBe(['default', 'fail-fast', 'log-and-continue', 'retry', 'robust']) + ->and($factory->get('log-and-continue')->handle('s', 'a', new RuntimeException('x'), 0))->toBe(CompensationErrorResult::Continue) + ->and($robust->handle('s', 'a', new RuntimeException('x'), 0))->toBe(CompensationErrorResult::Retry) + ->and($robust->handle('s', 'a', new RuntimeException('x'), 3))->toBe(CompensationErrorResult::Continue) + ->and($factory->get(OrchestrationSkipEverything::class)->handle('s', 'a', new RuntimeException('x'), 0))->toBe(CompensationErrorResult::SkipStep) + ->and($logger->messages('warning'))->not->toBe([]); +}); + +it('refuses an unknown handler name, listing the known ones', function (): void { + (new CompensationErrorHandlerFactory(new Container, new RecordingLogger))->get('pessimistic'); +})->throws(ConfigurationException::class, 'default, fail-fast, log-and-continue, retry, robust'); diff --git a/packages/orchestration/tests/Saga/Compensation/SagaCompensatorTest.php b/packages/orchestration/tests/Saga/Compensation/SagaCompensatorTest.php new file mode 100644 index 00000000..137784d3 --- /dev/null +++ b/packages/orchestration/tests/Saga/Compensation/SagaCompensatorTest.php @@ -0,0 +1,273 @@ +> $steps step id => dependsOn + * @param array $failing + * @param list $log + * @param list $critical + */ +function orchestrationCompensationSaga(array $steps, array $failing, array &$log, array $critical = [], int $compensationRetry = -1): SagaDefinition +{ + $builder = SagaBuilder::saga('order'); + // One counter per step, shared by reference: a loop variable captured by reference would be the SAME + // variable in every closure. + $remaining = $failing; + foreach ($steps as $id => $dependsOn) { + $id = (string) $id; + $builder = $builder->step($id) + ->dependsOn(...$dependsOn) + ->handler(static fn (): string => "did {$id}") + ->compensation(static function (mixed $result) use ($id, &$log, &$remaining): string { + $log[] = $id; + if (($remaining[$id] ?? 0) > 0) { + $remaining[$id]--; + + throw new RuntimeException("cannot undo {$id}"); + } + + return "undid {$id}"; + }) + ->compensationRetry($compensationRetry) + ->compensationBackoffMs(1) + ->compensationCritical(in_array($id, $critical, true)) + ->add(); + } + + return $builder->build(); +} + +/** @param list $completed */ +function orchestrationCompletedContext(SagaDefinition $saga, array $completed): ExecutionContext +{ + $context = ExecutionContext::start(ExecutionPattern::Saga, $saga->name, [], [], 'c-1', new DateTimeImmutable); + foreach ($completed as $id) { + $context->putResult($id, "did {$id}"); + $context->setStepStatus($id, StepStatus::Done); + $context->markCompleted($id); + } + + return $context; +} + +function orchestrationCompensator(CompensationPolicy $policy, CompensationErrorHandler $handler, ?RecordingOrchestrationEvents $events = null, ?RecordingLogger $logger = null, ?ResilienceRegistry $resilience = null): SagaCompensator +{ + return new SagaCompensator(TestSupport::invoker(), TestSupport::engineSupport(events: $events, logger: $logger), $policy, $handler, $resilience); +} + +it('STRICT_SEQUENTIAL undoes in reverse completion order and, by default, continues past a failure', function (): void { + $log = []; + $saga = orchestrationCompensationSaga(['a' => [], 'b' => ['a'], 'c' => ['b']], ['b' => PHP_INT_MAX], $log); + $context = orchestrationCompletedContext($saga, ['a', 'b', 'c']); + $events = new RecordingOrchestrationEvents; + + orchestrationCompensator(CompensationPolicy::StrictSequential, new DefaultCompensationErrorHandler, $events)->compensate($saga, $context); + + expect($log)->toBe(['c', 'b', 'a']) + ->and($context->stepStatus('c'))->toBe(StepStatus::Compensated) + ->and($context->stepStatus('b'))->toBe(StepStatus::CompensationFailed) + ->and($context->compensationError('b')?->getMessage())->toBe('cannot undo b') + ->and($context->stepStatus('a'))->toBe(StepStatus::Compensated) + ->and($context->compensationResult('a'))->toBe('undid a') + ->and($events->hooks())->toBe(['onCompensationStarted', 'onStepCompensated', 'onStepCompensationFailed', 'onStepCompensated']); +}); + +it('stops at the first failure when the handler fails the saga', function (): void { + $log = []; + $saga = orchestrationCompensationSaga(['a' => [], 'b' => ['a'], 'c' => ['b']], ['b' => PHP_INT_MAX], $log); + $context = orchestrationCompletedContext($saga, ['a', 'b', 'c']); + + expect(fn () => orchestrationCompensator(CompensationPolicy::StrictSequential, new FailFastCompensationErrorHandler)->compensate($saga, $context)) + ->toThrow(CompensationException::class, 'Saga [order] stopped compensating at step [b]: cannot undo b') + ->and($log)->toBe(['c', 'b']) + ->and($context->stepStatus('a'))->toBe(StepStatus::Done); +}); + +it('GROUPED_PARALLEL undoes layer by layer in reverse, declaration order inside a layer', function (): void { + $log = []; + $saga = orchestrationCompensationSaga(['a' => [], 'b' => ['a'], 'c' => ['a'], 'd' => ['b', 'c']], [], $log); + + orchestrationCompensator(CompensationPolicy::GroupedParallel, new DefaultCompensationErrorHandler)->compensate($saga, orchestrationCompletedContext($saga, ['a', 'c', 'b', 'd'])); + + expect($log)->toBe(['d', 'b', 'c', 'a']); +}); + +it('RETRY_WITH_BACKOFF retries a compensation compensationRetry times', function (): void { + $log = []; + $saga = orchestrationCompensationSaga(['a' => [], 'b' => ['a']], ['b' => 2], $log, compensationRetry: 2); + $context = orchestrationCompletedContext($saga, ['a', 'b']); + + orchestrationCompensator(CompensationPolicy::RetryWithBackoff, new DefaultCompensationErrorHandler)->compensate($saga, $context); + + expect($log)->toBe(['b', 'b', 'b', 'a']) + ->and($context->stepStatus('b'))->toBe(StepStatus::Compensated) + ->and($context->attempts('b:compensate'))->toBe(3); +}); + +it('lets the retry handler try a failed compensation again', function (): void { + $log = []; + $saga = orchestrationCompensationSaga(['a' => []], ['a' => 1], $log); + $context = orchestrationCompletedContext($saga, ['a']); + + orchestrationCompensator(CompensationPolicy::StrictSequential, new RetryWithBackoffCompensationErrorHandler(3))->compensate($saga, $context); + + expect($log)->toBe(['a', 'a'])->and($context->stepStatus('a'))->toBe(StepStatus::Compensated); +}); + +it('lets a handler mark a failed compensation as compensated', function (): void { + $log = []; + $saga = orchestrationCompensationSaga(['a' => []], ['a' => PHP_INT_MAX], $log); + $context = orchestrationCompletedContext($saga, ['a']); + $handler = new class implements CompensationErrorHandler + { + public function handle(string $sagaName, string $stepId, Throwable $error, int $attempt): CompensationErrorResult + { + return CompensationErrorResult::MarkCompensated; + } + }; + + orchestrationCompensator(CompensationPolicy::StrictSequential, $handler)->compensate($saga, $context); + + expect($context->stepStatus('a'))->toBe(StepStatus::Compensated); +}); + +it('CIRCUIT_BREAKER stops compensating after a critical step fails and leaves the rest for recovery', function (): void { + $log = []; + $logger = new RecordingLogger; + $saga = orchestrationCompensationSaga(['a' => [], 'b' => ['a'], 'c' => ['b']], ['c' => PHP_INT_MAX], $log, critical: ['c']); + $context = orchestrationCompletedContext($saga, ['a', 'b', 'c']); + + orchestrationCompensator(CompensationPolicy::CircuitBreaker, new DefaultCompensationErrorHandler, logger: $logger)->compensate($saga, $context); + + expect($log)->toBe(['c']) + ->and($context->stepStatus('b'))->toBe(StepStatus::Done) + ->and($context->stepStatus('a'))->toBe(StepStatus::Done) + ->and(implode("\n", $logger->messages('warning')))->toContain('left [b, a] uncompensated'); +}); + +it('CIRCUIT_BREAKER shares a firefly/resilience breaker per saga across executions', function (): void { + $log = []; + $resilience = ResilienceRegistry::fromConfig(new Config(new Repository(['firefly' => ['resilience' => ['circuit-breaker' => ['orchestration-compensation-order' => ['failure-threshold' => 1]]]]])), new InMemoryResilienceStore); + $saga = orchestrationCompensationSaga(['a' => [], 'b' => ['a'], 'c' => ['b']], ['c' => PHP_INT_MAX], $log); + $context = orchestrationCompletedContext($saga, ['a', 'b', 'c']); + + orchestrationCompensator(CompensationPolicy::CircuitBreaker, new DefaultCompensationErrorHandler, resilience: $resilience)->compensate($saga, $context); + + expect($log)->toBe(['c'])->and($context->stepStatus('b'))->toBe(StepStatus::Done); +}); + +it('BEST_EFFORT attempts every compensation even when the handler would fail the saga', function (): void { + $log = []; + $saga = orchestrationCompensationSaga(['a' => [], 'b' => ['a'], 'c' => ['b']], ['b' => PHP_INT_MAX, 'c' => PHP_INT_MAX], $log); + $context = orchestrationCompletedContext($saga, ['a', 'b', 'c']); + + orchestrationCompensator(CompensationPolicy::BestEffort, new FailFastCompensationErrorHandler)->compensate($saga, $context); + + expect($log)->toBe(['c', 'b', 'a'])->and($context->stepStatus('a'))->toBe(StepStatus::Compensated); +}); + +it('skips steps that never completed and steps without a compensation', function (): void { + $log = []; + $saga = SagaBuilder::saga('order') + ->step('a')->handler(static fn (): string => 'a')->add() + ->step('b')->handler(static fn (): string => 'b')->compensation(static function () use (&$log): void { + $log[] = 'b'; + })->add() + ->build(); + $context = orchestrationCompletedContext($saga, ['a']); + + orchestrationCompensator(CompensationPolicy::StrictSequential, new DefaultCompensationErrorHandler)->compensate($saga, $context); + + expect($log)->toBe([])->and($context->stepStatus('a'))->toBe(StepStatus::Done); +}); + +it('records a compensation that hits a downstream open breaker as failed instead of escaping the sweep', function (): void { + $log = []; + $saga = SagaBuilder::saga('order') + ->step('a')->handler(static fn (): string => 'a')->compensation(static function () use (&$log): string { + $log[] = 'a'; + + return 'undid a'; + })->add() + ->step('b')->dependsOn('a')->handler(static fn (): string => 'b')->compensation(static function () use (&$log): never { + $log[] = 'b'; + + throw new CircuitBreakerOpenException; + })->add() + ->build(); + + foreach ([CompensationPolicy::StrictSequential, CompensationPolicy::CircuitBreaker] as $policy) { + $log = []; + $context = orchestrationCompletedContext($saga, ['a', 'b']); + + orchestrationCompensator($policy, new DefaultCompensationErrorHandler)->compensate($saga, $context); + + expect($log)->toBe(['b', 'a']) + ->and($context->stepStatus('b'))->toBe(StepStatus::CompensationFailed) + ->and($context->compensationError('b'))->toBeInstanceOf(CircuitBreakerOpenException::class) + ->and($context->stepStatus('a'))->toBe(StepStatus::Compensated); + } +}); + +it('cuts a handler that keeps answering RETRY off after MAX_HANDLER_RETRIES retries', function (): void { + $log = []; + $logger = new RecordingLogger; + $saga = orchestrationCompensationSaga(['a' => []], ['a' => PHP_INT_MAX], $log); + $context = orchestrationCompletedContext($saga, ['a']); + $handler = new class implements CompensationErrorHandler + { + public function handle(string $sagaName, string $stepId, Throwable $error, int $attempt): CompensationErrorResult + { + return CompensationErrorResult::Retry; + } + }; + + orchestrationCompensator(CompensationPolicy::StrictSequential, $handler, logger: $logger)->compensate($saga, $context); + + expect($context->attempts('a:compensate'))->toBe(1 + SagaCompensator::MAX_HANDLER_RETRIES) + ->and($context->stepStatus('a'))->toBe(StepStatus::CompensationFailed) + ->and(implode("\n", $logger->messages('warning')))->toContain('after 25 handler retries'); +}); + +it('CIRCUIT_BREAKER refuses the next execution of the same saga while its breaker is open', function (): void { + $log = []; + $logger = new RecordingLogger; + $resilience = ResilienceRegistry::fromConfig(new Config(new Repository(['firefly' => ['resilience' => ['circuit-breaker' => ['orchestration-compensation-order' => ['failure-threshold' => 1]]]]])), new InMemoryResilienceStore); + $saga = orchestrationCompensationSaga(['a' => [], 'b' => ['a']], ['b' => PHP_INT_MAX], $log); + + orchestrationCompensator(CompensationPolicy::CircuitBreaker, new DefaultCompensationErrorHandler, resilience: $resilience)->compensate($saga, orchestrationCompletedContext($saga, ['a', 'b'])); + $log = []; + $second = orchestrationCompletedContext($saga, ['a', 'b']); + orchestrationCompensator(CompensationPolicy::CircuitBreaker, new DefaultCompensationErrorHandler, logger: $logger, resilience: $resilience)->compensate($saga, $second); + + expect($log)->toBe([]) + ->and($second->stepStatus('b'))->toBe(StepStatus::Done) + ->and($second->stepStatus('a'))->toBe(StepStatus::Done) + ->and(implode("\n", $logger->messages('warning')))->toContain('left [b, a] uncompensated'); +}); diff --git a/packages/orchestration/tests/Support/TestSupport.php b/packages/orchestration/tests/Support/TestSupport.php new file mode 100644 index 00000000..e4c5a571 --- /dev/null +++ b/packages/orchestration/tests/Support/TestSupport.php @@ -0,0 +1,63 @@ + Date: Mon, 12 Oct 2026 19:42:46 -0700 Subject: [PATCH 13/58] fix(orchestration): run compensation handler retries through the saga circuit breaker --- .../src/Saga/Compensation/SagaCompensator.php | 36 ++++++++++++++----- .../Saga/Compensation/SagaCompensatorTest.php | 34 ++++++++++++++++++ 2 files changed, 62 insertions(+), 8 deletions(-) diff --git a/packages/orchestration/src/Saga/Compensation/SagaCompensator.php b/packages/orchestration/src/Saga/Compensation/SagaCompensator.php index 79e5ae71..ccd7015f 100644 --- a/packages/orchestration/src/Saga/Compensation/SagaCompensator.php +++ b/packages/orchestration/src/Saga/Compensation/SagaCompensator.php @@ -34,7 +34,8 @@ * recovery or an operator. * - BEST_EFFORT: reverse completion order, every compensation attempted, FAIL_SAGA ignored. * - * The error handler's RETRY answer re-runs the compensation after the step's compensation backoff; a handler + * The error handler's RETRY answer re-runs the compensation after the step's compensation backoff (under + * CIRCUIT_BREAKER through the saga's breaker, so retries count towards its threshold); a handler * that keeps answering RETRY is cut off after MAX_HANDLER_RETRIES and treated as CONTINUE. */ final class SagaCompensator @@ -134,7 +135,11 @@ private function throughBreaker(SagaDefinition $saga, ExecutionContext $context, continue; } - $this->onFailure($saga, $context, $stepId, $error, honourFailSaga: true); + // The handler's retries go through the same breaker: they count towards its failure threshold, and a + // retry it refuses — or one that opens it — stops the sweep like any other open circuit. + if ($this->onFailure($saga, $context, $stepId, $error, honourFailSaga: true, breaker: $breaker)) { + $open = true; + } } if ($left !== []) { @@ -183,7 +188,13 @@ private function attempt(SagaDefinition $saga, ExecutionContext $context, string return null; } - private function onFailure(SagaDefinition $saga, ExecutionContext $context, string $stepId, Throwable $error, bool $honourFailSaga): void + /** + * Asks the error handler what to do with a failed compensation and carries out its answer. Under + * CIRCUIT_BREAKER `$breaker` is the saga's breaker and every RETRY runs through it; the method then + * returns true when that breaker refused a retry or opened after one, so the caller stops the sweep. + * Without a breaker it always returns false. + */ + private function onFailure(SagaDefinition $saga, ExecutionContext $context, string $stepId, Throwable $error, bool $honourFailSaga, ?CircuitBreaker $breaker = null): bool { $step = $saga->steps[$stepId]; @@ -191,13 +202,13 @@ private function onFailure(SagaDefinition $saga, ExecutionContext $context, stri $decision = $this->handler->handle($saga->name, $stepId, $error, $attempt); if ($decision === CompensationErrorResult::Continue || $decision === CompensationErrorResult::SkipStep) { - return; + return false; } if ($decision === CompensationErrorResult::MarkCompensated) { $this->markCompensated($saga, $context, $stepId); - return; + return false; } if ($decision === CompensationErrorResult::FailSaga) { @@ -205,7 +216,7 @@ private function onFailure(SagaDefinition $saga, ExecutionContext $context, stri throw new CompensationException("Saga [{$saga->name}] stopped compensating at step [{$stepId}]: {$error->getMessage()}", $error); } - return; + return false; } $backoff = $step->compensationOptions()->backoffMs; @@ -213,9 +224,16 @@ private function onFailure(SagaDefinition $saga, ExecutionContext $context, stri usleep($backoff * 1000); } - $retryError = $this->attempt($saga, $context, $stepId, new StepOptions(timeoutMs: $step->compensationOptions()->timeoutMs), null); + try { + $retryError = $this->attempt($saga, $context, $stepId, new StepOptions(timeoutMs: $step->compensationOptions()->timeoutMs), $breaker); + } catch (CircuitBreakerOpenException) { + return true; + } if ($retryError === null) { - return; + return false; + } + if ($breaker !== null && $breaker->state() === 'open') { + return true; } $error = $retryError; } @@ -223,6 +241,8 @@ private function onFailure(SagaDefinition $saga, ExecutionContext $context, stri $this->support->logger->warning("Saga [{$saga->name}] stopped retrying the compensation of step [{$stepId}] after ".self::MAX_HANDLER_RETRIES.' handler retries.', [ 'correlationId' => $context->correlationId, ]); + + return false; } private function markCompensated(SagaDefinition $saga, ExecutionContext $context, string $stepId): void diff --git a/packages/orchestration/tests/Saga/Compensation/SagaCompensatorTest.php b/packages/orchestration/tests/Saga/Compensation/SagaCompensatorTest.php index 137784d3..637a4cf1 100644 --- a/packages/orchestration/tests/Saga/Compensation/SagaCompensatorTest.php +++ b/packages/orchestration/tests/Saga/Compensation/SagaCompensatorTest.php @@ -271,3 +271,37 @@ public function handle(string $sagaName, string $stepId, Throwable $error, int $ ->and($second->stepStatus('a'))->toBe(StepStatus::Done) ->and(implode("\n", $logger->messages('warning')))->toContain('left [b, a] uncompensated'); }); + +it('CIRCUIT_BREAKER runs the handler retries through the saga breaker and stops the sweep once they open it', function (): void { + $log = []; + $logger = new RecordingLogger; + $resilience = ResilienceRegistry::fromConfig(new Config(new Repository(['firefly' => ['resilience' => ['circuit-breaker' => ['orchestration-compensation-order' => ['failure-threshold' => 2]]]]])), new InMemoryResilienceStore); + $saga = orchestrationCompensationSaga(['a' => [], 'b' => ['a'], 'c' => ['b']], ['c' => PHP_INT_MAX], $log); + $context = orchestrationCompletedContext($saga, ['a', 'b', 'c']); + + orchestrationCompensator(CompensationPolicy::CircuitBreaker, new RetryWithBackoffCompensationErrorHandler(3), logger: $logger, resilience: $resilience)->compensate($saga, $context); + + // The first try and ONE retry reach the breaker's threshold of two; the remaining two retries the handler + // asked for never hit the failing dependency, and the sweep leaves b and a for recovery. + expect($log)->toBe(['c', 'c']) + ->and($context->attempts('c:compensate'))->toBe(2) + ->and($resilience->circuitBreaker('orchestration-compensation-order')->state())->toBe('open') + ->and($context->stepStatus('c'))->toBe(StepStatus::CompensationFailed) + ->and($context->stepStatus('b'))->toBe(StepStatus::Done) + ->and($context->stepStatus('a'))->toBe(StepStatus::Done) + ->and(implode("\n", $logger->messages('warning')))->toContain('left [b, a] uncompensated'); +}); + +it('CIRCUIT_BREAKER counts a handler retry that succeeds as a breaker success and carries on', function (): void { + $log = []; + $resilience = ResilienceRegistry::fromConfig(new Config(new Repository(['firefly' => ['resilience' => ['circuit-breaker' => ['orchestration-compensation-order' => ['failure-threshold' => 2]]]]])), new InMemoryResilienceStore); + $saga = orchestrationCompensationSaga(['a' => [], 'b' => ['a']], ['b' => 1], $log); + $context = orchestrationCompletedContext($saga, ['a', 'b']); + + orchestrationCompensator(CompensationPolicy::CircuitBreaker, new RetryWithBackoffCompensationErrorHandler(3), resilience: $resilience)->compensate($saga, $context); + + expect($log)->toBe(['b', 'b', 'a']) + ->and($context->stepStatus('b'))->toBe(StepStatus::Compensated) + ->and($context->stepStatus('a'))->toBe(StepStatus::Compensated) + ->and($resilience->circuitBreaker('orchestration-compensation-order')->state())->toBe('closed'); +}); From 3f672eeff419a5a7c91efe4a0eb943f9db83c5b5 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 19:45:52 -0700 Subject: [PATCH 14/58] fix(orchestration): fall back to the default compensation breaker only when the saga has none configured --- .../src/Saga/Compensation/SagaCompensator.php | 13 +++++----- .../Saga/Compensation/SagaCompensatorTest.php | 26 +++++++++++++++++++ .../resilience/src/ResilienceRegistry.php | 13 ++++++++++ .../tests/ResilienceRegistryTest.php | 12 +++++++++ 4 files changed, 57 insertions(+), 7 deletions(-) diff --git a/packages/orchestration/src/Saga/Compensation/SagaCompensator.php b/packages/orchestration/src/Saga/Compensation/SagaCompensator.php index ccd7015f..dcd8ac79 100644 --- a/packages/orchestration/src/Saga/Compensation/SagaCompensator.php +++ b/packages/orchestration/src/Saga/Compensation/SagaCompensator.php @@ -4,7 +4,6 @@ namespace Firefly\Orchestration\Saga\Compensation; -use Firefly\Kernel\Exception\Framework\ConfigurationException; use Firefly\Kernel\Exception\Infrastructure\CircuitBreakerOpenException; use Firefly\Orchestration\Core\CompensationPolicy; use Firefly\Orchestration\Core\EngineSupport; @@ -272,12 +271,12 @@ private function byLayer(SagaDefinition $saga, array $completed): array private function breaker(string $sagaName): ?CircuitBreaker { - if ($this->resilience !== null) { - try { - return $this->resilience->circuitBreaker('orchestration-compensation-'.$sagaName); - } catch (ConfigurationException) { - // Not configured under firefly.resilience.circuit-breaker: fall through to the default breaker. - } + // Fall back to the default breaker only when firefly.resilience.circuit-breaker has no instance for this + // saga. A configured instance with an invalid value throws its ConfigurationException here instead of + // silently running on thresholds the operator never set. + $name = 'orchestration-compensation-'.$sagaName; + if ($this->resilience !== null && $this->resilience->hasCircuitBreaker($name)) { + return $this->resilience->circuitBreaker($name); } return $this->store === null ? null : new CircuitBreaker( diff --git a/packages/orchestration/tests/Saga/Compensation/SagaCompensatorTest.php b/packages/orchestration/tests/Saga/Compensation/SagaCompensatorTest.php index 637a4cf1..dcbb2eb7 100644 --- a/packages/orchestration/tests/Saga/Compensation/SagaCompensatorTest.php +++ b/packages/orchestration/tests/Saga/Compensation/SagaCompensatorTest.php @@ -3,6 +3,7 @@ declare(strict_types=1); use Firefly\Config\Config; +use Firefly\Kernel\Exception\Framework\ConfigurationException; use Firefly\Kernel\Exception\Infrastructure\CircuitBreakerOpenException; use Firefly\Orchestration\Core\CompensationPolicy; use Firefly\Orchestration\Core\Exception\CompensationException; @@ -305,3 +306,28 @@ public function handle(string $sagaName, string $stepId, Throwable $error, int $ ->and($context->stepStatus('a'))->toBe(StepStatus::Compensated) ->and($resilience->circuitBreaker('orchestration-compensation-order')->state())->toBe('closed'); }); + +it('CIRCUIT_BREAKER refuses to run on the default breaker when the configured saga breaker is invalid', function (): void { + $log = []; + $resilience = ResilienceRegistry::fromConfig(new Config(new Repository(['firefly' => ['resilience' => ['circuit-breaker' => ['orchestration-compensation-order' => ['wait-duration-in-open' => 'two minutes']]]]])), new InMemoryResilienceStore); + $saga = orchestrationCompensationSaga(['a' => [], 'b' => ['a']], [], $log); + $context = orchestrationCompletedContext($saga, ['a', 'b']); + $compensator = new SagaCompensator(TestSupport::invoker(), TestSupport::engineSupport(), CompensationPolicy::CircuitBreaker, new DefaultCompensationErrorHandler, $resilience, new InMemoryResilienceStore); + + expect(fn () => $compensator->compensate($saga, $context))->toThrow(ConfigurationException::class, 'Invalid duration [two minutes]') + ->and($log)->toBe([]); +}); + +it('CIRCUIT_BREAKER falls back to the default breaker only when no saga breaker is configured', function (): void { + $log = []; + $resilience = ResilienceRegistry::fromConfig(new Config(new Repository(['firefly' => ['resilience' => ['circuit-breaker' => ['payments' => ['failure-threshold' => 1]]]]])), new InMemoryResilienceStore); + $saga = orchestrationCompensationSaga(['a' => [], 'b' => ['a']], [], $log); + $context = orchestrationCompletedContext($saga, ['a', 'b']); + $compensator = new SagaCompensator(TestSupport::invoker(), TestSupport::engineSupport(), CompensationPolicy::CircuitBreaker, new DefaultCompensationErrorHandler, $resilience, new InMemoryResilienceStore); + + $compensator->compensate($saga, $context); + + expect($log)->toBe(['b', 'a']) + ->and($context->stepStatus('b'))->toBe(StepStatus::Compensated) + ->and($context->stepStatus('a'))->toBe(StepStatus::Compensated); +}); diff --git a/packages/resilience/src/ResilienceRegistry.php b/packages/resilience/src/ResilienceRegistry.php index 19c1e20b..cb59cce8 100644 --- a/packages/resilience/src/ResilienceRegistry.php +++ b/packages/resilience/src/ResilienceRegistry.php @@ -75,6 +75,19 @@ public function circuitBreaker(string $name): CircuitBreaker ); } + /** + * Whether firefly.resilience.circuit-breaker names an instance called $name. Lets a consumer that has its + * own default breaker fall back ONLY when the instance is absent: circuitBreaker() also throws a + * ConfigurationException for a configured instance with an invalid value (a bad duration, say), and that + * one must reach the operator rather than be mistaken for "not configured". + */ + public function hasCircuitBreaker(string $name): bool + { + $group = $this->config['circuit-breaker'] ?? null; + + return is_array($group) && array_key_exists($name, $group); + } + public function rateLimiter(string $name): RateLimiter { return $this->rateLimiters[$name] ??= new RateLimiter( diff --git a/packages/resilience/tests/ResilienceRegistryTest.php b/packages/resilience/tests/ResilienceRegistryTest.php index 4491bcf9..5b3bafb9 100644 --- a/packages/resilience/tests/ResilienceRegistryTest.php +++ b/packages/resilience/tests/ResilienceRegistryTest.php @@ -70,6 +70,18 @@ function makeResilienceRegistry(array $resilience = []): ResilienceRegistry expect(fn () => makeResilienceRegistry()->circuitBreaker('none'))->toThrow(ConfigurationException::class, '(none configured)'); }); +it('reports whether a circuit breaker instance is configured without building it', function () { + // An instance with an invalid value still counts as configured: hasCircuitBreaker() never parses it, so a + // caller that falls back on "absent" cannot mistake a misconfiguration for a missing instance. + $reg = makeResilienceRegistry(['circuit-breaker' => ['payments' => [], 'broken' => ['wait-duration-in-open' => 'two minutes']]]); + + expect($reg->hasCircuitBreaker('payments'))->toBeTrue() + ->and($reg->hasCircuitBreaker('broken'))->toBeTrue() + ->and($reg->hasCircuitBreaker('missing'))->toBeFalse() + ->and(makeResilienceRegistry()->hasCircuitBreaker('payments'))->toBeFalse() + ->and(fn () => $reg->circuitBreaker('broken'))->toThrow(ConfigurationException::class, 'Invalid duration [two minutes]'); +}); + it('wires the circuit breaker minimum-number-of-calls key through to the built instance', function () { // A registry key that never reaches the constructor is indistinguishable from a typo, so assert the // BEHAVIOUR the key buys rather than the object's shape: with a minimum of 3, two failures on a From 0e7257bcb9e91de91d9cd35aeecae3c0e1d13251 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 19:49:50 -0700 Subject: [PATCH 15/58] fix(orchestration): throw a CompensationException when the circuit-breaker policy stops the compensation sweep --- .../Core/Exception/CompensationException.php | 6 +- .../src/Saga/Compensation/SagaCompensator.php | 54 ++++++++++---- .../Saga/Compensation/SagaCompensatorTest.php | 70 +++++++++++++++---- 3 files changed, 104 insertions(+), 26 deletions(-) diff --git a/packages/orchestration/src/Core/Exception/CompensationException.php b/packages/orchestration/src/Core/Exception/CompensationException.php index 18c4f85e..c81bdfc3 100644 --- a/packages/orchestration/src/Core/Exception/CompensationException.php +++ b/packages/orchestration/src/Core/Exception/CompensationException.php @@ -6,7 +6,11 @@ use Throwable; -/** A compensation error handler answered FAIL_SAGA: compensation stopped and the saga ends FAILED. */ +/** + * Compensation stopped before every completed step was undone: an error handler answered FAIL_SAGA, or under + * CIRCUIT_BREAKER the saga's breaker opened or a `compensationCritical` step failed. The message names the + * step that stopped it and becomes the execution's recorded reason. + */ final class CompensationException extends OrchestrationException { public function __construct(string $message, ?Throwable $previous = null) diff --git a/packages/orchestration/src/Saga/Compensation/SagaCompensator.php b/packages/orchestration/src/Saga/Compensation/SagaCompensator.php index dcd8ac79..f92753ab 100644 --- a/packages/orchestration/src/Saga/Compensation/SagaCompensator.php +++ b/packages/orchestration/src/Saga/Compensation/SagaCompensator.php @@ -29,8 +29,9 @@ * - GROUPED_PARALLEL: reverse layer order, declaration order inside a layer (sequential: spec decision D2). * - RETRY_WITH_BACKOFF: reverse completion order with the step's compensation retry/backoff/timeout. * - CIRCUIT_BREAKER: reverse completion order through a firefly/resilience breaker per saga; a failed - * `compensationCritical` step, or an open breaker, stops the sweep and leaves the rest DONE (logged) for - * recovery or an operator. + * `compensationCritical` step, or an open breaker, stops the sweep and throws a CompensationException + * naming the step that stopped it and the steps left DONE. Nothing re-runs them automatically: the + * execution ends with that message as its reason, for an operator to act on. * - BEST_EFFORT: reverse completion order, every compensation attempted, FAIL_SAGA ignored. * * The error handler's RETRY answer re-runs the compensation after the step's compensation backoff (under @@ -97,20 +98,28 @@ private function sequentially(SagaDefinition $saga, ExecutionContext $context, a } /** + * Compensates in reverse completion order through the saga's breaker. When the breaker refuses a step (or + * opens during the sweep), or a `compensationCritical` step fails, the sweep stops and a + * CompensationException names the step that stopped it and the steps left DONE. The engine records that + * message as the execution's reason, so a FAILED or CANCELLED saga never claims a compensation it did not + * finish. + * * @param list $reversed */ private function throughBreaker(SagaDefinition $saga, ExecutionContext $context, array $reversed): void { $breaker = $this->breaker($saga->name); $left = []; - $open = false; + $stoppedAt = null; + $critical = false; + $lastError = null; foreach ($reversed as $stepId) { $step = $saga->steps[$stepId]; if ($step->compensation === null) { continue; } - if ($open) { + if ($stoppedAt !== null) { $left[] = $stepId; continue; @@ -118,8 +127,9 @@ private function throughBreaker(SagaDefinition $saga, ExecutionContext $context, try { $error = $this->attempt($saga, $context, $stepId, new StepOptions(timeoutMs: $step->compensationOptions()->timeoutMs), $breaker); - } catch (CircuitBreakerOpenException) { - $open = true; + } catch (CircuitBreakerOpenException $refused) { + $stoppedAt = $stepId; + $lastError ??= $refused; $left[] = $stepId; continue; @@ -128,8 +138,15 @@ private function throughBreaker(SagaDefinition $saga, ExecutionContext $context, if ($error === null) { continue; } - if ($step->compensationCritical || ($breaker !== null && $breaker->state() === 'open')) { - $open = true; + $lastError = $error; + if ($step->compensationCritical) { + $stoppedAt = $stepId; + $critical = true; + + continue; + } + if ($breaker !== null && $breaker->state() === 'open') { + $stoppedAt = $stepId; continue; } @@ -137,15 +154,28 @@ private function throughBreaker(SagaDefinition $saga, ExecutionContext $context, // The handler's retries go through the same breaker: they count towards its failure threshold, and a // retry it refuses — or one that opens it — stops the sweep like any other open circuit. if ($this->onFailure($saga, $context, $stepId, $error, honourFailSaga: true, breaker: $breaker)) { - $open = true; + $stoppedAt = $stepId; + $lastError = $context->compensationError($stepId) ?? $error; } } + if ($stoppedAt === null) { + return; + } + + $message = $critical + ? "Saga [{$saga->name}] stopped compensating at critical step [{$stoppedAt}]" + : "Saga [{$saga->name}] opened its compensation circuit at step [{$stoppedAt}]"; if ($left !== []) { - $this->support->logger->warning("Saga [{$saga->name}] opened its compensation circuit and left [".implode(', ', $left).'] uncompensated; they stay DONE for recovery or an operator.', [ - 'correlationId' => $context->correlationId, - ]); + $message .= '; ['.implode(', ', $left).'] left uncompensated'; } + + // Every compensated step was already checkpointed under COMPENSATING; the steps left stay DONE, and the + // exception ends the execution with this message as its reason, so an operator sees what to undo. + $this->support->logger->warning($message.'.', ['correlationId' => $context->correlationId]); + $this->support->checkpoint($context, ExecutionStatus::Compensating); + + throw new CompensationException($message.($lastError === null ? '.' : ": {$lastError->getMessage()}"), $lastError); } /** Runs one compensation; returns its error, or null when it succeeded. Throws only when `$breaker` refused admission. */ diff --git a/packages/orchestration/tests/Saga/Compensation/SagaCompensatorTest.php b/packages/orchestration/tests/Saga/Compensation/SagaCompensatorTest.php index dcbb2eb7..df75916a 100644 --- a/packages/orchestration/tests/Saga/Compensation/SagaCompensatorTest.php +++ b/packages/orchestration/tests/Saga/Compensation/SagaCompensatorTest.php @@ -64,6 +64,18 @@ function orchestrationCompensationSaga(array $steps, array $failing, array &$log return $builder->build(); } +/** Runs `$compensate`, which must stop the sweep, and returns the CompensationException it threw. */ +function orchestrationCompensationStopped(Closure $compensate): CompensationException +{ + try { + $compensate(); + } catch (CompensationException $stopped) { + return $stopped; + } + + throw new RuntimeException('Expected the compensation sweep to stop with a CompensationException.'); +} + /** @param list $completed */ function orchestrationCompletedContext(SagaDefinition $saga, array $completed): ExecutionContext { @@ -158,18 +170,46 @@ public function handle(string $sagaName, string $stepId, Throwable $error, int $ expect($context->stepStatus('a'))->toBe(StepStatus::Compensated); }); -it('CIRCUIT_BREAKER stops compensating after a critical step fails and leaves the rest for recovery', function (): void { +it('CIRCUIT_BREAKER stops compensating after a critical step fails and throws naming the steps left', function (): void { $log = []; $logger = new RecordingLogger; $saga = orchestrationCompensationSaga(['a' => [], 'b' => ['a'], 'c' => ['b']], ['c' => PHP_INT_MAX], $log, critical: ['c']); $context = orchestrationCompletedContext($saga, ['a', 'b', 'c']); + $handler = new class implements CompensationErrorHandler + { + public int $calls = 0; - orchestrationCompensator(CompensationPolicy::CircuitBreaker, new DefaultCompensationErrorHandler, logger: $logger)->compensate($saga, $context); + public function handle(string $sagaName, string $stepId, Throwable $error, int $attempt): CompensationErrorResult + { + $this->calls++; - expect($log)->toBe(['c']) + return CompensationErrorResult::Continue; + } + }; + + $stopped = orchestrationCompensationStopped(fn () => orchestrationCompensator(CompensationPolicy::CircuitBreaker, $handler, logger: $logger)->compensate($saga, $context)); + + expect($stopped->getMessage())->toBe('Saga [order] stopped compensating at critical step [c]; [b, a] left uncompensated: cannot undo c') + ->and($stopped->getPrevious())->toBeInstanceOf(RuntimeException::class) + ->and($handler->calls)->toBe(0) + ->and($log)->toBe(['c']) + ->and($context->stepStatus('c'))->toBe(StepStatus::CompensationFailed) ->and($context->stepStatus('b'))->toBe(StepStatus::Done) ->and($context->stepStatus('a'))->toBe(StepStatus::Done) - ->and(implode("\n", $logger->messages('warning')))->toContain('left [b, a] uncompensated'); + ->and(implode("\n", $logger->messages('warning')))->toContain('[b, a] left uncompensated'); +}); + +it('CIRCUIT_BREAKER still throws when the critical step that failed was the last one to undo', function (): void { + $log = []; + $saga = orchestrationCompensationSaga(['a' => [], 'b' => ['a']], ['a' => PHP_INT_MAX], $log, critical: ['a']); + $context = orchestrationCompletedContext($saga, ['a', 'b']); + + $stopped = orchestrationCompensationStopped(fn () => orchestrationCompensator(CompensationPolicy::CircuitBreaker, new DefaultCompensationErrorHandler)->compensate($saga, $context)); + + expect($stopped->getMessage())->toBe('Saga [order] stopped compensating at critical step [a]: cannot undo a') + ->and($log)->toBe(['b', 'a']) + ->and($context->stepStatus('b'))->toBe(StepStatus::Compensated) + ->and($context->stepStatus('a'))->toBe(StepStatus::CompensationFailed); }); it('CIRCUIT_BREAKER shares a firefly/resilience breaker per saga across executions', function (): void { @@ -178,9 +218,10 @@ public function handle(string $sagaName, string $stepId, Throwable $error, int $ $saga = orchestrationCompensationSaga(['a' => [], 'b' => ['a'], 'c' => ['b']], ['c' => PHP_INT_MAX], $log); $context = orchestrationCompletedContext($saga, ['a', 'b', 'c']); - orchestrationCompensator(CompensationPolicy::CircuitBreaker, new DefaultCompensationErrorHandler, resilience: $resilience)->compensate($saga, $context); + $stopped = orchestrationCompensationStopped(fn () => orchestrationCompensator(CompensationPolicy::CircuitBreaker, new DefaultCompensationErrorHandler, resilience: $resilience)->compensate($saga, $context)); - expect($log)->toBe(['c'])->and($context->stepStatus('b'))->toBe(StepStatus::Done); + expect($stopped->getMessage())->toBe('Saga [order] opened its compensation circuit at step [c]; [b, a] left uncompensated: cannot undo c') + ->and($log)->toBe(['c'])->and($context->stepStatus('b'))->toBe(StepStatus::Done); }); it('BEST_EFFORT attempts every compensation even when the handler would fail the saga', function (): void { @@ -262,15 +303,17 @@ public function handle(string $sagaName, string $stepId, Throwable $error, int $ $resilience = ResilienceRegistry::fromConfig(new Config(new Repository(['firefly' => ['resilience' => ['circuit-breaker' => ['orchestration-compensation-order' => ['failure-threshold' => 1]]]]])), new InMemoryResilienceStore); $saga = orchestrationCompensationSaga(['a' => [], 'b' => ['a']], ['b' => PHP_INT_MAX], $log); - orchestrationCompensator(CompensationPolicy::CircuitBreaker, new DefaultCompensationErrorHandler, resilience: $resilience)->compensate($saga, orchestrationCompletedContext($saga, ['a', 'b'])); + orchestrationCompensationStopped(fn () => orchestrationCompensator(CompensationPolicy::CircuitBreaker, new DefaultCompensationErrorHandler, resilience: $resilience)->compensate($saga, orchestrationCompletedContext($saga, ['a', 'b']))); $log = []; $second = orchestrationCompletedContext($saga, ['a', 'b']); - orchestrationCompensator(CompensationPolicy::CircuitBreaker, new DefaultCompensationErrorHandler, logger: $logger, resilience: $resilience)->compensate($saga, $second); + $stopped = orchestrationCompensationStopped(fn () => orchestrationCompensator(CompensationPolicy::CircuitBreaker, new DefaultCompensationErrorHandler, logger: $logger, resilience: $resilience)->compensate($saga, $second)); - expect($log)->toBe([]) + expect($stopped->getMessage())->toStartWith('Saga [order] opened its compensation circuit at step [b]; [b, a] left uncompensated') + ->and($stopped->getPrevious())->toBeInstanceOf(CircuitBreakerOpenException::class) + ->and($log)->toBe([]) ->and($second->stepStatus('b'))->toBe(StepStatus::Done) ->and($second->stepStatus('a'))->toBe(StepStatus::Done) - ->and(implode("\n", $logger->messages('warning')))->toContain('left [b, a] uncompensated'); + ->and(implode("\n", $logger->messages('warning')))->toContain('[b, a] left uncompensated'); }); it('CIRCUIT_BREAKER runs the handler retries through the saga breaker and stops the sweep once they open it', function (): void { @@ -280,17 +323,18 @@ public function handle(string $sagaName, string $stepId, Throwable $error, int $ $saga = orchestrationCompensationSaga(['a' => [], 'b' => ['a'], 'c' => ['b']], ['c' => PHP_INT_MAX], $log); $context = orchestrationCompletedContext($saga, ['a', 'b', 'c']); - orchestrationCompensator(CompensationPolicy::CircuitBreaker, new RetryWithBackoffCompensationErrorHandler(3), logger: $logger, resilience: $resilience)->compensate($saga, $context); + $stopped = orchestrationCompensationStopped(fn () => orchestrationCompensator(CompensationPolicy::CircuitBreaker, new RetryWithBackoffCompensationErrorHandler(3), logger: $logger, resilience: $resilience)->compensate($saga, $context)); // The first try and ONE retry reach the breaker's threshold of two; the remaining two retries the handler // asked for never hit the failing dependency, and the sweep leaves b and a for recovery. - expect($log)->toBe(['c', 'c']) + expect($stopped->getMessage())->toBe('Saga [order] opened its compensation circuit at step [c]; [b, a] left uncompensated: cannot undo c') + ->and($log)->toBe(['c', 'c']) ->and($context->attempts('c:compensate'))->toBe(2) ->and($resilience->circuitBreaker('orchestration-compensation-order')->state())->toBe('open') ->and($context->stepStatus('c'))->toBe(StepStatus::CompensationFailed) ->and($context->stepStatus('b'))->toBe(StepStatus::Done) ->and($context->stepStatus('a'))->toBe(StepStatus::Done) - ->and(implode("\n", $logger->messages('warning')))->toContain('left [b, a] uncompensated'); + ->and(implode("\n", $logger->messages('warning')))->toContain('[b, a] left uncompensated'); }); it('CIRCUIT_BREAKER counts a handler retry that succeeds as a breaker success and carries on', function (): void { From 45770aa1e4a0f1d74096c41e7b7e9b8c6b794f90 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 19:52:16 -0700 Subject: [PATCH 16/58] =?UTF-8?q?feat(orchestration):=20the=20saga=20engin?= =?UTF-8?q?e=20=E2=80=94=20layered=20execution,=20deadlines,=20idempotency?= =?UTF-8?q?,=20compensation,=20dead=20letters,=20resume?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../orchestration/src/Core/EngineSupport.php | 17 ++ .../src/Saga/Engine/SagaEngine.php | 209 ++++++++++++++++++ .../Saga/Engine/SagaExecutionOrchestrator.php | 151 +++++++++++++ .../tests/Core/EngineSupportTest.php | 30 +++ .../tests/Saga/Engine/SagaEngineTest.php | 209 ++++++++++++++++++ .../tests/Saga/Engine/SagaResumeTest.php | 151 +++++++++++++ .../tests/Support/TestEngines.php | 136 ++++++++++++ 7 files changed, 903 insertions(+) create mode 100644 packages/orchestration/src/Saga/Engine/SagaEngine.php create mode 100644 packages/orchestration/src/Saga/Engine/SagaExecutionOrchestrator.php create mode 100644 packages/orchestration/tests/Saga/Engine/SagaEngineTest.php create mode 100644 packages/orchestration/tests/Saga/Engine/SagaResumeTest.php create mode 100644 packages/orchestration/tests/Support/TestEngines.php diff --git a/packages/orchestration/src/Core/EngineSupport.php b/packages/orchestration/src/Core/EngineSupport.php index 31b31db2..f2bee5d1 100644 --- a/packages/orchestration/src/Core/EngineSupport.php +++ b/packages/orchestration/src/Core/EngineSupport.php @@ -8,6 +8,7 @@ use Firefly\Orchestration\Core\Callback\CallbackInvoker; use Firefly\Orchestration\Dlq\DeadLetterEntry; use Firefly\Orchestration\Dlq\DeadLetterService; +use Firefly\Orchestration\Event\OrchestrationEvent; use Firefly\Orchestration\Event\OrchestrationEventPublisher; use Firefly\Orchestration\Event\OrchestrationEvents; use Firefly\Orchestration\Event\OrchestrationTracing; @@ -83,6 +84,22 @@ public function deadLetter(ExecutionContext $context, ?string $stepId, Execution } } + /** + * Publishes an event without letting the transport fail the execution it describes: a broker that is down + * costs a log line, not a compensated saga. + */ + public function publish(OrchestrationEventPublisher $publisher, OrchestrationEvent $event): void + { + try { + $publisher->publish($event); + } catch (Throwable $error) { + $this->logger->warning("Orchestration event {$event->eventType} for [{$event->executionName}] could not be published: {$error->getMessage()}", [ + 'exception' => $error, + 'correlationId' => $event->correlationId, + ]); + } + } + public function elapsedMs(ExecutionContext $context): int { return max(0, (int) round(((float) $this->now()->format('U.u') - (float) $context->startedAt()->format('U.u')) * 1000)); diff --git a/packages/orchestration/src/Saga/Engine/SagaEngine.php b/packages/orchestration/src/Saga/Engine/SagaEngine.php new file mode 100644 index 00000000..d8283173 --- /dev/null +++ b/packages/orchestration/src/Saga/Engine/SagaEngine.php @@ -0,0 +1,209 @@ +registry; + } + + /** + * @param array|StepInputs $input + * @param array $headers + */ + public function execute(string|SagaDefinition $saga, array|StepInputs $input = [], array $headers = [], ?string $correlationId = null): SagaResult + { + $this->guard(); + $definition = is_string($saga) ? $this->registry->get($saga) : $saga; + $inputs = $input instanceof StepInputs ? $input : StepInputs::of($input); + $context = ExecutionContext::start(ExecutionPattern::Saga, $definition->name, $inputs->global(), $headers, $correlationId, $this->support->now()); + + return $this->support->admission->admit( + $definition->name, + fn (): SagaResult => $this->support->lock->synchronized($context->correlationId, fn (): SagaResult => $this->support->tracing->traceExecution( + $definition->name, + ExecutionPattern::Saga, + $context->correlationId, + fn (): SagaResult => $this->run($definition, $inputs, $context, fresh: true), + )), + static fn (SagaResult $result): bool => $result->success, + ); + } + + public function resume(string $correlationId): SagaResult + { + return $this->support->lock->synchronized($correlationId, function () use ($correlationId): SagaResult { + $state = $this->load($correlationId); + if ($state->isTerminal()) { + throw new InvalidExecutionStateException("Saga execution [{$correlationId}] is already {$state->status->value} and cannot be resumed."); + } + + $definition = $this->registry->get($state->executionName); + $context = ExecutionContext::fromState($state); + $inputs = StepInputs::of($state->input); + foreach ($state->stepInputs as $stepId => $value) { + $inputs = $inputs->forStep($stepId, $value); + } + + if ($state->status === ExecutionStatus::Compensating) { + return $this->fail($definition, $context, $this->firstError($context)); + } + + return $this->support->tracing->traceExecution($definition->name, ExecutionPattern::Saga, $correlationId, fn (): SagaResult => $this->run($definition, $inputs, $context, fresh: false)); + }); + } + + public function compensate(string $correlationId): SagaResult + { + return $this->support->lock->synchronized($correlationId, function () use ($correlationId): SagaResult { + $state = $this->load($correlationId); + if ($state->isTerminal() && $state->status !== ExecutionStatus::Completed) { + throw new InvalidExecutionStateException("Saga execution [{$correlationId}] is already {$state->status->value} and has nothing left to compensate."); + } + + $definition = $this->registry->get($state->executionName); + $context = ExecutionContext::fromState($state); + $this->support->checkpoint($context, ExecutionStatus::Compensating); + + $reason = 'compensated on request'; + try { + $this->compensator->compensate($definition, $context); + } catch (CompensationException $failure) { + $reason = $failure->getMessage(); + } + + return $this->finish($definition, $context, ExecutionStatus::Cancelled, null, $reason); + }); + } + + private function run(SagaDefinition $saga, StepInputs $inputs, ExecutionContext $context, bool $fresh): SagaResult + { + if ($fresh) { + foreach ($inputs->explicit() as $stepId => $value) { + $context->putStepInput($stepId, $value); + } + $this->support->checkpoint($context, ExecutionStatus::Running); + $this->support->publish($this->support->lifecycle, OrchestrationEvent::executionStarted($saga->name, $context->correlationId, ExecutionPattern::Saga, $this->support->now())); + $this->support->events->onStart($saga->name, $context->correlationId, ExecutionPattern::Saga); + } + + // A resumed execution gets a fresh budget from the moment it is resumed: measured from the original + // start, a saga recovery picks up after the stale threshold would always be past its deadline, and + // `recovery.action=resume` could only ever compensate it. + $deadline = $this->settings->defaultTimeout > 0 + ? ($fresh ? $context->startedAt() : $this->support->now())->modify(sprintf('+%d milliseconds', (int) round($this->settings->defaultTimeout * 1000))) + : null; + + if ($this->orchestrator->orchestrate($saga, $inputs, $context, $deadline)) { + $result = $this->finish($saga, $context, ExecutionStatus::Completed, null, null); + $this->support->callbacks->invoke($saga->onComplete, $context, $result); + + return $result; + } + + return $this->fail($saga, $context, $this->firstError($context)); + } + + private function fail(SagaDefinition $saga, ExecutionContext $context, Throwable $error): SagaResult + { + $this->support->checkpoint($context, ExecutionStatus::Compensating, $error->getMessage()); + + $compensationFailure = null; + try { + $this->compensator->compensate($saga, $context); + } catch (CompensationException $failure) { + $compensationFailure = $failure; + } + + foreach ($context->stepErrors() as $stepId => $stepError) { + $this->support->deadLetter($context, $stepId, ExecutionStatus::Failed, $stepError); + } + + $suppressed = $compensationFailure === null && $this->support->callbacks->suppresses($saga->onError, $error); + $result = $suppressed + ? $this->finish($saga, $context, ExecutionStatus::Completed, null, null) + : $this->finish($saga, $context, ExecutionStatus::Failed, $error, $compensationFailure?->getMessage() ?? $error->getMessage()); + + $this->support->callbacks->invoke($saga->onError, $context, $result, $error); + + return $result; + } + + private function finish(SagaDefinition $saga, ExecutionContext $context, ExecutionStatus $status, ?Throwable $error, ?string $reason): SagaResult + { + $now = $this->support->now(); + $report = ExecutionReportBuilder::fromContext($context, $status, $reason, $now, $saga->stepIds()); + $result = SagaResult::from($saga, $context, $status, $error, $report, $now); + + $this->support->checkpoint($context, $status, $reason, $report); + $this->support->publish($this->support->lifecycle, OrchestrationEvent::executionCompleted($saga->name, $context->correlationId, ExecutionPattern::Saga, $status, $now)); + $this->support->events->onCompleted($saga->name, $context->correlationId, ExecutionPattern::Saga, $status->isSuccess(), $this->support->elapsedMs($context)); + + return $result; + } + + private function firstError(ExecutionContext $context): Throwable + { + foreach ($context->stepErrors() as $error) { + return $error; + } + + return new OrchestrationException("Saga [{$context->executionName}] failed."); + } + + private function load(string $correlationId): ExecutionState + { + $state = $this->support->persistence->findById($correlationId); + if ($state === null || $state->pattern !== ExecutionPattern::Saga) { + throw new ExecutionNotFoundException($correlationId); + } + + return $state; + } + + private function guard(): void + { + if (! $this->settings->enabled) { + throw new PatternDisabledException(ExecutionPattern::Saga, 'firefly.orchestration.saga.enabled'); + } + } +} diff --git a/packages/orchestration/src/Saga/Engine/SagaExecutionOrchestrator.php b/packages/orchestration/src/Saga/Engine/SagaExecutionOrchestrator.php new file mode 100644 index 00000000..9cbd720b --- /dev/null +++ b/packages/orchestration/src/Saga/Engine/SagaExecutionOrchestrator.php @@ -0,0 +1,151 @@ +setTopologyLayers($saga->layers); + + foreach ($saga->layers as $layer) { + foreach ($layer as $stepId) { + if ($context->stepStatus($stepId)->isSettled()) { + continue; + } + if (! $this->runStep($saga, $saga->steps[$stepId], $inputs, $context, $deadline)) { + return false; + } + } + } + + return true; + } + + /** + * {correlationId}, {input.} and {header.} are replaced; anything else is kept literally. + */ + public static function interpolate(string $template, ExecutionContext $context, mixed $input): string + { + return (string) preg_replace_callback('/\{(correlationId|input\.[^}]+|header\.[^}]+)\}/', static function (array $match) use ($context, $input): string { + $token = $match[1]; + if ($token === 'correlationId') { + return $context->correlationId; + } + if (str_starts_with($token, 'input.')) { + $value = is_array($input) ? ($input[substr($token, 6)] ?? '') : ''; + + return is_scalar($value) ? (string) $value : ''; + } + + return $context->header(substr($token, 7)) ?? ''; + }, $template); + } + + /** A step-event key names an input key whose value is the event key; without such a key it is used literally. */ + public static function eventKey(string $key, mixed $input): string + { + if ($key === '') { + return ''; + } + $value = is_array($input) ? ($input[$key] ?? null) : null; + + return is_scalar($value) ? (string) $value : $key; + } + + private function runStep(SagaDefinition $saga, SagaStepDefinition $step, StepInputs $inputs, ExecutionContext $context, ?DateTimeImmutable $deadline): bool + { + $stepId = $step->id; + $now = $this->support->now(); + + if ($deadline !== null && $now > $deadline) { + return $this->failed($saga, $context, $stepId, new StepTimeoutException("Saga [{$saga->name}] passed its deadline (firefly.orchestration.saga.default-timeout) before step [{$stepId}].")); + } + + $input = $inputs->resolveFor($stepId, $context); + $key = $step->options->idempotencyKey === '' ? '' : self::interpolate($step->options->idempotencyKey, $context, $input); + if ($key !== '' && $context->hasIdempotencyKey($key)) { + $context->setStepStatus($stepId, StepStatus::Skipped); + $this->support->events->onStepSkippedIdempotent($saga->name, $context->correlationId, $stepId); + + return true; + } + + $context->putStepInput($stepId, $input); + $context->setStepStatus($stepId, StepStatus::Running); + $context->markStepStarted($stepId, $now); + $context->clearStepError($stepId); + $this->support->events->onStepStarted($saga->name, $context->correlationId, $stepId); + $this->support->checkpoint($context, ExecutionStatus::Running); + + $started = hrtime(true); + try { + $result = $this->invoker->invoke($saga->name, $stepId, $step->invocation, $step->options, $context, $input, $input); + } catch (Throwable $error) { + $context->setLatencyMs($stepId, self::since($started)); + + return $this->failed($saga, $context, $stepId, $error); + } + + $latency = self::since($started); + $context->putResult($stepId, $result); + $context->setLatencyMs($stepId, $latency); + $context->setStepStatus($stepId, StepStatus::Done); + $context->markCompleted($stepId); + if ($key !== '') { + $context->addIdempotencyKey($key); + } + $this->support->events->onStepSuccess($saga->name, $context->correlationId, $stepId, $context->attempts($stepId), $latency); + + $completed = OrchestrationEvent::stepCompleted($saga->name, $context->correlationId, ExecutionPattern::Saga, $stepId, $result, $this->support->now()); + if ($step->event !== null) { + $this->support->publish($this->support->stepEvents, $completed->withTopic($step->event->topic)->withType($step->event->type)->withKey(self::eventKey($step->event->key, $input))); + } + $this->support->publish($this->support->lifecycle, $completed); + $this->support->checkpoint($context, ExecutionStatus::Running); + + return true; + } + + private function failed(SagaDefinition $saga, ExecutionContext $context, string $stepId, Throwable $error): bool + { + $context->setStepStatus($stepId, $error instanceof TimeoutException ? StepStatus::TimedOut : StepStatus::Failed); + $context->putStepError($stepId, $error); + $this->support->events->onStepFailed($saga->name, $context->correlationId, $stepId, $error, $context->attempts($stepId)); + $this->support->publish($this->support->lifecycle, OrchestrationEvent::stepFailed($saga->name, $context->correlationId, ExecutionPattern::Saga, $stepId, $error, $this->support->now())); + + return false; + } + + private static function since(int|float $started): int + { + return (int) round((hrtime(true) - $started) / 1_000_000); + } +} diff --git a/packages/orchestration/tests/Core/EngineSupportTest.php b/packages/orchestration/tests/Core/EngineSupportTest.php index ec31d5b1..d3ef48cc 100644 --- a/packages/orchestration/tests/Core/EngineSupportTest.php +++ b/packages/orchestration/tests/Core/EngineSupportTest.php @@ -14,6 +14,8 @@ use Firefly\Orchestration\Dlq\MemoryDeadLetterStore; use Firefly\Orchestration\Event\NoOpOrchestrationEventPublisher; use Firefly\Orchestration\Event\NoOpOrchestrationTracing; +use Firefly\Orchestration\Event\OrchestrationEvent; +use Firefly\Orchestration\Event\OrchestrationEventPublisher; use Firefly\Orchestration\Persistence\ExecutionPersistenceProvider; use Firefly\Orchestration\Persistence\MemoryExecutionPersistenceProvider; use Firefly\Orchestration\Settings\BackpressureSettings; @@ -71,3 +73,31 @@ function orchestrationSupport(ExecutionPersistenceProvider $persistence, Recordi ->and($entry->input)->toBe(['orderId' => 7]) ->and($entry->headers)->toBe(['tenant' => 'acme']); }); + +it('publishes an event and survives a failing publisher by logging', function (): void { + $logger = new RecordingLogger; + $support = orchestrationSupport(new MemoryExecutionPersistenceProvider, $logger, new MemoryDeadLetterStore); + $event = OrchestrationEvent::executionStarted('order', 'c-1', ExecutionPattern::Saga, $support->now()); + $recording = new class implements OrchestrationEventPublisher + { + /** @var list */ + public array $published = []; + + public function publish(OrchestrationEvent $event): void + { + $this->published[] = $event->eventType; + } + }; + + $support->publish($recording, $event); + $support->publish(new class implements OrchestrationEventPublisher + { + public function publish(OrchestrationEvent $event): void + { + throw new RuntimeException('broker down'); + } + }, $event); + + expect($recording->published)->toBe([$event->eventType]) + ->and($logger->messages('warning')[0] ?? '')->toContain('could not be published: broker down'); +}); diff --git a/packages/orchestration/tests/Saga/Engine/SagaEngineTest.php b/packages/orchestration/tests/Saga/Engine/SagaEngineTest.php new file mode 100644 index 00000000..eead2e26 --- /dev/null +++ b/packages/orchestration/tests/Saga/Engine/SagaEngineTest.php @@ -0,0 +1,209 @@ + $log + */ +function orchestrationOrderSaga(array &$log, bool &$declined): SagaDefinition +{ + return SagaBuilder::saga('order') + ->step('reserve') + ->handler(static function (array $input) use (&$log): array { + $log[] = 'reserve'; + + return ['hold' => 'h-'.(is_scalar($input['orderId'] ?? null) ? $input['orderId'] : '?')]; + }) + ->compensation(static function (mixed $result) use (&$log): string { + $log[] = 'release '.(is_array($result) && is_string($result['hold'] ?? null) ? $result['hold'] : '?'); + + return 'released'; + }) + ->add() + ->step('charge')->dependsOn('reserve') + ->handler(static function () use (&$log, &$declined): string { + $log[] = 'charge'; + if ($declined) { + throw new RuntimeException('card declined'); + } + + return 'paid'; + }) + ->compensation(static function () use (&$log): void { + $log[] = 'refund'; + }) + ->stepEvent('payments', 'payment.captured', 'orderId') + ->add() + ->step('notify')->dependsOn('reserve') + ->handler(static function () use (&$log): string { + $log[] = 'notify'; + + return 'sent'; + }) + ->add() + ->build(); +} + +it('completes a saga, reports it, checkpoints it and calls #[OnSagaComplete]', function (): void { + $engines = TestEngines::create(['events' => ['enabled' => true]]); + $log = []; + $declined = false; + $completed = []; + $saga = orchestrationOrderSaga($log, $declined); + $saga = new SagaDefinition($saga->name, $saga->steps, onComplete: [CallbackDefinition::closure(static function (ExecutionContext $ctx, mixed $result) use (&$completed): void { + $completed[] = $result instanceof SagaResult ? $result->correlationId : 'none'; + })]); + + $result = $engines->sagas->execute($saga, ['orderId' => 7], ['tenant' => 'acme'], 'c-1'); + + expect($result->isSuccess())->toBeTrue() + ->and($result->status)->toBe(ExecutionStatus::Completed) + ->and($log)->toBe(['reserve', 'charge', 'notify']) + ->and($result->stepResults())->toBe(['reserve' => ['hold' => 'h-7'], 'charge' => 'paid', 'notify' => 'sent']) + ->and($result->report?->layers)->toBe([['reserve'], ['charge', 'notify']]) + ->and($result->headers)->toBe(['tenant' => 'acme']) + ->and($engines->persistence->findById('c-1')?->status)->toBe(ExecutionStatus::Completed) + ->and($engines->persistence->findById('c-1')?->report['status'] ?? null)->toBe('COMPLETED') + ->and($completed)->toBe(['c-1']) + ->and($engines->events->hooks())->toBe([ + 'onStart', 'onStepStarted', 'onStepSuccess', 'onStepStarted', 'onStepSuccess', 'onStepStarted', 'onStepSuccess', 'onCompleted', + ]) + ->and(array_column($engines->eda->published, 'eventType'))->toBe([ + 'orchestration.execution_started', 'orchestration.step_completed', 'payment.captured', + 'orchestration.step_completed', 'orchestration.step_completed', 'orchestration.execution_completed', + ]) + ->and($engines->eda->publishedOf('payment.captured')[0]['headers']['key'] ?? null)->toBe('7'); +}); + +it('compensates the completed steps in reverse, dead-letters the failure and calls #[OnSagaError]', function (): void { + $engines = TestEngines::create(); + $log = []; + $declined = true; + $errors = []; + $base = orchestrationOrderSaga($log, $declined); + $saga = new SagaDefinition($base->name, $base->steps, onError: [CallbackDefinition::closure(static function (ExecutionContext $ctx, mixed $error) use (&$errors): void { + $errors[] = $error instanceof Throwable ? $error->getMessage() : 'none'; + })]); + + $result = $engines->sagas->execute($saga, ['orderId' => 7], correlationId: 'c-2'); + + expect($result->isSuccess())->toBeFalse() + ->and($result->status)->toBe(ExecutionStatus::Failed) + ->and($result->error?->getMessage())->toBe('card declined') + ->and($log)->toBe(['reserve', 'charge', 'release h-7']) + ->and($result->failedSteps())->toBe(['charge']) + ->and($result->compensatedSteps())->toBe(['reserve']) + ->and($result->steps['notify']->status)->toBe(StepStatus::Pending) + ->and($errors)->toBe(['card declined']) + ->and($engines->persistence->findById('c-2')?->status)->toBe(ExecutionStatus::Failed) + ->and($engines->persistence->findById('c-2')?->failureReason)->toBe('card declined') + ->and($engines->deadLetterStore->findByCorrelationId('c-2')[0]->stepId ?? null)->toBe('charge') + ->and($engines->deadLetterStore->findByCorrelationId('c-2')[0]->input ?? null)->toBe(['orderId' => 7]); +}); + +it('turns a suppressed error into COMPLETED, after compensating', function (): void { + $engines = TestEngines::create(); + $log = []; + $declined = true; + $base = orchestrationOrderSaga($log, $declined); + $saga = new SagaDefinition($base->name, $base->steps, onError: [CallbackDefinition::closure(static function (): void {}, suppressError: true, errorTypes: [RuntimeException::class])]); + + $result = $engines->sagas->execute($saga, ['orderId' => 1]); + + expect($result->isSuccess())->toBeTrue() + ->and($result->status)->toBe(ExecutionStatus::Completed) + ->and($log)->toContain('release h-1'); +}); + +it('retries a step and records its attempts', function (): void { + $engines = TestEngines::create(); + $calls = 0; + $saga = SagaBuilder::saga('flaky')->step('call')->retry(2)->backoffMs(1)->handler(static function () use (&$calls): string { + if (++$calls < 3) { + throw new RuntimeException('busy'); + } + + return 'ok'; + })->add()->build(); + + $result = $engines->sagas->execute($saga); + + expect($result->isSuccess())->toBeTrue()->and($result->steps['call']->attempts)->toBe(3); +}); + +it('skips a step whose idempotency key was already recorded in this execution', function (): void { + $engines = TestEngines::create(); + $charged = 0; + $charge = static function () use (&$charged): string { + $charged++; + + return 'charged'; + }; + $saga = SagaBuilder::saga('double') + ->step('charge-a')->idempotencyKey('charge-{input.orderId}')->handler($charge)->add() + ->step('charge-b')->dependsOn('charge-a')->idempotencyKey('charge-{input.orderId}')->handler($charge)->add() + ->build(); + + $result = $engines->sagas->execute($saga, ['orderId' => 9]); + + expect($charged)->toBe(1) + ->and($result->steps['charge-b']->status)->toBe(StepStatus::Skipped) + ->and($engines->events->count('onStepSkippedIdempotent'))->toBe(1); +}); + +it('stops at the saga-wide deadline and compensates', function (): void { + $engines = TestEngines::create(['saga' => ['default-timeout' => '1s']]); + $log = []; + $saga = SagaBuilder::saga('slow') + ->step('a')->handler(static function () use ($engines, &$log): string { + $log[] = 'a'; + $engines->clock->advance(2.0); + + return 'a'; + })->compensation(static function () use (&$log): void { + $log[] = 'undo a'; + })->add() + ->step('b')->dependsOn('a')->handler(static function () use (&$log): string { + $log[] = 'b'; + + return 'b'; + })->add() + ->build(); + + $result = $engines->sagas->execute($saga); + + expect($log)->toBe(['a', 'undo a']) + ->and($result->steps['b']->status)->toBe(StepStatus::TimedOut) + ->and($result->error?->getMessage())->toContain('firefly.orchestration.saga.default-timeout'); +}); + +it('refuses executions while sagas are disabled, and under back-pressure', function (): void { + $saga = SagaBuilder::saga('any')->step('a')->handler(static fn (): string => 'a')->add()->build(); + $busy = TestEngines::create(['backpressure' => ['max-in-flight' => 1]]); + $busy->persistence->save(ExecutionPersistenceContract::state('open')); + + expect(fn () => TestEngines::create(['saga' => ['enabled' => false]])->sagas->execute($saga))->toThrow(PatternDisabledException::class, 'firefly.orchestration.saga.enabled') + ->and(fn () => $busy->sagas->execute($saga))->toThrow(BackpressureException::class); +}); + +it('starts a registered saga by name', function (): void { + $engines = TestEngines::create(); + $engines->sagaRegistry->register(SagaBuilder::saga('hello')->step('greet')->handler(static fn (array $input): string => 'hello '.(is_string($input['who'] ?? null) ? $input['who'] : ''))->add()->build()); + + expect($engines->sagas->execute('hello', ['who' => 'ada'])->resultOf('greet'))->toBe('hello ada'); +}); diff --git a/packages/orchestration/tests/Saga/Engine/SagaResumeTest.php b/packages/orchestration/tests/Saga/Engine/SagaResumeTest.php new file mode 100644 index 00000000..f537789d --- /dev/null +++ b/packages/orchestration/tests/Saga/Engine/SagaResumeTest.php @@ -0,0 +1,151 @@ + $log + */ +function orchestrationChainSaga(array &$log): SagaDefinition +{ + $builder = SagaBuilder::saga('chain'); + $previous = null; + foreach (['a', 'b', 'c'] as $id) { + $step = $builder->step($id) + ->handler(static function () use ($id, &$log): string { + $log[] = $id; + + return "did {$id}"; + }) + ->compensation(static function (mixed $result) use ($id, &$log): void { + $log[] = "undo {$id} (".(is_string($result) ? $result : '?').')'; + }); + $builder = ($previous === null ? $step : $step->dependsOn($previous))->add(); + $previous = $id; + } + + return $builder->build(); +} + +/** + * @param array $overrides + */ +function orchestrationCrashedChain(array $overrides): ExecutionState +{ + return ExecutionState::fromArray(array_merge([ + 'correlationId' => 'c-9', + 'executionName' => 'chain', + 'pattern' => 'SAGA', + 'status' => 'RUNNING', + 'startedAt' => '2026-10-12T09:00:00.000000+00:00', + 'updatedAt' => '2026-10-12T09:00:01.000000+00:00', + 'input' => ['orderId' => 7], + 'topologyLayers' => [['a'], ['b'], ['c']], + ], $overrides)); +} + +it('resumes a saga a crashed process left mid-step: settled steps are not run again', function (): void { + $engines = TestEngines::create(); + $log = []; + $engines->sagaRegistry->register(orchestrationChainSaga($log)); + $engines->persistence->save(orchestrationCrashedChain([ + 'stepStatuses' => ['a' => 'DONE', 'b' => 'RUNNING'], + 'stepResults' => ['a' => 'did a'], + 'stepAttempts' => ['a' => 1, 'b' => 1], + 'completionOrder' => ['a'], + ])); + + $result = $engines->sagas->resume('c-9'); + + expect($result->isSuccess())->toBeTrue() + ->and($log)->toBe(['b', 'c']) + ->and($result->steps['b']->attempts)->toBe(2) + ->and($result->resultOf('a'))->toBe('did a') + ->and($engines->persistence->findById('c-9')?->status)->toBe(ExecutionStatus::Completed) + ->and($engines->events->count('onStart'))->toBe(0); +}); + +it('continues an interrupted compensation without undoing a step twice', function (): void { + $engines = TestEngines::create(); + $log = []; + $engines->sagaRegistry->register(orchestrationChainSaga($log)); + $engines->persistence->save(orchestrationCrashedChain([ + 'status' => 'COMPENSATING', + 'stepStatuses' => ['a' => 'DONE', 'b' => 'COMPENSATED', 'c' => 'FAILED'], + 'stepResults' => ['a' => 'did a', 'b' => 'did b'], + 'stepErrors' => ['c' => 'c broke'], + 'completionOrder' => ['a', 'b'], + ])); + + $result = $engines->sagas->resume('c-9'); + + expect($log)->toBe(['undo a (did a)']) + ->and($result->status)->toBe(ExecutionStatus::Failed) + ->and($result->error?->getMessage())->toBe('c broke') + ->and($result->steps['a']->status)->toBe(StepStatus::Compensated); +}); + +it('compensates a completed saga on request and ends it CANCELLED', function (): void { + $engines = TestEngines::create(); + $log = []; + $engines->sagaRegistry->register(orchestrationChainSaga($log)); + $done = $engines->sagas->execute('chain', correlationId: 'c-1'); + + $cancelled = $engines->sagas->compensate('c-1'); + + expect($done->isSuccess())->toBeTrue() + ->and($cancelled->status)->toBe(ExecutionStatus::Cancelled) + ->and(array_slice($log, 3))->toBe(['undo c (did c)', 'undo b (did b)', 'undo a (did a)']) + ->and($engines->persistence->findById('c-1')?->status)->toBe(ExecutionStatus::Cancelled); +}); + +it('refuses to resume a finished or an unknown execution', function (): void { + $engines = TestEngines::create(); + $log = []; + $engines->sagaRegistry->register(orchestrationChainSaga($log)); + $engines->sagas->execute('chain', correlationId: 'c-1'); + + expect(fn () => $engines->sagas->resume('c-1'))->toThrow(InvalidExecutionStateException::class, 'COMPLETED') + ->and(fn () => $engines->sagas->resume('nope'))->toThrow(ExecutionNotFoundException::class); +}); + +it('gives a resumed saga a fresh deadline budget measured from the resume, and still enforces it', function (): void { + $engines = TestEngines::create(['saga' => ['default-timeout' => '1s']]); + $log = []; + $engines->sagaRegistry->register(SagaBuilder::saga('chain') + ->step('a')->handler(static fn (): string => 'did a')->add() + ->step('b')->dependsOn('a')->handler(static function () use ($engines, &$log): string { + $log[] = 'b'; + $engines->clock->advance(2.0); + + return 'did b'; + })->add() + ->step('c')->dependsOn('b')->handler(static function () use (&$log): string { + $log[] = 'c'; + + return 'did c'; + })->add() + ->build()); + $engines->persistence->save(orchestrationCrashedChain([ + 'stepStatuses' => ['a' => 'DONE'], + 'stepResults' => ['a' => 'did a'], + 'completionOrder' => ['a'], + ])); + + $result = $engines->sagas->resume('c-9'); + + expect($log)->toBe(['b']) + ->and($result->status)->toBe(ExecutionStatus::Failed) + ->and($result->steps['c']->status)->toBe(StepStatus::TimedOut) + ->and($result->error?->getMessage())->toContain('firefly.orchestration.saga.default-timeout'); +}); diff --git a/packages/orchestration/tests/Support/TestEngines.php b/packages/orchestration/tests/Support/TestEngines.php new file mode 100644 index 00000000..e1a865d2 --- /dev/null +++ b/packages/orchestration/tests/Support/TestEngines.php @@ -0,0 +1,136 @@ + $config + */ + private function __construct(array $config, ?ExecutionPersistenceProvider $persistence, ?DeadLetterStore $deadLetters) + { + $this->container = new Container; + $this->settings = OrchestrationSettings::fromConfig(new Config(new Repository(['firefly' => ['orchestration' => $config]]))); + $this->clock = new MutableClock; + $this->persistence = $persistence ?? new MemoryExecutionPersistenceProvider; + $this->deadLetterStore = $deadLetters ?? new MemoryDeadLetterStore; + $this->events = new RecordingOrchestrationEvents; + $this->logger = new RecordingLogger; + $this->eda = new RecordingEventPublisher; + $this->runner = new InvocationRunner($this->container, new ArgumentResolver); + $tracing = new NoOpOrchestrationTracing; + $this->invoker = new StepInvoker($this->runner, $tracing); + $this->conditions = new ConditionEvaluator; + $resilienceStore = new InMemoryResilienceStore; + + $breaker = $this->settings->backpressure->strategy === 'circuit-breaker' + ? new CircuitBreaker( + key: 'firefly:orchestration:backpressure', + store: $resilienceStore, + failureThreshold: $this->settings->backpressure->failureThreshold, + waitDurationInOpen: $this->settings->backpressure->recoveryTimeout, + halfOpenMaxCalls: $this->settings->backpressure->halfOpenMaxCalls, + ) + : null; + + $this->support = new EngineSupport( + persistence: $this->persistence, + deadLetters: new DeadLetterService($this->deadLetterStore, $this->events, $this->settings->dlq->enabled), + events: $this->events, + lifecycle: $this->settings->events->enabled ? new EdaOrchestrationEventPublisher($this->eda, $this->settings->events->defaultTopic) : new NoOpOrchestrationEventPublisher, + stepEvents: new EdaOrchestrationEventPublisher($this->eda, $this->settings->events->defaultTopic), + tracing: $tracing, + callbacks: new CallbackInvoker($this->container, $this->logger), + clock: $this->clock, + lock: new ExecutionLock(new NoneLock, $this->settings->lock->ttl, $this->settings->lock->wait), + admission: new ExecutionAdmission($this->persistence, $this->settings->backpressure, $breaker), + logger: $this->logger, + ); + + $this->sagaRegistry = new SagaRegistry; + $this->compensator = new SagaCompensator( + $this->invoker, + $this->support, + $this->settings->saga->compensationPolicy, + (new CompensationErrorHandlerFactory($this->container, $this->logger))->get($this->settings->saga->compensationErrorHandler), + ResilienceRegistry::fromConfig(new Config(new Repository(['firefly' => []])), $resilienceStore), + $resilienceStore, + ); + $this->sagas = new SagaEngine($this->sagaRegistry, new SagaExecutionOrchestrator($this->invoker, $this->support), $this->compensator, $this->support, $this->settings->saga); + } + + /** + * @param array $config + */ + public static function create(array $config = [], ?ExecutionPersistenceProvider $persistence = null, ?DeadLetterStore $deadLetters = null): self + { + return new self($config, $persistence, $deadLetters); + } +} From b3bcfad60a7936598dd89f5ae23708dbd1690ab1 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 19:55:35 -0700 Subject: [PATCH 17/58] fix(orchestration): refuse to start a saga under a correlation id that already exists --- .../src/Saga/Engine/SagaEngine.php | 34 ++++++++++++++---- .../tests/Saga/Engine/SagaEngineTest.php | 36 +++++++++++++++++++ 2 files changed, 64 insertions(+), 6 deletions(-) diff --git a/packages/orchestration/src/Saga/Engine/SagaEngine.php b/packages/orchestration/src/Saga/Engine/SagaEngine.php index d8283173..6194e0f3 100644 --- a/packages/orchestration/src/Saga/Engine/SagaEngine.php +++ b/packages/orchestration/src/Saga/Engine/SagaEngine.php @@ -57,14 +57,23 @@ public function execute(string|SagaDefinition $saga, array|StepInputs $input = [ $inputs = $input instanceof StepInputs ? $input : StepInputs::of($input); $context = ExecutionContext::start(ExecutionPattern::Saga, $definition->name, $inputs->global(), $headers, $correlationId, $this->support->now()); + // Checked twice: up front, so a retried call is refused before admission, whose circuit breaker would + // count the refusal as a failed execution; and again under the lock, so two concurrent calls with one id + // cannot both pass. + $this->refuseExisting($context->correlationId); + return $this->support->admission->admit( $definition->name, - fn (): SagaResult => $this->support->lock->synchronized($context->correlationId, fn (): SagaResult => $this->support->tracing->traceExecution( - $definition->name, - ExecutionPattern::Saga, - $context->correlationId, - fn (): SagaResult => $this->run($definition, $inputs, $context, fresh: true), - )), + fn (): SagaResult => $this->support->lock->synchronized($context->correlationId, function () use ($definition, $inputs, $context): SagaResult { + $this->refuseExisting($context->correlationId); + + return $this->support->tracing->traceExecution( + $definition->name, + ExecutionPattern::Saga, + $context->correlationId, + fn (): SagaResult => $this->run($definition, $inputs, $context, fresh: true), + ); + }), static fn (SagaResult $result): bool => $result->success, ); } @@ -190,6 +199,19 @@ private function firstError(ExecutionContext $context): Throwable return new OrchestrationException("Saga [{$context->executionName}] failed."); } + /** + * A caller-supplied correlation id that is already persisted belongs to an execution that exists: starting + * it again would overwrite its row (the first checkpoint is an upsert), run every step a second time with + * none of its idempotency keys, and lose its outcome or its compensation progress. + */ + private function refuseExisting(string $correlationId): void + { + $state = $this->support->persistence->findById($correlationId); + if ($state !== null) { + throw new InvalidExecutionStateException("Execution [{$correlationId}] already exists ({$state->pattern->value} {$state->executionName}, {$state->status->value}); use resume() or compensate() instead of starting it again."); + } + } + private function load(string $correlationId): ExecutionState { $state = $this->support->persistence->findById($correlationId); diff --git a/packages/orchestration/tests/Saga/Engine/SagaEngineTest.php b/packages/orchestration/tests/Saga/Engine/SagaEngineTest.php index eead2e26..7d0ce69c 100644 --- a/packages/orchestration/tests/Saga/Engine/SagaEngineTest.php +++ b/packages/orchestration/tests/Saga/Engine/SagaEngineTest.php @@ -4,6 +4,7 @@ use Firefly\Orchestration\Core\Callback\CallbackDefinition; use Firefly\Orchestration\Core\Exception\BackpressureException; +use Firefly\Orchestration\Core\Exception\InvalidExecutionStateException; use Firefly\Orchestration\Core\Exception\PatternDisabledException; use Firefly\Orchestration\Core\ExecutionContext; use Firefly\Orchestration\Core\ExecutionStatus; @@ -116,6 +117,41 @@ function orchestrationOrderSaga(array &$log, bool &$declined): SagaDefinition ->and($engines->deadLetterStore->findByCorrelationId('c-2')[0]->input ?? null)->toBe(['orderId' => 7]); }); +it('refuses to start an execution again under a correlation id that already exists', function (bool $declined, ExecutionStatus $status): void { + $engines = TestEngines::create(); + $log = []; + $saga = orchestrationOrderSaga($log, $declined); + $engines->sagas->execute($saga, ['orderId' => 7], correlationId: 'c-dup'); + $ran = $log; + $before = $engines->persistence->findById('c-dup'); + + expect(fn () => $engines->sagas->execute($saga, ['orderId' => 7], correlationId: 'c-dup')) + ->toThrow(InvalidExecutionStateException::class, "Execution [c-dup] already exists (SAGA order, {$status->value})"); + + expect($log)->toBe($ran) + ->and($engines->persistence->findById('c-dup')?->status)->toBe($status) + ->and($engines->persistence->findById('c-dup')?->report)->toBe($before?->report); +})->with([ + 'completed' => [false, ExecutionStatus::Completed], + 'failed' => [true, ExecutionStatus::Failed], +]); + +it('refuses to start over an execution that is still compensating, leaving its recovery state alone', function (): void { + $engines = TestEngines::create(); + $log = []; + $declined = false; + $saga = orchestrationOrderSaga($log, $declined); + $engines->sagas->execute($saga, ['orderId' => 7], correlationId: 'c-mid'); + $state = $engines->persistence->findById('c-mid') ?? throw new RuntimeException('c-mid was not persisted'); + $engines->persistence->save($state->withStatus(ExecutionStatus::Compensating, new DateTimeImmutable)); + + expect(fn () => $engines->sagas->execute($saga, ['orderId' => 7], correlationId: 'c-mid')) + ->toThrow(InvalidExecutionStateException::class, 'COMPENSATING'); + + expect($log)->toBe(['reserve', 'charge', 'notify']) + ->and($engines->persistence->findById('c-mid')?->status)->toBe(ExecutionStatus::Compensating); +}); + it('turns a suppressed error into COMPLETED, after compensating', function (): void { $engines = TestEngines::create(); $log = []; From e02ce4bc28c5cb86353439e80930213952c9ec3e Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 19:59:24 -0700 Subject: [PATCH 18/58] feat(orchestration): saga composition with data flow and compensation of the members that completed --- .../CompositionDataFlowManager.php | 53 +++++++ .../Saga/Composition/CompositionResult.php | 41 +++++ .../src/Saga/Composition/CompositionSaga.php | 26 ++++ .../Composition/CompositionSagaBuilder.php | 68 +++++++++ .../src/Saga/Composition/DataMapping.php | 16 ++ .../src/Saga/Composition/SagaComposition.php | 43 ++++++ .../Composition/SagaCompositionBuilder.php | 55 +++++++ .../src/Saga/Composition/SagaCompositor.php | 131 ++++++++++++++++ .../Saga/Composition/SagaCompositionTest.php | 43 ++++++ .../Saga/Composition/SagaCompositorTest.php | 142 ++++++++++++++++++ .../tests/Support/TestEngines.php | 5 + 11 files changed, 623 insertions(+) create mode 100644 packages/orchestration/src/Saga/Composition/CompositionDataFlowManager.php create mode 100644 packages/orchestration/src/Saga/Composition/CompositionResult.php create mode 100644 packages/orchestration/src/Saga/Composition/CompositionSaga.php create mode 100644 packages/orchestration/src/Saga/Composition/CompositionSagaBuilder.php create mode 100644 packages/orchestration/src/Saga/Composition/DataMapping.php create mode 100644 packages/orchestration/src/Saga/Composition/SagaComposition.php create mode 100644 packages/orchestration/src/Saga/Composition/SagaCompositionBuilder.php create mode 100644 packages/orchestration/src/Saga/Composition/SagaCompositor.php create mode 100644 packages/orchestration/tests/Saga/Composition/SagaCompositionTest.php create mode 100644 packages/orchestration/tests/Saga/Composition/SagaCompositorTest.php diff --git a/packages/orchestration/src/Saga/Composition/CompositionDataFlowManager.php b/packages/orchestration/src/Saga/Composition/CompositionDataFlowManager.php new file mode 100644 index 00000000..290c61e5 --- /dev/null +++ b/packages/orchestration/src/Saga/Composition/CompositionDataFlowManager.php @@ -0,0 +1,53 @@ + $global + * @param array $staticInput + * @param list $mappings + * @param array> $resultsByAlias alias => step id => step result + * @return array + */ + public function resolveInputs(array $global, array $staticInput, array $mappings, string $alias, array $resultsByAlias): array + { + $input = array_merge($global, $staticInput); + + foreach ($mappings as $mapping) { + if ($mapping->targetSaga !== $alias || ! isset($resultsByAlias[$mapping->sourceSaga])) { + continue; + } + + $value = $this->extract($resultsByAlias[$mapping->sourceSaga], $mapping->sourceField); + if ($value !== null) { + $input[$mapping->targetField] = $value; + } + } + + return $input; + } + + /** + * @param array $stepResults + */ + private function extract(array $stepResults, string $field): mixed + { + foreach ($stepResults as $result) { + if (is_array($result) && array_key_exists($field, $result)) { + return $result[$field]; + } + } + + return $stepResults[$field] ?? null; + } +} diff --git a/packages/orchestration/src/Saga/Composition/CompositionResult.php b/packages/orchestration/src/Saga/Composition/CompositionResult.php new file mode 100644 index 00000000..0c30940b --- /dev/null +++ b/packages/orchestration/src/Saga/Composition/CompositionResult.php @@ -0,0 +1,41 @@ + $sagaResults + * @param list $compensated + * @param list $skipped + */ + public function __construct( + public string $name, + public string $correlationId, + public bool $success, + public array $sagaResults, + public ?Throwable $error = null, + public array $compensated = [], + public array $skipped = [], + ) {} + + /** @return array */ + public function toArray(): array + { + return [ + 'name' => $this->name, + 'correlationId' => $this->correlationId, + 'success' => $this->success, + 'error' => $this->error?->getMessage(), + 'compensated' => $this->compensated, + 'skipped' => $this->skipped, + 'sagas' => array_map(static fn (SagaResult $result): array => $result->toArray(), $this->sagaResults), + ]; + } +} diff --git a/packages/orchestration/src/Saga/Composition/CompositionSaga.php b/packages/orchestration/src/Saga/Composition/CompositionSaga.php new file mode 100644 index 00000000..92814097 --- /dev/null +++ b/packages/orchestration/src/Saga/Composition/CompositionSaga.php @@ -0,0 +1,26 @@ + $dependsOn + * @param array $staticInput + */ + public function __construct( + public string $alias, + public string $sagaName, + public array $dependsOn = [], + public array $staticInput = [], + public bool $optional = false, + public string $condition = '', + ) {} +} diff --git a/packages/orchestration/src/Saga/Composition/CompositionSagaBuilder.php b/packages/orchestration/src/Saga/Composition/CompositionSagaBuilder.php new file mode 100644 index 00000000..6a5f05a5 --- /dev/null +++ b/packages/orchestration/src/Saga/Composition/CompositionSagaBuilder.php @@ -0,0 +1,68 @@ + */ + private array $dependsOn = []; + + /** @var array */ + private array $input = []; + + private bool $optional = false; + + private string $condition = ''; + + public function __construct( + private readonly SagaCompositionBuilder $composition, + private readonly string $alias, + ) { + $this->sagaName = $alias; + } + + public function sagaName(string $sagaName): self + { + $this->sagaName = $sagaName; + + return $this; + } + + public function dependsOn(string ...$aliases): self + { + $this->dependsOn = array_values([...$this->dependsOn, ...$aliases]); + + return $this; + } + + public function input(string $key, mixed $value): self + { + $this->input[$key] = $value; + + return $this; + } + + public function optional(bool $optional = true): self + { + $this->optional = $optional; + + return $this; + } + + public function condition(string $expression): self + { + $this->condition = $expression; + + return $this; + } + + public function add(): SagaCompositionBuilder + { + return $this->composition->addSaga(new CompositionSaga($this->alias, $this->sagaName, $this->dependsOn, $this->input, $this->optional, $this->condition)); + } +} diff --git a/packages/orchestration/src/Saga/Composition/DataMapping.php b/packages/orchestration/src/Saga/Composition/DataMapping.php new file mode 100644 index 00000000..1484537d --- /dev/null +++ b/packages/orchestration/src/Saga/Composition/DataMapping.php @@ -0,0 +1,16 @@ +> */ + public array $layers; + + /** + * @param array $sagas + * @param list $dataMappings + */ + public function __construct( + public string $name, + public array $sagas, + public array $dataMappings = [], + ) { + $dependencies = []; + foreach ($sagas as $alias => $member) { + $dependencies[(string) $alias] = $member->dependsOn; + } + $this->layers = TopologyBuilder::layers("Saga composition [{$name}]", $dependencies); + + foreach ($dataMappings as $mapping) { + foreach ([$mapping->sourceSaga, $mapping->targetSaga] as $alias) { + if (! isset($sagas[$alias])) { + throw new TopologyValidationException("Saga composition [{$name}] maps data ".($alias === $mapping->sourceSaga ? 'from' : 'to')." [{$alias}], which is not a member of it."); + } + } + } + } +} diff --git a/packages/orchestration/src/Saga/Composition/SagaCompositionBuilder.php b/packages/orchestration/src/Saga/Composition/SagaCompositionBuilder.php new file mode 100644 index 00000000..38472e26 --- /dev/null +++ b/packages/orchestration/src/Saga/Composition/SagaCompositionBuilder.php @@ -0,0 +1,55 @@ +saga('stock')->sagaName('reserve-stock')->add() + * ->saga('charge')->sagaName('charge-card')->dependsOn('stock')->add() + * ->dataFlow('stock', 'reservationId', 'charge', 'reservationId') + * ->build(); + */ +final class SagaCompositionBuilder +{ + /** @var array */ + private array $sagas = []; + + /** @var list */ + private array $dataMappings = []; + + private function __construct(private readonly string $name) {} + + public static function composition(string $name): self + { + return new self($name); + } + + public function saga(string $alias): CompositionSagaBuilder + { + return new CompositionSagaBuilder($this, $alias); + } + + public function dataFlow(string $sourceSaga, string $sourceField, string $targetSaga, string $targetField): self + { + $this->dataMappings[] = new DataMapping($sourceSaga, $sourceField, $targetSaga, $targetField); + + return $this; + } + + /** @internal called by CompositionSagaBuilder::add() */ + public function addSaga(CompositionSaga $saga): self + { + $this->sagas[$saga->alias] = $saga; + + return $this; + } + + public function build(): SagaComposition + { + return new SagaComposition($this->name, $this->sagas, $this->dataMappings); + } +} diff --git a/packages/orchestration/src/Saga/Composition/SagaCompositor.php b/packages/orchestration/src/Saga/Composition/SagaCompositor.php new file mode 100644 index 00000000..9a9dbe8e --- /dev/null +++ b/packages/orchestration/src/Saga/Composition/SagaCompositor.php @@ -0,0 +1,131 @@ + $input + * @param array $headers + */ + public function compose(SagaComposition $composition, array $input = [], array $headers = []): CompositionResult + { + foreach ($composition->sagas as $member) { + if ($member->condition !== '') { + $this->conditions->lint($member->condition, "Saga composition [{$composition->name}] member [{$member->alias}]"); + } + } + + $correlationId = Str::uuid()->toString(); + $this->events->onCompositionStarted($composition->name, $correlationId); + + /** @var array $results */ + $results = []; + $completed = []; + $skipped = []; + $unavailable = []; + + foreach ($composition->layers as $layer) { + foreach ($layer as $alias) { + $member = $composition->sagas[$alias]; + $resultsByAlias = array_map(static fn (SagaResult $result): array => $result->stepResults(), $results); + + try { + if (array_intersect($member->dependsOn, $unavailable) !== [] + || ($member->condition !== '' && ! $this->conditions->evaluate($member->condition, ['input' => $input, 'results' => $resultsByAlias, 'variables' => [], 'headers' => $headers]))) { + $skipped[] = $alias; + $unavailable[] = $alias; + + continue; + } + + // The composition's own headers win over a caller's header of the same name: a member must + // never claim to belong to a composition it is not part of. + $result = $this->engine->execute( + $member->sagaName, + $this->dataFlow->resolveInputs($input, $member->staticInput, $composition->dataMappings, $alias, $resultsByAlias), + array_merge($headers, [self::COMPOSITION_HEADER => $correlationId, self::ALIAS_HEADER => $alias]), + ); + } catch (Throwable $error) { + return $this->abort($composition, $correlationId, $results, $completed, $skipped, $error); + } + + $results[$alias] = $result; + + if ($result->isSuccess()) { + $completed[] = $alias; + + continue; + } + + if ($member->optional) { + $unavailable[] = $alias; + + continue; + } + + return $this->abort($composition, $correlationId, $results, $completed, $skipped, $result->error ?? new OrchestrationException("Composition member [{$alias}] failed.")); + } + } + + $this->events->onCompositionCompleted($composition->name, $correlationId, true); + + return new CompositionResult($composition->name, $correlationId, true, $results, null, [], $skipped); + } + + /** + * Compensates every completed member in reverse completion order and reports the composition as failed. + * + * @param array $results + * @param list $completed + * @param list $skipped + */ + private function abort(SagaComposition $composition, string $correlationId, array $results, array $completed, array $skipped, Throwable $error): CompositionResult + { + $compensated = []; + foreach (array_reverse($completed) as $done) { + try { + $this->engine->compensate($results[$done]->correlationId); + $compensated[] = $done; + } catch (Throwable) { + // Left COMPENSATING in the persistence port for recovery; the sweep continues. + } + } + $this->events->onCompositionCompleted($composition->name, $correlationId, false); + + return new CompositionResult($composition->name, $correlationId, false, $results, $error, $compensated, $skipped); + } +} diff --git a/packages/orchestration/tests/Saga/Composition/SagaCompositionTest.php b/packages/orchestration/tests/Saga/Composition/SagaCompositionTest.php new file mode 100644 index 00000000..73bb12e2 --- /dev/null +++ b/packages/orchestration/tests/Saga/Composition/SagaCompositionTest.php @@ -0,0 +1,43 @@ +saga('stock')->sagaName('reserve-stock')->input('warehouse', 'eu-1')->add() + ->saga('charge')->sagaName('charge-card')->dependsOn('stock')->add() + ->saga('notify')->sagaName('notify-customer')->dependsOn('charge')->optional()->condition('input["notify"] == true')->add() + ->dataFlow('stock', 'reservationId', 'charge', 'reservationId') + ->build(); + + expect($composition->layers)->toBe([['stock'], ['charge'], ['notify']]) + ->and($composition->sagas['stock']->staticInput)->toBe(['warehouse' => 'eu-1']) + ->and($composition->sagas['notify']->optional)->toBeTrue() + ->and($composition->sagas['notify']->condition)->toBe('input["notify"] == true') + ->and($composition->dataMappings)->toEqual([new DataMapping('stock', 'reservationId', 'charge', 'reservationId')]); +}); + +it('refuses a cycle and a data flow naming an unknown member', function (): void { + expect(fn () => SagaCompositionBuilder::composition('loop') + ->saga('a')->sagaName('x')->dependsOn('b')->add() + ->saga('b')->sagaName('y')->dependsOn('a')->add() + ->build())->toThrow(TopologyValidationException::class, 'Saga composition [loop]: the steps form a cycle') + ->and(fn () => SagaCompositionBuilder::composition('flow') + ->saga('a')->sagaName('x')->add() + ->dataFlow('ghost', 'id', 'a', 'id') + ->build())->toThrow(TopologyValidationException::class, 'Saga composition [flow] maps data from [ghost], which is not a member of it.'); +}); + +it('flows a field found inside a step result, or a whole step result, into the next member', function (): void { + $manager = new CompositionDataFlowManager; + $mappings = [new DataMapping('stock', 'reservationId', 'charge', 'reservation'), new DataMapping('stock', 'audit', 'charge', 'audit')]; + + expect($manager->resolveInputs(['amount' => 5, 'warehouse' => 'x'], ['warehouse' => 'eu-1'], $mappings, 'charge', [ + 'stock' => ['reserve' => ['reservationId' => 'r-1'], 'audit' => 'logged'], + ]))->toBe(['amount' => 5, 'warehouse' => 'eu-1', 'reservation' => 'r-1', 'audit' => 'logged']); +}); diff --git a/packages/orchestration/tests/Saga/Composition/SagaCompositorTest.php b/packages/orchestration/tests/Saga/Composition/SagaCompositorTest.php new file mode 100644 index 00000000..f829a768 --- /dev/null +++ b/packages/orchestration/tests/Saga/Composition/SagaCompositorTest.php @@ -0,0 +1,142 @@ + $log */ +function orchestrationCheckoutEngines(array &$log): TestEngines +{ + $engines = TestEngines::create(); + $engines->sagaRegistry->register(SagaBuilder::saga('reserve-stock') + ->step('reserve')->handler(static function (array $input) use (&$log): array { + $log[] = 'reserve in '.(is_string($input['warehouse'] ?? null) ? $input['warehouse'] : '?'); + + return ['reservationId' => 'r-1']; + })->compensation(static function () use (&$log): void { + $log[] = 'release stock'; + })->add()->build()); + $engines->sagaRegistry->register(SagaBuilder::saga('charge-card') + ->step('charge')->handler(static function (array $input) use (&$log): string { + $log[] = 'charge for '.(is_string($input['reservationId'] ?? null) ? $input['reservationId'] : '?'); + if (($input['amount'] ?? 0) > 100) { + throw new RuntimeException('limit exceeded'); + } + + return 'paid'; + })->add()->build()); + $engines->sagaRegistry->register(SagaBuilder::saga('notify-customer') + ->step('send')->handler(static function () use (&$log): string { + $log[] = 'notify'; + + return 'sent'; + })->add()->build()); + + return $engines; +} + +function orchestrationCheckout(): SagaComposition +{ + return SagaCompositionBuilder::composition('checkout') + ->saga('stock')->sagaName('reserve-stock')->input('warehouse', 'eu-1')->add() + ->saga('charge')->sagaName('charge-card')->dependsOn('stock')->add() + ->saga('notify')->sagaName('notify-customer')->dependsOn('charge')->condition('input["notify"] == true')->add() + ->dataFlow('stock', 'reservationId', 'charge', 'reservationId') + ->build(); +} + +it('runs members layer by layer, flows data between them and skips a member whose condition is false', function (): void { + $log = []; + $engines = orchestrationCheckoutEngines($log); + + $result = $engines->compositor->compose(orchestrationCheckout(), ['amount' => 40, 'notify' => false]); + + expect($result->success)->toBeTrue() + ->and($log)->toBe(['reserve in eu-1', 'charge for r-1']) + ->and($result->skipped)->toBe(['notify']) + ->and(array_keys($result->sagaResults))->toBe(['stock', 'charge']) + ->and($result->sagaResults['charge']->headers['x-firefly-composition'] ?? null)->toBe($result->correlationId) + ->and($engines->events->hooks())->toContain('onCompositionStarted') + ->and(array_slice($engines->events->hooks(), -1))->toBe(['onCompositionCompleted']); +}); + +it('compensates the members that completed, in reverse, when a required member fails', function (): void { + $log = []; + $engines = orchestrationCheckoutEngines($log); + + $result = $engines->compositor->compose(orchestrationCheckout(), ['amount' => 500, 'notify' => true]); + + expect($result->success)->toBeFalse() + ->and($result->error?->getMessage())->toBe('limit exceeded') + ->and($result->compensated)->toBe(['stock']) + ->and($log)->toBe(['reserve in eu-1', 'charge for r-1', 'release stock']) + ->and($engines->persistence->findById($result->sagaResults['stock']->correlationId)?->status)->toBe(ExecutionStatus::Cancelled); +}); + +it('tolerates an optional member failure and skips what depended on it', function (): void { + $log = []; + $engines = orchestrationCheckoutEngines($log); + $composition = SagaCompositionBuilder::composition('soft') + ->saga('charge')->sagaName('charge-card')->optional()->add() + ->saga('notify')->sagaName('notify-customer')->dependsOn('charge')->add() + ->build(); + + $result = $engines->compositor->compose($composition, ['amount' => 500]); + + expect($result->success)->toBeTrue() + ->and($result->sagaResults['charge']->isSuccess())->toBeFalse() + ->and($result->skipped)->toBe(['notify']) + ->and($log)->toBe(['charge for ?']); +}); + +it('compensates the members that completed when a later member cannot even start', function (): void { + $log = []; + $engines = orchestrationCheckoutEngines($log); + $composition = SagaCompositionBuilder::composition('broken') + ->saga('stock')->sagaName('reserve-stock')->add() + ->saga('ghost')->sagaName('no-such-saga')->dependsOn('stock')->add() + ->build(); + + $result = $engines->compositor->compose($composition); + + expect($result->success)->toBeFalse() + ->and($result->error?->getMessage())->toContain('no-such-saga') + ->and($result->compensated)->toBe(['stock']) + ->and($log)->toBe(['reserve in ?', 'release stock']) + ->and($engines->persistence->findById($result->sagaResults['stock']->correlationId)?->status)->toBe(ExecutionStatus::Cancelled) + ->and(array_slice($engines->events->calls, -1))->toBe([['hook' => 'onCompositionCompleted', 'args' => ['broken', $result->correlationId, false]]]); +}); + +it('compensates the members that completed when a condition cannot be evaluated', function (): void { + $log = []; + $engines = orchestrationCheckoutEngines($log); + $composition = SagaCompositionBuilder::composition('bad-condition') + ->saga('stock')->sagaName('reserve-stock')->add() + ->saga('notify')->sagaName('notify-customer')->dependsOn('stock')->condition('input.missing.deeper == 1')->add() + ->build(); + + $result = $engines->compositor->compose($composition); + + expect($result->success)->toBeFalse() + ->and($result->error?->getMessage())->toContain('could not be evaluated') + ->and($result->compensated)->toBe(['stock']) + ->and($log)->toBe(['reserve in ?', 'release stock']); +}); + +it('stamps the composition headers over a caller header of the same name', function (): void { + $log = []; + $engines = orchestrationCheckoutEngines($log); + $composition = SagaCompositionBuilder::composition('headers')->saga('stock')->sagaName('reserve-stock')->add()->build(); + + $result = $engines->compositor->compose($composition, [], ['x-firefly-composition' => 'forged', 'x-tenant' => 'acme']); + + expect($result->sagaResults['stock']->headers)->toMatchArray([ + 'x-firefly-composition' => $result->correlationId, + 'x-firefly-composition-alias' => 'stock', + 'x-tenant' => 'acme', + ]); +}); diff --git a/packages/orchestration/tests/Support/TestEngines.php b/packages/orchestration/tests/Support/TestEngines.php index e1a865d2..940bb95e 100644 --- a/packages/orchestration/tests/Support/TestEngines.php +++ b/packages/orchestration/tests/Support/TestEngines.php @@ -24,6 +24,8 @@ use Firefly\Orchestration\Persistence\MemoryExecutionPersistenceProvider; use Firefly\Orchestration\Saga\Compensation\CompensationErrorHandlerFactory; use Firefly\Orchestration\Saga\Compensation\SagaCompensator; +use Firefly\Orchestration\Saga\Composition\CompositionDataFlowManager; +use Firefly\Orchestration\Saga\Composition\SagaCompositor; use Firefly\Orchestration\Saga\Engine\SagaEngine; use Firefly\Orchestration\Saga\Engine\SagaExecutionOrchestrator; use Firefly\Orchestration\Saga\Registry\SagaRegistry; @@ -71,6 +73,8 @@ final class TestEngines public readonly SagaEngine $sagas; + public readonly SagaCompositor $compositor; + /** * @param array $config */ @@ -124,6 +128,7 @@ private function __construct(array $config, ?ExecutionPersistenceProvider $persi $resilienceStore, ); $this->sagas = new SagaEngine($this->sagaRegistry, new SagaExecutionOrchestrator($this->invoker, $this->support), $this->compensator, $this->support, $this->settings->saga); + $this->compositor = new SagaCompositor($this->sagas, $this->events, $this->conditions, new CompositionDataFlowManager); } /** From 8212410a5d6d4bd1ace4918e395852f5afb8f465 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 20:04:33 -0700 Subject: [PATCH 19/58] fix(orchestration): report the composition members a rollback could not undo and tolerate optional members that cannot start --- .../Saga/Composition/CompositionResult.php | 18 ++- .../src/Saga/Composition/CompositionSaga.php | 6 +- .../src/Saga/Composition/SagaCompositor.php | 89 ++++++++++++-- .../Saga/Composition/SagaCompositorTest.php | 93 ++++++++++++++- .../UnreadableExecutionPersistence.php | 111 ++++++++++++++++++ 5 files changed, 302 insertions(+), 15 deletions(-) create mode 100644 packages/orchestration/tests/Support/UnreadableExecutionPersistence.php diff --git a/packages/orchestration/src/Saga/Composition/CompositionResult.php b/packages/orchestration/src/Saga/Composition/CompositionResult.php index 0c30940b..102b681b 100644 --- a/packages/orchestration/src/Saga/Composition/CompositionResult.php +++ b/packages/orchestration/src/Saga/Composition/CompositionResult.php @@ -7,13 +7,23 @@ use Firefly\Orchestration\Saga\Engine\SagaResult; use Throwable; -/** What a composition did: each member's SagaResult by alias, what was compensated and what was skipped. */ +/** + * What a composition did: each member's SagaResult by alias, what was compensated and what was skipped. + * + * `compensated` lists only the members whose every completed, compensable step was undone. A member the + * rollback could not undo — compensate() refused it, or one of its compensations failed — is listed in + * `compensationFailures` instead, with why, so a caller can tell a clean rollback from a partial one. + * `optionalFailures` holds the optional members that failed or could not start, which the composition + * tolerated. + */ final readonly class CompositionResult { /** * @param array $sagaResults * @param list $compensated * @param list $skipped + * @param array $compensationFailures alias => why that member is not fully compensated + * @param array $optionalFailures alias => why that optional member failed or did not start */ public function __construct( public string $name, @@ -23,18 +33,24 @@ public function __construct( public ?Throwable $error = null, public array $compensated = [], public array $skipped = [], + public array $compensationFailures = [], + public array $optionalFailures = [], ) {} /** @return array */ public function toArray(): array { + $messages = static fn (Throwable $error): string => $error->getMessage(); + return [ 'name' => $this->name, 'correlationId' => $this->correlationId, 'success' => $this->success, 'error' => $this->error?->getMessage(), 'compensated' => $this->compensated, + 'compensationFailures' => array_map($messages, $this->compensationFailures), 'skipped' => $this->skipped, + 'optionalFailures' => array_map($messages, $this->optionalFailures), 'sagas' => array_map(static fn (SagaResult $result): array => $result->toArray(), $this->sagaResults), ]; } diff --git a/packages/orchestration/src/Saga/Composition/CompositionSaga.php b/packages/orchestration/src/Saga/Composition/CompositionSaga.php index 92814097..4d7517e1 100644 --- a/packages/orchestration/src/Saga/Composition/CompositionSaga.php +++ b/packages/orchestration/src/Saga/Composition/CompositionSaga.php @@ -6,8 +6,10 @@ /** * One member of a SagaComposition (jafly CompositionSaga): the saga it runs, under an alias other members - * depend on and map data from. An `optional` member may fail without failing the composition (its dependents - * are skipped); a `condition` (ExpressionLanguage over `input`, `results` by alias, `headers`) skips it. + * depend on and map data from. An `optional` member may fail — or fail to start at all: an unregistered saga, + * an admission refusal, a condition that cannot be evaluated — without failing the composition; its dependents + * are skipped and the reason lands in CompositionResult::$optionalFailures. A `condition` (ExpressionLanguage + * over `input`, `results` by alias, `headers`) that evaluates false skips it. */ final readonly class CompositionSaga { diff --git a/packages/orchestration/src/Saga/Composition/SagaCompositor.php b/packages/orchestration/src/Saga/Composition/SagaCompositor.php index 9a9dbe8e..96ced9cd 100644 --- a/packages/orchestration/src/Saga/Composition/SagaCompositor.php +++ b/packages/orchestration/src/Saga/Composition/SagaCompositor.php @@ -5,7 +5,9 @@ namespace Firefly\Orchestration\Saga\Composition; use Firefly\Orchestration\Core\Condition\ConditionEvaluator; +use Firefly\Orchestration\Core\Exception\CompensationException; use Firefly\Orchestration\Core\Exception\OrchestrationException; +use Firefly\Orchestration\Core\StepStatus; use Firefly\Orchestration\Event\OrchestrationEvents; use Firefly\Orchestration\Saga\Engine\SagaEngine; use Firefly\Orchestration\Saga\Engine\SagaResult; @@ -21,8 +23,20 @@ * A required member "fails" when its saga reports a failure, and equally when it cannot run at all — its saga * is not registered, admission refuses it, or its condition cannot be evaluated: either way the members before * it already did their work, and leaving them uncompensated would be the one outcome a composition exists to - * prevent. A member whose own compensation throws is left out of `compensated` and the sweep moves on to the - * next one; its execution stays COMPENSATING in the persistence port, where recovery finds it. + * prevent. An optional member is tolerated either way: a failed result or a refusal to start lands in + * CompositionResult::$optionalFailures, its dependents are skipped, and the composition carries on. + * + * The rollback sweep never stops at a member it cannot undo; it records it in CompositionResult:: + * $compensationFailures and reports it through OrchestrationEvents::onStepCompensationFailed() (the composition + * name, its correlation id and the member's alias). Only a member whose every completed, compensable step was + * undone is listed in `compensated`. What an uncompensated member's execution looks like depends on where the + * undo stopped: + * - compensate() refused before it started (the execution lock timed out, the store could not be read, the + * execution was already cancelled): the execution keeps its status — a COMPLETED member stays COMPLETED, + * which recovery never picks up, so the result is the only place the caller learns of it; + * - one of its compensations failed and the engine handled it (the error handler continued, FAIL_SAGA, an + * open compensation circuit): the execution ends CANCELLED with the failed steps marked in its report; + * - compensate() threw after it started: the execution stays COMPENSATING, where recovery finds it. */ final class SagaCompositor { @@ -57,6 +71,8 @@ public function compose(SagaComposition $composition, array $input = [], array $ $completed = []; $skipped = []; $unavailable = []; + /** @var array $optionalFailures */ + $optionalFailures = []; foreach ($composition->layers as $layer) { foreach ($layer as $alias) { @@ -80,7 +96,14 @@ public function compose(SagaComposition $composition, array $input = [], array $ array_merge($headers, [self::COMPOSITION_HEADER => $correlationId, self::ALIAS_HEADER => $alias]), ); } catch (Throwable $error) { - return $this->abort($composition, $correlationId, $results, $completed, $skipped, $error); + if ($member->optional) { + $optionalFailures[$alias] = $error; + $unavailable[] = $alias; + + continue; + } + + return $this->abort($composition, $correlationId, $results, $completed, $skipped, $optionalFailures, $error); } $results[$alias] = $result; @@ -91,41 +114,87 @@ public function compose(SagaComposition $composition, array $input = [], array $ continue; } + $error = $result->error ?? new OrchestrationException("Composition member [{$alias}] failed."); if ($member->optional) { + $optionalFailures[$alias] = $error; $unavailable[] = $alias; continue; } - return $this->abort($composition, $correlationId, $results, $completed, $skipped, $result->error ?? new OrchestrationException("Composition member [{$alias}] failed.")); + return $this->abort($composition, $correlationId, $results, $completed, $skipped, $optionalFailures, $error); } } $this->events->onCompositionCompleted($composition->name, $correlationId, true); - return new CompositionResult($composition->name, $correlationId, true, $results, null, [], $skipped); + return new CompositionResult($composition->name, $correlationId, true, $results, null, [], $skipped, [], $optionalFailures); } /** * Compensates every completed member in reverse completion order and reports the composition as failed. + * A member that cannot be fully undone is recorded and reported, and the sweep moves on to the next one. * * @param array $results * @param list $completed * @param list $skipped + * @param array $optionalFailures */ - private function abort(SagaComposition $composition, string $correlationId, array $results, array $completed, array $skipped, Throwable $error): CompositionResult + private function abort(SagaComposition $composition, string $correlationId, array $results, array $completed, array $skipped, array $optionalFailures, Throwable $error): CompositionResult { $compensated = []; + $failures = []; foreach (array_reverse($completed) as $done) { try { - $this->engine->compensate($results[$done]->correlationId); + $failure = $this->undo($done, $results[$done]); + } catch (Throwable $refused) { + $failure = $refused; + } + + if ($failure === null) { $compensated[] = $done; - } catch (Throwable) { - // Left COMPENSATING in the persistence port for recovery; the sweep continues. + + continue; } + + $failures[$done] = $failure; + $this->events->onStepCompensationFailed($composition->name, $correlationId, $done, $failure); } $this->events->onCompositionCompleted($composition->name, $correlationId, false); - return new CompositionResult($composition->name, $correlationId, false, $results, $error, $compensated, $skipped); + return new CompositionResult($composition->name, $correlationId, false, $results, $error, $compensated, $skipped, $failures, $optionalFailures); + } + + /** + * Compensates one completed member. Returns null when every step it completed that has a compensation was + * undone, and otherwise the reason it was not: SagaEngine::compensate() handles a failed compensation + * itself and still returns a CANCELLED result, so the outcome of each step is what tells the two apart. + */ + private function undo(string $alias, SagaResult $completed): ?Throwable + { + $undone = $this->engine->compensate($completed->correlationId); + $definition = $this->engine->registry()->get($completed->sagaName); + + $left = []; + $cause = null; + foreach ($completed->steps as $stepId => $outcome) { + if ($outcome->status !== StepStatus::Done || ($definition->steps[$stepId] ?? null)?->compensation === null) { + continue; + } + + $after = $undone->steps[$stepId] ?? null; + if ($after?->compensated !== true) { + $left[] = $stepId; + $cause ??= $after?->compensationError; + } + } + + if ($left === []) { + return null; + } + + $reason = $cause?->getMessage() ?? $undone->report->failureReason ?? 'not compensated'; + + return new CompensationException("Composition member [{$alias}] left [".implode(', ', $left)."] uncompensated: {$reason}", $cause); } } diff --git a/packages/orchestration/tests/Saga/Composition/SagaCompositorTest.php b/packages/orchestration/tests/Saga/Composition/SagaCompositorTest.php index f829a768..56aa01b6 100644 --- a/packages/orchestration/tests/Saga/Composition/SagaCompositorTest.php +++ b/packages/orchestration/tests/Saga/Composition/SagaCompositorTest.php @@ -2,16 +2,19 @@ declare(strict_types=1); +use Firefly\Orchestration\Core\Exception\CompensationException; +use Firefly\Orchestration\Core\Exception\DefinitionNotFoundException; use Firefly\Orchestration\Core\ExecutionStatus; use Firefly\Orchestration\Saga\Builder\SagaBuilder; use Firefly\Orchestration\Saga\Composition\SagaComposition; use Firefly\Orchestration\Saga\Composition\SagaCompositionBuilder; use Firefly\Orchestration\Tests\Support\TestEngines; +use Firefly\Orchestration\Tests\Support\UnreadableExecutionPersistence; /** @param list $log */ -function orchestrationCheckoutEngines(array &$log): TestEngines +function orchestrationCheckoutEngines(array &$log, ?TestEngines $engines = null): TestEngines { - $engines = TestEngines::create(); + $engines ??= TestEngines::create(); $engines->sagaRegistry->register(SagaBuilder::saga('reserve-stock') ->step('reserve')->handler(static function (array $input) use (&$log): array { $log[] = 'reserve in '.(is_string($input['warehouse'] ?? null) ? $input['warehouse'] : '?'); @@ -73,6 +76,7 @@ function orchestrationCheckout(): SagaComposition expect($result->success)->toBeFalse() ->and($result->error?->getMessage())->toBe('limit exceeded') ->and($result->compensated)->toBe(['stock']) + ->and($result->compensationFailures)->toBe([]) ->and($log)->toBe(['reserve in eu-1', 'charge for r-1', 'release stock']) ->and($engines->persistence->findById($result->sagaResults['stock']->correlationId)?->status)->toBe(ExecutionStatus::Cancelled); }); @@ -89,6 +93,7 @@ function orchestrationCheckout(): SagaComposition expect($result->success)->toBeTrue() ->and($result->sagaResults['charge']->isSuccess())->toBeFalse() + ->and(array_map(static fn (Throwable $error): string => $error->getMessage(), $result->optionalFailures))->toBe(['charge' => 'limit exceeded']) ->and($result->skipped)->toBe(['notify']) ->and($log)->toBe(['charge for ?']); }); @@ -140,3 +145,87 @@ function orchestrationCheckout(): SagaComposition 'x-tenant' => 'acme', ]); }); + +it('tolerates an optional member that cannot even start', function (): void { + $log = []; + $engines = orchestrationCheckoutEngines($log); + $composition = SagaCompositionBuilder::composition('soft-start') + ->saga('stock')->sagaName('reserve-stock')->add() + ->saga('ghost')->sagaName('no-such-saga')->dependsOn('stock')->optional()->add() + ->saga('after-ghost')->sagaName('notify-customer')->dependsOn('ghost')->add() + ->saga('notify')->sagaName('notify-customer')->dependsOn('stock')->add() + ->build(); + + $result = $engines->compositor->compose($composition); + + expect($result->success)->toBeTrue() + ->and($result->compensated)->toBe([]) + ->and(array_keys($result->optionalFailures))->toBe(['ghost']) + ->and($result->optionalFailures['ghost'])->toBeInstanceOf(DefinitionNotFoundException::class) + ->and($result->toArray()['optionalFailures'])->toBe(['ghost' => $result->optionalFailures['ghost']->getMessage()]) + ->and($result->skipped)->toBe(['after-ghost']) + ->and($log)->toBe(['reserve in ?', 'notify']) + ->and($engines->persistence->findById($result->sagaResults['stock']->correlationId)?->status)->toBe(ExecutionStatus::Completed); +}); + +it('does not report a member as compensated when its own compensation fails', function (string $handler): void { + $log = []; + $engines = orchestrationCheckoutEngines($log, TestEngines::create(['saga' => ['compensation-error-handler' => $handler]])); + $engines->sagaRegistry->register(SagaBuilder::saga('hold-seat') + ->step('hold')->handler(static fn (): string => 'held') + ->compensation(static function (): void { + throw new RuntimeException('seat service is down'); + })->add()->build()); + $composition = SagaCompositionBuilder::composition('trip') + ->saga('stock')->sagaName('reserve-stock')->add() + ->saga('seat')->sagaName('hold-seat')->dependsOn('stock')->add() + ->saga('charge')->sagaName('charge-card')->dependsOn('seat')->add() + ->build(); + + $result = $engines->compositor->compose($composition, ['amount' => 500]); + + expect($result->success)->toBeFalse() + ->and($result->compensated)->toBe(['stock']) + ->and(array_keys($result->compensationFailures))->toBe(['seat']) + ->and($result->compensationFailures['seat'])->toBeInstanceOf(CompensationException::class) + ->and($result->compensationFailures['seat']->getMessage())->toBe('Composition member [seat] left [hold] uncompensated: seat service is down') + ->and($result->toArray()['compensationFailures'])->toBe(['seat' => 'Composition member [seat] left [hold] uncompensated: seat service is down']) + ->and($log)->toBe(['reserve in ?', 'charge for ?', 'release stock']) + ->and($engines->events->calls)->toContain(['hook' => 'onStepCompensationFailed', 'args' => ['trip', $result->correlationId, 'seat', $result->compensationFailures['seat']->getMessage()]]); +})->with(['default', 'fail-fast']); + +it('keeps sweeping and reports the member compensate() refuses before it starts', function (): void { + $log = []; + $store = new UnreadableExecutionPersistence; + $engines = orchestrationCheckoutEngines($log, TestEngines::create([], $store)); + $engines->sagaRegistry->register(SagaBuilder::saga('hold-seat') + ->step('hold')->handler(static fn (): string => 'held') + ->compensation(static function () use (&$log): void { + $log[] = 'release seat'; + })->add()->build()); + $engines->sagaRegistry->register(SagaBuilder::saga('charge-and-lose-stock') + ->step('charge')->handler(static function () use ($store): never { + // The store stops answering for the stock member after it completed. + $store->refuseLoadsOf('reserve-stock'); + + throw new RuntimeException('card declined'); + })->add()->build()); + $composition = SagaCompositionBuilder::composition('trip') + ->saga('stock')->sagaName('reserve-stock')->add() + ->saga('seat')->sagaName('hold-seat')->dependsOn('stock')->add() + ->saga('charge')->sagaName('charge-and-lose-stock')->dependsOn('seat')->add() + ->build(); + + $result = $engines->compositor->compose($composition); + + expect($result->success)->toBeFalse() + ->and($result->error?->getMessage())->toBe('card declined') + ->and($result->compensated)->toBe(['seat']) + ->and(array_keys($result->compensationFailures))->toBe(['stock']) + ->and($result->compensationFailures['stock']->getMessage())->toStartWith('cannot read [') + ->and($log)->toBe(['reserve in ?', 'release seat']) + // Refused before it checkpointed COMPENSATING: the member is still COMPLETED, invisible to recovery. + ->and($store->peek($result->sagaResults['stock']->correlationId)?->status)->toBe(ExecutionStatus::Completed) + ->and($store->peek($result->sagaResults['seat']->correlationId)?->status)->toBe(ExecutionStatus::Cancelled) + ->and($engines->events->calls)->toContain(['hook' => 'onStepCompensationFailed', 'args' => ['trip', $result->correlationId, 'stock', $result->compensationFailures['stock']->getMessage()]]); +}); diff --git a/packages/orchestration/tests/Support/UnreadableExecutionPersistence.php b/packages/orchestration/tests/Support/UnreadableExecutionPersistence.php new file mode 100644 index 00000000..19c0f8cf --- /dev/null +++ b/packages/orchestration/tests/Support/UnreadableExecutionPersistence.php @@ -0,0 +1,111 @@ + */ + private array $refused = []; + + public function __construct() + { + $this->inner = new MemoryExecutionPersistenceProvider; + } + + public function refuseLoadsOf(string $executionName): void + { + $this->refused[] = $executionName; + } + + /** Reads past the refusal, so a test can assert what the store holds. */ + public function peek(string $correlationId): ?ExecutionState + { + return $this->inner->findById($correlationId); + } + + public function save(ExecutionState $state): void + { + $this->inner->save($state); + } + + public function findById(string $correlationId): ?ExecutionState + { + $state = $this->inner->findById($correlationId); + if ($state !== null && in_array($state->executionName, $this->refused, true)) { + throw new RuntimeException("cannot read [{$correlationId}]"); + } + + return $state; + } + + public function find(ExecutionCriteria $criteria): array + { + return $this->inner->find($criteria); + } + + public function findInFlight(int $limit = 100): array + { + return $this->inner->findInFlight($limit); + } + + public function findStale(DateTimeImmutable $before, int $limit = 100): array + { + return $this->inner->findStale($before, $limit); + } + + public function findDue(DateTimeImmutable $now, int $limit = 100): array + { + return $this->inner->findDue($now, $limit); + } + + public function findBySearchAttributes(array $criteria, int $limit = 100): array + { + return $this->inner->findBySearchAttributes($criteria, $limit); + } + + public function updateStatus(string $correlationId, ExecutionStatus $status, DateTimeImmutable $now): void + { + $this->inner->updateStatus($correlationId, $status, $now); + } + + public function delete(string $correlationId): bool + { + return $this->inner->delete($correlationId); + } + + public function cleanup(DateTimeImmutable $before): int + { + return $this->inner->cleanup($before); + } + + public function countInFlight(): int + { + return $this->inner->countInFlight(); + } + + public function countByStatus(): array + { + return $this->inner->countByStatus(); + } + + public function isHealthy(): bool + { + return $this->inner->isHealthy(); + } +} From 21e636ed87d19fb9fae0b50a686dc6f79bc98332 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 20:10:11 -0700 Subject: [PATCH 20/58] feat(orchestration): TCC transactions with ordered participants, per-phase retries, phase checkpoints and composition --- .../src/Tcc/Attributes/CancelMethod.php | 21 ++ .../src/Tcc/Attributes/ConfirmMethod.php | 21 ++ .../src/Tcc/Attributes/FromTry.php | 14 + .../src/Tcc/Attributes/OnTccComplete.php | 17 ++ .../src/Tcc/Attributes/OnTccError.php | 22 ++ .../src/Tcc/Attributes/ScheduledTcc.php | 23 ++ .../orchestration/src/Tcc/Attributes/Tcc.php | 26 ++ .../src/Tcc/Attributes/TccEvent.php | 18 ++ .../src/Tcc/Attributes/TccParticipant.php | 26 ++ .../src/Tcc/Attributes/TryMethod.php | 18 ++ .../src/Tcc/Builder/TccBuilder.php | 106 ++++++++ .../src/Tcc/Builder/TccParticipantBuilder.php | 107 ++++++++ .../src/Tcc/Composition/TccComposition.php | 40 +++ .../Tcc/Composition/TccCompositionBuilder.php | 49 ++++ .../Tcc/Composition/TccCompositionEntry.php | 22 ++ .../TccCompositionEntryBuilder.php | 68 +++++ .../Tcc/Composition/TccCompositionResult.php | 56 ++++ .../src/Tcc/Composition/TccCompositor.php | 188 ++++++++++++++ .../src/Tcc/Engine/ParticipantOutcome.php | 45 ++++ .../src/Tcc/Engine/TccEngine.php | 193 ++++++++++++++ .../Tcc/Engine/TccExecutionOrchestrator.php | 243 ++++++++++++++++++ .../src/Tcc/Engine/TccInputs.php | 48 ++++ .../src/Tcc/Engine/TccResult.php | 105 ++++++++ .../src/Tcc/Engine/TccRunOutcome.php | 20 ++ .../src/Tcc/Registry/TccDefinition.php | 89 +++++++ .../Tcc/Registry/TccParticipantDefinition.php | 85 ++++++ .../src/Tcc/Registry/TccPhaseDefinition.php | 32 +++ .../src/Tcc/Registry/TccRegistry.php | 46 ++++ .../tests/Support/TestEngines.php | 14 + .../tests/Tcc/TccCompositorTest.php | 140 ++++++++++ .../tests/Tcc/TccDefinitionTest.php | 36 +++ .../orchestration/tests/Tcc/TccEngineTest.php | 178 +++++++++++++ .../orchestration/tests/Tcc/TccResumeTest.php | 119 +++++++++ 33 files changed, 2235 insertions(+) create mode 100644 packages/orchestration/src/Tcc/Attributes/CancelMethod.php create mode 100644 packages/orchestration/src/Tcc/Attributes/ConfirmMethod.php create mode 100644 packages/orchestration/src/Tcc/Attributes/FromTry.php create mode 100644 packages/orchestration/src/Tcc/Attributes/OnTccComplete.php create mode 100644 packages/orchestration/src/Tcc/Attributes/OnTccError.php create mode 100644 packages/orchestration/src/Tcc/Attributes/ScheduledTcc.php create mode 100644 packages/orchestration/src/Tcc/Attributes/Tcc.php create mode 100644 packages/orchestration/src/Tcc/Attributes/TccEvent.php create mode 100644 packages/orchestration/src/Tcc/Attributes/TccParticipant.php create mode 100644 packages/orchestration/src/Tcc/Attributes/TryMethod.php create mode 100644 packages/orchestration/src/Tcc/Builder/TccBuilder.php create mode 100644 packages/orchestration/src/Tcc/Builder/TccParticipantBuilder.php create mode 100644 packages/orchestration/src/Tcc/Composition/TccComposition.php create mode 100644 packages/orchestration/src/Tcc/Composition/TccCompositionBuilder.php create mode 100644 packages/orchestration/src/Tcc/Composition/TccCompositionEntry.php create mode 100644 packages/orchestration/src/Tcc/Composition/TccCompositionEntryBuilder.php create mode 100644 packages/orchestration/src/Tcc/Composition/TccCompositionResult.php create mode 100644 packages/orchestration/src/Tcc/Composition/TccCompositor.php create mode 100644 packages/orchestration/src/Tcc/Engine/ParticipantOutcome.php create mode 100644 packages/orchestration/src/Tcc/Engine/TccEngine.php create mode 100644 packages/orchestration/src/Tcc/Engine/TccExecutionOrchestrator.php create mode 100644 packages/orchestration/src/Tcc/Engine/TccInputs.php create mode 100644 packages/orchestration/src/Tcc/Engine/TccResult.php create mode 100644 packages/orchestration/src/Tcc/Engine/TccRunOutcome.php create mode 100644 packages/orchestration/src/Tcc/Registry/TccDefinition.php create mode 100644 packages/orchestration/src/Tcc/Registry/TccParticipantDefinition.php create mode 100644 packages/orchestration/src/Tcc/Registry/TccPhaseDefinition.php create mode 100644 packages/orchestration/src/Tcc/Registry/TccRegistry.php create mode 100644 packages/orchestration/tests/Tcc/TccCompositorTest.php create mode 100644 packages/orchestration/tests/Tcc/TccDefinitionTest.php create mode 100644 packages/orchestration/tests/Tcc/TccEngineTest.php create mode 100644 packages/orchestration/tests/Tcc/TccResumeTest.php diff --git a/packages/orchestration/src/Tcc/Attributes/CancelMethod.php b/packages/orchestration/src/Tcc/Attributes/CancelMethod.php new file mode 100644 index 00000000..21a1e1a5 --- /dev/null +++ b/packages/orchestration/src/Tcc/Attributes/CancelMethod.php @@ -0,0 +1,21 @@ + $errorTypes exception class names; empty matches every error + */ + public function __construct( + public array $errorTypes = [], + public bool $suppressError = false, + public bool $async = false, + public int $priority = 0, + ) {} +} diff --git a/packages/orchestration/src/Tcc/Attributes/ScheduledTcc.php b/packages/orchestration/src/Tcc/Attributes/ScheduledTcc.php new file mode 100644 index 00000000..9eed5a77 --- /dev/null +++ b/packages/orchestration/src/Tcc/Attributes/ScheduledTcc.php @@ -0,0 +1,23 @@ + */ + private array $participants = []; + + private int $timeoutMs = -1; + + private bool $retryEnabled = true; + + private int $maxRetries = 3; + + private int $backoffMs = 1000; + + private string $triggerEventType = ''; + + /** @var list */ + private array $onComplete = []; + + /** @var list */ + private array $onError = []; + + private function __construct(private readonly string $name) {} + + public static function tcc(string $name): self + { + return new self($name); + } + + public function timeoutMs(int $milliseconds): self + { + $this->timeoutMs = $milliseconds; + + return $this; + } + + public function retry(bool $enabled, int $maxRetries = 3, int $backoffMs = 1000): self + { + $this->retryEnabled = $enabled; + $this->maxRetries = $maxRetries; + $this->backoffMs = $backoffMs; + + return $this; + } + + public function triggerEventType(string $eventType): self + { + $this->triggerEventType = $eventType; + + return $this; + } + + public function participant(string $id): TccParticipantBuilder + { + return new TccParticipantBuilder($this, $this->name, $id); + } + + /** + * @param Closure(ExecutionContext, mixed): void $callback + */ + public function onComplete(Closure $callback, int $priority = 0, bool $async = false): self + { + $this->onComplete[] = CallbackDefinition::closure($callback, $async, $priority); + + return $this; + } + + /** + * @param Closure(ExecutionContext, mixed): void $callback + * @param list $errorTypes + */ + public function onError(Closure $callback, bool $suppressError = false, array $errorTypes = [], int $priority = 0, bool $async = false): self + { + $this->onError[] = CallbackDefinition::closure($callback, $async, $priority, $errorTypes, $suppressError); + + return $this; + } + + /** @internal called by TccParticipantBuilder::add() */ + public function addParticipant(TccParticipantDefinition $participant): self + { + $this->participants[$participant->id] = $participant; + + return $this; + } + + public function build(): TccDefinition + { + return new TccDefinition($this->name, $this->participants, $this->timeoutMs, $this->retryEnabled, $this->maxRetries, $this->backoffMs, $this->triggerEventType, $this->onComplete, $this->onError); + } +} diff --git a/packages/orchestration/src/Tcc/Builder/TccParticipantBuilder.php b/packages/orchestration/src/Tcc/Builder/TccParticipantBuilder.php new file mode 100644 index 00000000..e9618bad --- /dev/null +++ b/packages/orchestration/src/Tcc/Builder/TccParticipantBuilder.php @@ -0,0 +1,107 @@ + */ + private array $phases = []; + + public function __construct( + private readonly TccBuilder $tcc, + private readonly string $tccName, + private readonly string $id, + ) {} + + public function order(int $order): self + { + $this->order = $order; + + return $this; + } + + public function timeoutMs(int $milliseconds): self + { + $this->timeoutMs = $milliseconds; + + return $this; + } + + public function optional(bool $optional = true): self + { + $this->optional = $optional; + + return $this; + } + + public function jitter(bool $enabled = true, float $factor = 0.5): self + { + $this->jitter = $enabled; + $this->jitterFactor = $factor; + + return $this; + } + + public function tryWith(Closure $phase, int $timeoutMs = -1, int $retry = -1, int $backoffMs = -1): self + { + return $this->phase('try', $phase, $timeoutMs, $retry, $backoffMs); + } + + public function confirmWith(Closure $phase, int $timeoutMs = -1, int $retry = -1, int $backoffMs = -1): self + { + return $this->phase('confirm', $phase, $timeoutMs, $retry, $backoffMs); + } + + public function cancelWith(Closure $phase, int $timeoutMs = -1, int $retry = -1, int $backoffMs = -1): self + { + return $this->phase('cancel', $phase, $timeoutMs, $retry, $backoffMs); + } + + public function event(string $topic, string $eventType, string $key = ''): self + { + $this->event = new StepEventDefinition($topic, $eventType, $key); + + return $this; + } + + public function add(): TccBuilder + { + if (! isset($this->phases['try'], $this->phases['confirm'], $this->phases['cancel'])) { + throw new TopologyValidationException("TCC [{$this->tccName}] participant [{$this->id}] needs try, confirm and cancel; it declares ".($this->phases === [] ? 'none' : implode(', ', array_keys($this->phases))).'.'); + } + + return $this->tcc->addParticipant(new TccParticipantDefinition( + $this->id, $this->order, $this->phases['try'], $this->phases['confirm'], $this->phases['cancel'], + $this->timeoutMs, $this->optional, $this->jitter, $this->jitterFactor, $this->event, + )); + } + + private function phase(string $name, Closure $phase, int $timeoutMs, int $retry, int $backoffMs): self + { + $this->phases[$name] = new TccPhaseDefinition(new ClosureInvocation($phase, "{$this->tccName}.{$this->id}:{$name}"), $timeoutMs, $retry, $backoffMs); + + return $this; + } +} diff --git a/packages/orchestration/src/Tcc/Composition/TccComposition.php b/packages/orchestration/src/Tcc/Composition/TccComposition.php new file mode 100644 index 00000000..d3eb4131 --- /dev/null +++ b/packages/orchestration/src/Tcc/Composition/TccComposition.php @@ -0,0 +1,40 @@ +> */ + public array $layers; + + /** + * @param array $members + * @param list $dataMappings + */ + public function __construct( + public string $name, + public array $members, + public array $dataMappings = [], + ) { + $dependencies = []; + foreach ($members as $alias => $member) { + $dependencies[(string) $alias] = $member->dependsOn; + } + $this->layers = TopologyBuilder::layers("TCC composition [{$name}]", $dependencies); + + foreach ($dataMappings as $mapping) { + foreach ([$mapping->sourceSaga, $mapping->targetSaga] as $alias) { + if (! isset($members[$alias])) { + throw new TopologyValidationException("TCC composition [{$name}] maps data ".($alias === $mapping->sourceSaga ? 'from' : 'to')." [{$alias}], which is not a member of it."); + } + } + } + } +} diff --git a/packages/orchestration/src/Tcc/Composition/TccCompositionBuilder.php b/packages/orchestration/src/Tcc/Composition/TccCompositionBuilder.php new file mode 100644 index 00000000..2ff90053 --- /dev/null +++ b/packages/orchestration/src/Tcc/Composition/TccCompositionBuilder.php @@ -0,0 +1,49 @@ + */ + private array $members = []; + + /** @var list */ + private array $dataMappings = []; + + private function __construct(private readonly string $name) {} + + public static function composition(string $name): self + { + return new self($name); + } + + public function tcc(string $alias): TccCompositionEntryBuilder + { + return new TccCompositionEntryBuilder($this, $alias); + } + + public function dataFlow(string $source, string $sourceField, string $target, string $targetField): self + { + $this->dataMappings[] = new DataMapping($source, $sourceField, $target, $targetField); + + return $this; + } + + /** @internal called by TccCompositionEntryBuilder::add() */ + public function addMember(TccCompositionEntry $member): self + { + $this->members[$member->alias] = $member; + + return $this; + } + + public function build(): TccComposition + { + return new TccComposition($this->name, $this->members, $this->dataMappings); + } +} diff --git a/packages/orchestration/src/Tcc/Composition/TccCompositionEntry.php b/packages/orchestration/src/Tcc/Composition/TccCompositionEntry.php new file mode 100644 index 00000000..70e76387 --- /dev/null +++ b/packages/orchestration/src/Tcc/Composition/TccCompositionEntry.php @@ -0,0 +1,22 @@ + $dependsOn + * @param array $staticInput + */ + public function __construct( + public string $alias, + public string $tccName, + public array $dependsOn = [], + public array $staticInput = [], + public bool $optional = false, + public string $condition = '', + ) {} +} diff --git a/packages/orchestration/src/Tcc/Composition/TccCompositionEntryBuilder.php b/packages/orchestration/src/Tcc/Composition/TccCompositionEntryBuilder.php new file mode 100644 index 00000000..ad014adb --- /dev/null +++ b/packages/orchestration/src/Tcc/Composition/TccCompositionEntryBuilder.php @@ -0,0 +1,68 @@ + */ + private array $dependsOn = []; + + /** @var array */ + private array $input = []; + + private bool $optional = false; + + private string $condition = ''; + + public function __construct( + private readonly TccCompositionBuilder $composition, + private readonly string $alias, + ) { + $this->tccName = $alias; + } + + public function tccName(string $tccName): self + { + $this->tccName = $tccName; + + return $this; + } + + public function dependsOn(string ...$aliases): self + { + $this->dependsOn = array_values([...$this->dependsOn, ...$aliases]); + + return $this; + } + + public function input(string $key, mixed $value): self + { + $this->input[$key] = $value; + + return $this; + } + + public function optional(bool $optional = true): self + { + $this->optional = $optional; + + return $this; + } + + public function condition(string $expression): self + { + $this->condition = $expression; + + return $this; + } + + public function add(): TccCompositionBuilder + { + return $this->composition->addMember(new TccCompositionEntry($this->alias, $this->tccName, $this->dependsOn, $this->input, $this->optional, $this->condition)); + } +} diff --git a/packages/orchestration/src/Tcc/Composition/TccCompositionResult.php b/packages/orchestration/src/Tcc/Composition/TccCompositionResult.php new file mode 100644 index 00000000..996fb78f --- /dev/null +++ b/packages/orchestration/src/Tcc/Composition/TccCompositionResult.php @@ -0,0 +1,56 @@ + $results + * @param list $compensated + * @param list $skipped + * @param array $compensationFailures alias => why that member is not fully cancelled + * @param array $optionalFailures alias => why that optional member did not confirm or start + */ + public function __construct( + public string $name, + public string $correlationId, + public bool $success, + public array $results, + public ?Throwable $error = null, + public array $compensated = [], + public array $skipped = [], + public array $compensationFailures = [], + public array $optionalFailures = [], + ) {} + + /** @return array */ + public function toArray(): array + { + $messages = static fn (Throwable $error): string => $error->getMessage(); + + return [ + 'name' => $this->name, + 'correlationId' => $this->correlationId, + 'success' => $this->success, + 'error' => $this->error?->getMessage(), + 'compensated' => $this->compensated, + 'compensationFailures' => array_map($messages, $this->compensationFailures), + 'skipped' => $this->skipped, + 'optionalFailures' => array_map($messages, $this->optionalFailures), + 'tccs' => array_map(static fn (TccResult $result): array => $result->toArray(), $this->results), + ]; + } +} diff --git a/packages/orchestration/src/Tcc/Composition/TccCompositor.php b/packages/orchestration/src/Tcc/Composition/TccCompositor.php new file mode 100644 index 00000000..4a8c64ef --- /dev/null +++ b/packages/orchestration/src/Tcc/Composition/TccCompositor.php @@ -0,0 +1,188 @@ + $input + * @param array $headers + */ + public function compose(TccComposition $composition, array $input = [], array $headers = []): TccCompositionResult + { + if (! $this->settings->compositionEnabled) { + throw new PatternDisabledException(ExecutionPattern::Tcc, 'firefly.orchestration.tcc.composition.enabled'); + } + + foreach ($composition->members as $member) { + if ($member->condition !== '') { + $this->conditions->lint($member->condition, "TCC composition [{$composition->name}] member [{$member->alias}]"); + } + } + + $correlationId = Str::uuid()->toString(); + $this->events->onCompositionStarted($composition->name, $correlationId); + + /** @var array $results */ + $results = []; + $completed = []; + $skipped = []; + $unavailable = []; + /** @var array $optionalFailures */ + $optionalFailures = []; + + foreach ($composition->layers as $layer) { + foreach ($layer as $alias) { + $member = $composition->members[$alias]; + $resultsByAlias = array_map(static fn (TccResult $result): array => $result->tryResults(), $results); + + try { + if (array_intersect($member->dependsOn, $unavailable) !== [] + || ($member->condition !== '' && ! $this->conditions->evaluate($member->condition, ['input' => $input, 'results' => $resultsByAlias, 'variables' => [], 'headers' => $headers]))) { + $skipped[] = $alias; + $unavailable[] = $alias; + + continue; + } + + // The composition's own headers win over a caller's header of the same name: a member must + // never claim to belong to a composition it is not part of. + $result = $this->engine->execute( + $member->tccName, + $this->dataFlow->resolveInputs($input, $member->staticInput, $composition->dataMappings, $alias, $resultsByAlias), + array_merge($headers, [SagaCompositor::COMPOSITION_HEADER => $correlationId, SagaCompositor::ALIAS_HEADER => $alias]), + ); + } catch (Throwable $error) { + if ($member->optional) { + $optionalFailures[$alias] = $error; + $unavailable[] = $alias; + + continue; + } + + return $this->abort($composition, $correlationId, $results, $completed, $skipped, $optionalFailures, $error); + } + + $results[$alias] = $result; + + if ($result->isConfirmed()) { + $completed[] = $alias; + + continue; + } + + $error = $result->error ?? new OrchestrationException("TCC composition member [{$alias}] did not confirm."); + if ($member->optional) { + $optionalFailures[$alias] = $error; + $unavailable[] = $alias; + + continue; + } + + return $this->abort($composition, $correlationId, $results, $completed, $skipped, $optionalFailures, $error); + } + } + + $this->events->onCompositionCompleted($composition->name, $correlationId, true); + + return new TccCompositionResult($composition->name, $correlationId, true, $results, null, [], $skipped, [], $optionalFailures); + } + + /** + * Cancels every confirmed member in reverse completion order and reports the composition as failed. A + * member that cannot be fully cancelled is recorded and reported, and the sweep moves on to the next one. + * + * @param array $results + * @param list $completed + * @param list $skipped + * @param array $optionalFailures + */ + private function abort(TccComposition $composition, string $correlationId, array $results, array $completed, array $skipped, array $optionalFailures, Throwable $error): TccCompositionResult + { + $compensated = []; + $failures = []; + foreach (array_reverse($completed) as $done) { + try { + $failure = $this->undo($done, $results[$done]); + } catch (Throwable $refused) { + $failure = $refused; + } + + if ($failure === null) { + $compensated[] = $done; + + continue; + } + + $failures[$done] = $failure; + $this->events->onStepCompensationFailed($composition->name, $correlationId, $done, $failure); + } + $this->events->onCompositionCompleted($composition->name, $correlationId, false); + + return new TccCompositionResult($composition->name, $correlationId, false, $results, $error, $compensated, $skipped, $failures, $optionalFailures); + } + + /** + * Cancels one confirmed member. Returns null when it ended CANCELED, and otherwise why it did not: + * TccEngine::compensate() handles a failed cancel itself and returns a FAILED result. + */ + private function undo(string $alias, TccResult $confirmed): ?Throwable + { + $undone = $this->engine->compensate($confirmed->correlationId); + if ($undone->status === ExecutionStatus::Canceled) { + return null; + } + + $left = []; + foreach ($undone->participants as $participantId => $outcome) { + if ($outcome->status === StepStatus::CancelFailed) { + $left[] = $participantId; + } + } + $reason = $undone->report->failureReason ?? 'not canceled'; + + return new CompensationException("TCC composition member [{$alias}] left [".implode(', ', $left)."] uncanceled: {$reason}"); + } +} diff --git a/packages/orchestration/src/Tcc/Engine/ParticipantOutcome.php b/packages/orchestration/src/Tcc/Engine/ParticipantOutcome.php new file mode 100644 index 00000000..cd8fb131 --- /dev/null +++ b/packages/orchestration/src/Tcc/Engine/ParticipantOutcome.php @@ -0,0 +1,45 @@ +status, [StepStatus::Done, StepStatus::Confirmed, StepStatus::ConfirmFailed, StepStatus::Canceled, StepStatus::CancelFailed], true); + } + + public function confirmed(): bool + { + return $this->status === StepStatus::Confirmed; + } + + public function canceled(): bool + { + return $this->status === StepStatus::Canceled; + } + + /** @return array */ + public function toArray(): array + { + return [ + 'id' => $this->id, 'status' => $this->status->value, 'tryResult' => StateSerializer::normalize($this->tryResult, 'tryResult'), + 'error' => $this->error?->getMessage(), 'attempts' => $this->attempts, + ]; + } +} diff --git a/packages/orchestration/src/Tcc/Engine/TccEngine.php b/packages/orchestration/src/Tcc/Engine/TccEngine.php new file mode 100644 index 00000000..05843f4b --- /dev/null +++ b/packages/orchestration/src/Tcc/Engine/TccEngine.php @@ -0,0 +1,193 @@ +registry; + } + + /** + * @param array|TccInputs $input + * @param array $headers + */ + public function execute(string|TccDefinition $tcc, array|TccInputs $input = [], array $headers = [], ?string $correlationId = null): TccResult + { + if (! $this->settings->enabled) { + throw new PatternDisabledException(ExecutionPattern::Tcc, 'firefly.orchestration.tcc.enabled'); + } + + $definition = is_string($tcc) ? $this->registry->get($tcc) : $tcc; + $inputs = $input instanceof TccInputs ? $input : TccInputs::of($input); + $context = ExecutionContext::start(ExecutionPattern::Tcc, $definition->name, $inputs->global(), $headers, $correlationId, $this->support->now()); + + // Checked twice, as SagaEngine does: up front, so a retried call is refused before admission, whose + // circuit breaker would count the refusal as a failed execution; and again under the lock, so two + // concurrent calls with one id cannot both pass. + $this->refuseExisting($context->correlationId); + + return $this->support->admission->admit( + $definition->name, + fn (): TccResult => $this->support->lock->synchronized($context->correlationId, function () use ($definition, $inputs, $context): TccResult { + $this->refuseExisting($context->correlationId); + + return $this->support->tracing->traceExecution( + $definition->name, + ExecutionPattern::Tcc, + $context->correlationId, + function () use ($definition, $inputs, $context): TccResult { + foreach ($inputs->explicit() as $participantId => $value) { + $context->putStepInput($participantId, $value); + } + $this->support->checkpoint($context, ExecutionStatus::Trying); + $this->support->publish($this->support->lifecycle, OrchestrationEvent::executionStarted($definition->name, $context->correlationId, ExecutionPattern::Tcc, $this->support->now())); + $this->support->events->onStart($definition->name, $context->correlationId, ExecutionPattern::Tcc); + + return $this->finish($definition, $context, $this->orchestrator->orchestrate($definition, $inputs, $context)); + }, + ); + }), + static fn (TccResult $result): bool => $result->success, + ); + } + + public function resume(string $correlationId): TccResult + { + return $this->support->lock->synchronized($correlationId, function () use ($correlationId): TccResult { + $state = $this->load($correlationId); + if ($state->isTerminal()) { + throw new InvalidExecutionStateException("TCC execution [{$correlationId}] is already {$state->status->value} and cannot be resumed."); + } + + $definition = $this->registry->get($state->executionName); + $context = ExecutionContext::fromState($state); + $inputs = TccInputs::of($state->input); + foreach ($state->stepInputs as $participantId => $value) { + $inputs = $inputs->forParticipant($participantId, $value); + } + + return $this->support->tracing->traceExecution($definition->name, ExecutionPattern::Tcc, $correlationId, fn (): TccResult => $this->finish($definition, $context, $this->orchestrator->orchestrate($definition, $inputs, $context))); + }); + } + + public function compensate(string $correlationId): TccResult + { + return $this->support->lock->synchronized($correlationId, function () use ($correlationId): TccResult { + $state = $this->load($correlationId); + if ($state->isTerminal() && $state->status !== ExecutionStatus::Confirmed) { + throw new InvalidExecutionStateException("TCC execution [{$correlationId}] is already {$state->status->value} and has nothing left to cancel."); + } + + $definition = $this->registry->get($state->executionName); + $context = ExecutionContext::fromState($state); + + return $this->finish($definition, $context, $this->orchestrator->cancelAll($definition, $context), onRequest: true); + }); + } + + /** + * @param bool $onRequest compensate(): no dead letter and no callbacks, the cancel was asked for + */ + private function finish(TccDefinition $tcc, ExecutionContext $context, TccRunOutcome $outcome, bool $onRequest = false): TccResult + { + $now = $this->support->now(); + $error = $outcome->error; + $suppressed = ! $onRequest && $outcome->status !== ExecutionStatus::Confirmed && $error !== null && $this->support->callbacks->suppresses($tcc->onError, $error); + $reason = match (true) { + $outcome->status === ExecutionStatus::Confirmed => null, + $error !== null => $error->getMessage(), + $outcome->status === ExecutionStatus::Canceled => $onRequest ? 'canceled on request' : 'canceled', + default => self::cancelFailures($context), + }; + $report = ExecutionReportBuilder::fromContext($context, $outcome->status, $reason, $now, $tcc->participantIds()); + $result = TccResult::from($tcc, $context, $outcome, $outcome->status === ExecutionStatus::Confirmed || $suppressed, $report, $now); + + $this->support->checkpoint($context, $outcome->status, $reason, $report); + if (! $onRequest && $error !== null) { + $this->support->deadLetter($context, $outcome->failedParticipantId, $outcome->status, $error); + } + $this->support->publish($this->support->lifecycle, OrchestrationEvent::executionCompleted($tcc->name, $context->correlationId, ExecutionPattern::Tcc, $outcome->status, $now)); + $this->support->events->onCompleted($tcc->name, $context->correlationId, ExecutionPattern::Tcc, $outcome->status === ExecutionStatus::Confirmed, $this->support->elapsedMs($context)); + + if ($onRequest) { + return $result; + } + + if ($outcome->status === ExecutionStatus::Confirmed) { + $this->support->callbacks->invoke($tcc->onComplete, $context, $result); + } else { + $this->support->callbacks->invoke($tcc->onError, $context, $result, $error); + } + + return $result; + } + + /** Why a cancel sweep that raised no error of its own still ended FAILED: the cancels that failed. */ + private static function cancelFailures(ExecutionContext $context): string + { + $failures = []; + foreach ($context->compensationErrors() as $participantId => $failure) { + $failures[] = "[{$participantId}] {$failure->getMessage()}"; + } + + return $failures === [] ? 'canceled' : 'cancel failed: '.implode('; ', $failures); + } + + /** + * A caller-supplied correlation id that is already persisted belongs to an execution that exists: starting + * it again would overwrite its row and try every participant a second time over reservations it already + * holds. + */ + private function refuseExisting(string $correlationId): void + { + $state = $this->support->persistence->findById($correlationId); + if ($state !== null) { + throw new InvalidExecutionStateException("Execution [{$correlationId}] already exists ({$state->pattern->value} {$state->executionName}, {$state->status->value}); use resume() or compensate() instead of starting it again."); + } + } + + private function load(string $correlationId): ExecutionState + { + $state = $this->support->persistence->findById($correlationId); + if ($state === null || $state->pattern !== ExecutionPattern::Tcc) { + throw new ExecutionNotFoundException($correlationId); + } + + return $state; + } +} diff --git a/packages/orchestration/src/Tcc/Engine/TccExecutionOrchestrator.php b/packages/orchestration/src/Tcc/Engine/TccExecutionOrchestrator.php new file mode 100644 index 00000000..37d42888 --- /dev/null +++ b/packages/orchestration/src/Tcc/Engine/TccExecutionOrchestrator.php @@ -0,0 +1,243 @@ +currentPhase() ?? TccPhase::Try; + + if ($phase === TccPhase::Try) { + $failure = $this->tryPhase($tcc, $inputs, $context); + if ($failure !== null) { + return $this->cancelPhase($tcc, $context, $failure[0], TccPhase::Try, $failure[1], includeConfirmed: false); + } + $phase = TccPhase::Confirm; + } + + if ($phase === TccPhase::Confirm) { + $failure = $this->confirmPhase($tcc, $context); + if ($failure !== null) { + return $this->cancelPhase($tcc, $context, $failure[0], TccPhase::Confirm, $failure[1], includeConfirmed: false); + } + + return new TccRunOutcome(ExecutionStatus::Confirmed); + } + + [$failedId, $failedPhase, $error] = $this->recordedFailure($tcc, $context); + + return $this->cancelPhase($tcc, $context, $failedId, $failedPhase, $error, includeConfirmed: false); + } + + public function cancelAll(TccDefinition $tcc, ExecutionContext $context): TccRunOutcome + { + return $this->cancelPhase($tcc, $context, null, null, null, includeConfirmed: true); + } + + /** @return array{0: string, 1: Throwable}|null the failed required participant and its error */ + private function tryPhase(TccDefinition $tcc, TccInputs $inputs, ExecutionContext $context): ?array + { + $context->setCurrentPhase(TccPhase::Try); + $this->support->events->onPhaseStarted($tcc->name, $context->correlationId, TccPhase::Try); + $this->support->checkpoint($context, ExecutionStatus::Trying); + $started = hrtime(true); + + foreach ($tcc->ordered() as $participant) { + $status = $context->stepStatus($participant->id); + if ($status === StepStatus::Done || ($participant->optional && $status->isFailure())) { + continue; + } + + $input = $inputs->inputFor($participant->id); + $context->putStepInput($participant->id, $input); + $context->setStepStatus($participant->id, StepStatus::Running); + $context->markStepStarted($participant->id, $this->support->now()); + $this->support->events->onParticipantStarted($tcc->name, $context->correlationId, $participant->id, TccPhase::Try); + $this->support->checkpoint($context, ExecutionStatus::Trying); + + $attemptStarted = hrtime(true); + try { + $result = $this->invoker->invoke($tcc->name, $participant->id, $participant->try->invocation, $this->options($participant, TccPhase::Try, $tcc), $context, $input, $input); + } catch (Throwable $error) { + $context->setStepStatus($participant->id, $error instanceof TimeoutException ? StepStatus::TimedOut : StepStatus::Failed); + $context->putStepError($participant->id, $error); + $this->support->events->onParticipantFailed($tcc->name, $context->correlationId, $participant->id, TccPhase::Try, $error); + $this->support->events->onStepFailed($tcc->name, $context->correlationId, $participant->id, $error, $context->attempts($participant->id)); + $this->support->publish($this->support->lifecycle, OrchestrationEvent::stepFailed($tcc->name, $context->correlationId, ExecutionPattern::Tcc, $participant->id, $error, $this->support->now())); + $this->support->checkpoint($context, ExecutionStatus::Trying); + + if ($participant->optional) { + continue; + } + + $this->support->events->onPhaseFailed($tcc->name, $context->correlationId, TccPhase::Try, $error); + + return [$participant->id, $error]; + } + + $latency = self::since($attemptStarted); + $context->putTryResult($participant->id, $result); + $context->putResult($participant->id, $result); + $context->setLatencyMs($participant->id, $latency); + $context->setStepStatus($participant->id, StepStatus::Done); + $context->markCompleted($participant->id); + $this->support->events->onParticipantSuccess($tcc->name, $context->correlationId, $participant->id, TccPhase::Try); + $this->support->events->onStepSuccess($tcc->name, $context->correlationId, $participant->id, $context->attempts($participant->id), $latency); + $this->support->checkpoint($context, ExecutionStatus::Trying); + } + + $this->support->events->onPhaseCompleted($tcc->name, $context->correlationId, TccPhase::Try, self::since($started)); + + return null; + } + + /** @return array{0: string, 1: Throwable}|null */ + private function confirmPhase(TccDefinition $tcc, ExecutionContext $context): ?array + { + $context->setCurrentPhase(TccPhase::Confirm); + $this->support->events->onPhaseStarted($tcc->name, $context->correlationId, TccPhase::Confirm); + $this->support->checkpoint($context, ExecutionStatus::Confirming); + $started = hrtime(true); + + foreach ($tcc->ordered() as $participant) { + if ($context->stepStatus($participant->id) !== StepStatus::Done) { + continue; + } + + $this->support->events->onParticipantStarted($tcc->name, $context->correlationId, $participant->id, TccPhase::Confirm); + try { + $result = $this->invoker->invoke($tcc->name, $participant->id.':confirm', $participant->confirm->invocation, $this->options($participant, TccPhase::Confirm, $tcc), $context, $context->stepInput($participant->id), $context->tryResult($participant->id)); + } catch (Throwable $error) { + $context->setStepStatus($participant->id, StepStatus::ConfirmFailed); + $context->putStepError($participant->id, $error); + $this->support->events->onParticipantFailed($tcc->name, $context->correlationId, $participant->id, TccPhase::Confirm, $error); + $this->support->events->onPhaseFailed($tcc->name, $context->correlationId, TccPhase::Confirm, $error); + $this->support->publish($this->support->lifecycle, OrchestrationEvent::stepFailed($tcc->name, $context->correlationId, ExecutionPattern::Tcc, $participant->id, $error, $this->support->now())); + $this->support->checkpoint($context, ExecutionStatus::Confirming); + + return [$participant->id, $error]; + } + + $context->setStepStatus($participant->id, StepStatus::Confirmed); + $this->support->events->onParticipantSuccess($tcc->name, $context->correlationId, $participant->id, TccPhase::Confirm); + + $confirmed = OrchestrationEvent::stepCompleted($tcc->name, $context->correlationId, ExecutionPattern::Tcc, $participant->id, $result, $this->support->now()); + if ($participant->event !== null) { + $this->support->publish($this->support->stepEvents, $confirmed->withTopic($participant->event->topic)->withType($participant->event->type)->withKey(SagaExecutionOrchestrator::eventKey($participant->event->key, $context->stepInput($participant->id)))); + } + $this->support->publish($this->support->lifecycle, $confirmed); + $this->support->checkpoint($context, ExecutionStatus::Confirming); + } + + $this->support->events->onPhaseCompleted($tcc->name, $context->correlationId, TccPhase::Confirm, self::since($started)); + + return null; + } + + private function cancelPhase(TccDefinition $tcc, ExecutionContext $context, ?string $failedId, ?TccPhase $failedPhase, ?Throwable $error, bool $includeConfirmed): TccRunOutcome + { + $context->setCurrentPhase(TccPhase::Cancel); + $this->support->events->onPhaseStarted($tcc->name, $context->correlationId, TccPhase::Cancel); + $this->support->checkpoint($context, ExecutionStatus::Canceling); + $started = hrtime(true); + $cancelFailed = false; + $cancelable = $includeConfirmed + ? [StepStatus::Done, StepStatus::Confirmed, StepStatus::ConfirmFailed, StepStatus::CancelFailed] + : [StepStatus::Done, StepStatus::ConfirmFailed, StepStatus::CancelFailed]; + + foreach (array_reverse($tcc->ordered()) as $participant) { + if (! in_array($context->stepStatus($participant->id), $cancelable, true)) { + continue; + } + + $this->support->events->onParticipantStarted($tcc->name, $context->correlationId, $participant->id, TccPhase::Cancel); + try { + $this->invoker->invoke($tcc->name, $participant->id.':cancel', $participant->cancel->invocation, $this->options($participant, TccPhase::Cancel, $tcc), $context, $context->stepInput($participant->id), $context->tryResult($participant->id)); + $context->setStepStatus($participant->id, StepStatus::Canceled); + $this->support->events->onParticipantSuccess($tcc->name, $context->correlationId, $participant->id, TccPhase::Cancel); + } catch (Throwable $cancelError) { + $cancelFailed = true; + $context->setStepStatus($participant->id, StepStatus::CancelFailed); + $context->putCompensationError($participant->id, $cancelError); + $this->support->events->onParticipantFailed($tcc->name, $context->correlationId, $participant->id, TccPhase::Cancel, $cancelError); + } + $this->support->checkpoint($context, ExecutionStatus::Canceling); + } + + if ($cancelFailed) { + $this->support->events->onPhaseFailed($tcc->name, $context->correlationId, TccPhase::Cancel, null); + } else { + $this->support->events->onPhaseCompleted($tcc->name, $context->correlationId, TccPhase::Cancel, self::since($started)); + } + + $status = $cancelFailed || $failedPhase === TccPhase::Confirm ? ExecutionStatus::Failed : ExecutionStatus::Canceled; + + return new TccRunOutcome($status, $failedId, $failedPhase, $error); + } + + /** @return array{0: ?string, 1: ?TccPhase, 2: ?Throwable} the failure a resumed CANCELING transaction was cancelling for */ + private function recordedFailure(TccDefinition $tcc, ExecutionContext $context): array + { + foreach ($tcc->ordered() as $participant) { + $status = $context->stepStatus($participant->id); + if (in_array($status, [StepStatus::Failed, StepStatus::TimedOut, StepStatus::ConfirmFailed], true)) { + return [ + $participant->id, + $status === StepStatus::ConfirmFailed ? TccPhase::Confirm : TccPhase::Try, + $context->stepError($participant->id) ?? new OrchestrationException("TCC [{$tcc->name}] participant [{$participant->id}] failed."), + ]; + } + } + + return [null, null, null]; + } + + private function options(TccParticipantDefinition $participant, TccPhase $phase, TccDefinition $tcc): StepOptions + { + return $participant->options($phase, $tcc, $this->settings->defaultTimeout); + } + + private static function since(int|float $started): int + { + return (int) round((hrtime(true) - $started) / 1_000_000); + } +} diff --git a/packages/orchestration/src/Tcc/Engine/TccInputs.php b/packages/orchestration/src/Tcc/Engine/TccInputs.php new file mode 100644 index 00000000..c926e3a8 --- /dev/null +++ b/packages/orchestration/src/Tcc/Engine/TccInputs.php @@ -0,0 +1,48 @@ + $global + * @param array $values + */ + private function __construct( + private array $global, + private array $values, + ) {} + + /** + * @param array $global + */ + public static function of(array $global = []): self + { + return new self($global, []); + } + + public function forParticipant(string $participantId, mixed $input): self + { + return new self($this->global, [...$this->values, $participantId => $input]); + } + + public function inputFor(string $participantId): mixed + { + return array_key_exists($participantId, $this->values) ? $this->values[$participantId] : $this->global; + } + + /** @return array */ + public function global(): array + { + return $this->global; + } + + /** @return array */ + public function explicit(): array + { + return $this->values; + } +} diff --git a/packages/orchestration/src/Tcc/Engine/TccResult.php b/packages/orchestration/src/Tcc/Engine/TccResult.php new file mode 100644 index 00000000..7062694e --- /dev/null +++ b/packages/orchestration/src/Tcc/Engine/TccResult.php @@ -0,0 +1,105 @@ + $participants + */ + public function __construct( + public string $tccName, + public string $correlationId, + public ExecutionStatus $status, + public bool $success, + public array $participants, + public ?string $failedParticipantId, + public ?TccPhase $failedPhase, + public ?Throwable $error, + public ?ExecutionReport $report, + public DateTimeImmutable $startedAt, + public DateTimeImmutable $completedAt, + ) {} + + public static function from(TccDefinition $tcc, ExecutionContext $context, TccRunOutcome $outcome, bool $success, ?ExecutionReport $report, DateTimeImmutable $now): self + { + $participants = []; + foreach ($tcc->participantIds() as $id) { + $participants[$id] = new ParticipantOutcome($id, $context->stepStatus($id), $context->tryResult($id), $context->stepError($id), $context->attempts($id)); + } + + return new self($tcc->name, $context->correlationId, $outcome->status, $success, $participants, $outcome->failedParticipantId, $outcome->failedPhase, $outcome->error, $report, $context->startedAt(), $now); + } + + public function isConfirmed(): bool + { + return $this->status === ExecutionStatus::Confirmed; + } + + public function isCanceled(): bool + { + return $this->status === ExecutionStatus::Canceled; + } + + public function isFailed(): bool + { + return $this->status === ExecutionStatus::Failed; + } + + public function isSuccess(): bool + { + return $this->success; + } + + public function tryResultOf(string $participantId): mixed + { + return ($this->participants[$participantId] ?? null)?->tryResult; + } + + /** @return array */ + public function tryResults(): array + { + $results = []; + foreach ($this->participants as $id => $participant) { + if ($participant->tryResult !== null) { + $results[$id] = $participant->tryResult; + } + } + + return $results; + } + + /** @return array */ + public function toArray(): array + { + return [ + 'tccName' => $this->tccName, + 'correlationId' => $this->correlationId, + 'status' => $this->status->value, + 'success' => $this->success, + 'failedParticipantId' => $this->failedParticipantId, + 'failedPhase' => $this->failedPhase?->value, + 'error' => $this->error?->getMessage(), + 'startedAt' => ExecutionState::instant($this->startedAt), + 'completedAt' => ExecutionState::instant($this->completedAt), + 'participants' => array_map(static fn (ParticipantOutcome $participant): array => $participant->toArray(), $this->participants), + 'report' => $this->report?->toArray(), + ]; + } +} diff --git a/packages/orchestration/src/Tcc/Engine/TccRunOutcome.php b/packages/orchestration/src/Tcc/Engine/TccRunOutcome.php new file mode 100644 index 00000000..eac90049 --- /dev/null +++ b/packages/orchestration/src/Tcc/Engine/TccRunOutcome.php @@ -0,0 +1,20 @@ + $participants + * @param list $onComplete + * @param list $onError + */ + public function __construct( + public string $name, + public array $participants, + public int $timeoutMs = -1, + public bool $retryEnabled = true, + public int $maxRetries = 3, + public int $backoffMs = 1000, + public string $triggerEventType = '', + public array $onComplete = [], + public array $onError = [], + ) { + if ($participants === []) { + throw new TopologyValidationException("TCC [{$name}] declares no participants."); + } + } + + /** @return list */ + public function ordered(): array + { + $participants = array_values($this->participants); + usort($participants, static fn (TccParticipantDefinition $a, TccParticipantDefinition $b): int => $a->order <=> $b->order); + + return $participants; + } + + public function participant(string $id): TccParticipantDefinition + { + return $this->participants[$id] ?? throw new DefinitionNotFoundException("TCC [{$this->name}] has no participant [{$id}]."); + } + + /** @return list */ + public function participantIds(): array + { + return array_map(static fn (TccParticipantDefinition $participant): string => $participant->id, $this->ordered()); + } + + /** @return array */ + public function toArray(): array + { + return [ + 'name' => $this->name, 'timeoutMs' => $this->timeoutMs, 'retryEnabled' => $this->retryEnabled, + 'maxRetries' => $this->maxRetries, 'backoffMs' => $this->backoffMs, 'triggerEventType' => $this->triggerEventType, + 'participants' => array_values(array_map(static fn (TccParticipantDefinition $participant): array => $participant->toArray(), $this->participants)), + 'onComplete' => array_map(static fn (CallbackDefinition $callback): array => $callback->toArray(), $this->onComplete), + 'onError' => array_map(static fn (CallbackDefinition $callback): array => $callback->toArray(), $this->onError), + ]; + } + + /** @param array $row */ + public static function fromArray(array $row): self + { + $participants = []; + foreach (Rows::rows($row, 'participants') as $participantRow) { + $participant = TccParticipantDefinition::fromArray($participantRow); + $participants[$participant->id] = $participant; + } + + return new self( + name: Rows::string($row, 'name'), + participants: $participants, + timeoutMs: Rows::int($row, 'timeoutMs', -1), + retryEnabled: Rows::bool($row, 'retryEnabled', true), + maxRetries: Rows::int($row, 'maxRetries', 3), + backoffMs: Rows::int($row, 'backoffMs', 1000), + triggerEventType: Rows::string($row, 'triggerEventType'), + onComplete: array_map(static fn (array $callback): CallbackDefinition => CallbackDefinition::fromArray($callback), Rows::rows($row, 'onComplete')), + onError: array_map(static fn (array $callback): CallbackDefinition => CallbackDefinition::fromArray($callback), Rows::rows($row, 'onError')), + ); + } +} diff --git a/packages/orchestration/src/Tcc/Registry/TccParticipantDefinition.php b/packages/orchestration/src/Tcc/Registry/TccParticipantDefinition.php new file mode 100644 index 00000000..8166cb63 --- /dev/null +++ b/packages/orchestration/src/Tcc/Registry/TccParticipantDefinition.php @@ -0,0 +1,85 @@ + $this->try, + TccPhase::Confirm => $this->confirm, + TccPhase::Cancel => $this->cancel, + }; + } + + public function options(TccPhase $phase, TccDefinition $tcc, float $defaultTimeoutSeconds): StepOptions + { + $definition = $this->phase($phase); + $timeout = match (true) { + $definition->timeoutMs >= 0 => $definition->timeoutMs, + $this->timeoutMs >= 0 => $this->timeoutMs, + $tcc->timeoutMs >= 0 => $tcc->timeoutMs, + default => (int) round($defaultTimeoutSeconds * 1000), + }; + $retry = $definition->retry >= 0 ? $definition->retry : ($tcc->retryEnabled ? $tcc->maxRetries : 0); + $backoff = $definition->backoffMs >= 0 ? $definition->backoffMs : $tcc->backoffMs; + + return StepOptions::of($retry, $backoff, $timeout, $this->jitter, $this->jitterFactor); + } + + /** @return array */ + public function toArray(): array + { + return [ + 'id' => $this->id, 'order' => $this->order, 'try' => $this->try->toArray(), 'confirm' => $this->confirm->toArray(), + 'cancel' => $this->cancel->toArray(), 'timeoutMs' => $this->timeoutMs, 'optional' => $this->optional, + 'jitter' => $this->jitter, 'jitterFactor' => $this->jitterFactor, 'event' => $this->event?->toArray(), + ]; + } + + /** @param array $row */ + public static function fromArray(array $row): self + { + $event = Rows::nullableMap($row, 'event'); + + return new self( + id: Rows::string($row, 'id'), + order: Rows::int($row, 'order'), + try: TccPhaseDefinition::fromArray(Rows::map($row, 'try')), + confirm: TccPhaseDefinition::fromArray(Rows::map($row, 'confirm')), + cancel: TccPhaseDefinition::fromArray(Rows::map($row, 'cancel')), + timeoutMs: Rows::int($row, 'timeoutMs', -1), + optional: Rows::bool($row, 'optional'), + jitter: Rows::bool($row, 'jitter'), + jitterFactor: Rows::float($row, 'jitterFactor'), + event: $event === null ? null : StepEventDefinition::fromArray($event), + ); + } +} diff --git a/packages/orchestration/src/Tcc/Registry/TccPhaseDefinition.php b/packages/orchestration/src/Tcc/Registry/TccPhaseDefinition.php new file mode 100644 index 00000000..d93ce55a --- /dev/null +++ b/packages/orchestration/src/Tcc/Registry/TccPhaseDefinition.php @@ -0,0 +1,32 @@ + */ + public function toArray(): array + { + return ['invocation' => $this->invocation->toArray(), 'timeoutMs' => $this->timeoutMs, 'retry' => $this->retry, 'backoffMs' => $this->backoffMs]; + } + + /** @param array $row */ + public static function fromArray(array $row): self + { + return new self(MethodInvocation::fromArray(Rows::map($row, 'invocation')), Rows::int($row, 'timeoutMs', -1), Rows::int($row, 'retry', -1), Rows::int($row, 'backoffMs', -1)); + } +} diff --git a/packages/orchestration/src/Tcc/Registry/TccRegistry.php b/packages/orchestration/src/Tcc/Registry/TccRegistry.php new file mode 100644 index 00000000..a4c0f585 --- /dev/null +++ b/packages/orchestration/src/Tcc/Registry/TccRegistry.php @@ -0,0 +1,46 @@ + */ + private array $tccs = []; + + public function register(TccDefinition $tcc): void + { + if (isset($this->tccs[$tcc->name])) { + throw new TopologyValidationException("TCC [{$tcc->name}] is defined twice."); + } + + $this->tccs[$tcc->name] = $tcc; + } + + public function get(string $name): TccDefinition + { + return $this->tccs[$name] ?? throw new DefinitionNotFoundException("No TCC [{$name}] is registered."); + } + + public function has(string $name): bool + { + return isset($this->tccs[$name]); + } + + /** @return array */ + public function all(): array + { + return $this->tccs; + } + + /** @return list */ + public function names(): array + { + return array_map('strval', array_keys($this->tccs)); + } +} diff --git a/packages/orchestration/tests/Support/TestEngines.php b/packages/orchestration/tests/Support/TestEngines.php index 940bb95e..7ae02e42 100644 --- a/packages/orchestration/tests/Support/TestEngines.php +++ b/packages/orchestration/tests/Support/TestEngines.php @@ -29,6 +29,10 @@ use Firefly\Orchestration\Saga\Engine\SagaEngine; use Firefly\Orchestration\Saga\Engine\SagaExecutionOrchestrator; use Firefly\Orchestration\Saga\Registry\SagaRegistry; +use Firefly\Orchestration\Tcc\Composition\TccCompositor; +use Firefly\Orchestration\Tcc\Engine\TccEngine; +use Firefly\Orchestration\Tcc\Engine\TccExecutionOrchestrator; +use Firefly\Orchestration\Tcc\Registry\TccRegistry; use Firefly\Resilience\CircuitBreaker; use Firefly\Resilience\ResilienceRegistry; use Firefly\Resilience\Store\InMemoryResilienceStore; @@ -75,6 +79,12 @@ final class TestEngines public readonly SagaCompositor $compositor; + public readonly TccRegistry $tccRegistry; + + public readonly TccEngine $tccs; + + public readonly TccCompositor $tccCompositor; + /** * @param array $config */ @@ -129,6 +139,10 @@ private function __construct(array $config, ?ExecutionPersistenceProvider $persi ); $this->sagas = new SagaEngine($this->sagaRegistry, new SagaExecutionOrchestrator($this->invoker, $this->support), $this->compensator, $this->support, $this->settings->saga); $this->compositor = new SagaCompositor($this->sagas, $this->events, $this->conditions, new CompositionDataFlowManager); + + $this->tccRegistry = new TccRegistry; + $this->tccs = new TccEngine($this->tccRegistry, new TccExecutionOrchestrator($this->invoker, $this->support, $this->settings->tcc), $this->support, $this->settings->tcc); + $this->tccCompositor = new TccCompositor($this->tccs, $this->events, $this->conditions, new CompositionDataFlowManager, $this->settings->tcc); } /** diff --git a/packages/orchestration/tests/Tcc/TccCompositorTest.php b/packages/orchestration/tests/Tcc/TccCompositorTest.php new file mode 100644 index 00000000..a19d8b21 --- /dev/null +++ b/packages/orchestration/tests/Tcc/TccCompositorTest.php @@ -0,0 +1,140 @@ + $log */ +function orchestrationTccPair(TestEngines $engines, array &$log): void +{ + foreach (['hold-funds' => false, 'book-seat' => true] as $name => $failsWhenAsked) { + $engines->tccRegistry->register(TccBuilder::tcc($name)->retry(false) + ->participant('main') + ->tryWith(static function (array $input) use ($name, $failsWhenAsked, &$log): array { + $log[] = "try {$name}"; + if ($failsWhenAsked && ($input['fail'] ?? false) === true) { + throw new RuntimeException("{$name} unavailable"); + } + + return ['ref' => "{$name}-ref", 'holdRef' => is_string($input['holdRef'] ?? null) ? $input['holdRef'] : null]; + }) + ->confirmWith(static function () use ($name, &$log): void { + $log[] = "confirm {$name}"; + }) + ->cancelWith(static function () use ($name, &$log): void { + $log[] = "cancel {$name}"; + }) + ->add()->build()); + } +} + +it('confirms members in dependency order, flowing try results forward', function (): void { + $engines = TestEngines::create(); + $log = []; + orchestrationTccPair($engines, $log); + $composition = TccCompositionBuilder::composition('trip') + ->tcc('funds')->tccName('hold-funds')->add() + ->tcc('seat')->tccName('book-seat')->dependsOn('funds')->add() + ->dataFlow('funds', 'ref', 'seat', 'holdRef') + ->build(); + + $result = $engines->tccCompositor->compose($composition); + + expect($result->success)->toBeTrue() + ->and($result->results['seat']->tryResultOf('main'))->toBe(['ref' => 'book-seat-ref', 'holdRef' => 'hold-funds-ref']) + ->and($log)->toBe(['try hold-funds', 'confirm hold-funds', 'try book-seat', 'confirm book-seat']); +}); + +it('cancels the confirmed members when a later member fails', function (): void { + $engines = TestEngines::create(); + $log = []; + orchestrationTccPair($engines, $log); + $composition = TccCompositionBuilder::composition('trip') + ->tcc('funds')->tccName('hold-funds')->add() + ->tcc('seat')->tccName('book-seat')->dependsOn('funds')->add() + ->build(); + + $result = $engines->tccCompositor->compose($composition, ['fail' => true]); + + expect($result->success)->toBeFalse() + ->and($result->compensated)->toBe(['funds']) + ->and($log)->toBe(['try hold-funds', 'confirm hold-funds', 'try book-seat', 'cancel hold-funds']) + ->and($engines->persistence->findById($result->results['funds']->correlationId)?->status)->toBe(ExecutionStatus::Canceled); +}); + +it('refuses compositions while tcc.composition is disabled', function (): void { + $engines = TestEngines::create(['tcc' => ['composition' => ['enabled' => false]]]); + + $engines->tccCompositor->compose(TccCompositionBuilder::composition('x')->tcc('a')->tccName('a')->add()->build()); +})->throws(PatternDisabledException::class, 'firefly.orchestration.tcc.composition.enabled'); + +it('rolls back the confirmed members when a required member cannot start at all', function (): void { + $engines = TestEngines::create(); + $log = []; + orchestrationTccPair($engines, $log); + $composition = TccCompositionBuilder::composition('trip') + ->tcc('funds')->tccName('hold-funds')->add() + ->tcc('hotel')->tccName('book-hotel')->dependsOn('funds')->add() + ->build(); + + $result = $engines->tccCompositor->compose($composition); + + expect($result->success)->toBeFalse() + ->and($result->error)->toBeInstanceOf(DefinitionNotFoundException::class) + ->and($result->compensated)->toBe(['funds']) + ->and($log)->toBe(['try hold-funds', 'confirm hold-funds', 'cancel hold-funds']); +}); + +it('tolerates an optional member that fails or cannot start, and skips its dependents', function (): void { + $engines = TestEngines::create(); + $log = []; + orchestrationTccPair($engines, $log); + $composition = TccCompositionBuilder::composition('trip') + ->tcc('funds')->tccName('hold-funds')->add() + ->tcc('seat')->tccName('book-seat')->optional()->add() + ->tcc('hotel')->tccName('book-hotel')->optional()->add() + ->tcc('meal')->tccName('hold-funds')->dependsOn('seat')->add() + ->build(); + + $result = $engines->tccCompositor->compose($composition, ['fail' => true]); + + expect($result->success)->toBeTrue() + ->and(array_keys($result->optionalFailures))->toBe(['seat', 'hotel']) + ->and($result->optionalFailures['seat']->getMessage())->toBe('book-seat unavailable') + ->and($result->skipped)->toBe(['meal']) + ->and($log)->toBe(['try hold-funds', 'confirm hold-funds', 'try book-seat']); +}); + +it('reports a confirmed member whose cancel failed instead of listing it as compensated', function (): void { + $engines = TestEngines::create(); + $log = []; + $engines->tccRegistry->register(TccBuilder::tcc('stubborn')->retry(false) + ->participant('main') + ->tryWith(static fn (): string => 'held') + ->confirmWith(static fn (): null => null) + ->cancelWith(static function (): void { + throw new RuntimeException('ledger offline'); + }) + ->add()->build()); + orchestrationTccPair($engines, $log); + $composition = TccCompositionBuilder::composition('trip') + ->tcc('stubborn')->add() + ->tcc('funds')->tccName('hold-funds')->dependsOn('stubborn')->add() + ->tcc('seat')->tccName('book-seat')->dependsOn('funds')->add() + ->build(); + + $result = $engines->tccCompositor->compose($composition, ['fail' => true]); + + expect($result->success)->toBeFalse() + ->and($result->compensated)->toBe(['funds']) + ->and(array_keys($result->compensationFailures))->toBe(['stubborn']) + ->and($result->compensationFailures['stubborn']->getMessage())->toBe('TCC composition member [stubborn] left [main] uncanceled: cancel failed: [main] ledger offline') + ->and($engines->persistence->findById($result->results['stubborn']->correlationId)?->status)->toBe(ExecutionStatus::Failed) + ->and($engines->events->hooks())->toContain('onStepCompensationFailed') + ->and($result->toArray()['compensationFailures'])->toBe(['stubborn' => $result->compensationFailures['stubborn']->getMessage()]); +}); diff --git a/packages/orchestration/tests/Tcc/TccDefinitionTest.php b/packages/orchestration/tests/Tcc/TccDefinitionTest.php new file mode 100644 index 00000000..c2343ac4 --- /dev/null +++ b/packages/orchestration/tests/Tcc/TccDefinitionTest.php @@ -0,0 +1,36 @@ + new TccPhaseDefinition(new MethodInvocation('App\\Debit', $method, [new ParameterBinding('in', ParameterBinding::IMPLICIT)]), $timeout, $retry); + $participant = static fn (string $id, int $order, int $timeout = -1): TccParticipantDefinition => new TccParticipantDefinition($id, $order, $phase('try', 200), $phase('confirm', -1, 5), $phase('cancel'), $timeout); + $tcc = new TccDefinition('transfer', ['credit' => $participant('credit', 2), 'debit' => $participant('debit', 1, 900), 'audit' => $participant('audit', 2)], timeoutMs: 1500, maxRetries: 2, backoffMs: 10); + + expect($tcc->participantIds())->toBe(['debit', 'credit', 'audit']) + ->and($tcc->participant('debit')->options(TccPhase::Try, $tcc, 30.0))->toEqual(new StepOptions(2, 10, 200, false, 0.0)) + ->and($tcc->participant('debit')->options(TccPhase::Confirm, $tcc, 30.0))->toEqual(new StepOptions(5, 10, 900, false, 0.0)) + ->and($tcc->participant('credit')->options(TccPhase::Cancel, $tcc, 30.0))->toEqual(new StepOptions(2, 10, 1500, false, 0.0)) + ->and(TccDefinition::fromArray($tcc->toArray()))->toEqual($tcc); +}); + +it('falls back to the configured default timeout and honours retryEnabled', function (): void { + $tcc = TccBuilder::tcc('t')->retry(false)->participant('p')->tryWith(static fn (): string => 'x')->confirmWith(static fn (): null => null)->cancelWith(static fn (): null => null)->add()->build(); + + expect($tcc->participant('p')->options(TccPhase::Try, $tcc, 30.0))->toEqual(new StepOptions(0, 1000, 30000, false, 0.0)); +}); + +it('refuses a TCC without participants and a participant without all three phases', function (): void { + expect(fn () => new TccDefinition('empty', []))->toThrow(TopologyValidationException::class, 'TCC [empty] declares no participants.') + ->and(fn () => TccBuilder::tcc('half')->participant('p')->tryWith(static fn (): string => 'x')->add())->toThrow(TopologyValidationException::class, 'TCC [half] participant [p] needs try, confirm and cancel'); +}); diff --git a/packages/orchestration/tests/Tcc/TccEngineTest.php b/packages/orchestration/tests/Tcc/TccEngineTest.php new file mode 100644 index 00000000..3fc89b6b --- /dev/null +++ b/packages/orchestration/tests/Tcc/TccEngineTest.php @@ -0,0 +1,178 @@ +:" that throw (`times` left, PHP_INT_MAX = always). + * + * @param list $log + * @param array $fail + */ +function orchestrationTransfer(array &$log, array $fail = [], bool $optionalCredit = false, int $confirmRetry = -1): TccDefinition +{ + $builder = TccBuilder::tcc('transfer')->retry(true, 0, 1); + foreach (['debit' => 1, 'credit' => 2] as $id => $order) { + $phase = static function (string $name) use ($id, &$log, &$fail): Closure { + return static function (mixed $implicit, ExecutionContext $ctx) use ($id, $name, &$log, &$fail): string { + $log[] = "{$name} {$id}".($name === 'try' ? '' : ' ('.(is_string($implicit) ? $implicit : '?').')'); + if (($fail["{$id}:{$name}"] ?? 0) > 0) { + $fail["{$id}:{$name}"]--; + + throw new RuntimeException("{$name} {$id} failed"); + } + + return "{$id}-{$name}"; + }; + }; + $builder = $builder->participant($id)->order($order)->optional($optionalCredit && $id === 'credit') + ->tryWith($phase('try')) + ->confirmWith($phase('confirm'), retry: $confirmRetry) + ->cancelWith($phase('cancel')) + ->event('ledger', "ledger.{$id}", 'account') + ->add(); + } + + return $builder->build(); +} + +it('tries every participant in order, then confirms them in order', function (): void { + $engines = TestEngines::create(); + $log = []; + + $result = $engines->tccs->execute(orchestrationTransfer($log), ['account' => 'acc-1'], correlationId: 't-1'); + + expect($result->isConfirmed())->toBeTrue() + ->and($result->status)->toBe(ExecutionStatus::Confirmed) + ->and($log)->toBe(['try debit', 'try credit', 'confirm debit (debit-try)', 'confirm credit (credit-try)']) + ->and($result->tryResultOf('credit'))->toBe('credit-try') + ->and($result->participants['debit']->status)->toBe(StepStatus::Confirmed) + ->and($engines->persistence->findById('t-1')?->status)->toBe(ExecutionStatus::Confirmed) + ->and(array_column($engines->eda->published, 'eventType'))->toBe(['ledger.debit', 'ledger.credit']) + ->and($engines->eda->published[0]['headers']['key'] ?? null)->toBe('acc-1') + ->and($engines->events->hooks())->toContain('onPhaseStarted', 'onPhaseCompleted', 'onParticipantSuccess'); +}); + +it('cancels the tried participants in reverse when a try fails', function (): void { + $engines = TestEngines::create(); + $log = []; + + $result = $engines->tccs->execute(orchestrationTransfer($log, ['credit:try' => PHP_INT_MAX]), correlationId: 't-2'); + + expect($result->isCanceled())->toBeTrue() + ->and($result->failedParticipantId)->toBe('credit') + ->and($result->failedPhase)->toBe(TccPhase::Try) + ->and($result->error?->getMessage())->toBe('try credit failed') + ->and($log)->toBe(['try debit', 'try credit', 'cancel debit (debit-try)']) + ->and($result->participants['debit']->status)->toBe(StepStatus::Canceled) + ->and($engines->deadLetterStore->findByCorrelationId('t-2')[0]->stepId ?? null)->toBe('credit'); +}); + +it('cancels what it could not confirm and ends FAILED when a confirm fails', function (): void { + $engines = TestEngines::create(); + $log = []; + + $result = $engines->tccs->execute(orchestrationTransfer($log, ['credit:confirm' => PHP_INT_MAX])); + + expect($result->isFailed())->toBeTrue() + ->and($result->failedPhase)->toBe(TccPhase::Confirm) + ->and($log)->toBe(['try debit', 'try credit', 'confirm debit (debit-try)', 'confirm credit (credit-try)', 'cancel credit (credit-try)']) + ->and($result->participants['debit']->status)->toBe(StepStatus::Confirmed) + ->and($result->participants['credit']->status)->toBe(StepStatus::Canceled); +}); + +it('lets an optional participant fail its try without cancelling the others', function (): void { + $engines = TestEngines::create(); + $log = []; + + $result = $engines->tccs->execute(orchestrationTransfer($log, ['credit:try' => PHP_INT_MAX], optionalCredit: true)); + + expect($result->isConfirmed())->toBeTrue() + ->and($log)->toBe(['try debit', 'try credit', 'confirm debit (debit-try)']) + ->and($result->participants['credit']->status)->toBe(StepStatus::Failed); +}); + +it('retries a phase with its own retry count', function (): void { + $engines = TestEngines::create(); + $log = []; + + $result = $engines->tccs->execute(orchestrationTransfer($log, ['credit:confirm' => 1], confirmRetry: 1)); + + expect($result->isConfirmed())->toBeTrue() + ->and($result->participants['credit']->attempts)->toBe(1); +}); + +it('refuses executions while TCC is disabled', function (): void { + $log = []; + TestEngines::create(['tcc' => ['enabled' => false]])->tccs->execute(orchestrationTransfer($log)); +})->throws(PatternDisabledException::class, 'firefly.orchestration.tcc.enabled'); + +it('refuses to start a TCC under a correlation id that already exists', function (): void { + $engines = TestEngines::create(); + $log = []; + $engines->tccs->execute(orchestrationTransfer($log), correlationId: 't-dup'); + + expect(fn () => $engines->tccs->execute(orchestrationTransfer($log), correlationId: 't-dup')) + ->toThrow(InvalidExecutionStateException::class, 'Execution [t-dup] already exists (TCC transfer, CONFIRMED)') + ->and($log)->toHaveCount(4); +}); + +it('calls OnTccComplete after a confirm and OnTccError after a cancel, and suppressError reports a success', function (): void { + $engines = TestEngines::create(); + $calls = []; + $tcc = static function (bool $fails, bool $suppress) use (&$calls): TccDefinition { + return TccBuilder::tcc($fails ? 'failing' : 'fine')->retry(false) + ->participant('p') + ->tryWith(static function () use ($fails): string { + if ($fails) { + throw new RuntimeException('no stock'); + } + + return 'held'; + }) + ->confirmWith(static fn (): null => null) + ->cancelWith(static fn (): null => null) + ->add() + ->onComplete(static function (ExecutionContext $ctx) use (&$calls): void { + $calls[] = "complete {$ctx->executionName}"; + }) + ->onError(static function (ExecutionContext $ctx) use (&$calls): void { + $calls[] = "error {$ctx->executionName}"; + }, suppressError: $suppress) + ->build(); + }; + + $confirmed = $engines->tccs->execute($tcc(false, false)); + $canceled = $engines->tccs->execute($tcc(true, false)); + $suppressed = TestEngines::create()->tccs->execute($tcc(true, true)); + + expect($confirmed->isSuccess())->toBeTrue() + ->and($canceled->isSuccess())->toBeFalse() + ->and($canceled->isCanceled())->toBeTrue() + ->and($suppressed->isCanceled())->toBeTrue() + ->and($suppressed->isSuccess())->toBeTrue() + ->and($calls)->toBe(['complete fine', 'error failing', 'error failing']); +}); + +it('ends FAILED and reports the participant whose cancel failed', function (): void { + $engines = TestEngines::create(); + $log = []; + + $result = $engines->tccs->execute(orchestrationTransfer($log, ['credit:try' => PHP_INT_MAX, 'debit:cancel' => PHP_INT_MAX]), correlationId: 't-3'); + + expect($result->isFailed())->toBeTrue() + ->and($result->failedParticipantId)->toBe('credit') + ->and($result->failedPhase)->toBe(TccPhase::Try) + ->and($result->participants['debit']->status)->toBe(StepStatus::CancelFailed) + ->and($engines->persistence->findById('t-3')?->compensationErrors)->toBe(['debit' => 'cancel debit failed']) + ->and($engines->events->hooks())->toContain('onPhaseFailed'); +}); diff --git a/packages/orchestration/tests/Tcc/TccResumeTest.php b/packages/orchestration/tests/Tcc/TccResumeTest.php new file mode 100644 index 00000000..b47ebd63 --- /dev/null +++ b/packages/orchestration/tests/Tcc/TccResumeTest.php @@ -0,0 +1,119 @@ +tccRegistry->register(TccBuilder::tcc('transfer') + ->participant('debit')->order(1)->tryWith($phase('try', 'debit'))->confirmWith($phase('confirm', 'debit'))->cancelWith($phase('cancel', 'debit'))->add() + ->participant('credit')->order(2)->tryWith($phase('try', 'credit'))->confirmWith($phase('confirm', 'credit'))->cancelWith($phase('cancel', 'credit'))->add() + ->build()); + $engines->persistence->save(ExecutionState::fromArray([ + 'correlationId' => 't-9', 'executionName' => 'transfer', 'pattern' => 'TCC', 'status' => 'CONFIRMING', + 'startedAt' => '2026-10-12T09:00:00.000000+00:00', 'updatedAt' => '2026-10-12T09:00:01.000000+00:00', + 'currentPhase' => 'CONFIRM', + 'stepStatuses' => ['debit' => 'CONFIRMED', 'credit' => 'DONE'], + 'tryResults' => ['debit' => 'debit-try', 'credit' => 'credit-try'], + 'stepResults' => ['debit' => 'debit-try', 'credit' => 'credit-try'], + 'completionOrder' => ['debit', 'credit'], + ])); + + $result = $engines->tccs->resume('t-9'); + + expect($result->isConfirmed())->toBeTrue() + ->and($log)->toBe(['confirm credit']) + ->and($engines->persistence->findById('t-9')?->status)->toBe(ExecutionStatus::Confirmed); +}); + +it('cancels every tried participant of a confirmed TCC on request', function (): void { + $engines = TestEngines::create(); + $log = []; + $phase = static function (string $name, string $id) use (&$log): Closure { + return static function () use ($name, $id, &$log): string { + $log[] = "{$name} {$id}"; + + return $id; + }; + }; + $engines->tccRegistry->register(TccBuilder::tcc('transfer') + ->participant('debit')->order(1)->tryWith($phase('try', 'debit'))->confirmWith($phase('confirm', 'debit'))->cancelWith($phase('cancel', 'debit'))->add() + ->participant('credit')->order(2)->tryWith($phase('try', 'credit'))->confirmWith($phase('confirm', 'credit'))->cancelWith($phase('cancel', 'credit'))->add() + ->build()); + $engines->tccs->execute('transfer', correlationId: 't-1'); + + $canceled = $engines->tccs->compensate('t-1'); + + expect($canceled->status)->toBe(ExecutionStatus::Canceled) + ->and(array_slice($log, 4))->toBe(['cancel credit', 'cancel debit']); +}); + +it('resumes a TCC interrupted in CANCELING, cancels what is left and reports the recorded failure', function (): void { + $engines = TestEngines::create(); + $log = []; + $phase = static function (string $name, string $id) use (&$log): Closure { + return static function () use ($name, $id, &$log): string { + $log[] = "{$name} {$id}"; + + return $id; + }; + }; + $engines->tccRegistry->register(TccBuilder::tcc('transfer') + ->participant('debit')->order(1)->tryWith($phase('try', 'debit'))->confirmWith($phase('confirm', 'debit'))->cancelWith($phase('cancel', 'debit'))->add() + ->participant('credit')->order(2)->tryWith($phase('try', 'credit'))->confirmWith($phase('confirm', 'credit'))->cancelWith($phase('cancel', 'credit'))->add() + ->build()); + $engines->persistence->save(ExecutionState::fromArray([ + 'correlationId' => 't-10', 'executionName' => 'transfer', 'pattern' => 'TCC', 'status' => 'CANCELING', + 'startedAt' => '2026-10-12T09:00:00.000000+00:00', 'updatedAt' => '2026-10-12T09:00:01.000000+00:00', + 'currentPhase' => 'CANCEL', + 'stepStatuses' => ['debit' => 'DONE', 'credit' => 'FAILED'], + 'stepErrors' => ['credit' => 'credit refused'], + 'tryResults' => ['debit' => 'debit'], + 'completionOrder' => ['debit'], + ])); + + $result = $engines->tccs->resume('t-10'); + + expect($result->isCanceled())->toBeTrue() + ->and($result->failedParticipantId)->toBe('credit') + ->and($result->failedPhase)->toBe(TccPhase::Try) + ->and($result->error?->getMessage())->toBe('credit refused') + ->and($log)->toBe(['cancel debit']) + ->and($engines->deadLetterStore->findByCorrelationId('t-10'))->toHaveCount(1); +}); + +it('neither dead-letters nor calls OnTccError when a TCC is cancelled on request', function (): void { + $engines = TestEngines::create(); + $errors = 0; + $engines->tccRegistry->register(TccBuilder::tcc('hold') + ->participant('p')->tryWith(static fn (): string => 'held')->confirmWith(static fn (): null => null)->cancelWith(static fn (): null => null)->add() + ->onError(static function () use (&$errors): void { + $errors++; + }) + ->build()); + $engines->tccs->execute('hold', correlationId: 't-11'); + + $canceled = $engines->tccs->compensate('t-11'); + + expect($canceled->isCanceled())->toBeTrue() + ->and($canceled->report?->failureReason)->toBe('canceled on request') + ->and($errors)->toBe(0) + ->and($engines->deadLetterStore->findByCorrelationId('t-11'))->toBe([]) + ->and(fn () => $engines->tccs->compensate('t-11'))->toThrow(InvalidExecutionStateException::class, 'already CANCELED'); +}); From 3576fbfd9019b2b79090a14e019545924409c300 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 20:14:41 -0700 Subject: [PATCH 21/58] fix(orchestration): checkpoint a required TCC failure under CANCEL so a resume never retries it or reports CONFIRMED --- .../Tcc/Engine/TccExecutionOrchestrator.php | 35 ++++- .../Support/RecordingExecutionPersistence.php | 95 ++++++++++++++ .../orchestration/tests/Tcc/TccResumeTest.php | 123 ++++++++++++++++++ 3 files changed, 248 insertions(+), 5 deletions(-) create mode 100644 packages/orchestration/tests/Support/RecordingExecutionPersistence.php diff --git a/packages/orchestration/src/Tcc/Engine/TccExecutionOrchestrator.php b/packages/orchestration/src/Tcc/Engine/TccExecutionOrchestrator.php index 37d42888..44c4af21 100644 --- a/packages/orchestration/src/Tcc/Engine/TccExecutionOrchestrator.php +++ b/packages/orchestration/src/Tcc/Engine/TccExecutionOrchestrator.php @@ -32,7 +32,9 @@ * * The phase is checkpointed as it starts and every participant result as it lands, so resume() picks up a * CONFIRMING or CANCELING transaction exactly where a crash left it — which is why confirm and cancel must be - * idempotent. + * idempotent. A required failure is checkpointed together with the switch to CANCEL, and a resumed state that + * still holds one goes straight to CANCEL, so a resume never retries an exhausted participant or reports a + * transaction with a CONFIRM_FAILED participant as CONFIRMED. */ final class TccExecutionOrchestrator { @@ -46,6 +48,13 @@ public function orchestrate(TccDefinition $tcc, TccInputs $inputs, ExecutionCont { $phase = $context->currentPhase() ?? TccPhase::Try; + // A state checkpointed before the switch to CANCEL was saved (or by an older engine) can still say TRY or + // CONFIRM while already holding a required failure; running that phase again would retry an exhausted + // participant or skip a CONFIRM_FAILED one and report CONFIRMED. Such a transaction is cancelling. + if ($phase !== TccPhase::Cancel && $this->recordedFailure($tcc, $context)[0] !== null) { + $phase = TccPhase::Cancel; + } + if ($phase === TccPhase::Try) { $failure = $this->tryPhase($tcc, $inputs, $context); if ($failure !== null) { @@ -103,12 +112,18 @@ private function tryPhase(TccDefinition $tcc, TccInputs $inputs, ExecutionContex $this->support->events->onParticipantFailed($tcc->name, $context->correlationId, $participant->id, TccPhase::Try, $error); $this->support->events->onStepFailed($tcc->name, $context->correlationId, $participant->id, $error, $context->attempts($participant->id)); $this->support->publish($this->support->lifecycle, OrchestrationEvent::stepFailed($tcc->name, $context->correlationId, ExecutionPattern::Tcc, $participant->id, $error, $this->support->now())); - $this->support->checkpoint($context, ExecutionStatus::Trying); if ($participant->optional) { + $this->support->checkpoint($context, ExecutionStatus::Trying); + continue; } + // The failure checkpoint already records the switch to CANCEL, so a crash before cancelPhase() + // resumes into cancelling instead of trying the exhausted participant again. + $context->setCurrentPhase(TccPhase::Cancel); + $this->support->checkpoint($context, ExecutionStatus::Canceling); + $this->support->events->onPhaseFailed($tcc->name, $context->correlationId, TccPhase::Try, $error); return [$participant->id, $error]; @@ -152,7 +167,10 @@ private function confirmPhase(TccDefinition $tcc, ExecutionContext $context): ?a $this->support->events->onParticipantFailed($tcc->name, $context->correlationId, $participant->id, TccPhase::Confirm, $error); $this->support->events->onPhaseFailed($tcc->name, $context->correlationId, TccPhase::Confirm, $error); $this->support->publish($this->support->lifecycle, OrchestrationEvent::stepFailed($tcc->name, $context->correlationId, ExecutionPattern::Tcc, $participant->id, $error, $this->support->now())); - $this->support->checkpoint($context, ExecutionStatus::Confirming); + // Saved as CANCEL already: a crash before cancelPhase() must resume into cancelling, not back + // into CONFIRM, which would skip this participant and report the transaction CONFIRMED. + $context->setCurrentPhase(TccPhase::Cancel); + $this->support->checkpoint($context, ExecutionStatus::Canceling); return [$participant->id, $error]; } @@ -214,12 +232,19 @@ private function cancelPhase(TccDefinition $tcc, ExecutionContext $context, ?str return new TccRunOutcome($status, $failedId, $failedPhase, $error); } - /** @return array{0: ?string, 1: ?TccPhase, 2: ?Throwable} the failure a resumed CANCELING transaction was cancelling for */ + /** + * The required failure a resumed transaction is cancelling for: a CONFIRM_FAILED participant, or a FAILED or + * TIMED_OUT one that is not optional (an optional participant's try failure never cancels the transaction). + * + * @return array{0: ?string, 1: ?TccPhase, 2: ?Throwable} + */ private function recordedFailure(TccDefinition $tcc, ExecutionContext $context): array { foreach ($tcc->ordered() as $participant) { $status = $context->stepStatus($participant->id); - if (in_array($status, [StepStatus::Failed, StepStatus::TimedOut, StepStatus::ConfirmFailed], true)) { + $required = $status === StepStatus::ConfirmFailed + || (! $participant->optional && in_array($status, [StepStatus::Failed, StepStatus::TimedOut], true)); + if ($required) { return [ $participant->id, $status === StepStatus::ConfirmFailed ? TccPhase::Confirm : TccPhase::Try, diff --git a/packages/orchestration/tests/Support/RecordingExecutionPersistence.php b/packages/orchestration/tests/Support/RecordingExecutionPersistence.php new file mode 100644 index 00000000..e983dc34 --- /dev/null +++ b/packages/orchestration/tests/Support/RecordingExecutionPersistence.php @@ -0,0 +1,95 @@ + */ + public array $saved = []; + + private readonly MemoryExecutionPersistenceProvider $inner; + + public function __construct() + { + $this->inner = new MemoryExecutionPersistenceProvider; + } + + public function save(ExecutionState $state): void + { + $this->saved[] = ExecutionState::fromArray($state->toArray()); + $this->inner->save($state); + } + + public function findById(string $correlationId): ?ExecutionState + { + return $this->inner->findById($correlationId); + } + + public function find(ExecutionCriteria $criteria): array + { + return $this->inner->find($criteria); + } + + public function findInFlight(int $limit = 100): array + { + return $this->inner->findInFlight($limit); + } + + public function findStale(DateTimeImmutable $before, int $limit = 100): array + { + return $this->inner->findStale($before, $limit); + } + + public function findDue(DateTimeImmutable $now, int $limit = 100): array + { + return $this->inner->findDue($now, $limit); + } + + public function findBySearchAttributes(array $criteria, int $limit = 100): array + { + return $this->inner->findBySearchAttributes($criteria, $limit); + } + + public function updateStatus(string $correlationId, ExecutionStatus $status, DateTimeImmutable $now): void + { + $this->inner->updateStatus($correlationId, $status, $now); + } + + public function delete(string $correlationId): bool + { + return $this->inner->delete($correlationId); + } + + public function cleanup(DateTimeImmutable $before): int + { + return $this->inner->cleanup($before); + } + + public function countInFlight(): int + { + return $this->inner->countInFlight(); + } + + public function countByStatus(): array + { + return $this->inner->countByStatus(); + } + + public function isHealthy(): bool + { + return $this->inner->isHealthy(); + } +} diff --git a/packages/orchestration/tests/Tcc/TccResumeTest.php b/packages/orchestration/tests/Tcc/TccResumeTest.php index b47ebd63..8ef5efa3 100644 --- a/packages/orchestration/tests/Tcc/TccResumeTest.php +++ b/packages/orchestration/tests/Tcc/TccResumeTest.php @@ -4,9 +4,11 @@ use Firefly\Orchestration\Core\Exception\InvalidExecutionStateException; use Firefly\Orchestration\Core\ExecutionStatus; +use Firefly\Orchestration\Core\StepStatus; use Firefly\Orchestration\Core\TccPhase; use Firefly\Orchestration\Persistence\ExecutionState; use Firefly\Orchestration\Tcc\Builder\TccBuilder; +use Firefly\Orchestration\Tests\Support\RecordingExecutionPersistence; use Firefly\Orchestration\Tests\Support\TestEngines; it('resumes a TCC interrupted in CONFIRMING and confirms only what is left', function (): void { @@ -117,3 +119,124 @@ ->and($engines->deadLetterStore->findByCorrelationId('t-11'))->toBe([]) ->and(fn () => $engines->tccs->compensate('t-11'))->toThrow(InvalidExecutionStateException::class, 'already CANCELED'); }); + +it('resumes a CONFIRMING TCC with a CONFIRM_FAILED participant into CANCEL and ends FAILED', function (): void { + $engines = TestEngines::create(); + $log = []; + $phase = static function (string $name, string $id) use (&$log): Closure { + return static function () use ($name, $id, &$log): string { + $log[] = "{$name} {$id}"; + + return $id; + }; + }; + $engines->tccRegistry->register(TccBuilder::tcc('transfer') + ->participant('debit')->order(1)->tryWith($phase('try', 'debit'))->confirmWith($phase('confirm', 'debit'))->cancelWith($phase('cancel', 'debit'))->add() + ->participant('credit')->order(2)->tryWith($phase('try', 'credit'))->confirmWith($phase('confirm', 'credit'))->cancelWith($phase('cancel', 'credit'))->add() + ->build()); + // The process died between the CONFIRM_FAILED checkpoint and the switch to CANCEL (an engine that saved the + // failure under CONFIRM): resuming into CONFIRM would confirm credit and report a false CONFIRMED. + $engines->persistence->save(ExecutionState::fromArray([ + 'correlationId' => 't-12', 'executionName' => 'transfer', 'pattern' => 'TCC', 'status' => 'CONFIRMING', + 'startedAt' => '2026-10-12T09:00:00.000000+00:00', 'updatedAt' => '2026-10-12T09:00:01.000000+00:00', + 'currentPhase' => 'CONFIRM', + 'stepStatuses' => ['debit' => 'CONFIRM_FAILED', 'credit' => 'DONE'], + 'stepErrors' => ['debit' => 'debit confirm refused'], + 'tryResults' => ['debit' => 'debit', 'credit' => 'credit'], + 'completionOrder' => ['debit', 'credit'], + ])); + + $result = $engines->tccs->resume('t-12'); + + expect($result->status)->toBe(ExecutionStatus::Failed) + ->and($result->failedParticipantId)->toBe('debit') + ->and($result->failedPhase)->toBe(TccPhase::Confirm) + ->and($result->error?->getMessage())->toBe('debit confirm refused') + ->and($log)->toBe(['cancel credit', 'cancel debit']) + ->and($engines->persistence->findById('t-12')?->status)->toBe(ExecutionStatus::Failed) + ->and($engines->deadLetterStore->findByCorrelationId('t-12'))->toHaveCount(1); +}); + +it('resumes a TRYING TCC with a FAILED required participant into CANCEL without trying it again', function (): void { + $engines = TestEngines::create(); + $log = []; + $phase = static function (string $name, string $id) use (&$log): Closure { + return static function () use ($name, $id, &$log): string { + $log[] = "{$name} {$id}"; + + return $id; + }; + }; + $engines->tccRegistry->register(TccBuilder::tcc('transfer') + ->participant('audit')->order(0)->optional()->tryWith($phase('try', 'audit'))->confirmWith($phase('confirm', 'audit'))->cancelWith($phase('cancel', 'audit'))->add() + ->participant('debit')->order(1)->tryWith($phase('try', 'debit'))->confirmWith($phase('confirm', 'debit'))->cancelWith($phase('cancel', 'debit'))->add() + ->participant('credit')->order(2)->tryWith($phase('try', 'credit'))->confirmWith($phase('confirm', 'credit'))->cancelWith($phase('cancel', 'credit'))->add() + ->participant('notify')->order(3)->tryWith($phase('try', 'notify'))->confirmWith($phase('confirm', 'notify'))->cancelWith($phase('cancel', 'notify'))->add() + ->build()); + // credit used up its retries; the optional audit's earlier failure is not what the transaction cancels for. + $engines->persistence->save(ExecutionState::fromArray([ + 'correlationId' => 't-13', 'executionName' => 'transfer', 'pattern' => 'TCC', 'status' => 'TRYING', + 'startedAt' => '2026-10-12T09:00:00.000000+00:00', 'updatedAt' => '2026-10-12T09:00:01.000000+00:00', + 'currentPhase' => 'TRY', + 'stepStatuses' => ['audit' => 'FAILED', 'debit' => 'DONE', 'credit' => 'FAILED'], + 'stepErrors' => ['audit' => 'audit down', 'credit' => 'credit refused'], + 'tryResults' => ['debit' => 'debit'], + 'completionOrder' => ['debit'], + ])); + + $result = $engines->tccs->resume('t-13'); + + expect($result->isCanceled())->toBeTrue() + ->and($result->failedParticipantId)->toBe('credit') + ->and($result->failedPhase)->toBe(TccPhase::Try) + ->and($result->error?->getMessage())->toBe('credit refused') + ->and($log)->toBe(['cancel debit']); +}); + +it('checkpoints a required failure under CANCEL, so a crash right after it resumes into cancelling', function (string $phase, StepStatus $failed, ExecutionStatus $ending): void { + $register = static function (TestEngines $engines, array &$log) use ($phase): void { + $call = static function (string $name, string $id, bool $fails = false) use (&$log): Closure { + return static function () use ($name, $id, $fails, &$log): string { + $log[] = "{$name} {$id}"; + if ($fails) { + throw new RuntimeException("{$name} {$id} refused"); + } + + return $id; + }; + }; + $engines->tccRegistry->register(TccBuilder::tcc('transfer') + ->participant('debit')->order(1)->tryWith($call('try', 'debit'))->confirmWith($call('confirm', 'debit'))->cancelWith($call('cancel', 'debit'))->add() + ->participant('credit')->order(2)->tryWith($call('try', 'credit', $phase === 'TRY'))->confirmWith($call('confirm', 'credit', $phase === 'CONFIRM'))->cancelWith($call('cancel', 'credit'))->add() + ->build()); + }; + $recording = new RecordingExecutionPersistence; + $first = TestEngines::create(persistence: $recording); + $firstLog = []; + $register($first, $firstLog); + $first->tccs->execute('transfer', correlationId: 't-14'); + + // The first checkpoint that records credit's failure is what a crash in the listeners' window leaves behind. + $crashed = null; + foreach ($recording->saved as $state) { + if ($state->stepStatus('credit') === $failed) { + $crashed = $state; + break; + } + } + expect($crashed?->currentPhase)->toBe(TccPhase::Cancel); + + $second = TestEngines::create(); + $log = []; + $register($second, $log); + $second->persistence->save($crashed ?? throw new LogicException('no failure checkpoint')); + $result = $second->tccs->resume('t-14'); + + expect($result->status)->toBe($ending) + ->and($result->failedParticipantId)->toBe('credit') + ->and($result->failedPhase?->value)->toBe($phase) + ->and($log)->toBe($phase === 'TRY' ? ['cancel debit'] : ['cancel credit']); // a confirmed debit is not cancelled +})->with([ + 'a try failure' => ['TRY', StepStatus::Failed, ExecutionStatus::Canceled], + 'a confirm failure' => ['CONFIRM', StepStatus::ConfirmFailed, ExecutionStatus::Failed], +]); From b0a6ccd7b3aaae196c6022a6ca7387701418778a Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 20:24:18 -0700 Subject: [PATCH 22/58] fix(orchestration): honor the TCC composition compensation policy and resume an interrupted cancel on request as one --- .../src/Tcc/Composition/TccCompositor.php | 171 ++++++++++++++++-- .../src/Tcc/Engine/TccEngine.php | 28 ++- .../Tcc/Engine/TccExecutionOrchestrator.php | 31 +++- .../tests/Support/TestEngines.php | 13 +- .../tests/Tcc/TccCompositorTest.php | 138 ++++++++++++++ .../orchestration/tests/Tcc/TccResumeTest.php | 145 +++++++++++++++ .../resilience/src/ResilienceRegistry.php | 8 + .../tests/ResilienceRegistryTest.php | 9 + skeleton/config/firefly.php | 9 +- 9 files changed, 528 insertions(+), 24 deletions(-) diff --git a/packages/orchestration/src/Tcc/Composition/TccCompositor.php b/packages/orchestration/src/Tcc/Composition/TccCompositor.php index 4a8c64ef..6afee35e 100644 --- a/packages/orchestration/src/Tcc/Composition/TccCompositor.php +++ b/packages/orchestration/src/Tcc/Composition/TccCompositor.php @@ -4,6 +4,8 @@ namespace Firefly\Orchestration\Tcc\Composition; +use Firefly\Kernel\Exception\Infrastructure\CircuitBreakerOpenException; +use Firefly\Orchestration\Core\CompensationPolicy; use Firefly\Orchestration\Core\Condition\ConditionEvaluator; use Firefly\Orchestration\Core\Exception\CompensationException; use Firefly\Orchestration\Core\Exception\OrchestrationException; @@ -17,6 +19,10 @@ use Firefly\Orchestration\Settings\TccSettings; use Firefly\Orchestration\Tcc\Engine\TccEngine; use Firefly\Orchestration\Tcc\Engine\TccResult; +use Firefly\Resilience\CircuitBreaker; +use Firefly\Resilience\ResilienceRegistry; +use Firefly\Resilience\Retry; +use Firefly\Resilience\Store\ResilienceStore; use Illuminate\Support\Str; use Throwable; @@ -25,25 +31,46 @@ * run layer by layer through TccEngine, so each is a normal, persisted, recoverable TCC execution carrying the * composition's correlation id in its headers. A member that ends CONFIRMED is final from its own point of * view, so when a later required member fails, each confirmed member is undone through - * TccEngine::compensate() — its participants' cancel methods with their try results — in reverse completion - * order (spec decision D14). Data flows from members' try results. + * TccEngine::compensate() — its participants' cancel methods with their try results — in the order the + * compensation policy below sets (spec decision D14). Data flows from members' try results. * * Failure handling follows SagaCompositor: a required member "fails" when it does not confirm and equally * when it cannot run at all (its TCC is not registered, admission refuses it, its condition cannot be * evaluated); an optional member is tolerated either way and lands in TccCompositionResult::$optionalFailures, - * its dependents skipped. The rollback sweep never stops at a member it cannot undo: it records it in - * $compensationFailures and reports it through OrchestrationEvents::onStepCompensationFailed() (composition - * name, its correlation id, the member's alias). A member whose compensate() was refused keeps its CONFIRMED - * status; one whose cancel failed ends FAILED with the participant marked CANCEL_FAILED. + * its dependents skipped. A member the rollback cannot undo is recorded in $compensationFailures and reported + * through OrchestrationEvents::onStepCompensationFailed() (composition name, its correlation id, the member's + * alias). A member whose compensate() was refused keeps its CONFIRMED status; one whose cancel failed ends + * FAILED with the participant marked CANCEL_FAILED. + * + * The rollback follows `firefly.orchestration.tcc.composition.compensation-policy`, with SagaCompensator's + * meanings applied to members: + * + * - STRICT_SEQUENTIAL: reverse completion order, one compensate() per member, the sweep moving on past a + * member it cannot undo. There is no compensation error handler between members to stop it, so + * BEST_EFFORT behaves exactly the same. + * - GROUPED_PARALLEL: reverse layer order, declaration order inside a layer (sequentially, as for sagas). + * - RETRY_WITH_BACKOFF: reverse completion order; a member whose cancel failed is compensated again through + * firefly/resilience's Retry (`firefly.resilience.retry.orchestration-tcc-composition-`, or + * three attempts backing off 100 ms then 200 ms) — the engine retries only its CANCEL_FAILED participants. + * - CIRCUIT_BREAKER: reverse completion order through a firefly/resilience breaker per composition + * (`firefly.resilience.circuit-breaker.orchestration-tcc-composition-`, or five failures and + * thirty seconds open); once it is open the sweep stops, and every member it did not cancel is recorded as + * a compensation failure. + * - BEST_EFFORT: reverse completion order, every member attempted. */ final class TccCompositor { + /** The firefly/resilience instance names this compositor looks up, followed by the composition's name. */ + public const string RESILIENCE_PREFIX = 'orchestration-tcc-composition-'; + public function __construct( private readonly TccEngine $engine, private readonly OrchestrationEvents $events, private readonly ConditionEvaluator $conditions, private readonly CompositionDataFlowManager $dataFlow, private readonly TccSettings $settings, + private readonly ?ResilienceRegistry $resilience = null, + private readonly ?ResilienceStore $store = null, ) {} /** @@ -131,8 +158,11 @@ public function compose(TccComposition $composition, array $input = [], array $h } /** - * Cancels every confirmed member in reverse completion order and reports the composition as failed. A - * member that cannot be fully cancelled is recorded and reported, and the sweep moves on to the next one. + * Cancels every confirmed member, in the order and with the failure handling that + * `firefly.orchestration.tcc.composition.compensation-policy` names, and reports the composition as failed. + * A member that cannot be fully cancelled is recorded and reported; only CIRCUIT_BREAKER stops the sweep, + * and then every member it did not reach is recorded too, so `compensated` never claims a member that is + * still CONFIRMED. * * @param array $results * @param list $completed @@ -141,13 +171,26 @@ public function compose(TccComposition $composition, array $input = [], array $h */ private function abort(TccComposition $composition, string $correlationId, array $results, array $completed, array $skipped, array $optionalFailures, Throwable $error): TccCompositionResult { + $policy = $this->settings->compositionCompensationPolicy; + $breaker = $policy === CompensationPolicy::CircuitBreaker ? $this->breaker($composition->name) : null; + $retry = $policy === CompensationPolicy::RetryWithBackoff ? $this->retry($composition->name) : null; + $stoppedAt = null; $compensated = []; $failures = []; - foreach (array_reverse($completed) as $done) { - try { - $failure = $this->undo($done, $results[$done]); - } catch (Throwable $refused) { - $failure = $refused; + + foreach ($this->rollbackOrder($composition, $completed, $policy) as $done) { + if ($stoppedAt !== null) { + $failure = new CompensationException("TCC composition [{$composition->name}] did not cancel member [{$done}]: its compensation circuit opened at member [{$stoppedAt}]."); + } else { + try { + $failure = $this->attempt($done, $results[$done], $breaker, $retry); + } catch (CircuitBreakerOpenException $open) { + $stoppedAt = $done; + $failure = new CompensationException("TCC composition [{$composition->name}] did not cancel member [{$done}]: its compensation circuit is open.", $open); + } + if ($failure !== null && $stoppedAt === null && $breaker !== null && $breaker->state() === 'open') { + $stoppedAt = $done; + } } if ($failure === null) { @@ -164,6 +207,108 @@ private function abort(TccComposition $composition, string $correlationId, array return new TccCompositionResult($composition->name, $correlationId, false, $results, $error, $compensated, $skipped, $failures, $optionalFailures); } + /** + * The members to cancel, in the policy's order. GROUPED_PARALLEL walks the composition's layers in reverse, + * members of one layer in declaration order (sequentially: PHP runs one thing at a time, as for sagas); + * every other policy cancels in reverse completion order. + * + * @param list $completed + * @return list + */ + private function rollbackOrder(TccComposition $composition, array $completed, CompensationPolicy $policy): array + { + if ($policy !== CompensationPolicy::GroupedParallel) { + return array_reverse($completed); + } + + $order = []; + foreach (array_reverse($composition->layers) as $layer) { + foreach ($layer as $alias) { + if (in_array($alias, $completed, true)) { + $order[] = $alias; + } + } + } + + return $order; + } + + /** + * Cancels one member under the policy: through `$retry` (RETRY_WITH_BACKOFF), which asks compensate() again + * after a failed cancel — the engine then retries only the participants left CANCEL_FAILED — or through + * `$breaker` (CIRCUIT_BREAKER), where every failed member counts towards the breaker's threshold. Returns + * why the member is not cancelled, or null. Throws only when OUR breaker refused to admit the attempt: a + * cancel that itself ran into some other open breaker failed like any other cancel. + */ + private function attempt(string $alias, TccResult $confirmed, ?CircuitBreaker $breaker, ?Retry $retry): ?Throwable + { + $ran = false; + $once = function () use ($alias, $confirmed, &$ran): null { + $ran = true; + $failure = $this->undo($alias, $confirmed); + if ($failure !== null) { + throw $failure; + } + + return null; + }; + + try { + if ($breaker !== null) { + $breaker->call($once); + } elseif ($retry !== null) { + $retry->call($once); + } else { + $once(); + } + } catch (CircuitBreakerOpenException $refused) { + if (! $ran) { + throw $refused; + } + + return $refused; + } catch (Throwable $failure) { + return $failure; + } + + return null; + } + + /** + * The composition's breaker: `firefly.resilience.circuit-breaker.orchestration-tcc-composition-` when + * configured (an invalid value in it throws its ConfigurationException here rather than being ignored), + * otherwise a default breaker (five failures, thirty seconds open) on the resilience store. Without a + * store there is no breaker and CIRCUIT_BREAKER cancels like STRICT_SEQUENTIAL. + */ + private function breaker(string $composition): ?CircuitBreaker + { + $name = self::RESILIENCE_PREFIX.$composition; + if ($this->resilience !== null && $this->resilience->hasCircuitBreaker($name)) { + return $this->resilience->circuitBreaker($name); + } + + return $this->store === null ? null : new CircuitBreaker( + key: 'firefly:orchestration:tcc-composition:'.$composition, + store: $this->store, + failureThreshold: 5, + waitDurationInOpen: 30.0, + ); + } + + /** + * The composition's retry: `firefly.resilience.retry.orchestration-tcc-composition-` when configured, + * otherwise three attempts backing off 100 ms then 200 ms. + */ + private function retry(string $composition): Retry + { + $name = self::RESILIENCE_PREFIX.$composition; + if ($this->resilience !== null && $this->resilience->hasRetry($name)) { + return $this->resilience->retry($name); + } + + return new Retry(maxAttempts: 3, waitDuration: 0.1, backoffMultiplier: 2.0); + } + /** * Cancels one confirmed member. Returns null when it ended CANCELED, and otherwise why it did not: * TccEngine::compensate() handles a failed cancel itself and returns a FAILED result. diff --git a/packages/orchestration/src/Tcc/Engine/TccEngine.php b/packages/orchestration/src/Tcc/Engine/TccEngine.php index 05843f4b..5abbd2bb 100644 --- a/packages/orchestration/src/Tcc/Engine/TccEngine.php +++ b/packages/orchestration/src/Tcc/Engine/TccEngine.php @@ -11,6 +11,7 @@ use Firefly\Orchestration\Core\ExecutionContext; use Firefly\Orchestration\Core\ExecutionPattern; use Firefly\Orchestration\Core\ExecutionStatus; +use Firefly\Orchestration\Core\StepStatus; use Firefly\Orchestration\Event\OrchestrationEvent; use Firefly\Orchestration\Persistence\ExecutionState; use Firefly\Orchestration\Report\ExecutionReportBuilder; @@ -26,7 +27,9 @@ * As with SagaEngine, the outcome is always a TccResult — a failed phase is cancelled and reported, never * thrown. What throws is the refusal to run at all: an unknown TCC, a disabled pattern, back-pressure, a * correlation id that already exists, an execution that is not there or is already finished. A cancel on - * request (compensate()) is not a failure of the transaction: it neither dead-letters nor calls #[OnTccError]. + * request (compensate()) is not a failure of the transaction: it neither dead-letters nor calls #[OnTccError], + * and a resume() after a crash in the middle of one finishes it as a cancel on request. A cancel on request + * that ended FAILED with CANCEL_FAILED participants can be compensated again to retry just those cancels. */ final class TccEngine { @@ -101,6 +104,12 @@ public function resume(string $correlationId): TccResult $inputs = $inputs->forParticipant($participantId, $value); } + // A cancel on request interrupted by a crash goes on as one: confirmed participants are cancelled too, + // and the cancel neither dead-letters nor calls #[OnTccError], exactly as compensate() would have. + if ($this->orchestrator->cancelRequested($context)) { + return $this->support->tracing->traceExecution($definition->name, ExecutionPattern::Tcc, $correlationId, fn (): TccResult => $this->finish($definition, $context, $this->orchestrator->cancelAll($definition, $context), onRequest: true)); + } + return $this->support->tracing->traceExecution($definition->name, ExecutionPattern::Tcc, $correlationId, fn (): TccResult => $this->finish($definition, $context, $this->orchestrator->orchestrate($definition, $inputs, $context))); }); } @@ -109,12 +118,12 @@ public function compensate(string $correlationId): TccResult { return $this->support->lock->synchronized($correlationId, function () use ($correlationId): TccResult { $state = $this->load($correlationId); - if ($state->isTerminal() && $state->status !== ExecutionStatus::Confirmed) { + $context = ExecutionContext::fromState($state); + if ($state->isTerminal() && $state->status !== ExecutionStatus::Confirmed && ! $this->retriesCancel($state, $context)) { throw new InvalidExecutionStateException("TCC execution [{$correlationId}] is already {$state->status->value} and has nothing left to cancel."); } $definition = $this->registry->get($state->executionName); - $context = ExecutionContext::fromState($state); return $this->finish($definition, $context, $this->orchestrator->cancelAll($definition, $context), onRequest: true); }); @@ -157,6 +166,19 @@ private function finish(TccDefinition $tcc, ExecutionContext $context, TccRunOut return $result; } + /** + * A cancel on request that ended FAILED because some cancels failed may be asked again: the sweep skips + * what it already cancelled and retries only the CANCEL_FAILED participants (TccCompositor's + * retry-with-backoff policy relies on it). A transaction that FAILED on its own — a confirm failure — is + * not reopened: its confirmed participants stay confirmed, as the TCC protocol left them. + */ + private function retriesCancel(ExecutionState $state, ExecutionContext $context): bool + { + return $state->status === ExecutionStatus::Failed + && $this->orchestrator->cancelRequested($context) + && in_array(StepStatus::CancelFailed, $state->stepStatuses, true); + } + /** Why a cancel sweep that raised no error of its own still ended FAILED: the cancels that failed. */ private static function cancelFailures(ExecutionContext $context): string { diff --git a/packages/orchestration/src/Tcc/Engine/TccExecutionOrchestrator.php b/packages/orchestration/src/Tcc/Engine/TccExecutionOrchestrator.php index 44c4af21..545a9fad 100644 --- a/packages/orchestration/src/Tcc/Engine/TccExecutionOrchestrator.php +++ b/packages/orchestration/src/Tcc/Engine/TccExecutionOrchestrator.php @@ -35,9 +35,18 @@ * idempotent. A required failure is checkpointed together with the switch to CANCEL, and a resumed state that * still holds one goes straight to CANCEL, so a resume never retries an exhausted participant or reports a * transaction with a CONFIRM_FAILED participant as CONFIRMED. + * + * A cancel on request (cancelAll(), behind TccEngine::compensate()) is recorded in the context as the + * CANCEL_REQUESTED variable before its first CANCELING checkpoint. Without it a crash in the middle of undoing a + * CONFIRMED transaction would resume into the failure-driven CANCEL, which leaves confirmed participants + * confirmed, and report a clean rollback over resources that are still committed. TccEngine::resume() reads it + * through cancelRequested() and finishes such a transaction as the cancel on request it was. */ final class TccExecutionOrchestrator { + /** The context variable that marks a cancel on request; reserved, an application never sets it. */ + public const string CANCEL_REQUESTED = 'firefly.tcc.cancel-requested'; + public function __construct( private readonly StepInvoker $invoker, private readonly EngineSupport $support, @@ -79,9 +88,19 @@ public function orchestrate(TccDefinition $tcc, TccInputs $inputs, ExecutionCont public function cancelAll(TccDefinition $tcc, ExecutionContext $context): TccRunOutcome { + // Set before cancelPhase() saves its first CANCELING checkpoint, so every checkpoint of this sweep says + // it is a cancel on request and a resume after a crash cancels the confirmed participants too. + $context->setVariable(self::CANCEL_REQUESTED, true); + return $this->cancelPhase($tcc, $context, null, null, null, includeConfirmed: true); } + /** Whether this transaction is (or was) being cancelled on request, by cancelAll(). */ + public function cancelRequested(ExecutionContext $context): bool + { + return $context->variable(self::CANCEL_REQUESTED) === true; + } + /** @return array{0: string, 1: Throwable}|null the failed required participant and its error */ private function tryPhase(TccDefinition $tcc, TccInputs $inputs, ExecutionContext $context): ?array { @@ -236,18 +255,26 @@ private function cancelPhase(TccDefinition $tcc, ExecutionContext $context, ?str * The required failure a resumed transaction is cancelling for: a CONFIRM_FAILED participant, or a FAILED or * TIMED_OUT one that is not optional (an optional participant's try failure never cancels the transaction). * + * The CONFIRM_FAILED participant is itself cancelled by the sweep, which overwrites its status with CANCELED + * or CANCEL_FAILED; a crash after that cancel must still resume into a FAILED transaction with its confirm + * error. Its step error survives the overwrite, and only a confirm failure leaves one on a participant the + * sweep can cancel (a try failure stays FAILED and is never cancelled), so that pair also reads as the + * confirm failure. + * * @return array{0: ?string, 1: ?TccPhase, 2: ?Throwable} */ private function recordedFailure(TccDefinition $tcc, ExecutionContext $context): array { foreach ($tcc->ordered() as $participant) { $status = $context->stepStatus($participant->id); - $required = $status === StepStatus::ConfirmFailed + $confirmFailed = $status === StepStatus::ConfirmFailed + || (in_array($status, [StepStatus::Canceled, StepStatus::CancelFailed], true) && $context->stepError($participant->id) !== null); + $required = $confirmFailed || (! $participant->optional && in_array($status, [StepStatus::Failed, StepStatus::TimedOut], true)); if ($required) { return [ $participant->id, - $status === StepStatus::ConfirmFailed ? TccPhase::Confirm : TccPhase::Try, + $confirmFailed ? TccPhase::Confirm : TccPhase::Try, $context->stepError($participant->id) ?? new OrchestrationException("TCC [{$tcc->name}] participant [{$participant->id}] failed."), ]; } diff --git a/packages/orchestration/tests/Support/TestEngines.php b/packages/orchestration/tests/Support/TestEngines.php index 7ae02e42..e6b7f6ed 100644 --- a/packages/orchestration/tests/Support/TestEngines.php +++ b/packages/orchestration/tests/Support/TestEngines.php @@ -87,8 +87,9 @@ final class TestEngines /** * @param array $config + * @param array $resilience */ - private function __construct(array $config, ?ExecutionPersistenceProvider $persistence, ?DeadLetterStore $deadLetters) + private function __construct(array $config, ?ExecutionPersistenceProvider $persistence, ?DeadLetterStore $deadLetters, array $resilience) { $this->container = new Container; $this->settings = OrchestrationSettings::fromConfig(new Config(new Repository(['firefly' => ['orchestration' => $config]]))); @@ -103,6 +104,7 @@ private function __construct(array $config, ?ExecutionPersistenceProvider $persi $this->invoker = new StepInvoker($this->runner, $tracing); $this->conditions = new ConditionEvaluator; $resilienceStore = new InMemoryResilienceStore; + $resilienceRegistry = ResilienceRegistry::fromConfig(new Config(new Repository(['firefly' => ['resilience' => $resilience]])), $resilienceStore); $breaker = $this->settings->backpressure->strategy === 'circuit-breaker' ? new CircuitBreaker( @@ -134,7 +136,7 @@ private function __construct(array $config, ?ExecutionPersistenceProvider $persi $this->support, $this->settings->saga->compensationPolicy, (new CompensationErrorHandlerFactory($this->container, $this->logger))->get($this->settings->saga->compensationErrorHandler), - ResilienceRegistry::fromConfig(new Config(new Repository(['firefly' => []])), $resilienceStore), + $resilienceRegistry, $resilienceStore, ); $this->sagas = new SagaEngine($this->sagaRegistry, new SagaExecutionOrchestrator($this->invoker, $this->support), $this->compensator, $this->support, $this->settings->saga); @@ -142,14 +144,15 @@ private function __construct(array $config, ?ExecutionPersistenceProvider $persi $this->tccRegistry = new TccRegistry; $this->tccs = new TccEngine($this->tccRegistry, new TccExecutionOrchestrator($this->invoker, $this->support, $this->settings->tcc), $this->support, $this->settings->tcc); - $this->tccCompositor = new TccCompositor($this->tccs, $this->events, $this->conditions, new CompositionDataFlowManager, $this->settings->tcc); + $this->tccCompositor = new TccCompositor($this->tccs, $this->events, $this->conditions, new CompositionDataFlowManager, $this->settings->tcc, $resilienceRegistry, $resilienceStore); } /** * @param array $config + * @param array $resilience the `firefly.resilience` section (named retry and breaker instances) */ - public static function create(array $config = [], ?ExecutionPersistenceProvider $persistence = null, ?DeadLetterStore $deadLetters = null): self + public static function create(array $config = [], ?ExecutionPersistenceProvider $persistence = null, ?DeadLetterStore $deadLetters = null, array $resilience = []): self { - return new self($config, $persistence, $deadLetters); + return new self($config, $persistence, $deadLetters, $resilience); } } diff --git a/packages/orchestration/tests/Tcc/TccCompositorTest.php b/packages/orchestration/tests/Tcc/TccCompositorTest.php index a19d8b21..8e6209e6 100644 --- a/packages/orchestration/tests/Tcc/TccCompositorTest.php +++ b/packages/orchestration/tests/Tcc/TccCompositorTest.php @@ -138,3 +138,141 @@ function orchestrationTccPair(TestEngines $engines, array &$log): void ->and($engines->events->hooks())->toContain('onStepCompensationFailed') ->and($result->toArray()['compensationFailures'])->toBe(['stubborn' => $result->compensationFailures['stubborn']->getMessage()]); }); + +/** + * One single-participant TCC per name, logging "cancel ". `$cancelFailures` says how many times a member's + * cancel fails before it succeeds (PHP_INT_MAX: always); the TCC named `fail` refuses its try. + * + * @param list $names + * @param list $log + * @param array $cancelFailures + */ +function orchestrationTccMembers(TestEngines $engines, array $names, array &$log, array $cancelFailures = []): void +{ + // One array shared by reference: a per-iteration `$failures` captured by reference would be one variable + // that every later iteration overwrites. + $remaining = $cancelFailures; + foreach ([...$names, 'fail'] as $name) { + $engines->tccRegistry->register(TccBuilder::tcc($name)->retry(false) + ->participant('main') + ->tryWith(static function () use ($name): string { + if ($name === 'fail') { + throw new RuntimeException('fail refused'); + } + + return $name; + }) + ->confirmWith(static fn (): null => null) + ->cancelWith(static function () use ($name, &$log, &$remaining): void { + $log[] = "cancel {$name}"; + if (($remaining[$name] ?? 0) > 0) { + $remaining[$name]--; + + throw new RuntimeException("{$name} cancel failed"); + } + }) + ->add()->build()); + } +} + +it('cancels the confirmed members layer by layer in reverse under grouped-parallel', function (): void { + $engines = TestEngines::create(['tcc' => ['composition' => ['compensation-policy' => 'grouped-parallel']]]); + $log = []; + orchestrationTccMembers($engines, ['a', 'b', 'c'], $log); + // Completion order is a, b, c; reverse completion would cancel c, b, a. The layers are [a, b] then [c]. + $composition = TccCompositionBuilder::composition('trip') + ->tcc('a')->add() + ->tcc('b')->add() + ->tcc('c')->dependsOn('a')->add() + ->tcc('fail')->dependsOn('c')->add() + ->build(); + + $result = $engines->tccCompositor->compose($composition); + + expect($result->success)->toBeFalse() + ->and($result->compensated)->toBe(['c', 'a', 'b']) + ->and($log)->toBe(['cancel c', 'cancel a', 'cancel b']); +}); + +it('compensates a member again after a failed cancel under retry-with-backoff', function (): void { + $engines = TestEngines::create( + ['tcc' => ['composition' => ['compensation-policy' => 'retry-with-backoff']]], + resilience: ['retry' => ['orchestration-tcc-composition-trip' => ['max-attempts' => 3]]], + ); + $log = []; + orchestrationTccMembers($engines, ['a', 'flaky'], $log, ['flaky' => 2]); + $composition = TccCompositionBuilder::composition('trip') + ->tcc('a')->add() + ->tcc('flaky')->dependsOn('a')->add() + ->tcc('fail')->dependsOn('flaky')->add() + ->build(); + + $result = $engines->tccCompositor->compose($composition); + + expect($result->success)->toBeFalse() + ->and($result->compensated)->toBe(['flaky', 'a']) + ->and($result->compensationFailures)->toBe([]) + ->and($log)->toBe(['cancel flaky', 'cancel flaky', 'cancel flaky', 'cancel a']) + ->and($engines->persistence->findById($result->results['flaky']->correlationId)?->status)->toBe(ExecutionStatus::Canceled); +}); + +it('does not compensate a member twice under strict-sequential', function (): void { + $engines = TestEngines::create(resilience: ['retry' => ['orchestration-tcc-composition-trip' => ['max-attempts' => 3]]]); + $log = []; + orchestrationTccMembers($engines, ['a', 'flaky'], $log, ['flaky' => 1]); + $composition = TccCompositionBuilder::composition('trip') + ->tcc('a')->add() + ->tcc('flaky')->dependsOn('a')->add() + ->tcc('fail')->dependsOn('flaky')->add() + ->build(); + + $result = $engines->tccCompositor->compose($composition); + + expect($result->compensated)->toBe(['a']) + ->and(array_keys($result->compensationFailures))->toBe(['flaky']) + ->and($log)->toBe(['cancel flaky', 'cancel a']); +}); + +it('stops the rollback once the composition breaker opens and records the members it left', function (): void { + $engines = TestEngines::create( + ['tcc' => ['composition' => ['compensation-policy' => 'circuit-breaker']]], + resilience: ['circuit-breaker' => ['orchestration-tcc-composition-trip' => ['failure-threshold' => 1]]], + ); + $log = []; + orchestrationTccMembers($engines, ['a', 'b', 'c'], $log, ['c' => PHP_INT_MAX]); + $composition = TccCompositionBuilder::composition('trip') + ->tcc('a')->add() + ->tcc('b')->dependsOn('a')->add() + ->tcc('c')->dependsOn('b')->add() + ->tcc('fail')->dependsOn('c')->add() + ->build(); + + $result = $engines->tccCompositor->compose($composition); + + expect($result->success)->toBeFalse() + ->and($result->compensated)->toBe([]) + ->and(array_keys($result->compensationFailures))->toBe(['c', 'b', 'a']) + ->and($result->compensationFailures['b']->getMessage())->toBe('TCC composition [trip] did not cancel member [b]: its compensation circuit opened at member [c].') + ->and($log)->toBe(['cancel c']) + ->and($engines->persistence->findById($result->results['b']->correlationId)?->status)->toBe(ExecutionStatus::Confirmed) + ->and($engines->persistence->findById($result->results['a']->correlationId)?->status)->toBe(ExecutionStatus::Confirmed) + ->and($engines->events->count('onStepCompensationFailed'))->toBe(3); +}); + +it('attempts every member past a failed cancel under best-effort', function (): void { + $engines = TestEngines::create(['tcc' => ['composition' => ['compensation-policy' => 'best-effort']]]); + $log = []; + orchestrationTccMembers($engines, ['a', 'b', 'c'], $log, ['c' => PHP_INT_MAX]); + $composition = TccCompositionBuilder::composition('trip') + ->tcc('a')->add() + ->tcc('b')->dependsOn('a')->add() + ->tcc('c')->dependsOn('b')->add() + ->tcc('fail')->dependsOn('c')->add() + ->build(); + + $result = $engines->tccCompositor->compose($composition); + + expect($result->compensated)->toBe(['b', 'a']) + ->and(array_keys($result->compensationFailures))->toBe(['c']) + ->and($log)->toBe(['cancel c', 'cancel b', 'cancel a']); +}); diff --git a/packages/orchestration/tests/Tcc/TccResumeTest.php b/packages/orchestration/tests/Tcc/TccResumeTest.php index 8ef5efa3..2e730930 100644 --- a/packages/orchestration/tests/Tcc/TccResumeTest.php +++ b/packages/orchestration/tests/Tcc/TccResumeTest.php @@ -240,3 +240,148 @@ 'a try failure' => ['TRY', StepStatus::Failed, ExecutionStatus::Canceled], 'a confirm failure' => ['CONFIRM', StepStatus::ConfirmFailed, ExecutionStatus::Failed], ]); + +it('resumes a cancel on request interrupted mid-sweep as a cancel on request, confirmed participants included', function (): void { + $register = static function (TestEngines $engines, array &$log, int &$errors): void { + $call = static function (string $name, string $id) use (&$log): Closure { + return static function () use ($name, $id, &$log): string { + $log[] = "{$name} {$id}"; + + return $id; + }; + }; + $engines->tccRegistry->register(TccBuilder::tcc('transfer') + ->participant('debit')->order(1)->tryWith($call('try', 'debit'))->confirmWith($call('confirm', 'debit'))->cancelWith($call('cancel', 'debit'))->add() + ->participant('credit')->order(2)->tryWith($call('try', 'credit'))->confirmWith($call('confirm', 'credit'))->cancelWith($call('cancel', 'credit'))->add() + ->onError(static function () use (&$errors): void { + $errors++; + }) + ->build()); + }; + $recording = new RecordingExecutionPersistence; + $first = TestEngines::create(persistence: $recording); + $firstLog = []; + $firstErrors = 0; + $register($first, $firstLog, $firstErrors); + $first->tccs->execute('transfer', correlationId: 't-15'); + $first->tccs->compensate('t-15'); + + // The checkpoint after credit's cancel landed and before debit's: a crash there leaves a CONFIRMED debit. + $crashed = null; + foreach ($recording->saved as $state) { + if ($state->stepStatus('credit') === StepStatus::Canceled && $state->stepStatus('debit') === StepStatus::Confirmed) { + $crashed = $state; + break; + } + } + expect($crashed?->status)->toBe(ExecutionStatus::Canceling) + ->and($crashed?->currentPhase)->toBe(TccPhase::Cancel); + + $second = TestEngines::create(); + $log = []; + $errors = 0; + $register($second, $log, $errors); + $second->persistence->save($crashed ?? throw new LogicException('no mid-sweep checkpoint')); + $result = $second->tccs->resume('t-15'); + + expect($result->status)->toBe(ExecutionStatus::Canceled) + ->and($result->report?->failureReason)->toBe('canceled on request') + ->and($log)->toBe(['cancel debit']) + ->and($second->persistence->findById('t-15')?->stepStatus('debit'))->toBe(StepStatus::Canceled) + ->and($second->deadLetterStore->findByCorrelationId('t-15'))->toBe([]) + ->and($errors)->toBe(0); +}); + +it('resumes a confirm failure whose failed participant was already cancelled as FAILED with its confirm error', function (): void { + $register = static function (TestEngines $engines, array &$log): void { + $call = static function (string $name, string $id, bool $fails = false) use (&$log): Closure { + return static function () use ($name, $id, $fails, &$log): string { + $log[] = "{$name} {$id}"; + if ($fails) { + throw new RuntimeException("{$name} {$id} refused"); + } + + return $id; + }; + }; + $engines->tccRegistry->register(TccBuilder::tcc('transfer')->retry(false) + ->participant('debit')->order(1)->tryWith($call('try', 'debit'))->confirmWith($call('confirm', 'debit'))->cancelWith($call('cancel', 'debit'))->add() + ->participant('credit')->order(2)->tryWith($call('try', 'credit'))->confirmWith($call('confirm', 'credit', true))->cancelWith($call('cancel', 'credit'))->add() + ->participant('notify')->order(3)->tryWith($call('try', 'notify'))->confirmWith($call('confirm', 'notify'))->cancelWith($call('cancel', 'notify'))->add() + ->build()); + }; + $recording = new RecordingExecutionPersistence; + $first = TestEngines::create(persistence: $recording); + $firstLog = []; + $register($first, $firstLog); + $first->tccs->execute('transfer', correlationId: 't-16'); + + // The sweep cancels notify, then the CONFIRM_FAILED credit, overwriting its status with CANCELED; a crash + // right after that checkpoint must not lose the confirm failure. + $crashed = null; + foreach ($recording->saved as $state) { + if ($state->status === ExecutionStatus::Canceling && $state->stepStatus('credit') === StepStatus::Canceled) { + $crashed = $state; + break; + } + } + + $second = TestEngines::create(); + $log = []; + $register($second, $log); + $second->persistence->save($crashed ?? throw new LogicException('no mid-sweep checkpoint')); + $result = $second->tccs->resume('t-16'); + + expect($result->status)->toBe(ExecutionStatus::Failed) + ->and($result->failedParticipantId)->toBe('credit') + ->and($result->failedPhase)->toBe(TccPhase::Confirm) + ->and($result->error?->getMessage())->toBe('confirm credit refused') + ->and($log)->toBe([]) + ->and($second->persistence->findById('t-16')?->stepStatus('debit'))->toBe(StepStatus::Confirmed) + ->and($second->deadLetterStore->findByCorrelationId('t-16'))->toHaveCount(1); +}); + +it('compensates a cancel on request that ended FAILED again, retrying only the cancels that failed', function (): void { + $engines = TestEngines::create(); + $log = []; + $creditFails = 1; + $engines->tccRegistry->register(TccBuilder::tcc('transfer')->retry(false) + ->participant('debit')->order(1)->tryWith(static fn (): string => 'debit')->confirmWith(static fn (): null => null) + ->cancelWith(static function () use (&$log): void { + $log[] = 'cancel debit'; + })->add() + ->participant('credit')->order(2)->tryWith(static fn (): string => 'credit')->confirmWith(static fn (): null => null) + ->cancelWith(static function () use (&$log, &$creditFails): void { + $log[] = 'cancel credit'; + if ($creditFails-- > 0) { + throw new RuntimeException('ledger offline'); + } + })->add() + ->build()); + $engines->tccs->execute('transfer', correlationId: 't-17'); + + $failed = $engines->tccs->compensate('t-17'); + $retried = $engines->tccs->compensate('t-17'); + + expect($failed->status)->toBe(ExecutionStatus::Failed) + ->and($retried->status)->toBe(ExecutionStatus::Canceled) + ->and($log)->toBe(['cancel credit', 'cancel debit', 'cancel credit']) + ->and($engines->deadLetterStore->findByCorrelationId('t-17'))->toBe([]); +}); + +it('does not reopen a TCC that FAILED on a confirm failure', function (): void { + $engines = TestEngines::create(); + $engines->tccRegistry->register(TccBuilder::tcc('transfer')->retry(false) + ->participant('debit')->order(1)->tryWith(static fn (): string => 'debit')->confirmWith(static fn (): null => null)->cancelWith(static fn (): null => null)->add() + ->participant('credit')->order(2)->tryWith(static fn (): string => 'credit') + ->confirmWith(static function (): void { + throw new RuntimeException('credit confirm refused'); + }) + ->cancelWith(static function (): void { + throw new RuntimeException('credit cancel refused'); + })->add() + ->build()); + + expect($engines->tccs->execute('transfer', correlationId: 't-18')->status)->toBe(ExecutionStatus::Failed) + ->and(fn () => $engines->tccs->compensate('t-18'))->toThrow(InvalidExecutionStateException::class, 'already FAILED'); +}); diff --git a/packages/resilience/src/ResilienceRegistry.php b/packages/resilience/src/ResilienceRegistry.php index cb59cce8..b984f59f 100644 --- a/packages/resilience/src/ResilienceRegistry.php +++ b/packages/resilience/src/ResilienceRegistry.php @@ -88,6 +88,14 @@ public function hasCircuitBreaker(string $name): bool return is_array($group) && array_key_exists($name, $group); } + /** Whether firefly.resilience.retry names an instance called $name — hasCircuitBreaker()'s twin for retries. */ + public function hasRetry(string $name): bool + { + $group = $this->config['retry'] ?? null; + + return is_array($group) && array_key_exists($name, $group); + } + public function rateLimiter(string $name): RateLimiter { return $this->rateLimiters[$name] ??= new RateLimiter( diff --git a/packages/resilience/tests/ResilienceRegistryTest.php b/packages/resilience/tests/ResilienceRegistryTest.php index 5b3bafb9..6174aa0c 100644 --- a/packages/resilience/tests/ResilienceRegistryTest.php +++ b/packages/resilience/tests/ResilienceRegistryTest.php @@ -82,6 +82,15 @@ function makeResilienceRegistry(array $resilience = []): ResilienceRegistry ->and(fn () => $reg->circuitBreaker('broken'))->toThrow(ConfigurationException::class, 'Invalid duration [two minutes]'); }); +it('reports whether a retry instance is configured without building it', function () { + $reg = makeResilienceRegistry(['retry' => ['payments' => []]]); + + expect($reg->hasRetry('payments'))->toBeTrue() + ->and($reg->hasRetry('missing'))->toBeFalse() + ->and(makeResilienceRegistry()->hasRetry('payments'))->toBeFalse() + ->and(makeResilienceRegistry(['circuit-breaker' => ['payments' => []]])->hasRetry('payments'))->toBeFalse(); +}); + it('wires the circuit breaker minimum-number-of-calls key through to the built instance', function () { // A registry key that never reaches the constructor is indistinguishable from a typo, so assert the // BEHAVIOUR the key buys rather than the object's shape: with a minimum of 3, two failures on a diff --git a/skeleton/config/firefly.php b/skeleton/config/firefly.php index 53ec581c..5e42a305 100644 --- a/skeleton/config/firefly.php +++ b/skeleton/config/firefly.php @@ -2577,7 +2577,14 @@ | firefly.resilience.circuit-breaker.orchestration-compensation-) or best-effort. | `compensation-error-handler` is default, fail-fast, log-and-continue, retry, robust, or the class | name of your own CompensationErrorHandler bean. The TCC `default-timeout` applies to a phase that - | sets no timeoutMs. Defaults: enabled true, strict-sequential, 5m, default, 30s. + | sets no timeoutMs. The TCC `composition.compensation-policy` decides how a failed TCC composition + | cancels its confirmed members: strict-sequential and best-effort (reverse completion order, every + | member attempted), grouped-parallel (layer by layer in reverse), retry-with-backoff (a failed + | member is compensated again through firefly.resilience.retry.orchestration-tcc-composition- + | , else 3 attempts from 100ms) or circuit-breaker (stops once firefly.resilience. + | circuit-breaker.orchestration-tcc-composition- opens, else 5 failures; the members + | left are reported as compensation failures). Defaults: enabled true, strict-sequential, 5m, + | default, 30s, composition enabled true and strict-sequential. */ 'saga' => [ 'enabled' => env('FIREFLY_ORCHESTRATION_SAGA_ENABLED', true), From d2177ef96859b15894123e731cc72feefad0e8a7 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 20:30:35 -0700 Subject: [PATCH 23/58] fix(orchestration): cancel TCC participants whose try timed out or was interrupted, with a null try result --- .../src/Tcc/Attributes/CancelMethod.php | 5 ++ .../src/Tcc/Engine/TccEngine.php | 3 +- .../Tcc/Engine/TccExecutionOrchestrator.php | 86 ++++++++++++++----- .../orchestration/tests/Tcc/TccEngineTest.php | 54 ++++++++++++ .../orchestration/tests/Tcc/TccResumeTest.php | 69 +++++++++++++++ 5 files changed, 196 insertions(+), 21 deletions(-) diff --git a/packages/orchestration/src/Tcc/Attributes/CancelMethod.php b/packages/orchestration/src/Tcc/Attributes/CancelMethod.php index 21a1e1a5..b0b987f7 100644 --- a/packages/orchestration/src/Tcc/Attributes/CancelMethod.php +++ b/packages/orchestration/src/Tcc/Attributes/CancelMethod.php @@ -9,6 +9,11 @@ /** * The participant's CANCEL: release the reservation; receives the try result. Called for tried participants * when the transaction cancels, and for confirmed ones too when a composition or recovery compensates it. + * + * A participant whose try timed out, or was interrupted by a crash, may still have reserved, so it is + * cancelled too — with a null try result (the TCC "empty rollback"). A cancel must therefore accept null and + * release whatever it holds for this transaction (find it by the participant input), do nothing when it holds + * nothing, and be idempotent, since a resume can repeat it. */ #[Attribute(Attribute::TARGET_METHOD)] final readonly class CancelMethod diff --git a/packages/orchestration/src/Tcc/Engine/TccEngine.php b/packages/orchestration/src/Tcc/Engine/TccEngine.php index 5abbd2bb..7d553e55 100644 --- a/packages/orchestration/src/Tcc/Engine/TccEngine.php +++ b/packages/orchestration/src/Tcc/Engine/TccEngine.php @@ -22,7 +22,8 @@ /** * Runs TCC transactions (jafly TccEngine): execute(), resume() from the checkpointed phase, compensate() * (cancel every tried participant, confirmed ones included — a composition or recovery undoing a CONFIRMED - * transaction). Every entry point holds the execution lock; execute() passes back-pressure admission. + * transaction; "tried" covers a try that timed out or was left RUNNING by a crash, cancelled with a null try + * result). Every entry point holds the execution lock; execute() passes back-pressure admission. * * As with SagaEngine, the outcome is always a TccResult — a failed phase is cancelled and reported, never * thrown. What throws is the refusal to run at all: an unknown TCC, a disabled pattern, back-pressure, a diff --git a/packages/orchestration/src/Tcc/Engine/TccExecutionOrchestrator.php b/packages/orchestration/src/Tcc/Engine/TccExecutionOrchestrator.php index 545a9fad..9233bbf8 100644 --- a/packages/orchestration/src/Tcc/Engine/TccExecutionOrchestrator.php +++ b/packages/orchestration/src/Tcc/Engine/TccExecutionOrchestrator.php @@ -30,6 +30,14 @@ * ending FAILED (some participants already confirmed — jafly parity). * - CANCEL: every tried-but-unconfirmed participant in REVERSE order; a cancel failure ends FAILED. * + * "Tried" includes a participant that may have reserved without reporting it: one whose try TIMED_OUT (outside + * pcntl whole-second deadlines the time limiter only notices the overrun after the try body has returned, so + * the reservation exists — and each retry made another), and one left RUNNING by a crash in the middle of its + * try. Both are cancelled with a null try result, the TCC "empty rollback": a cancel method must accept a null + * try result and release whatever the participant holds for this transaction, or do nothing when it holds + * nothing. An optional participant whose try timed out is cancelled the same way after a successful CONFIRM, + * so a confirmed transaction does not keep its reservation either; the transaction stays CONFIRMED. + * * The phase is checkpointed as it starts and every participant result as it lands, so resume() picks up a * CONFIRMING or CANCELING transaction exactly where a crash left it — which is why confirm and cancel must be * idempotent. A required failure is checkpointed together with the switch to CANCEL, and a resumed state that @@ -78,6 +86,8 @@ public function orchestrate(TccDefinition $tcc, TccInputs $inputs, ExecutionCont return $this->cancelPhase($tcc, $context, $failure[0], TccPhase::Confirm, $failure[1], includeConfirmed: false); } + $this->releaseTimedOutOptionals($tcc, $context); + return new TccRunOutcome(ExecutionStatus::Confirmed); } @@ -217,25 +227,19 @@ private function cancelPhase(TccDefinition $tcc, ExecutionContext $context, ?str $this->support->checkpoint($context, ExecutionStatus::Canceling); $started = hrtime(true); $cancelFailed = false; + // TIMED_OUT and RUNNING participants may hold a reservation they never reported (see the class + // docblock); they are cancelled with a null try result. A plain try failure (FAILED) reserved nothing. $cancelable = $includeConfirmed - ? [StepStatus::Done, StepStatus::Confirmed, StepStatus::ConfirmFailed, StepStatus::CancelFailed] - : [StepStatus::Done, StepStatus::ConfirmFailed, StepStatus::CancelFailed]; + ? [StepStatus::Done, StepStatus::Confirmed, StepStatus::ConfirmFailed, StepStatus::CancelFailed, StepStatus::TimedOut, StepStatus::Running] + : [StepStatus::Done, StepStatus::ConfirmFailed, StepStatus::CancelFailed, StepStatus::TimedOut, StepStatus::Running]; foreach (array_reverse($tcc->ordered()) as $participant) { if (! in_array($context->stepStatus($participant->id), $cancelable, true)) { continue; } - $this->support->events->onParticipantStarted($tcc->name, $context->correlationId, $participant->id, TccPhase::Cancel); - try { - $this->invoker->invoke($tcc->name, $participant->id.':cancel', $participant->cancel->invocation, $this->options($participant, TccPhase::Cancel, $tcc), $context, $context->stepInput($participant->id), $context->tryResult($participant->id)); - $context->setStepStatus($participant->id, StepStatus::Canceled); - $this->support->events->onParticipantSuccess($tcc->name, $context->correlationId, $participant->id, TccPhase::Cancel); - } catch (Throwable $cancelError) { + if (! $this->cancelOne($tcc, $participant, $context)) { $cancelFailed = true; - $context->setStepStatus($participant->id, StepStatus::CancelFailed); - $context->putCompensationError($participant->id, $cancelError); - $this->support->events->onParticipantFailed($tcc->name, $context->correlationId, $participant->id, TccPhase::Cancel, $cancelError); } $this->support->checkpoint($context, ExecutionStatus::Canceling); } @@ -251,15 +255,56 @@ private function cancelPhase(TccDefinition $tcc, ExecutionContext $context, ?str return new TccRunOutcome($status, $failedId, $failedPhase, $error); } + /** + * After a successful CONFIRM, cancels the optional participants whose try timed out: their try may have + * reserved after all (see the class docblock), and nothing else would ever release it. A failure here is + * recorded as CANCEL_FAILED — compensate() retries it with the rest — and does not undo the CONFIRMED + * outcome, which the confirmed participants already committed. + */ + private function releaseTimedOutOptionals(TccDefinition $tcc, ExecutionContext $context): void + { + foreach (array_reverse($tcc->ordered()) as $participant) { + if (! $participant->optional || $context->stepStatus($participant->id) !== StepStatus::TimedOut) { + continue; + } + + $this->cancelOne($tcc, $participant, $context); + $this->support->checkpoint($context, ExecutionStatus::Confirming); + } + } + + /** Runs one participant's cancel with its try result (null when the try never reported one); false when it failed. */ + private function cancelOne(TccDefinition $tcc, TccParticipantDefinition $participant, ExecutionContext $context): bool + { + $this->support->events->onParticipantStarted($tcc->name, $context->correlationId, $participant->id, TccPhase::Cancel); + try { + $this->invoker->invoke($tcc->name, $participant->id.':cancel', $participant->cancel->invocation, $this->options($participant, TccPhase::Cancel, $tcc), $context, $context->stepInput($participant->id), $context->tryResult($participant->id)); + } catch (Throwable $cancelError) { + $context->setStepStatus($participant->id, StepStatus::CancelFailed); + $context->putCompensationError($participant->id, $cancelError); + $this->support->events->onParticipantFailed($tcc->name, $context->correlationId, $participant->id, TccPhase::Cancel, $cancelError); + + return false; + } + + $context->setStepStatus($participant->id, StepStatus::Canceled); + $this->support->events->onParticipantSuccess($tcc->name, $context->correlationId, $participant->id, TccPhase::Cancel); + + return true; + } + /** * The required failure a resumed transaction is cancelling for: a CONFIRM_FAILED participant, or a FAILED or * TIMED_OUT one that is not optional (an optional participant's try failure never cancels the transaction). * - * The CONFIRM_FAILED participant is itself cancelled by the sweep, which overwrites its status with CANCELED - * or CANCEL_FAILED; a crash after that cancel must still resume into a FAILED transaction with its confirm - * error. Its step error survives the overwrite, and only a confirm failure leaves one on a participant the - * sweep can cancel (a try failure stays FAILED and is never cancelled), so that pair also reads as the - * confirm failure. + * The sweep overwrites the status of the participants it cancels with CANCELED or CANCEL_FAILED, the failed + * one included: a CONFIRM_FAILED participant, and a TIMED_OUT one (cancelled because its try may have + * reserved). A crash after that cancel must still resume with the original failure. The step error survives + * the overwrite (cancel errors are compensation errors), and the try result tells the two apart: a + * participant that reached CONFIRM has one, a try that timed out never stored one. So a cancelled + * participant with a step error and a try result is the confirm failure, and one with a step error and no + * try result is a timed-out try — a required failure only when the participant is not optional. A plain + * FAILED try is never cancelled and keeps its status. * * @return array{0: ?string, 1: ?TccPhase, 2: ?Throwable} */ @@ -267,10 +312,11 @@ private function recordedFailure(TccDefinition $tcc, ExecutionContext $context): { foreach ($tcc->ordered() as $participant) { $status = $context->stepStatus($participant->id); - $confirmFailed = $status === StepStatus::ConfirmFailed - || (in_array($status, [StepStatus::Canceled, StepStatus::CancelFailed], true) && $context->stepError($participant->id) !== null); - $required = $confirmFailed - || (! $participant->optional && in_array($status, [StepStatus::Failed, StepStatus::TimedOut], true)); + $cancelledFailure = in_array($status, [StepStatus::Canceled, StepStatus::CancelFailed], true) && $context->stepError($participant->id) !== null; + $tried = array_key_exists($participant->id, $context->tryResults()); + $confirmFailed = $status === StepStatus::ConfirmFailed || ($cancelledFailure && $tried); + $tryFailed = in_array($status, [StepStatus::Failed, StepStatus::TimedOut], true) || ($cancelledFailure && ! $tried); + $required = $confirmFailed || (! $participant->optional && $tryFailed); if ($required) { return [ $participant->id, diff --git a/packages/orchestration/tests/Tcc/TccEngineTest.php b/packages/orchestration/tests/Tcc/TccEngineTest.php index 3fc89b6b..d88e763d 100644 --- a/packages/orchestration/tests/Tcc/TccEngineTest.php +++ b/packages/orchestration/tests/Tcc/TccEngineTest.php @@ -176,3 +176,57 @@ function orchestrationTransfer(array &$log, array $fail = [], bool $optionalCred ->and($engines->persistence->findById('t-3')?->compensationErrors)->toBe(['debit' => 'cancel debit failed']) ->and($engines->events->hooks())->toContain('onPhaseFailed'); }); + +/** + * debit (order 1) and credit (order 2); credit's try reserves, then overruns its 10 ms deadline — a sub-second + * deadline is only checked after the call returns, so every attempt has reserved by the time it times out. + * + * @param list $log + */ +function orchestrationSlowCredit(array &$log, bool $optionalCredit = false, int $maxRetries = 1): TccDefinition +{ + $call = static function (string $name, string $id, bool $slow = false) use (&$log): Closure { + return static function (mixed $implicit) use ($name, $id, $slow, &$log): string { + $log[] = "{$name} {$id}".($name === 'cancel' ? ' ('.(is_string($implicit) ? $implicit : 'null').')' : ''); + if ($slow) { + usleep(30_000); + } + + return "{$id}-{$name}"; + }; + }; + + return TccBuilder::tcc('transfer')->retry(true, $maxRetries, 0) + ->participant('debit')->order(1)->tryWith($call('try', 'debit'))->confirmWith($call('confirm', 'debit'))->cancelWith($call('cancel', 'debit'))->add() + ->participant('credit')->order(2)->optional($optionalCredit)->tryWith($call('try', 'credit', slow: true), timeoutMs: 10)->confirmWith($call('confirm', 'credit'))->cancelWith($call('cancel', 'credit'))->add() + ->build(); +} + +it('cancels a required participant whose try timed out after reserving, with a null try result', function (): void { + $engines = TestEngines::create(); + $log = []; + + $result = $engines->tccs->execute(orchestrationSlowCredit($log), correlationId: 't-timeout'); + + expect($result->isCanceled())->toBeTrue() + ->and($result->failedParticipantId)->toBe('credit') + ->and($result->failedPhase)->toBe(TccPhase::Try) + // The first attempt and its retry both reserved; the cancel releases what credit holds, then debit's. + ->and($log)->toBe(['try debit', 'try credit', 'try credit', 'cancel credit (null)', 'cancel debit (debit-try)']) + ->and($result->participants['credit']->status)->toBe(StepStatus::Canceled) + ->and($result->participants['debit']->status)->toBe(StepStatus::Canceled) + ->and($engines->persistence->findById('t-timeout')?->status)->toBe(ExecutionStatus::Canceled); +}); + +it('cancels an optional participant whose try timed out once the others confirm, and stays CONFIRMED', function (): void { + $engines = TestEngines::create(); + $log = []; + + $result = $engines->tccs->execute(orchestrationSlowCredit($log, optionalCredit: true, maxRetries: 0), correlationId: 't-opt-timeout'); + + expect($result->isConfirmed())->toBeTrue() + ->and($log)->toBe(['try debit', 'try credit', 'confirm debit', 'cancel credit (null)']) + ->and($result->participants['debit']->status)->toBe(StepStatus::Confirmed) + ->and($result->participants['credit']->status)->toBe(StepStatus::Canceled) + ->and($engines->persistence->findById('t-opt-timeout')?->status)->toBe(ExecutionStatus::Confirmed); +}); diff --git a/packages/orchestration/tests/Tcc/TccResumeTest.php b/packages/orchestration/tests/Tcc/TccResumeTest.php index 2e730930..6c5d220b 100644 --- a/packages/orchestration/tests/Tcc/TccResumeTest.php +++ b/packages/orchestration/tests/Tcc/TccResumeTest.php @@ -385,3 +385,72 @@ expect($engines->tccs->execute('transfer', correlationId: 't-18')->status)->toBe(ExecutionStatus::Failed) ->and(fn () => $engines->tccs->compensate('t-18'))->toThrow(InvalidExecutionStateException::class, 'already FAILED'); }); + +it('cancels a participant left RUNNING by a crash mid-try when a TRYING TCC is compensated', function (): void { + $engines = TestEngines::create(); + $log = []; + $phase = static function (string $name, string $id) use (&$log): Closure { + return static function (mixed $implicit) use ($name, $id, &$log): string { + $log[] = "{$name} {$id} (".(is_string($implicit) ? $implicit : 'null').')'; + + return $id; + }; + }; + $engines->tccRegistry->register(TccBuilder::tcc('transfer') + ->participant('debit')->order(1)->tryWith($phase('try', 'debit'))->confirmWith($phase('confirm', 'debit'))->cancelWith($phase('cancel', 'debit'))->add() + ->participant('credit')->order(2)->tryWith($phase('try', 'credit'))->confirmWith($phase('confirm', 'credit'))->cancelWith($phase('cancel', 'credit'))->add() + ->participant('notify')->order(3)->tryWith($phase('try', 'notify'))->confirmWith($phase('confirm', 'notify'))->cancelWith($phase('cancel', 'notify'))->add() + ->build()); + // The process died while credit's try was running: it may have reserved, notify was never reached. + $engines->persistence->save(ExecutionState::fromArray([ + 'correlationId' => 't-20', 'executionName' => 'transfer', 'pattern' => 'TCC', 'status' => 'TRYING', + 'startedAt' => '2026-10-12T09:00:00.000000+00:00', 'updatedAt' => '2026-10-12T09:00:01.000000+00:00', + 'currentPhase' => 'TRY', + 'stepStatuses' => ['debit' => 'DONE', 'credit' => 'RUNNING'], + 'stepInputs' => ['debit' => 'acc-1', 'credit' => 'acc-2'], + 'tryResults' => ['debit' => 'debit-hold'], + 'completionOrder' => ['debit'], + ])); + + $result = $engines->tccs->compensate('t-20'); + + expect($result->isCanceled())->toBeTrue() + ->and($log)->toBe(['cancel credit (null)', 'cancel debit (debit-hold)']) + ->and($result->participants['credit']->status)->toBe(StepStatus::Canceled) + ->and($result->participants['notify']->status)->toBe(StepStatus::Pending); +}); + +it('resumes a cancel whose timed-out required participant was already cancelled as CANCELED with its try error', function (): void { + $engines = TestEngines::create(); + $log = []; + $phase = static function (string $name, string $id) use (&$log): Closure { + return static function () use ($name, $id, &$log): string { + $log[] = "{$name} {$id}"; + + return $id; + }; + }; + $engines->tccRegistry->register(TccBuilder::tcc('transfer') + ->participant('debit')->order(1)->tryWith($phase('try', 'debit'))->confirmWith($phase('confirm', 'debit'))->cancelWith($phase('cancel', 'debit'))->add() + ->participant('credit')->order(2)->tryWith($phase('try', 'credit'))->confirmWith($phase('confirm', 'credit'))->cancelWith($phase('cancel', 'credit'))->add() + ->build()); + // credit's try timed out and the sweep already cancelled it (no try result: it never reported one); the + // crash came before debit's cancel. The resume must report the try failure, not a confirm failure. + $engines->persistence->save(ExecutionState::fromArray([ + 'correlationId' => 't-21', 'executionName' => 'transfer', 'pattern' => 'TCC', 'status' => 'CANCELING', + 'startedAt' => '2026-10-12T09:00:00.000000+00:00', 'updatedAt' => '2026-10-12T09:00:01.000000+00:00', + 'currentPhase' => 'CANCEL', + 'stepStatuses' => ['debit' => 'DONE', 'credit' => 'CANCELED'], + 'stepErrors' => ['credit' => 'credit try timed out'], + 'tryResults' => ['debit' => 'debit'], + 'completionOrder' => ['debit'], + ])); + + $result = $engines->tccs->resume('t-21'); + + expect($result->status)->toBe(ExecutionStatus::Canceled) + ->and($result->failedParticipantId)->toBe('credit') + ->and($result->failedPhase)->toBe(TccPhase::Try) + ->and($result->error?->getMessage())->toBe('credit try timed out') + ->and($log)->toBe(['cancel debit']); +}); From 2e8dd1e530516d4a7a6f7d1cbbd557a2dfa12e71 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 20:32:14 -0700 Subject: [PATCH 24/58] feat(orchestration): workflow attributes, validated definitions with waits, children and queries, and the builder DSL --- .../src/Workflow/Attributes/ChildWorkflow.php | 23 ++ .../Workflow/Attributes/CompensationStep.php | 14 ++ .../Workflow/Attributes/OnStepComplete.php | 21 ++ .../Attributes/OnWorkflowComplete.php | 17 ++ .../Workflow/Attributes/OnWorkflowError.php | 27 +++ .../Workflow/Attributes/ScheduledWorkflow.php | 23 ++ .../src/Workflow/Attributes/WaitForAll.php | 21 ++ .../src/Workflow/Attributes/WaitForAny.php | 24 ++ .../src/Workflow/Attributes/WaitForSignal.php | 21 ++ .../src/Workflow/Attributes/WaitForTimer.php | 21 ++ .../src/Workflow/Attributes/Workflow.php | 34 +++ .../src/Workflow/Attributes/WorkflowQuery.php | 18 ++ .../src/Workflow/Attributes/WorkflowStep.php | 36 +++ .../src/Workflow/Builder/WorkflowBuilder.php | 205 ++++++++++++++++++ .../Workflow/Builder/WorkflowStepBuilder.php | 199 +++++++++++++++++ .../src/Workflow/Engine/ContinueAsNew.php | 18 ++ .../Registry/ChildWorkflowDefinition.php | 15 ++ .../src/Workflow/Registry/SignalWait.php | 14 ++ .../src/Workflow/Registry/TimerWait.php | 14 ++ .../src/Workflow/Registry/WaitDefinition.php | 77 +++++++ .../Workflow/Registry/WorkflowDefinition.php | 140 ++++++++++++ .../Registry/WorkflowQueryDefinition.php | 17 ++ .../Workflow/Registry/WorkflowRegistry.php | 46 ++++ .../Registry/WorkflowStepDefinition.php | 70 ++++++ .../tests/Workflow/WorkflowBuilderTest.php | 60 +++++ .../tests/Workflow/WorkflowDefinitionTest.php | 78 +++++++ 26 files changed, 1253 insertions(+) create mode 100644 packages/orchestration/src/Workflow/Attributes/ChildWorkflow.php create mode 100644 packages/orchestration/src/Workflow/Attributes/CompensationStep.php create mode 100644 packages/orchestration/src/Workflow/Attributes/OnStepComplete.php create mode 100644 packages/orchestration/src/Workflow/Attributes/OnWorkflowComplete.php create mode 100644 packages/orchestration/src/Workflow/Attributes/OnWorkflowError.php create mode 100644 packages/orchestration/src/Workflow/Attributes/ScheduledWorkflow.php create mode 100644 packages/orchestration/src/Workflow/Attributes/WaitForAll.php create mode 100644 packages/orchestration/src/Workflow/Attributes/WaitForAny.php create mode 100644 packages/orchestration/src/Workflow/Attributes/WaitForSignal.php create mode 100644 packages/orchestration/src/Workflow/Attributes/WaitForTimer.php create mode 100644 packages/orchestration/src/Workflow/Attributes/Workflow.php create mode 100644 packages/orchestration/src/Workflow/Attributes/WorkflowQuery.php create mode 100644 packages/orchestration/src/Workflow/Attributes/WorkflowStep.php create mode 100644 packages/orchestration/src/Workflow/Builder/WorkflowBuilder.php create mode 100644 packages/orchestration/src/Workflow/Builder/WorkflowStepBuilder.php create mode 100644 packages/orchestration/src/Workflow/Engine/ContinueAsNew.php create mode 100644 packages/orchestration/src/Workflow/Registry/ChildWorkflowDefinition.php create mode 100644 packages/orchestration/src/Workflow/Registry/SignalWait.php create mode 100644 packages/orchestration/src/Workflow/Registry/TimerWait.php create mode 100644 packages/orchestration/src/Workflow/Registry/WaitDefinition.php create mode 100644 packages/orchestration/src/Workflow/Registry/WorkflowDefinition.php create mode 100644 packages/orchestration/src/Workflow/Registry/WorkflowQueryDefinition.php create mode 100644 packages/orchestration/src/Workflow/Registry/WorkflowRegistry.php create mode 100644 packages/orchestration/src/Workflow/Registry/WorkflowStepDefinition.php create mode 100644 packages/orchestration/tests/Workflow/WorkflowBuilderTest.php create mode 100644 packages/orchestration/tests/Workflow/WorkflowDefinitionTest.php diff --git a/packages/orchestration/src/Workflow/Attributes/ChildWorkflow.php b/packages/orchestration/src/Workflow/Attributes/ChildWorkflow.php new file mode 100644 index 00000000..f3f36308 --- /dev/null +++ b/packages/orchestration/src/Workflow/Attributes/ChildWorkflow.php @@ -0,0 +1,23 @@ + $stepIds + */ + public function __construct( + public array $stepIds = [], + public bool $async = false, + public int $priority = 0, + ) {} +} diff --git a/packages/orchestration/src/Workflow/Attributes/OnWorkflowComplete.php b/packages/orchestration/src/Workflow/Attributes/OnWorkflowComplete.php new file mode 100644 index 00000000..ad169792 --- /dev/null +++ b/packages/orchestration/src/Workflow/Attributes/OnWorkflowComplete.php @@ -0,0 +1,17 @@ + $errorTypes exception class names; empty matches every error + * @param list $stepIds + */ + public function __construct( + public array $errorTypes = [], + public array $stepIds = [], + public bool $async = false, + public int $priority = 0, + public bool $suppressError = false, + ) {} +} diff --git a/packages/orchestration/src/Workflow/Attributes/ScheduledWorkflow.php b/packages/orchestration/src/Workflow/Attributes/ScheduledWorkflow.php new file mode 100644 index 00000000..4afa2704 --- /dev/null +++ b/packages/orchestration/src/Workflow/Attributes/ScheduledWorkflow.php @@ -0,0 +1,23 @@ + $signals + * @param list $timers + */ + public function __construct( + public array $signals = [], + public array $timers = [], + ) {} +} diff --git a/packages/orchestration/src/Workflow/Attributes/WaitForAny.php b/packages/orchestration/src/Workflow/Attributes/WaitForAny.php new file mode 100644 index 00000000..5ea3aa22 --- /dev/null +++ b/packages/orchestration/src/Workflow/Attributes/WaitForAny.php @@ -0,0 +1,24 @@ + $signals + * @param list $timers + */ + public function __construct( + public array $signals = [], + public array $timers = [], + ) {} +} diff --git a/packages/orchestration/src/Workflow/Attributes/WaitForSignal.php b/packages/orchestration/src/Workflow/Attributes/WaitForSignal.php new file mode 100644 index 00000000..2a0e658b --- /dev/null +++ b/packages/orchestration/src/Workflow/Attributes/WaitForSignal.php @@ -0,0 +1,21 @@ +`. A signal sent before the step is reached is buffered. With + * `timeoutMs`, the step FAILS (SignalTimeoutException) if the signal has not arrived in time. + */ +#[Attribute(Attribute::TARGET_METHOD)] +final readonly class WaitForSignal +{ + public function __construct( + public string $name, + public int $timeoutMs = 0, + ) {} +} diff --git a/packages/orchestration/src/Workflow/Attributes/WaitForTimer.php b/packages/orchestration/src/Workflow/Attributes/WaitForTimer.php new file mode 100644 index 00000000..d331202b --- /dev/null +++ b/packages/orchestration/src/Workflow/Attributes/WaitForTimer.php @@ -0,0 +1,21 @@ +:timer:`. + */ +#[Attribute(Attribute::TARGET_METHOD)] +final readonly class WaitForTimer +{ + public function __construct( + public int $delayMs, + public string $timerId = '', + ) {} +} diff --git a/packages/orchestration/src/Workflow/Attributes/Workflow.php b/packages/orchestration/src/Workflow/Attributes/Workflow.php new file mode 100644 index 00000000..9c6f30a4 --- /dev/null +++ b/packages/orchestration/src/Workflow/Attributes/Workflow.php @@ -0,0 +1,34 @@ + $dependsOn + */ + public function __construct( + public string $id = '', + public string $name = '', + public string $description = '', + public array $dependsOn = [], + public string $outputEventType = '', + public int $timeoutMs = 0, + public int $maxRetries = -1, + public int $retryDelayMs = -1, + public string $condition = '', + public bool $async = false, + public bool $compensatable = false, + public string $compensationMethod = '', + ) {} +} diff --git a/packages/orchestration/src/Workflow/Builder/WorkflowBuilder.php b/packages/orchestration/src/Workflow/Builder/WorkflowBuilder.php new file mode 100644 index 00000000..6afa747e --- /dev/null +++ b/packages/orchestration/src/Workflow/Builder/WorkflowBuilder.php @@ -0,0 +1,205 @@ + */ + private array $steps = []; + + private string $name = ''; + + private string $description = ''; + + private string $version = '1.0'; + + private TriggerMode $triggerMode = TriggerMode::Sync; + + private string $triggerEventType = ''; + + private int $timeoutMs = 0; + + private int $maxRetries = 0; + + private int $retryDelayMs = 1000; + + private bool $publishEvents = false; + + /** @var array */ + private array $queries = []; + + /** @var list */ + private array $onStepComplete = []; + + /** @var list */ + private array $onComplete = []; + + /** @var list */ + private array $onError = []; + + private function __construct(private readonly string $id) {} + + public static function workflow(string $id): self + { + return new self($id); + } + + public function name(string $name): self + { + $this->name = $name; + + return $this; + } + + public function description(string $description): self + { + $this->description = $description; + + return $this; + } + + public function version(string $version): self + { + $this->version = $version; + + return $this; + } + + public function triggerMode(TriggerMode $mode): self + { + $this->triggerMode = $mode; + + return $this; + } + + public function triggerEventType(string $eventType): self + { + $this->triggerEventType = $eventType; + + return $this; + } + + public function timeoutMs(int $milliseconds): self + { + $this->timeoutMs = $milliseconds; + + return $this; + } + + public function retries(int $maxRetries, int $retryDelayMs = 1000): self + { + $this->maxRetries = $maxRetries; + $this->retryDelayMs = $retryDelayMs; + + return $this; + } + + public function publishEvents(bool $publish = true): self + { + $this->publishEvents = $publish; + + return $this; + } + + public function step(string $id): WorkflowStepBuilder + { + return new WorkflowStepBuilder($this, $this->id, $id); + } + + /** + * @param Closure(ExecutionContext, mixed ...): mixed $query + */ + public function query(string $name, Closure $query): self + { + $this->queries[$name] = new WorkflowQueryDefinition($name, new ClosureInvocation($query, "{$this->id}?{$name}")); + + return $this; + } + + /** + * @param Closure(ExecutionContext, mixed): void $callback + * @param list $stepIds + */ + public function onStepComplete(Closure $callback, array $stepIds = [], int $priority = 0, bool $async = false): self + { + $this->onStepComplete[] = CallbackDefinition::closure($callback, $async, $priority, stepIds: $stepIds); + + return $this; + } + + /** + * @param Closure(ExecutionContext, mixed): void $callback + */ + public function onComplete(Closure $callback, int $priority = 0, bool $async = false): self + { + $this->onComplete[] = CallbackDefinition::closure($callback, $async, $priority); + + return $this; + } + + /** + * @param Closure(ExecutionContext, mixed): void $callback + * @param list $errorTypes + * @param list $stepIds + */ + public function onError(Closure $callback, bool $suppressError = false, array $errorTypes = [], array $stepIds = [], int $priority = 0, bool $async = false): self + { + $this->onError[] = CallbackDefinition::closure($callback, $async, $priority, $errorTypes, $suppressError, $stepIds); + + return $this; + } + + /** @internal the workflow-level retries a step with maxRetries -1 inherits */ + public function inheritedRetries(): int + { + return $this->maxRetries; + } + + /** @internal the workflow-level retry delay a step with retryDelayMs -1 inherits */ + public function inheritedRetryDelayMs(): int + { + return $this->retryDelayMs; + } + + /** + * @internal called by WorkflowStepBuilder::add(). A second step with an id already added refuses, rather + * than silently replacing the first. + */ + public function addStep(WorkflowStepDefinition $step): self + { + if (isset($this->steps[$step->id])) { + throw new TopologyValidationException("Workflow [{$this->id}] declares step [{$step->id}] twice."); + } + + $this->steps[$step->id] = $step; + + return $this; + } + + public function build(): WorkflowDefinition + { + return new WorkflowDefinition( + $this->id, $this->steps, $this->name, $this->description, $this->version, $this->triggerMode, + $this->triggerEventType, $this->timeoutMs, $this->maxRetries, $this->retryDelayMs, $this->publishEvents, 0, + $this->queries, $this->onStepComplete, $this->onComplete, $this->onError, + ); + } +} diff --git a/packages/orchestration/src/Workflow/Builder/WorkflowStepBuilder.php b/packages/orchestration/src/Workflow/Builder/WorkflowStepBuilder.php new file mode 100644 index 00000000..924e6b92 --- /dev/null +++ b/packages/orchestration/src/Workflow/Builder/WorkflowStepBuilder.php @@ -0,0 +1,199 @@ + */ + private array $dependsOn = []; + + private string $name = ''; + + private string $description = ''; + + private string $condition = ''; + + private int $timeoutMs = 0; + + private int $maxRetries = -1; + + private int $retryDelayMs = -1; + + private string $outputEventType = ''; + + private ?WaitDefinition $wait = null; + + private ?ChildWorkflowDefinition $child = null; + + private ?Closure $handler = null; + + private ?Closure $compensation = null; + + private ?string $variable = null; + + public function __construct( + private readonly WorkflowBuilder $workflow, + private readonly string $workflowId, + private readonly string $id, + ) {} + + public function name(string $name): self + { + $this->name = $name; + + return $this; + } + + public function description(string $description): self + { + $this->description = $description; + + return $this; + } + + public function dependsOn(string ...$ids): self + { + $this->dependsOn = array_values([...$this->dependsOn, ...$ids]); + + return $this; + } + + public function condition(string $expression): self + { + $this->condition = $expression; + + return $this; + } + + public function timeoutMs(int $milliseconds): self + { + $this->timeoutMs = $milliseconds; + + return $this; + } + + public function retries(int $maxRetries, int $retryDelayMs = -1): self + { + $this->maxRetries = $maxRetries; + $this->retryDelayMs = $retryDelayMs; + + return $this; + } + + public function outputEventType(string $eventType): self + { + $this->outputEventType = $eventType; + + return $this; + } + + public function waitForSignal(string $name, int $timeoutMs = 0): self + { + $this->wait = WaitDefinition::signal($name, $timeoutMs); + + return $this; + } + + public function waitForTimer(int $delayMs, string $timerId = ''): self + { + $this->wait = WaitDefinition::timer($delayMs, $timerId === '' ? "{$this->id}:timer:0" : $timerId); + + return $this; + } + + /** + * @param list $signals + * @param list $timers + */ + public function waitForAll(array $signals, array $timers = []): self + { + $this->wait = WaitDefinition::all($signals, $timers); + + return $this; + } + + /** + * @param list $signals + * @param list $timers + */ + public function waitForAny(array $signals, array $timers = []): self + { + $this->wait = WaitDefinition::any($signals, $timers); + + return $this; + } + + public function childWorkflow(string $workflowId, bool $waitForCompletion = true, int $timeoutMs = 0): self + { + $this->child = new ChildWorkflowDefinition($workflowId, $waitForCompletion, $timeoutMs); + + return $this; + } + + public function handler(Closure $handler): self + { + $this->handler = $handler; + + return $this; + } + + public function compensation(Closure $compensation): self + { + $this->compensation = $compensation; + + return $this; + } + + public function setVariable(string $name): self + { + $this->variable = $name; + + return $this; + } + + public function add(): WorkflowBuilder + { + $handler = $this->handler; + if ($handler === null) { + if ($this->wait === null && $this->child === null) { + throw new TopologyValidationException("Workflow [{$this->workflowId}] step [{$this->id}] has no handler."); + } + $handler = static fn (mixed $input): mixed => $input; + } + + return $this->workflow->addStep(new WorkflowStepDefinition( + id: $this->id, + invocation: new ClosureInvocation($handler, "{$this->workflowId}.{$this->id}", $this->variable), + dependsOn: $this->dependsOn, + name: $this->name, + description: $this->description, + outputEventType: $this->outputEventType, + options: StepOptions::of( + $this->maxRetries >= 0 ? $this->maxRetries : $this->workflow->inheritedRetries(), + $this->retryDelayMs >= 0 ? $this->retryDelayMs : $this->workflow->inheritedRetryDelayMs(), + $this->timeoutMs, + ), + condition: $this->condition, + compensatable: $this->compensation !== null, + compensation: $this->compensation === null ? null : new ClosureInvocation($this->compensation, "{$this->workflowId}.{$this->id}:compensate"), + wait: $this->wait, + child: $this->child, + )); + } +} diff --git a/packages/orchestration/src/Workflow/Engine/ContinueAsNew.php b/packages/orchestration/src/Workflow/Engine/ContinueAsNew.php new file mode 100644 index 00000000..5d673639 --- /dev/null +++ b/packages/orchestration/src/Workflow/Engine/ContinueAsNew.php @@ -0,0 +1,18 @@ + $input + */ + public function __construct(public array $input = []) {} +} diff --git a/packages/orchestration/src/Workflow/Registry/ChildWorkflowDefinition.php b/packages/orchestration/src/Workflow/Registry/ChildWorkflowDefinition.php new file mode 100644 index 00000000..caf6ff5a --- /dev/null +++ b/packages/orchestration/src/Workflow/Registry/ChildWorkflowDefinition.php @@ -0,0 +1,15 @@ + $signals + * @param list $timers + */ + public function __construct( + public string $kind, + public array $signals = [], + public array $timers = [], + ) {} + + public static function signal(string $name, int $timeoutMs = 0): self + { + return new self(self::SIGNAL, [new SignalWait($name, $timeoutMs)]); + } + + public static function timer(int $delayMs, string $id): self + { + return new self(self::TIMER, [], [new TimerWait($id, $delayMs)]); + } + + /** + * @param list $signals + * @param list $timers + */ + public static function all(array $signals, array $timers = []): self + { + return new self(self::ALL, $signals, $timers); + } + + /** + * @param list $signals + * @param list $timers + */ + public static function any(array $signals, array $timers = []): self + { + return new self(self::ANY, $signals, $timers); + } + + /** @return array */ + public function toArray(): array + { + return [ + 'kind' => $this->kind, + 'signals' => array_map(static fn (SignalWait $signal): array => ['name' => $signal->name, 'timeoutMs' => $signal->timeoutMs], $this->signals), + 'timers' => array_map(static fn (TimerWait $timer): array => ['id' => $timer->id, 'delayMs' => $timer->delayMs], $this->timers), + ]; + } + + /** @param array $row */ + public static function fromArray(array $row): self + { + return new self( + Rows::string($row, 'kind', self::SIGNAL), + array_map(static fn (array $signal): SignalWait => new SignalWait(Rows::string($signal, 'name'), Rows::int($signal, 'timeoutMs')), Rows::rows($row, 'signals')), + array_map(static fn (array $timer): TimerWait => new TimerWait(Rows::string($timer, 'id'), Rows::int($timer, 'delayMs')), Rows::rows($row, 'timers')), + ); + } +} diff --git a/packages/orchestration/src/Workflow/Registry/WorkflowDefinition.php b/packages/orchestration/src/Workflow/Registry/WorkflowDefinition.php new file mode 100644 index 00000000..24ead912 --- /dev/null +++ b/packages/orchestration/src/Workflow/Registry/WorkflowDefinition.php @@ -0,0 +1,140 @@ +> */ + public array $layers; + + /** + * @param array $steps + * @param array $queries + * @param list $onStepComplete + * @param list $onComplete + * @param list $onError + */ + public function __construct( + public string $id, + public array $steps, + public string $name = '', + public string $description = '', + public string $version = '1.0', + public TriggerMode $triggerMode = TriggerMode::Sync, + public string $triggerEventType = '', + public int $timeoutMs = 0, + public int $maxRetries = 0, + public int $retryDelayMs = 1000, + public bool $publishEvents = false, + public int $layerConcurrency = 0, + public array $queries = [], + public array $onStepComplete = [], + public array $onComplete = [], + public array $onError = [], + ) { + $dependencies = []; + foreach ($steps as $stepId => $step) { + if ((string) $stepId !== $step->id) { + throw new TopologyValidationException("Workflow [{$id}]: step [{$step->id}] is filed under key [{$stepId}]; a step must be keyed by its own id."); + } + $dependencies[(string) $stepId] = $step->dependsOn; + if ($step->compensatable && $step->compensation === null) { + throw new TopologyValidationException("Workflow [{$id}] step [{$step->id}] is compensatable but declares no compensation (compensationMethod or #[CompensationStep])."); + } + } + + $this->layers = TopologyBuilder::layers("Workflow [{$id}]", $dependencies); + } + + public function step(string $stepId): WorkflowStepDefinition + { + return $this->steps[$stepId] ?? throw new DefinitionNotFoundException("Workflow [{$this->id}] has no step [{$stepId}]."); + } + + /** @return list */ + public function stepIds(): array + { + return array_map('strval', array_keys($this->steps)); + } + + public function query(string $name): WorkflowQueryDefinition + { + return $this->queries[$name] ?? throw new DefinitionNotFoundException("Workflow [{$this->id}] declares no query [{$name}]."); + } + + /** @return array */ + public function toArray(): array + { + return [ + 'id' => $this->id, 'name' => $this->name, 'description' => $this->description, 'version' => $this->version, + 'triggerMode' => $this->triggerMode->value, 'triggerEventType' => $this->triggerEventType, 'timeoutMs' => $this->timeoutMs, + 'maxRetries' => $this->maxRetries, 'retryDelayMs' => $this->retryDelayMs, 'publishEvents' => $this->publishEvents, + 'layerConcurrency' => $this->layerConcurrency, + 'steps' => array_values(array_map(static fn (WorkflowStepDefinition $step): array => $step->toArray(), $this->steps)), + 'queries' => array_values(array_map(static fn (WorkflowQueryDefinition $query): array => ['name' => $query->name, 'invocation' => $query->invocation->toArray(), 'contextFirst' => $query->contextFirst], $this->queries)), + 'onStepComplete' => array_map(static fn (CallbackDefinition $callback): array => $callback->toArray(), $this->onStepComplete), + 'onComplete' => array_map(static fn (CallbackDefinition $callback): array => $callback->toArray(), $this->onComplete), + 'onError' => array_map(static fn (CallbackDefinition $callback): array => $callback->toArray(), $this->onError), + ]; + } + + /** @param array $row */ + public static function fromArray(array $row): self + { + $steps = []; + foreach (Rows::rows($row, 'steps') as $stepRow) { + $step = WorkflowStepDefinition::fromArray($stepRow); + $steps[$step->id] = $step; + } + + $queries = []; + foreach (Rows::rows($row, 'queries') as $queryRow) { + $name = Rows::string($queryRow, 'name'); + $queries[$name] = new WorkflowQueryDefinition($name, MethodInvocation::fromArray(Rows::map($queryRow, 'invocation')), Rows::bool($queryRow, 'contextFirst', true)); + } + + return new self( + id: Rows::string($row, 'id'), + steps: $steps, + name: Rows::string($row, 'name'), + description: Rows::string($row, 'description'), + version: Rows::string($row, 'version', '1.0'), + triggerMode: TriggerMode::tryFrom(Rows::string($row, 'triggerMode')) ?? TriggerMode::Sync, + triggerEventType: Rows::string($row, 'triggerEventType'), + timeoutMs: Rows::int($row, 'timeoutMs'), + maxRetries: Rows::int($row, 'maxRetries'), + retryDelayMs: Rows::int($row, 'retryDelayMs', 1000), + publishEvents: Rows::bool($row, 'publishEvents'), + layerConcurrency: Rows::int($row, 'layerConcurrency'), + queries: $queries, + onStepComplete: self::callbacks($row, 'onStepComplete'), + onComplete: self::callbacks($row, 'onComplete'), + onError: self::callbacks($row, 'onError'), + ); + } + + /** + * @param array $row + * @return list + */ + private static function callbacks(array $row, string $key): array + { + return array_map(static fn (array $callback): CallbackDefinition => CallbackDefinition::fromArray($callback), Rows::rows($row, $key)); + } +} diff --git a/packages/orchestration/src/Workflow/Registry/WorkflowQueryDefinition.php b/packages/orchestration/src/Workflow/Registry/WorkflowQueryDefinition.php new file mode 100644 index 00000000..7518fa19 --- /dev/null +++ b/packages/orchestration/src/Workflow/Registry/WorkflowQueryDefinition.php @@ -0,0 +1,17 @@ + */ + private array $workflows = []; + + public function register(WorkflowDefinition $workflow): void + { + if (isset($this->workflows[$workflow->id])) { + throw new TopologyValidationException("Workflow [{$workflow->id}] is defined twice."); + } + + $this->workflows[$workflow->id] = $workflow; + } + + public function get(string $id): WorkflowDefinition + { + return $this->workflows[$id] ?? throw new DefinitionNotFoundException("No workflow [{$id}] is registered."); + } + + public function has(string $id): bool + { + return isset($this->workflows[$id]); + } + + /** @return array */ + public function all(): array + { + return $this->workflows; + } + + /** @return list */ + public function ids(): array + { + return array_map('strval', array_keys($this->workflows)); + } +} diff --git a/packages/orchestration/src/Workflow/Registry/WorkflowStepDefinition.php b/packages/orchestration/src/Workflow/Registry/WorkflowStepDefinition.php new file mode 100644 index 00000000..d5f2f572 --- /dev/null +++ b/packages/orchestration/src/Workflow/Registry/WorkflowStepDefinition.php @@ -0,0 +1,70 @@ + $dependsOn + */ + public function __construct( + public string $id, + public Invocation $invocation, + public array $dependsOn = [], + public string $name = '', + public string $description = '', + public string $outputEventType = '', + public StepOptions $options = new StepOptions, + public string $condition = '', + public bool $async = false, + public bool $compensatable = false, + public ?Invocation $compensation = null, + public ?WaitDefinition $wait = null, + public ?ChildWorkflowDefinition $child = null, + ) {} + + /** @return array */ + public function toArray(): array + { + return [ + 'id' => $this->id, 'invocation' => $this->invocation->toArray(), 'dependsOn' => $this->dependsOn, + 'name' => $this->name, 'description' => $this->description, 'outputEventType' => $this->outputEventType, + 'options' => $this->options->toArray(), 'condition' => $this->condition, 'async' => $this->async, + 'compensatable' => $this->compensatable, 'compensation' => $this->compensation?->toArray(), + 'wait' => $this->wait?->toArray(), + 'child' => $this->child === null ? null : ['workflowId' => $this->child->workflowId, 'waitForCompletion' => $this->child->waitForCompletion, 'timeoutMs' => $this->child->timeoutMs], + ]; + } + + /** @param array $row */ + public static function fromArray(array $row): self + { + $compensation = Rows::nullableMap($row, 'compensation'); + $wait = Rows::nullableMap($row, 'wait'); + $child = Rows::nullableMap($row, 'child'); + + return new self( + id: Rows::string($row, 'id'), + invocation: MethodInvocation::fromArray(Rows::map($row, 'invocation')), + dependsOn: Rows::strings($row, 'dependsOn'), + name: Rows::string($row, 'name'), + description: Rows::string($row, 'description'), + outputEventType: Rows::string($row, 'outputEventType'), + options: StepOptions::fromArray(Rows::map($row, 'options')), + condition: Rows::string($row, 'condition'), + async: Rows::bool($row, 'async'), + compensatable: Rows::bool($row, 'compensatable'), + compensation: $compensation === null ? null : MethodInvocation::fromArray($compensation), + wait: $wait === null ? null : WaitDefinition::fromArray($wait), + child: $child === null ? null : new ChildWorkflowDefinition(Rows::string($child, 'workflowId'), Rows::bool($child, 'waitForCompletion', true), Rows::int($child, 'timeoutMs')), + ); + } +} diff --git a/packages/orchestration/tests/Workflow/WorkflowBuilderTest.php b/packages/orchestration/tests/Workflow/WorkflowBuilderTest.php new file mode 100644 index 00000000..37064c59 --- /dev/null +++ b/packages/orchestration/tests/Workflow/WorkflowBuilderTest.php @@ -0,0 +1,60 @@ +name('Onboarding')->triggerMode(TriggerMode::Async)->triggerEventType('users.registered')->timeoutMs(60_000)->retries(2, 50)->publishEvents() + ->step('create')->handler(static fn (array $input): string => 'account')->compensation(static fn (): null => null)->add() + ->step('verify')->dependsOn('create')->waitForSignal('email-verified', 3_600_000)->add() + ->step('welcome')->dependsOn('verify')->retries(0)->outputEventType('users.welcomed')->condition('input["send"] == true')->handler(static fn (): string => 'sent')->add() + ->query('progress', static fn (ExecutionContext $ctx): int => count($ctx->results())) + ->onComplete(static function (): void {}) + ->build(); + + expect($workflow->triggerMode)->toBe(TriggerMode::Async) + ->and($workflow->publishEvents)->toBeTrue() + ->and($workflow->step('create')->options)->toEqual(new StepOptions(2, 50)) + ->and($workflow->step('create')->compensatable)->toBeTrue() + ->and($workflow->step('welcome')->options)->toEqual(new StepOptions(0, 50)) + ->and($workflow->step('verify')->wait)->toEqual(new WaitDefinition(WaitDefinition::SIGNAL, [new SignalWait('email-verified', 3_600_000)])) + ->and($workflow->layers)->toBe([['create'], ['verify'], ['welcome']]) + ->and(array_keys($workflow->queries))->toBe(['progress']) + ->and($workflow->onComplete)->toHaveCount(1); +}); + +it('refuses a step id declared twice and a step with neither handler, wait nor child', function (): void { + $builder = WorkflowBuilder::workflow('dup')->step('a')->handler(static fn (): null => null)->add(); + + expect(fn () => $builder->step('a')->handler(static fn (): null => null)->add()) + ->toThrow(TopologyValidationException::class, 'Workflow [dup] declares step [a] twice.') + ->and(fn () => WorkflowBuilder::workflow('bare')->step('x')->add()) + ->toThrow(TopologyValidationException::class, 'Workflow [bare] step [x] has no handler.'); +}); + +it('passes the input through a wait-only or child-only step and names a default timer', function (): void { + $workflow = WorkflowBuilder::workflow('pass') + ->step('pause')->waitForTimer(500)->add() + ->step('spawn')->dependsOn('pause')->childWorkflow('child', false, 1_000)->setVariable('childId')->add() + ->build(); + + $pause = $workflow->step('pause'); + $spawn = $workflow->step('spawn'); + + expect($pause->wait)->toEqual(WaitDefinition::timer(500, 'pause:timer:0')) + ->and($pause->invocation)->toBeInstanceOf(ClosureInvocation::class) + ->and($pause->invocation instanceof ClosureInvocation ? ($pause->invocation->closure)(['k' => 1]) : null)->toBe(['k' => 1]) + ->and($spawn->child)->toEqual(new ChildWorkflowDefinition('child', false, 1_000)) + ->and($spawn->invocation->setVariable())->toBe('childId') + ->and($spawn->compensatable)->toBeFalse(); +}); diff --git a/packages/orchestration/tests/Workflow/WorkflowDefinitionTest.php b/packages/orchestration/tests/Workflow/WorkflowDefinitionTest.php new file mode 100644 index 00000000..4ba80cc6 --- /dev/null +++ b/packages/orchestration/tests/Workflow/WorkflowDefinitionTest.php @@ -0,0 +1,78 @@ + new MethodInvocation('App\\Approval', $name); + + return new WorkflowDefinition( + id: 'approval', + steps: [ + 'submit' => new WorkflowStepDefinition('submit', $method('submit'), outputEventType: 'approval.submitted', compensatable: true, compensation: $method('withdraw')), + 'approve' => new WorkflowStepDefinition('approve', $method('approve'), ['submit'], wait: WaitDefinition::any([new SignalWait('approved', 86_400_000)], [new TimerWait('escalate', 3_600_000)])), + 'archive' => new WorkflowStepDefinition('archive', $method('archive'), ['approve'], options: new StepOptions(retry: 2, backoffMs: 10), condition: 'variables["signals.approved"] != null'), + 'audit' => new WorkflowStepDefinition('audit', $method('audit'), ['submit'], child: new ChildWorkflowDefinition('audit-trail', true, 60_000)), + ], + name: 'Expense approval', + version: '2.0', + triggerMode: TriggerMode::Async, + triggerEventType: 'expenses.submitted', + timeoutMs: 604_800_000, + queries: ['status' => new WorkflowQueryDefinition('status', $method('status'))], + ); +} + +it('layers its steps and keeps waits, children, conditions and queries', function (): void { + $workflow = orchestrationApprovalDefinition(); + + expect($workflow->layers)->toBe([['submit'], ['approve', 'audit'], ['archive']]) + ->and($workflow->step('approve')->wait?->kind)->toBe(WaitDefinition::ANY) + ->and($workflow->step('audit')->child?->workflowId)->toBe('audit-trail') + ->and($workflow->query('status')->name)->toBe('status') + ->and(fn () => $workflow->query('nope'))->toThrow(DefinitionNotFoundException::class, 'Workflow [approval] declares no query [nope]') + ->and(WorkflowDefinition::fromArray($workflow->toArray()))->toEqual($workflow); +}); + +it('refuses a compensatable step without a compensation', function (): void { + new WorkflowDefinition('w', ['a' => new WorkflowStepDefinition('a', new ClosureInvocation(static fn (): null => null), compensatable: true)]); +})->throws(TopologyValidationException::class, 'Workflow [w] step [a] is compensatable but declares no compensation'); + +it('registers each workflow once', function (): void { + $registry = new WorkflowRegistry; + $registry->register(orchestrationApprovalDefinition()); + + expect($registry->get('approval')->version)->toBe('2.0') + ->and($registry->ids())->toBe(['approval']) + ->and(fn () => $registry->register(orchestrationApprovalDefinition()))->toThrow(TopologyValidationException::class, 'Workflow [approval] is defined twice') + ->and(fn () => $registry->get('ghost'))->toThrow(DefinitionNotFoundException::class); +}); + +it('builds every wait kind', function (): void { + expect(WaitDefinition::signal('go', 50))->toEqual(new WaitDefinition(WaitDefinition::SIGNAL, [new SignalWait('go', 50)])) + ->and(WaitDefinition::timer(10, 't'))->toEqual(new WaitDefinition(WaitDefinition::TIMER, [], [new TimerWait('t', 10)])) + ->and(WaitDefinition::fromArray(WaitDefinition::all([new SignalWait('a')], [new TimerWait('b', 5)])->toArray()))->toEqual(WaitDefinition::all([new SignalWait('a')], [new TimerWait('b', 5)])); +}); + +it('refuses a step filed under a key that is not its own id', function (): void { + new WorkflowDefinition('w', ['a' => new WorkflowStepDefinition('b', new ClosureInvocation(static fn (): null => null))]); +})->throws(TopologyValidationException::class, 'Workflow [w]: step [b] is filed under key [a]; a step must be keyed by its own id.'); + +it('refuses a dependency on a step it does not declare', function (): void { + new WorkflowDefinition('w', ['a' => new WorkflowStepDefinition('a', new ClosureInvocation(static fn (): null => null), ['ghost'])]); +})->throws(TopologyValidationException::class, 'Workflow [w]: step [a] depends on [ghost], which is not a step of it.'); From c61a49ee8e869fa3a4932ee77579b00226f519df Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 20:34:50 -0700 Subject: [PATCH 25/58] fix(orchestration): validate every workflow step's wait at definition time and refuse a manifest wait without a known kind --- .../src/Workflow/Registry/WaitDefinition.php | 83 ++++++++++++++++++- .../Workflow/Registry/WorkflowDefinition.php | 4 +- .../tests/Workflow/WorkflowBuilderTest.php | 4 + .../tests/Workflow/WorkflowDefinitionTest.php | 30 +++++++ 4 files changed, 117 insertions(+), 4 deletions(-) diff --git a/packages/orchestration/src/Workflow/Registry/WaitDefinition.php b/packages/orchestration/src/Workflow/Registry/WaitDefinition.php index 3238c258..8d119ff9 100644 --- a/packages/orchestration/src/Workflow/Registry/WaitDefinition.php +++ b/packages/orchestration/src/Workflow/Registry/WaitDefinition.php @@ -4,9 +4,16 @@ namespace Firefly\Orchestration\Workflow\Registry; +use Firefly\Orchestration\Core\Exception\TopologyValidationException; use Firefly\Orchestration\Core\Rows; -/** What a step waits for before it runs: one signal, one timer, all of a set, or any of a set. */ +/** + * What a step waits for before it runs: one signal, one timer, all of a set, or any of a set. The value object + * itself accepts anything so a compiled manifest and the builder share one gate: WorkflowDefinition calls + * validate() for every step, which refuses an unknown kind, a SIGNAL or TIMER wait without exactly its one + * member, an empty ALL or ANY set (an empty ANY suspends forever and an empty ALL resolves at once), a negative + * delay or timeout, a blank or duplicate signal name or timer id. + */ final readonly class WaitDefinition { public const string SIGNAL = 'signal'; @@ -27,6 +34,64 @@ public function __construct( public array $timers = [], ) {} + /** @return list */ + public static function kinds(): array + { + return [self::SIGNAL, self::TIMER, self::ALL, self::ANY]; + } + + /** + * Refuses a wait the engine could not honor. `$where` names the owner, as in "Workflow [w] step [s]". + * + * @throws TopologyValidationException + */ + public function validate(string $where): void + { + if (! in_array($this->kind, self::kinds(), true)) { + throw new TopologyValidationException("{$where} waits with an unknown kind [{$this->kind}]; expected one of [".implode(', ', self::kinds()).'].'); + } + + $signals = count($this->signals); + $timers = count($this->timers); + if ($this->kind === self::SIGNAL && ($signals !== 1 || $timers !== 0)) { + throw new TopologyValidationException("{$where} has a signal wait that must name exactly one signal and no timers (it has {$signals} signals and {$timers} timers)."); + } + if ($this->kind === self::TIMER && ($timers !== 1 || $signals !== 0)) { + throw new TopologyValidationException("{$where} has a timer wait that must name exactly one timer and no signals (it has {$timers} timers and {$signals} signals)."); + } + if (($this->kind === self::ALL || $this->kind === self::ANY) && $signals + $timers === 0) { + throw new TopologyValidationException("{$where} has a wait for {$this->kind} with no signals and no timers."); + } + + $names = []; + foreach ($this->signals as $signal) { + if ($signal->name === '') { + throw new TopologyValidationException("{$where} waits for a signal with a blank name."); + } + if ($signal->timeoutMs < 0) { + throw new TopologyValidationException("{$where} waits for signal [{$signal->name}] with a negative timeout ({$signal->timeoutMs} ms)."); + } + if (isset($names[$signal->name])) { + throw new TopologyValidationException("{$where} waits for signal [{$signal->name}] twice."); + } + $names[$signal->name] = true; + } + + $ids = []; + foreach ($this->timers as $timer) { + if ($timer->id === '') { + throw new TopologyValidationException("{$where} waits for a timer with a blank id."); + } + if ($timer->delayMs < 0) { + throw new TopologyValidationException("{$where} waits for timer [{$timer->id}] with a negative delay ({$timer->delayMs} ms)."); + } + if (isset($ids[$timer->id])) { + throw new TopologyValidationException("{$where} waits for timer [{$timer->id}] twice."); + } + $ids[$timer->id] = true; + } + } + public static function signal(string $name, int $timeoutMs = 0): self { return new self(self::SIGNAL, [new SignalWait($name, $timeoutMs)]); @@ -65,11 +130,23 @@ public function toArray(): array ]; } - /** @param array $row */ + /** + * Rebuilds a wait from its array form. The kind has no default: a manifest row with a missing or unknown kind + * is refused here rather than silently read as a signal wait. + * + * @param array $row + * + * @throws TopologyValidationException + */ public static function fromArray(array $row): self { + $kind = Rows::string($row, 'kind'); + if (! in_array($kind, self::kinds(), true)) { + throw new TopologyValidationException('A workflow wait has '.($kind === '' ? 'no kind' : "an unknown kind [{$kind}]").'; expected one of ['.implode(', ', self::kinds()).'].'); + } + return new self( - Rows::string($row, 'kind', self::SIGNAL), + $kind, array_map(static fn (array $signal): SignalWait => new SignalWait(Rows::string($signal, 'name'), Rows::int($signal, 'timeoutMs')), Rows::rows($row, 'signals')), array_map(static fn (array $timer): TimerWait => new TimerWait(Rows::string($timer, 'id'), Rows::int($timer, 'delayMs')), Rows::rows($row, 'timers')), ); diff --git a/packages/orchestration/src/Workflow/Registry/WorkflowDefinition.php b/packages/orchestration/src/Workflow/Registry/WorkflowDefinition.php index 24ead912..eb885760 100644 --- a/packages/orchestration/src/Workflow/Registry/WorkflowDefinition.php +++ b/packages/orchestration/src/Workflow/Registry/WorkflowDefinition.php @@ -16,7 +16,8 @@ * A durable workflow (jafly WorkflowDefinition). Layers are computed and the definition validated at * construction: the topology, every step keyed by its own id (so the array form round-trips unchanged), and * every `compensatable` step having a compensation (a "missing compensation" the boot must refuse rather than - * discover on the first failure). + * discover on the first failure), and every step's wait well formed (WaitDefinition::validate(); the builder + * skips the attribute scanner, so this is the one gate every path goes through). */ final readonly class WorkflowDefinition { @@ -57,6 +58,7 @@ public function __construct( if ($step->compensatable && $step->compensation === null) { throw new TopologyValidationException("Workflow [{$id}] step [{$step->id}] is compensatable but declares no compensation (compensationMethod or #[CompensationStep])."); } + $step->wait?->validate("Workflow [{$id}] step [{$step->id}]"); } $this->layers = TopologyBuilder::layers("Workflow [{$id}]", $dependencies); diff --git a/packages/orchestration/tests/Workflow/WorkflowBuilderTest.php b/packages/orchestration/tests/Workflow/WorkflowBuilderTest.php index 37064c59..65c9a9e9 100644 --- a/packages/orchestration/tests/Workflow/WorkflowBuilderTest.php +++ b/packages/orchestration/tests/Workflow/WorkflowBuilderTest.php @@ -58,3 +58,7 @@ ->and($spawn->invocation->setVariable())->toBe('childId') ->and($spawn->compensatable)->toBeFalse(); }); + +it('refuses an empty wait-for-any built through the DSL', function (): void { + WorkflowBuilder::workflow('stuck')->step('hold')->waitForAny([])->add()->build(); +})->throws(TopologyValidationException::class, 'Workflow [stuck] step [hold] has a wait for any with no signals and no timers.'); diff --git a/packages/orchestration/tests/Workflow/WorkflowDefinitionTest.php b/packages/orchestration/tests/Workflow/WorkflowDefinitionTest.php index 4ba80cc6..21e29343 100644 --- a/packages/orchestration/tests/Workflow/WorkflowDefinitionTest.php +++ b/packages/orchestration/tests/Workflow/WorkflowDefinitionTest.php @@ -76,3 +76,33 @@ function orchestrationApprovalDefinition(): WorkflowDefinition it('refuses a dependency on a step it does not declare', function (): void { new WorkflowDefinition('w', ['a' => new WorkflowStepDefinition('a', new ClosureInvocation(static fn (): null => null), ['ghost'])]); })->throws(TopologyValidationException::class, 'Workflow [w]: step [a] depends on [ghost], which is not a step of it.'); + +function orchestrationWaitingWorkflow(WaitDefinition $wait): WorkflowDefinition +{ + return new WorkflowDefinition('w', ['s' => new WorkflowStepDefinition('s', new ClosureInvocation(static fn (): null => null), wait: $wait)]); +} + +it('refuses a malformed wait', function (WaitDefinition $wait, string $message): void { + expect(fn () => orchestrationWaitingWorkflow($wait))->toThrow(TopologyValidationException::class, $message); +})->with([ + 'empty any' => [WaitDefinition::any([]), 'Workflow [w] step [s] has a wait for any with no signals and no timers.'], + 'empty all' => [WaitDefinition::all([]), 'Workflow [w] step [s] has a wait for all with no signals and no timers.'], + 'unknown kind' => [new WaitDefinition('sometimes', [new SignalWait('go')]), 'Workflow [w] step [s] waits with an unknown kind [sometimes]'], + 'signal without a signal' => [new WaitDefinition(WaitDefinition::SIGNAL), 'Workflow [w] step [s] has a signal wait that must name exactly one signal and no timers (it has 0 signals and 0 timers).'], + 'signal with a timer' => [new WaitDefinition(WaitDefinition::SIGNAL, [new SignalWait('go')], [new TimerWait('t', 1)]), 'it has 1 signals and 1 timers'], + 'timer without a timer' => [new WaitDefinition(WaitDefinition::TIMER), 'Workflow [w] step [s] has a timer wait that must name exactly one timer and no signals'], + 'negative delay' => [WaitDefinition::timer(-1, 't'), 'Workflow [w] step [s] waits for timer [t] with a negative delay (-1 ms).'], + 'negative timeout' => [WaitDefinition::signal('go', -5), 'Workflow [w] step [s] waits for signal [go] with a negative timeout (-5 ms).'], + 'blank signal' => [WaitDefinition::signal(''), 'Workflow [w] step [s] waits for a signal with a blank name.'], + 'duplicate signal' => [WaitDefinition::all([new SignalWait('go'), new SignalWait('go')]), 'Workflow [w] step [s] waits for signal [go] twice.'], + 'duplicate timer' => [WaitDefinition::any([], [new TimerWait('t', 1), new TimerWait('t', 2)]), 'Workflow [w] step [s] waits for timer [t] twice.'], +]); + +it('refuses a manifest wait with a missing or unknown kind instead of reading it as a signal wait', function (): void { + expect(fn () => WaitDefinition::fromArray(['signals' => [['name' => 'go']]])) + ->toThrow(TopologyValidationException::class, 'A workflow wait has no kind; expected one of [signal, timer, all, any].') + ->and(fn () => WaitDefinition::fromArray(['kind' => 'maybe'])) + ->toThrow(TopologyValidationException::class, 'A workflow wait has an unknown kind [maybe]') + ->and(fn () => WorkflowDefinition::fromArray(['id' => 'w', 'steps' => [['id' => 's', 'invocation' => [], 'wait' => ['kind' => 'any']]]])) + ->toThrow(TopologyValidationException::class, 'Workflow [w] step [s] has a wait for any with no signals and no timers.'); +}); From 5b48e365ca3c0d8c5600743319c7214a749d1852 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 20:38:53 -0700 Subject: [PATCH 26/58] feat(orchestration): durable workflows with signals, timers, children, continue-as-new and queries, plus dead-letter retry --- .../src/Core/ExecutionOutcome.php | 58 +++ .../orchestration/src/Core/Orchestrator.php | 55 ++ .../src/Dlq/DeadLetterRetrier.php | 37 ++ .../src/Workflow/Engine/WaitEvaluator.php | 104 ++++ .../src/Workflow/Engine/WorkflowEngine.php | 474 ++++++++++++++++++ .../src/Workflow/Engine/WorkflowExecutor.php | 249 +++++++++ .../Workflow/Engine/WorkflowRunOutcome.php | 64 +++ .../Workflow/Queue/BusWorkflowDispatcher.php | 35 ++ .../src/Workflow/Queue/RunWorkflowJob.php | 25 + .../src/Workflow/Queue/WorkflowDispatcher.php | 11 + .../tests/Core/OrchestratorTest.php | 67 +++ .../Support/RecordingWorkflowDispatcher.php | 32 ++ .../tests/Support/TestEngines.php | 30 ++ .../Workflow/Engine/WorkflowChildrenTest.php | 60 +++ .../Workflow/Engine/WorkflowEngineTest.php | 139 +++++ .../Workflow/Engine/WorkflowLifecycleTest.php | 60 +++ .../Workflow/Engine/WorkflowWaitsTest.php | 96 ++++ 17 files changed, 1596 insertions(+) create mode 100644 packages/orchestration/src/Core/ExecutionOutcome.php create mode 100644 packages/orchestration/src/Core/Orchestrator.php create mode 100644 packages/orchestration/src/Dlq/DeadLetterRetrier.php create mode 100644 packages/orchestration/src/Workflow/Engine/WaitEvaluator.php create mode 100644 packages/orchestration/src/Workflow/Engine/WorkflowEngine.php create mode 100644 packages/orchestration/src/Workflow/Engine/WorkflowExecutor.php create mode 100644 packages/orchestration/src/Workflow/Engine/WorkflowRunOutcome.php create mode 100644 packages/orchestration/src/Workflow/Queue/BusWorkflowDispatcher.php create mode 100644 packages/orchestration/src/Workflow/Queue/RunWorkflowJob.php create mode 100644 packages/orchestration/src/Workflow/Queue/WorkflowDispatcher.php create mode 100644 packages/orchestration/tests/Core/OrchestratorTest.php create mode 100644 packages/orchestration/tests/Support/RecordingWorkflowDispatcher.php create mode 100644 packages/orchestration/tests/Workflow/Engine/WorkflowChildrenTest.php create mode 100644 packages/orchestration/tests/Workflow/Engine/WorkflowEngineTest.php create mode 100644 packages/orchestration/tests/Workflow/Engine/WorkflowLifecycleTest.php create mode 100644 packages/orchestration/tests/Workflow/Engine/WorkflowWaitsTest.php diff --git a/packages/orchestration/src/Core/ExecutionOutcome.php b/packages/orchestration/src/Core/ExecutionOutcome.php new file mode 100644 index 00000000..4bcb83bc --- /dev/null +++ b/packages/orchestration/src/Core/ExecutionOutcome.php @@ -0,0 +1,58 @@ + $payload + */ + public function __construct( + public string $correlationId, + public ExecutionPattern $pattern, + public string $name, + public ExecutionStatus $status, + public bool $success, + public array $payload, + ) {} + + public static function fromSaga(SagaResult $result): self + { + return new self($result->correlationId, ExecutionPattern::Saga, $result->sagaName, $result->status, $result->success, $result->toArray()); + } + + public static function fromTcc(TccResult $result): self + { + return new self($result->correlationId, ExecutionPattern::Tcc, $result->tccName, $result->status, $result->success, $result->toArray()); + } + + public static function fromWorkflow(ExecutionState $state): self + { + $success = ! in_array($state->status, [ExecutionStatus::Failed, ExecutionStatus::TimedOut, ExecutionStatus::Cancelled], true); + + return new self($state->correlationId, ExecutionPattern::Workflow, $state->executionName, $state->status, $success, $state->toArray()); + } + + /** @return array */ + public function toArray(): array + { + return [ + 'correlationId' => $this->correlationId, + 'pattern' => $this->pattern->value, + 'name' => $this->name, + 'status' => $this->status->value, + 'success' => $this->success, + 'result' => $this->payload, + ]; + } +} diff --git a/packages/orchestration/src/Core/Orchestrator.php b/packages/orchestration/src/Core/Orchestrator.php new file mode 100644 index 00000000..863cceed --- /dev/null +++ b/packages/orchestration/src/Core/Orchestrator.php @@ -0,0 +1,55 @@ + $input + * @param array $headers + */ + public function start(ExecutionPattern $pattern, string $name, array $input = [], array $headers = []): ExecutionOutcome + { + return match ($pattern) { + ExecutionPattern::Saga => ExecutionOutcome::fromSaga($this->sagas->execute($name, $input, $headers)), + ExecutionPattern::Tcc => ExecutionOutcome::fromTcc($this->tccs->execute($name, $input, $headers)), + ExecutionPattern::Workflow => ExecutionOutcome::fromWorkflow($this->workflows->start($name, $input, $headers)), + }; + } + + public function find(string $correlationId): ?ExecutionState + { + return $this->persistence->findById($correlationId); + } + + public function sagas(): SagaEngine + { + return $this->sagas; + } + + public function tccs(): TccEngine + { + return $this->tccs; + } + + public function workflows(): WorkflowEngine + { + return $this->workflows; + } +} diff --git a/packages/orchestration/src/Dlq/DeadLetterRetrier.php b/packages/orchestration/src/Dlq/DeadLetterRetrier.php new file mode 100644 index 00000000..167a1669 --- /dev/null +++ b/packages/orchestration/src/Dlq/DeadLetterRetrier.php @@ -0,0 +1,37 @@ +deadLetters->get($id); + $headers = $entry->headers + [ + DeadLetterEntry::RETRY_OF_HEADER => $entry->id, + DeadLetterEntry::RETRY_COUNT_HEADER => (string) $entry->retryCount, + ]; + + $outcome = $this->orchestrator->start($entry->pattern, $entry->executionName, $entry->input, $headers); + $this->deadLetters->delete($id); + + return $outcome; + } +} diff --git a/packages/orchestration/src/Workflow/Engine/WaitEvaluator.php b/packages/orchestration/src/Workflow/Engine/WaitEvaluator.php new file mode 100644 index 00000000..4f745264 --- /dev/null +++ b/packages/orchestration/src/Workflow/Engine/WaitEvaluator.php @@ -0,0 +1,104 @@ +`. A signal whose timeout timer fired first throws + * SignalTimeoutException, which fails the step. + */ +final class WaitEvaluator +{ + public function __construct(private readonly OrchestrationEvents $events) {} + + public static function timerId(string $stepId, TimerWait $timer, int $index): string + { + return $timer->id !== '' ? $timer->id : "{$stepId}:timer:{$index}"; + } + + public function satisfied(string $workflowId, WorkflowStepDefinition $step, ExecutionContext $context, DateTimeImmutable $now): bool + { + $wait = $step->wait; + if ($wait === null) { + return true; + } + + $timerIds = []; + foreach ($wait->timers as $index => $timer) { + $id = self::timerId($step->id, $timer, $index); + $timerIds[] = $id; + if (! $context->hasTimer($id)) { + $context->scheduleTimer($id, $now->modify(sprintf('+%d milliseconds', $timer->delayMs))); + } + } + foreach ($wait->signals as $signal) { + $timeoutId = "{$step->id}:signal-timeout:{$signal->name}"; + if ($signal->timeoutMs > 0 && ! $context->hasSignal($signal->name) && ! $context->hasTimer($timeoutId)) { + $context->scheduleTimer($timeoutId, $now->modify(sprintf('+%d milliseconds', $signal->timeoutMs))); + } + } + + $delivered = array_values(array_filter($wait->signals, static fn (SignalWait $signal): bool => $context->hasSignal($signal->name))); + $fired = array_values(array_filter($timerIds, static fn (string $id): bool => $context->timerFired($id, $now))); + + $satisfied = match ($wait->kind) { + WaitDefinition::SIGNAL, WaitDefinition::ALL => count($delivered) === count($wait->signals) && count($fired) === count($timerIds), + WaitDefinition::TIMER => count($fired) === count($timerIds), + default => $delivered !== [] || $fired !== [], + }; + + if (! $satisfied) { + foreach ($wait->signals as $signal) { + $timeoutId = "{$step->id}:signal-timeout:{$signal->name}"; + if (! $context->hasSignal($signal->name) && $context->timerFired($timeoutId, $now)) { + $context->removeTimer($timeoutId); + + throw new SignalTimeoutException("Workflow [{$workflowId}] step [{$step->id}] waited longer than {$signal->timeoutMs} ms for signal [{$signal->name}]."); + } + } + + return false; + } + + if ($wait->kind === WaitDefinition::ANY) { + $trigger = $delivered !== [] ? $delivered[0]->name : $fired[0]; + $context->setVariable('wait.trigger', $trigger); + if ($delivered !== []) { + $context->setVariable('signals.'.$delivered[0]->name, $context->consumeSignal($delivered[0]->name)); + } + } else { + foreach ($delivered as $signal) { + $context->setVariable('signals.'.$signal->name, $context->consumeSignal($signal->name)); + } + } + + foreach ($fired as $id) { + $this->events->onTimerFired($workflowId, $context->correlationId, $id); + } + foreach ($timerIds as $id) { + $context->removeTimer($id); + } + foreach ($wait->signals as $signal) { + $context->removeTimer("{$step->id}:signal-timeout:{$signal->name}"); + } + + return true; + } +} diff --git a/packages/orchestration/src/Workflow/Engine/WorkflowEngine.php b/packages/orchestration/src/Workflow/Engine/WorkflowEngine.php new file mode 100644 index 00000000..1444fd11 --- /dev/null +++ b/packages/orchestration/src/Workflow/Engine/WorkflowEngine.php @@ -0,0 +1,474 @@ + */ + private array $driving = []; + + public function __construct( + private readonly WorkflowRegistry $registry, + private readonly WorkflowExecutor $executor, + private readonly EngineSupport $support, + private readonly WorkflowSettings $settings, + private readonly WorkflowDispatcher $dispatcher, + private readonly Container $container, + private readonly int $batchSize = 10, + ) {} + + public function registry(): WorkflowRegistry + { + return $this->registry; + } + + /** + * @param array $input + * @param array $headers + */ + public function start(string|WorkflowDefinition $workflow, array $input = [], array $headers = [], ?string $correlationId = null, ?string $parentCorrelationId = null): ExecutionState + { + if (! $this->settings->enabled) { + throw new PatternDisabledException(ExecutionPattern::Workflow, 'firefly.orchestration.workflow.enabled'); + } + + $definition = is_string($workflow) ? $this->registry->get($workflow) : $workflow; + $context = ExecutionContext::start(ExecutionPattern::Workflow, $definition->id, $input, $headers, $correlationId, $this->support->now()); + $context->setParentCorrelationId($parentCorrelationId); + + return $this->support->admission->admit( + $definition->id, + function () use ($definition, $context): ExecutionState { + if ($definition->triggerMode === TriggerMode::Async) { + $pending = $this->support->checkpoint($context, ExecutionStatus::Pending); + $this->dispatcher->dispatch($context->correlationId); + + return $this->support->persistence->findById($context->correlationId) ?? $pending; + } + + return $this->support->lock->synchronized($context->correlationId, fn (): ExecutionState => $this->support->tracing->traceExecution( + $definition->id, + ExecutionPattern::Workflow, + $context->correlationId, + fn (): ExecutionState => $this->begin($definition, $context), + )); + }, + static fn (ExecutionState $state): bool => ! in_array($state->status, [ExecutionStatus::Failed, ExecutionStatus::TimedOut], true), + ); + } + + /** The queued job's entry point: starts a PENDING execution; anything else is returned untouched. */ + public function run(string $correlationId): ExecutionState + { + return $this->support->lock->synchronized($correlationId, function () use ($correlationId): ExecutionState { + $state = $this->load($correlationId); + if ($state->status !== ExecutionStatus::Pending) { + return $state; + } + + $definition = $this->registry->get($state->executionName); + $context = ExecutionContext::fromState($state); + + return $this->support->tracing->traceExecution($definition->id, ExecutionPattern::Workflow, $correlationId, fn (): ExecutionState => $this->begin($definition, $context)); + }); + } + + public function signal(string $correlationId, string $signal, mixed $payload = null): ExecutionState + { + return $this->support->lock->synchronized($correlationId, function () use ($correlationId, $signal, $payload): ExecutionState { + $state = $this->load($correlationId); + if ($state->isTerminal()) { + throw new InvalidExecutionStateException("Workflow execution [{$correlationId}] is already {$state->status->value} and accepts no signals."); + } + + $definition = $this->registry->get($state->executionName); + $context = ExecutionContext::fromState($state); + $context->receiveSignal($signal, $payload); + $this->support->events->onSignalDelivered($definition->id, $correlationId, $signal); + + return $state->status === ExecutionStatus::Waiting + ? $this->drive($definition, $context) + : $this->support->checkpoint($context, $state->status); + }); + } + + /** @return list the executions this tick advanced */ + public function resumeDue(): array + { + $advanced = []; + foreach ($this->support->persistence->findDue($this->support->now(), $this->batchSize) as $due) { + if ($due->pattern !== ExecutionPattern::Workflow) { + continue; + } + + try { + $advanced[] = $this->support->lock->synchronized($due->correlationId, function () use ($due): ExecutionState { + $state = $this->load($due->correlationId); + if ($state->status !== ExecutionStatus::Waiting) { + return $state; + } + + return $this->drive($this->registry->get($state->executionName), ExecutionContext::fromState($state)); + }); + } catch (ExecutionLockedException) { + // Another process is advancing it right now; the next tick sees whatever it left. + } + } + + return $advanced; + } + + public function suspend(string $correlationId, string $reason = ''): ExecutionState + { + return $this->support->lock->synchronized($correlationId, function () use ($correlationId, $reason): ExecutionState { + $state = $this->load($correlationId); + if (! $state->status->canSuspend()) { + throw new InvalidExecutionStateException("Workflow execution [{$correlationId}] is {$state->status->value} and cannot be suspended."); + } + + $context = ExecutionContext::fromState($state); + $context->setVariable('suspendedFrom', $state->status->value); + $context->setVariable('suspendReason', $reason); + $this->support->events->onWorkflowSuspended($state->executionName, $correlationId, $reason); + + return $this->support->checkpoint($context, ExecutionStatus::Suspended); + }); + } + + public function resume(string $correlationId): ExecutionState + { + return $this->support->lock->synchronized($correlationId, function () use ($correlationId): ExecutionState { + $state = $this->load($correlationId); + if (! $state->status->canResume()) { + throw new InvalidExecutionStateException("Workflow execution [{$correlationId}] is {$state->status->value}, not SUSPENDED."); + } + + $definition = $this->registry->get($state->executionName); + $context = ExecutionContext::fromState($state); + $this->support->events->onWorkflowResumed($definition->id, $correlationId); + + return $context->variable('suspendedFrom') === ExecutionStatus::Pending->value + ? $this->begin($definition, $context) + : $this->drive($definition, $context); + }); + } + + public function cancel(string $correlationId): ExecutionState + { + return $this->support->lock->synchronized($correlationId, function () use ($correlationId): ExecutionState { + $state = $this->load($correlationId); + if ($state->isTerminal()) { + throw new InvalidExecutionStateException("Workflow execution [{$correlationId}] is already {$state->status->value}."); + } + + $context = ExecutionContext::fromState($state); + $this->cancelChildren($context); + $cancelled = $this->support->checkpoint($context, ExecutionStatus::Cancelled, 'cancelled'); + $this->support->publish($this->support->lifecycle, OrchestrationEvent::executionCompleted($state->executionName, $correlationId, ExecutionPattern::Workflow, ExecutionStatus::Cancelled, $this->support->now())); + $this->support->events->onCompleted($state->executionName, $correlationId, ExecutionPattern::Workflow, false, $this->support->elapsedMs($context)); + $this->notifyParent($cancelled); + + return $cancelled; + }); + } + + public function compensate(string $correlationId): ExecutionState + { + return $this->support->lock->synchronized($correlationId, function () use ($correlationId): ExecutionState { + $state = $this->load($correlationId); + if ($state->isTerminal() && $state->status !== ExecutionStatus::Completed) { + throw new InvalidExecutionStateException("Workflow execution [{$correlationId}] is already {$state->status->value} and has nothing left to compensate."); + } + + $definition = $this->registry->get($state->executionName); + $context = ExecutionContext::fromState($state); + $this->cancelChildren($context); + $this->executor->compensate($definition, $context); + + return $this->finish($definition, $context, ExecutionStatus::Cancelled, 'compensated on request'); + }); + } + + /** + * @param list $arguments + */ + public function query(string $correlationId, string $name, array $arguments = []): mixed + { + $state = $this->load($correlationId); + $query = $this->registry->get($state->executionName)->query($name); + $context = ExecutionContext::fromState($state); + $invocation = $query->invocation; + + if ($invocation instanceof ClosureInvocation) { + return ($invocation->closure)($context, ...$arguments); + } + if ($invocation instanceof MethodInvocation) { + $bean = $this->container->make($invocation->class); + $method = $invocation->method; + if (is_object($bean) && method_exists($bean, $method)) { + return $bean->{$method}(...($query->contextFirst ? [$context, ...$arguments] : $arguments)); + } + } + + throw new OrchestrationException("Workflow query [{$name}] of [{$state->executionName}] cannot be invoked.", 'ORCHESTRATION_UNKNOWN_QUERY'); + } + + /** + * @param array $attributes + */ + public function upsertSearchAttributes(string $correlationId, array $attributes): ExecutionState + { + return $this->support->lock->synchronized($correlationId, function () use ($correlationId, $attributes): ExecutionState { + $state = $this->load($correlationId); + $context = ExecutionContext::fromState($state); + foreach ($attributes as $key => $value) { + $context->setSearchAttribute($key, $value); + } + + return $this->support->checkpoint($context, $state->status, $state->failureReason, null); + }); + } + + /** + * @param array $criteria + * @return list + */ + public function search(array $criteria, int $limit = 100): array + { + return array_values(array_filter( + $this->support->persistence->findBySearchAttributes($criteria, $limit), + static fn (ExecutionState $state): bool => $state->pattern === ExecutionPattern::Workflow, + )); + } + + public function find(string $correlationId): ?ExecutionState + { + $state = $this->support->persistence->findById($correlationId); + + return $state !== null && $state->pattern === ExecutionPattern::Workflow ? $state : null; + } + + /** @return list */ + public function findByStatus(ExecutionStatus $status, int $limit = 50): array + { + return $this->support->persistence->find(new ExecutionCriteria(ExecutionPattern::Workflow, $status, limit: $limit)); + } + + private function begin(WorkflowDefinition $workflow, ExecutionContext $context): ExecutionState + { + $this->support->checkpoint($context, ExecutionStatus::Running); + $this->support->publish($this->support->lifecycle, OrchestrationEvent::executionStarted($workflow->id, $context->correlationId, ExecutionPattern::Workflow, $this->support->now())); + $this->support->events->onStart($workflow->id, $context->correlationId, ExecutionPattern::Workflow); + + return $this->drive($workflow, $context); + } + + private function drive(WorkflowDefinition $workflow, ExecutionContext $context): ExecutionState + { + $this->driving[$context->correlationId] = true; + try { + $context->setDeadline($workflow->timeoutMs > 0 ? $context->startedAt()->modify(sprintf('+%d milliseconds', $workflow->timeoutMs)) : null); + + while (true) { + $outcome = $this->executor->run($workflow, $context); + + if ($outcome->kind === WorkflowRunOutcome::CHILD && $this->startChild($workflow, $context, (string) $outcome->stepId, $outcome->childInput)) { + continue; + } + + return match ($outcome->kind) { + WorkflowRunOutcome::CHILD, WorkflowRunOutcome::WAITING => $this->support->checkpoint($context, ExecutionStatus::Waiting), + WorkflowRunOutcome::CONTINUE => $this->continueAsNew($workflow, $context, $outcome), + WorkflowRunOutcome::FAILED => $this->fail($workflow, $context, $outcome), + default => $this->complete($workflow, $context), + }; + } + } finally { + unset($this->driving[$context->correlationId]); + } + } + + /** + * Starts the child of `$stepId`; true when the step is already settled (the child finished inline, or the + * parent does not wait for it) and the drive can go on, false when the parent now waits for the child. + * + * @param array $input + */ + private function startChild(WorkflowDefinition $workflow, ExecutionContext $context, string $stepId, array $input): bool + { + $child = $workflow->step($stepId)->child; + if ($child === null) { + return true; + } + + $childState = $this->start($child->workflowId, $input, $context->headers(), null, $context->correlationId); + $context->setChild($stepId, $childState->correlationId); + $this->support->events->onChildWorkflowStarted($workflow->id, $context->correlationId, $child->workflowId, $childState->correlationId); + + if (! $child->waitForCompletion) { + $this->executor->finishStep($workflow, $context, $stepId, ['correlationId' => $childState->correlationId, 'status' => $childState->status->value]); + + return true; + } + + if ($childState->isTerminal()) { + $this->executor->completeChild($workflow, $context, $stepId, $childState); + + return true; + } + + if ($child->timeoutMs > 0) { + $context->scheduleTimer("{$stepId}:child-timeout", $this->support->now()->modify(sprintf('+%d milliseconds', $child->timeoutMs))); + } + $context->setStepStatus($stepId, StepStatus::Waiting); + $context->setWaitingOn($stepId); + + return false; + } + + private function continueAsNew(WorkflowDefinition $workflow, ExecutionContext $context, WorkflowRunOutcome $outcome): ExecutionState + { + $next = $outcome->continueAsNew ?? new ContinueAsNew; + $newId = Str::uuid()->toString(); + $context->setVariable('continuedAs', $newId); + $this->finish($workflow, $context, ExecutionStatus::Completed, null, notify: false); + $this->support->events->onContinueAsNew($workflow->id, $context->correlationId, $newId); + + $headers = $context->headers(); + $headers[self::CONTINUED_FROM_HEADER] ??= $context->correlationId; + + return $this->start($workflow, $next->input, $headers, $newId, $context->parentCorrelationId()); + } + + private function complete(WorkflowDefinition $workflow, ExecutionContext $context): ExecutionState + { + return $this->finish($workflow, $context, ExecutionStatus::Completed, null); + } + + private function fail(WorkflowDefinition $workflow, ExecutionContext $context, WorkflowRunOutcome $outcome): ExecutionState + { + $error = $outcome->error ?? new OrchestrationException("Workflow [{$workflow->id}] failed."); + $stepId = $outcome->stepId === '' ? null : $outcome->stepId; + + $this->support->checkpoint($context, ExecutionStatus::Compensating, $error->getMessage()); + $this->cancelChildren($context); + $this->executor->compensate($workflow, $context); + $this->support->deadLetter($context, $stepId, $outcome->timedOut ? ExecutionStatus::TimedOut : ExecutionStatus::Failed, $error); + + $suppressed = false; + foreach ($workflow->onError as $callback) { + if ($callback->suppressError && $callback->matchesError($error) && $callback->matchesStep($stepId)) { + $suppressed = true; + } + } + + $status = $suppressed ? ExecutionStatus::Completed : ($outcome->timedOut ? ExecutionStatus::TimedOut : ExecutionStatus::Failed); + $state = $this->finish($workflow, $context, $status, $suppressed ? null : $error->getMessage(), callbacks: false); + $this->support->callbacks->invoke($workflow->onError, $context, $state, $error, $stepId); + + return $state; + } + + private function finish(WorkflowDefinition $workflow, ExecutionContext $context, ExecutionStatus $status, ?string $reason, bool $notify = true, bool $callbacks = true): ExecutionState + { + $now = $this->support->now(); + $report = ExecutionReportBuilder::fromContext($context, $status, $reason, $now, $workflow->stepIds()); + $state = $this->support->checkpoint($context, $status, $reason, $report); + + $this->support->publish($this->support->lifecycle, OrchestrationEvent::executionCompleted($workflow->id, $context->correlationId, ExecutionPattern::Workflow, $status, $now)); + $this->support->events->onCompleted($workflow->id, $context->correlationId, ExecutionPattern::Workflow, $status->isSuccess(), $this->support->elapsedMs($context)); + + if ($callbacks && $status === ExecutionStatus::Completed) { + $this->support->callbacks->invoke($workflow->onComplete, $context, $state); + } + if ($notify) { + $this->notifyParent($state); + } + + return $state; + } + + private function notifyParent(ExecutionState $child): void + { + $parentId = $child->parentCorrelationId; + if ($parentId === null || isset($this->driving[$parentId])) { + return; + } + + $this->support->lock->synchronized($parentId, function () use ($parentId, $child): void { + $parent = $this->support->persistence->findById($parentId); + if ($parent === null || $parent->status !== ExecutionStatus::Waiting) { + return; + } + + $key = $child->headers[self::CONTINUED_FROM_HEADER] ?? $child->correlationId; + $stepId = array_search($key, $parent->children, true); + if (! is_string($stepId) || $parent->waitingOn !== $stepId) { + return; + } + + $definition = $this->registry->get($parent->executionName); + $context = ExecutionContext::fromState($parent); + $this->executor->completeChild($definition, $context, $stepId, $child); + $this->drive($definition, $context); + }); + } + + private function cancelChildren(ExecutionContext $context): void + { + foreach ($context->children() as $childId) { + $child = $this->support->persistence->findById($childId); + if ($child !== null && ! $child->isTerminal()) { + $this->cancel($childId); + } + } + } + + private function load(string $correlationId): ExecutionState + { + $state = $this->support->persistence->findById($correlationId); + if ($state === null || $state->pattern !== ExecutionPattern::Workflow) { + throw new ExecutionNotFoundException($correlationId); + } + + return $state; + } +} diff --git a/packages/orchestration/src/Workflow/Engine/WorkflowExecutor.php b/packages/orchestration/src/Workflow/Engine/WorkflowExecutor.php new file mode 100644 index 00000000..c28e8866 --- /dev/null +++ b/packages/orchestration/src/Workflow/Engine/WorkflowExecutor.php @@ -0,0 +1,249 @@ +result(). + */ +final class WorkflowExecutor +{ + public function __construct( + private readonly StepInvoker $invoker, + private readonly EngineSupport $support, + private readonly ConditionEvaluator $conditions, + private readonly WaitEvaluator $waits, + ) {} + + public function run(WorkflowDefinition $workflow, ExecutionContext $context): WorkflowRunOutcome + { + $context->setTopologyLayers($workflow->layers); + $now = $this->support->now(); + + $deadline = $context->deadline(); + if ($deadline !== null && $now >= $deadline) { + return WorkflowRunOutcome::failed($context->waitingOn() ?? '', new StepTimeoutException("Workflow [{$workflow->id}] passed its timeoutMs deadline of {$workflow->timeoutMs} ms."), timedOut: true); + } + + foreach ($workflow->layers as $layer) { + foreach ($layer as $stepId) { + $step = $workflow->steps[$stepId]; + $status = $context->stepStatus($stepId); + + if ($status->isSettled()) { + continue; + } + if ($status->isFailure()) { + return WorkflowRunOutcome::failed($stepId, $context->stepError($stepId) ?? new OrchestrationException("Workflow [{$workflow->id}] step [{$stepId}] failed.")); + } + + if ($status === StepStatus::Pending && $step->condition !== '' && ! $this->conditions->evaluate($step->condition, self::conditionValues($context))) { + $context->setStepStatus($stepId, StepStatus::Skipped); + $this->support->events->onStepSkipped($workflow->id, $context->correlationId, $stepId); + $this->support->checkpoint($context, ExecutionStatus::Running); + + continue; + } + + if ($step->wait !== null && $status !== StepStatus::Running) { + try { + $satisfied = $this->waits->satisfied($workflow->id, $step, $context, $now); + } catch (Throwable $error) { + return $this->failed($workflow, $context, $stepId, $error); + } + if (! $satisfied) { + $context->setStepStatus($stepId, StepStatus::Waiting); + $context->setWaitingOn($stepId); + + return WorkflowRunOutcome::waiting($stepId); + } + } + + if ($step->child !== null && $context->child($stepId) !== null) { + if ($context->timerFired("{$stepId}:child-timeout", $now)) { + $context->removeTimer("{$stepId}:child-timeout"); + + return $this->failed($workflow, $context, $stepId, new StepTimeoutException("Workflow [{$workflow->id}] step [{$stepId}]: child workflow [{$step->child->workflowId}] did not finish within {$step->child->timeoutMs} ms.")); + } + $context->setStepStatus($stepId, StepStatus::Waiting); + $context->setWaitingOn($stepId); + + return WorkflowRunOutcome::waiting($stepId); + } + + $started = hrtime(true); + try { + $result = $this->invoke($workflow, $step, $context); + } catch (Throwable $error) { + $context->setLatencyMs($stepId, self::since($started)); + + return $this->failed($workflow, $context, $stepId, $error); + } + + if ($step->child !== null) { + return WorkflowRunOutcome::child($stepId, self::childInput($result)); + } + + if ($result instanceof ContinueAsNew) { + $context->setStepStatus($stepId, StepStatus::Done); + $context->markCompleted($stepId); + + return WorkflowRunOutcome::continueAsNew($stepId, $result); + } + + $this->finishStep($workflow, $context, $stepId, $result, self::since($started)); + } + } + + return WorkflowRunOutcome::completed(); + } + + public function finishStep(WorkflowDefinition $workflow, ExecutionContext $context, string $stepId, mixed $result, int $latencyMs = 0): void + { + $step = $workflow->step($stepId); + $context->putResult($stepId, $result); + $context->setLatencyMs($stepId, $latencyMs); + $context->setStepStatus($stepId, StepStatus::Done); + $context->markCompleted($stepId); + $context->setWaitingOn(null); + $this->support->events->onStepSuccess($workflow->id, $context->correlationId, $stepId, $context->attempts($stepId), $latencyMs); + + $completed = OrchestrationEvent::stepCompleted($workflow->id, $context->correlationId, ExecutionPattern::Workflow, $stepId, $result, $this->support->now()); + if ($step->outputEventType !== '') { + $this->support->publish($this->support->stepEvents, $completed->withType($step->outputEventType)); + } + if ($workflow->publishEvents) { + $this->support->publish($this->support->lifecycle, $completed); + } + + $this->support->callbacks->invoke($workflow->onStepComplete, $context, $result, null, $stepId); + $this->support->checkpoint($context, ExecutionStatus::Running); + } + + public function completeChild(WorkflowDefinition $workflow, ExecutionContext $context, string $stepId, ExecutionState $child): void + { + $success = $child->status->isSuccess(); + $this->support->events->onChildWorkflowCompleted($workflow->id, $context->correlationId, $child->correlationId, $success); + $context->removeTimer("{$stepId}:child-timeout"); + + if ($success) { + $this->finishStep($workflow, $context, $stepId, ['correlationId' => $child->correlationId, 'status' => $child->status->value, 'results' => $child->stepResults]); + + return; + } + + $error = new OrchestrationException( + "Child workflow [{$child->executionName}] ({$child->correlationId}) of step [{$stepId}] ended {$child->status->value}".($child->failureReason === null ? '.' : ": {$child->failureReason}"), + 'ORCHESTRATION_CHILD_FAILED', + ); + $context->setWaitingOn(null); + $this->failed($workflow, $context, $stepId, $error); + } + + public function compensate(WorkflowDefinition $workflow, ExecutionContext $context): void + { + $order = array_values(array_filter( + array_reverse($context->completionOrder()), + static fn (string $stepId): bool => isset($workflow->steps[$stepId]) + && $workflow->steps[$stepId]->compensatable + && $workflow->steps[$stepId]->compensation !== null + && $context->stepStatus($stepId) === StepStatus::Done, + )); + if ($order === []) { + return; + } + + $this->support->events->onCompensationStarted($workflow->id, $context->correlationId); + foreach ($order as $stepId) { + $step = $workflow->steps[$stepId]; + $compensation = $step->compensation; + if ($compensation === null) { + continue; + } + + try { + $result = $this->invoker->invoke($workflow->id, $stepId.':compensate', $compensation, $step->options->withoutIdempotency(), $context, $context->stepInput($stepId), $context->result($stepId)); + $context->putCompensationResult($stepId, $result); + $context->setStepStatus($stepId, StepStatus::Compensated); + $this->support->events->onStepCompensated($workflow->id, $context->correlationId, $stepId); + } catch (Throwable $error) { + $context->putCompensationError($stepId, $error); + $context->setStepStatus($stepId, StepStatus::CompensationFailed); + $this->support->events->onStepCompensationFailed($workflow->id, $context->correlationId, $stepId, $error); + } + $this->support->checkpoint($context, ExecutionStatus::Compensating); + } + } + + private function invoke(WorkflowDefinition $workflow, WorkflowStepDefinition $step, ExecutionContext $context): mixed + { + $input = $context->input(); + $context->putStepInput($step->id, $input); + $context->setStepStatus($step->id, StepStatus::Running); + $context->markStepStarted($step->id, $this->support->now()); + $context->setWaitingOn(null); + $context->clearStepError($step->id); + $this->support->events->onStepStarted($workflow->id, $context->correlationId, $step->id); + $this->support->checkpoint($context, ExecutionStatus::Running); + + return $this->invoker->invoke($workflow->id, $step->id, $step->invocation, $step->options, $context, $input, $input); + } + + private function failed(WorkflowDefinition $workflow, ExecutionContext $context, string $stepId, Throwable $error): WorkflowRunOutcome + { + $context->setStepStatus($stepId, $error instanceof TimeoutException ? StepStatus::TimedOut : StepStatus::Failed); + $context->putStepError($stepId, $error); + $this->support->events->onStepFailed($workflow->id, $context->correlationId, $stepId, $error, $context->attempts($stepId)); + $this->support->publish($this->support->lifecycle, OrchestrationEvent::stepFailed($workflow->id, $context->correlationId, ExecutionPattern::Workflow, $stepId, $error, $this->support->now())); + + return WorkflowRunOutcome::failed($stepId, $error); + } + + /** @return array */ + private static function conditionValues(ExecutionContext $context): array + { + return ['input' => $context->input(), 'results' => $context->results(), 'variables' => $context->variables(), 'headers' => $context->headers()]; + } + + /** @return array */ + private static function childInput(mixed $result): array + { + if (! is_array($result)) { + return $result === null ? [] : ['value' => $result]; + } + + $input = []; + foreach ($result as $key => $value) { + $input[(string) $key] = $value; + } + + return $input; + } + + private static function since(int|float $started): int + { + return (int) round((hrtime(true) - $started) / 1_000_000); + } +} diff --git a/packages/orchestration/src/Workflow/Engine/WorkflowRunOutcome.php b/packages/orchestration/src/Workflow/Engine/WorkflowRunOutcome.php new file mode 100644 index 00000000..23588ce5 --- /dev/null +++ b/packages/orchestration/src/Workflow/Engine/WorkflowRunOutcome.php @@ -0,0 +1,64 @@ + $childInput + */ + private function __construct( + public string $kind, + public ?string $stepId = null, + public ?Throwable $error = null, + public array $childInput = [], + public ?ContinueAsNew $continueAsNew = null, + public bool $timedOut = false, + ) {} + + public static function completed(): self + { + return new self(self::COMPLETED); + } + + public static function waiting(string $stepId): self + { + return new self(self::WAITING, $stepId); + } + + public static function failed(string $stepId, Throwable $error, bool $timedOut = false): self + { + return new self(self::FAILED, $stepId, $error, timedOut: $timedOut); + } + + /** + * @param array $input + */ + public static function child(string $stepId, array $input): self + { + return new self(self::CHILD, $stepId, childInput: $input); + } + + public static function continueAsNew(string $stepId, ContinueAsNew $next): self + { + return new self(self::CONTINUE, $stepId, continueAsNew: $next); + } +} diff --git a/packages/orchestration/src/Workflow/Queue/BusWorkflowDispatcher.php b/packages/orchestration/src/Workflow/Queue/BusWorkflowDispatcher.php new file mode 100644 index 00000000..7cb72cc3 --- /dev/null +++ b/packages/orchestration/src/Workflow/Queue/BusWorkflowDispatcher.php @@ -0,0 +1,35 @@ +connection !== null) { + $job->onConnection($this->connection); + } + if ($this->queue !== null) { + $job->onQueue($this->queue); + } + + $this->bus->dispatch($job); + } +} diff --git a/packages/orchestration/src/Workflow/Queue/RunWorkflowJob.php b/packages/orchestration/src/Workflow/Queue/RunWorkflowJob.php new file mode 100644 index 00000000..1aab1233 --- /dev/null +++ b/packages/orchestration/src/Workflow/Queue/RunWorkflowJob.php @@ -0,0 +1,25 @@ +run($this->correlationId); + } +} diff --git a/packages/orchestration/src/Workflow/Queue/WorkflowDispatcher.php b/packages/orchestration/src/Workflow/Queue/WorkflowDispatcher.php new file mode 100644 index 00000000..56c3ed5e --- /dev/null +++ b/packages/orchestration/src/Workflow/Queue/WorkflowDispatcher.php @@ -0,0 +1,11 @@ +sagaRegistry->register(SagaBuilder::saga('s')->step('a')->handler(static fn (): string => 'a')->add()->build()); + $engines->tccRegistry->register(TccBuilder::tcc('t')->participant('p')->tryWith(static fn (): string => 'held')->confirmWith(static fn (): null => null)->cancelWith(static fn (): null => null)->add()->build()); + $engines->workflowRegistry->register(WorkflowBuilder::workflow('w')->step('a')->handler(static fn (): string => 'a')->add()->build()); + + $saga = $engines->orchestrator->start(ExecutionPattern::Saga, 's', ['x' => 1]); + $tcc = $engines->orchestrator->start(ExecutionPattern::Tcc, 't'); + $workflow = $engines->orchestrator->start(ExecutionPattern::Workflow, 'w'); + + expect([$saga->status, $tcc->status, $workflow->status])->toBe([ExecutionStatus::Completed, ExecutionStatus::Confirmed, ExecutionStatus::Completed]) + ->and($saga->success && $tcc->success && $workflow->success)->toBeTrue() + ->and($saga->toArray())->toMatchArray(['pattern' => 'SAGA', 'name' => 's', 'status' => 'COMPLETED', 'success' => true]) + ->and($engines->orchestrator->find($tcc->correlationId)?->pattern)->toBe(ExecutionPattern::Tcc); +}); + +it('retries a dead letter by starting the definition again with its input, and counts the retries', function (): void { + $engines = TestEngines::create(); + $switch = new ArrayObject(['fail' => true]); + $engines->sagaRegistry->register(SagaBuilder::saga('flaky')->step('call')->handler(static function (array $input) use ($switch): string { + if ($switch['fail']) { + throw new RuntimeException('down'); + } + + return 'ok '.(is_int($input['id'] ?? null) ? $input['id'] : 0); + })->add()->build()); + $engines->sagas->execute('flaky', ['id' => 5], ['tenant' => 'acme']); + $first = $engines->deadLetterStore->findAll()[0]; + + $again = $engines->retrier->retry($first->id); + $second = $engines->deadLetterStore->findAll()[0]; + $switch['fail'] = false; + $ok = $engines->retrier->retry($second->id); + + expect($again->success)->toBeFalse() + ->and($engines->deadLetterStore->findById($first->id))->toBeNull() + ->and($second->retryCount)->toBe(1) + ->and($second->headers)->toBe(['tenant' => 'acme']) + ->and($ok->success)->toBeTrue() + ->and(data_get($ok->payload, 'steps.call.result'))->toBe('ok 5') + ->and($engines->deadLetterStore->count())->toBe(0) + ->and(DeadLetterEntry::RETRY_COUNT_HEADER)->toBe('x-firefly-dlq-retry-count'); +}); + +it('keeps the dead letter when its retry cannot start', function (): void { + $engines = TestEngines::create(); + $engines->sagaRegistry->register(SagaBuilder::saga('gone')->step('call')->handler(static fn (): never => throw new RuntimeException('down'))->add()->build()); + $engines->sagas->execute('gone', ['id' => 7]); + $entry = $engines->deadLetterStore->findAll()[0]; + $withoutDefinition = TestEngines::create(deadLetters: $engines->deadLetterStore); + + expect(fn () => $withoutDefinition->retrier->retry($entry->id))->toThrow(DefinitionNotFoundException::class) + ->and($engines->deadLetterStore->findById($entry->id)?->id)->toBe($entry->id); +}); diff --git a/packages/orchestration/tests/Support/RecordingWorkflowDispatcher.php b/packages/orchestration/tests/Support/RecordingWorkflowDispatcher.php new file mode 100644 index 00000000..65c6da0f --- /dev/null +++ b/packages/orchestration/tests/Support/RecordingWorkflowDispatcher.php @@ -0,0 +1,32 @@ + */ + public array $dispatched = []; + + public function dispatch(string $correlationId): void + { + $this->dispatched[] = $correlationId; + } + + /** @return list */ + public function runAll(WorkflowEngine $engine): array + { + $states = []; + while ($this->dispatched !== []) { + $states[] = $engine->run(array_shift($this->dispatched)); + } + + return $states; + } +} diff --git a/packages/orchestration/tests/Support/TestEngines.php b/packages/orchestration/tests/Support/TestEngines.php index e6b7f6ed..15f7a24c 100644 --- a/packages/orchestration/tests/Support/TestEngines.php +++ b/packages/orchestration/tests/Support/TestEngines.php @@ -12,7 +12,9 @@ use Firefly\Orchestration\Core\ExecutionAdmission; use Firefly\Orchestration\Core\ExecutionLock; use Firefly\Orchestration\Core\InvocationRunner; +use Firefly\Orchestration\Core\Orchestrator; use Firefly\Orchestration\Core\StepInvoker; +use Firefly\Orchestration\Dlq\DeadLetterRetrier; use Firefly\Orchestration\Dlq\DeadLetterService; use Firefly\Orchestration\Dlq\DeadLetterStore; use Firefly\Orchestration\Dlq\MemoryDeadLetterStore; @@ -33,6 +35,10 @@ use Firefly\Orchestration\Tcc\Engine\TccEngine; use Firefly\Orchestration\Tcc\Engine\TccExecutionOrchestrator; use Firefly\Orchestration\Tcc\Registry\TccRegistry; +use Firefly\Orchestration\Workflow\Engine\WaitEvaluator; +use Firefly\Orchestration\Workflow\Engine\WorkflowEngine; +use Firefly\Orchestration\Workflow\Engine\WorkflowExecutor; +use Firefly\Orchestration\Workflow\Registry\WorkflowRegistry; use Firefly\Resilience\CircuitBreaker; use Firefly\Resilience\ResilienceRegistry; use Firefly\Resilience\Store\InMemoryResilienceStore; @@ -85,6 +91,16 @@ final class TestEngines public readonly TccCompositor $tccCompositor; + public readonly WorkflowRegistry $workflowRegistry; + + public readonly RecordingWorkflowDispatcher $dispatcher; + + public readonly WorkflowEngine $workflows; + + public readonly Orchestrator $orchestrator; + + public readonly DeadLetterRetrier $retrier; + /** * @param array $config * @param array $resilience @@ -145,6 +161,20 @@ private function __construct(array $config, ?ExecutionPersistenceProvider $persi $this->tccRegistry = new TccRegistry; $this->tccs = new TccEngine($this->tccRegistry, new TccExecutionOrchestrator($this->invoker, $this->support, $this->settings->tcc), $this->support, $this->settings->tcc); $this->tccCompositor = new TccCompositor($this->tccs, $this->events, $this->conditions, new CompositionDataFlowManager, $this->settings->tcc, $resilienceRegistry, $resilienceStore); + + $this->workflowRegistry = new WorkflowRegistry; + $this->dispatcher = new RecordingWorkflowDispatcher; + $this->workflows = new WorkflowEngine( + $this->workflowRegistry, + new WorkflowExecutor($this->invoker, $this->support, $this->conditions, new WaitEvaluator($this->events)), + $this->support, + $this->settings->workflow, + $this->dispatcher, + $this->container, + $this->settings->backpressure->batchSize, + ); + $this->orchestrator = new Orchestrator($this->sagas, $this->tccs, $this->workflows, $this->persistence); + $this->retrier = new DeadLetterRetrier($this->support->deadLetters, $this->orchestrator); } /** diff --git a/packages/orchestration/tests/Workflow/Engine/WorkflowChildrenTest.php b/packages/orchestration/tests/Workflow/Engine/WorkflowChildrenTest.php new file mode 100644 index 00000000..6440ee5c --- /dev/null +++ b/packages/orchestration/tests/Workflow/Engine/WorkflowChildrenTest.php @@ -0,0 +1,60 @@ +workflowRegistry->register(WorkflowBuilder::workflow('check')->triggerMode($childMode) + ->step('run')->handler(static function (array $input) use ($childFails): string { + if ($childFails) { + throw new RuntimeException('check failed'); + } + + return 'checked '.(is_string($input['subject'] ?? null) ? $input['subject'] : '?'); + })->add()->build()); + $engines->workflowRegistry->register(WorkflowBuilder::workflow('parent') + ->step('spawn')->childWorkflow('check')->handler(static fn (): array => ['subject' => 'ada'])->add() + ->step('after')->dependsOn('spawn')->handler(static fn (): string => 'after child')->add() + ->build()); + + return $engines; +} + +it('runs a SYNC child inline and takes its results as the step result', function (): void { + $engines = orchestrationChildEngines(TriggerMode::Sync); + + $parent = $engines->workflows->start('parent', correlationId: 'p-1'); + $childId = $parent->children['spawn'] ?? ''; + + expect($parent->status)->toBe(ExecutionStatus::Completed) + ->and(data_get($parent->stepResults, 'spawn.results'))->toBe(['run' => 'checked ada']) + ->and($engines->workflows->find($childId)?->parentCorrelationId)->toBe('p-1') + ->and($engines->events->count('onChildWorkflowStarted'))->toBe(1); +}); + +it('waits durably for an ASYNC child and resumes when the worker finishes it', function (): void { + $engines = orchestrationChildEngines(TriggerMode::Async); + + $waiting = $engines->workflows->start('parent', correlationId: 'p-2'); + $engines->dispatcher->runAll($engines->workflows); + + expect($waiting->status)->toBe(ExecutionStatus::Waiting) + ->and($waiting->waitingOn)->toBe('spawn') + ->and($engines->workflows->find('p-2')?->status)->toBe(ExecutionStatus::Completed) + ->and($engines->events->count('onChildWorkflowCompleted'))->toBe(1); +}); + +it('fails the parent step when the child fails', function (): void { + $engines = orchestrationChildEngines(TriggerMode::Sync, childFails: true); + + $parent = $engines->workflows->start('parent', correlationId: 'p-3'); + + expect($parent->status)->toBe(ExecutionStatus::Failed) + ->and($parent->failureReason)->toContain('ended FAILED: check failed'); +}); diff --git a/packages/orchestration/tests/Workflow/Engine/WorkflowEngineTest.php b/packages/orchestration/tests/Workflow/Engine/WorkflowEngineTest.php new file mode 100644 index 00000000..974aaddb --- /dev/null +++ b/packages/orchestration/tests/Workflow/Engine/WorkflowEngineTest.php @@ -0,0 +1,139 @@ +workflowRegistry->register(WorkflowBuilder::workflow('onboard') + ->step('create')->handler(static function (array $input) use (&$log): array { + $log[] = 'create'; + + return ['userId' => 'u-'.(is_string($input['email'] ?? null) ? $input['email'] : '')]; + })->setVariable('user')->outputEventType('users.created')->add() + ->step('vip')->dependsOn('create')->condition('input["tier"] == "gold"')->handler(static function () use (&$log): string { + $log[] = 'vip'; + + return 'vip'; + })->add() + ->step('welcome')->dependsOn('create')->handler(static function (mixed $input, ExecutionContext $ctx) use (&$log): string { + $user = $ctx->variable('user'); + $log[] = 'welcome '.(is_array($user) && is_string($user['userId'] ?? null) ? $user['userId'] : '?'); + + return 'sent'; + })->add() + ->onStepComplete(static function (ExecutionContext $ctx, mixed $result) use (&$completed): void { + $completed[] = is_string($result) ? $result : 'array'; + }) + ->build()); + + $state = $engines->workflows->start('onboard', ['email' => 'ada', 'tier' => 'silver'], correlationId: 'w-1'); + + expect($state->status)->toBe(ExecutionStatus::Completed) + ->and($log)->toBe(['create', 'welcome u-ada']) + ->and($state->stepStatus('vip'))->toBe(StepStatus::Skipped) + ->and($state->stepResults['welcome'] ?? null)->toBe('sent') + ->and($state->report['status'] ?? null)->toBe('COMPLETED') + ->and($completed)->toBe(['array', 'sent']) + ->and(array_column($engines->eda->published, 'eventType'))->toBe(['users.created']) + ->and($engines->workflows->find('w-1')?->status)->toBe(ExecutionStatus::Completed); +}); + +it('compensates completed compensatable steps in reverse when a step fails, and dead-letters it', function (): void { + $engines = TestEngines::create(); + $log = []; + $engines->workflowRegistry->register(WorkflowBuilder::workflow('provision') + ->step('vm')->handler(static fn (): string => 'vm-1')->compensation(static function (mixed $result) use (&$log): void { + $log[] = 'destroy '.(is_string($result) ? $result : '?'); + })->add() + ->step('dns')->dependsOn('vm')->handler(static fn (): string => 'dns-1')->compensation(static function () use (&$log): void { + $log[] = 'remove dns'; + })->add() + ->step('tls')->dependsOn('dns')->handler(static fn (): never => throw new RuntimeException('acme down'))->add() + ->build()); + + $state = $engines->workflows->start('provision', correlationId: 'w-2'); + + expect($state->status)->toBe(ExecutionStatus::Failed) + ->and($state->failureReason)->toBe('acme down') + ->and($log)->toBe(['remove dns', 'destroy vm-1']) + ->and($state->stepStatus('vm'))->toBe(StepStatus::Compensated) + ->and($engines->deadLetterStore->findByCorrelationId('w-2')[0]->stepId ?? null)->toBe('tls'); +}); + +it('ends COMPLETED when an error callback suppresses the failure', function (): void { + $engines = TestEngines::create(); + $engines->workflowRegistry->register(WorkflowBuilder::workflow('soft') + ->step('try')->handler(static fn (): never => throw new RuntimeException('nope'))->add() + ->onError(static function (): void {}, suppressError: true, stepIds: ['try']) + ->build()); + + expect($engines->workflows->start('soft')->status)->toBe(ExecutionStatus::Completed); +}); + +it('runs an ASYNC workflow from the queue, and a duplicate delivery is a no-op', function (): void { + $engines = TestEngines::create(); + $runs = 0; + $engines->workflowRegistry->register(WorkflowBuilder::workflow('report')->triggerMode(TriggerMode::Async) + ->step('build')->handler(static function () use (&$runs): string { + $runs++; + + return 'pdf'; + })->add()->build()); + + $pending = $engines->workflows->start('report', correlationId: 'w-3'); + + expect($pending->status)->toBe(ExecutionStatus::Pending) + ->and($engines->dispatcher->dispatched)->toBe(['w-3']); + + $states = $engines->dispatcher->runAll($engines->workflows); + + expect($states[0]->status)->toBe(ExecutionStatus::Completed) + ->and($engines->workflows->run('w-3')->status)->toBe(ExecutionStatus::Completed) + ->and($runs)->toBe(1); +}); + +it('continues as new, carrying the parent link forward and pointing the old run at the new one', function (): void { + $engines = TestEngines::create(); + $engines->workflowRegistry->register(WorkflowBuilder::workflow('poll') + ->step('tick')->handler(static function (array $input): mixed { + $n = is_int($input['n'] ?? null) ? $input['n'] : 0; + + return $n < 2 ? new ContinueAsNew(['n' => $n + 1]) : "done at {$n}"; + })->add()->build()); + + $last = $engines->workflows->start('poll', ['n' => 0], correlationId: 'w-4'); + $first = $engines->workflows->find('w-4'); + + expect($last->status)->toBe(ExecutionStatus::Completed) + ->and($last->stepResults['tick'] ?? null)->toBe('done at 2') + ->and($first?->status)->toBe(ExecutionStatus::Completed) + ->and(is_string($first?->variables['continuedAs'] ?? null))->toBeTrue() + ->and($last->headers[WorkflowEngine::CONTINUED_FROM_HEADER] ?? null)->toBe('w-4') + ->and($engines->events->count('onContinueAsNew'))->toBe(2); +}); + +it('times a workflow out at its deadline, waits included', function (): void { + $engines = TestEngines::create(); + $engines->workflowRegistry->register(WorkflowBuilder::workflow('deadline')->timeoutMs(1000) + ->step('wait')->waitForSignal('never')->add()->build()); + + $waiting = $engines->workflows->start('deadline', correlationId: 'w-5'); + $engines->clock->advance(2.0); + $engines->workflows->resumeDue(); + + expect($waiting->status)->toBe(ExecutionStatus::Waiting) + ->and(ExecutionState::instant($waiting->wakeAt ?? new DateTimeImmutable))->toBe('2026-10-12T10:00:01.000000+00:00') + ->and($engines->workflows->find('w-5')?->status)->toBe(ExecutionStatus::TimedOut); +}); diff --git a/packages/orchestration/tests/Workflow/Engine/WorkflowLifecycleTest.php b/packages/orchestration/tests/Workflow/Engine/WorkflowLifecycleTest.php new file mode 100644 index 00000000..75bbf94b --- /dev/null +++ b/packages/orchestration/tests/Workflow/Engine/WorkflowLifecycleTest.php @@ -0,0 +1,60 @@ +workflowRegistry->register(WorkflowBuilder::workflow('order') + ->step('place')->handler(static function (array $input, ExecutionContext $ctx): string { + $ctx->setSearchAttribute('customer', is_string($input['customer'] ?? null) ? $input['customer'] : '?'); + + return 'placed'; + })->add() + ->step('ship')->dependsOn('place')->waitForSignal('paid')->handler(static fn (): string => 'shipped')->add() + ->query('progress', static function (ExecutionContext $ctx, mixed ...$arguments): string { + $unit = is_string($arguments[0] ?? null) ? $arguments[0] : 'steps'; + + return count($ctx->results())." {$unit}"; + }) + ->build()); + + return $engines; +} + +it('suspends and resumes a waiting workflow, signals included', function (): void { + $engines = orchestrationLifecycleEngines(); + $engines->workflows->start('order', ['customer' => 'ada'], correlationId: 'o-1'); + + $suspended = $engines->workflows->suspend('o-1', 'fraud review'); + $buffered = $engines->workflows->signal('o-1', 'paid'); + $resumed = $engines->workflows->resume('o-1'); + + expect($suspended->status)->toBe(ExecutionStatus::Suspended) + ->and($suspended->wakeAt)->toBeNull() + ->and($buffered->status)->toBe(ExecutionStatus::Suspended) + ->and($resumed->status)->toBe(ExecutionStatus::Completed) + ->and($engines->events->hooks())->toContain('onWorkflowSuspended', 'onWorkflowResumed') + ->and(fn () => $engines->workflows->resume('o-1'))->toThrow(InvalidExecutionStateException::class); +}); + +it('cancels a workflow and answers queries and search from its checkpoint', function (): void { + $engines = orchestrationLifecycleEngines(); + $engines->workflows->start('order', ['customer' => 'ada'], correlationId: 'o-1'); + $engines->workflows->start('order', ['customer' => 'grace'], correlationId: 'o-2'); + $engines->workflows->upsertSearchAttributes('o-2', ['priority' => 'high']); + + $cancelled = $engines->workflows->cancel('o-1'); + + expect($cancelled->status)->toBe(ExecutionStatus::Cancelled) + ->and($engines->workflows->query('o-2', 'progress', ['results']))->toBe('1 results') + ->and(array_map(static fn ($s): string => $s->correlationId, $engines->workflows->search(['customer' => 'grace', 'priority' => 'high'])))->toBe(['o-2']) + ->and(array_map(static fn ($s): string => $s->correlationId, $engines->workflows->findByStatus(ExecutionStatus::Waiting)))->toBe(['o-2']) + ->and(fn () => $engines->workflows->cancel('o-1'))->toThrow(InvalidExecutionStateException::class); +}); diff --git a/packages/orchestration/tests/Workflow/Engine/WorkflowWaitsTest.php b/packages/orchestration/tests/Workflow/Engine/WorkflowWaitsTest.php new file mode 100644 index 00000000..10aa0437 --- /dev/null +++ b/packages/orchestration/tests/Workflow/Engine/WorkflowWaitsTest.php @@ -0,0 +1,96 @@ +workflowRegistry->register(WorkflowBuilder::workflow('approval') + ->step('submit')->handler(static fn (): string => 'submitted')->add() + ->step('approve')->dependsOn('submit')->waitForSignal('approved')->handler(static function (mixed $input, ExecutionContext $ctx): string { + $payload = $ctx->variable('signals.approved'); + + return 'approved by '.(is_array($payload) && is_string($payload['by'] ?? null) ? $payload['by'] : '?'); + })->add() + ->build()); + + $waiting = $engines->workflows->start('approval', correlationId: 'a-1'); + $done = $engines->workflows->signal('a-1', 'approved', ['by' => 'grace']); + + expect($waiting->status)->toBe(ExecutionStatus::Waiting) + ->and($waiting->waitingOn)->toBe('approve') + ->and($waiting->stepStatus('approve'))->toBe(StepStatus::Waiting) + ->and($done->status)->toBe(ExecutionStatus::Completed) + ->and($done->stepResults['approve'] ?? null)->toBe('approved by grace') + ->and($engines->events->count('onSignalDelivered'))->toBe(1) + ->and(fn () => $engines->workflows->signal('a-1', 'approved'))->toThrow(InvalidExecutionStateException::class, 'accepts no signals'); +}); + +it('buffers a signal that arrives before its step and fires timers from the tick', function (): void { + $engines = TestEngines::create(); + $engines->workflowRegistry->register(WorkflowBuilder::workflow('cooldown') + ->step('pause')->waitForTimer(600_000, 'cool')->add() + ->step('go')->dependsOn('pause')->waitForSignal('go')->handler(static fn (): string => 'went')->add() + ->build()); + + $engines->workflows->start('cooldown', correlationId: 'c-1'); + $stillWaiting = $engines->workflows->signal('c-1', 'go'); + $engines->clock->advance(599.0); + $early = $engines->workflows->resumeDue(); + $engines->clock->advance(1.0); + $resumed = $engines->workflows->resumeDue(); + + expect($stillWaiting->status)->toBe(ExecutionStatus::Waiting) + ->and($stillWaiting->waitingOn)->toBe('pause') + ->and($early)->toBe([]) + ->and($resumed[0]->status ?? null)->toBe(ExecutionStatus::Completed) + ->and($engines->events->count('onTimerFired'))->toBe(1); +}); + +it('fails a step whose signal did not arrive in time, and compensates', function (): void { + $engines = TestEngines::create(); + $log = []; + $engines->workflowRegistry->register(WorkflowBuilder::workflow('kyc') + ->step('open')->handler(static fn (): string => 'account')->compensation(static function () use (&$log): void { + $log[] = 'close account'; + })->add() + ->step('documents')->dependsOn('open')->waitForSignal('documents', 1000)->add() + ->build()); + + $engines->workflows->start('kyc', correlationId: 'k-1'); + $engines->clock->advance(1.5); + $engines->workflows->resumeDue(); + $state = $engines->workflows->find('k-1'); + + expect($state?->status)->toBe(ExecutionStatus::Failed) + ->and($state?->stepStatus('documents'))->toBe(StepStatus::TimedOut) + ->and($state?->failureReason)->toContain('waited longer than 1000 ms for signal [documents]') + ->and($log)->toBe(['close account']); +}); + +it('waits for all of a set, and for any of a set', function (): void { + $engines = TestEngines::create(); + $engines->workflowRegistry->register(WorkflowBuilder::workflow('all') + ->step('both')->waitForAll([new SignalWait('legal'), new SignalWait('finance')])->handler(static fn (): string => 'signed off')->add()->build()); + $engines->workflowRegistry->register(WorkflowBuilder::workflow('any') + ->step('first')->waitForAny([new SignalWait('manual')], [new TimerWait('auto', 60_000)])->handler(static fn (mixed $input, ExecutionContext $ctx): string => 'by '.(is_string($ctx->variable('wait.trigger')) ? $ctx->variable('wait.trigger') : '?'))->add()->build()); + + $engines->workflows->start('all', correlationId: 'all-1'); + $half = $engines->workflows->signal('all-1', 'legal'); + $full = $engines->workflows->signal('all-1', 'finance'); + $engines->workflows->start('any', correlationId: 'any-1'); + $engines->clock->advance(60.0); + $byTimer = $engines->workflows->resumeDue()[0] ?? null; + + expect($half->status)->toBe(ExecutionStatus::Waiting) + ->and($full->status)->toBe(ExecutionStatus::Completed) + ->and($byTimer?->stepResults['first'] ?? null)->toBe('by auto'); +}); From 29b9af8d58a6324714cc632ca4f8181de8410f71 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 20:43:33 -0700 Subject: [PATCH 27/58] fix(orchestration): keep a cancelled or compensated parent from being re-driven by its children, and keep the tick going past an execution it cannot drive --- .../src/Workflow/Engine/WorkflowEngine.php | 57 ++++++++++++++++--- .../Workflow/Engine/WorkflowChildrenTest.php | 52 +++++++++++++++++ .../Workflow/Engine/WorkflowWaitsTest.php | 25 ++++++++ 3 files changed, 125 insertions(+), 9 deletions(-) diff --git a/packages/orchestration/src/Workflow/Engine/WorkflowEngine.php b/packages/orchestration/src/Workflow/Engine/WorkflowEngine.php index 1444fd11..e82ceb4c 100644 --- a/packages/orchestration/src/Workflow/Engine/WorkflowEngine.php +++ b/packages/orchestration/src/Workflow/Engine/WorkflowEngine.php @@ -27,6 +27,7 @@ use Firefly\Orchestration\Workflow\Registry\WorkflowRegistry; use Illuminate\Contracts\Container\Container; use Illuminate\Support\Str; +use Throwable; /** * Runs durable workflows (jafly WorkflowEngine plus its Signal/Timer/ChildWorkflow/ContinueAsNew/Query/Search @@ -36,15 +37,20 @@ * A SYNC start drives inline; an ASYNC start persists PENDING and dispatches a RunWorkflowJob. A WAITING * workflow is re-driven by signal(), by the tick (resumeDue()) once a timer or its deadline is due, and by a * child's completion. A child that finishes while its parent is being driven in THIS process is completed - * inline by that drive; one that finishes elsewhere (a queue worker) wakes the parent itself. + * inline by that drive; one that finishes elsewhere (a queue worker) wakes the parent itself. A parent that is + * being cancelled or compensated cascades to its children, and those children do not wake it back up: the + * parent is settling, so it is not re-driven from inside its own cancel()/compensate(). */ final class WorkflowEngine { public const string CONTINUED_FROM_HEADER = 'x-firefly-continued-from'; - /** @var array */ + /** @var array the executions this process is driving right now */ private array $driving = []; + /** @var array the executions this process is cancelling or compensating right now */ + private array $settling = []; + public function __construct( private readonly WorkflowRegistry $registry, private readonly WorkflowExecutor $executor, @@ -150,6 +156,13 @@ public function resumeDue(): array }); } catch (ExecutionLockedException) { // Another process is advancing it right now; the next tick sees whatever it left. + } catch (Throwable $error) { + // One execution that cannot be driven (its definition was removed in a deploy, a handler blew up + // outside a step) must not stop the rest of the batch: report it and move on to the next row. + $this->support->logger->warning("Workflow [{$due->executionName}] execution [{$due->correlationId}] could not be resumed by the tick: {$error->getMessage()}", [ + 'exception' => $error, + 'correlationId' => $due->correlationId, + ]); } } @@ -200,10 +213,14 @@ public function cancel(string $correlationId): ExecutionState } $context = ExecutionContext::fromState($state); - $this->cancelChildren($context); - $cancelled = $this->support->checkpoint($context, ExecutionStatus::Cancelled, 'cancelled'); - $this->support->publish($this->support->lifecycle, OrchestrationEvent::executionCompleted($state->executionName, $correlationId, ExecutionPattern::Workflow, ExecutionStatus::Cancelled, $this->support->now())); - $this->support->events->onCompleted($state->executionName, $correlationId, ExecutionPattern::Workflow, false, $this->support->elapsedMs($context)); + $cancelled = $this->settle($correlationId, function () use ($context, $state, $correlationId): ExecutionState { + $this->cancelChildren($context); + $cancelled = $this->support->checkpoint($context, ExecutionStatus::Cancelled, 'cancelled'); + $this->support->publish($this->support->lifecycle, OrchestrationEvent::executionCompleted($state->executionName, $correlationId, ExecutionPattern::Workflow, ExecutionStatus::Cancelled, $this->support->now())); + $this->support->events->onCompleted($state->executionName, $correlationId, ExecutionPattern::Workflow, false, $this->support->elapsedMs($context)); + + return $cancelled; + }); $this->notifyParent($cancelled); return $cancelled; @@ -220,8 +237,10 @@ public function compensate(string $correlationId): ExecutionState $definition = $this->registry->get($state->executionName); $context = ExecutionContext::fromState($state); - $this->cancelChildren($context); - $this->executor->compensate($definition, $context); + $this->settle($correlationId, function () use ($definition, $context): void { + $this->cancelChildren($context); + $this->executor->compensate($definition, $context); + }); return $this->finish($definition, $context, ExecutionStatus::Cancelled, 'compensated on request'); }); @@ -429,7 +448,7 @@ private function finish(WorkflowDefinition $workflow, ExecutionContext $context, private function notifyParent(ExecutionState $child): void { $parentId = $child->parentCorrelationId; - if ($parentId === null || isset($this->driving[$parentId])) { + if ($parentId === null || isset($this->driving[$parentId]) || isset($this->settling[$parentId])) { return; } @@ -452,6 +471,26 @@ private function notifyParent(ExecutionState $child): void }); } + /** + * Runs `$work` with `$correlationId` marked as settling, so a child it cancels does not wake it back up + * through notifyParent(): the lock is reentrant, and the parent is still WAITING on that child until + * `$work` has moved it on. + * + * @template T + * + * @param callable(): T $work + * @return T + */ + private function settle(string $correlationId, callable $work): mixed + { + $this->settling[$correlationId] = true; + try { + return $work(); + } finally { + unset($this->settling[$correlationId]); + } + } + private function cancelChildren(ExecutionContext $context): void { foreach ($context->children() as $childId) { diff --git a/packages/orchestration/tests/Workflow/Engine/WorkflowChildrenTest.php b/packages/orchestration/tests/Workflow/Engine/WorkflowChildrenTest.php index 6440ee5c..af047324 100644 --- a/packages/orchestration/tests/Workflow/Engine/WorkflowChildrenTest.php +++ b/packages/orchestration/tests/Workflow/Engine/WorkflowChildrenTest.php @@ -58,3 +58,55 @@ function orchestrationChildEngines(TriggerMode $childMode, bool $childFails = fa expect($parent->status)->toBe(ExecutionStatus::Failed) ->and($parent->failureReason)->toContain('ended FAILED: check failed'); }); + +/** + * @param list $log + */ +function orchestrationPayingParentEngines(array &$log): TestEngines +{ + $engines = TestEngines::create(); + $engines->workflowRegistry->register(WorkflowBuilder::workflow('slow-check')->triggerMode(TriggerMode::Async) + ->step('run')->handler(static fn (): string => 'checked')->add()->build()); + $engines->workflowRegistry->register(WorkflowBuilder::workflow('paying-parent') + ->step('pay')->handler(static fn (): string => 'paid')->compensation(static function () use (&$log): void { + $log[] = 'refund'; + })->add() + ->step('spawn')->dependsOn('pay')->childWorkflow('slow-check')->handler(static fn (): array => [])->add() + ->build()); + + return $engines; +} + +it('cancels a parent waiting on an ASYNC child without re-driving the parent from the child', function (): void { + $log = []; + $engines = orchestrationPayingParentEngines($log); + + $waiting = $engines->workflows->start('paying-parent', correlationId: 'pp-1'); + $childId = $waiting->children['spawn'] ?? ''; + $cancelled = $engines->workflows->cancel('pp-1'); + + expect($waiting->status)->toBe(ExecutionStatus::Waiting) + ->and($cancelled->status)->toBe(ExecutionStatus::Cancelled) + ->and($engines->workflows->find('pp-1')?->status)->toBe(ExecutionStatus::Cancelled) + ->and($engines->workflows->find($childId)?->status)->toBe(ExecutionStatus::Cancelled) + ->and($log)->toBe([]) + ->and($engines->deadLetterStore->findByCorrelationId('pp-1'))->toBe([]) + ->and($engines->events->count('onCompleted'))->toBe(2); +}); + +it('compensates a parent waiting on an ASYNC child exactly once and does not dead-letter it', function (): void { + $log = []; + $engines = orchestrationPayingParentEngines($log); + + $waiting = $engines->workflows->start('paying-parent', correlationId: 'pp-2'); + $childId = $waiting->children['spawn'] ?? ''; + $compensated = $engines->workflows->compensate('pp-2'); + + expect($compensated->status)->toBe(ExecutionStatus::Cancelled) + ->and($compensated->failureReason)->toBe('compensated on request') + ->and($engines->workflows->find('pp-2')?->status)->toBe(ExecutionStatus::Cancelled) + ->and($engines->workflows->find($childId)?->status)->toBe(ExecutionStatus::Cancelled) + ->and($log)->toBe(['refund']) + ->and($engines->deadLetterStore->findByCorrelationId('pp-2'))->toBe([]) + ->and($engines->events->count('onCompleted'))->toBe(2); +}); diff --git a/packages/orchestration/tests/Workflow/Engine/WorkflowWaitsTest.php b/packages/orchestration/tests/Workflow/Engine/WorkflowWaitsTest.php index 10aa0437..4ea09006 100644 --- a/packages/orchestration/tests/Workflow/Engine/WorkflowWaitsTest.php +++ b/packages/orchestration/tests/Workflow/Engine/WorkflowWaitsTest.php @@ -94,3 +94,28 @@ ->and($full->status)->toBe(ExecutionStatus::Completed) ->and($byTimer?->stepResults['first'] ?? null)->toBe('by auto'); }); + +it('keeps advancing the other due executions when one of them cannot be driven', function (): void { + $timer = static fn (string $id) => WorkflowBuilder::workflow($id) + ->step('pause')->waitForTimer(60_000)->add() + ->step('go')->dependsOn('pause')->handler(static fn (): string => 'went')->add() + ->build(); + $before = TestEngines::create(); + $before->workflowRegistry->register($timer('retired')); + $before->workflowRegistry->register($timer('kept')); + $before->workflows->start('retired', correlationId: 'a-retired'); + $before->workflows->start('kept', correlationId: 'b-kept'); + + // A deploy drops [retired] while its execution still waits on a timer that is due first. + $after = TestEngines::create(persistence: $before->persistence); + $after->workflowRegistry->register($timer('kept')); + $after->clock->advance(61.0); + $advanced = $after->workflows->resumeDue(); + $warnings = $after->logger->messages('warning'); + + expect(array_map(static fn ($state): string => $state->correlationId, $advanced))->toBe(['b-kept']) + ->and($after->workflows->find('b-kept')?->status)->toBe(ExecutionStatus::Completed) + ->and($after->workflows->find('a-retired')?->status)->toBe(ExecutionStatus::Waiting) + ->and($warnings)->toHaveCount(1) + ->and($warnings[0] ?? '')->toContain('a-retired'); +}); From 80860fc25db02a3030ea2cd3f803a53dbaf04b73 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 20:57:42 -0700 Subject: [PATCH 28/58] fix(orchestration): re-check a waiting parent's child on every drive, keep the continue-as-new origin out of children, fail a parent whose child cannot start, and refuse a taken workflow correlation id --- .../src/Workflow/Engine/WorkflowEngine.php | 83 ++++++++-- .../src/Workflow/Engine/WorkflowExecutor.php | 42 ++++- .../Support/RecordingWorkflowDispatcher.php | 11 ++ .../tests/Support/ScriptedLock.php | 31 ++++ .../tests/Support/TestEngines.php | 10 +- .../Workflow/Engine/WorkflowChildrenTest.php | 148 ++++++++++++++++++ .../Workflow/Engine/WorkflowEngineTest.php | 29 ++++ 7 files changed, 339 insertions(+), 15 deletions(-) create mode 100644 packages/orchestration/tests/Support/ScriptedLock.php diff --git a/packages/orchestration/src/Workflow/Engine/WorkflowEngine.php b/packages/orchestration/src/Workflow/Engine/WorkflowEngine.php index e82ceb4c..52baad73 100644 --- a/packages/orchestration/src/Workflow/Engine/WorkflowEngine.php +++ b/packages/orchestration/src/Workflow/Engine/WorkflowEngine.php @@ -79,23 +79,29 @@ public function start(string|WorkflowDefinition $workflow, array $input = [], ar $definition = is_string($workflow) ? $this->registry->get($workflow) : $workflow; $context = ExecutionContext::start(ExecutionPattern::Workflow, $definition->id, $input, $headers, $correlationId, $this->support->now()); $context->setParentCorrelationId($parentCorrelationId); + $this->refuseExisting($context->correlationId); return $this->support->admission->admit( $definition->id, function () use ($definition, $context): ExecutionState { if ($definition->triggerMode === TriggerMode::Async) { + $this->refuseExisting($context->correlationId); $pending = $this->support->checkpoint($context, ExecutionStatus::Pending); $this->dispatcher->dispatch($context->correlationId); return $this->support->persistence->findById($context->correlationId) ?? $pending; } - return $this->support->lock->synchronized($context->correlationId, fn (): ExecutionState => $this->support->tracing->traceExecution( - $definition->id, - ExecutionPattern::Workflow, - $context->correlationId, - fn (): ExecutionState => $this->begin($definition, $context), - )); + return $this->support->lock->synchronized($context->correlationId, function () use ($definition, $context): ExecutionState { + $this->refuseExisting($context->correlationId); + + return $this->support->tracing->traceExecution( + $definition->id, + ExecutionPattern::Workflow, + $context->correlationId, + fn (): ExecutionState => $this->begin($definition, $context), + ); + }); }, static fn (ExecutionState $state): bool => ! in_array($state->status, [ExecutionStatus::Failed, ExecutionStatus::TimedOut], true), ); @@ -329,8 +335,18 @@ private function drive(WorkflowDefinition $workflow, ExecutionContext $context): while (true) { $outcome = $this->executor->run($workflow, $context); - if ($outcome->kind === WorkflowRunOutcome::CHILD && $this->startChild($workflow, $context, (string) $outcome->stepId, $outcome->childInput)) { - continue; + if ($outcome->kind === WorkflowRunOutcome::CHILD) { + $stepId = (string) $outcome->stepId; + try { + if ($this->startChild($workflow, $context, $stepId, $outcome->childInput)) { + continue; + } + } catch (Throwable $error) { + // The child could not be started at all (its definition is gone, the pattern is disabled, + // admission refused it, the ASYNC dispatch failed): that is this step failing, so the + // parent compensates, dead-letters and ends FAILED instead of staying RUNNING. + $outcome = $this->executor->failed($workflow, $context, $stepId, $error); + } } return match ($outcome->kind) { @@ -358,7 +374,23 @@ private function startChild(WorkflowDefinition $workflow, ExecutionContext $cont return true; } - $childState = $this->start($child->workflowId, $input, $context->headers(), null, $context->correlationId); + // The child is a chain of its own: the parent's continue-as-new origin must not leak into it, or the + // child's completion would be matched against the parent's original run id instead of its own. + $headers = $context->headers(); + unset($headers[self::CONTINUED_FROM_HEADER]); + + $childId = Str::uuid()->toString(); + try { + $childState = $this->start($child->workflowId, $input, $headers, $childId, $context->correlationId); + } catch (Throwable $error) { + // A child that got as far as its first checkpoint (an ASYNC dispatch that failed) is recorded, so + // the parent's failure cancels it rather than leaving it PENDING with nobody waiting for it. + if ($this->support->persistence->findById($childId) !== null) { + $context->setChild($stepId, $childId); + } + + throw $error; + } $context->setChild($stepId, $childState->correlationId); $this->support->events->onChildWorkflowStarted($workflow->id, $context->correlationId, $child->workflowId, $childState->correlationId); @@ -452,6 +484,26 @@ private function notifyParent(ExecutionState $child): void return; } + try { + $this->wakeParent($parentId, $child); + } catch (ExecutionLockedException $locked) { + if ($locked->correlationId !== $parentId) { + throw $locked; + } + // The child is already checkpointed terminal; failing it now would only fail its job, and a redelivery + // is a no-op. Whoever holds the parent is driving it (a signal, the tick, a resume), and every drive of + // a parent waiting on a child reads the child's persisted state first (WorkflowExecutor), so that + // drive, or the next one, settles the step this wake-up could not. + $this->support->logger->warning("Workflow execution [{$child->correlationId}] finished but could not wake its parent [{$parentId}]: {$locked->getMessage()}", [ + 'exception' => $locked, + 'correlationId' => $child->correlationId, + 'parentCorrelationId' => $parentId, + ]); + } + } + + private function wakeParent(string $parentId, ExecutionState $child): void + { $this->support->lock->synchronized($parentId, function () use ($parentId, $child): void { $parent = $this->support->persistence->findById($parentId); if ($parent === null || $parent->status !== ExecutionStatus::Waiting) { @@ -501,6 +553,19 @@ private function cancelChildren(ExecutionContext $context): void } } + /** + * The first checkpoint is an upsert, so starting with a correlation id that is already taken would reset that + * execution to RUNNING and run every step again (SagaEngine and TccEngine refuse it the same way). + * Continue-as-new always takes a fresh id. + */ + private function refuseExisting(string $correlationId): void + { + $state = $this->support->persistence->findById($correlationId); + if ($state !== null) { + throw new InvalidExecutionStateException("Execution [{$correlationId}] already exists ({$state->pattern->value} {$state->executionName}, {$state->status->value}); signal(), resume() or cancel() it instead of starting it again."); + } + } + private function load(string $correlationId): ExecutionState { $state = $this->support->persistence->findById($correlationId); diff --git a/packages/orchestration/src/Workflow/Engine/WorkflowExecutor.php b/packages/orchestration/src/Workflow/Engine/WorkflowExecutor.php index c28e8866..f916c821 100644 --- a/packages/orchestration/src/Workflow/Engine/WorkflowExecutor.php +++ b/packages/orchestration/src/Workflow/Engine/WorkflowExecutor.php @@ -81,7 +81,20 @@ public function run(WorkflowDefinition $workflow, ExecutionContext $context): Wo } } - if ($step->child !== null && $context->child($stepId) !== null) { + $childId = $step->child === null ? null : $context->child($stepId); + if ($step->child !== null && $childId !== null) { + // The child's own wake-up (WorkflowEngine::notifyParent) can be missed: the parent was + // SUSPENDED when the child finished, or its lock was busy. So every drive of a parent still + // waiting on a child looks at the child's persisted state first, and settles the step itself. + $child = $this->settledChild($childId); + if ($child !== null) { + $this->completeChild($workflow, $context, $stepId, $child); + if ($context->stepStatus($stepId)->isFailure()) { + return WorkflowRunOutcome::failed($stepId, $context->stepError($stepId) ?? new OrchestrationException("Workflow [{$workflow->id}] step [{$stepId}] failed.")); + } + + continue; + } if ($context->timerFired("{$stepId}:child-timeout", $now)) { $context->removeTimer("{$stepId}:child-timeout"); @@ -211,7 +224,11 @@ private function invoke(WorkflowDefinition $workflow, WorkflowStepDefinition $st return $this->invoker->invoke($workflow->id, $step->id, $step->invocation, $step->options, $context, $input, $input); } - private function failed(WorkflowDefinition $workflow, ExecutionContext $context, string $stepId, Throwable $error): WorkflowRunOutcome + /** + * Records `$error` as the failure of `$stepId` and returns the FAILED outcome the engine settles (compensate, + * dead-letter, FAILED); the engine calls it too when a child step's child cannot even be started. + */ + public function failed(WorkflowDefinition $workflow, ExecutionContext $context, string $stepId, Throwable $error): WorkflowRunOutcome { $context->setStepStatus($stepId, $error instanceof TimeoutException ? StepStatus::TimedOut : StepStatus::Failed); $context->putStepError($stepId, $error); @@ -221,6 +238,27 @@ private function failed(WorkflowDefinition $workflow, ExecutionContext $context, return WorkflowRunOutcome::failed($stepId, $error); } + /** + * The final run of child `$childId` once it is terminal, following its continue-as-new chain (a run that + * continued as new is COMPLETED, but the child is only finished when the last run of the chain is); null + * while the child is still going, or when its row is gone. + */ + private function settledChild(string $childId): ?ExecutionState + { + $seen = []; + $state = $this->support->persistence->findById($childId); + while ($state !== null && $state->isTerminal()) { + $next = $state->variables['continuedAs'] ?? null; + if (! is_string($next) || $next === '' || isset($seen[$next])) { + return $state; + } + $seen[$next] = true; + $state = $this->support->persistence->findById($next); + } + + return null; + } + /** @return array */ private static function conditionValues(ExecutionContext $context): array { diff --git a/packages/orchestration/tests/Support/RecordingWorkflowDispatcher.php b/packages/orchestration/tests/Support/RecordingWorkflowDispatcher.php index 65c6da0f..21f87b19 100644 --- a/packages/orchestration/tests/Support/RecordingWorkflowDispatcher.php +++ b/packages/orchestration/tests/Support/RecordingWorkflowDispatcher.php @@ -7,6 +7,7 @@ use Firefly\Orchestration\Persistence\ExecutionState; use Firefly\Orchestration\Workflow\Engine\WorkflowEngine; use Firefly\Orchestration\Workflow\Queue\WorkflowDispatcher; +use Throwable; /** Holds ASYNC starts instead of queueing them; runAll() plays the queue worker. */ final class RecordingWorkflowDispatcher implements WorkflowDispatcher @@ -14,8 +15,18 @@ final class RecordingWorkflowDispatcher implements WorkflowDispatcher /** @var list */ public array $dispatched = []; + /** When set, the next dispatch throws it (a queue that is down) instead of recording the start. */ + public ?Throwable $failNext = null; + public function dispatch(string $correlationId): void { + if ($this->failNext !== null) { + $error = $this->failNext; + $this->failNext = null; + + throw $error; + } + $this->dispatched[] = $correlationId; } diff --git a/packages/orchestration/tests/Support/ScriptedLock.php b/packages/orchestration/tests/Support/ScriptedLock.php new file mode 100644 index 00000000..a394535c --- /dev/null +++ b/packages/orchestration/tests/Support/ScriptedLock.php @@ -0,0 +1,31 @@ + */ + private array $heldElsewhere = []; + + public function holdElsewhere(string $name): void + { + $this->heldElsewhere[$name] = true; + } + + public function releaseElsewhere(string $name): void + { + unset($this->heldElsewhere[$name]); + } + + public function tryAcquire(string $name, float $ttlSeconds): bool + { + return ! isset($this->heldElsewhere[$name]); + } + + public function release(string $name): void {} +} diff --git a/packages/orchestration/tests/Support/TestEngines.php b/packages/orchestration/tests/Support/TestEngines.php index 15f7a24c..707ae123 100644 --- a/packages/orchestration/tests/Support/TestEngines.php +++ b/packages/orchestration/tests/Support/TestEngines.php @@ -42,6 +42,7 @@ use Firefly\Resilience\CircuitBreaker; use Firefly\Resilience\ResilienceRegistry; use Firefly\Resilience\Store\InMemoryResilienceStore; +use Firefly\Scheduling\Lock\DistributedLock; use Firefly\Scheduling\Lock\NoneLock; use Firefly\Testing\Double\RecordingEventPublisher; use Illuminate\Config\Repository; @@ -105,7 +106,7 @@ final class TestEngines * @param array $config * @param array $resilience */ - private function __construct(array $config, ?ExecutionPersistenceProvider $persistence, ?DeadLetterStore $deadLetters, array $resilience) + private function __construct(array $config, ?ExecutionPersistenceProvider $persistence, ?DeadLetterStore $deadLetters, array $resilience, ?DistributedLock $lock) { $this->container = new Container; $this->settings = OrchestrationSettings::fromConfig(new Config(new Repository(['firefly' => ['orchestration' => $config]]))); @@ -141,7 +142,7 @@ private function __construct(array $config, ?ExecutionPersistenceProvider $persi tracing: $tracing, callbacks: new CallbackInvoker($this->container, $this->logger), clock: $this->clock, - lock: new ExecutionLock(new NoneLock, $this->settings->lock->ttl, $this->settings->lock->wait), + lock: new ExecutionLock($lock ?? new NoneLock, $this->settings->lock->ttl, $this->settings->lock->wait), admission: new ExecutionAdmission($this->persistence, $this->settings->backpressure, $breaker), logger: $this->logger, ); @@ -180,9 +181,10 @@ private function __construct(array $config, ?ExecutionPersistenceProvider $persi /** * @param array $config * @param array $resilience the `firefly.resilience` section (named retry and breaker instances) + * @param DistributedLock|null $lock the lock backend under ExecutionLock (NoneLock by default) */ - public static function create(array $config = [], ?ExecutionPersistenceProvider $persistence = null, ?DeadLetterStore $deadLetters = null, array $resilience = []): self + public static function create(array $config = [], ?ExecutionPersistenceProvider $persistence = null, ?DeadLetterStore $deadLetters = null, array $resilience = [], ?DistributedLock $lock = null): self { - return new self($config, $persistence, $deadLetters, $resilience); + return new self($config, $persistence, $deadLetters, $resilience, $lock); } } diff --git a/packages/orchestration/tests/Workflow/Engine/WorkflowChildrenTest.php b/packages/orchestration/tests/Workflow/Engine/WorkflowChildrenTest.php index af047324..fc604897 100644 --- a/packages/orchestration/tests/Workflow/Engine/WorkflowChildrenTest.php +++ b/packages/orchestration/tests/Workflow/Engine/WorkflowChildrenTest.php @@ -2,10 +2,15 @@ declare(strict_types=1); +use Firefly\Orchestration\Core\ExecutionLock; use Firefly\Orchestration\Core\ExecutionStatus; +use Firefly\Orchestration\Core\StepStatus; use Firefly\Orchestration\Core\TriggerMode; +use Firefly\Orchestration\Tests\Support\ScriptedLock; use Firefly\Orchestration\Tests\Support\TestEngines; use Firefly\Orchestration\Workflow\Builder\WorkflowBuilder; +use Firefly\Orchestration\Workflow\Engine\ContinueAsNew; +use Firefly\Orchestration\Workflow\Engine\WorkflowEngine; function orchestrationChildEngines(TriggerMode $childMode, bool $childFails = false): TestEngines { @@ -110,3 +115,146 @@ function orchestrationPayingParentEngines(array &$log): TestEngines ->and($engines->deadLetterStore->findByCorrelationId('pp-2'))->toBe([]) ->and($engines->events->count('onCompleted'))->toBe(2); }); + +it('settles a child that finished while its parent was SUSPENDED when the parent is resumed', function (): void { + $engines = orchestrationChildEngines(TriggerMode::Async); + + $engines->workflows->start('parent', correlationId: 'p-4'); + $engines->workflows->suspend('p-4'); + $engines->dispatcher->runAll($engines->workflows); + $childId = $engines->workflows->find('p-4')?->children['spawn'] ?? ''; + $suspended = $engines->workflows->find('p-4'); + + $resumed = $engines->workflows->resume('p-4'); + + expect($engines->workflows->find($childId)?->status)->toBe(ExecutionStatus::Completed) + ->and($suspended?->status)->toBe(ExecutionStatus::Suspended) + ->and($resumed->status)->toBe(ExecutionStatus::Completed) + ->and(data_get($resumed->stepResults, 'spawn.results'))->toBe(['run' => 'checked ada']) + ->and($resumed->stepResults['after'] ?? null)->toBe('after child') + ->and($engines->events->count('onChildWorkflowCompleted'))->toBe(1); +}); + +it('follows a child continue-as-new chain to its last run when the parent re-checks the child', function (): void { + $engines = TestEngines::create(); + $engines->workflowRegistry->register(WorkflowBuilder::workflow('looping-check')->triggerMode(TriggerMode::Async) + ->step('run')->handler(static fn (array $input): mixed => ($input['round'] ?? 0) === 0 ? new ContinueAsNew(['round' => 1]) : 'second round')->add() + ->build()); + $engines->workflowRegistry->register(WorkflowBuilder::workflow('chain-parent') + ->step('spawn')->childWorkflow('looping-check')->handler(static fn (): array => [])->add() + ->build()); + + $engines->workflows->start('chain-parent', correlationId: 'p-5'); + $engines->workflows->suspend('p-5'); + $engines->dispatcher->runAll($engines->workflows); + $firstRun = $engines->workflows->find($engines->workflows->find('p-5')?->children['spawn'] ?? ''); + $lastRunId = $firstRun?->variables['continuedAs'] ?? null; + + $resumed = $engines->workflows->resume('p-5'); + + expect($firstRun?->status)->toBe(ExecutionStatus::Completed) + ->and($resumed->status)->toBe(ExecutionStatus::Completed) + ->and(data_get($resumed->stepResults, 'spawn.correlationId'))->toBe($lastRunId) + ->and(data_get($resumed->stepResults, 'spawn.results'))->toBe(['run' => 'second round']); +}); + +it('keeps a child finished while its parent is locked elsewhere, and the next drive of the parent settles it', function (): void { + $lock = new ScriptedLock; + $engines = TestEngines::create(['lock' => ['wait' => 0]], lock: $lock); + $engines->workflowRegistry->register(WorkflowBuilder::workflow('check')->triggerMode(TriggerMode::Async) + ->step('run')->handler(static fn (): string => 'checked')->add()->build()); + $engines->workflowRegistry->register(WorkflowBuilder::workflow('parent') + ->step('spawn')->childWorkflow('check')->handler(static fn (): array => [])->add() + ->build()); + + $engines->workflows->start('parent', correlationId: 'p-6'); + $lock->holdElsewhere(ExecutionLock::name('p-6')); + $ran = $engines->dispatcher->runAll($engines->workflows); + $whileLocked = $engines->workflows->find('p-6'); + $lock->releaseElsewhere(ExecutionLock::name('p-6')); + + $driven = $engines->workflows->signal('p-6', 'nudge'); + + expect($ran[0]->status ?? null)->toBe(ExecutionStatus::Completed) + ->and($whileLocked?->status)->toBe(ExecutionStatus::Waiting) + ->and($engines->logger->messages('warning')[0] ?? '')->toContain('could not wake its parent [p-6]') + ->and($driven->status)->toBe(ExecutionStatus::Completed) + ->and(data_get($driven->stepResults, 'spawn.results'))->toBe(['run' => 'checked']); +}); + +it('wakes a parent that continued as new when its ASYNC child completes', function (): void { + $engines = TestEngines::create(); + $engines->workflowRegistry->register(WorkflowBuilder::workflow('check')->triggerMode(TriggerMode::Async) + ->step('run')->handler(static fn (): string => 'checked')->add()->build()); + $engines->workflowRegistry->register(WorkflowBuilder::workflow('loop') + ->step('tick')->handler(static fn (array $input): mixed => ($input['round'] ?? 0) === 0 ? new ContinueAsNew(['round' => 1]) : 'ticked')->add() + ->step('spawn')->dependsOn('tick')->childWorkflow('check')->handler(static fn (): array => [])->add() + ->build()); + + $waiting = $engines->workflows->start('loop', correlationId: 'loop-1'); + $childId = $waiting->children['spawn'] ?? ''; + $engines->dispatcher->runAll($engines->workflows); + + expect($waiting->correlationId)->not->toBe('loop-1') + ->and($waiting->status)->toBe(ExecutionStatus::Waiting) + ->and($waiting->headers[WorkflowEngine::CONTINUED_FROM_HEADER] ?? null)->toBe('loop-1') + ->and($engines->workflows->find($childId)?->headers)->not->toHaveKey(WorkflowEngine::CONTINUED_FROM_HEADER) + ->and($engines->workflows->find($waiting->correlationId)?->status)->toBe(ExecutionStatus::Completed); +}); + +it('fails, compensates and dead-letters a parent whose child workflow is not registered', function (): void { + $log = []; + $engines = TestEngines::create(); + $engines->workflowRegistry->register(WorkflowBuilder::workflow('orphan-parent') + ->step('pay')->handler(static fn (): string => 'paid')->compensation(static function () use (&$log): void { + $log[] = 'refund'; + })->add() + ->step('spawn')->dependsOn('pay')->childWorkflow('never-registered')->handler(static fn (): array => [])->add() + ->build()); + + $failed = $engines->workflows->start('orphan-parent', correlationId: 'p-7'); + + expect($failed->status)->toBe(ExecutionStatus::Failed) + ->and($failed->stepStatus('spawn'))->toBe(StepStatus::Failed) + ->and($failed->failureReason)->toContain('never-registered') + ->and($log)->toBe(['refund']) + ->and($engines->workflows->find('p-7')?->status)->toBe(ExecutionStatus::Failed) + ->and($engines->deadLetterStore->findByCorrelationId('p-7'))->toHaveCount(1); +}); + +it('fails, compensates and dead-letters a parent whose child is refused by back-pressure', function (): void { + $log = []; + $engines = TestEngines::create(['backpressure' => ['max-in-flight' => 1]]); + $engines->workflowRegistry->register(WorkflowBuilder::workflow('slow-check')->triggerMode(TriggerMode::Async) + ->step('run')->handler(static fn (): string => 'checked')->add()->build()); + $engines->workflowRegistry->register(WorkflowBuilder::workflow('paying-parent') + ->step('pay')->handler(static fn (): string => 'paid')->compensation(static function () use (&$log): void { + $log[] = 'refund'; + })->add() + ->step('spawn')->dependsOn('pay')->childWorkflow('slow-check')->handler(static fn (): array => [])->add() + ->build()); + + $failed = $engines->workflows->start('paying-parent', correlationId: 'p-8'); + + expect($failed->status)->toBe(ExecutionStatus::Failed) + ->and($failed->stepStatus('spawn'))->toBe(StepStatus::Failed) + ->and($failed->failureReason)->toContain('max-in-flight') + ->and($failed->children)->toBe([]) + ->and($log)->toBe(['refund']) + ->and($engines->deadLetterStore->findByCorrelationId('p-8'))->toHaveCount(1); +}); + +it('cancels a child whose ASYNC dispatch failed when the parent fails over it', function (): void { + $log = []; + $engines = orchestrationPayingParentEngines($log); + $engines->dispatcher->failNext = new RuntimeException('queue is down'); + + $failed = $engines->workflows->start('paying-parent', correlationId: 'p-9'); + $childId = $failed->children['spawn'] ?? ''; + + expect($failed->status)->toBe(ExecutionStatus::Failed) + ->and($failed->failureReason)->toBe('queue is down') + ->and($engines->workflows->find($childId)?->status)->toBe(ExecutionStatus::Cancelled) + ->and($log)->toBe(['refund']) + ->and($engines->deadLetterStore->findByCorrelationId('p-9'))->toHaveCount(1); +}); diff --git a/packages/orchestration/tests/Workflow/Engine/WorkflowEngineTest.php b/packages/orchestration/tests/Workflow/Engine/WorkflowEngineTest.php index 974aaddb..4c88fbf5 100644 --- a/packages/orchestration/tests/Workflow/Engine/WorkflowEngineTest.php +++ b/packages/orchestration/tests/Workflow/Engine/WorkflowEngineTest.php @@ -2,6 +2,7 @@ declare(strict_types=1); +use Firefly\Orchestration\Core\Exception\InvalidExecutionStateException; use Firefly\Orchestration\Core\ExecutionContext; use Firefly\Orchestration\Core\ExecutionStatus; use Firefly\Orchestration\Core\StepStatus; @@ -137,3 +138,31 @@ ->and(ExecutionState::instant($waiting->wakeAt ?? new DateTimeImmutable))->toBe('2026-10-12T10:00:01.000000+00:00') ->and($engines->workflows->find('w-5')?->status)->toBe(ExecutionStatus::TimedOut); }); + +it('refuses to start a workflow under a correlation id that is already taken, and leaves that execution alone', function (TriggerMode $mode): void { + $engines = TestEngines::create(); + $runs = 0; + $engines->workflowRegistry->register(WorkflowBuilder::workflow('once')->triggerMode($mode) + ->step('wait')->waitForSignal('go')->add() + ->step('work')->dependsOn('wait')->handler(static function () use (&$runs): string { + $runs++; + + return 'worked'; + })->add() + ->build()); + $engines->workflows->start('once', correlationId: 'w-dup'); + $engines->dispatcher->runAll($engines->workflows); + $engines->workflows->signal('w-dup', 'go'); + $before = $engines->workflows->find('w-dup'); + + expect(fn () => $engines->workflows->start('once', correlationId: 'w-dup')) + ->toThrow(InvalidExecutionStateException::class, 'already exists'); + + expect($before?->status)->toBe(ExecutionStatus::Completed) + ->and($engines->workflows->find('w-dup')?->toArray())->toBe($before?->toArray()) + ->and($engines->dispatcher->dispatched)->toBe([]) + ->and($runs)->toBe(1); +})->with([ + 'sync' => [TriggerMode::Sync], + 'async' => [TriggerMode::Async], +]); From 1301d88efaada6dc8d6f2db7b6b9cf6820bdf1a7 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 21:04:08 -0700 Subject: [PATCH 29/58] fix(orchestration): record a child's origin run so a child that continued as new still wakes its parent, keep the report and deadline on a search-attribute upsert, back off due workflows the tick cannot drive, and lock the ASYNC start --- .../src/Core/ExecutionContext.php | 5 + .../src/Workflow/Engine/WorkflowEngine.php | 187 ++++++++++++++---- .../Workflow/Engine/WorkflowChildrenTest.php | 31 +++ .../Workflow/Engine/WorkflowEngineTest.php | 26 +++ .../Workflow/Engine/WorkflowLifecycleTest.php | 29 +++ .../Workflow/Engine/WorkflowWaitsTest.php | 31 +++ 6 files changed, 275 insertions(+), 34 deletions(-) diff --git a/packages/orchestration/src/Core/ExecutionContext.php b/packages/orchestration/src/Core/ExecutionContext.php index 2b221ee0..3c9dd578 100644 --- a/packages/orchestration/src/Core/ExecutionContext.php +++ b/packages/orchestration/src/Core/ExecutionContext.php @@ -255,6 +255,11 @@ public function setVariable(string $name, mixed $value): void $this->variables[$name] = $value; } + public function removeVariable(string $name): void + { + unset($this->variables[$name]); + } + /** @return array */ public function results(): array { diff --git a/packages/orchestration/src/Workflow/Engine/WorkflowEngine.php b/packages/orchestration/src/Workflow/Engine/WorkflowEngine.php index 52baad73..ad962d92 100644 --- a/packages/orchestration/src/Workflow/Engine/WorkflowEngine.php +++ b/packages/orchestration/src/Workflow/Engine/WorkflowEngine.php @@ -45,6 +45,16 @@ final class WorkflowEngine { public const string CONTINUED_FROM_HEADER = 'x-firefly-continued-from'; + /** The timer that holds a tick back-off (resumeDue()); it is only ever a wake-up, never a step's wait. */ + public const string BACKOFF_TIMER = 'tick:backoff'; + + /** The variable counting the ticks in a row that could not drive an execution. */ + public const string BACKOFF_FAILURES = 'tickFailures'; + + private const int BACKOFF_BASE_SECONDS = 60; + + private const int BACKOFF_MAX_SECONDS = 3600; + /** @var array the executions this process is driving right now */ private array $driving = []; @@ -85,11 +95,16 @@ public function start(string|WorkflowDefinition $workflow, array $input = [], ar $definition->id, function () use ($definition, $context): ExecutionState { if ($definition->triggerMode === TriggerMode::Async) { - $this->refuseExisting($context->correlationId); - $pending = $this->support->checkpoint($context, ExecutionStatus::Pending); - $this->dispatcher->dispatch($context->correlationId); - - return $this->support->persistence->findById($context->correlationId) ?? $pending; + // Under the lock like a SYNC start: the existence check and the PENDING upsert must be one + // step, or two starts racing for a caller-supplied id both pass the check, the second + // overwrites the first one's input, and both dispatch a job. + return $this->support->lock->synchronized($context->correlationId, function () use ($context): ExecutionState { + $this->refuseExisting($context->correlationId); + $pending = $this->support->checkpoint($context, ExecutionStatus::Pending); + $this->dispatcher->dispatch($context->correlationId); + + return $this->support->persistence->findById($context->correlationId) ?? $pending; + }); } return $this->support->lock->synchronized($context->correlationId, function () use ($definition, $context): ExecutionState { @@ -142,35 +157,65 @@ public function signal(string $correlationId, string $signal, mixed $payload = n }); } - /** @return list the executions this tick advanced */ + /** + * Drives every due WAITING workflow, `batchSize` rows at a time, earliest wake-up first. + * + * One execution that cannot be driven (its definition was removed in a deploy, or is not deployed on this + * node yet; a handler blew up outside a step) is logged and backed off (backOff()): its wake-up moves + * forward, so it leaves the head of the due queue instead of coming back first on every tick and, once a + * batch of them is due, starving every healthy timer and deadline. It is not failed: a rolling deploy can + * leave a node that does not know a workflow yet, and that is no reason to end the execution. When a batch + * had to back rows off, the tick asks for the next batch, so the healthy rows behind them still move now. + * + * @return list the executions this tick advanced + */ public function resumeDue(): array { $advanced = []; - foreach ($this->support->persistence->findDue($this->support->now(), $this->batchSize) as $due) { - if ($due->pattern !== ExecutionPattern::Workflow) { - continue; - } + $seen = []; + do { + $fresh = 0; + $backedOff = false; + foreach ($this->support->persistence->findDue($this->support->now(), $this->batchSize) as $due) { + if (isset($seen[$due->correlationId])) { + continue; + } + $seen[$due->correlationId] = true; + $fresh++; + if ($due->pattern !== ExecutionPattern::Workflow) { + continue; + } - try { - $advanced[] = $this->support->lock->synchronized($due->correlationId, function () use ($due): ExecutionState { - $state = $this->load($due->correlationId); - if ($state->status !== ExecutionStatus::Waiting) { - return $state; - } + try { + $advanced[] = $this->support->lock->synchronized($due->correlationId, function () use ($due): ExecutionState { + $state = $this->load($due->correlationId); + if ($state->status !== ExecutionStatus::Waiting) { + return $state; + } - return $this->drive($this->registry->get($state->executionName), ExecutionContext::fromState($state)); - }); - } catch (ExecutionLockedException) { - // Another process is advancing it right now; the next tick sees whatever it left. - } catch (Throwable $error) { - // One execution that cannot be driven (its definition was removed in a deploy, a handler blew up - // outside a step) must not stop the rest of the batch: report it and move on to the next row. - $this->support->logger->warning("Workflow [{$due->executionName}] execution [{$due->correlationId}] could not be resumed by the tick: {$error->getMessage()}", [ - 'exception' => $error, - 'correlationId' => $due->correlationId, - ]); + try { + $context = ExecutionContext::fromState($state); + $context->removeTimer(self::BACKOFF_TIMER); + $context->removeVariable(self::BACKOFF_FAILURES); + + return $this->drive($this->registry->get($state->executionName), $context); + } catch (Throwable $error) { + $this->backOff($due->correlationId, $state); + + throw $error; + } + }); + } catch (ExecutionLockedException) { + // Another process is advancing it right now; the next tick sees whatever it left. + } catch (Throwable $error) { + $backedOff = true; + $this->support->logger->warning("Workflow [{$due->executionName}] execution [{$due->correlationId}] could not be resumed by the tick: {$error->getMessage()}", [ + 'exception' => $error, + 'correlationId' => $due->correlationId, + ]); + } } - } + } while ($backedOff && $fresh > 0 && count($advanced) < $this->batchSize); return $advanced; } @@ -287,8 +332,18 @@ public function upsertSearchAttributes(string $correlationId, array $attributes) foreach ($attributes as $key => $value) { $context->setSearchAttribute($key, $value); } + // The deadline is not persisted, only folded into wakeAt: without it a WAITING workflow's wake-up + // would lose the deadline, and the tick would never time it out. + if ($this->registry->has($state->executionName)) { + $this->applyDeadline($this->registry->get($state->executionName), $context); + } + + // Saved as is rather than through checkpoint(): the context does not carry the execution report, + // and checkpointing a finished workflow without it would erase the report the upsert must keep. + $updated = $context->toState($state->status, $this->support->now(), $state->failureReason, $state->report); + $this->support->persistence->save($updated); - return $this->support->checkpoint($context, $state->status, $state->failureReason, null); + return $updated; }); } @@ -330,7 +385,7 @@ private function drive(WorkflowDefinition $workflow, ExecutionContext $context): { $this->driving[$context->correlationId] = true; try { - $context->setDeadline($workflow->timeoutMs > 0 ? $context->startedAt()->modify(sprintf('+%d milliseconds', $workflow->timeoutMs)) : null); + $this->applyDeadline($workflow, $context); while (true) { $outcome = $this->executor->run($workflow, $context); @@ -391,8 +446,12 @@ private function startChild(WorkflowDefinition $workflow, ExecutionContext $cont throw $error; } - $context->setChild($stepId, $childState->correlationId); - $this->support->events->onChildWorkflowStarted($workflow->id, $context->correlationId, $child->workflowId, $childState->correlationId); + // The id this parent generated, not $childState's: a child that continued as new inline returns its LAST + // run, but its completion is matched on the origin (wakeParent() keys on CONTINUED_FROM_HEADER) and + // WorkflowExecutor::settledChild() follows the continuedAs chain from it, so the origin is what stays + // recorded; recording the last run would leave the parent waiting forever once that run waits. + $context->setChild($stepId, $childId); + $this->support->events->onChildWorkflowStarted($workflow->id, $context->correlationId, $child->workflowId, $childId); if (! $child->waitForCompletion) { $this->executor->finishStep($workflow, $context, $stepId, ['correlationId' => $childState->correlationId, 'status' => $childState->status->value]); @@ -546,10 +605,70 @@ private function settle(string $correlationId, callable $work): mixed private function cancelChildren(ExecutionContext $context): void { foreach ($context->children() as $childId) { - $child = $this->support->persistence->findById($childId); + $child = $this->lastRun($childId); if ($child !== null && ! $child->isTerminal()) { - $this->cancel($childId); + $this->cancel($child->correlationId); + } + } + } + + /** + * The latest run of the chain that starts at `$correlationId`: a run that continued as new is COMPLETED and + * points at its successor through the `continuedAs` variable, and the live run is the last one. + */ + private function lastRun(string $correlationId): ?ExecutionState + { + $seen = []; + $state = $this->support->persistence->findById($correlationId); + while ($state !== null) { + $next = $state->variables['continuedAs'] ?? null; + if (! is_string($next) || $next === '' || isset($seen[$next])) { + return $state; + } + $seen[$next] = true; + $successor = $this->support->persistence->findById($next); + if ($successor === null) { + return $state; + } + $state = $successor; + } + + return null; + } + + private function applyDeadline(WorkflowDefinition $workflow, ExecutionContext $context): void + { + $context->setDeadline($workflow->timeoutMs > 0 ? $context->startedAt()->modify(sprintf('+%d milliseconds', $workflow->timeoutMs)) : null); + } + + /** + * Moves the wake-up of an execution the tick could not drive forward, doubling from BACKOFF_BASE_SECONDS up + * to BACKOFF_MAX_SECONDS with every tick in a row that fails on it. It is written on the latest checkpoint + * (the failed drive may have written one) and only while that is still WAITING; the deadline is left out of + * the wake-up on purpose, since a deadline already past would keep the row at the head of the queue. The + * next drive that gets through clears the back-off and recomputes the wake-up from the definition. + */ + private function backOff(string $correlationId, ExecutionState $before): void + { + try { + $state = $this->support->persistence->findById($correlationId) ?? $before; + if ($state->status !== ExecutionStatus::Waiting) { + return; } + + $previous = $before->variables[self::BACKOFF_FAILURES] ?? 0; + $failures = (is_int($previous) ? $previous : 0) + 1; + $delay = min(self::BACKOFF_MAX_SECONDS, self::BACKOFF_BASE_SECONDS * 2 ** min($failures - 1, 16)); + + $context = ExecutionContext::fromState($state); + $context->setVariable(self::BACKOFF_FAILURES, $failures); + $context->scheduleTimer(self::BACKOFF_TIMER, $this->support->now()->modify(sprintf('+%d seconds', $delay))); + $this->support->checkpoint($context, ExecutionStatus::Waiting, $state->failureReason); + } catch (Throwable $error) { + $this->support->logger->warning("Workflow execution [{$correlationId}] could not be backed off by the tick: {$error->getMessage()}", [ + 'exception' => $error, + 'correlationId' => $correlationId, + ]); } } diff --git a/packages/orchestration/tests/Workflow/Engine/WorkflowChildrenTest.php b/packages/orchestration/tests/Workflow/Engine/WorkflowChildrenTest.php index fc604897..21695c62 100644 --- a/packages/orchestration/tests/Workflow/Engine/WorkflowChildrenTest.php +++ b/packages/orchestration/tests/Workflow/Engine/WorkflowChildrenTest.php @@ -258,3 +258,34 @@ function orchestrationPayingParentEngines(array &$log): TestEngines ->and($log)->toBe(['refund']) ->and($engines->deadLetterStore->findByCorrelationId('p-9'))->toHaveCount(1); }); + +it('wakes a parent whose child continued as new inline and then waited, and cancels the live run of that chain', function (): void { + $engines = TestEngines::create(); + $engines->workflowRegistry->register(WorkflowBuilder::workflow('looping-check') + ->step('tick')->handler(static fn (array $input): mixed => ($input['round'] ?? 0) === 0 ? new ContinueAsNew(['round' => 1]) : 'ticked')->add() + ->step('approve')->dependsOn('tick')->waitForSignal('go')->handler(static fn (): string => 'approved')->add() + ->build()); + $engines->workflowRegistry->register(WorkflowBuilder::workflow('pp') + ->step('spawn')->childWorkflow('looping-check')->handler(static fn (): array => [])->add() + ->step('after')->dependsOn('spawn')->handler(static fn (): string => 'after child')->add() + ->build()); + + $waiting = $engines->workflows->start('pp', correlationId: 'pp-cn'); + $originId = $waiting->children['spawn'] ?? ''; + $lastRunId = $engines->workflows->find($originId)?->variables['continuedAs'] ?? null; + $engines->workflows->signal(is_string($lastRunId) ? $lastRunId : '', 'go'); + $parent = $engines->workflows->find('pp-cn'); + + $engines->workflows->start('pp', correlationId: 'pp-cancel'); + $cancelOrigin = $engines->workflows->find('pp-cancel')?->children['spawn'] ?? ''; + $cancelLastRun = $engines->workflows->find($cancelOrigin)?->variables['continuedAs'] ?? null; + $engines->workflows->cancel('pp-cancel'); + + expect($waiting->status)->toBe(ExecutionStatus::Waiting) + ->and($engines->workflows->find($originId)?->status)->toBe(ExecutionStatus::Completed) + ->and($lastRunId)->toBeString()->not->toBe($originId) + ->and($parent?->status)->toBe(ExecutionStatus::Completed) + ->and(data_get($parent?->stepResults, 'spawn.correlationId'))->toBe($lastRunId) + ->and(data_get($parent?->stepResults, 'spawn.results'))->toBe(['tick' => 'ticked', 'approve' => 'approved']) + ->and($engines->workflows->find(is_string($cancelLastRun) ? $cancelLastRun : '')?->status)->toBe(ExecutionStatus::Cancelled); +}); diff --git a/packages/orchestration/tests/Workflow/Engine/WorkflowEngineTest.php b/packages/orchestration/tests/Workflow/Engine/WorkflowEngineTest.php index 4c88fbf5..87b07763 100644 --- a/packages/orchestration/tests/Workflow/Engine/WorkflowEngineTest.php +++ b/packages/orchestration/tests/Workflow/Engine/WorkflowEngineTest.php @@ -2,12 +2,15 @@ declare(strict_types=1); +use Firefly\Orchestration\Core\Exception\ExecutionLockedException; use Firefly\Orchestration\Core\Exception\InvalidExecutionStateException; use Firefly\Orchestration\Core\ExecutionContext; +use Firefly\Orchestration\Core\ExecutionLock; use Firefly\Orchestration\Core\ExecutionStatus; use Firefly\Orchestration\Core\StepStatus; use Firefly\Orchestration\Core\TriggerMode; use Firefly\Orchestration\Persistence\ExecutionState; +use Firefly\Orchestration\Tests\Support\ScriptedLock; use Firefly\Orchestration\Tests\Support\TestEngines; use Firefly\Orchestration\Workflow\Builder\WorkflowBuilder; use Firefly\Orchestration\Workflow\Engine\ContinueAsNew; @@ -166,3 +169,26 @@ 'sync' => [TriggerMode::Sync], 'async' => [TriggerMode::Async], ]); + +it('takes the execution lock around an ASYNC start, so a start racing another one for the same id writes nothing', function (): void { + $lock = new ScriptedLock; + $engines = TestEngines::create(['lock' => ['wait' => 0]], lock: $lock); + $engines->workflowRegistry->register(WorkflowBuilder::workflow('queued')->triggerMode(TriggerMode::Async) + ->step('work')->handler(static fn (): string => 'worked')->add() + ->build()); + + // Another process is starting [w-race] right now and holds its lock between the existence check and the write. + $lock->holdElsewhere(ExecutionLock::name('w-race')); + + expect(fn () => $engines->workflows->start('queued', ['by' => 'second'], correlationId: 'w-race')) + ->toThrow(ExecutionLockedException::class); + expect($engines->workflows->find('w-race'))->toBeNull() + ->and($engines->dispatcher->dispatched)->toBe([]); + + $lock->releaseElsewhere(ExecutionLock::name('w-race')); + $pending = $engines->workflows->start('queued', ['by' => 'first'], correlationId: 'w-race'); + + expect($pending->status)->toBe(ExecutionStatus::Pending) + ->and($pending->input)->toBe(['by' => 'first']) + ->and($engines->dispatcher->dispatched)->toBe(['w-race']); +}); diff --git a/packages/orchestration/tests/Workflow/Engine/WorkflowLifecycleTest.php b/packages/orchestration/tests/Workflow/Engine/WorkflowLifecycleTest.php index 75bbf94b..4717ef8b 100644 --- a/packages/orchestration/tests/Workflow/Engine/WorkflowLifecycleTest.php +++ b/packages/orchestration/tests/Workflow/Engine/WorkflowLifecycleTest.php @@ -58,3 +58,32 @@ function orchestrationLifecycleEngines(): TestEngines ->and(array_map(static fn ($s): string => $s->correlationId, $engines->workflows->findByStatus(ExecutionStatus::Waiting)))->toBe(['o-2']) ->and(fn () => $engines->workflows->cancel('o-1'))->toThrow(InvalidExecutionStateException::class); }); + +it('keeps the execution report of a finished workflow when its search attributes are upserted', function (): void { + $engines = TestEngines::create(); + $engines->workflowRegistry->register(WorkflowBuilder::workflow('one-step') + ->step('only')->handler(static fn (): string => 'done')->add() + ->build()); + + $completed = $engines->workflows->start('one-step', correlationId: 'w-1'); + $upserted = $engines->workflows->upsertSearchAttributes('w-1', ['k' => 'v']); + + expect($completed->report)->not->toBeNull() + ->and($upserted->report)->toBe($completed->report) + ->and($engines->workflows->find('w-1')?->report)->toBe($completed->report) + ->and($engines->workflows->find('w-1')?->searchAttributes)->toBe(['k' => 'v']); +}); + +it('keeps the deadline of a waiting workflow in its wake-up when its search attributes are upserted', function (): void { + $engines = TestEngines::create(); + $engines->workflowRegistry->register(WorkflowBuilder::workflow('bounded')->timeoutMs(5_000) + ->step('wait')->waitForSignal('go')->add() + ->build()); + + $waiting = $engines->workflows->start('bounded', correlationId: 'w-2'); + $upserted = $engines->workflows->upsertSearchAttributes('w-2', ['k' => 'v']); + + expect($waiting->wakeAt)->not->toBeNull() + ->and($upserted->wakeAt)->toEqual($waiting->wakeAt) + ->and($engines->workflows->find('w-2')?->wakeAt)->toEqual($waiting->wakeAt); +}); diff --git a/packages/orchestration/tests/Workflow/Engine/WorkflowWaitsTest.php b/packages/orchestration/tests/Workflow/Engine/WorkflowWaitsTest.php index 4ea09006..95d5547a 100644 --- a/packages/orchestration/tests/Workflow/Engine/WorkflowWaitsTest.php +++ b/packages/orchestration/tests/Workflow/Engine/WorkflowWaitsTest.php @@ -119,3 +119,34 @@ ->and($warnings)->toHaveCount(1) ->and($warnings[0] ?? '')->toContain('a-retired'); }); + +it('backs off due executions it cannot drive, so a full batch of them does not starve a healthy one', function (): void { + $timer = static fn (string $id) => WorkflowBuilder::workflow($id) + ->step('pause')->waitForTimer(60_000)->add() + ->step('go')->dependsOn('pause')->handler(static fn (): string => 'went')->add() + ->build(); + $before = TestEngines::create(['backpressure' => ['batch-size' => 3]]); + $before->workflowRegistry->register($timer('retired')); + $before->workflowRegistry->register($timer('kept')); + foreach (range(1, 5) as $n) { + $before->workflows->start('retired', correlationId: "a-retired-{$n}"); + } + $before->workflows->start('kept', correlationId: 'b-kept'); + + // A deploy drops [retired]: five undrivable rows are due ahead of the healthy one, more than one batch. + $after = TestEngines::create(['backpressure' => ['batch-size' => 3]], persistence: $before->persistence); + $after->workflowRegistry->register($timer('kept')); + $after->clock->advance(61.0); + $advanced = $after->workflows->resumeDue(); + $retired = $after->workflows->find('a-retired-1'); + $warnings = count($after->logger->messages('warning')); + $again = $after->workflows->resumeDue(); + + expect(array_map(static fn ($state): string => $state->correlationId, $advanced))->toBe(['b-kept']) + ->and($after->workflows->find('b-kept')?->status)->toBe(ExecutionStatus::Completed) + ->and($retired?->status)->toBe(ExecutionStatus::Waiting) + ->and($retired?->wakeAt)->toEqual($after->clock->now()->modify('+60 seconds')) + ->and($warnings)->toBe(5) + ->and($again)->toBe([]) + ->and(count($after->logger->messages('warning')))->toBe(5); +}); From a45d362ad2c68c3b1a88721d956797407d7e8544 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 21:09:40 -0700 Subject: [PATCH 30/58] feat(orchestration): portable database persistence and dead-letter store, proven on PostgreSQL, MySQL, MariaDB and SQLite --- ...00_create_firefly_orchestration_tables.php | 31 +++ .../src/Dlq/DatabaseDeadLetterStore.php | 109 +++++++++ .../DatabaseExecutionPersistenceProvider.php | 231 ++++++++++++++++++ .../src/Persistence/OrchestrationSchema.php | 152 ++++++++++++ .../tests/Dlq/DatabaseDeadLetterStoreTest.php | 37 +++ .../DatabaseDeadLetterIntegrationTest.php | 35 +++ .../DatabasePersistenceIntegrationTest.php | 40 +++ .../DatabaseSagaIntegrationTest.php | 40 +++ ...tabaseExecutionPersistenceProviderTest.php | 59 +++++ .../Persistence/OrchestrationSchemaTest.php | 38 +++ .../tests/Support/IntegrationDatabase.php | 90 +++++++ 11 files changed, 862 insertions(+) create mode 100644 packages/orchestration/database/migrations/2026_10_12_000000_create_firefly_orchestration_tables.php create mode 100644 packages/orchestration/src/Dlq/DatabaseDeadLetterStore.php create mode 100644 packages/orchestration/src/Persistence/DatabaseExecutionPersistenceProvider.php create mode 100644 packages/orchestration/src/Persistence/OrchestrationSchema.php create mode 100644 packages/orchestration/tests/Dlq/DatabaseDeadLetterStoreTest.php create mode 100644 packages/orchestration/tests/Integration/DatabaseDeadLetterIntegrationTest.php create mode 100644 packages/orchestration/tests/Integration/DatabasePersistenceIntegrationTest.php create mode 100644 packages/orchestration/tests/Integration/DatabaseSagaIntegrationTest.php create mode 100644 packages/orchestration/tests/Persistence/DatabaseExecutionPersistenceProviderTest.php create mode 100644 packages/orchestration/tests/Persistence/OrchestrationSchemaTest.php create mode 100644 packages/orchestration/tests/Support/IntegrationDatabase.php diff --git a/packages/orchestration/database/migrations/2026_10_12_000000_create_firefly_orchestration_tables.php b/packages/orchestration/database/migrations/2026_10_12_000000_create_firefly_orchestration_tables.php new file mode 100644 index 00000000..b51bb380 --- /dev/null +++ b/packages/orchestration/database/migrations/2026_10_12_000000_create_firefly_orchestration_tables.php @@ -0,0 +1,31 @@ +getConnection())); + } + + public function down(): void + { + OrchestrationSchema::drop(Schema::connection($this->getConnection())); + } +}; diff --git a/packages/orchestration/src/Dlq/DatabaseDeadLetterStore.php b/packages/orchestration/src/Dlq/DatabaseDeadLetterStore.php new file mode 100644 index 00000000..3263ab2f --- /dev/null +++ b/packages/orchestration/src/Dlq/DatabaseDeadLetterStore.php @@ -0,0 +1,109 @@ +id, OrchestrationSchema::ID_LENGTH); + OrchestrationSchema::fit('executionName', $entry->executionName, OrchestrationSchema::NAME_LENGTH); + OrchestrationSchema::fit('correlationId', $entry->correlationId, OrchestrationSchema::ID_LENGTH); + OrchestrationSchema::fit('stepId', $entry->stepId, OrchestrationSchema::NAME_LENGTH); + OrchestrationSchema::fit('errorType', $entry->errorType, OrchestrationSchema::TYPE_LENGTH); + + $this->table()->upsert([[ + 'id' => $entry->id, + 'execution_name' => $entry->executionName, + 'correlation_id' => $entry->correlationId, + 'pattern' => $entry->pattern->value, + 'step_id' => $entry->stepId, + 'status_at_failure' => $entry->statusAtFailure->value, + 'error_message' => $entry->errorMessage, + 'error_type' => $entry->errorType, + 'input' => StateSerializer::encodeArray($entry->input), + 'headers' => StateSerializer::encodeArray($entry->headers), + 'retry_count' => $entry->retryCount, + 'created_at' => OrchestrationSchema::instant($entry->createdAt), + 'last_retried_at' => $entry->lastRetriedAt === null ? null : OrchestrationSchema::instant($entry->lastRetriedAt), + ]], ['id'], ['retry_count', 'last_retried_at', 'error_message', 'error_type']); + } + + public function findById(string $id): ?DeadLetterEntry + { + return $this->entries($this->table()->where('id', $id)->limit(1))[0] ?? null; + } + + public function findAll(int $limit = 100): array + { + return $this->entries($this->table()->orderByDesc('created_at')->orderBy('id')->limit(max(0, $limit))); + } + + public function findByExecutionName(string $executionName, int $limit = 100): array + { + return $this->entries($this->table()->where('execution_name', $executionName)->orderByDesc('created_at')->orderBy('id')->limit(max(0, $limit))); + } + + public function findByCorrelationId(string $correlationId): array + { + return $this->entries($this->table()->where('correlation_id', $correlationId)->orderByDesc('created_at')->orderBy('id')); + } + + public function delete(string $id): bool + { + return $this->table()->where('id', $id)->delete() > 0; + } + + public function count(): int + { + return $this->table()->count(); + } + + private function table(): Builder + { + return $this->connection->table(OrchestrationSchema::DEAD_LETTERS); + } + + /** @return list */ + private function entries(Builder $query): array + { + $entries = []; + foreach ($query->get() as $row) { + $data = (array) $row; + $entries[] = DeadLetterEntry::fromArray([ + 'id' => $data['id'] ?? '', + 'executionName' => $data['execution_name'] ?? '', + 'correlationId' => $data['correlation_id'] ?? '', + 'pattern' => $data['pattern'] ?? '', + 'stepId' => $data['step_id'] ?? null, + 'statusAtFailure' => $data['status_at_failure'] ?? '', + 'errorMessage' => $data['error_message'] ?? '', + 'errorType' => $data['error_type'] ?? '', + 'input' => is_string($data['input'] ?? null) ? StateSerializer::decodeArray($data['input']) : [], + 'headers' => is_string($data['headers'] ?? null) ? StateSerializer::decodeArray($data['headers']) : [], + 'retryCount' => $data['retry_count'] ?? 0, + 'createdAt' => is_string($data['created_at'] ?? null) ? ExecutionState::instant(new DateTimeImmutable($data['created_at'], new DateTimeZone('UTC'))) : 'now', + 'lastRetriedAt' => is_string($data['last_retried_at'] ?? null) ? ExecutionState::instant(new DateTimeImmutable($data['last_retried_at'], new DateTimeZone('UTC'))) : null, + ]); + } + + return $entries; + } +} diff --git a/packages/orchestration/src/Persistence/DatabaseExecutionPersistenceProvider.php b/packages/orchestration/src/Persistence/DatabaseExecutionPersistenceProvider.php new file mode 100644 index 00000000..f8c1bc33 --- /dev/null +++ b/packages/orchestration/src/Persistence/DatabaseExecutionPersistenceProvider.php @@ -0,0 +1,231 @@ +correlationId, OrchestrationSchema::ID_LENGTH); + OrchestrationSchema::fit('parentCorrelationId', $state->parentCorrelationId, OrchestrationSchema::ID_LENGTH); + OrchestrationSchema::fit('executionName', $state->executionName, OrchestrationSchema::NAME_LENGTH); + + $attributes = []; + foreach ($state->searchAttributes as $key => $value) { + $key = (string) $key; + $value = ExecutionState::attributeValue($value); + OrchestrationSchema::fit('searchAttributes key', $key, OrchestrationSchema::ATTRIBUTE_KEY_LENGTH); + OrchestrationSchema::fit("searchAttributes.{$key}", $value, OrchestrationSchema::ATTRIBUTE_VALUE_LENGTH); + $attributes[] = ['correlation_id' => $state->correlationId, 'attribute_key' => $key, 'attribute_value' => $value]; + } + + $row = [ + 'correlation_id' => $state->correlationId, + 'execution_name' => $state->executionName, + 'pattern' => $state->pattern->value, + 'status' => $state->status->value, + 'parent_correlation_id' => $state->parentCorrelationId, + 'started_at' => OrchestrationSchema::instant($state->startedAt), + 'updated_at' => OrchestrationSchema::instant($state->updatedAt), + 'completed_at' => $state->completedAt === null ? null : OrchestrationSchema::instant($state->completedAt), + 'wake_at' => $state->wakeAt === null ? null : OrchestrationSchema::instant($state->wakeAt), + 'payload' => StateSerializer::encode($state), + ]; + + $this->connection->transaction(function () use ($row, $state, $attributes): void { + $this->connection->table(OrchestrationSchema::EXECUTIONS)->upsert([$row], ['correlation_id'], self::UPDATED); + $this->connection->table(OrchestrationSchema::SEARCH_ATTRIBUTES)->where('correlation_id', $state->correlationId)->delete(); + + if ($attributes !== []) { + $this->connection->table(OrchestrationSchema::SEARCH_ATTRIBUTES)->insert($attributes); + } + }); + } + + public function findById(string $correlationId): ?ExecutionState + { + $payload = $this->connection->table(OrchestrationSchema::EXECUTIONS)->where('correlation_id', $correlationId)->value('payload'); + + return is_string($payload) ? StateSerializer::decode($payload) : null; + } + + public function find(ExecutionCriteria $criteria): array + { + $query = $this->executions(); + if ($criteria->pattern !== null) { + $query->where('pattern', $criteria->pattern->value); + } + if ($criteria->status !== null) { + $query->where('status', $criteria->status->value); + } + if ($criteria->name !== null) { + $query->where('execution_name', $criteria->name); + } + + return $this->newestFirst($query, $criteria->limit); + } + + public function findInFlight(int $limit = 100): array + { + return $this->newestFirst($this->executions()->whereNotIn('status', self::values(ExecutionStatus::terminal())), $limit); + } + + public function findStale(DateTimeImmutable $before, int $limit = 100): array + { + return $this->newestFirst( + $this->executions()->whereIn('status', self::values(ExecutionStatus::recoverable()))->where('updated_at', '<', OrchestrationSchema::instant($before)), + $limit, + ); + } + + public function findDue(DateTimeImmutable $now, int $limit = 100): array + { + return $this->decode($this->executions() + ->where('status', ExecutionStatus::Waiting->value) + ->whereNotNull('wake_at') + ->where('wake_at', '<=', OrchestrationSchema::instant($now)) + ->orderBy('wake_at') + ->orderBy('correlation_id') + ->limit(max(0, $limit)) + ->pluck('payload') + ->all()); + } + + public function findBySearchAttributes(array $criteria, int $limit = 100): array + { + $query = $this->connection->table(OrchestrationSchema::EXECUTIONS.' as e'); + foreach ($criteria as $key => $value) { + $query->whereExists(static function (Builder $attribute) use ($key, $value): void { + $attribute->selectRaw('1') + ->from(OrchestrationSchema::SEARCH_ATTRIBUTES.' as a') + ->whereColumn('a.correlation_id', 'e.correlation_id') + ->where('a.attribute_key', $key) + ->where('a.attribute_value', ExecutionState::attributeValue($value)); + }); + } + + return $this->decode($query->orderByDesc('e.updated_at')->orderBy('e.correlation_id')->limit(max(0, $limit))->pluck('e.payload')->all()); + } + + public function updateStatus(string $correlationId, ExecutionStatus $status, DateTimeImmutable $now): void + { + $state = $this->findById($correlationId); + if ($state !== null) { + $this->save($state->withStatus($status, $now)); + } + } + + public function delete(string $correlationId): bool + { + return $this->connection->transaction(function () use ($correlationId): bool { + $this->connection->table(OrchestrationSchema::SEARCH_ATTRIBUTES)->where('correlation_id', $correlationId)->delete(); + + return $this->connection->table(OrchestrationSchema::EXECUTIONS)->where('correlation_id', $correlationId)->delete() > 0; + }); + } + + public function cleanup(DateTimeImmutable $before): int + { + $ids = array_values(array_filter($this->executions() + ->whereIn('status', self::values(ExecutionStatus::terminal())) + ->where('updated_at', '<', OrchestrationSchema::instant($before)) + ->pluck('correlation_id') + ->all(), 'is_string')); + + foreach (array_chunk($ids, 500) as $chunk) { + $this->connection->transaction(function () use ($chunk): void { + $this->connection->table(OrchestrationSchema::SEARCH_ATTRIBUTES)->whereIn('correlation_id', $chunk)->delete(); + $this->connection->table(OrchestrationSchema::EXECUTIONS)->whereIn('correlation_id', $chunk)->delete(); + }); + } + + return count($ids); + } + + public function countInFlight(): int + { + return $this->executions()->whereNotIn('status', self::values(ExecutionStatus::terminal()))->count(); + } + + public function countByStatus(): array + { + $counts = []; + foreach ($this->executions()->select('status')->selectRaw('count(*) as total')->groupBy('status')->get() as $row) { + $status = isset($row->status) && is_string($row->status) ? $row->status : null; + $total = isset($row->total) && is_numeric($row->total) ? (int) $row->total : 0; + if ($status !== null) { + $counts[$status] = $total; + } + } + + return $counts; + } + + public function isHealthy(): bool + { + try { + $this->executions()->limit(1)->get(['correlation_id']); + + return true; + } catch (Throwable) { + return false; + } + } + + private function executions(): Builder + { + return $this->connection->table(OrchestrationSchema::EXECUTIONS); + } + + /** @return list */ + private function newestFirst(Builder $query, int $limit): array + { + return $this->decode($query->orderByDesc('updated_at')->orderBy('correlation_id')->limit(max(0, $limit))->pluck('payload')->all()); + } + + /** + * @param array $payloads + * @return list + */ + private function decode(array $payloads): array + { + $states = []; + foreach ($payloads as $payload) { + if (is_string($payload)) { + $states[] = StateSerializer::decode($payload); + } + } + + return $states; + } + + /** + * @param list $statuses + * @return list + */ + private static function values(array $statuses): array + { + return array_map(static fn (ExecutionStatus $status): string => $status->value, $statuses); + } +} diff --git a/packages/orchestration/src/Persistence/OrchestrationSchema.php b/packages/orchestration/src/Persistence/OrchestrationSchema.php new file mode 100644 index 00000000..1e50d2c0 --- /dev/null +++ b/packages/orchestration/src/Persistence/OrchestrationSchema.php @@ -0,0 +1,152 @@ + $length) { + throw new OrchestrationException("{$field} is ".mb_strlen($value, 'UTF-8')." characters long; the orchestration database tables hold at most {$length}.", 'ORCHESTRATION_UNSERIALIZABLE'); + } + } + + public static function instant(DateTimeImmutable $at): string + { + return $at->setTimezone(new DateTimeZone('UTC'))->format('Y-m-d H:i:s.u'); + } + + public static function create(Builder $schema): void + { + $collation = self::binaryCollation($schema); + $binary = static function (ColumnDefinition $column) use ($collation): void { + if ($collation === self::SQL_SERVER_BINARY) { + $column->collation($collation); + } elseif ($collation !== null) { + $column->charset('utf8mb4')->collation($collation); + } + }; + + if (! $schema->hasTable(self::EXECUTIONS)) { + $schema->create(self::EXECUTIONS, static function (Blueprint $table) use ($binary): void { + $binary($table->string('correlation_id', self::ID_LENGTH)->primary()); + $binary($table->string('execution_name', self::NAME_LENGTH)); + $table->string('pattern', 16); + $table->string('status', 24); + $binary($table->string('parent_correlation_id', self::ID_LENGTH)->nullable()); + $table->dateTime('started_at', 6); + $table->dateTime('updated_at', 6); + $table->dateTime('completed_at', 6)->nullable(); + $table->dateTime('wake_at', 6)->nullable(); + $table->longText('payload'); + $table->index(['status', 'updated_at'], 'firefly_orchestration_executions_status_updated'); + $table->index(['pattern', 'status'], 'firefly_orchestration_executions_pattern_status'); + $table->index(['wake_at'], 'firefly_orchestration_executions_wake'); + $table->index(['execution_name'], 'firefly_orchestration_executions_name'); + }); + } + + if (! $schema->hasTable(self::SEARCH_ATTRIBUTES)) { + $schema->create(self::SEARCH_ATTRIBUTES, static function (Blueprint $table) use ($binary): void { + $binary($table->string('correlation_id', self::ID_LENGTH)); + $binary($table->string('attribute_key', self::ATTRIBUTE_KEY_LENGTH)); + $binary($table->string('attribute_value', self::ATTRIBUTE_VALUE_LENGTH)); + $table->primary(['correlation_id', 'attribute_key'], 'firefly_orchestration_search_attributes_pk'); + $table->index(['attribute_key', 'attribute_value'], 'firefly_orchestration_search_attributes_lookup'); + }); + } + + if (! $schema->hasTable(self::DEAD_LETTERS)) { + $schema->create(self::DEAD_LETTERS, static function (Blueprint $table) use ($binary): void { + $binary($table->string('id', self::ID_LENGTH)->primary()); + $binary($table->string('execution_name', self::NAME_LENGTH)); + $binary($table->string('correlation_id', self::ID_LENGTH)); + $table->string('pattern', 16); + $table->string('step_id', self::NAME_LENGTH)->nullable(); + $table->string('status_at_failure', 24); + $table->longText('error_message'); + $table->string('error_type', self::TYPE_LENGTH); + $table->longText('input'); + $table->longText('headers'); + $table->unsignedInteger('retry_count')->default(0); + $table->dateTime('created_at', 6); + $table->dateTime('last_retried_at', 6)->nullable(); + $table->index(['execution_name'], 'firefly_orchestration_dead_letters_name'); + $table->index(['correlation_id'], 'firefly_orchestration_dead_letters_correlation'); + $table->index(['created_at'], 'firefly_orchestration_dead_letters_created'); + }); + } + } + + public static function drop(Builder $schema): void + { + $schema->dropIfExists(self::DEAD_LETTERS); + $schema->dropIfExists(self::SEARCH_ATTRIBUTES); + $schema->dropIfExists(self::EXECUTIONS); + } + + private const string SQL_SERVER_BINARY = 'Latin1_General_100_BIN2'; + + private static function binaryCollation(Builder $schema): ?string + { + $connection = $schema->getConnection(); + if ($connection instanceof SqlServerConnection) { + return self::SQL_SERVER_BINARY; + } + if (! $connection instanceof MySqlConnection) { + return null; + } + + $modern = version_compare($connection->getServerVersion(), $connection->isMaria() ? '10.2.2' : '8.0.1', '>='); + + return $modern ? ($connection->isMaria() ? 'utf8mb4_nopad_bin' : 'utf8mb4_0900_bin') : 'utf8mb4_bin'; + } +} diff --git a/packages/orchestration/tests/Dlq/DatabaseDeadLetterStoreTest.php b/packages/orchestration/tests/Dlq/DatabaseDeadLetterStoreTest.php new file mode 100644 index 00000000..12c045d6 --- /dev/null +++ b/packages/orchestration/tests/Dlq/DatabaseDeadLetterStoreTest.php @@ -0,0 +1,37 @@ + $assert) { + it("SQLite dead-letter store {$case}", function () use ($assert): void { + $assert(new DatabaseDeadLetterStore(IntegrationDatabase::fresh(IntegrationDatabase::sqlite()))); + }); +} + +it('refuses, before any write, a dead letter whose identity outgrows its column', function (string $field, string $id, string $name, string $correlationId): void { + $connection = IntegrationDatabase::fresh(IntegrationDatabase::sqlite()); + $store = new DatabaseDeadLetterStore($connection); + + expect(fn () => $store->save(DeadLetterStoreContract::entry($id, $name, $correlationId))) + ->toThrow(OrchestrationException::class, $field) + ->and($store->count())->toBe(0); +})->with([ + 'id' => ['id', str_repeat('d', 65), 'order', 'c-1'], + 'execution name' => ['executionName', 'd-1', str_repeat('n', 192), 'c-1'], + 'correlation id' => ['correlationId', 'd-1', 'order', str_repeat('c', 65)], +]); + +it('keeps an error message of any length', function (): void { + $store = new DatabaseDeadLetterStore(IntegrationDatabase::fresh(IntegrationDatabase::sqlite())); + $entry = DeadLetterEntry::fromArray(['errorMessage' => str_repeat('é', 70_000)] + DeadLetterStoreContract::entry('d-1')->toArray()); + + $store->save($entry); + + expect($store->findById('d-1')?->errorMessage)->toBe($entry->errorMessage); +}); diff --git a/packages/orchestration/tests/Integration/DatabaseDeadLetterIntegrationTest.php b/packages/orchestration/tests/Integration/DatabaseDeadLetterIntegrationTest.php new file mode 100644 index 00000000..66be7630 --- /dev/null +++ b/packages/orchestration/tests/Integration/DatabaseDeadLetterIntegrationTest.php @@ -0,0 +1,35 @@ + [$driver, $variable, $extension]) { + foreach (DeadLetterStoreContract::cases() as $case => $assert) { + it("{$server} dead-letter store {$case}", function () use ($assert, $driver, $variable): void { + $connection = IntegrationDatabase::fresh(IntegrationDatabase::connection($driver, $variable)); + try { + $assert(new DatabaseDeadLetterStore($connection)); + } finally { + $connection->disconnect(); + } + })->group('integration')->skip(IntegrationDatabase::skipReason($variable, $extension) !== null, (string) IntegrationDatabase::skipReason($variable, $extension)); + } + + it("keeps an error message longer than a MySQL TEXT column on {$server}", function () use ($driver, $variable): void { + $connection = IntegrationDatabase::fresh(IntegrationDatabase::connection($driver, $variable)); + try { + $store = new DatabaseDeadLetterStore($connection); + $entry = DeadLetterEntry::fromArray(['errorMessage' => str_repeat('é', 70_000)] + DeadLetterStoreContract::entry('d-1')->toArray()); + + $store->save($entry); + + expect($store->findById('d-1')?->errorMessage)->toBe($entry->errorMessage); + } finally { + $connection->disconnect(); + } + })->group('integration')->skip(IntegrationDatabase::skipReason($variable, $extension) !== null, (string) IntegrationDatabase::skipReason($variable, $extension)); +} diff --git a/packages/orchestration/tests/Integration/DatabasePersistenceIntegrationTest.php b/packages/orchestration/tests/Integration/DatabasePersistenceIntegrationTest.php new file mode 100644 index 00000000..7c86ad80 --- /dev/null +++ b/packages/orchestration/tests/Integration/DatabasePersistenceIntegrationTest.php @@ -0,0 +1,40 @@ + [$driver, $variable, $extension]) { + foreach (ExecutionPersistenceContract::cases() as $case => $assert) { + it("{$server} database persistence {$case}", function () use ($assert, $driver, $variable): void { + $connection = IntegrationDatabase::fresh(IntegrationDatabase::connection($driver, $variable)); + try { + $assert(new DatabaseExecutionPersistenceProvider($connection)); + } finally { + $connection->disconnect(); + } + })->group('integration')->skip(IntegrationDatabase::skipReason($variable, $extension) !== null, (string) IntegrationDatabase::skipReason($variable, $extension)); + } + + it("keeps full-width ids, names and search attributes and refuses wider ones on {$server}", function () use ($driver, $variable): void { + $connection = IntegrationDatabase::fresh(IntegrationDatabase::connection($driver, $variable)); + try { + $provider = new DatabaseExecutionPersistenceProvider($connection); + $id = str_repeat('i', 64); + $state = ExecutionPersistenceContract::state($id, overrides: ['executionName' => str_repeat('ñ', 191), 'searchAttributes' => [str_repeat('k', 128) => str_repeat('é', 191)]]); + + $provider->save($state); + + expect($provider->findById($id)?->toArray())->toBe($state->toArray()) + ->and(ExecutionPersistenceContract::ids($provider->findBySearchAttributes([str_repeat('k', 128) => str_repeat('é', 191)])))->toBe([$id]) + ->and(fn () => $provider->save(ExecutionPersistenceContract::state('wide', overrides: ['searchAttributes' => ['customer' => str_repeat('é', 192)]])))->toThrow(OrchestrationException::class, 'searchAttributes.customer') + ->and($connection->table(OrchestrationSchema::EXECUTIONS)->count())->toBe(1); + } finally { + $connection->disconnect(); + } + })->group('integration')->skip(IntegrationDatabase::skipReason($variable, $extension) !== null, (string) IntegrationDatabase::skipReason($variable, $extension)); +} diff --git a/packages/orchestration/tests/Integration/DatabaseSagaIntegrationTest.php b/packages/orchestration/tests/Integration/DatabaseSagaIntegrationTest.php new file mode 100644 index 00000000..578f5a70 --- /dev/null +++ b/packages/orchestration/tests/Integration/DatabaseSagaIntegrationTest.php @@ -0,0 +1,40 @@ + [$driver, $variable, $extension]) { + it("runs, compensates and dead-letters a saga, and parks and wakes a workflow, on {$server}", function () use ($driver, $variable): void { + $connection = IntegrationDatabase::fresh(IntegrationDatabase::connection($driver, $variable)); + try { + $engines = TestEngines::create(persistence: new DatabaseExecutionPersistenceProvider($connection), deadLetters: new DatabaseDeadLetterStore($connection)); + $saga = SagaBuilder::saga('order') + ->step('reserve')->handler(static fn (): array => ['hold' => 'h-1', 'note' => 'café'])->compensation(static fn (): string => 'released')->add() + ->step('charge')->dependsOn('reserve')->handler(static fn (): never => throw new RuntimeException('card declined'))->add() + ->build(); + $engines->workflowRegistry->register(WorkflowBuilder::workflow('approval') + ->step('approve')->waitForTimer(60_000, 'deadline')->add()->build()); + + $failed = $engines->sagas->execute($saga, ['orderId' => 7], correlationId: "it-{$variable}-saga"); + $waiting = $engines->workflows->start('approval', correlationId: "it-{$variable}-wf"); + $engines->clock->advance(61.0); + $woken = $engines->workflows->resumeDue(); + + expect($failed->status)->toBe(ExecutionStatus::Failed) + ->and($engines->persistence->findById("it-{$variable}-saga")?->compensationResults)->toBe(['reserve' => 'released']) + ->and($engines->persistence->findById("it-{$variable}-saga")?->stepResults['reserve'] ?? null)->toBe(['hold' => 'h-1', 'note' => 'café']) + ->and($engines->deadLetterStore->findByCorrelationId("it-{$variable}-saga")[0]->stepId ?? null)->toBe('charge') + ->and($waiting->status)->toBe(ExecutionStatus::Waiting) + ->and($woken[0]->status ?? null)->toBe(ExecutionStatus::Completed); + } finally { + $connection->disconnect(); + } + })->group('integration')->skip(IntegrationDatabase::skipReason($variable, $extension) !== null, (string) IntegrationDatabase::skipReason($variable, $extension)); +} diff --git a/packages/orchestration/tests/Persistence/DatabaseExecutionPersistenceProviderTest.php b/packages/orchestration/tests/Persistence/DatabaseExecutionPersistenceProviderTest.php new file mode 100644 index 00000000..79bf1ac2 --- /dev/null +++ b/packages/orchestration/tests/Persistence/DatabaseExecutionPersistenceProviderTest.php @@ -0,0 +1,59 @@ + $assert) { + it("SQLite database persistence {$case}", function () use ($assert): void { + $assert(new DatabaseExecutionPersistenceProvider(IntegrationDatabase::fresh(IntegrationDatabase::sqlite()))); + }); +} + +it('keeps one row per execution and checkpoints a saga through it', function (): void { + $connection = IntegrationDatabase::fresh(IntegrationDatabase::sqlite()); + $engines = TestEngines::create(persistence: new DatabaseExecutionPersistenceProvider($connection)); + $saga = SagaBuilder::saga('db')->step('a')->handler(static fn (): array => ['n' => 1])->add()->step('b')->dependsOn('a')->handler(static fn (): string => 'b')->add()->build(); + + $result = $engines->sagas->execute($saga, correlationId: 'db-1'); + + expect($result->isSuccess())->toBeTrue() + ->and($connection->table(OrchestrationSchema::EXECUTIONS)->count())->toBe(1) + ->and($connection->table(OrchestrationSchema::EXECUTIONS)->value('status'))->toBe('COMPLETED') + ->and($engines->persistence->findById('db-1')?->stepResults)->toBe(['a' => ['n' => 1], 'b' => 'b']) + ->and($engines->persistence->findById('db-1')?->status)->toBe(ExecutionStatus::Completed); +}); + +it('refuses, before any write, a value longer than its column, as every driver would have to', function (string $field, array $overrides): void { + $connection = IntegrationDatabase::fresh(IntegrationDatabase::sqlite()); + $provider = new DatabaseExecutionPersistenceProvider($connection); + /** @var array $overrides */ + expect(fn () => $provider->save(ExecutionPersistenceContract::state('c-1', overrides: $overrides))) + ->toThrow(OrchestrationException::class, $field) + ->and($connection->table(OrchestrationSchema::EXECUTIONS)->count())->toBe(0) + ->and($connection->table(OrchestrationSchema::SEARCH_ATTRIBUTES)->count())->toBe(0); +})->with([ + 'correlation id' => ['correlationId', ['correlationId' => str_repeat('c', 65)]], + 'parent correlation id' => ['parentCorrelationId', ['parentCorrelationId' => str_repeat('p', 65)]], + 'execution name' => ['executionName', ['executionName' => str_repeat('n', 192)]], + 'search-attribute key' => ['searchAttributes', ['searchAttributes' => [str_repeat('k', 129) => 'v']]], + 'search-attribute value' => ['searchAttributes.customer', ['searchAttributes' => ['customer' => str_repeat('é', 192)]]], +]); + +it('accepts values exactly as long as their columns, counting characters rather than bytes', function (): void { + $provider = new DatabaseExecutionPersistenceProvider(IntegrationDatabase::fresh(IntegrationDatabase::sqlite())); + $id = str_repeat('i', 64); + $state = ExecutionPersistenceContract::state($id, overrides: ['executionName' => str_repeat('ñ', 191), 'searchAttributes' => [str_repeat('k', 128) => str_repeat('é', 191)]]); + + $provider->save($state); + + expect($provider->findById($id)?->toArray())->toBe($state->toArray()) + ->and(ExecutionPersistenceContract::ids($provider->findBySearchAttributes([str_repeat('k', 128) => str_repeat('é', 191)])))->toBe([$id]); +}); diff --git a/packages/orchestration/tests/Persistence/OrchestrationSchemaTest.php b/packages/orchestration/tests/Persistence/OrchestrationSchemaTest.php new file mode 100644 index 00000000..11c03ffe --- /dev/null +++ b/packages/orchestration/tests/Persistence/OrchestrationSchemaTest.php @@ -0,0 +1,38 @@ +getSchemaBuilder(); + + OrchestrationSchema::create($schema); + OrchestrationSchema::create($schema); + + expect($schema->hasTable(OrchestrationSchema::EXECUTIONS))->toBeTrue() + ->and($schema->hasColumns(OrchestrationSchema::EXECUTIONS, ['correlation_id', 'execution_name', 'pattern', 'status', 'parent_correlation_id', 'started_at', 'updated_at', 'completed_at', 'wake_at', 'payload']))->toBeTrue() + ->and($schema->hasColumns(OrchestrationSchema::SEARCH_ATTRIBUTES, ['correlation_id', 'attribute_key', 'attribute_value']))->toBeTrue() + ->and($schema->hasColumns(OrchestrationSchema::DEAD_LETTERS, ['id', 'execution_name', 'correlation_id', 'pattern', 'step_id', 'status_at_failure', 'error_message', 'error_type', 'input', 'headers', 'retry_count', 'created_at', 'last_retried_at']))->toBeTrue() + ->and(OrchestrationSchema::instant(new DateTimeImmutable('2026-10-12T12:00:00.5+02:00')))->toBe('2026-10-12 10:00:00.500000'); + + OrchestrationSchema::drop($schema); + + expect($schema->hasTable(OrchestrationSchema::EXECUTIONS))->toBeFalse(); +}); + +it('gives SQL Server the binary collation MySQL and MariaDB get, so ids and search attributes compare case-sensitively', function (): void { + $connection = new SqlServerConnection(static fn (): never => throw new RuntimeException('No server: the DDL is only pretended.'), 'orchestration', '', ['driver' => 'sqlsrv', 'name' => 'orchestration-sqlsrv']); + + $ddl = implode("\n", array_column($connection->pretend(static function () use ($connection): void { + OrchestrationSchema::create($connection->getSchemaBuilder()); + }), 'query')); + + expect($ddl)->toContain('"correlation_id" nvarchar(64) collate Latin1_General_100_BIN2 not null, "execution_name" nvarchar(191) collate Latin1_General_100_BIN2 not null') + ->and($ddl)->toContain('"attribute_key" nvarchar(128) collate Latin1_General_100_BIN2 not null, "attribute_value" nvarchar(191) collate Latin1_General_100_BIN2 not null') + ->and($ddl)->toContain('"id" nvarchar(64) collate Latin1_General_100_BIN2 not null') + ->and($ddl)->toContain('"pattern" nvarchar(16) not null'); +}); diff --git a/packages/orchestration/tests/Support/IntegrationDatabase.php b/packages/orchestration/tests/Support/IntegrationDatabase.php new file mode 100644 index 00000000..24f550e4 --- /dev/null +++ b/packages/orchestration/tests/Support/IntegrationDatabase.php @@ -0,0 +1,90 @@ + */ + public static function servers(): array + { + return [ + 'PostgreSQL' => ['pgsql', 'FIREFLY_PG_DSN', 'pdo_pgsql'], + 'MySQL' => ['mysql', 'FIREFLY_MYSQL_DSN', 'pdo_mysql'], + 'MariaDB' => ['mariadb', 'FIREFLY_MARIADB_DSN', 'pdo_mysql'], + 'SQL Server' => ['sqlsrv', 'FIREFLY_SQLSRV_DSN', 'pdo_sqlsrv'], + ]; + } + + public static function skipReason(string $variable, string $extension): ?string + { + $dsn = getenv($variable); + if (! is_string($dsn) || $dsn === '') { + return "Set {$variable} to an isolated server."; + } + + return extension_loaded($extension) ? null : "The {$extension} extension is not loaded."; + } + + public static function connection(string $driver, string $variable): Connection + { + $dsn = getenv($variable); + if (! is_string($dsn) || $dsn === '') { + throw new RuntimeException("{$variable} is not set."); + } + + $parts = []; + foreach (explode(';', $dsn) as $pair) { + [$key, $value] = array_pad(explode('=', $pair, 2), 2, ''); + $parts[trim($key)] = trim($value); + } + + $config = [ + 'driver' => $driver, + 'host' => $parts['host'] ?? '127.0.0.1', + 'port' => (int) ($parts['port'] ?? 0), + 'database' => $parts['dbname'] ?? '', + 'username' => $parts['user'] ?? '', + 'password' => $parts['password'] ?? '', + 'prefix' => '', + ]; + $config += match ($driver) { + 'pgsql' => ['charset' => 'utf8', 'search_path' => 'public', 'sslmode' => 'prefer'], + 'sqlsrv' => ['charset' => 'utf8', 'trust_server_certificate' => true], + default => ['charset' => 'utf8mb4', 'collation' => 'utf8mb4_unicode_ci'], + }; + + $capsule = new Manager; + $capsule->addConnection($config, $variable); + + return $capsule->getConnection($variable); + } + + public static function sqlite(string $database = ':memory:'): Connection + { + $capsule = new Manager; + $capsule->addConnection(['driver' => 'sqlite', 'database' => $database, 'prefix' => '', 'foreign_key_constraints' => true], 'orchestration-sqlite'); + + return $capsule->getConnection('orchestration-sqlite'); + } + + public static function fresh(Connection $connection): Connection + { + OrchestrationSchema::drop($connection->getSchemaBuilder()); + OrchestrationSchema::create($connection->getSchemaBuilder()); + + return $connection; + } +} From f0a2357364f995706c2c93e6d360b4e3aff38541 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 21:15:00 -0700 Subject: [PATCH 31/58] fix(orchestration): rewrite search attributes by exact key and upsert so concurrent saves cannot gap-lock deadlock on MySQL and MariaDB, and retry a top-level save on a deadlock --- .../DatabaseExecutionPersistenceProvider.php | 40 ++++++++++++++++-- .../DatabasePersistenceIntegrationTest.php | 41 +++++++++++++++++++ 2 files changed, 78 insertions(+), 3 deletions(-) diff --git a/packages/orchestration/src/Persistence/DatabaseExecutionPersistenceProvider.php b/packages/orchestration/src/Persistence/DatabaseExecutionPersistenceProvider.php index f8c1bc33..ead087f2 100644 --- a/packages/orchestration/src/Persistence/DatabaseExecutionPersistenceProvider.php +++ b/packages/orchestration/src/Persistence/DatabaseExecutionPersistenceProvider.php @@ -18,9 +18,20 @@ * same transaction. Joining the caller's transaction is deliberate: a step that writes business rows inside * a #[Transactional] service commits or rolls back its checkpoint with them. Every value bound for a * width-limited column is checked (OrchestrationSchema::fit()) before the first statement runs. + * + * Search attributes are rewritten without a range DELETE: under InnoDB's REPEATABLE READ a DELETE that matches + * nothing (every first save) takes a gap lock, and two workers saving new executions whose ids share an index + * gap then deadlock on each other's INSERT (MySQL and MariaDB error 1213). save() instead reads the stored + * keys with a plain select, deletes only the obsolete ones by exact primary key (record locks, no gap + * locks) and upserts the new set on [correlation_id, attribute_key]. A top-level save() also retries a + * deadlock or serialization failure up to ATTEMPTS times; inside a caller's transaction Laravel cannot + * retry a nested block, so the error reaches the caller, whose own transaction decides what to do. */ final class DatabaseExecutionPersistenceProvider implements ExecutionPersistenceProvider { + /** How many times a top-level save() runs when the database reports a deadlock or serialization failure. */ + private const int ATTEMPTS = 3; + private const array UPDATED = ['execution_name', 'pattern', 'status', 'parent_correlation_id', 'updated_at', 'completed_at', 'wake_at', 'payload']; public function __construct(private readonly ConnectionInterface $connection) {} @@ -55,12 +66,35 @@ public function save(ExecutionState $state): void $this->connection->transaction(function () use ($row, $state, $attributes): void { $this->connection->table(OrchestrationSchema::EXECUTIONS)->upsert([$row], ['correlation_id'], self::UPDATED); - $this->connection->table(OrchestrationSchema::SEARCH_ATTRIBUTES)->where('correlation_id', $state->correlationId)->delete(); + + $obsolete = array_values(array_diff($this->attributeKeys($state->correlationId), array_column($attributes, 'attribute_key'))); + if ($obsolete !== []) { + $this->connection->table(OrchestrationSchema::SEARCH_ATTRIBUTES) + ->where('correlation_id', $state->correlationId) + ->whereIn('attribute_key', $obsolete) + ->delete(); + } if ($attributes !== []) { - $this->connection->table(OrchestrationSchema::SEARCH_ATTRIBUTES)->insert($attributes); + $this->connection->table(OrchestrationSchema::SEARCH_ATTRIBUTES)->upsert($attributes, ['correlation_id', 'attribute_key'], ['attribute_value']); } - }); + }, self::ATTEMPTS); + } + + /** + * The search-attribute keys stored for one execution, read with a plain (non-locking) select. The + * execution row's upsert above already holds that row's lock, so no other save of the same execution + * can change these keys before this transaction ends; the engine's execution lock already keeps two + * saves of one execution apart, so the read never needs to lock the attribute rows themselves. + * + * @return list + */ + private function attributeKeys(string $correlationId): array + { + return array_values(array_filter($this->connection->table(OrchestrationSchema::SEARCH_ATTRIBUTES) + ->where('correlation_id', $correlationId) + ->pluck('attribute_key') + ->all(), 'is_string')); } public function findById(string $correlationId): ?ExecutionState diff --git a/packages/orchestration/tests/Integration/DatabasePersistenceIntegrationTest.php b/packages/orchestration/tests/Integration/DatabasePersistenceIntegrationTest.php index 7c86ad80..83421278 100644 --- a/packages/orchestration/tests/Integration/DatabasePersistenceIntegrationTest.php +++ b/packages/orchestration/tests/Integration/DatabasePersistenceIntegrationTest.php @@ -20,6 +20,47 @@ })->group('integration')->skip(IntegrationDatabase::skipReason($variable, $extension) !== null, (string) IntegrationDatabase::skipReason($variable, $extension)); } + it("saves two new executions with search attributes from two workers at once without a gap-lock wait on {$server}", function () use ($driver, $variable): void { + $first = IntegrationDatabase::fresh(IntegrationDatabase::connection($driver, $variable)); + $second = IntegrationDatabase::connection($driver, $variable); + try { + // A lock wait (the first half of the MySQL/MariaDB deadlock) surfaces in one second instead of hanging. + match ($driver) { + 'mysql', 'mariadb' => $second->statement('SET SESSION innodb_lock_wait_timeout = 1'), + 'pgsql' => $second->statement("SET lock_timeout = '1s'"), + 'sqlsrv' => $second->statement('SET LOCK_TIMEOUT 1000'), + default => null, + }; + $one = new DatabaseExecutionPersistenceProvider($first); + $two = new DatabaseExecutionPersistenceProvider($second); + $one->save(ExecutionPersistenceContract::state('a0', overrides: ['searchAttributes' => ['customer' => 'ada']])); + $one->save(ExecutionPersistenceContract::state('b0', overrides: ['searchAttributes' => ['customer' => 'ada']])); + + // Worker one holds its open checkpoint transaction while worker two saves an id in the same index gap. + $first->beginTransaction(); + $one->save(ExecutionPersistenceContract::state('a1', overrides: ['searchAttributes' => ['customer' => 'grace', 'vip' => true]])); + $two->save(ExecutionPersistenceContract::state('a2', overrides: ['searchAttributes' => ['customer' => 'alan', 'vip' => true]])); + $first->commit(); + + // A re-save drops an obsolete key, rewrites a kept one and adds a new one, under the same interleaving. + $first->beginTransaction(); + $one->save(ExecutionPersistenceContract::state('a1', overrides: ['searchAttributes' => ['customer' => 'grace', 'region' => 'eu']])); + $two->save(ExecutionPersistenceContract::state('a2', overrides: ['searchAttributes' => ['customer' => 'barbara']])); + $first->commit(); + + expect(ExecutionPersistenceContract::ids($one->findBySearchAttributes(['vip' => true])))->toBe([]) + ->and(ExecutionPersistenceContract::ids($one->findBySearchAttributes(['customer' => 'grace', 'region' => 'eu'])))->toBe(['a1']) + ->and(ExecutionPersistenceContract::ids($two->findBySearchAttributes(['customer' => 'barbara'])))->toBe(['a2']) + ->and($first->table(OrchestrationSchema::SEARCH_ATTRIBUTES)->whereIn('correlation_id', ['a1', 'a2'])->count())->toBe(3); + } finally { + if ($first->transactionLevel() > 0) { + $first->rollBack(); + } + $first->disconnect(); + $second->disconnect(); + } + })->group('integration')->skip(IntegrationDatabase::skipReason($variable, $extension) !== null, (string) IntegrationDatabase::skipReason($variable, $extension)); + it("keeps full-width ids, names and search attributes and refuses wider ones on {$server}", function () use ($driver, $variable): void { $connection = IntegrationDatabase::fresh(IntegrationDatabase::connection($driver, $variable)); try { From a80a6fb8545580086cd456235a04c0af24b56b9c Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 21:24:13 -0700 Subject: [PATCH 32/58] feat(orchestration): scan sagas, TCCs and workflows into a compiled manifest that firefly:cache emits --- book/src-es/02-dependency-injection.md | 2 +- book/src-es/03-configuration.md | 2 +- book/src-es/07-cqrs.md | 6 +- book/src-es/08-eda-outbox.md | 2 +- book/src-es/09-transactions.md | 2 +- book/src-es/13-cli-cache.md | 17 +- book/src/02-dependency-injection.md | 2 +- book/src/03-configuration.md | 2 +- book/src/07-cqrs.md | 6 +- book/src/08-eda-outbox.md | 2 +- book/src/09-transactions.md | 2 +- book/src/13-cli-cache.md | 17 +- deptrac.yaml | 5 +- docs/cli.md | 7 +- docs/tutorial.es.md | 7 +- docs/tutorial.md | 6 +- packages/cli/composer.json | 1 + .../src/Boot/FireflyCacheServiceProvider.php | 4 + packages/cli/src/Cache/FireflyCachePaths.php | 2 + .../cli/src/Cache/ManifestCacheWriter.php | 8 + .../tests/Cache/ManifestCacheWriterTest.php | 2 +- packages/context/src/Scan/AppScan.php | 2 + packages/context/tests/Scan/AppScanTest.php | 2 +- .../src/Manifest/OrchestrationManifest.php | 110 +++ .../OrchestrationManifestCompiler.php | 30 + .../src/Manifest/ScheduledExecution.php | 58 ++ .../src/Scanner/OrchestrationScanner.php | 678 ++++++++++++++++++ .../Manifest/OrchestrationManifestTest.php | 26 + .../tests/ReflectionFreeOrchestrationTest.php | 4 +- .../Scanner/OrchestrationScannerTest.php | 110 +++ .../Bad/BadCondition/BadConditionWorkflow.php | 15 + .../ScannerFixtures/Bad/Cycle/CycleSaga.php | 18 + .../Bad/HalfParticipant/HalfParticipant.php | 19 + .../Bad/HalfParticipant/HalfTcc.php | 10 + .../Bad/InitialDelay/InitialDelaySaga.php | 17 + .../MissingCompensationSaga.php | 15 + .../Bad/OrphanExternal/OrphanStep.php | 13 + .../Bad/TwoImplicit/TwoImplicitSaga.php | 15 + .../ScannerFixtures/Good/ApprovalWorkflow.php | 57 ++ .../Good/CreditParticipant.php | 26 + .../ScannerFixtures/Good/DebitParticipant.php | 30 + .../ScannerFixtures/Good/InventoryService.php | 21 + .../tests/ScannerFixtures/Good/OrderSaga.php | 57 ++ .../ScannerFixtures/Good/TransferTcc.php | 16 + tests/DocsProseIsRealTest.php | 8 +- 45 files changed, 1419 insertions(+), 42 deletions(-) create mode 100644 packages/orchestration/src/Manifest/OrchestrationManifest.php create mode 100644 packages/orchestration/src/Manifest/OrchestrationManifestCompiler.php create mode 100644 packages/orchestration/src/Manifest/ScheduledExecution.php create mode 100644 packages/orchestration/src/Scanner/OrchestrationScanner.php create mode 100644 packages/orchestration/tests/Manifest/OrchestrationManifestTest.php create mode 100644 packages/orchestration/tests/Scanner/OrchestrationScannerTest.php create mode 100644 packages/orchestration/tests/ScannerFixtures/Bad/BadCondition/BadConditionWorkflow.php create mode 100644 packages/orchestration/tests/ScannerFixtures/Bad/Cycle/CycleSaga.php create mode 100644 packages/orchestration/tests/ScannerFixtures/Bad/HalfParticipant/HalfParticipant.php create mode 100644 packages/orchestration/tests/ScannerFixtures/Bad/HalfParticipant/HalfTcc.php create mode 100644 packages/orchestration/tests/ScannerFixtures/Bad/InitialDelay/InitialDelaySaga.php create mode 100644 packages/orchestration/tests/ScannerFixtures/Bad/MissingCompensation/MissingCompensationSaga.php create mode 100644 packages/orchestration/tests/ScannerFixtures/Bad/OrphanExternal/OrphanStep.php create mode 100644 packages/orchestration/tests/ScannerFixtures/Bad/TwoImplicit/TwoImplicitSaga.php create mode 100644 packages/orchestration/tests/ScannerFixtures/Good/ApprovalWorkflow.php create mode 100644 packages/orchestration/tests/ScannerFixtures/Good/CreditParticipant.php create mode 100644 packages/orchestration/tests/ScannerFixtures/Good/DebitParticipant.php create mode 100644 packages/orchestration/tests/ScannerFixtures/Good/InventoryService.php create mode 100644 packages/orchestration/tests/ScannerFixtures/Good/OrderSaga.php create mode 100644 packages/orchestration/tests/ScannerFixtures/Good/TransferTcc.php diff --git a/book/src-es/02-dependency-injection.md b/book/src-es/02-dependency-injection.md index 0ef00a1a..173b98c3 100644 --- a/book/src-es/02-dependency-injection.md +++ b/book/src-es/02-dependency-injection.md @@ -367,7 +367,7 @@ php artisan firefly:cache ``` ``` -firefly:cache — wrote 14 manifest(s) + 0 proxy(ies) to /path/to/my-app/bootstrap/cache/firefly +firefly:cache — wrote 15 manifest(s) + 0 proxy(ies) to /path/to/my-app/bootstrap/cache/firefly ``` Ya ejecutaste esto una vez, de forma indirecta — `firefly new` lo llama por ti en su script `post-create-project-cmd`, que es la razón por la que la aplicación del Inicio rápido arrancó correctamente sin que ejecutaras el comando a mano ni una sola vez. De aquí en adelante, cada vez que añadas o cambies un `#[Service]`, `#[Repository]`, `#[Configuration]` o cualquier otro atributo de Firefly, vuelve a ejecutar `firefly:cache` para recompilar el manifiesto; `php artisan firefly:clear` elimina la caché compilada y vuelve a la ruta de escaneo de desarrollo (más lenta, basada en reflexión). diff --git a/book/src-es/03-configuration.md b/book/src-es/03-configuration.md index 2f35577f..a9c8b3e1 100644 --- a/book/src-es/03-configuration.md +++ b/book/src-es/03-configuration.md @@ -231,7 +231,7 @@ foreach ($this->classesIn($prefix, $dir) as $class) { } ``` -`ConfigManifestCompiler` escribe esa lista en `bootstrap/cache/firefly/config-properties.php` con la misma técnica basada en `var_export()`, sin reflexión, que usó el manifiesto de componentes. `php artisan firefly:cache` ejecuta este escaneo como uno de los doce pares escáner/compilador que orquesta — nunca invocas `ConfigPropertiesScanner` tú mismo. +`ConfigManifestCompiler` escribe esa lista en `bootstrap/cache/firefly/config-properties.php` con la misma técnica basada en `var_export()`, sin reflexión, que usó el manifiesto de componentes. `php artisan firefly:cache` ejecuta este escaneo como uno de los trece pares escáner/compilador que orquesta — nunca invocas `ConfigPropertiesScanner` tú mismo. En el arranque, `Firefly\Cli\Boot\FireflyCacheServiceProvider` carga ese manifiesto compilado y se lo entrega directamente a `Firefly\Config\Registrar\ConfigRegistrar::register()`, que vincula el DTO de cada descriptor como singleton del contenedor en un solo paso: diff --git a/book/src-es/07-cqrs.md b/book/src-es/07-cqrs.md index 119f2f9e..95af9d88 100644 --- a/book/src-es/07-cqrs.md +++ b/book/src-es/07-cqrs.md @@ -456,11 +456,11 @@ El tipo exterior te dice *qué operación del bus* falló; el tipo interior — `HandlerScanner` es el **único** punto de reflexión en `packages/cqrs/src` (la misma invariante forzada por grep que el Capítulo 6 describió para `firefly/domain`). Recorre una vez las raíces PSR-4 de la aplicación, resuelve cada `#[CommandHandler]`/`#[QueryHandler]` (ejecutando exactamente la inferencia de `messageType()` que acabas de leer) en un `HandlerDescriptor`, y entrega la lista completa a `HandlerManifestCompiler`, que la `var_export()`a en un archivo PHP de array plano — la misma técnica sin reflexión que el Capítulo 2 introdujo para el manifiesto de componentes y el Capítulo 3 para `#[ConfigProperties]`. -Nunca ejecutas `HandlerScanner` tú mismo. Es uno de los **doce** pares escáner/compilador que `php artisan firefly:cache` orquesta en un solo paso, junto al de componentes/contexto, rutas, propiedades de configuración y (del Capítulo 9) el transaccional: +Nunca ejecutas `HandlerScanner` tú mismo. Es uno de los **trece** pares escáner/compilador que `php artisan firefly:cache` orquesta en un solo paso, junto al de componentes/contexto, rutas, propiedades de configuración y (del Capítulo 9) el transaccional: ```bash $ php artisan firefly:cache -firefly:cache — wrote 14 manifest(s) + 1 proxy(ies) to /path/to/my-app/bootstrap/cache/firefly +firefly:cache — wrote 15 manifest(s) + 1 proxy(ies) to /path/to/my-app/bootstrap/cache/firefly ``` El archivo compilado aterriza en `bootstrap/cache/firefly/handlers.php`; `FireflyCacheServiceProvider` lo vincula directamente sobre la entrada del contenedor `HandlerManifest` — sobrescribiendo incondicionalmente el `HandlerManifest` **vacío** por defecto que `CqrsWiringProvider` vincula cuando todavía no existe ninguna caché (el mismo patrón de retirada al estilo `#[ConditionalOnMissingBean]` que el Capítulo 2 te mostró para los beans, aplicado aquí a todo un manifiesto compilado). `CqrsHandlerWiringPass` recorre entonces ese manifiesto en el arranque y llena el `HandlerRegistry` que conociste arriba con un invocador por descriptor — cada invocador resuelve su bean manejador **de nuevo desde el contenedor en cada despacho**, nunca en caché en tiempo de cableado, así que un manejador `#[Transactional]` como `OpenWalletHandler` siempre se observa a través de su proxy generado (Capítulo 9), nunca a través de la clase desnuda, sin proxy. @@ -479,7 +479,7 @@ Equivócate en esto en una aplicación — añade un manejador, olvida volver a | La tubería acotada | correlacionar → validar → autorizar → (obtener de caché, solo consultas) → resolver e invocar → (guardar en caché, solo consultas) → métricas | | `CommandProcessingException` / `QueryProcessingException` | Envoltorios que preservan la categoría; el fallo original siempre es alcanzable vía `getPrevious()` | | `HandlerRegistry` | Una clase de comando/consulta se mapea a exactamente un manejador — un registro duplicado falla ruidosamente en tiempo de cableado | -| `HandlerScanner` → `HandlerManifest` | El único punto de reflexión; compilado por `firefly:cache` (uno de doce pares escáner/compilador) en un manifiesto sin reflexión | +| `HandlerScanner` → `HandlerManifest` | El único punto de reflexión; compilado por `firefly:cache` (uno de trece pares escáner/compilador) en un manifiesto sin reflexión | --- diff --git a/book/src-es/08-eda-outbox.md b/book/src-es/08-eda-outbox.md index 581f52c4..c0ab483b 100644 --- a/book/src-es/08-eda-outbox.md +++ b/book/src-es/08-eda-outbox.md @@ -183,7 +183,7 @@ final class OrderService Aquí el destino (`'orders'`) y el tipo de evento (`'order.placed'`) son cadenas deliberadamente distintas, y el patrón `'order.*'` coincide con la **segunda** — precisamente para que este ejemplo no pueda confundirse con evidencia de que los patrones alguna vez miran el destino. -`EventListenerScanner` (`packages/eda/src/Scanner/EventListenerScanner.php`) es el único sitio de reflexión en `firefly/eda`, recorre una vez las raíces PSR-4 de la aplicación y compila cada método `#[EventListener]` en un `EventListenerDescriptor`; `EventListenerManifest` carga el resultado sin reflexión alguna, exactamente el mismo idioma de `HandlerManifest` que te mostró el Capítulo 7. `firefly:cache` ejecuta este escáner como uno de sus doce pares, escribiendo `bootstrap/cache/firefly/event-listeners.php`; `FireflyCacheServiceProvider` lo vincula, sustituyendo el valor por defecto vacío con el que arrancaría una aplicación sin caché. `EventListenerWiringPass` recorre después ese manifiesto al arrancar — en *todo* proceso, tanto en la petición web como en el worker de cola — envuelve cada invocación de destino en el decorador de reintento/DLQ de más abajo, y llama a `$bus->subscribe($pattern, $wrapped)` por cada patrón, resolviendo el bean de destino **fresco desde el contenedor en cada despacho**. +`EventListenerScanner` (`packages/eda/src/Scanner/EventListenerScanner.php`) es el único sitio de reflexión en `firefly/eda`, recorre una vez las raíces PSR-4 de la aplicación y compila cada método `#[EventListener]` en un `EventListenerDescriptor`; `EventListenerManifest` carga el resultado sin reflexión alguna, exactamente el mismo idioma de `HandlerManifest` que te mostró el Capítulo 7. `firefly:cache` ejecuta este escáner como uno de sus trece pares, escribiendo `bootstrap/cache/firefly/event-listeners.php`; `FireflyCacheServiceProvider` lo vincula, sustituyendo el valor por defecto vacío con el que arrancaría una aplicación sin caché. `EventListenerWiringPass` recorre después ese manifiesto al arrancar — en *todo* proceso, tanto en la petición web como en el worker de cola — envuelve cada invocación de destino en el decorador de reintento/DLQ de más abajo, y llama a `$bus->subscribe($pattern, $wrapped)` por cada patrón, resolviendo el bean de destino **fresco desde el contenedor en cada despacho**. --- diff --git a/book/src-es/09-transactions.md b/book/src-es/09-transactions.md index 0fcc240a..e3d11bf6 100644 --- a/book/src-es/09-transactions.md +++ b/book/src-es/09-transactions.md @@ -404,7 +404,7 @@ Cuatro detalles, cada uno de los cuales acabarás necesitando: El Capítulo 2 te enseñó la forma de una clase `#[Configuration]` y sus métodos fábrica `#[Bean]`. Durante una versión de este framework, toda aplicación LaraFly tuvo que escribir una de ellas a mano — un `#[Configuration]` cuyo único `#[Bean]` cargaba el `TransactionalManifest` cacheado — o `#[Transactional]` no hacía absolutamente nada. **Ya no necesitas esa clase**, y merece una página: el fallo que aquella clase sorteaba fue de los más afilados que ha tenido este framework, y su arreglo es una pequeña lección sobre cómo se resuelve cada artefacto compilado. -`php artisan firefly:cache` emite **tres** artefactos separados que lee la maquinaria de proxies. Ejecuta `TransactionalScanner`/`TransactionalManifestCompiler` — uno de los doce pares scanner/compiler — para escribir los datos del manifiesto en `bootstrap/cache/firefly/transactional.php`. Ejecuta `ProxyPlanCompiler` sobre el `ProxyPlan` fusionado para escribir `bootstrap/cache/firefly/proxy-plan.php` — el artefacto que decide qué beans reciben siquiera un proxy y qué advice ejecuta cada uno de sus métodos. Y ejecuta `ProxyClassGenerator` para emitir un archivo fuente `{Target}__FireflyTransactionalProxy` **por cada clase que el plan nombra** — cada clase que reclama algún `AdviceSource`, así que un bean que solo lleva reglas de seguridad de método también recibe uno — en un directorio `proxies/`, más un classmap `proxies.php`. +`php artisan firefly:cache` emite **tres** artefactos separados que lee la maquinaria de proxies. Ejecuta `TransactionalScanner`/`TransactionalManifestCompiler` — uno de los trece pares scanner/compiler — para escribir los datos del manifiesto en `bootstrap/cache/firefly/transactional.php`. Ejecuta `ProxyPlanCompiler` sobre el `ProxyPlan` fusionado para escribir `bootstrap/cache/firefly/proxy-plan.php` — el artefacto que decide qué beans reciben siquiera un proxy y qué advice ejecuta cada uno de sus métodos. Y ejecuta `ProxyClassGenerator` para emitir un archivo fuente `{Target}__FireflyTransactionalProxy` **por cada clase que el plan nombra** — cada clase que reclama algún `AdviceSource`, así que un bean que solo lleva reglas de seguridad de método también recibe uno — en un directorio `proxies/`, más un classmap `proxies.php`. Durante mucho tiempo, nada cargaba el primero de esos tres. `DataAutoConfiguration` vinculaba un `TransactionalManifest` *vacío* incondicionalmente, el `transactional.php` compilado solo lo referenciaba su propia declaración, y por eso `hasProxyFor()` era siempre falso y `TransactionalBeanPostProcessor` devolvía cada bean sin envolver. `#[Transactional]` era un **no-op silencioso** en cualquier aplicación que no escribiera a mano su propia configuración de manifiesto — que es exactamente lo que aquel `#[Configuration]` manuscrito existía para hacer, y exactamente el modo de fallo del que avisaba la sección anterior, llegando desde el propio lado del framework. diff --git a/book/src-es/13-cli-cache.md b/book/src-es/13-cli-cache.md index 535dfceb..d99c7a08 100644 --- a/book/src-es/13-cli-cache.md +++ b/book/src-es/13-cli-cache.md @@ -2,7 +2,7 @@ # La CLI y la Caché Sin Reflexión {.chtitle} -Al terminar este capítulo sabrás exactamente qué hace `php artisan firefly:cache` — los trece pasos de escáner-y-compilador que ejecuta, en orden, sobre las propias raíces PSR-4 de tu aplicación, y los manifiestos de `bootstrap/cache/firefly/` y las clases proxy generadas (una por cada clase que nombra el plan de proxies) que escribe — cómo `FireflyCacheServiceProvider` vuelve a cargar esa caché en el arranque con cero reflexión, el inverso exacto de `firefly:clear`, los comandos de actuator-sobre-CLI (`firefly:about`/`:routes`/`:health`/`:metrics`) que reutilizan los endpoints del Capítulo 11 en-proceso sin ida y vuelta HTTP, los ocho generadores `make:firefly-*`, y cómo el `firefly new` de `firefly/installer` te lleva de la nada a una aplicación cacheada y en funcionamiento en un solo comando. +Al terminar este capítulo sabrás exactamente qué hace `php artisan firefly:cache` — los catorce pasos de escáner-y-compilador que ejecuta, en orden, sobre las propias raíces PSR-4 de tu aplicación, y los manifiestos de `bootstrap/cache/firefly/` y las clases proxy generadas (una por cada clase que nombra el plan de proxies) que escribe — cómo `FireflyCacheServiceProvider` vuelve a cargar esa caché en el arranque con cero reflexión, el inverso exacto de `firefly:clear`, los comandos de actuator-sobre-CLI (`firefly:about`/`:routes`/`:health`/`:metrics`) que reutilizan los endpoints del Capítulo 11 en-proceso sin ida y vuelta HTTP, los ocho generadores `make:firefly-*`, y cómo el `firefly new` de `firefly/installer` te lleva de la nada a una aplicación cacheada y en funcionamiento en un solo comando. !!! note "Término nuevo: arranque sin reflexión" Cada capítulo anterior a este ha mencionado, de pasada, que un manifiesto compilado reemplaza a "escanea el sistema de archivos con reflexión de PHP en cada arranque". Un **arranque sin reflexión** es lo que obtienes una vez que `firefly:cache` ha ejecutado realmente: `FireflyCacheServiceProvider` carga manifiestos precomputados directamente desde archivos PHP bajo `bootstrap/cache/firefly/`, de modo que ningún atributo, ninguna clase y ningún método se somete jamás a reflexión en tiempo de petición — el escaneo ocurrió una vez, en tiempo de despliegue, no en cada petición. @@ -125,6 +125,12 @@ final class ManifestCacheWriter $files[] = $dir.'/'.FireflyCachePaths::SCHEDULED, ); + // orchestration — #[Saga] / #[Tcc] / #[Workflow] definitions and their schedules. + (new OrchestrationManifestCompiler)->write( + (new OrchestrationScanner)->scan($psr4), + $files[] = $dir.'/'.FireflyCachePaths::ORCHESTRATION, + ); + // security (new SecurityMethodManifestCompiler)->write( (new MethodSecurityScanner)->scan($psr4), @@ -146,7 +152,7 @@ final class ManifestCacheWriter } ``` -Eso son doce llamadas `scan()`-luego-`write()` — cada capacidad desde el escaneo de componentes del Capítulo 2 hasta el manifiesto de seguridad de método del Capítulo 10, cada una ya presentada en un capítulo anterior como un escaneo en-proceso — más un paso más, `writeProxies()`, que genera las clases proxy que te enseñó el Capítulo 9, usando el mismísimo `ProxyClassGenerator` sin reflexión que usa el propio runtime cuando aún no existe caché: +Eso son trece llamadas `scan()`-luego-`write()` — cada capacidad desde el escaneo de componentes del Capítulo 2 hasta el manifiesto de seguridad de método del Capítulo 10, más el manifiesto de orquestación con las definiciones `#[Saga]`, `#[Tcc]` y `#[Workflow]`, cada una ya presentada en un capítulo anterior como un escaneo en-proceso — más un paso más, `writeProxies()`, que genera las clases proxy que te enseñó el Capítulo 9, usando el mismísimo `ProxyClassGenerator` sin reflexión que usa el propio runtime cuando aún no existe caché: ```php @@ -195,7 +201,7 @@ final class ManifestCacheWriter } ``` -Trece pasos en total dejan catorce artefactos bajo `bootstrap/cache/firefly/`: +Catorce pasos en total dejan quince artefactos bajo `bootstrap/cache/firefly/`: ``` bootstrap/cache/firefly/ @@ -209,6 +215,7 @@ bootstrap/cache/firefly/ ├── event-listeners.php # #[EventListener] manifest ├── message-listeners.php # #[MessageListener] manifest ├── scheduled.php # #[Scheduled] manifest +├── orchestration.php # #[Saga]/#[Tcc]/#[Workflow] definitions and their schedules ├── security-methods.php # #[PreAuthorize]/#[Secured]/#[RolesAllowed] manifest ├── transactional.php # #[Transactional] method manifest ├── proxy-plan.php # qué beans reciben un proxy, y qué advice ejecuta cada método @@ -313,7 +320,7 @@ php artisan firefly:cache ``` ``` -firefly:cache — wrote 14 manifest(s) + 3 proxy(ies) to bootstrap/cache/firefly +firefly:cache — wrote 15 manifest(s) + 3 proxy(ies) to bootstrap/cache/firefly firefly:cache — skipped 1 stale manifest entry naming a class that no longer exists: App\Security\ControlPlaneJwksProvider ``` @@ -521,7 +528,7 @@ El Capítulo 13A presenta `firefly:flags` para inspeccionar y cambiar banderas e | Concepto | Qué hace | |---|---| -| `firefly:cache` | Ejecuta el par escáner→compilador propio de cada capacidad (13 pasos, 14 artefactos) sobre `firefly.scan.paths`; escribe manifiestos + un archivo de clase proxy generado por cada clase que nombra el plan de proxies a `bootstrap/cache/firefly/` | +| `firefly:cache` | Ejecuta el par escáner→compilador propio de cada capacidad (14 pasos, 15 artefactos) sobre `firefly.scan.paths`; escribe manifiestos + un archivo de clase proxy generado por cada clase que nombra el plan de proxies a `bootstrap/cache/firefly/` | | `FireflyCacheServiceProvider` | Enlaza cada manifiesto cacheado incondicionalmente (gana sobre todo valor por defecto vacío) + instala el autoloader de classmap de proxies — antes de que se ejecute cualquier resolución de bean | | `firefly:clear` | Borra recursivamente solo el directorio de caché; el siguiente arranque recurre al escaneo en-proceso | | `firefly:about` / `:routes` / `:health` / `:metrics` | Renderiza los endpoints de actuator del Capítulo 11 en el terminal, en-proceso, sin ida y vuelta HTTP | diff --git a/book/src/02-dependency-injection.md b/book/src/02-dependency-injection.md index 8f0103ba..8dfc6d89 100644 --- a/book/src/02-dependency-injection.md +++ b/book/src/02-dependency-injection.md @@ -367,7 +367,7 @@ php artisan firefly:cache ``` ``` -firefly:cache — wrote 14 manifest(s) + 0 proxy(ies) to /path/to/my-app/bootstrap/cache/firefly +firefly:cache — wrote 15 manifest(s) + 0 proxy(ies) to /path/to/my-app/bootstrap/cache/firefly ``` You already ran this once, indirectly — `firefly new` calls it for you in its `post-create-project-cmd` script, which is why the Quick Start's application booted correctly without you ever running the command by hand. From here on, any time you add or change a `#[Service]`, `#[Repository]`, `#[Configuration]`, or any other Firefly attribute, re-run `firefly:cache` to recompile the manifest; `php artisan firefly:clear` deletes the compiled cache and falls back to the slower, reflection-based development scan. diff --git a/book/src/03-configuration.md b/book/src/03-configuration.md index e57e002b..47502f11 100644 --- a/book/src/03-configuration.md +++ b/book/src/03-configuration.md @@ -231,7 +231,7 @@ foreach ($this->classesIn($prefix, $dir) as $class) { } ``` -`ConfigManifestCompiler` writes that list to `bootstrap/cache/firefly/config-properties.php` with the same `var_export()`-based, zero-reflection technique the component manifest used. `php artisan firefly:cache` runs this scan as one step among the twelve scanner/compiler pairs it orchestrates — you never invoke `ConfigPropertiesScanner` yourself. +`ConfigManifestCompiler` writes that list to `bootstrap/cache/firefly/config-properties.php` with the same `var_export()`-based, zero-reflection technique the component manifest used. `php artisan firefly:cache` runs this scan as one step among the thirteen scanner/compiler pairs it orchestrates — you never invoke `ConfigPropertiesScanner` yourself. At boot, `Firefly\Cli\Boot\FireflyCacheServiceProvider` loads that compiled manifest and hands it straight to `Firefly\Config\Registrar\ConfigRegistrar::register()`, which binds every descriptor's DTO as a container singleton in one pass: diff --git a/book/src/07-cqrs.md b/book/src/07-cqrs.md index 957f321d..da940320 100644 --- a/book/src/07-cqrs.md +++ b/book/src/07-cqrs.md @@ -456,11 +456,11 @@ The outer type tells you *which bus operation* failed; the inner type — reache `HandlerScanner` is the **sole** reflection site in `packages/cqrs/src` (the same grep-enforced invariant Chapter 6 described for `firefly/domain`). It walks the app's PSR-4 roots once, resolves every `#[CommandHandler]`/`#[QueryHandler]` (running exactly the `messageType()` inference you just read) into a `HandlerDescriptor`, and hands the whole list to `HandlerManifestCompiler`, which `var_export()`s it to a plain PHP array file — the same zero-reflection technique Chapter 2 introduced for the component manifest and Chapter 3 for `#[ConfigProperties]`. -You never run `HandlerScanner` yourself. It is one of the **twelve** scanner/compiler pairs `php artisan firefly:cache` orchestrates in one pass, alongside the component/context, route, config-properties, and (Chapter 9's) transactional scanners: +You never run `HandlerScanner` yourself. It is one of the **thirteen** scanner/compiler pairs `php artisan firefly:cache` orchestrates in one pass, alongside the component/context, route, config-properties, and (Chapter 9's) transactional scanners: ```bash $ php artisan firefly:cache -firefly:cache — wrote 14 manifest(s) + 1 proxy(ies) to /path/to/my-app/bootstrap/cache/firefly +firefly:cache — wrote 15 manifest(s) + 1 proxy(ies) to /path/to/my-app/bootstrap/cache/firefly ``` The compiled file lands at `bootstrap/cache/firefly/handlers.php`; `FireflyCacheServiceProvider` binds it straight onto the `HandlerManifest` container entry — unconditionally overriding the **empty** default `CqrsWiringProvider` binds when no cache exists yet (the same `#[ConditionalOnMissingBean]`-style backing-off pattern Chapter 2 showed you for beans, applied here to a whole compiled manifest). `CqrsHandlerWiringPass` then walks that manifest at boot and populates the `HandlerRegistry` you met above with one invoker per descriptor — each invoker resolving its handler bean **fresh from the container on every dispatch**, never cached at wiring time, so a `#[Transactional]` handler like `OpenWalletHandler` is always observed through its generated proxy (Chapter 9), never through the bare, unproxied class. @@ -479,7 +479,7 @@ Get this wrong in an application — add a handler, forget to re-run `firefly:ca | The bounded pipeline | correlate → validate → authorize → (cache-get, query only) → resolve + invoke → (cache-put, query only) → metrics | | `CommandProcessingException` / `QueryProcessingException` | Category-preserving wrappers; the original fault is always reachable via `getPrevious()` | | `HandlerRegistry` | One command/query class maps to exactly one handler — a duplicate registration fails loud at wiring time | -| `HandlerScanner` → `HandlerManifest` | The sole reflection site; compiled by `firefly:cache` (one of twelve scanner/compiler pairs) into a zero-reflection manifest | +| `HandlerScanner` → `HandlerManifest` | The sole reflection site; compiled by `firefly:cache` (one of thirteen scanner/compiler pairs) into a zero-reflection manifest | --- diff --git a/book/src/08-eda-outbox.md b/book/src/08-eda-outbox.md index 06218bd7..f28e537a 100644 --- a/book/src/08-eda-outbox.md +++ b/book/src/08-eda-outbox.md @@ -183,7 +183,7 @@ final class OrderService Here the destination (`'orders'`) and the event type (`'order.placed'`) are deliberately different strings, and the `'order.*'` pattern matches the **second** one — precisely so this example cannot be mistaken for evidence that patterns ever look at the destination. -`EventListenerScanner` (`packages/eda/src/Scanner/EventListenerScanner.php`) is the sole reflection site in `firefly/eda`, walking the app's PSR-4 roots once and compiling every `#[EventListener]` method into an `EventListenerDescriptor`; `EventListenerManifest` loads the result with zero reflection, exactly the `HandlerManifest` idiom Chapter 7 just showed you. `firefly:cache` runs this scanner as one of its twelve pairs, writing `bootstrap/cache/firefly/event-listeners.php`; `FireflyCacheServiceProvider` binds it, overriding the empty default an uncached app would otherwise boot with. `EventListenerWiringPass` then walks that manifest at boot — in *every* process, web request and queue worker alike — wraps each target invocation in the retry/DLQ decorator below, and calls `$bus->subscribe($pattern, $wrapped)` per pattern, resolving the target bean **fresh from the container on every dispatch**. +`EventListenerScanner` (`packages/eda/src/Scanner/EventListenerScanner.php`) is the sole reflection site in `firefly/eda`, walking the app's PSR-4 roots once and compiling every `#[EventListener]` method into an `EventListenerDescriptor`; `EventListenerManifest` loads the result with zero reflection, exactly the `HandlerManifest` idiom Chapter 7 just showed you. `firefly:cache` runs this scanner as one of its thirteen pairs, writing `bootstrap/cache/firefly/event-listeners.php`; `FireflyCacheServiceProvider` binds it, overriding the empty default an uncached app would otherwise boot with. `EventListenerWiringPass` then walks that manifest at boot — in *every* process, web request and queue worker alike — wraps each target invocation in the retry/DLQ decorator below, and calls `$bus->subscribe($pattern, $wrapped)` per pattern, resolving the target bean **fresh from the container on every dispatch**. --- diff --git a/book/src/09-transactions.md b/book/src/09-transactions.md index 736c2ae4..bc6aef3c 100644 --- a/book/src/09-transactions.md +++ b/book/src/09-transactions.md @@ -404,7 +404,7 @@ Four details, each of which you will eventually need: Chapter 2 taught you the shape of a `#[Configuration]` class and its `#[Bean]` factory methods. For one release of this framework, every LaraFly application had to write one of them by hand — a `#[Configuration]` whose single `#[Bean]` loaded the cached `TransactionalManifest` — or `#[Transactional]` did nothing at all. **You no longer need that class**, and it is worth a page: the bug it worked around was one of the sharpest this framework has had, and the fix is a small lesson in how every compiled artifact is resolved. -`php artisan firefly:cache` emits **three** separate artifacts the proxy machinery reads. It runs `TransactionalScanner`/`TransactionalManifestCompiler` — one of the twelve scanner/compiler pairs — to write the manifest data to `bootstrap/cache/firefly/transactional.php`. It runs `ProxyPlanCompiler` over the merged `ProxyPlan` to write `bootstrap/cache/firefly/proxy-plan.php` — the artifact that decides which beans get a proxy at all and which advice each of their methods runs. And it runs `ProxyClassGenerator` to emit one `{Target}__FireflyTransactionalProxy` source file **per class the plan names** — every class any `AdviceSource` claims, so a bean carrying only method-security rules gets one too — into a `proxies/` directory, plus a `proxies.php` classmap. +`php artisan firefly:cache` emits **three** separate artifacts the proxy machinery reads. It runs `TransactionalScanner`/`TransactionalManifestCompiler` — one of the thirteen scanner/compiler pairs — to write the manifest data to `bootstrap/cache/firefly/transactional.php`. It runs `ProxyPlanCompiler` over the merged `ProxyPlan` to write `bootstrap/cache/firefly/proxy-plan.php` — the artifact that decides which beans get a proxy at all and which advice each of their methods runs. And it runs `ProxyClassGenerator` to emit one `{Target}__FireflyTransactionalProxy` source file **per class the plan names** — every class any `AdviceSource` claims, so a bean carrying only method-security rules gets one too — into a `proxies/` directory, plus a `proxies.php` classmap. For a long time, nothing loaded the first of those three. `DataAutoConfiguration` bound an unconditional *empty* `TransactionalManifest`, the compiled `transactional.php` was referenced only by its own declaration, and so `hasProxyFor()` was always false and `TransactionalBeanPostProcessor` handed back every bean unwrapped. `#[Transactional]` was a **silent no-op** in any application that did not hand-write its own manifest configuration — which is exactly what that hand-written `#[Configuration]` existed to do, and exactly the failure mode the previous section warned about, arriving from the framework's own side. diff --git a/book/src/13-cli-cache.md b/book/src/13-cli-cache.md index 6a625374..d4c29581 100644 --- a/book/src/13-cli-cache.md +++ b/book/src/13-cli-cache.md @@ -2,7 +2,7 @@ # The CLI and the Zero-Reflection Cache {.chtitle} -By the end of this chapter you will know exactly what `php artisan firefly:cache` does — the thirteen scanner-and-compiler steps it runs, in order, over your application's own PSR-4 roots, and the `bootstrap/cache/firefly/` manifests and generated proxy classes (one per class the proxy plan names) it writes — how `FireflyCacheServiceProvider` loads that cache back in at boot with zero reflection, `firefly:clear`'s exact inverse, the actuator-over-CLI commands (`firefly:about`/`:routes`/`:health`/`:metrics`) that reuse Chapter 11's endpoints in-process with no HTTP round trip, the eight `make:firefly-*` generators, and how `firefly/installer`'s `firefly new` gets you from nothing to a cached, running application in one command. +By the end of this chapter you will know exactly what `php artisan firefly:cache` does — the fourteen scanner-and-compiler steps it runs, in order, over your application's own PSR-4 roots, and the `bootstrap/cache/firefly/` manifests and generated proxy classes (one per class the proxy plan names) it writes — how `FireflyCacheServiceProvider` loads that cache back in at boot with zero reflection, `firefly:clear`'s exact inverse, the actuator-over-CLI commands (`firefly:about`/`:routes`/`:health`/`:metrics`) that reuse Chapter 11's endpoints in-process with no HTTP round trip, the eight `make:firefly-*` generators, and how `firefly/installer`'s `firefly new` gets you from nothing to a cached, running application in one command. !!! note "New term: zero-reflection boot" Every chapter before this one has mentioned, in passing, that a compiled manifest replaces "scan the filesystem with PHP reflection at every boot." A **zero-reflection boot** is what you get once `firefly:cache` has actually run: `FireflyCacheServiceProvider` loads pre-computed manifests straight from PHP files under `bootstrap/cache/firefly/`, so no attribute, no class, and no method is ever reflected on at request time — the scan happened once, at deploy time, not on every request. @@ -125,6 +125,12 @@ final class ManifestCacheWriter $files[] = $dir.'/'.FireflyCachePaths::SCHEDULED, ); + // orchestration — #[Saga] / #[Tcc] / #[Workflow] definitions and their schedules. + (new OrchestrationManifestCompiler)->write( + (new OrchestrationScanner)->scan($psr4), + $files[] = $dir.'/'.FireflyCachePaths::ORCHESTRATION, + ); + // security (new SecurityMethodManifestCompiler)->write( (new MethodSecurityScanner)->scan($psr4), @@ -146,7 +152,7 @@ final class ManifestCacheWriter } ``` -That is twelve `scan()`-then-`write()` calls — every capability from Chapter 2's component scan through Chapter 10's method-security manifest, each already introduced in an earlier chapter as an in-process scan — plus one more step, `writeProxies()`, that generates the proxy classes Chapter 9 taught you about, using the exact same reflection-free `ProxyClassGenerator` the runtime itself uses when no cache exists yet: +That is thirteen `scan()`-then-`write()` calls — every capability from Chapter 2's component scan through Chapter 10's method-security manifest, plus the orchestration manifest of `#[Saga]`, `#[Tcc]` and `#[Workflow]` definitions, each already introduced in an earlier chapter as an in-process scan — plus one more step, `writeProxies()`, that generates the proxy classes Chapter 9 taught you about, using the exact same reflection-free `ProxyClassGenerator` the runtime itself uses when no cache exists yet: ```php @@ -195,7 +201,7 @@ final class ManifestCacheWriter } ``` -Thirteen steps in total land fourteen artifacts under `bootstrap/cache/firefly/`: +Fourteen steps in total land fifteen artifacts under `bootstrap/cache/firefly/`: ``` bootstrap/cache/firefly/ @@ -209,6 +215,7 @@ bootstrap/cache/firefly/ ├── event-listeners.php # #[EventListener] manifest ├── message-listeners.php # #[MessageListener] manifest ├── scheduled.php # #[Scheduled] manifest +├── orchestration.php # #[Saga]/#[Tcc]/#[Workflow] definitions and their schedules ├── security-methods.php # #[PreAuthorize]/#[Secured]/#[RolesAllowed] manifest ├── transactional.php # #[Transactional] method manifest ├── proxy-plan.php # which beans get a proxy, and which advice each method runs @@ -313,7 +320,7 @@ php artisan firefly:cache ``` ``` -firefly:cache — wrote 14 manifest(s) + 3 proxy(ies) to bootstrap/cache/firefly +firefly:cache — wrote 15 manifest(s) + 3 proxy(ies) to bootstrap/cache/firefly firefly:cache — skipped 1 stale manifest entry naming a class that no longer exists: App\Security\ControlPlaneJwksProvider ``` @@ -521,7 +528,7 @@ Chapter 13A introduces `firefly:flags` for inspecting and changing runtime flags | Concept | What it does | |---|---| -| `firefly:cache` | Runs every capability's own scanner→compiler pair (13 steps, 14 artifacts) over `firefly.scan.paths`; writes manifests + one generated proxy class file per class the proxy plan names to `bootstrap/cache/firefly/` | +| `firefly:cache` | Runs every capability's own scanner→compiler pair (14 steps, 15 artifacts) over `firefly.scan.paths`; writes manifests + one generated proxy class file per class the proxy plan names to `bootstrap/cache/firefly/` | | `FireflyCacheServiceProvider` | Binds each cached manifest unconditionally (wins over every empty default) + installs the proxy classmap autoloader — before any bean resolution runs | | `firefly:clear` | Recursively deletes only the cache dir; the next boot falls back to in-process scanning | | `firefly:about` / `:routes` / `:health` / `:metrics` | Renders Chapter 11's actuator endpoints at the terminal, in-process, no HTTP round trip | diff --git a/deptrac.yaml b/deptrac.yaml index 45311b79..27a6f6ec 100644 --- a/deptrac.yaml +++ b/deptrac.yaml @@ -568,7 +568,7 @@ deptrac: - Observability - FeatureFlags - # Cli is the top-of-stack developer-experience package: firefly:cache drives each of the 12 + # Cli is the top-of-stack developer-experience package: firefly:cache drives each of the 13 # compile-producing packages' scanner/compiler pair directly (D4 DIRECT DEPENDENCY). UNLIKE Testing # (whose composer.json genuinely requires the full runtime family, so it legitimately lists all 18 # layers), firefly/cli's composer.json requires exactly the packages ManifestCacheWriter/CacheCommand/ @@ -581,6 +581,8 @@ deptrac: # FeatureFlags is listed for the same reason: ManifestCacheWriter's planner names FeatureFlagAdviceSource, so # firefly:cache compiles #[FeatureFlag] into the same proxy plan; packages/cli/composer.json requires it # accordingly. + # Orchestration is listed for the same reason: ManifestCacheWriter compiles #[Saga]/#[Tcc]/#[Workflow] through + # OrchestrationScanner into orchestration.php, and packages/cli/composer.json requires it accordingly. # It is depended on by NONE. It does NOT depend on Testing (dev-only). Cli: - Kernel @@ -601,6 +603,7 @@ deptrac: - Observability - Resilience - FeatureFlags + - Orchestration # The bundled installer reads the framework version for its local skeleton repository. Installer: diff --git a/docs/cli.md b/docs/cli.md index f208be69..03dcdd18 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -20,10 +20,10 @@ php artisan firefly:cache ``` The zero-reflection payoff: it runs every settled package's existing scanner → compiler pair over -`config('firefly.scan.paths')` (a PSR-4 map, e.g. `'App\\' => app_path()`) and writes fourteen compiled +`config('firefly.scan.paths')` (a PSR-4 map, e.g. `'App\\' => app_path()`) and writes fifteen compiled artifacts — every manifest below, unconditionally, whether or not the application has anything to put in it — plus one generated proxy class per planned bean, into `bootstrap/cache/firefly/`. It prints what it wrote: -`firefly:cache — wrote 14 manifest(s) + N proxy(ies) to `. +`firefly:cache — wrote 15 manifest(s) + N proxy(ies) to `. ``` bootstrap/cache/firefly/ @@ -37,6 +37,7 @@ bootstrap/cache/firefly/ ├── event-listeners.php # #[EventListener] manifest ├── message-listeners.php # #[MessageListener] manifest ├── scheduled.php # #[Scheduled] manifest +├── orchestration.php # #[Saga]/#[Tcc]/#[Workflow] definitions and their schedules ├── security-methods.php # #[PreAuthorize]/#[Secured]/#[RolesAllowed] manifest ├── transactional.php # #[Transactional] method manifest ├── proxy-plan.php # the compiled ProxyPlan: which beans get a proxy, and the advice each method runs @@ -110,7 +111,7 @@ php artisan firefly:cache ``` ``` -firefly:cache — wrote 14 manifest(s) + 3 proxy(ies) to bootstrap/cache/firefly +firefly:cache — wrote 15 manifest(s) + 3 proxy(ies) to bootstrap/cache/firefly firefly:cache — skipped 1 stale manifest entry naming a class that no longer exists: App\Security\ControlPlaneJwksProvider ``` diff --git a/docs/tutorial.es.md b/docs/tutorial.es.md index a9e7619a..7bce932f 100644 --- a/docs/tutorial.es.md +++ b/docs/tutorial.es.md @@ -1093,17 +1093,18 @@ php artisan firefly:cache ``` ``` -firefly:cache — wrote 14 manifest(s) + 1 proxy(ies) to /path/to/my-app/bootstrap/cache/firefly +firefly:cache — wrote 15 manifest(s) + 1 proxy(ies) to /path/to/my-app/bootstrap/cache/firefly ``` -El recuento de manifiestos es fijo, catorce, porque `ManifestCacheWriter` escribe cada artefacto sin +El recuento de manifiestos es fijo, quince, porque `ManifestCacheWriter` escribe cada artefacto sin condiciones — una aplicación sin ningún método `#[Scheduled]` obtiene igualmente un `scheduled.php` vacío, que es lo que permite a la ruta de arranque leer "el fichero no está" como "nadie ha compilado todavía" y no como "no hay nada que cargar". Este único comando ejecuta *cada* par escáner → compilador ya asentado de cada paquete sobre `config('firefly.scan.paths')` y escribe, en `bootstrap/cache/firefly/`: los manifiestos de DI/context/config-properties, la tabla de rutas compilada, el manifiesto de handlers `#[ControllerAdvice]`, el manifiesto de restricciones de validación, el manifiesto de handlers de CQRS, los -manifiestos de listeners de evento/mensaje, el manifiesto de tareas programadas, el manifiesto de métodos de +manifiestos de listeners de evento/mensaje, el manifiesto de tareas programadas, el manifiesto de orquestación +(`#[Saga]`/`#[Tcc]`/`#[Workflow]`), el manifiesto de métodos de seguridad, el manifiesto de `#[Transactional]`, el **plan de proxies** (`proxy-plan.php` — qué beans reciben un proxy y qué advice ejecuta cada uno de sus métodos) y el classmap de proxies, **y** un fichero de clase `RenameGreetingHandler__FireflyTransactionalProxy` generado (el "1 proxy" de arriba). Se emite un proxy por diff --git a/docs/tutorial.md b/docs/tutorial.md index 681f77f2..e7184f57 100644 --- a/docs/tutorial.md +++ b/docs/tutorial.md @@ -1074,17 +1074,17 @@ php artisan firefly:cache ``` ``` -firefly:cache — wrote 14 manifest(s) + 1 proxy(ies) to /path/to/my-app/bootstrap/cache/firefly +firefly:cache — wrote 15 manifest(s) + 1 proxy(ies) to /path/to/my-app/bootstrap/cache/firefly ``` -The manifest count is fixed at fourteen, because `ManifestCacheWriter` writes every artifact unconditionally +The manifest count is fixed at fifteen, because `ManifestCacheWriter` writes every artifact unconditionally — an application with no `#[Scheduled]` method still gets an empty `scheduled.php`, which is what lets the boot path treat "the file is missing" as "nobody has compiled yet" rather than as "there is nothing to load". This single command drives *every* settled package's existing scanner → compiler pair over `config('firefly.scan.paths')` and writes, into `bootstrap/cache/firefly/`: the DI/context/config-properties manifests, the compiled route table, the `#[ControllerAdvice]` handler manifest, the validation constraint manifest, the CQRS handler manifest, the event/message-listener manifests, the scheduled-task manifest, the -security-method manifest, the `#[Transactional]` manifest, the **proxy plan** (`proxy-plan.php` — which +orchestration manifest (`#[Saga]`/`#[Tcc]`/`#[Workflow]`), the security-method manifest, the `#[Transactional]` manifest, the **proxy plan** (`proxy-plan.php` — which beans get a proxy and which advice each of their methods runs) and the proxy classmap, **and** a generated `RenameGreetingHandler__FireflyTransactionalProxy` class file (the "1 proxy" above). A proxy is emitted per class the *plan* claims, which is every class any advice source names — so a `#[Service]` carrying only diff --git a/packages/cli/composer.json b/packages/cli/composer.json index 82f1e05b..3455988b 100644 --- a/packages/cli/composer.json +++ b/packages/cli/composer.json @@ -33,6 +33,7 @@ "firefly/kernel": "*@dev", "firefly/messaging": "*@dev", "firefly/observability": "*@dev", + "firefly/orchestration": "*@dev", "firefly/resilience": "*@dev", "firefly/scheduling": "*@dev", "firefly/security": "*@dev", diff --git a/packages/cli/src/Boot/FireflyCacheServiceProvider.php b/packages/cli/src/Boot/FireflyCacheServiceProvider.php index bae0b395..28f84d83 100644 --- a/packages/cli/src/Boot/FireflyCacheServiceProvider.php +++ b/packages/cli/src/Boot/FireflyCacheServiceProvider.php @@ -11,6 +11,7 @@ use Firefly\Cqrs\Handler\HandlerManifest; use Firefly\Eda\Listener\EventListenerManifest; use Firefly\Messaging\Listener\MessageListenerManifest; +use Firefly\Orchestration\Manifest\OrchestrationManifest; use Firefly\Scheduling\Schedule\ScheduledManifest; use Firefly\Security\Access\Method\SecurityMethodManifest; use Firefly\Validation\Constraint\ConstraintManifest; @@ -55,6 +56,9 @@ public function register(): void if (is_file($path = $dir.'/'.FireflyCachePaths::SCHEDULED)) { $this->app->instance(ScheduledManifest::class, ScheduledManifest::load($path)); } + if (is_file($path = $dir.'/'.FireflyCachePaths::ORCHESTRATION)) { + $this->app->instance(OrchestrationManifest::class, OrchestrationManifest::load($path)); + } if (is_file($path = $dir.'/'.FireflyCachePaths::SECURITY_METHODS)) { $this->app->instance(SecurityMethodManifest::class, SecurityMethodManifest::load($path)); } diff --git a/packages/cli/src/Cache/FireflyCachePaths.php b/packages/cli/src/Cache/FireflyCachePaths.php index efae4b3d..afee3ff5 100644 --- a/packages/cli/src/Cache/FireflyCachePaths.php +++ b/packages/cli/src/Cache/FireflyCachePaths.php @@ -29,6 +29,8 @@ final class FireflyCachePaths public const string SCHEDULED = 'scheduled.php'; + public const string ORCHESTRATION = 'orchestration.php'; + public const string SECURITY_METHODS = 'security-methods.php'; public const string TRANSACTIONAL = 'transactional.php'; diff --git a/packages/cli/src/Cache/ManifestCacheWriter.php b/packages/cli/src/Cache/ManifestCacheWriter.php index 99d5c6fd..5c1b67d2 100644 --- a/packages/cli/src/Cache/ManifestCacheWriter.php +++ b/packages/cli/src/Cache/ManifestCacheWriter.php @@ -21,6 +21,8 @@ use Firefly\Messaging\Listener\MessageListenerManifestCompiler; use Firefly\Messaging\Scanner\MessageListenerScanner; use Firefly\Observability\Method\ObservabilityAdviceSource; +use Firefly\Orchestration\Manifest\OrchestrationManifestCompiler; +use Firefly\Orchestration\Scanner\OrchestrationScanner; use Firefly\Resilience\Method\ResilienceAdviceSource; use Firefly\Scheduling\Scanner\ScheduledScanner; use Firefly\Scheduling\Schedule\ScheduledManifestCompiler; @@ -187,6 +189,12 @@ public function writeManifests(array $psr4, string $dir): CacheReport $files[] = $dir.'/'.FireflyCachePaths::SCHEDULED, ); + // orchestration — #[Saga] / #[Tcc] / #[Workflow] definitions and their schedules. + (new OrchestrationManifestCompiler)->write( + (new OrchestrationScanner)->scan($psr4), + $files[] = $dir.'/'.FireflyCachePaths::ORCHESTRATION, + ); + // security (new SecurityMethodManifestCompiler)->write( (new MethodSecurityScanner)->scan($psr4), diff --git a/packages/cli/tests/Cache/ManifestCacheWriterTest.php b/packages/cli/tests/Cache/ManifestCacheWriterTest.php index bcd8a598..2eb28424 100644 --- a/packages/cli/tests/Cache/ManifestCacheWriterTest.php +++ b/packages/cli/tests/Cache/ManifestCacheWriterTest.php @@ -22,7 +22,7 @@ function fixturesPsr4(): array foreach ([ FireflyCachePaths::COMPONENT, FireflyCachePaths::CONTEXT, FireflyCachePaths::CONFIG_PROPERTIES, FireflyCachePaths::ROUTES, FireflyCachePaths::CONSTRAINTS, FireflyCachePaths::HANDLERS, - FireflyCachePaths::EVENT_LISTENERS, FireflyCachePaths::MESSAGE_LISTENERS, FireflyCachePaths::SCHEDULED, + FireflyCachePaths::EVENT_LISTENERS, FireflyCachePaths::MESSAGE_LISTENERS, FireflyCachePaths::SCHEDULED, FireflyCachePaths::ORCHESTRATION, FireflyCachePaths::SECURITY_METHODS, FireflyCachePaths::TRANSACTIONAL, FireflyCachePaths::PROXY_PLAN, ] as $basename) { $path = $dir.'/'.$basename; diff --git a/packages/context/src/Scan/AppScan.php b/packages/context/src/Scan/AppScan.php index 747a774d..d433b38a 100644 --- a/packages/context/src/Scan/AppScan.php +++ b/packages/context/src/Scan/AppScan.php @@ -62,6 +62,8 @@ final class AppScan public const string SCHEDULED = 'scheduled.php'; + public const string ORCHESTRATION = 'orchestration.php'; + public const string SECURITY_METHODS = 'security-methods.php'; public const string TRANSACTIONAL = 'transactional.php'; diff --git a/packages/context/tests/Scan/AppScanTest.php b/packages/context/tests/Scan/AppScanTest.php index 8b6c7197..10ff4c87 100644 --- a/packages/context/tests/Scan/AppScanTest.php +++ b/packages/context/tests/Scan/AppScanTest.php @@ -74,7 +74,7 @@ function appScanContainer(array $firefly = []): Container foreach ([ AppScan::COMPONENT, AppScan::CONTEXT, AppScan::CONFIG_PROPERTIES, AppScan::ROUTES, AppScan::EXCEPTION_HANDLERS, AppScan::CONSTRAINTS, AppScan::HANDLERS, AppScan::EVENT_LISTENERS, - AppScan::MESSAGE_LISTENERS, AppScan::SCHEDULED, AppScan::SECURITY_METHODS, AppScan::TRANSACTIONAL, + AppScan::MESSAGE_LISTENERS, AppScan::SCHEDULED, AppScan::ORCHESTRATION, AppScan::SECURITY_METHODS, AppScan::TRANSACTIONAL, AppScan::PROXY_MAP, AppScan::PROXY_PLAN, ] as $basename) { expect($source)->toContain("'{$basename}'"); diff --git a/packages/orchestration/src/Manifest/OrchestrationManifest.php b/packages/orchestration/src/Manifest/OrchestrationManifest.php new file mode 100644 index 00000000..d3fcec6a --- /dev/null +++ b/packages/orchestration/src/Manifest/OrchestrationManifest.php @@ -0,0 +1,110 @@ +> $sagas + * @param list> $tccs + * @param list> $workflows + * @param list> $schedules + */ + public function __construct( + private readonly array $sagas = [], + private readonly array $tccs = [], + private readonly array $workflows = [], + private readonly array $schedules = [], + ) {} + + public static function empty(): self + { + return new self; + } + + /** + * @param list $sagas + * @param list $tccs + * @param list $workflows + * @param list $schedules + */ + public static function fromDefinitions(array $sagas, array $tccs, array $workflows, array $schedules): self + { + return new self( + array_map(static fn (SagaDefinition $saga): array => $saga->toArray(), $sagas), + array_map(static fn (TccDefinition $tcc): array => $tcc->toArray(), $tccs), + array_map(static fn (WorkflowDefinition $workflow): array => $workflow->toArray(), $workflows), + array_map(static fn (ScheduledExecution $schedule): array => $schedule->toArray(), $schedules), + ); + } + + /** @return list */ + public function sagas(): array + { + return array_map(static fn (array $row): SagaDefinition => SagaDefinition::fromArray($row), $this->sagas); + } + + /** @return list */ + public function tccs(): array + { + return array_map(static fn (array $row): TccDefinition => TccDefinition::fromArray($row), $this->tccs); + } + + /** @return list */ + public function workflows(): array + { + return array_map(static fn (array $row): WorkflowDefinition => WorkflowDefinition::fromArray($row), $this->workflows); + } + + /** @return list */ + public function schedules(): array + { + return array_map(static fn (array $row): ScheduledExecution => ScheduledExecution::fromArray($row), $this->schedules); + } + + public function isEmpty(): bool + { + return $this->sagas === [] && $this->tccs === [] && $this->workflows === [] && $this->schedules === []; + } + + /** @return array{sagas: list>, tccs: list>, workflows: list>, schedules: list>} */ + public function toArray(): array + { + return ['sagas' => $this->sagas, 'tccs' => $this->tccs, 'workflows' => $this->workflows, 'schedules' => $this->schedules]; + } + + /** @param array $data */ + public static function fromArray(array $data): self + { + return new self(Rows::rows($data, 'sagas'), Rows::rows($data, 'tccs'), Rows::rows($data, 'workflows'), Rows::rows($data, 'schedules')); + } + + public static function load(string $path): self + { + if (! is_file($path)) { + throw new ConfigurationException("Orchestration manifest not found at {$path}. Run php artisan firefly:cache."); + } + + /** @var mixed $data */ + $data = require $path; + if (! is_array($data)) { + throw new ConfigurationException("Orchestration manifest at {$path} did not return an array."); + } + + return self::fromArray($data); + } +} diff --git a/packages/orchestration/src/Manifest/OrchestrationManifestCompiler.php b/packages/orchestration/src/Manifest/OrchestrationManifestCompiler.php new file mode 100644 index 00000000..23ebacc2 --- /dev/null +++ b/packages/orchestration/src/Manifest/OrchestrationManifestCompiler.php @@ -0,0 +1,30 @@ +toArray(), true) + .";\n"; + } + + public function write(OrchestrationManifest $manifest, string $path): void + { + $dir = dirname($path); + if (! is_dir($dir) && ! mkdir($dir, 0o775, true) && ! is_dir($dir)) { + throw new ConfigurationException("Could not create manifest directory {$dir}."); + } + + if (file_put_contents($path, $this->compile($manifest)) === false) { + throw new ConfigurationException("Could not write the orchestration manifest to {$path}."); + } + } +} diff --git a/packages/orchestration/src/Manifest/ScheduledExecution.php b/packages/orchestration/src/Manifest/ScheduledExecution.php new file mode 100644 index 00000000..6b6f777d --- /dev/null +++ b/packages/orchestration/src/Manifest/ScheduledExecution.php @@ -0,0 +1,58 @@ + $input + */ + public function __construct( + public ExecutionPattern $pattern, + public string $name, + public ?string $cron = null, + public ?string $zone = null, + public ?string $fixedRate = null, + public ?string $fixedDelay = null, + public array $input = [], + public string $description = '', + ) {} + + public function lockName(int $index): string + { + return "firefly:orchestration:scheduled:{$this->pattern->value}:{$this->name}:{$index}"; + } + + /** @return array */ + public function toArray(): array + { + return [ + 'pattern' => $this->pattern->value, 'name' => $this->name, 'cron' => $this->cron, 'zone' => $this->zone, + 'fixedRate' => $this->fixedRate, 'fixedDelay' => $this->fixedDelay, 'input' => $this->input, 'description' => $this->description, + ]; + } + + /** @param array $row */ + public static function fromArray(array $row): self + { + return new self( + ExecutionPattern::tryFrom(Rows::string($row, 'pattern')) ?? ExecutionPattern::Saga, + Rows::string($row, 'name'), + Rows::nullableString($row, 'cron'), + Rows::nullableString($row, 'zone'), + Rows::nullableString($row, 'fixedRate'), + Rows::nullableString($row, 'fixedDelay'), + Rows::map($row, 'input'), + Rows::string($row, 'description'), + ); + } +} diff --git a/packages/orchestration/src/Scanner/OrchestrationScanner.php b/packages/orchestration/src/Scanner/OrchestrationScanner.php new file mode 100644 index 00000000..336cd75d --- /dev/null +++ b/packages/orchestration/src/Scanner/OrchestrationScanner.php @@ -0,0 +1,678 @@ +conditions = $conditions ?? new ConditionEvaluator; + } + + /** + * @param array $psr4 namespace prefix => absolute directory + */ + public function scan(array $psr4): OrchestrationManifest + { + $classes = AppScan::classes($psr4); + sort($classes); + + /** @var array, 1: Saga}> $sagas */ + $sagas = []; + /** @var array, 1: ReflectionMethod, 2: ExternalSagaStep}>> $external */ + $external = []; + /** @var array, 1: ReflectionMethod}>> $compensations */ + $compensations = []; + /** @var array, 1: Tcc}> $tccs */ + $tccs = []; + /** @var array, 1: TccParticipant}>> $participants */ + $participants = []; + /** @var array $workflows */ + $workflows = []; + /** @var array> $schedules */ + $schedules = [ExecutionPattern::Saga->value => [], ExecutionPattern::Tcc->value => [], ExecutionPattern::Workflow->value => []]; + + foreach ($classes as $class) { + $reflection = new ReflectionClass($class); + if ($reflection->isAbstract() || $reflection->isInterface() || $reflection->isTrait() || $reflection->isEnum()) { + continue; + } + + foreach ($reflection->getMethods(ReflectionMethod::IS_PUBLIC) as $method) { + foreach ($method->getAttributes(ExternalSagaStep::class) as $attribute) { + $step = $attribute->newInstance(); + $external[$step->saga][] = [$reflection, $method, $step]; + } + foreach ($method->getAttributes(CompensationSagaStep::class) as $attribute) { + $compensation = $attribute->newInstance(); + if (isset($compensations[$compensation->saga][$compensation->forStepId])) { + throw new TopologyValidationException("Saga [{$compensation->saga}] step [{$compensation->forStepId}] has two #[CompensationSagaStep] methods; a step has one compensation."); + } + $compensations[$compensation->saga][$compensation->forStepId] = [$reflection, $method]; + } + } + + $saga = self::classAttribute($reflection, Saga::class); + if ($saga !== null) { + if (isset($sagas[$saga->name])) { + throw new TopologyValidationException("Saga [{$saga->name}] is declared by {$sagas[$saga->name][0]->getName()} and by {$class}."); + } + $sagas[$saga->name] = [$reflection, $saga]; + foreach ($reflection->getAttributes(ScheduledSaga::class) as $attribute) { + $schedule = $attribute->newInstance(); + $this->addSchedule($schedules, ExecutionPattern::Saga, $saga->name, $class, $schedule->cron, $schedule->zone, $schedule->enabled, $schedule->fixedDelay, $schedule->fixedRate, $schedule->initialDelay, $schedule->input, $schedule->description); + } + } + + $tcc = self::classAttribute($reflection, Tcc::class); + if ($tcc !== null) { + if (isset($tccs[$tcc->name])) { + throw new TopologyValidationException("TCC [{$tcc->name}] is declared by {$tccs[$tcc->name][0]->getName()} and by {$class}."); + } + $tccs[$tcc->name] = [$reflection, $tcc]; + foreach ($reflection->getAttributes(ScheduledTcc::class) as $attribute) { + $schedule = $attribute->newInstance(); + $this->addSchedule($schedules, ExecutionPattern::Tcc, $tcc->name, $class, $schedule->cron, $schedule->zone, $schedule->enabled, $schedule->fixedDelay, $schedule->fixedRate, $schedule->initialDelay, $schedule->input, $schedule->description); + } + } + + $participant = self::classAttribute($reflection, TccParticipant::class); + if ($participant !== null) { + $participants[$participant->tcc][] = [$reflection, $participant]; + } + + $workflow = self::classAttribute($reflection, Workflow::class); + if ($workflow !== null) { + $definition = $this->workflow($reflection, $workflow); + if (isset($workflows[$definition->id])) { + throw new TopologyValidationException("Workflow [{$definition->id}] is declared twice ({$class})."); + } + $workflows[$definition->id] = $definition; + foreach ($reflection->getAttributes(ScheduledWorkflow::class) as $attribute) { + $schedule = $attribute->newInstance(); + $this->addSchedule($schedules, ExecutionPattern::Workflow, $definition->id, $class, $schedule->cron, $schedule->zone, $schedule->enabled, $schedule->fixedDelay, $schedule->fixedRate, $schedule->initialDelay, $schedule->input, $schedule->description); + } + } + } + + foreach ($external as $sagaName => $steps) { + if (! isset($sagas[$sagaName])) { + throw new TopologyValidationException("#[ExternalSagaStep] {$steps[0][0]->getName()}::{$steps[0][1]->getName()}() joins saga [{$sagaName}], but no #[Saga(name: '{$sagaName}')] class exists."); + } + } + foreach ($compensations as $sagaName => $byStep) { + if (! isset($sagas[$sagaName])) { + throw new TopologyValidationException("#[CompensationSagaStep] for saga [{$sagaName}] names no #[Saga(name: '{$sagaName}')] class."); + } + } + foreach ($participants as $tccName => $members) { + if (! isset($tccs[$tccName])) { + throw new TopologyValidationException("#[TccParticipant] {$members[0][0]->getName()} joins TCC [{$tccName}], but no #[Tcc(name: '{$tccName}')] class exists."); + } + } + + $sagaDefinitions = []; + foreach ($sagas as $name => [$reflection, $saga]) { + $sagaDefinitions[] = $this->saga($reflection, $saga, $external[$name] ?? [], $compensations[$name] ?? []); + } + usort($sagaDefinitions, static fn (SagaDefinition $a, SagaDefinition $b): int => $a->name <=> $b->name); + + $tccDefinitions = []; + foreach ($tccs as $name => [$reflection, $tcc]) { + $tccDefinitions[] = $this->tcc($reflection, $tcc, $participants[$name] ?? []); + } + usort($tccDefinitions, static fn (TccDefinition $a, TccDefinition $b): int => $a->name <=> $b->name); + + $workflowDefinitions = array_values($workflows); + usort($workflowDefinitions, static fn (WorkflowDefinition $a, WorkflowDefinition $b): int => $a->id <=> $b->id); + + $ordered = []; + foreach ($schedules as $group) { + usort($group, static fn (ScheduledExecution $a, ScheduledExecution $b): int => $a->name <=> $b->name); + $ordered = [...$ordered, ...$group]; + } + + return OrchestrationManifest::fromDefinitions($sagaDefinitions, $tccDefinitions, $workflowDefinitions, $ordered); + } + + /** + * @param ReflectionClass $class + * @param list, 1: ReflectionMethod, 2: ExternalSagaStep}> $external + * @param array, 1: ReflectionMethod}> $compensations + */ + private function saga(ReflectionClass $class, Saga $saga, array $external, array $compensations): SagaDefinition + { + /** @var array, 1: ReflectionMethod, 2: SagaStep|ExternalSagaStep}> $declared */ + $declared = []; + foreach ($class->getMethods(ReflectionMethod::IS_PUBLIC) as $method) { + foreach ($method->getAttributes(SagaStep::class) as $attribute) { + $step = $attribute->newInstance(); + if (isset($declared[$step->id])) { + throw new TopologyValidationException("Saga [{$saga->name}] declares step [{$step->id}] twice."); + } + $declared[$step->id] = [$class, $method, $step]; + } + } + foreach ($external as [$owner, $method, $step]) { + if (isset($declared[$step->id])) { + throw new TopologyValidationException("Saga [{$saga->name}] declares step [{$step->id}] twice (#[ExternalSagaStep] on {$owner->getName()})."); + } + $declared[$step->id] = [$owner, $method, $step]; + } + + $steps = []; + foreach ($declared as $id => [$owner, $method, $step]) { + $id = (string) $id; + $where = "Saga [{$saga->name}] step [{$id}]"; + + $compensation = null; + if ($step->compensate !== '') { + $compensation = $this->invocation($owner, $this->namedMethod($owner, $step->compensate, "{$where} names compensate: '{$step->compensate}', but"), "{$where} compensation"); + } + if (isset($compensations[$id])) { + if ($compensation !== null) { + throw new TopologyValidationException("{$where} declares compensate: and a #[CompensationSagaStep]; a step has one compensation."); + } + $compensation = $this->invocation($compensations[$id][0], $compensations[$id][1], "{$where} compensation"); + } + + $event = self::methodAttribute($method, StepEvent::class); + $steps[$id] = new SagaStepDefinition( + id: $id, + invocation: $this->invocation($owner, $method, $where), + dependsOn: $step->dependsOn, + options: StepOptions::of($step->retry, $step->backoffMs, $step->timeoutMs, $step->jitter, $step->jitterFactor, $step->idempotencyKey), + compensation: $compensation, + compensationRetry: $step->compensationRetry, + compensationBackoffMs: $step->compensationBackoffMs, + compensationTimeoutMs: $step->compensationTimeoutMs, + compensationCritical: $step->compensationCritical, + cpuBound: $step->cpuBound, + event: $event === null ? null : new StepEventDefinition($event->topic, $event->type, $event->key), + ); + } + + foreach (array_keys($compensations) as $stepId) { + if (! isset($steps[(string) $stepId])) { + throw new TopologyValidationException("#[CompensationSagaStep] for saga [{$saga->name}] names step [{$stepId}], which the saga does not declare."); + } + } + + return new SagaDefinition($saga->name, $steps, $saga->layerConcurrency, $saga->triggerEventType, $this->callbacks($class, OnSagaComplete::class), $this->callbacks($class, OnSagaError::class)); + } + + /** + * @param ReflectionClass $class + * @param list, 1: TccParticipant}> $participants + */ + private function tcc(ReflectionClass $class, Tcc $tcc, array $participants): TccDefinition + { + $definitions = []; + foreach ($participants as [$participantClass, $participant]) { + $where = "TCC [{$tcc->name}] participant [{$participant->id}] ({$participantClass->getName()})"; + if (isset($definitions[$participant->id])) { + throw new TopologyValidationException("{$where} is declared twice."); + } + + /** @var array{try: list, confirm: list, cancel: list} $phases */ + $phases = ['try' => [], 'confirm' => [], 'cancel' => []]; + foreach ($participantClass->getMethods(ReflectionMethod::IS_PUBLIC) as $method) { + foreach ($method->getAttributes(TryMethod::class) as $attribute) { + $phases['try'][] = [$method, $attribute->newInstance()]; + } + foreach ($method->getAttributes(ConfirmMethod::class) as $attribute) { + $phases['confirm'][] = [$method, $attribute->newInstance()]; + } + foreach ($method->getAttributes(CancelMethod::class) as $attribute) { + $phases['cancel'][] = [$method, $attribute->newInstance()]; + } + } + if (count($phases['try']) !== 1 || count($phases['confirm']) !== 1 || count($phases['cancel']) !== 1) { + throw new TopologyValidationException("{$where} needs exactly one #[TryMethod], #[ConfirmMethod] and #[CancelMethod]; it declares ".count($phases['try']).' / '.count($phases['confirm']).' / '.count($phases['cancel']).'.'); + } + + [$tryMethod, $try] = $phases['try'][0]; + [$confirmMethod, $confirm] = $phases['confirm'][0]; + [$cancelMethod, $cancel] = $phases['cancel'][0]; + $event = self::classAttribute($participantClass, TccEvent::class); + + $definitions[$participant->id] = new TccParticipantDefinition( + id: $participant->id, + order: $participant->order, + try: new TccPhaseDefinition($this->invocation($participantClass, $tryMethod, "{$where} try"), $try->timeoutMs, $try->retry, $try->backoffMs), + confirm: new TccPhaseDefinition($this->invocation($participantClass, $confirmMethod, "{$where} confirm"), $confirm->timeoutMs, $confirm->retry, $confirm->backoffMs), + cancel: new TccPhaseDefinition($this->invocation($participantClass, $cancelMethod, "{$where} cancel"), $cancel->timeoutMs, $cancel->retry, $cancel->backoffMs), + timeoutMs: $participant->timeoutMs, + optional: $participant->optional, + jitter: $participant->jitter, + jitterFactor: $participant->jitterFactor, + event: $event === null ? null : new StepEventDefinition($event->topic, $event->eventType, $event->key), + ); + } + + return new TccDefinition( + $tcc->name, $definitions, $tcc->timeoutMs, $tcc->retryEnabled, $tcc->maxRetries, $tcc->backoffMs, $tcc->triggerEventType, + $this->callbacks($class, OnTccComplete::class), $this->callbacks($class, OnTccError::class), + ); + } + + /** + * @param ReflectionClass $class + */ + private function workflow(ReflectionClass $class, Workflow $workflow): WorkflowDefinition + { + $id = $workflow->id !== '' ? $workflow->id : $class->getShortName(); + + $compensations = []; + foreach ($class->getMethods(ReflectionMethod::IS_PUBLIC) as $method) { + foreach ($method->getAttributes(CompensationStep::class) as $attribute) { + $compensates = $attribute->newInstance()->compensates; + if (isset($compensations[$compensates])) { + throw new TopologyValidationException("Workflow [{$id}] step [{$compensates}] has two #[CompensationStep] methods."); + } + $compensations[$compensates] = $method; + } + } + + $steps = []; + $queries = []; + foreach ($class->getMethods(ReflectionMethod::IS_PUBLIC) as $method) { + foreach ($method->getAttributes(WorkflowStep::class) as $attribute) { + $step = $attribute->newInstance(); + $stepId = $step->id !== '' ? $step->id : $method->getName(); + $where = "Workflow [{$id}] step [{$stepId}]"; + if (isset($steps[$stepId])) { + throw new TopologyValidationException("{$where} is declared twice."); + } + if ($step->condition !== '') { + $this->conditions->lint($step->condition, $where); + } + + $compensation = null; + if ($step->compensationMethod !== '') { + $compensation = $this->invocation($class, $this->namedMethod($class, $step->compensationMethod, "{$where} names compensationMethod: '{$step->compensationMethod}', but"), "{$where} compensation"); + } + if (isset($compensations[$stepId])) { + if ($compensation !== null) { + throw new TopologyValidationException("{$where} declares compensationMethod: and a #[CompensationStep]; a step has one compensation."); + } + $compensation = $this->invocation($class, $compensations[$stepId], "{$where} compensation"); + } + + $child = self::methodAttribute($method, ChildWorkflow::class); + $steps[$stepId] = new WorkflowStepDefinition( + id: $stepId, + invocation: $this->invocation($class, $method, $where), + dependsOn: $step->dependsOn, + name: $step->name, + description: $step->description, + outputEventType: $step->outputEventType, + options: StepOptions::of( + $step->maxRetries >= 0 ? $step->maxRetries : $workflow->maxRetries, + $step->retryDelayMs >= 0 ? $step->retryDelayMs : $workflow->retryDelayMs, + $step->timeoutMs, + ), + condition: $step->condition, + async: $step->async, + compensatable: $step->compensatable || $compensation !== null, + compensation: $compensation, + wait: $this->wait($method, $stepId, $where), + child: $child === null ? null : new ChildWorkflowDefinition($child->workflowId, $child->waitForCompletion, $child->timeoutMs), + ); + } + + foreach ($method->getAttributes(WorkflowQuery::class) as $attribute) { + $query = $attribute->newInstance(); + $first = $method->getParameters()[0] ?? null; + $type = $first?->getType(); + $queries[$query->name] = new WorkflowQueryDefinition( + $query->name, + new MethodInvocation($class->getName(), $method->getName()), + $type instanceof ReflectionNamedType && $type->getName() === ExecutionContext::class, + ); + } + } + + foreach (array_keys($compensations) as $stepId) { + if (! isset($steps[(string) $stepId])) { + throw new TopologyValidationException("#[CompensationStep(compensates: '{$stepId}')] in workflow [{$id}] names a step it does not declare."); + } + } + + return new WorkflowDefinition( + id: $id, + steps: $steps, + name: $workflow->name, + description: $workflow->description, + version: $workflow->version, + triggerMode: $workflow->triggerMode, + triggerEventType: $workflow->triggerEventType, + timeoutMs: $workflow->timeoutMs, + maxRetries: $workflow->maxRetries, + retryDelayMs: $workflow->retryDelayMs, + publishEvents: $workflow->publishEvents, + layerConcurrency: $workflow->layerConcurrency, + queries: $queries, + onStepComplete: $this->callbacks($class, OnStepComplete::class), + onComplete: $this->callbacks($class, OnWorkflowComplete::class), + onError: $this->callbacks($class, OnWorkflowError::class), + ); + } + + private function wait(ReflectionMethod $method, string $stepId, string $where): ?WaitDefinition + { + $found = []; + foreach ([WaitForSignal::class, WaitForTimer::class, WaitForAll::class, WaitForAny::class] as $type) { + foreach ($method->getAttributes($type) as $attribute) { + $found[] = $attribute->newInstance(); + } + } + if ($found === []) { + return null; + } + if (count($found) > 1) { + throw new TopologyValidationException("{$where} declares more than one wait; combine them with #[WaitForAll] or #[WaitForAny]."); + } + + $wait = $found[0]; + if ($wait instanceof WaitForSignal) { + return WaitDefinition::signal($wait->name, $wait->timeoutMs); + } + if ($wait instanceof WaitForTimer) { + return WaitDefinition::timer($wait->delayMs, $wait->timerId !== '' ? $wait->timerId : "{$stepId}:timer:0"); + } + + // Only #[WaitForAll] and #[WaitForAny] are left: both carry signals and timers, and differ in the kind. + $signals = array_map(static fn (WaitForSignal $signal): SignalWait => new SignalWait($signal->name, $signal->timeoutMs), $wait->signals); + $timers = []; + foreach ($wait->timers as $index => $timer) { + $timers[] = new TimerWait($timer->timerId !== '' ? $timer->timerId : "{$stepId}:timer:{$index}", $timer->delayMs); + } + + return $wait instanceof WaitForAll ? WaitDefinition::all($signals, $timers) : WaitDefinition::any($signals, $timers); + } + + /** + * @param ReflectionClass $class + * @param class-string $attributeClass + * @return list + */ + private function callbacks(ReflectionClass $class, string $attributeClass): array + { + $definitions = []; + foreach ($class->getMethods(ReflectionMethod::IS_PUBLIC) as $method) { + foreach ($method->getAttributes($attributeClass) as $attribute) { + $callback = $attribute->newInstance(); + $parameters = $this->callbackParameters($method); + $owner = $class->getName(); + $name = $method->getName(); + + $definitions[] = match (true) { + $callback instanceof OnSagaError, $callback instanceof OnTccError => new CallbackDefinition($owner, $name, $parameters, $callback->async, $callback->priority, $callback->errorTypes, $callback->suppressError), + $callback instanceof OnWorkflowError => new CallbackDefinition($owner, $name, $parameters, $callback->async, $callback->priority, $callback->errorTypes, $callback->suppressError, $callback->stepIds), + $callback instanceof OnStepComplete => new CallbackDefinition($owner, $name, $parameters, $callback->async, $callback->priority, stepIds: $callback->stepIds), + $callback instanceof OnSagaComplete, $callback instanceof OnTccComplete, $callback instanceof OnWorkflowComplete => new CallbackDefinition($owner, $name, $parameters, $callback->async, $callback->priority), + default => throw new LogicException("Unsupported callback attribute {$attributeClass}."), + }; + } + } + + return $definitions; + } + + /** @return list */ + private function callbackParameters(ReflectionMethod $method): array + { + $kinds = []; + foreach ($method->getParameters() as $parameter) { + $type = $parameter->getType(); + $name = $type instanceof ReflectionNamedType ? $type->getName() : ''; + $kinds[] = match (true) { + $name === ExecutionContext::class => CallbackDefinition::CONTEXT, + $name !== '' && is_a($name, Throwable::class, true) => CallbackDefinition::ERROR, + default => CallbackDefinition::RESULT, + }; + } + + return $kinds; + } + + /** + * @param ReflectionClass $class + */ + private function invocation(ReflectionClass $class, ReflectionMethod $method, string $where): MethodInvocation + { + return new MethodInvocation($class->getName(), $method->getName(), $this->bindings($class, $method, $where), self::methodAttribute($method, SetVariable::class)?->name); + } + + /** + * @param ReflectionClass $class + * @return list + */ + private function bindings(ReflectionClass $class, ReflectionMethod $method, string $where): array + { + $bindings = []; + $implicit = false; + + foreach ($method->getParameters() as $parameter) { + $name = $parameter->getName(); + $required = $parameter->getAttributes(Required::class) !== []; + $binding = $this->annotated($parameter, $name, $required); + + if ($binding === null) { + $type = $parameter->getType(); + if ($type instanceof ReflectionNamedType && $type->getName() === ExecutionContext::class) { + $binding = new ParameterBinding($name, ParameterBinding::CONTEXT); + } elseif (! $implicit) { + $binding = new ParameterBinding($name, ParameterBinding::IMPLICIT, required: $required); + $implicit = true; + } elseif ($parameter->isDefaultValueAvailable()) { + $binding = new ParameterBinding($name, ParameterBinding::DEFAULT); + } else { + throw new TopologyValidationException("{$where}: parameter \${$name} of {$class->getName()}::{$method->getName()}() cannot be bound — only one parameter receives the input implicitly. Annotate it with #[Input], #[FromStep], #[FromTry], #[FromCompensationResult], #[CompensationError], #[Header], #[Headers], #[Variable], #[Variables] or #[CorrelationId], or give it a default."); + } + } + + $bindings[] = $binding; + } + + return $bindings; + } + + private function annotated(ReflectionParameter $parameter, string $name, bool $required): ?ParameterBinding + { + foreach ($parameter->getAttributes() as $attribute) { + if (! class_exists($attribute->getName())) { + continue; + } + + $instance = $attribute->newInstance(); + $binding = match (true) { + $instance instanceof Input => new ParameterBinding($name, $instance->key === '' ? ParameterBinding::INPUT : ParameterBinding::INPUT_KEY, $instance->key, $required), + $instance instanceof FromStep => new ParameterBinding($name, ParameterBinding::FROM_STEP, $instance->stepId, $required), + $instance instanceof FromCompensationResult => new ParameterBinding($name, ParameterBinding::FROM_COMPENSATION_RESULT, $instance->stepId, $required), + $instance instanceof CompensationError => new ParameterBinding($name, ParameterBinding::COMPENSATION_ERROR, $instance->stepId, $required), + $instance instanceof FromTry => new ParameterBinding($name, ParameterBinding::FROM_TRY, $instance->participantId, $required), + $instance instanceof Header => new ParameterBinding($name, ParameterBinding::HEADER, $instance->name, $required), + $instance instanceof Headers => new ParameterBinding($name, ParameterBinding::HEADERS, '', $required), + $instance instanceof Variable => new ParameterBinding($name, ParameterBinding::VARIABLE, $instance->name, $required), + $instance instanceof Variables => new ParameterBinding($name, ParameterBinding::VARIABLES, '', $required), + $instance instanceof CorrelationId => new ParameterBinding($name, ParameterBinding::CORRELATION_ID, '', $required), + default => null, + }; + + if ($binding !== null) { + return $binding; + } + } + + return null; + } + + /** + * @param ReflectionClass $class + */ + private function namedMethod(ReflectionClass $class, string $name, string $prefix): ReflectionMethod + { + if (! $class->hasMethod($name) || ! $class->getMethod($name)->isPublic()) { + throw new TopologyValidationException("{$prefix} {$class->getName()} has no public method {$name}()."); + } + + return $class->getMethod($name); + } + + /** + * @param array> $schedules + */ + private function addSchedule(array &$schedules, ExecutionPattern $pattern, string $name, string $class, string $cron, string $zone, bool $enabled, int $fixedDelay, int $fixedRate, int $initialDelay, string $input, string $description): void + { + if (! $enabled) { + return; + } + + $where = '#[Scheduled'.ucfirst(strtolower($pattern->value)).'] on '.$class; + if ($initialDelay !== 0) { + throw new TopologyValidationException("{$where} sets initialDelay: {$initialDelay}; a scheduled orchestration cannot honour an initial delay. Start it from a #[Scheduled(initialDelay: …)] method instead."); + } + if (count(array_filter([$cron !== '', $fixedRate > 0, $fixedDelay > 0])) !== 1) { + throw new TopologyValidationException("{$where} needs exactly one of cron, fixedRate or fixedDelay (milliseconds, greater than zero)."); + } + + try { + $decoded = json_decode($input, true, 512, JSON_THROW_ON_ERROR); + } catch (JsonException $error) { + throw new TopologyValidationException("{$where} input is not JSON: {$error->getMessage()}", $error); + } + if (! is_array($decoded)) { + throw new TopologyValidationException("{$where} input must be a JSON object."); + } + $values = []; + foreach ($decoded as $key => $value) { + $values[(string) $key] = $value; + } + + $schedules[$pattern->value][] = new ScheduledExecution( + $pattern, + $name, + $cron !== '' ? $cron : null, + $zone !== '' ? $zone : null, + $fixedRate > 0 ? "{$fixedRate}ms" : null, + $fixedDelay > 0 ? "{$fixedDelay}ms" : null, + $values, + $description, + ); + } + + /** + * @template T of object + * + * @param ReflectionClass $class + * @param class-string $attribute + * @return T|null + */ + private static function classAttribute(ReflectionClass $class, string $attribute): ?object + { + $found = $class->getAttributes($attribute); + + return $found === [] ? null : $found[0]->newInstance(); + } + + /** + * @template T of object + * + * @param class-string $attribute + * @return T|null + */ + private static function methodAttribute(ReflectionMethod $method, string $attribute): ?object + { + $found = $method->getAttributes($attribute); + + return $found === [] ? null : $found[0]->newInstance(); + } +} diff --git a/packages/orchestration/tests/Manifest/OrchestrationManifestTest.php b/packages/orchestration/tests/Manifest/OrchestrationManifestTest.php new file mode 100644 index 00000000..a0813bbf --- /dev/null +++ b/packages/orchestration/tests/Manifest/OrchestrationManifestTest.php @@ -0,0 +1,26 @@ +scan(['Firefly\\Orchestration\\Tests\\ScannerFixtures\\Good\\' => dirname(__DIR__).'/ScannerFixtures/Good']); + $path = sys_get_temp_dir().'/firefly-orchestration-manifest-'.bin2hex(random_bytes(6)).'.php'; + + try { + (new OrchestrationManifestCompiler)->write($manifest, $path); + $loaded = OrchestrationManifest::load($path); + + expect((string) file_get_contents($path))->toStartWith("and($loaded->toArray())->toBe($manifest->toArray()) + ->and($loaded->sagas())->toEqual($manifest->sagas()) + ->and($loaded->workflows())->toEqual($manifest->workflows()) + ->and(OrchestrationManifest::empty()->isEmpty())->toBeTrue() + ->and($loaded->isEmpty())->toBeFalse(); + } finally { + @unlink($path); + } +}); diff --git a/packages/orchestration/tests/ReflectionFreeOrchestrationTest.php b/packages/orchestration/tests/ReflectionFreeOrchestrationTest.php index db34bf87..8f9d3703 100644 --- a/packages/orchestration/tests/ReflectionFreeOrchestrationTest.php +++ b/packages/orchestration/tests/ReflectionFreeOrchestrationTest.php @@ -24,8 +24,8 @@ function orchestrationReflectionHits(string $dir): array /** * Production loads the compiled OrchestrationManifest by require; reflection is allowed in exactly one file, - * the scanner that compiles it (Task 16 adds it and widens this list to that one name). + * the scanner that compiles it (at firefly:cache time, or as the uncached development fallback). */ it('confines reflection in firefly/orchestration to the scanner', function (): void { - expect(orchestrationReflectionHits(__DIR__.'/../src'))->toBe([]); + expect(orchestrationReflectionHits(__DIR__.'/../src'))->toBe(['OrchestrationScanner.php']); }); diff --git a/packages/orchestration/tests/Scanner/OrchestrationScannerTest.php b/packages/orchestration/tests/Scanner/OrchestrationScannerTest.php new file mode 100644 index 00000000..1e8f8c32 --- /dev/null +++ b/packages/orchestration/tests/Scanner/OrchestrationScannerTest.php @@ -0,0 +1,110 @@ +scan(['Firefly\\Orchestration\\Tests\\ScannerFixtures\\Good\\' => dirname(__DIR__).'/ScannerFixtures/Good']); +} + +it('compiles a saga: steps, external steps, compensations, parameter plans, events and callbacks', function (): void { + $saga = orchestrationScanGood()->sagas()[0]; + + expect($saga->name)->toBe('order') + ->and($saga->triggerEventType)->toBe('orders.placed') + ->and($saga->stepIds())->toBe(['reserve', 'charge', 'notify']) + ->and($saga->layers)->toBe([['reserve'], ['charge', 'notify']]) + ->and($saga->step('reserve')->options)->toEqual(new StepOptions(2, 5, 1000, false, 0.5, 'reserve-{input.orderId}')) + ->and($saga->step('reserve')->invocation)->toEqual(new MethodInvocation(OrderSaga::class, 'reserve', [ + new ParameterBinding('orderId', ParameterBinding::INPUT_KEY, 'orderId'), + new ParameterBinding('ctx', ParameterBinding::CONTEXT), + ])) + ->and($saga->step('reserve')->compensation)->toEqual(new MethodInvocation(OrderSaga::class, 'release', [ + new ParameterBinding('hold', ParameterBinding::IMPLICIT), + new ParameterBinding('input', ParameterBinding::INPUT), + ])) + ->and($saga->step('charge')->invocation)->toEqual(new MethodInvocation(OrderSaga::class, 'charge', [ + new ParameterBinding('hold', ParameterBinding::FROM_STEP, 'reserve'), + new ParameterBinding('tenant', ParameterBinding::HEADER, 'tenant'), + new ParameterBinding('limit', ParameterBinding::VARIABLE, 'limit', required: true), + ], 'receipt')) + ->and($saga->step('charge')->compensationCritical)->toBeTrue() + ->and($saga->step('charge')->event?->toArray())->toBe(['topic' => 'payments', 'type' => 'payment.captured', 'key' => 'orderId']) + ->and($saga->step('notify')->invocation)->toEqual(new MethodInvocation(InventoryService::class, 'notify', [new ParameterBinding('id', ParameterBinding::CORRELATION_ID)])) + ->and($saga->step('notify')->compensation)->toEqual(new MethodInvocation(InventoryService::class, 'unnotify')) + ->and($saga->onComplete)->toEqual([new CallbackDefinition(OrderSaga::class, 'done', [CallbackDefinition::RESULT, CallbackDefinition::CONTEXT], priority: 5)]) + ->and($saga->onError)->toEqual([new CallbackDefinition(OrderSaga::class, 'failed', [CallbackDefinition::ERROR], errorTypes: [RuntimeException::class])]); +}); + +it('compiles a TCC from its participant classes, ordered, with per-phase options', function (): void { + $tcc = orchestrationScanGood()->tccs()[0]; + + expect($tcc->name)->toBe('transfer') + ->and($tcc->participantIds())->toBe(['debit', 'credit']) + ->and($tcc->participant('debit')->try->timeoutMs)->toBe(500) + ->and($tcc->participant('debit')->confirm->invocation)->toEqual(new MethodInvocation(DebitParticipant::class, 'commit', [new ParameterBinding('hold', ParameterBinding::FROM_TRY)])) + ->and($tcc->participant('debit')->cancel->invocation)->toEqual(new MethodInvocation(DebitParticipant::class, 'release', [new ParameterBinding('hold', ParameterBinding::FROM_TRY, 'debit')])) + ->and($tcc->participant('debit')->cancel->retry)->toBe(2) + ->and($tcc->participant('debit')->event?->type)->toBe('ledger.debited') + ->and($tcc->participant('credit')->optional)->toBeTrue() + ->and($tcc->maxRetries)->toBe(1) + ->and($tcc->onComplete[0]->parameters)->toBe([CallbackDefinition::RESULT]); +}); + +it('compiles a workflow: inherited retries, waits, children, compensations, conditions, queries and callbacks', function (): void { + $workflow = orchestrationScanGood()->workflows()[0]; + + expect($workflow->id)->toBe('approval') + ->and($workflow->triggerMode)->toBe(TriggerMode::Async) + ->and($workflow->layers)->toBe([['submit'], ['decide', 'audit']]) + ->and($workflow->step('submit')->options)->toEqual(new StepOptions(1, 10)) + ->and($workflow->step('submit')->compensation)->toEqual(new MethodInvocation(ApprovalWorkflow::class, 'withdraw', [new ParameterBinding('result', ParameterBinding::IMPLICIT)])) + ->and($workflow->step('decide')->options)->toEqual(new StepOptions(0, 10)) + ->and($workflow->step('decide')->wait?->kind)->toBe(WaitDefinition::ANY) + ->and($workflow->step('decide')->wait?->signals[0]->timeoutMs)->toBe(86400000) + ->and($workflow->step('decide')->wait?->timers[0]->id)->toBe('escalate') + ->and($workflow->step('decide')->condition)->toBe('input["amount"] > 100') + ->and($workflow->step('audit')->child?->workflowId)->toBe('audit-trail') + ->and($workflow->query('status')->contextFirst)->toBeTrue() + ->and($workflow->onError[0]->stepIds)->toBe(['decide']) + ->and($workflow->onError[0]->suppressError)->toBeTrue(); +}); + +it('compiles the schedules', function (): void { + $schedules = orchestrationScanGood()->schedules(); + + expect(array_map(static fn ($s): array => $s->toArray(), $schedules))->toBe([ + ['pattern' => 'SAGA', 'name' => 'order', 'cron' => '0 3 * * *', 'zone' => null, 'fixedRate' => null, 'fixedDelay' => null, 'input' => ['source' => 'nightly'], 'description' => 'nightly reconciliation'], + ['pattern' => 'WORKFLOW', 'name' => 'approval', 'cron' => null, 'zone' => null, 'fixedRate' => '60000ms', 'fixedDelay' => null, 'input' => [], 'description' => ''], + ]) + ->and($schedules[0]->lockName(0))->toBe('firefly:orchestration:scheduled:SAGA:order:0') + ->and($schedules[0]->pattern)->toBe(ExecutionPattern::Saga); +}); + +it('refuses every invalid definition at scan time, naming it', function (string $case, string $message): void { + expect(fn () => (new OrchestrationScanner)->scan(["Firefly\\Orchestration\\Tests\\ScannerFixtures\\Bad\\{$case}\\" => dirname(__DIR__)."/ScannerFixtures/Bad/{$case}"])) + ->toThrow(TopologyValidationException::class, $message); +})->with([ + ['Cycle', 'Saga [cycle]: the steps form a cycle'], + ['MissingCompensation', "names compensate: 'nope'"], + ['TwoImplicit', 'cannot be bound'], + ['HalfParticipant', 'needs exactly one #[TryMethod], #[ConfirmMethod] and #[CancelMethod]'], + ['BadCondition', 'does not parse'], + ['InitialDelay', 'initialDelay'], + ['OrphanExternal', "no #[Saga(name: 'ghost')]"], +]); diff --git a/packages/orchestration/tests/ScannerFixtures/Bad/BadCondition/BadConditionWorkflow.php b/packages/orchestration/tests/ScannerFixtures/Bad/BadCondition/BadConditionWorkflow.php new file mode 100644 index 00000000..5f79cf60 --- /dev/null +++ b/packages/orchestration/tests/ScannerFixtures/Bad/BadCondition/BadConditionWorkflow.php @@ -0,0 +1,15 @@ + $input */ + #[WorkflowStep(id: 'submit', outputEventType: 'approval.submitted', compensatable: true, compensationMethod: 'withdraw')] + public function submit(array $input): string + { + return 'submitted'; + } + + public function withdraw(string $result): void {} + + #[WorkflowStep(id: 'decide', dependsOn: ['submit'], maxRetries: 0, condition: 'input["amount"] > 100')] + #[WaitForAny(signals: [new WaitForSignal('approved', 86400000)], timers: [new WaitForTimer(3600000, 'escalate')])] + public function decide(#[Variable('signals.approved')] mixed $approval): string + { + return 'decided'; + } + + /** @return array{subject: string} */ + #[WorkflowStep(id: 'audit', dependsOn: ['submit'])] + #[ChildWorkflow(workflowId: 'audit-trail', timeoutMs: 1000)] + public function audit(): array + { + return ['subject' => 'x']; + } + + #[WorkflowQuery('status')] + public function status(ExecutionContext $ctx, string $format = 'short'): string + { + return 'ok'; + } + + #[OnWorkflowError(stepIds: ['decide'], suppressError: true)] + public function onDecideError(Throwable $error, ExecutionContext $ctx): void {} +} diff --git a/packages/orchestration/tests/ScannerFixtures/Good/CreditParticipant.php b/packages/orchestration/tests/ScannerFixtures/Good/CreditParticipant.php new file mode 100644 index 00000000..12ec396f --- /dev/null +++ b/packages/orchestration/tests/ScannerFixtures/Good/CreditParticipant.php @@ -0,0 +1,26 @@ + $input */ + #[TryMethod(timeoutMs: 500)] + public function hold(array $input): string + { + return 'hold-1'; + } + + #[ConfirmMethod] + public function commit(#[FromTry] string $hold): void {} + + #[CancelMethod(retry: 2)] + public function release(#[FromTry('debit')] string $hold): void {} +} diff --git a/packages/orchestration/tests/ScannerFixtures/Good/InventoryService.php b/packages/orchestration/tests/ScannerFixtures/Good/InventoryService.php new file mode 100644 index 00000000..4e49f6f6 --- /dev/null +++ b/packages/orchestration/tests/ScannerFixtures/Good/InventoryService.php @@ -0,0 +1,21 @@ + "h-{$orderId}"]; + } + + /** + * @param array $hold + * @param array $input + */ + public function release(array $hold, #[Input] array $input): void {} + + /** @param array $hold */ + #[SagaStep(id: 'charge', dependsOn: ['reserve'], compensate: 'refund', compensationCritical: true)] + #[StepEvent(topic: 'payments', type: 'payment.captured', key: 'orderId')] + #[SetVariable('receipt')] + public function charge(#[FromStep('reserve')] array $hold, #[Header('tenant')] ?string $tenant, #[Required] #[Variable('limit')] mixed $limit = null): string + { + return 'paid'; + } + + public function refund(string $receipt): void {} + + #[OnSagaComplete(priority: 5)] + public function done(SagaResult $result, ExecutionContext $ctx): void {} + + #[OnSagaError(errorTypes: [RuntimeException::class])] + public function failed(Throwable $error): void {} +} diff --git a/packages/orchestration/tests/ScannerFixtures/Good/TransferTcc.php b/packages/orchestration/tests/ScannerFixtures/Good/TransferTcc.php new file mode 100644 index 00000000..d1e5b5b4 --- /dev/null +++ b/packages/orchestration/tests/ScannerFixtures/Good/TransferTcc.php @@ -0,0 +1,16 @@ +toBe(['artifacts' => 14, 'pairs' => 12]); + expect($figures)->toBe(['artifacts' => 15, 'pairs' => 13]); - // A STEP is not a pair: writeManifests() makes twelve scanner-then-compiler calls and writeProxies() is - // the thirteenth step, which is why the chapter's own body says "twelve `scan()`-then-`write()` calls ... + // A STEP is not a pair: writeManifests() makes thirteen scanner-then-compiler calls and writeProxies() is + // the fourteenth step, which is why the chapter's own body says "thirteen `scan()`-then-`write()` calls ... // plus one more step". Derived as pairs + 1 rather than typed, so adding a capability moves the pair // count, the step count and the sentences that quote either of them together. $steps = $figures['pairs'] + 1; From 4b077b4c24f3dfe52f42b7ea5b446f0039d406c7 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 21:32:56 -0700 Subject: [PATCH 33/58] feat(orchestration): auto-configure the engines, register and validate definitions at boot, and start executions from EDA triggers --- docs/modules.md | 4 +- .../firefly-orchestration-components.php | 421 ++++++++++++++++++ .../cache/firefly-orchestration-context.php | 343 ++++++++++++++ .../src/Boot/EventGatewayWiringPass.php | 57 +++ .../src/Boot/OrchestrationRegistryPass.php | 130 ++++++ .../src/OrchestrationAutoConfiguration.php | 418 +++++++++++++++++ .../src/OrchestrationWiringProvider.php | 51 ++- .../src/Validation/OrchestrationValidator.php | 75 ++++ .../src/Validation/ValidationIssue.php | 18 + .../tests/Capstone/BootValidationTest.php | 36 ++ .../tests/Capstone/CapstoneSagaTest.php | 48 ++ .../tests/Capstone/CapstoneTccTest.php | 27 ++ .../tests/Capstone/CapstoneWorkflowTest.php | 30 ++ .../tests/CapstoneFixtures/CheckoutHook.php | 21 + .../tests/CapstoneFixtures/CheckoutSaga.php | 69 +++ .../tests/CapstoneFixtures/CreditAccount.php | 43 ++ .../tests/CapstoneFixtures/DebitAccount.php | 40 ++ .../tests/CapstoneFixtures/Ledger.php | 20 + .../CapstoneFixtures/OnboardingWorkflow.php | 55 +++ .../tests/CapstoneFixtures/TransferTcc.php | 10 + .../UnknownChild/OrphanParent.php | 21 + .../orchestration/tests/PackageBootsTest.php | 59 ++- .../Support/OrchestrationCapstoneTestCase.php | 73 +++ .../Support/OrchestrationFixturesProvider.php | 42 ++ .../Validation/OrchestrationValidatorTest.php | 49 ++ 25 files changed, 2146 insertions(+), 14 deletions(-) create mode 100644 packages/orchestration/src/Boot/EventGatewayWiringPass.php create mode 100644 packages/orchestration/src/Boot/OrchestrationRegistryPass.php create mode 100644 packages/orchestration/src/OrchestrationAutoConfiguration.php create mode 100644 packages/orchestration/src/Validation/OrchestrationValidator.php create mode 100644 packages/orchestration/src/Validation/ValidationIssue.php create mode 100644 packages/orchestration/tests/Capstone/BootValidationTest.php create mode 100644 packages/orchestration/tests/Capstone/CapstoneSagaTest.php create mode 100644 packages/orchestration/tests/Capstone/CapstoneTccTest.php create mode 100644 packages/orchestration/tests/Capstone/CapstoneWorkflowTest.php create mode 100644 packages/orchestration/tests/CapstoneFixtures/CheckoutHook.php create mode 100644 packages/orchestration/tests/CapstoneFixtures/CheckoutSaga.php create mode 100644 packages/orchestration/tests/CapstoneFixtures/CreditAccount.php create mode 100644 packages/orchestration/tests/CapstoneFixtures/DebitAccount.php create mode 100644 packages/orchestration/tests/CapstoneFixtures/Ledger.php create mode 100644 packages/orchestration/tests/CapstoneFixtures/OnboardingWorkflow.php create mode 100644 packages/orchestration/tests/CapstoneFixtures/TransferTcc.php create mode 100644 packages/orchestration/tests/InvalidFixtures/UnknownChild/OrphanParent.php create mode 100644 packages/orchestration/tests/Support/OrchestrationCapstoneTestCase.php create mode 100644 packages/orchestration/tests/Support/OrchestrationFixturesProvider.php create mode 100644 packages/orchestration/tests/Validation/OrchestrationValidatorTest.php diff --git a/docs/modules.md b/docs/modules.md index 6629fec4..7fdb64ee 100644 --- a/docs/modules.md +++ b/docs/modules.md @@ -6,10 +6,10 @@ Several components carry more than one guide: `firefly/data` answers for *Data & *Relational Data* and *Transactions*. Laravel discovers the root library's providers, and configuration selects optional features. A bean you declare wins over the framework default. Two idioms deliver that outcome, and -which one you are looking at matters the moment you go reading the source. **18 of the 31 packages** carry a +which one you are looking at matters the moment you go reading the source. **19 of the 31 packages** carry a `#[Configuration]` class whose `#[Bean]` methods sit behind `#[ConditionalOnMissingBean]`, so a bean of yours makes the framework's back off silently — `packages/data/src/DataAutoConfiguration.php` is the model. The -**other 13** have no such class to go looking for; `firefly/web`, the one nearly every reader installs, guards +**other 12** have no such class to go looking for; `firefly/web`, the one nearly every reader installs, guards each binding in `WebServiceProvider::register()` with a `bound()` check instead, first-one-wins by hand.
diff --git a/packages/orchestration/cache/firefly-orchestration-components.php b/packages/orchestration/cache/firefly-orchestration-components.php index b2ca1444..835bdf22 100644 --- a/packages/orchestration/cache/firefly-orchestration-components.php +++ b/packages/orchestration/cache/firefly-orchestration-components.php @@ -5,4 +5,425 @@ // Generated by firefly/container. Do not edit. return [ + 0 => [ + 'class' => 'Firefly\\Orchestration\\OrchestrationAutoConfiguration', + 'stereotype' => 'configuration', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 1000, + 'qualifier' => null, + 'interfaces' => [ + ], + 'beans' => [ + 0 => [ + 'method' => 'orchestrationSettings', + 'returns' => 'Firefly\\Orchestration\\OrchestrationSettings', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 0, + 'lazy' => false, + 'dependencies' => [ + 0 => 'Firefly\\Config\\Config', + ], + ], + 1 => [ + 'method' => 'orchestrationClock', + 'returns' => 'Firefly\\Orchestration\\Core\\Clock', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 0, + 'lazy' => true, + 'dependencies' => [ + ], + ], + 2 => [ + 'method' => 'executionPersistenceProvider', + 'returns' => 'Firefly\\Orchestration\\Persistence\\ExecutionPersistenceProvider', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 0, + 'lazy' => true, + 'dependencies' => [ + 0 => 'Firefly\\Orchestration\\OrchestrationSettings', + 1 => 'Illuminate\\Container\\Container', + ], + ], + 3 => [ + 'method' => 'orchestrationDeadLetterStore', + 'returns' => 'Firefly\\Orchestration\\Dlq\\DeadLetterStore', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 0, + 'lazy' => true, + 'dependencies' => [ + 0 => 'Firefly\\Orchestration\\OrchestrationSettings', + 1 => 'Illuminate\\Container\\Container', + ], + ], + 4 => [ + 'method' => 'orchestrationMetrics', + 'returns' => 'Firefly\\Orchestration\\Event\\OrchestrationMetrics', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 0, + 'lazy' => true, + 'dependencies' => [ + ], + ], + 5 => [ + 'method' => 'orchestrationTracing', + 'returns' => 'Firefly\\Orchestration\\Event\\OrchestrationTracing', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 0, + 'lazy' => true, + 'dependencies' => [ + ], + ], + 6 => [ + 'method' => 'orchestrationEvents', + 'returns' => 'Firefly\\Orchestration\\Event\\CompositeOrchestrationEvents', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 0, + 'lazy' => true, + 'dependencies' => [ + 0 => 'Firefly\\Orchestration\\OrchestrationSettings', + 1 => 'Firefly\\Orchestration\\Event\\OrchestrationMetrics', + 2 => 'Firefly\\Container\\Container', + 3 => 'Illuminate\\Container\\Container', + ], + ], + 7 => [ + 'method' => 'orchestrationEventPublisher', + 'returns' => 'Firefly\\Orchestration\\Event\\OrchestrationEventPublisher', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 0, + 'lazy' => true, + 'dependencies' => [ + 0 => 'Firefly\\Orchestration\\OrchestrationSettings', + 1 => 'Firefly\\Eda\\EventPublisher', + ], + ], + 8 => [ + 'method' => 'executionLock', + 'returns' => 'Firefly\\Orchestration\\Core\\ExecutionLock', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 0, + 'lazy' => true, + 'dependencies' => [ + 0 => 'Firefly\\Orchestration\\OrchestrationSettings', + 1 => 'Firefly\\Scheduling\\Lock\\DistributedLock', + ], + ], + 9 => [ + 'method' => 'executionAdmission', + 'returns' => 'Firefly\\Orchestration\\Core\\ExecutionAdmission', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 0, + 'lazy' => true, + 'dependencies' => [ + 0 => 'Firefly\\Orchestration\\OrchestrationSettings', + 1 => 'Firefly\\Orchestration\\Persistence\\ExecutionPersistenceProvider', + 2 => 'Firefly\\Resilience\\Store\\ResilienceStore', + ], + ], + 10 => [ + 'method' => 'deadLetterService', + 'returns' => 'Firefly\\Orchestration\\Dlq\\DeadLetterService', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 0, + 'lazy' => true, + 'dependencies' => [ + 0 => 'Firefly\\Orchestration\\OrchestrationSettings', + 1 => 'Firefly\\Orchestration\\Dlq\\DeadLetterStore', + 2 => 'Firefly\\Orchestration\\Event\\CompositeOrchestrationEvents', + ], + ], + 11 => [ + 'method' => 'callbackInvoker', + 'returns' => 'Firefly\\Orchestration\\Core\\Callback\\CallbackInvoker', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 0, + 'lazy' => true, + 'dependencies' => [ + 0 => 'Illuminate\\Container\\Container', + ], + ], + 12 => [ + 'method' => 'engineSupport', + 'returns' => 'Firefly\\Orchestration\\Core\\EngineSupport', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 0, + 'lazy' => true, + 'dependencies' => [ + 0 => 'Firefly\\Orchestration\\OrchestrationSettings', + 1 => 'Firefly\\Orchestration\\Persistence\\ExecutionPersistenceProvider', + 2 => 'Firefly\\Orchestration\\Dlq\\DeadLetterService', + 3 => 'Firefly\\Orchestration\\Event\\CompositeOrchestrationEvents', + 4 => 'Firefly\\Orchestration\\Event\\OrchestrationEventPublisher', + 5 => 'Firefly\\Eda\\EventPublisher', + 6 => 'Firefly\\Orchestration\\Event\\OrchestrationTracing', + 7 => 'Firefly\\Orchestration\\Core\\Callback\\CallbackInvoker', + 8 => 'Firefly\\Orchestration\\Core\\Clock', + 9 => 'Firefly\\Orchestration\\Core\\ExecutionLock', + 10 => 'Firefly\\Orchestration\\Core\\ExecutionAdmission', + 11 => 'Illuminate\\Container\\Container', + ], + ], + 13 => [ + 'method' => 'conditionEvaluator', + 'returns' => 'Firefly\\Orchestration\\Core\\Condition\\ConditionEvaluator', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 0, + 'lazy' => true, + 'dependencies' => [ + ], + ], + 14 => [ + 'method' => 'invocationRunner', + 'returns' => 'Firefly\\Orchestration\\Core\\InvocationRunner', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 0, + 'lazy' => true, + 'dependencies' => [ + 0 => 'Illuminate\\Container\\Container', + ], + ], + 15 => [ + 'method' => 'stepInvoker', + 'returns' => 'Firefly\\Orchestration\\Core\\StepInvoker', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 0, + 'lazy' => true, + 'dependencies' => [ + 0 => 'Firefly\\Orchestration\\Core\\InvocationRunner', + 1 => 'Firefly\\Orchestration\\Core\\EngineSupport', + ], + ], + 16 => [ + 'method' => 'sagaRegistry', + 'returns' => 'Firefly\\Orchestration\\Saga\\Registry\\SagaRegistry', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 0, + 'lazy' => true, + 'dependencies' => [ + ], + ], + 17 => [ + 'method' => 'tccRegistry', + 'returns' => 'Firefly\\Orchestration\\Tcc\\Registry\\TccRegistry', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 0, + 'lazy' => true, + 'dependencies' => [ + ], + ], + 18 => [ + 'method' => 'workflowRegistry', + 'returns' => 'Firefly\\Orchestration\\Workflow\\Registry\\WorkflowRegistry', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 0, + 'lazy' => true, + 'dependencies' => [ + ], + ], + 19 => [ + 'method' => 'eventGateway', + 'returns' => 'Firefly\\Orchestration\\Event\\EventGateway', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 0, + 'lazy' => true, + 'dependencies' => [ + ], + ], + 20 => [ + 'method' => 'compensationErrorHandlerFactory', + 'returns' => 'Firefly\\Orchestration\\Saga\\Compensation\\CompensationErrorHandlerFactory', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 0, + 'lazy' => true, + 'dependencies' => [ + 0 => 'Illuminate\\Container\\Container', + ], + ], + 21 => [ + 'method' => 'sagaCompensator', + 'returns' => 'Firefly\\Orchestration\\Saga\\Compensation\\SagaCompensator', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 0, + 'lazy' => true, + 'dependencies' => [ + 0 => 'Firefly\\Orchestration\\OrchestrationSettings', + 1 => 'Firefly\\Orchestration\\Core\\StepInvoker', + 2 => 'Firefly\\Orchestration\\Core\\EngineSupport', + 3 => 'Firefly\\Orchestration\\Saga\\Compensation\\CompensationErrorHandlerFactory', + 4 => 'Firefly\\Resilience\\ResilienceRegistry', + 5 => 'Firefly\\Resilience\\Store\\ResilienceStore', + ], + ], + 22 => [ + 'method' => 'sagaEngine', + 'returns' => 'Firefly\\Orchestration\\Saga\\Engine\\SagaEngine', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 0, + 'lazy' => true, + 'dependencies' => [ + 0 => 'Firefly\\Orchestration\\OrchestrationSettings', + 1 => 'Firefly\\Orchestration\\Saga\\Registry\\SagaRegistry', + 2 => 'Firefly\\Orchestration\\Core\\StepInvoker', + 3 => 'Firefly\\Orchestration\\Saga\\Compensation\\SagaCompensator', + 4 => 'Firefly\\Orchestration\\Core\\EngineSupport', + ], + ], + 23 => [ + 'method' => 'sagaCompositor', + 'returns' => 'Firefly\\Orchestration\\Saga\\Composition\\SagaCompositor', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 0, + 'lazy' => true, + 'dependencies' => [ + 0 => 'Firefly\\Orchestration\\Saga\\Engine\\SagaEngine', + 1 => 'Firefly\\Orchestration\\Core\\EngineSupport', + 2 => 'Firefly\\Orchestration\\Core\\Condition\\ConditionEvaluator', + ], + ], + 24 => [ + 'method' => 'tccEngine', + 'returns' => 'Firefly\\Orchestration\\Tcc\\Engine\\TccEngine', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 0, + 'lazy' => true, + 'dependencies' => [ + 0 => 'Firefly\\Orchestration\\OrchestrationSettings', + 1 => 'Firefly\\Orchestration\\Tcc\\Registry\\TccRegistry', + 2 => 'Firefly\\Orchestration\\Core\\StepInvoker', + 3 => 'Firefly\\Orchestration\\Core\\EngineSupport', + ], + ], + 25 => [ + 'method' => 'tccCompositor', + 'returns' => 'Firefly\\Orchestration\\Tcc\\Composition\\TccCompositor', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 0, + 'lazy' => true, + 'dependencies' => [ + 0 => 'Firefly\\Orchestration\\OrchestrationSettings', + 1 => 'Firefly\\Orchestration\\Tcc\\Engine\\TccEngine', + 2 => 'Firefly\\Orchestration\\Core\\EngineSupport', + 3 => 'Firefly\\Orchestration\\Core\\Condition\\ConditionEvaluator', + 4 => 'Firefly\\Resilience\\ResilienceRegistry', + 5 => 'Firefly\\Resilience\\Store\\ResilienceStore', + ], + ], + 26 => [ + 'method' => 'workflowDispatcher', + 'returns' => 'Firefly\\Orchestration\\Workflow\\Queue\\WorkflowDispatcher', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 0, + 'lazy' => true, + 'dependencies' => [ + 0 => 'Firefly\\Orchestration\\OrchestrationSettings', + 1 => 'Illuminate\\Contracts\\Bus\\Dispatcher', + ], + ], + 27 => [ + 'method' => 'workflowEngine', + 'returns' => 'Firefly\\Orchestration\\Workflow\\Engine\\WorkflowEngine', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 0, + 'lazy' => true, + 'dependencies' => [ + 0 => 'Firefly\\Orchestration\\OrchestrationSettings', + 1 => 'Firefly\\Orchestration\\Workflow\\Registry\\WorkflowRegistry', + 2 => 'Firefly\\Orchestration\\Core\\StepInvoker', + 3 => 'Firefly\\Orchestration\\Core\\EngineSupport', + 4 => 'Firefly\\Orchestration\\Core\\Condition\\ConditionEvaluator', + 5 => 'Firefly\\Orchestration\\Workflow\\Queue\\WorkflowDispatcher', + 6 => 'Illuminate\\Container\\Container', + ], + ], + 28 => [ + 'method' => 'orchestrator', + 'returns' => 'Firefly\\Orchestration\\Core\\Orchestrator', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 0, + 'lazy' => true, + 'dependencies' => [ + 0 => 'Firefly\\Orchestration\\Saga\\Engine\\SagaEngine', + 1 => 'Firefly\\Orchestration\\Tcc\\Engine\\TccEngine', + 2 => 'Firefly\\Orchestration\\Workflow\\Engine\\WorkflowEngine', + 3 => 'Firefly\\Orchestration\\Persistence\\ExecutionPersistenceProvider', + ], + ], + 29 => [ + 'method' => 'deadLetterRetrier', + 'returns' => 'Firefly\\Orchestration\\Dlq\\DeadLetterRetrier', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 0, + 'lazy' => true, + 'dependencies' => [ + 0 => 'Firefly\\Orchestration\\Dlq\\DeadLetterService', + 1 => 'Firefly\\Orchestration\\Core\\Orchestrator', + ], + ], + ], + 'lazy' => false, + 'dependencies' => [ + ], + ], ]; diff --git a/packages/orchestration/cache/firefly-orchestration-context.php b/packages/orchestration/cache/firefly-orchestration-context.php index 4a1771ff..adedc364 100644 --- a/packages/orchestration/cache/firefly-orchestration-context.php +++ b/packages/orchestration/cache/firefly-orchestration-context.php @@ -5,4 +5,347 @@ // Generated by firefly/context. Do not edit. return [ + 0 => [ + 'class' => 'Firefly\\Orchestration\\OrchestrationAutoConfiguration', + 'postConstruct' => [ + ], + 'preDestroy' => [ + ], + 'listeners' => [ + ], + 'conditions' => [ + ], + 'beanConditions' => [ + 0 => [ + 'method' => 'orchestrationSettings', + 'conditions' => [ + 0 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnMissingBean', + 'args' => [ + 0 => 'Firefly\\Orchestration\\OrchestrationSettings', + ], + ], + ], + ], + 1 => [ + 'method' => 'orchestrationClock', + 'conditions' => [ + 0 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnMissingBean', + 'args' => [ + 0 => 'Firefly\\Orchestration\\Core\\Clock', + ], + ], + ], + ], + 2 => [ + 'method' => 'executionPersistenceProvider', + 'conditions' => [ + 0 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnMissingBean', + 'args' => [ + 0 => 'Firefly\\Orchestration\\Persistence\\ExecutionPersistenceProvider', + ], + ], + ], + ], + 3 => [ + 'method' => 'orchestrationDeadLetterStore', + 'conditions' => [ + 0 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnMissingBean', + 'args' => [ + 0 => 'Firefly\\Orchestration\\Dlq\\DeadLetterStore', + ], + ], + ], + ], + 4 => [ + 'method' => 'orchestrationMetrics', + 'conditions' => [ + 0 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnMissingBean', + 'args' => [ + 0 => 'Firefly\\Orchestration\\Event\\OrchestrationMetrics', + ], + ], + ], + ], + 5 => [ + 'method' => 'orchestrationTracing', + 'conditions' => [ + 0 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnMissingBean', + 'args' => [ + 0 => 'Firefly\\Orchestration\\Event\\OrchestrationTracing', + ], + ], + ], + ], + 6 => [ + 'method' => 'orchestrationEvents', + 'conditions' => [ + 0 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnMissingBean', + 'args' => [ + 0 => 'Firefly\\Orchestration\\Event\\CompositeOrchestrationEvents', + ], + ], + ], + ], + 7 => [ + 'method' => 'orchestrationEventPublisher', + 'conditions' => [ + 0 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnMissingBean', + 'args' => [ + 0 => 'Firefly\\Orchestration\\Event\\OrchestrationEventPublisher', + ], + ], + ], + ], + 8 => [ + 'method' => 'executionLock', + 'conditions' => [ + 0 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnMissingBean', + 'args' => [ + 0 => 'Firefly\\Orchestration\\Core\\ExecutionLock', + ], + ], + ], + ], + 9 => [ + 'method' => 'executionAdmission', + 'conditions' => [ + 0 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnMissingBean', + 'args' => [ + 0 => 'Firefly\\Orchestration\\Core\\ExecutionAdmission', + ], + ], + ], + ], + 10 => [ + 'method' => 'deadLetterService', + 'conditions' => [ + 0 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnMissingBean', + 'args' => [ + 0 => 'Firefly\\Orchestration\\Dlq\\DeadLetterService', + ], + ], + ], + ], + 11 => [ + 'method' => 'callbackInvoker', + 'conditions' => [ + 0 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnMissingBean', + 'args' => [ + 0 => 'Firefly\\Orchestration\\Core\\Callback\\CallbackInvoker', + ], + ], + ], + ], + 12 => [ + 'method' => 'engineSupport', + 'conditions' => [ + 0 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnMissingBean', + 'args' => [ + 0 => 'Firefly\\Orchestration\\Core\\EngineSupport', + ], + ], + ], + ], + 13 => [ + 'method' => 'conditionEvaluator', + 'conditions' => [ + 0 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnMissingBean', + 'args' => [ + 0 => 'Firefly\\Orchestration\\Core\\Condition\\ConditionEvaluator', + ], + ], + ], + ], + 14 => [ + 'method' => 'invocationRunner', + 'conditions' => [ + 0 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnMissingBean', + 'args' => [ + 0 => 'Firefly\\Orchestration\\Core\\InvocationRunner', + ], + ], + ], + ], + 15 => [ + 'method' => 'stepInvoker', + 'conditions' => [ + 0 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnMissingBean', + 'args' => [ + 0 => 'Firefly\\Orchestration\\Core\\StepInvoker', + ], + ], + ], + ], + 16 => [ + 'method' => 'sagaRegistry', + 'conditions' => [ + 0 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnMissingBean', + 'args' => [ + 0 => 'Firefly\\Orchestration\\Saga\\Registry\\SagaRegistry', + ], + ], + ], + ], + 17 => [ + 'method' => 'tccRegistry', + 'conditions' => [ + 0 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnMissingBean', + 'args' => [ + 0 => 'Firefly\\Orchestration\\Tcc\\Registry\\TccRegistry', + ], + ], + ], + ], + 18 => [ + 'method' => 'workflowRegistry', + 'conditions' => [ + 0 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnMissingBean', + 'args' => [ + 0 => 'Firefly\\Orchestration\\Workflow\\Registry\\WorkflowRegistry', + ], + ], + ], + ], + 19 => [ + 'method' => 'eventGateway', + 'conditions' => [ + 0 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnMissingBean', + 'args' => [ + 0 => 'Firefly\\Orchestration\\Event\\EventGateway', + ], + ], + ], + ], + 20 => [ + 'method' => 'compensationErrorHandlerFactory', + 'conditions' => [ + 0 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnMissingBean', + 'args' => [ + 0 => 'Firefly\\Orchestration\\Saga\\Compensation\\CompensationErrorHandlerFactory', + ], + ], + ], + ], + 21 => [ + 'method' => 'sagaCompensator', + 'conditions' => [ + 0 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnMissingBean', + 'args' => [ + 0 => 'Firefly\\Orchestration\\Saga\\Compensation\\SagaCompensator', + ], + ], + ], + ], + 22 => [ + 'method' => 'sagaEngine', + 'conditions' => [ + 0 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnMissingBean', + 'args' => [ + 0 => 'Firefly\\Orchestration\\Saga\\Engine\\SagaEngine', + ], + ], + ], + ], + 23 => [ + 'method' => 'sagaCompositor', + 'conditions' => [ + 0 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnMissingBean', + 'args' => [ + 0 => 'Firefly\\Orchestration\\Saga\\Composition\\SagaCompositor', + ], + ], + ], + ], + 24 => [ + 'method' => 'tccEngine', + 'conditions' => [ + 0 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnMissingBean', + 'args' => [ + 0 => 'Firefly\\Orchestration\\Tcc\\Engine\\TccEngine', + ], + ], + ], + ], + 25 => [ + 'method' => 'tccCompositor', + 'conditions' => [ + 0 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnMissingBean', + 'args' => [ + 0 => 'Firefly\\Orchestration\\Tcc\\Composition\\TccCompositor', + ], + ], + ], + ], + 26 => [ + 'method' => 'workflowDispatcher', + 'conditions' => [ + 0 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnMissingBean', + 'args' => [ + 0 => 'Firefly\\Orchestration\\Workflow\\Queue\\WorkflowDispatcher', + ], + ], + ], + ], + 27 => [ + 'method' => 'workflowEngine', + 'conditions' => [ + 0 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnMissingBean', + 'args' => [ + 0 => 'Firefly\\Orchestration\\Workflow\\Engine\\WorkflowEngine', + ], + ], + ], + ], + 28 => [ + 'method' => 'orchestrator', + 'conditions' => [ + 0 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnMissingBean', + 'args' => [ + 0 => 'Firefly\\Orchestration\\Core\\Orchestrator', + ], + ], + ], + ], + 29 => [ + 'method' => 'deadLetterRetrier', + 'conditions' => [ + 0 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnMissingBean', + 'args' => [ + 0 => 'Firefly\\Orchestration\\Dlq\\DeadLetterRetrier', + ], + ], + ], + ], + ], + ], ]; diff --git a/packages/orchestration/src/Boot/EventGatewayWiringPass.php b/packages/orchestration/src/Boot/EventGatewayWiringPass.php new file mode 100644 index 00000000..d8162fda --- /dev/null +++ b/packages/orchestration/src/Boot/EventGatewayWiringPass.php @@ -0,0 +1,57 @@ +container; + if (! $container->bound(EventGateway::class) || ! $container->bound(EventPublisher::class)) { + return; + } + + /** @var EventGateway $gateway */ + $gateway = $container->make(EventGateway::class); + if ($gateway->count() === 0) { + return; + } + + /** @var EventPublisher $publisher */ + $publisher = $container->make(EventPublisher::class); + foreach ($gateway->eventTypes() as $eventType) { + $publisher->subscribe($eventType, static function (EventEnvelope $envelope) use ($gateway, $eventType): void { + $payload = []; + foreach ($envelope->payload as $key => $value) { + $payload[(string) $key] = $value; + } + $gateway->route($eventType, $payload); + }); + } + } +} diff --git a/packages/orchestration/src/Boot/OrchestrationRegistryPass.php b/packages/orchestration/src/Boot/OrchestrationRegistryPass.php new file mode 100644 index 00000000..a0553cc6 --- /dev/null +++ b/packages/orchestration/src/Boot/OrchestrationRegistryPass.php @@ -0,0 +1,130 @@ +container; + if (! $container->bound(OrchestrationManifest::class) || ! $container->bound(OrchestrationSettings::class)) { + return; + } + + /** @var OrchestrationManifest $manifest */ + $manifest = $container->make(OrchestrationManifest::class); + /** @var OrchestrationSettings $settings */ + $settings = $container->make(OrchestrationSettings::class); + /** @var SagaRegistry $sagas */ + $sagas = $container->make(SagaRegistry::class); + /** @var TccRegistry $tccs */ + $tccs = $container->make(TccRegistry::class); + /** @var WorkflowRegistry $workflows */ + $workflows = $container->make(WorkflowRegistry::class); + + if ($settings->saga->enabled) { + foreach ($manifest->sagas() as $saga) { + $sagas->register($saga); + } + } + if ($settings->tcc->enabled) { + foreach ($manifest->tccs() as $tcc) { + $tccs->register($tcc); + } + } + if ($settings->workflow->enabled) { + foreach ($manifest->workflows() as $workflow) { + $workflows->register($workflow); + } + } + + if ($settings->validationEnabled) { + /** @var ConditionEvaluator $conditions */ + $conditions = $container->make(ConditionEvaluator::class); + $logger = $container->bound(LoggerInterface::class) ? $container->make(LoggerInterface::class) : new NullLogger; + $validator = new OrchestrationValidator($conditions); + $validator->enforce($validator->validate($sagas, $tccs, $workflows), $settings->failOnWarning, $logger); + } + + if (! $settings->events->gatewayEnabled) { + return; + } + + /** @var EventGateway $gateway */ + $gateway = $container->make(EventGateway::class); + $start = static fn (ExecutionPattern $pattern, string $name): Closure => self::starter($container, $pattern, $name); + + foreach ($sagas->all() as $saga) { + if ($saga->triggerEventType !== '') { + $gateway->register($saga->triggerEventType, $saga->name, ExecutionPattern::Saga, $start(ExecutionPattern::Saga, $saga->name)); + } + } + foreach ($tccs->all() as $tcc) { + if ($tcc->triggerEventType !== '') { + $gateway->register($tcc->triggerEventType, $tcc->name, ExecutionPattern::Tcc, $start(ExecutionPattern::Tcc, $tcc->name)); + } + } + foreach ($workflows->all() as $workflow) { + if ($workflow->triggerEventType !== '') { + $gateway->register($workflow->triggerEventType, $workflow->id, ExecutionPattern::Workflow, $start(ExecutionPattern::Workflow, $workflow->id)); + } + } + } + + /** + * The executor the gateway runs for one trigger: it resolves the Orchestrator only when the first event + * arrives, so registering triggers at boot builds no engine. + * + * @return Closure(array): mixed + */ + private static function starter(Container $container, ExecutionPattern $pattern, string $name): Closure + { + return static function (array $payload) use ($container, $pattern, $name): mixed { + $input = []; + foreach ($payload as $key => $value) { + $input[(string) $key] = $value; + } + /** @var Orchestrator $orchestrator */ + $orchestrator = $container->make(Orchestrator::class); + + return $orchestrator->start($pattern, $name, $input); + }; + } +} diff --git a/packages/orchestration/src/OrchestrationAutoConfiguration.php b/packages/orchestration/src/OrchestrationAutoConfiguration.php new file mode 100644 index 00000000..468f8f11 --- /dev/null +++ b/packages/orchestration/src/OrchestrationAutoConfiguration.php @@ -0,0 +1,418 @@ +persistence; + + return match ($persistence->provider) { + 'cache' => new CacheExecutionPersistenceProvider(self::cacheStore($app, $persistence->cacheStore), $persistence->keyPrefix, $persistence->keyTtl), + 'database' => new DatabaseExecutionPersistenceProvider(self::connection($app, $persistence->connection, 'firefly.orchestration.persistence.provider')), + 'event-sourced' => throw new ConfigurationException('Configuration key [firefly.orchestration.persistence.provider] is event-sourced, which firefly/eventsourcing supplies; install it, or choose memory, cache or database.'), + default => new MemoryExecutionPersistenceProvider, + }; + } + + #[Bean] + #[Lazy] + #[ConditionalOnMissingBean(DeadLetterStore::class)] + public function orchestrationDeadLetterStore(OrchestrationSettings $settings, Container $app): DeadLetterStore + { + return $settings->dlq->store === 'database' + ? new DatabaseDeadLetterStore(self::connection($app, $settings->persistence->connection, 'firefly.orchestration.dlq.store')) + : new MemoryDeadLetterStore; + } + + #[Bean] + #[Lazy] + #[ConditionalOnMissingBean(OrchestrationMetrics::class)] + public function orchestrationMetrics(): OrchestrationMetrics + { + return new NoOpOrchestrationMetrics; + } + + #[Bean] + #[Lazy] + #[ConditionalOnMissingBean(OrchestrationTracing::class)] + public function orchestrationTracing(): OrchestrationTracing + { + return new NoOpOrchestrationTracing; + } + + #[Bean] + #[Lazy] + #[ConditionalOnMissingBean(CompositeOrchestrationEvents::class)] + public function orchestrationEvents(OrchestrationSettings $settings, OrchestrationMetrics $metrics, FireflyContainer $beans, Container $app): CompositeOrchestrationEvents + { + $logger = self::logger($app); + $listeners = [new LoggerOrchestrationEvents($logger)]; + if ($settings->metricsEnabled) { + $listeners[] = new MetricsOrchestrationEvents($metrics); + } + + return new CompositeOrchestrationEvents($listeners, static fn (): array => $beans->getAll(OrchestrationEvents::class), $logger); + } + + #[Bean] + #[Lazy] + #[ConditionalOnMissingBean(OrchestrationEventPublisher::class)] + public function orchestrationEventPublisher(OrchestrationSettings $settings, EventPublisher $publisher): OrchestrationEventPublisher + { + return $settings->events->enabled ? new EdaOrchestrationEventPublisher($publisher, $settings->events->defaultTopic) : new NoOpOrchestrationEventPublisher; + } + + #[Bean] + #[Lazy] + #[ConditionalOnMissingBean(ExecutionLock::class)] + public function executionLock(OrchestrationSettings $settings, DistributedLock $lock): ExecutionLock + { + return new ExecutionLock($lock, $settings->lock->ttl, $settings->lock->wait); + } + + #[Bean] + #[Lazy] + #[ConditionalOnMissingBean(ExecutionAdmission::class)] + public function executionAdmission(OrchestrationSettings $settings, ExecutionPersistenceProvider $persistence, ResilienceStore $store): ExecutionAdmission + { + $backpressure = $settings->backpressure; + $breaker = $backpressure->strategy === 'circuit-breaker' + ? new CircuitBreaker( + key: 'firefly:orchestration:backpressure', + store: $store, + failureThreshold: $backpressure->failureThreshold, + waitDurationInOpen: $backpressure->recoveryTimeout, + halfOpenMaxCalls: $backpressure->halfOpenMaxCalls, + ) + : null; + + return new ExecutionAdmission($persistence, $backpressure, $breaker); + } + + #[Bean] + #[Lazy] + #[ConditionalOnMissingBean(DeadLetterService::class)] + public function deadLetterService(OrchestrationSettings $settings, DeadLetterStore $store, CompositeOrchestrationEvents $events): DeadLetterService + { + return new DeadLetterService($store, $events, $settings->dlq->enabled); + } + + #[Bean] + #[Lazy] + #[ConditionalOnMissingBean(CallbackInvoker::class)] + public function callbackInvoker(Container $app): CallbackInvoker + { + return new CallbackInvoker($app, self::logger($app)); + } + + #[Bean] + #[Lazy] + #[ConditionalOnMissingBean(EngineSupport::class)] + public function engineSupport( + OrchestrationSettings $settings, + ExecutionPersistenceProvider $persistence, + DeadLetterService $deadLetters, + CompositeOrchestrationEvents $events, + OrchestrationEventPublisher $lifecycle, + EventPublisher $eda, + OrchestrationTracing $tracing, + CallbackInvoker $callbacks, + Clock $clock, + ExecutionLock $lock, + ExecutionAdmission $admission, + Container $app, + ): EngineSupport { + return new EngineSupport( + persistence: $persistence, + deadLetters: $deadLetters, + events: $events, + lifecycle: $lifecycle, + stepEvents: new EdaOrchestrationEventPublisher($eda, $settings->events->defaultTopic), + tracing: $settings->tracingEnabled ? $tracing : new NoOpOrchestrationTracing, + callbacks: $callbacks, + clock: $clock, + lock: $lock, + admission: $admission, + logger: self::logger($app), + ); + } + + #[Bean] + #[Lazy] + #[ConditionalOnMissingBean(ConditionEvaluator::class)] + public function conditionEvaluator(): ConditionEvaluator + { + return new ConditionEvaluator; + } + + #[Bean] + #[Lazy] + #[ConditionalOnMissingBean(InvocationRunner::class)] + public function invocationRunner(Container $app): InvocationRunner + { + return new InvocationRunner($app, new ArgumentResolver); + } + + #[Bean] + #[Lazy] + #[ConditionalOnMissingBean(StepInvoker::class)] + public function stepInvoker(InvocationRunner $runner, EngineSupport $support): StepInvoker + { + return new StepInvoker($runner, $support->tracing); + } + + #[Bean] + #[Lazy] + #[ConditionalOnMissingBean(SagaRegistry::class)] + public function sagaRegistry(): SagaRegistry + { + return new SagaRegistry; + } + + #[Bean] + #[Lazy] + #[ConditionalOnMissingBean(TccRegistry::class)] + public function tccRegistry(): TccRegistry + { + return new TccRegistry; + } + + #[Bean] + #[Lazy] + #[ConditionalOnMissingBean(WorkflowRegistry::class)] + public function workflowRegistry(): WorkflowRegistry + { + return new WorkflowRegistry; + } + + #[Bean] + #[Lazy] + #[ConditionalOnMissingBean(EventGateway::class)] + public function eventGateway(): EventGateway + { + return new EventGateway; + } + + #[Bean] + #[Lazy] + #[ConditionalOnMissingBean(CompensationErrorHandlerFactory::class)] + public function compensationErrorHandlerFactory(Container $app): CompensationErrorHandlerFactory + { + return new CompensationErrorHandlerFactory($app, self::logger($app)); + } + + #[Bean] + #[Lazy] + #[ConditionalOnMissingBean(SagaCompensator::class)] + public function sagaCompensator(OrchestrationSettings $settings, StepInvoker $invoker, EngineSupport $support, CompensationErrorHandlerFactory $handlers, ResilienceRegistry $resilience, ResilienceStore $store): SagaCompensator + { + return new SagaCompensator($invoker, $support, $settings->saga->compensationPolicy, $handlers->get($settings->saga->compensationErrorHandler), $resilience, $store); + } + + #[Bean] + #[Lazy] + #[ConditionalOnMissingBean(SagaEngine::class)] + public function sagaEngine(OrchestrationSettings $settings, SagaRegistry $registry, StepInvoker $invoker, SagaCompensator $compensator, EngineSupport $support): SagaEngine + { + return new SagaEngine($registry, new SagaExecutionOrchestrator($invoker, $support), $compensator, $support, $settings->saga); + } + + #[Bean] + #[Lazy] + #[ConditionalOnMissingBean(SagaCompositor::class)] + public function sagaCompositor(SagaEngine $engine, EngineSupport $support, ConditionEvaluator $conditions): SagaCompositor + { + return new SagaCompositor($engine, $support->events, $conditions, new CompositionDataFlowManager); + } + + #[Bean] + #[Lazy] + #[ConditionalOnMissingBean(TccEngine::class)] + public function tccEngine(OrchestrationSettings $settings, TccRegistry $registry, StepInvoker $invoker, EngineSupport $support): TccEngine + { + return new TccEngine($registry, new TccExecutionOrchestrator($invoker, $support, $settings->tcc), $support, $settings->tcc); + } + + #[Bean] + #[Lazy] + #[ConditionalOnMissingBean(TccCompositor::class)] + public function tccCompositor(OrchestrationSettings $settings, TccEngine $engine, EngineSupport $support, ConditionEvaluator $conditions, ResilienceRegistry $resilience, ResilienceStore $store): TccCompositor + { + // Task 12 fix: the compositor honours tcc.composition.compensation-policy and needs the resilience + // registry and store for retry-with-backoff / circuit-breaker, exactly as SagaCompensator does. + return new TccCompositor($engine, $support->events, $conditions, new CompositionDataFlowManager, $settings->tcc, $resilience, $store); + } + + #[Bean] + #[Lazy] + #[ConditionalOnMissingBean(WorkflowDispatcher::class)] + public function workflowDispatcher(OrchestrationSettings $settings, Dispatcher $bus): WorkflowDispatcher + { + return new BusWorkflowDispatcher($bus, $settings->workflow->queueConnection, $settings->workflow->queueName); + } + + #[Bean] + #[Lazy] + #[ConditionalOnMissingBean(WorkflowEngine::class)] + public function workflowEngine(OrchestrationSettings $settings, WorkflowRegistry $registry, StepInvoker $invoker, EngineSupport $support, ConditionEvaluator $conditions, WorkflowDispatcher $dispatcher, Container $app): WorkflowEngine + { + return new WorkflowEngine( + $registry, + new WorkflowExecutor($invoker, $support, $conditions, new WaitEvaluator($support->events)), + $support, + $settings->workflow, + $dispatcher, + $app, + $settings->backpressure->batchSize, + ); + } + + #[Bean] + #[Lazy] + #[ConditionalOnMissingBean(Orchestrator::class)] + public function orchestrator(SagaEngine $sagas, TccEngine $tccs, WorkflowEngine $workflows, ExecutionPersistenceProvider $persistence): Orchestrator + { + return new Orchestrator($sagas, $tccs, $workflows, $persistence); + } + + #[Bean] + #[Lazy] + #[ConditionalOnMissingBean(DeadLetterRetrier::class)] + public function deadLetterRetrier(DeadLetterService $deadLetters, Orchestrator $orchestrator): DeadLetterRetrier + { + return new DeadLetterRetrier($deadLetters, $orchestrator); + } + + private static function cacheStore(Container $app, ?string $store): CacheRepository + { + if (! $app->bound(CacheFactory::class)) { + throw new ConfigurationException('Configuration key [firefly.orchestration.persistence.provider] is cache but no cache manager is bound.'); + } + + /** @var CacheFactory $cache */ + $cache = $app->make(CacheFactory::class); + + return $cache->store($store); + } + + private static function connection(Container $app, ?string $connection, string $key): ConnectionInterface + { + $databases = $app->bound('db') ? $app->make('db') : null; + if (! $databases instanceof ConnectionResolverInterface) { + throw new ConfigurationException("Configuration key [{$key}] is database but no database connection resolver (db) is bound."); + } + + return $databases->connection($connection); + } + + private static function logger(Container $app): LoggerInterface + { + $logger = $app->bound(LoggerInterface::class) ? $app->make(LoggerInterface::class) : null; + + return $logger instanceof LoggerInterface ? $logger : new NullLogger; + } +} diff --git a/packages/orchestration/src/OrchestrationWiringProvider.php b/packages/orchestration/src/OrchestrationWiringProvider.php index b83d4634..45b37344 100644 --- a/packages/orchestration/src/OrchestrationWiringProvider.php +++ b/packages/orchestration/src/OrchestrationWiringProvider.php @@ -4,23 +4,66 @@ namespace Firefly\Orchestration; +use Firefly\Config\Config; use Firefly\Context\Boot\BootPass; use Firefly\Context\Boot\FireflyServiceProvider; +use Firefly\Context\Scan\AppScan; +use Firefly\Orchestration\Boot\EventGatewayWiringPass; +use Firefly\Orchestration\Boot\OrchestrationRegistryPass; +use Firefly\Orchestration\Manifest\OrchestrationManifest; +use Firefly\Orchestration\Scanner\OrchestrationScanner; +use Illuminate\Contracts\Container\Container; /** * The boot-pass half of firefly/orchestration. It cannot ride on OrchestrationServiceProvider (an * AutoConfiguration records candidacy only), so — exactly like SchedulingWiringProvider and - * FeatureFlagsWiringProvider — this plain FireflyServiceProvider binds the compiled OrchestrationManifest, - * contributes the registry, gateway, schedule and route passes, registers the console commands and loads the - * database migration. Both providers are listed in extra.laravel.providers. + * FeatureFlagsWiringProvider — this plain FireflyServiceProvider: + * + * - binds the OrchestrationManifest behind a bound() guard: the compiled `orchestration.php` first, then an + * in-process scan of `firefly.scan.paths`, then an empty manifest (the AppScan convention), lazily, so a + * cached application that has firefly/cli pays nothing; + * - contributes the registry and gateway passes; + * - publishes the migration (`--tag=firefly-orchestration-migrations`) and loads it while persistence or + * the DLQ is `database`. */ final class OrchestrationWiringProvider extends FireflyServiceProvider { + public const string MIGRATION = '2026_10_12_000000_create_firefly_orchestration_tables.php'; + + public function register(): void + { + if (! $this->app->bound(OrchestrationManifest::class)) { + $this->app->singleton(OrchestrationManifest::class, static function (Container $app): OrchestrationManifest { + if (($file = AppScan::cachedFile($app, AppScan::ORCHESTRATION)) !== null) { + return OrchestrationManifest::load($file); + } + + $paths = AppScan::paths($app); + + return $paths === [] ? OrchestrationManifest::empty() : (new OrchestrationScanner)->scan($paths); + }); + } + + parent::register(); + } + /** * @return list */ public function passes(): array { - return []; + return [new OrchestrationRegistryPass, new EventGatewayWiringPass]; + } + + public function boot(): void + { + $migrations = dirname(__DIR__).'/database/migrations'; + $this->publishes([$migrations.'/'.self::MIGRATION => $this->app->databasePath('migrations/'.self::MIGRATION)], 'firefly-orchestration-migrations'); + + $config = new Config($this->app->make('config')); + if ($config->string('firefly.orchestration.persistence.provider', 'memory') === 'database' + || $config->string('firefly.orchestration.dlq.store', 'memory') === 'database') { + $this->loadMigrationsFrom($migrations); + } } } diff --git a/packages/orchestration/src/Validation/OrchestrationValidator.php b/packages/orchestration/src/Validation/OrchestrationValidator.php new file mode 100644 index 00000000..9ec125c7 --- /dev/null +++ b/packages/orchestration/src/Validation/OrchestrationValidator.php @@ -0,0 +1,75 @@ + */ + public function validate(SagaRegistry $sagas, TccRegistry $tccs, WorkflowRegistry $workflows): array + { + $issues = []; + + foreach ($sagas->all() as $saga) { + foreach ($saga->steps as $step) { + if ($step->compensationCritical && $step->compensation === null) { + $issues[] = new ValidationIssue(ValidationIssue::WARNING, "Saga [{$saga->name}] step [{$step->id}] is compensationCritical but has no compensation."); + } + } + } + + foreach ($workflows->all() as $workflow) { + foreach ($workflow->steps as $step) { + if ($step->child !== null && ! $workflows->has($step->child->workflowId)) { + $issues[] = new ValidationIssue(ValidationIssue::ERROR, "Workflow [{$workflow->id}] step [{$step->id}] starts child workflow [{$step->child->workflowId}], which is not registered."); + } + if ($step->condition !== '') { + try { + $this->conditions->lint($step->condition, "Workflow [{$workflow->id}] step [{$step->id}]"); + } catch (TopologyValidationException $error) { + $issues[] = new ValidationIssue(ValidationIssue::ERROR, $error->getMessage()); + } + } + } + } + + return $issues; + } + + /** + * @param list $issues + */ + public function enforce(array $issues, bool $failOnWarning, LoggerInterface $logger): void + { + $refusals = []; + foreach ($issues as $issue) { + if ($issue->severity === ValidationIssue::ERROR || $failOnWarning) { + $refusals[] = $issue->message; + + continue; + } + $logger->warning("Orchestration validation: {$issue->message}"); + } + + if ($refusals !== []) { + throw new TopologyValidationException("Orchestration definitions are invalid:\n - ".implode("\n - ", $refusals)); + } + } +} diff --git a/packages/orchestration/src/Validation/ValidationIssue.php b/packages/orchestration/src/Validation/ValidationIssue.php new file mode 100644 index 00000000..ec2c086d --- /dev/null +++ b/packages/orchestration/src/Validation/ValidationIssue.php @@ -0,0 +1,18 @@ + []], + [ResilienceServiceProvider::class, SchedulingServiceProvider::class, SchedulingWiringProvider::class, EdaServiceProvider::class, EdaWiringProvider::class, OrchestrationServiceProvider::class, OrchestrationWiringProvider::class], + bindings: [ + OrchestrationManifest::class => (new OrchestrationScanner)->scan(['Firefly\\Orchestration\\Tests\\InvalidFixtures\\UnknownChild\\' => dirname(__DIR__).'/InvalidFixtures/UnknownChild']), + CacheRepositoryContract::class => new CacheRepository(new ArrayStore), + CacheFactoryContract::class => new class implements CacheFactoryContract + { + public function store($name = null) + { + return new CacheRepository(new ArrayStore); + } + }, + ], + ); +})->throws(TopologyValidationException::class, 'Workflow [orphan-parent] step [spawn] starts child workflow [does-not-exist], which is not registered'); diff --git a/packages/orchestration/tests/Capstone/CapstoneSagaTest.php b/packages/orchestration/tests/Capstone/CapstoneSagaTest.php new file mode 100644 index 00000000..827fb44a --- /dev/null +++ b/packages/orchestration/tests/Capstone/CapstoneSagaTest.php @@ -0,0 +1,48 @@ +app()->make(EventPublisher::class)->subscribe('*', static function (EventEnvelope $envelope) use (&$seen): void { + $seen[] = $envelope->eventType; + }); + + $outcome = $this->app()->make(Orchestrator::class)->start(ExecutionPattern::Saga, 'checkout', ['orderId' => 7, 'amount' => 40]); + + expect($this->app()->make(SagaRegistry::class)->names())->toBe(['checkout']) + ->and($outcome->success)->toBeTrue() + ->and($this->app()->make(Ledger::class)->lines)->toBe(['hook start checkout', 'reserve 7', 'charge 40 for r-7', 'completed checkout']) + ->and(DB::table(OrchestrationSchema::EXECUTIONS)->where('correlation_id', $outcome->correlationId)->value('status'))->toBe('COMPLETED') + ->and($seen)->toContain('orchestration.execution_started', 'payment.captured', 'orchestration.execution_completed'); +}); + +it('compensates in reverse, calls #[OnSagaError] and dead-letters into the database', function (): void { + /** @var OrchestrationCapstoneTestCase $this */ + $outcome = $this->app()->make(Orchestrator::class)->start(ExecutionPattern::Saga, 'checkout', ['orderId' => 8, 'amount' => 500]); + + expect($outcome->success)->toBeFalse() + ->and($this->app()->make(Ledger::class)->lines)->toBe(['hook start checkout', 'reserve 8', 'release r-8', 'failed: amount 500 exceeds the limit']) + ->and(DB::table(OrchestrationSchema::DEAD_LETTERS)->where('correlation_id', $outcome->correlationId)->value('step_id'))->toBe('charge') + ->and(DB::table(OrchestrationSchema::EXECUTIONS)->where('correlation_id', $outcome->correlationId)->value('status'))->toBe('FAILED'); +}); + +it('starts the saga from its trigger event on the EDA bus', function (): void { + /** @var OrchestrationCapstoneTestCase $this */ + $this->app()->make(EventPublisher::class)->publish('orders', 'orders.placed', ['orderId' => 9, 'amount' => 10]); + + expect($this->app()->make(Ledger::class)->lines)->toContain('reserve 9', 'charge 10 for r-9'); +}); diff --git a/packages/orchestration/tests/Capstone/CapstoneTccTest.php b/packages/orchestration/tests/Capstone/CapstoneTccTest.php new file mode 100644 index 00000000..15a9dee2 --- /dev/null +++ b/packages/orchestration/tests/Capstone/CapstoneTccTest.php @@ -0,0 +1,27 @@ +app()->make(Orchestrator::class)->start(ExecutionPattern::Tcc, 'transfer', ['amount' => 25, 'to' => 'acc-2']); + + expect($outcome->status)->toBe(ExecutionStatus::Confirmed) + ->and($this->app()->make(Ledger::class)->lines)->toBe(['hook start transfer', 'hold 25', 'credit hold acc-2', 'commit hold-25', 'credit commit credit-acc-2']); +}); + +it('cancels the tried participant when another try fails', function (): void { + /** @var OrchestrationCapstoneTestCase $this */ + $outcome = $this->app()->make(Orchestrator::class)->start(ExecutionPattern::Tcc, 'transfer', ['amount' => 25, 'to' => 'closed']); + + expect($outcome->status)->toBe(ExecutionStatus::Canceled) + ->and($this->app()->make(Ledger::class)->lines)->toBe(['hook start transfer', 'hold 25', 'release hold-25']); +}); diff --git a/packages/orchestration/tests/Capstone/CapstoneWorkflowTest.php b/packages/orchestration/tests/Capstone/CapstoneWorkflowTest.php new file mode 100644 index 00000000..cfd3a38f --- /dev/null +++ b/packages/orchestration/tests/Capstone/CapstoneWorkflowTest.php @@ -0,0 +1,30 @@ +app()->make(WorkflowEngine::class); + + $started = $workflows->start('onboarding', ['email' => 'ada@example.com'], correlationId: 'onb-1'); + $signalled = $workflows->signal('onb-1', 'email-verified', ['token' => 't-1']); + usleep(5_000); + $workflows->resumeDue(); + $finished = $workflows->find('onb-1'); + + expect($started->status)->toBe(ExecutionStatus::Waiting) + ->and($started->waitingOn)->toBe('verify') + ->and($signalled->status)->toBe(ExecutionStatus::Waiting) + ->and($signalled->waitingOn)->toBe('cool-off') + ->and($finished?->status)->toBe(ExecutionStatus::Completed) + ->and($this->app()->make(Ledger::class)->lines)->toBe(['hook start onboarding', 'register ada@example.com', 'verified', 'cooled off']) + ->and($workflows->query('onb-1', 'progress'))->toBe(3) + ->and(array_map(static fn ($s): string => $s->correlationId, $workflows->search(['email' => 'ada@example.com'])))->toBe(['onb-1']); +}); diff --git a/packages/orchestration/tests/CapstoneFixtures/CheckoutHook.php b/packages/orchestration/tests/CapstoneFixtures/CheckoutHook.php new file mode 100644 index 00000000..065aed8c --- /dev/null +++ b/packages/orchestration/tests/CapstoneFixtures/CheckoutHook.php @@ -0,0 +1,21 @@ +ledger->record("hook start {$name}"); + } +} diff --git a/packages/orchestration/tests/CapstoneFixtures/CheckoutSaga.php b/packages/orchestration/tests/CapstoneFixtures/CheckoutSaga.php new file mode 100644 index 00000000..1ed80490 --- /dev/null +++ b/packages/orchestration/tests/CapstoneFixtures/CheckoutSaga.php @@ -0,0 +1,69 @@ +ledger->record("reserve {$orderId}"); + + return ['reservation' => "r-{$orderId}"]; + } + + /** @param array{reservation: string} $reservation */ + public function release(array $reservation): void + { + $this->ledger->record("release {$reservation['reservation']}"); + } + + /** @param array{reservation: string} $reservation */ + #[SagaStep(id: 'charge', dependsOn: ['reserve'], compensate: 'refund', retry: 1, backoffMs: 1)] + #[StepEvent(topic: 'payments', type: 'payment.captured', key: 'orderId')] + public function charge(#[Input('amount')] int $amount, #[FromStep('reserve')] array $reservation): string + { + if ($amount > 100) { + throw new RuntimeException("amount {$amount} exceeds the limit"); + } + $this->ledger->record("charge {$amount} for {$reservation['reservation']}"); + + return 'receipt-1'; + } + + public function refund(string $receipt): void + { + $this->ledger->record("refund {$receipt}"); + } + + #[OnSagaComplete] + public function completed(SagaResult $result): void + { + $this->ledger->record("completed {$result->sagaName}"); + } + + #[OnSagaError] + public function failed(Throwable $error): void + { + $this->ledger->record("failed: {$error->getMessage()}"); + } +} diff --git a/packages/orchestration/tests/CapstoneFixtures/CreditAccount.php b/packages/orchestration/tests/CapstoneFixtures/CreditAccount.php new file mode 100644 index 00000000..d66e0a20 --- /dev/null +++ b/packages/orchestration/tests/CapstoneFixtures/CreditAccount.php @@ -0,0 +1,43 @@ +ledger->record("credit hold {$account}"); + + return "credit-{$account}"; + } + + #[ConfirmMethod] + public function commit(string $hold): void + { + $this->ledger->record("credit commit {$hold}"); + } + + #[CancelMethod] + public function release(string $hold): void + { + $this->ledger->record("credit release {$hold}"); + } +} diff --git a/packages/orchestration/tests/CapstoneFixtures/DebitAccount.php b/packages/orchestration/tests/CapstoneFixtures/DebitAccount.php new file mode 100644 index 00000000..b52f2783 --- /dev/null +++ b/packages/orchestration/tests/CapstoneFixtures/DebitAccount.php @@ -0,0 +1,40 @@ +ledger->record("hold {$amount}"); + + return "hold-{$amount}"; + } + + #[ConfirmMethod] + public function commit(#[FromTry] string $hold): void + { + $this->ledger->record("commit {$hold}"); + } + + #[CancelMethod] + public function release(#[FromTry] string $hold): void + { + $this->ledger->record("release {$hold}"); + } +} diff --git a/packages/orchestration/tests/CapstoneFixtures/Ledger.php b/packages/orchestration/tests/CapstoneFixtures/Ledger.php new file mode 100644 index 00000000..c8a8fc91 --- /dev/null +++ b/packages/orchestration/tests/CapstoneFixtures/Ledger.php @@ -0,0 +1,20 @@ + */ + public array $lines = []; + + public function record(string $line): void + { + $this->lines[] = $line; + } +} diff --git a/packages/orchestration/tests/CapstoneFixtures/OnboardingWorkflow.php b/packages/orchestration/tests/CapstoneFixtures/OnboardingWorkflow.php new file mode 100644 index 00000000..b45ccb79 --- /dev/null +++ b/packages/orchestration/tests/CapstoneFixtures/OnboardingWorkflow.php @@ -0,0 +1,55 @@ +setSearchAttribute('email', $email); + $this->ledger->record("register {$email}"); + + return "user-{$email}"; + } + + #[WorkflowStep(id: 'verify', dependsOn: ['register'])] + #[WaitForSignal('email-verified')] + public function verify(#[Variable('signals.email-verified')] mixed $proof): string + { + $this->ledger->record('verified'); + + return 'verified'; + } + + #[WorkflowStep(id: 'cool-off', dependsOn: ['verify'])] + #[WaitForTimer(delayMs: 1, timerId: 'cool-off')] + public function coolOff(): string + { + $this->ledger->record('cooled off'); + + return 'cooled'; + } + + #[WorkflowQuery('progress')] + public function progress(ExecutionContext $ctx): int + { + return count($ctx->results()); + } +} diff --git a/packages/orchestration/tests/CapstoneFixtures/TransferTcc.php b/packages/orchestration/tests/CapstoneFixtures/TransferTcc.php new file mode 100644 index 00000000..8a756e0a --- /dev/null +++ b/packages/orchestration/tests/CapstoneFixtures/TransferTcc.php @@ -0,0 +1,10 @@ + */ + #[WorkflowStep(id: 'spawn')] + #[ChildWorkflow(workflowId: 'does-not-exist')] + public function spawn(): array + { + return []; + } +} diff --git a/packages/orchestration/tests/PackageBootsTest.php b/packages/orchestration/tests/PackageBootsTest.php index e33aa412..c35fb5e9 100644 --- a/packages/orchestration/tests/PackageBootsTest.php +++ b/packages/orchestration/tests/PackageBootsTest.php @@ -3,19 +3,62 @@ declare(strict_types=1); use Firefly\Context\Boot\ApplicationContext; +use Firefly\Eda\EdaServiceProvider; +use Firefly\Eda\EdaWiringProvider; +use Firefly\Kernel\Exception\Framework\ConfigurationException; +use Firefly\Orchestration\Manifest\OrchestrationManifest; use Firefly\Orchestration\OrchestrationServiceProvider; +use Firefly\Orchestration\OrchestrationSettings; use Firefly\Orchestration\OrchestrationWiringProvider; +use Firefly\Orchestration\Persistence\ExecutionPersistenceProvider; +use Firefly\Orchestration\Persistence\MemoryExecutionPersistenceProvider; +use Firefly\Orchestration\Saga\Engine\SagaEngine; +use Firefly\Orchestration\Saga\Registry\SagaRegistry; +use Firefly\Resilience\ResilienceServiceProvider; +use Firefly\Scheduling\SchedulingServiceProvider; +use Firefly\Scheduling\SchedulingWiringProvider; +use Illuminate\Cache\ArrayStore; +use Illuminate\Cache\Repository as CacheRepository; +use Illuminate\Contracts\Cache\Factory as CacheFactoryContract; +use Illuminate\Contracts\Cache\Repository as CacheRepositoryContract; +use Illuminate\Foundation\Application; /** - * The two discovered providers boot over the real pipeline next to the bootstrap provider — exactly what - * `composer require firefly/orchestration` does. Task 17 extends this file once the package has beans. + * The discovered providers boot over the real pipeline next to the ones they depend on — exactly what + * `composer require firefly/orchestration` brings. The cache bindings are what a real application has + * (scheduling's lock and resilience's store take a Repository; see the scheduling-postgres coexistence test). + * + * @param array $orchestration */ -it('boots green when discovered alongside the bootstrap provider', function (): void { - $app = fireflyApplication( - ['firefly' => []], - [OrchestrationServiceProvider::class, OrchestrationWiringProvider::class], - needs: ['cache'], +function orchestrationBareApp(array $orchestration = []): Application +{ + return fireflyApplication( + ['firefly' => ['orchestration' => $orchestration]], + [ResilienceServiceProvider::class, SchedulingServiceProvider::class, SchedulingWiringProvider::class, EdaServiceProvider::class, EdaWiringProvider::class, OrchestrationServiceProvider::class, OrchestrationWiringProvider::class], + bindings: [ + CacheRepositoryContract::class => new CacheRepository(new ArrayStore), + CacheFactoryContract::class => new class implements CacheFactoryContract + { + public function store($name = null) + { + return new CacheRepository(new ArrayStore); + } + }, + ], ); +} - expect($app->make(ApplicationContext::class))->toBeInstanceOf(ApplicationContext::class); +it('boots green with the settings eager and the engines lazily available', function (): void { + $app = orchestrationBareApp(); + + expect($app->make(ApplicationContext::class))->toBeInstanceOf(ApplicationContext::class) + ->and($app->make(OrchestrationSettings::class)->persistence->provider)->toBe('memory') + ->and($app->make(OrchestrationManifest::class)->isEmpty())->toBeTrue() + ->and($app->make(ExecutionPersistenceProvider::class))->toBeInstanceOf(MemoryExecutionPersistenceProvider::class) + ->and($app->make(SagaEngine::class))->toBeInstanceOf(SagaEngine::class) + ->and($app->make(SagaRegistry::class)->names())->toBe([]); }); + +it('refuses at boot a configuration it cannot honour', function (): void { + orchestrationBareApp(['persistence' => ['provider' => 'redis']]); +})->throws(ConfigurationException::class, 'firefly.orchestration.persistence.provider'); diff --git a/packages/orchestration/tests/Support/OrchestrationCapstoneTestCase.php b/packages/orchestration/tests/Support/OrchestrationCapstoneTestCase.php new file mode 100644 index 00000000..0856e10c --- /dev/null +++ b/packages/orchestration/tests/Support/OrchestrationCapstoneTestCase.php @@ -0,0 +1,73 @@ + */ + public static function fixtures(): array + { + return ['Firefly\\Orchestration\\Tests\\CapstoneFixtures\\' => dirname(__DIR__).'/CapstoneFixtures']; + } + + /** @return list */ + protected function fireflyProviders(): array + { + return [ + ResilienceServiceProvider::class, + SchedulingServiceProvider::class, + SchedulingWiringProvider::class, + EdaServiceProvider::class, + EdaWiringProvider::class, + OrchestrationServiceProvider::class, + OrchestrationWiringProvider::class, + OrchestrationFixturesProvider::class, + ]; + } + + /** @return array */ + protected function configOverrides(): array + { + return [ + 'cache.default' => 'array', + 'queue.default' => 'sync', + 'firefly.orchestration.persistence.provider' => 'database', + 'firefly.orchestration.dlq.store' => 'database', + 'firefly.orchestration.events.enabled' => true, + ]; + } + + protected function defineFireflyEnvironment(Application $app): void + { + $app->instance(OrchestrationManifest::class, (new OrchestrationScanner)->scan(self::fixtures())); + } + + protected function setUp(): void + { + parent::setUp(); + + OrchestrationSchema::create(DB::connection()->getSchemaBuilder()); + } +} diff --git a/packages/orchestration/tests/Support/OrchestrationFixturesProvider.php b/packages/orchestration/tests/Support/OrchestrationFixturesProvider.php new file mode 100644 index 00000000..7de4a69c --- /dev/null +++ b/packages/orchestration/tests/Support/OrchestrationFixturesProvider.php @@ -0,0 +1,42 @@ +write(OrchestrationCapstoneTestCase::fixtures(), $components, $context); + $paths = [$components, $context]; + } + + return $paths; + } +} diff --git a/packages/orchestration/tests/Validation/OrchestrationValidatorTest.php b/packages/orchestration/tests/Validation/OrchestrationValidatorTest.php new file mode 100644 index 00000000..2f46035c --- /dev/null +++ b/packages/orchestration/tests/Validation/OrchestrationValidatorTest.php @@ -0,0 +1,49 @@ +register(SagaBuilder::saga('s')->step('a')->compensationCritical()->handler(static fn (): null => null)->add()->build()); + $workflows = new WorkflowRegistry; + $workflows->register(WorkflowBuilder::workflow('parent')->step('spawn')->childWorkflow('ghost')->add()->build()); + $workflows->register(WorkflowBuilder::workflow('typo')->step('a')->condition('input[')->handler(static fn (): null => null)->add()->build()); + + return [$sagas, new TccRegistry, $workflows]; +} + +it('reports unknown children and bad conditions as errors and critical steps without compensation as warnings', function (): void { + [$sagas, $tccs, $workflows] = orchestrationRegistries(); + + $issues = (new OrchestrationValidator(new ConditionEvaluator))->validate($sagas, $tccs, $workflows); + + expect(array_map(static fn (ValidationIssue $issue): string => $issue->severity, $issues))->toBe([ValidationIssue::WARNING, ValidationIssue::ERROR, ValidationIssue::ERROR]) + ->and($issues[0]->message)->toContain('Saga [s] step [a] is compensationCritical but has no compensation') + ->and($issues[1]->message)->toContain('Workflow [parent] step [spawn] starts child workflow [ghost], which is not registered') + ->and($issues[2]->message)->toContain('does not parse'); +}); + +it('refuses on errors, and on warnings only when asked', function (): void { + $logger = new RecordingLogger; + $validator = new OrchestrationValidator(new ConditionEvaluator); + $warning = [new ValidationIssue(ValidationIssue::WARNING, 'careful')]; + + $validator->enforce($warning, false, $logger); + + expect($logger->messages('warning'))->toBe(['Orchestration validation: careful']) + ->and(fn () => $validator->enforce($warning, true, $logger))->toThrow(TopologyValidationException::class, 'careful') + ->and(fn () => $validator->enforce([new ValidationIssue(ValidationIssue::ERROR, 'broken')], false, $logger))->toThrow(TopologyValidationException::class, 'broken'); +}); From d6b13dd3b0b5c3385d8ac4a2fe2bba0dfb14690b Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 21:39:57 -0700 Subject: [PATCH 34/58] fix(orchestration): derive a trigger execution's correlation id from the event id and carry the envelope headers, so a redelivered event is acknowledged instead of starting the definition again --- .../src/Boot/EventGatewayWiringPass.php | 9 +++-- .../src/Boot/OrchestrationRegistryPass.php | 30 +++++++++++++++-- .../orchestration/src/Core/Orchestrator.php | 12 ++++--- .../orchestration/src/Event/EventGateway.php | 33 +++++++++++++++---- .../tests/Capstone/CapstoneSagaTest.php | 25 ++++++++++++++ .../tests/Event/EventGatewayTest.php | 17 ++++++++++ skeleton/config/firefly.php | 5 ++- 7 files changed, 116 insertions(+), 15 deletions(-) diff --git a/packages/orchestration/src/Boot/EventGatewayWiringPass.php b/packages/orchestration/src/Boot/EventGatewayWiringPass.php index d8162fda..d9d1c735 100644 --- a/packages/orchestration/src/Boot/EventGatewayWiringPass.php +++ b/packages/orchestration/src/Boot/EventGatewayWiringPass.php @@ -15,7 +15,9 @@ * Subscribes every trigger event type the gateway knows on the application's EventPublisher (WiringPasses, * order 30 — after the registry pass filled the gateway). It runs in every process, web and worker alike, so a * queue-driven EDA transport delivers triggers to whichever process consumes the event (the - * EventListenerWiringPass rule). A delivered envelope's payload is the execution input. + * EventListenerWiringPass rule). A delivered envelope's payload is the execution input; its headers, plus the + * envelope's `eventId` and `eventType`, are the execution headers (what #[Header] and #[Headers] inject); and its + * correlation id is EventGateway::correlationIdFor() the event, so a redelivery is acknowledged, not re-run. */ final class EventGatewayWiringPass implements BootPass { @@ -50,7 +52,10 @@ public function run(BootContext $context): void foreach ($envelope->payload as $key => $value) { $payload[(string) $key] = $value; } - $gateway->route($eventType, $payload); + $headers = $envelope->headers; + $headers['eventId'] = $envelope->eventId; + $headers['eventType'] = $envelope->eventType; + $gateway->route($eventType, $payload, $headers, EventGateway::correlationIdFor($envelope->eventType, $envelope->eventId)); }); } } diff --git a/packages/orchestration/src/Boot/OrchestrationRegistryPass.php b/packages/orchestration/src/Boot/OrchestrationRegistryPass.php index a0553cc6..e850b7a4 100644 --- a/packages/orchestration/src/Boot/OrchestrationRegistryPass.php +++ b/packages/orchestration/src/Boot/OrchestrationRegistryPass.php @@ -9,6 +9,8 @@ use Firefly\Context\Boot\BootPass; use Firefly\Context\Boot\BootPhase; use Firefly\Orchestration\Core\Condition\ConditionEvaluator; +use Firefly\Orchestration\Core\Exception\InvalidExecutionStateException; +use Firefly\Orchestration\Core\ExecutionOutcome; use Firefly\Orchestration\Core\ExecutionPattern; use Firefly\Orchestration\Core\Orchestrator; use Firefly\Orchestration\Event\EventGateway; @@ -114,17 +116,41 @@ public function run(BootContext $context): void * * @return Closure(array): mixed */ + /** + * The executor the gateway calls for one delivered trigger. The gateway hands it a correlation id derived + * from the triggering event, so a redelivery of that event (every EDA transport delivers at least once) + * names an execution that already exists: the start is skipped before it runs, or, when two deliveries + * race, the engine refuses the loser under the execution lock. Either way the redelivery is acknowledged as + * a no-op (null) instead of starting the definition again or failing the delivery into a retry loop. + */ private static function starter(Container $container, ExecutionPattern $pattern, string $name): Closure { - return static function (array $payload) use ($container, $pattern, $name): mixed { + return static function (array $payload, array $headers = [], ?string $correlationId = null) use ($container, $pattern, $name): ?ExecutionOutcome { $input = []; foreach ($payload as $key => $value) { $input[(string) $key] = $value; } + $carried = []; + foreach ($headers as $key => $value) { + if (is_string($value)) { + $carried[(string) $key] = $value; + } + } /** @var Orchestrator $orchestrator */ $orchestrator = $container->make(Orchestrator::class); + if ($correlationId !== null && $orchestrator->find($correlationId) !== null) { + return null; + } + + try { + return $orchestrator->start($pattern, $name, $input, $carried, $correlationId); + } catch (InvalidExecutionStateException $refused) { + if ($correlationId !== null && $orchestrator->find($correlationId) !== null) { + return null; + } - return $orchestrator->start($pattern, $name, $input); + throw $refused; + } }; } } diff --git a/packages/orchestration/src/Core/Orchestrator.php b/packages/orchestration/src/Core/Orchestrator.php index 863cceed..77130731 100644 --- a/packages/orchestration/src/Core/Orchestrator.php +++ b/packages/orchestration/src/Core/Orchestrator.php @@ -21,15 +21,19 @@ public function __construct( ) {} /** + * A null $correlationId lets the engine mint a fresh one. A caller that needs a start to be idempotent (the + * event gateway under at-least-once delivery) passes a stable id instead: every engine refuses an id that + * already exists, under the execution lock, with an InvalidExecutionStateException. + * * @param array $input * @param array $headers */ - public function start(ExecutionPattern $pattern, string $name, array $input = [], array $headers = []): ExecutionOutcome + public function start(ExecutionPattern $pattern, string $name, array $input = [], array $headers = [], ?string $correlationId = null): ExecutionOutcome { return match ($pattern) { - ExecutionPattern::Saga => ExecutionOutcome::fromSaga($this->sagas->execute($name, $input, $headers)), - ExecutionPattern::Tcc => ExecutionOutcome::fromTcc($this->tccs->execute($name, $input, $headers)), - ExecutionPattern::Workflow => ExecutionOutcome::fromWorkflow($this->workflows->start($name, $input, $headers)), + ExecutionPattern::Saga => ExecutionOutcome::fromSaga($this->sagas->execute($name, $input, $headers, $correlationId)), + ExecutionPattern::Tcc => ExecutionOutcome::fromTcc($this->tccs->execute($name, $input, $headers, $correlationId)), + ExecutionPattern::Workflow => ExecutionOutcome::fromWorkflow($this->workflows->start($name, $input, $headers, $correlationId)), }; } diff --git a/packages/orchestration/src/Event/EventGateway.php b/packages/orchestration/src/Event/EventGateway.php index 76ec18eb..ccf24653 100644 --- a/packages/orchestration/src/Event/EventGateway.php +++ b/packages/orchestration/src/Event/EventGateway.php @@ -11,16 +11,23 @@ /** * Starts executions from EDA events (jafly EventGateway). OrchestrationRegistryPass registers one executor per * definition that declares a `triggerEventType`; EventGatewayWiringPass subscribes each event type on the - * application's EventPublisher, and a delivered envelope's payload becomes the execution input. One event type - * starts exactly one definition: a second registration refuses the boot rather than racing two executions. + * application's EventPublisher, and a delivered envelope's payload becomes the execution input, its headers (plus + * `eventId` and `eventType`) the execution headers. One event type starts exactly one definition: a second + * registration refuses the boot rather than racing two executions. + * + * Delivery is at least once on every transport (Kafka, RabbitMQ, the database outbox), so the execution's + * correlation id is derived from the triggering event (correlationIdFor()): a redelivered event names the + * execution its first delivery started, and the executor acknowledges it as a no-op instead of starting again. */ final class EventGateway { - /** @var array): mixed}> */ + /** @var array, array, ?string): mixed}> */ private array $triggers = []; /** - * @param Closure(array): mixed $executor + * The executor is called with the payload, the headers and the correlation id the execution must use. + * + * @param Closure(array, array, ?string): mixed $executor */ public function register(string $eventType, string $target, ExecutionPattern $pattern, Closure $executor): void { @@ -35,12 +42,26 @@ public function register(string $eventType, string $target, ExecutionPattern $pa /** * @param array $payload + * @param array $headers */ - public function route(string $eventType, array $payload): mixed + public function route(string $eventType, array $payload, array $headers = [], ?string $correlationId = null): mixed { $trigger = $this->triggers[$eventType] ?? null; - return $trigger === null ? null : ($trigger['executor'])($payload); + return $trigger === null ? null : ($trigger['executor'])($payload, $headers, $correlationId); + } + + /** + * The correlation id of the execution an event starts: a name-based (version 5 layout) UUID over the event + * type and the event id, so it is stable across redeliveries, fits the ledger's id column whatever the + * producer's id looks like, and never collides with the random ids code-started executions get. + */ + public static function correlationIdFor(string $eventType, string $eventId): string + { + $hash = sha1("firefly.orchestration.trigger\0{$eventType}\0{$eventId}"); + $variant = dechex((hexdec($hash[16]) & 0x3) | 0x8); + + return substr($hash, 0, 8).'-'.substr($hash, 8, 4).'-5'.substr($hash, 13, 3).'-'.$variant.substr($hash, 17, 3).'-'.substr($hash, 20, 12); } public function has(string $eventType): bool diff --git a/packages/orchestration/tests/Capstone/CapstoneSagaTest.php b/packages/orchestration/tests/Capstone/CapstoneSagaTest.php index 827fb44a..fae0ee69 100644 --- a/packages/orchestration/tests/Capstone/CapstoneSagaTest.php +++ b/packages/orchestration/tests/Capstone/CapstoneSagaTest.php @@ -2,10 +2,12 @@ declare(strict_types=1); +use Firefly\Eda\Bus\SubscriberRegistry; use Firefly\Eda\EventEnvelope; use Firefly\Eda\EventPublisher; use Firefly\Orchestration\Core\ExecutionPattern; use Firefly\Orchestration\Core\Orchestrator; +use Firefly\Orchestration\Event\EventGateway; use Firefly\Orchestration\Persistence\OrchestrationSchema; use Firefly\Orchestration\Saga\Registry\SagaRegistry; use Firefly\Orchestration\Tests\CapstoneFixtures\Ledger; @@ -46,3 +48,26 @@ expect($this->app()->make(Ledger::class)->lines)->toContain('reserve 9', 'charge 10 for r-9'); }); + +it('acknowledges a redelivered trigger event as a no-op instead of running the saga twice', function (): void { + /** @var OrchestrationCapstoneTestCase $this */ + $delivered = []; + $this->app()->make(EventPublisher::class)->subscribe('orders.placed', static function (EventEnvelope $envelope) use (&$delivered): void { + $delivered[] = $envelope; + }); + $this->app()->make(EventPublisher::class)->publish('orders', 'orders.placed', ['orderId' => 11, 'amount' => 20], ['tenant' => 'acme']); + expect($delivered)->toHaveCount(1); + $envelope = $delivered[0]; + + // The broker hands the same envelope over again (a consumer crashed before acking it). + $this->app()->make(SubscriberRegistry::class)->deliver($envelope); + + $correlationId = EventGateway::correlationIdFor('orders.placed', $envelope->eventId); + $state = $this->app()->make(Orchestrator::class)->find($correlationId); + + expect(array_values(array_filter($this->app()->make(Ledger::class)->lines, static fn (string $line): bool => str_starts_with($line, 'reserve') || str_starts_with($line, 'charge')))) + ->toBe(['reserve 11', 'charge 20 for r-11']) + ->and(DB::table(OrchestrationSchema::EXECUTIONS)->where('execution_name', 'checkout')->count())->toBe(1) + ->and($state?->status->value)->toBe('COMPLETED') + ->and($state?->headers)->toMatchArray(['tenant' => 'acme', 'eventId' => $envelope->eventId, 'eventType' => 'orders.placed']); +}); diff --git a/packages/orchestration/tests/Event/EventGatewayTest.php b/packages/orchestration/tests/Event/EventGatewayTest.php index e515245b..8958e33f 100644 --- a/packages/orchestration/tests/Event/EventGatewayTest.php +++ b/packages/orchestration/tests/Event/EventGatewayTest.php @@ -23,3 +23,20 @@ $gateway->register('orders.placed', 'a', ExecutionPattern::Saga, static fn (array $payload): null => null); $gateway->register('orders.placed', 'b', ExecutionPattern::Workflow, static fn (array $payload): null => null); })->throws(ConfigurationException::class, 'orders.placed'); + +it('hands the executor the headers and the correlation id of the delivery', function (): void { + $gateway = new EventGateway; + $gateway->register('orders.placed', 'order-saga', ExecutionPattern::Saga, static fn (array $payload, array $headers, ?string $correlationId): array => [$headers, $correlationId]); + + expect($gateway->route('orders.placed', [], ['eventId' => 'e-1'], 'c-1'))->toBe([['eventId' => 'e-1'], 'c-1']); +}); + +it('derives a stable, column-sized correlation id from the triggering event', function (): void { + $id = EventGateway::correlationIdFor('orders.placed', 'e-1'); + + expect($id)->toBe(EventGateway::correlationIdFor('orders.placed', 'e-1')) + ->and($id)->toMatch('/^[0-9a-f]{8}-[0-9a-f]{4}-5[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/') + ->and($id)->not->toBe(EventGateway::correlationIdFor('orders.placed', 'e-2')) + ->and($id)->not->toBe(EventGateway::correlationIdFor('orders.cancelled', 'e-1')) + ->and(strlen(EventGateway::correlationIdFor(str_repeat('very.long.event.type.', 20), str_repeat('x', 500))))->toBe(36); +}); diff --git a/skeleton/config/firefly.php b/skeleton/config/firefly.php index 5e42a305..6ba15f54 100644 --- a/skeleton/config/firefly.php +++ b/skeleton/config/firefly.php @@ -2662,7 +2662,10 @@ 'default-topic' => env('FIREFLY_ORCHESTRATION_EVENTS_TOPIC', ''), ], - // Start the definition whose triggerEventType matches an EDA event, with the event payload as input. + // Start the definition whose triggerEventType matches an EDA event, with the event payload as input and its + // headers (plus eventId and eventType) as headers. Delivery is at least once, so the execution's correlation id + // is derived from the event id: a redelivered event finds its execution already there and is acknowledged as a + // no-op instead of starting it again. 'event-gateway' => ['enabled' => true], // The REST surface (start, list, show, signal, suspend/resume/cancel, query, DLQ). Off by default; protect `path` From 8a596f68a7d3c3e7ca5d2da9906034f9fa4ccd7b Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 21:48:24 -0700 Subject: [PATCH 35/58] feat(orchestration): recovery that resumes or compensates killed executions, the scheduled tick and sweep, and scheduled orchestrations --- .../firefly-orchestration-components.php | 18 +++ .../cache/firefly-orchestration-context.php | 11 ++ .../src/Boot/OrchestrationSchedulePass.php | 84 +++++++++++ .../src/Boot/ScheduledExecutionsPass.php | 106 +++++++++++++ .../src/OrchestrationAutoConfiguration.php | 9 ++ .../src/OrchestrationWiringProvider.php | 7 +- .../Recovery/OrchestrationRecoveryTask.php | 18 +++ .../src/Recovery/OrchestrationTickTask.php | 24 +++ .../src/Recovery/RecoveryReport.php | 29 ++++ .../src/Recovery/RecoveryService.php | 141 ++++++++++++++++++ .../src/Workflow/Engine/WorkflowEngine.php | 20 +++ .../tests/Capstone/CapstoneSchedulingTest.php | 37 +++++ .../tests/CapstoneFixtures/CheckoutSaga.php | 2 + .../CrashRecoveryIntegrationTest.php | 34 +++++ .../tests/Recovery/CrashRecoveryTest.php | 86 +++++++++++ .../tests/Recovery/RecoveryServiceTest.php | 70 +++++++++ .../tests/Support/CrashScenarios.php | 102 +++++++++++++ .../tests/Support/TestEngines.php | 7 + .../tests/Support/crash-orchestration.php | 33 ++++ 19 files changed, 836 insertions(+), 2 deletions(-) create mode 100644 packages/orchestration/src/Boot/OrchestrationSchedulePass.php create mode 100644 packages/orchestration/src/Boot/ScheduledExecutionsPass.php create mode 100644 packages/orchestration/src/Recovery/OrchestrationRecoveryTask.php create mode 100644 packages/orchestration/src/Recovery/OrchestrationTickTask.php create mode 100644 packages/orchestration/src/Recovery/RecoveryReport.php create mode 100644 packages/orchestration/src/Recovery/RecoveryService.php create mode 100644 packages/orchestration/tests/Capstone/CapstoneSchedulingTest.php create mode 100644 packages/orchestration/tests/Integration/CrashRecoveryIntegrationTest.php create mode 100644 packages/orchestration/tests/Recovery/CrashRecoveryTest.php create mode 100644 packages/orchestration/tests/Recovery/RecoveryServiceTest.php create mode 100644 packages/orchestration/tests/Support/CrashScenarios.php create mode 100644 packages/orchestration/tests/Support/crash-orchestration.php diff --git a/packages/orchestration/cache/firefly-orchestration-components.php b/packages/orchestration/cache/firefly-orchestration-components.php index 835bdf22..a7f58a65 100644 --- a/packages/orchestration/cache/firefly-orchestration-components.php +++ b/packages/orchestration/cache/firefly-orchestration-components.php @@ -421,6 +421,24 @@ 1 => 'Firefly\\Orchestration\\Core\\Orchestrator', ], ], + 30 => [ + 'method' => 'recoveryService', + 'returns' => 'Firefly\\Orchestration\\Recovery\\RecoveryService', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 0, + 'lazy' => true, + 'dependencies' => [ + 0 => 'Firefly\\Orchestration\\OrchestrationSettings', + 1 => 'Firefly\\Orchestration\\Persistence\\ExecutionPersistenceProvider', + 2 => 'Firefly\\Orchestration\\Saga\\Engine\\SagaEngine', + 3 => 'Firefly\\Orchestration\\Tcc\\Engine\\TccEngine', + 4 => 'Firefly\\Orchestration\\Workflow\\Engine\\WorkflowEngine', + 5 => 'Firefly\\Orchestration\\Core\\Clock', + 6 => 'Illuminate\\Container\\Container', + ], + ], ], 'lazy' => false, 'dependencies' => [ diff --git a/packages/orchestration/cache/firefly-orchestration-context.php b/packages/orchestration/cache/firefly-orchestration-context.php index adedc364..2d2e08a1 100644 --- a/packages/orchestration/cache/firefly-orchestration-context.php +++ b/packages/orchestration/cache/firefly-orchestration-context.php @@ -346,6 +346,17 @@ ], ], ], + 30 => [ + 'method' => 'recoveryService', + 'conditions' => [ + 0 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnMissingBean', + 'args' => [ + 0 => 'Firefly\\Orchestration\\Recovery\\RecoveryService', + ], + ], + ], + ], ], ], ]; diff --git a/packages/orchestration/src/Boot/OrchestrationSchedulePass.php b/packages/orchestration/src/Boot/OrchestrationSchedulePass.php new file mode 100644 index 00000000..5ba67ded --- /dev/null +++ b/packages/orchestration/src/Boot/OrchestrationSchedulePass.php @@ -0,0 +1,84 @@ +container; + if (! class_exists(ScheduledManifest::class) || ! $container->bound(ScheduledManifest::class) || ! $container->bound(OrchestrationSettings::class)) { + return; + } + + /** @var OrchestrationSettings $settings */ + $settings = $container->make(OrchestrationSettings::class); + $durable = $settings->persistence->provider !== 'memory'; + $declaresWorkflows = $container->bound(OrchestrationManifest::class) && $this->declaresWorkflows($container->make(OrchestrationManifest::class)); + + $descriptors = []; + if ($settings->workflow->enabled && $settings->workflow->tickEnabled && ($durable || $declaresWorkflows)) { + $descriptors[] = new ScheduledDescriptor( + class: OrchestrationTickTask::class, + method: 'tick', + fixedRate: OrchestrationSettings::interval($settings->workflow->tickInterval), + lockName: OrchestrationTickTask::LOCK, + lockTtl: '300s', + ); + } + if ($settings->recovery->enabled && $durable) { + $descriptors[] = new ScheduledDescriptor( + class: OrchestrationRecoveryTask::class, + method: 'recover', + fixedRate: OrchestrationSettings::interval($settings->persistence->cleanupInterval), + lockName: OrchestrationRecoveryTask::LOCK, + lockTtl: '600s', + ); + } + if ($descriptors === []) { + return; + } + + /** @var ScheduledManifest $manifest */ + $manifest = $container->make(ScheduledManifest::class); + $container->instance(ScheduledManifest::class, new ScheduledManifest([...$manifest->all(), ...$descriptors])); + } + + private function declaresWorkflows(mixed $manifest): bool + { + return $manifest instanceof OrchestrationManifest && $manifest->toArray()['workflows'] !== []; + } +} diff --git a/packages/orchestration/src/Boot/ScheduledExecutionsPass.php b/packages/orchestration/src/Boot/ScheduledExecutionsPass.php new file mode 100644 index 00000000..18c3a366 --- /dev/null +++ b/packages/orchestration/src/Boot/ScheduledExecutionsPass.php @@ -0,0 +1,106 @@ +container; + if (! $container->bound(OrchestrationManifest::class)) { + return; + } + + /** @var OrchestrationManifest $manifest */ + $manifest = $container->make(OrchestrationManifest::class); + $schedules = $manifest->schedules(); + if ($schedules === []) { + return; + } + + $container->afterResolving(Schedule::class, static function (Schedule $schedule) use ($container, $schedules): void { + foreach ($schedules as $index => $scheduled) { + $event = $schedule->call(static function () use ($container, $scheduled, $index): void { + self::start($container, $scheduled, $index); + }); + + Cadence::of(new ScheduledDescriptor( + class: Orchestrator::class, + method: 'start', + cron: $scheduled->cron, + fixedRate: $scheduled->fixedRate, + fixedDelay: $scheduled->fixedDelay, + ))->apply($event); + + if ($scheduled->zone !== null) { + $event->timezone($scheduled->zone); + } + $event->description($scheduled->description !== '' ? $scheduled->description : "orchestration {$scheduled->pattern->value} {$scheduled->name}"); + } + }); + } + + private static function start(Container $container, ScheduledExecution $scheduled, int $index): void + { + $name = $scheduled->lockName($index); + $lock = null; + + try { + /** @var DistributedLock $lock */ + $lock = $container->make(DistributedLock::class); + if (! $lock->tryAcquire($name, 300.0)) { + $lock = null; + + return; // another node starts it this tick + } + + /** @var Orchestrator $orchestrator */ + $orchestrator = $container->make(Orchestrator::class); + $orchestrator->start($scheduled->pattern, $scheduled->name, $scheduled->input, ['x-firefly-scheduled' => 'true']); + } catch (Throwable $error) { + $logger = $container->bound(LoggerInterface::class) ? $container->make(LoggerInterface::class) : null; + $message = "Scheduled orchestration {$scheduled->pattern->value} [{$scheduled->name}] failed to start: {$error->getMessage()}"; + if ($logger instanceof LoggerInterface) { + $logger->error($message, ['exception' => $error]); + } else { + error_log($message); + } + } finally { + $lock?->release($name); + } + } +} diff --git a/packages/orchestration/src/OrchestrationAutoConfiguration.php b/packages/orchestration/src/OrchestrationAutoConfiguration.php index 468f8f11..c0e15968 100644 --- a/packages/orchestration/src/OrchestrationAutoConfiguration.php +++ b/packages/orchestration/src/OrchestrationAutoConfiguration.php @@ -45,6 +45,7 @@ use Firefly\Orchestration\Persistence\DatabaseExecutionPersistenceProvider; use Firefly\Orchestration\Persistence\ExecutionPersistenceProvider; use Firefly\Orchestration\Persistence\MemoryExecutionPersistenceProvider; +use Firefly\Orchestration\Recovery\RecoveryService; use Firefly\Orchestration\Saga\Compensation\CompensationErrorHandlerFactory; use Firefly\Orchestration\Saga\Compensation\SagaCompensator; use Firefly\Orchestration\Saga\Composition\CompositionDataFlowManager; @@ -387,6 +388,14 @@ public function deadLetterRetrier(DeadLetterService $deadLetters, Orchestrator $ return new DeadLetterRetrier($deadLetters, $orchestrator); } + #[Bean] + #[Lazy] + #[ConditionalOnMissingBean(RecoveryService::class)] + public function recoveryService(OrchestrationSettings $settings, ExecutionPersistenceProvider $persistence, SagaEngine $sagas, TccEngine $tccs, WorkflowEngine $workflows, Clock $clock, Container $app): RecoveryService + { + return new RecoveryService($persistence, $sagas, $tccs, $workflows, $settings->recovery, $settings->persistence, $clock, self::logger($app), $settings->backpressure->batchSize); + } + private static function cacheStore(Container $app, ?string $store): CacheRepository { if (! $app->bound(CacheFactory::class)) { diff --git a/packages/orchestration/src/OrchestrationWiringProvider.php b/packages/orchestration/src/OrchestrationWiringProvider.php index 45b37344..aec5c2e8 100644 --- a/packages/orchestration/src/OrchestrationWiringProvider.php +++ b/packages/orchestration/src/OrchestrationWiringProvider.php @@ -10,6 +10,8 @@ use Firefly\Context\Scan\AppScan; use Firefly\Orchestration\Boot\EventGatewayWiringPass; use Firefly\Orchestration\Boot\OrchestrationRegistryPass; +use Firefly\Orchestration\Boot\OrchestrationSchedulePass; +use Firefly\Orchestration\Boot\ScheduledExecutionsPass; use Firefly\Orchestration\Manifest\OrchestrationManifest; use Firefly\Orchestration\Scanner\OrchestrationScanner; use Illuminate\Contracts\Container\Container; @@ -22,7 +24,8 @@ * - binds the OrchestrationManifest behind a bound() guard: the compiled `orchestration.php` first, then an * in-process scan of `firefly.scan.paths`, then an empty manifest (the AppScan convention), lazily, so a * cached application that has firefly/cli pays nothing; - * - contributes the registry and gateway passes; + * - contributes the registry, gateway, schedule (the tick and the recovery sweep) and scheduled-execution + * (#[ScheduledSaga] / #[ScheduledTcc] / #[ScheduledWorkflow]) passes; * - publishes the migration (`--tag=firefly-orchestration-migrations`) and loads it while persistence or * the DLQ is `database`. */ @@ -52,7 +55,7 @@ public function register(): void */ public function passes(): array { - return [new OrchestrationRegistryPass, new EventGatewayWiringPass]; + return [new OrchestrationRegistryPass, new EventGatewayWiringPass, new OrchestrationSchedulePass, new ScheduledExecutionsPass]; } public function boot(): void diff --git a/packages/orchestration/src/Recovery/OrchestrationRecoveryTask.php b/packages/orchestration/src/Recovery/OrchestrationRecoveryTask.php new file mode 100644 index 00000000..179e0c82 --- /dev/null +++ b/packages/orchestration/src/Recovery/OrchestrationRecoveryTask.php @@ -0,0 +1,18 @@ +recovery->recover(); + } +} diff --git a/packages/orchestration/src/Recovery/OrchestrationTickTask.php b/packages/orchestration/src/Recovery/OrchestrationTickTask.php new file mode 100644 index 00000000..6f153aaf --- /dev/null +++ b/packages/orchestration/src/Recovery/OrchestrationTickTask.php @@ -0,0 +1,24 @@ +workflows->resumeDue()); + } +} diff --git a/packages/orchestration/src/Recovery/RecoveryReport.php b/packages/orchestration/src/Recovery/RecoveryReport.php new file mode 100644 index 00000000..0fef4225 --- /dev/null +++ b/packages/orchestration/src/Recovery/RecoveryReport.php @@ -0,0 +1,29 @@ + $stale + * @param list $resumed + * @param list $compensated + * @param array $errors + */ + public function __construct( + public array $stale = [], + public array $resumed = [], + public array $compensated = [], + public array $errors = [], + public int $cleaned = 0, + ) {} + + /** @return array{stale: list, resumed: list, compensated: list, errors: array, cleaned: int} */ + public function toArray(): array + { + return ['stale' => $this->stale, 'resumed' => $this->resumed, 'compensated' => $this->compensated, 'errors' => $this->errors, 'cleaned' => $this->cleaned]; + } +} diff --git a/packages/orchestration/src/Recovery/RecoveryService.php b/packages/orchestration/src/Recovery/RecoveryService.php new file mode 100644 index 00000000..e1f2ffc6 --- /dev/null +++ b/packages/orchestration/src/Recovery/RecoveryService.php @@ -0,0 +1,141 @@ + at most $limit (the batch size by default), newest update first */ + public function findStale(?int $limit = null): array + { + return $this->persistence->findStale($this->staleBefore(), $limit ?? $this->batchSize); + } + + public function cleanup(): int + { + return $this->persistence->cleanup($this->clock->now()->modify(sprintf('-%d seconds', (int) round($this->persistenceSettings->retentionPeriod)))); + } + + public function recover(): RecoveryReport + { + if (! $this->settings->enabled) { + return new RecoveryReport; + } + + $stale = []; + $resumed = []; + $compensated = []; + $errors = []; + + $before = $this->staleBefore(); + $seen = []; + do { + $fresh = 0; + $failed = false; + // Wide enough to reach past every row this sweep already saw, so a row that failed is not fetched + // in place of the next one. + foreach ($this->persistence->findStale($before, $this->batchSize + count($seen)) as $state) { + $id = $state->correlationId; + if (isset($seen[$id])) { + continue; + } + $seen[$id] = true; + $fresh++; + $stale[] = $id; + + try { + if ($this->settings->action === RecoverySettings::RESUME) { + $this->resume($state); + $resumed[] = $id; + } elseif ($this->settings->action === RecoverySettings::COMPENSATE) { + $this->compensate($state); + $compensated[] = $id; + } else { + $this->logger->warning("Orchestration recovery found {$state->pattern->value} [{$state->executionName}] ({$id}) stale in {$state->status->value} since ".ExecutionState::instant($state->updatedAt).'; firefly.orchestration.recovery.action is report, so it is left as it is.', ['correlationId' => $id]); + } + } catch (Throwable $error) { + $failed = true; + $errors[$id] = $error->getMessage(); + $this->logger->error("Orchestration recovery of [{$id}] failed: {$error->getMessage()}", ['correlationId' => $id, 'exception' => $error]); + } + + if (count($stale) - count($errors) >= $this->batchSize) { + break 2; + } + } + } while ($failed && $fresh > 0 && count($seen) < $this->batchSize * self::MAX_BATCHES); + + return new RecoveryReport($stale, $resumed, $compensated, $errors, $this->cleanup()); + } + + private function staleBefore(): DateTimeImmutable + { + return $this->clock->now()->modify(sprintf('-%d milliseconds', (int) round($this->settings->staleThreshold * 1000))); + } + + private function resume(ExecutionState $state): void + { + if ($state->pattern === ExecutionPattern::Saga) { + $this->sagas->resume($state->correlationId); + } elseif ($state->pattern === ExecutionPattern::Tcc) { + $this->tccs->resume($state->correlationId); + } else { + $this->workflows->recover($state->correlationId); + } + } + + private function compensate(ExecutionState $state): void + { + if ($state->pattern === ExecutionPattern::Saga) { + $this->sagas->compensate($state->correlationId); + } elseif ($state->pattern === ExecutionPattern::Tcc) { + $this->tccs->compensate($state->correlationId); + } else { + $this->workflows->compensate($state->correlationId); + } + } +} diff --git a/packages/orchestration/src/Workflow/Engine/WorkflowEngine.php b/packages/orchestration/src/Workflow/Engine/WorkflowEngine.php index ad962d92..48bdf2a0 100644 --- a/packages/orchestration/src/Workflow/Engine/WorkflowEngine.php +++ b/packages/orchestration/src/Workflow/Engine/WorkflowEngine.php @@ -255,6 +255,26 @@ public function resume(string $correlationId): ExecutionState }); } + /** + * Re-drives a workflow a crashed process left in a recoverable status (the recovery sweep's `resume` + * action): a PENDING one is begun, a RUNNING or COMPENSATING one is driven from its last checkpoint, so the + * step that was running when the process died runs again — make workflow steps idempotent. + */ + public function recover(string $correlationId): ExecutionState + { + return $this->support->lock->synchronized($correlationId, function () use ($correlationId): ExecutionState { + $state = $this->load($correlationId); + if (! $state->status->isRecoverable()) { + throw new InvalidExecutionStateException("Workflow execution [{$correlationId}] is {$state->status->value}; only an interrupted execution can be recovered."); + } + + $definition = $this->registry->get($state->executionName); + $context = ExecutionContext::fromState($state); + + return $state->status === ExecutionStatus::Pending ? $this->begin($definition, $context) : $this->drive($definition, $context); + }); + } + public function cancel(string $correlationId): ExecutionState { return $this->support->lock->synchronized($correlationId, function () use ($correlationId): ExecutionState { diff --git a/packages/orchestration/tests/Capstone/CapstoneSchedulingTest.php b/packages/orchestration/tests/Capstone/CapstoneSchedulingTest.php new file mode 100644 index 00000000..8a9c5f31 --- /dev/null +++ b/packages/orchestration/tests/Capstone/CapstoneSchedulingTest.php @@ -0,0 +1,37 @@ + "{$task->class}::{$task->method} {$task->fixedRate} {$task->lockName}", $this->app()->make(ScheduledManifest::class)->all()); + + expect($tasks)->toContain(OrchestrationTickTask::class.'::tick 60s firefly:orchestration:tick') + ->toContain(OrchestrationRecoveryTask::class.'::recover 3600s firefly:orchestration:recovery'); +}); + +it('wires #[ScheduledSaga] onto Laravel\'s schedule and starts the saga when it runs', function (): void { + /** @var OrchestrationCapstoneTestCase $this */ + $events = array_values(array_filter( + $this->app()->make(Schedule::class)->events(), + static fn (Event $event): bool => $event->description === 'nightly checkout', + )); + + expect($events)->toHaveCount(1) + ->and($events[0]->expression)->toBe('0 2 * * *'); + + $events[0]->run($this->app()); + + expect($this->app()->make(Ledger::class)->lines)->toContain('reserve 1', 'charge 5 for r-1'); +}); diff --git a/packages/orchestration/tests/CapstoneFixtures/CheckoutSaga.php b/packages/orchestration/tests/CapstoneFixtures/CheckoutSaga.php index 1ed80490..8be61857 100644 --- a/packages/orchestration/tests/CapstoneFixtures/CheckoutSaga.php +++ b/packages/orchestration/tests/CapstoneFixtures/CheckoutSaga.php @@ -11,6 +11,7 @@ use Firefly\Orchestration\Saga\Attributes\OnSagaError; use Firefly\Orchestration\Saga\Attributes\Saga; use Firefly\Orchestration\Saga\Attributes\SagaStep; +use Firefly\Orchestration\Saga\Attributes\ScheduledSaga; use Firefly\Orchestration\Saga\Attributes\StepEvent; use Firefly\Orchestration\Saga\Engine\SagaResult; use RuntimeException; @@ -18,6 +19,7 @@ #[Service] #[Saga(name: 'checkout', triggerEventType: 'orders.placed')] +#[ScheduledSaga(cron: '0 2 * * *', input: '{"orderId": 1, "amount": 5}', description: 'nightly checkout')] final class CheckoutSaga { public function __construct(private readonly Ledger $ledger) {} diff --git a/packages/orchestration/tests/Integration/CrashRecoveryIntegrationTest.php b/packages/orchestration/tests/Integration/CrashRecoveryIntegrationTest.php new file mode 100644 index 00000000..06718ded --- /dev/null +++ b/packages/orchestration/tests/Integration/CrashRecoveryIntegrationTest.php @@ -0,0 +1,34 @@ + [$driver, $variable, $extension]) { + foreach (['resume' => ['reserve', 'charge'], 'compensate' => ['reserve', 'release']] as $action => $expected) { + it("recovers a saga killed mid-step on {$server} with action {$action}", function () use ($driver, $variable, $action, $expected): void { + $connection = IntegrationDatabase::fresh(IntegrationDatabase::connection($driver, $variable)); + $marks = sys_get_temp_dir().'/firefly-orchestration-crash-'.bin2hex(random_bytes(6)).'.marks'; + $correlationId = "crash-{$action}-".bin2hex(random_bytes(4)); + + try { + expect(CrashScenarios::crash("{$driver}|{$variable}", 'saga', $correlationId, $marks))->toBe(70); + + $engines = TestEngines::create(['recovery' => ['action' => $action, 'stale-threshold' => '1m']], new DatabaseExecutionPersistenceProvider($connection), new DatabaseDeadLetterStore($connection)); + CrashScenarios::register($engines, $marks, crash: false); + $engines->clock->advance(120.0); + $engines->recovery->recover(); + + expect(CrashScenarios::marks($marks))->toBe($expected) + ->and($engines->persistence->findById($correlationId)?->status)->toBe($action === 'resume' ? ExecutionStatus::Completed : ExecutionStatus::Cancelled); + } finally { + $connection->disconnect(); + } + })->group('integration')->skip(IntegrationDatabase::skipReason($variable, $extension) !== null, (string) IntegrationDatabase::skipReason($variable, $extension)); + } +} diff --git a/packages/orchestration/tests/Recovery/CrashRecoveryTest.php b/packages/orchestration/tests/Recovery/CrashRecoveryTest.php new file mode 100644 index 00000000..dd42171a --- /dev/null +++ b/packages/orchestration/tests/Recovery/CrashRecoveryTest.php @@ -0,0 +1,86 @@ + $recovery + * @return array{0: TestEngines, 1: string} + */ +function orchestrationCrashed(string $scenario, string $correlationId, array $recovery): array +{ + $base = sys_get_temp_dir().'/firefly-orchestration-crash-'.bin2hex(random_bytes(6)); + $database = $base.'.sqlite'; + $marks = $base.'.marks'; + touch($database); + IntegrationDatabase::fresh(IntegrationDatabase::sqlite($database)); + + expect(CrashScenarios::crash("sqlite:{$database}", $scenario, $correlationId, $marks))->toBe(70); + + $connection = IntegrationDatabase::sqlite($database); + $engines = TestEngines::create(['recovery' => $recovery], new DatabaseExecutionPersistenceProvider($connection), new DatabaseDeadLetterStore($connection)); + CrashScenarios::register($engines, $marks, crash: false); + + return [$engines, $marks]; +} + +it('leaves a killed saga RUNNING mid-step, and resume finishes it without repeating the settled step', function (): void { + [$engines, $marks] = orchestrationCrashed('saga', 'crash-1', ['action' => 'resume', 'stale-threshold' => '1m']); + $crashed = $engines->persistence->findById('crash-1'); + $engines->clock->advance(120.0); + + $report = $engines->recovery->recover(); + + expect($crashed?->status)->toBe(ExecutionStatus::Running) + ->and($crashed?->stepStatus('reserve'))->toBe(StepStatus::Done) + ->and($crashed?->stepStatus('charge'))->toBe(StepStatus::Running) + ->and($report->stale)->toBe(['crash-1']) + ->and($report->resumed)->toBe(['crash-1']) + ->and(CrashScenarios::marks($marks))->toBe(['reserve', 'charge']) + ->and($engines->persistence->findById('crash-1')?->status)->toBe(ExecutionStatus::Completed); +}); + +it('compensates a killed saga when the recovery action is compensate', function (): void { + [$engines, $marks] = orchestrationCrashed('saga', 'crash-2', ['action' => 'compensate', 'stale-threshold' => '1m']); + $engines->clock->advance(120.0); + + $report = $engines->recovery->recover(); + + expect($report->compensated)->toBe(['crash-2']) + ->and(CrashScenarios::marks($marks))->toBe(['reserve', 'release']) + ->and($engines->persistence->findById('crash-2')?->status)->toBe(ExecutionStatus::Cancelled); +}); + +it('only reports a killed saga by default, and leaves a fresh one alone', function (): void { + [$engines] = orchestrationCrashed('saga', 'crash-3', ['stale-threshold' => '1h']); + + $fresh = $engines->recovery->recover(); + $engines->clock->advance(7200.0); + $stale = $engines->recovery->recover(); + + expect($fresh->stale)->toBe([]) + ->and($stale->stale)->toBe(['crash-3']) + ->and($stale->resumed)->toBe([]) + ->and($engines->persistence->findById('crash-3')?->status)->toBe(ExecutionStatus::Running) + ->and(implode("\n", $engines->logger->messages('warning')))->toContain('recovery.action is report'); +}); + +it('resumes a TCC killed in CONFIRMING and confirms only the participant that was interrupted', function (): void { + [$engines, $marks] = orchestrationCrashed('tcc', 'crash-4', ['action' => 'resume', 'stale-threshold' => '1m']); + $crashed = $engines->persistence->findById('crash-4'); + $engines->clock->advance(120.0); + + $engines->recovery->recover(); + + expect($crashed?->status)->toBe(ExecutionStatus::Confirming) + ->and($crashed?->stepStatus('debit'))->toBe(StepStatus::Confirmed) + ->and(CrashScenarios::marks($marks))->toBe(['try debit', 'try credit', 'confirm debit', 'confirm credit']) + ->and($engines->persistence->findById('crash-4')?->status)->toBe(ExecutionStatus::Confirmed); +}); diff --git a/packages/orchestration/tests/Recovery/RecoveryServiceTest.php b/packages/orchestration/tests/Recovery/RecoveryServiceTest.php new file mode 100644 index 00000000..89c34f89 --- /dev/null +++ b/packages/orchestration/tests/Recovery/RecoveryServiceTest.php @@ -0,0 +1,70 @@ + ['retention-period' => 'P1D']]); + $engines->persistence->save(ExecutionPersistenceContract::state('old', ExecutionPattern::Saga, ExecutionStatus::Completed, '2026-10-10T09:00:00+00:00')); + $engines->persistence->save(ExecutionPersistenceContract::state('recent', ExecutionPattern::Saga, ExecutionStatus::Failed, '2026-10-12T09:00:00+00:00')); + $engines->persistence->save(ExecutionPersistenceContract::state('parked', ExecutionPattern::Workflow, ExecutionStatus::Waiting, '2026-10-01T09:00:00+00:00')); + + $report = $engines->recovery->recover(); + + expect($report->cleaned)->toBe(1) + ->and($report->stale)->toBe([]) + ->and($engines->persistence->findById('old'))->toBeNull() + ->and($engines->persistence->findById('parked'))->not->toBeNull(); +}); + +it('recovers an interrupted workflow by driving it again', function (): void { + $engines = TestEngines::create(['recovery' => ['action' => 'resume', 'stale-threshold' => '1m']]); + $engines->workflowRegistry->register(WorkflowBuilder::workflow('w')->step('a')->handler(static fn (): string => 'a')->add()->build()); + $engines->persistence->save(ExecutionPersistenceContract::state('w-1', ExecutionPattern::Workflow, ExecutionStatus::Running, '2026-10-12T09:00:00+00:00', [ + 'executionName' => 'w', 'stepStatuses' => ['a' => 'RUNNING'], 'stepResults' => [], + ])); + + $report = $engines->recovery->recover(); + + expect($report->resumed)->toBe(['w-1']) + ->and($engines->persistence->findById('w-1')?->status)->toBe(ExecutionStatus::Completed); +}); + +it('records a recovery that fails instead of aborting the sweep', function (): void { + $engines = TestEngines::create(['recovery' => ['action' => 'resume', 'stale-threshold' => '1m']]); + $engines->persistence->save(ExecutionPersistenceContract::state('ghost', ExecutionPattern::Saga, ExecutionStatus::Running, '2026-10-12T09:00:00+00:00', ['executionName' => 'unregistered'])); + + $report = $engines->recovery->recover(); + + expect($report->errors)->toHaveKey('ghost') + ->and($report->errors['ghost'])->toContain('No saga [unregistered] is registered'); +}); + +it('looks past a full batch of executions it cannot recover, so they never starve an older one it can', function (): void { + $engines = TestEngines::create(['recovery' => ['action' => 'resume', 'stale-threshold' => '1m'], 'backpressure' => ['batch-size' => 2]]); + $engines->workflowRegistry->register(WorkflowBuilder::workflow('w')->step('a')->handler(static fn (): string => 'a')->add()->build()); + $engines->persistence->save(ExecutionPersistenceContract::state('w-old', ExecutionPattern::Workflow, ExecutionStatus::Running, '2026-10-12T07:00:00+00:00', [ + 'executionName' => 'w', 'stepStatuses' => ['a' => 'RUNNING'], 'stepResults' => [], + ])); + $engines->persistence->save(ExecutionPersistenceContract::state('ghost-1', ExecutionPattern::Saga, ExecutionStatus::Running, '2026-10-12T09:00:00+00:00', ['executionName' => 'unregistered'])); + $engines->persistence->save(ExecutionPersistenceContract::state('ghost-2', ExecutionPattern::Saga, ExecutionStatus::Running, '2026-10-12T09:30:00+00:00', ['executionName' => 'unregistered'])); + + $report = $engines->recovery->recover(); + + expect($report->stale)->toBe(['ghost-2', 'ghost-1', 'w-old']) + ->and(array_keys($report->errors))->toBe(['ghost-2', 'ghost-1']) + ->and($report->resumed)->toBe(['w-old']) + ->and($engines->persistence->findById('w-old')?->status)->toBe(ExecutionStatus::Completed); +}); + +it('does nothing while recovery is disabled', function (): void { + $engines = TestEngines::create(['recovery' => ['enabled' => false]]); + $engines->persistence->save(ExecutionPersistenceContract::state('old', ExecutionPattern::Saga, ExecutionStatus::Completed, '2026-01-01T09:00:00+00:00')); + + expect($engines->recovery->recover()->toArray())->toBe(['stale' => [], 'resumed' => [], 'compensated' => [], 'errors' => [], 'cleaned' => 0]); +}); diff --git a/packages/orchestration/tests/Support/CrashScenarios.php b/packages/orchestration/tests/Support/CrashScenarios.php new file mode 100644 index 00000000..192018e7 --- /dev/null +++ b/packages/orchestration/tests/Support/CrashScenarios.php @@ -0,0 +1,102 @@ +sagaRegistry->register(SagaBuilder::saga('crash-saga') + ->step('reserve')->handler(static function () use ($mark): array { + $mark('reserve'); + + return ['hold' => 'h-1']; + })->compensation(static function () use ($mark): void { + $mark('release'); + })->add() + ->step('charge')->dependsOn('reserve')->handler(static function () use ($mark, $crash): string { + if ($crash) { + exit(70); + } + $mark('charge'); + + return 'paid'; + })->add() + ->build()); + + $engines->tccRegistry->register(TccBuilder::tcc('crash-tcc')->retry(false) + ->participant('debit')->order(1) + ->tryWith(static function () use ($mark): string { + $mark('try debit'); + + return 'd'; + }) + ->confirmWith(static function () use ($mark): void { + $mark('confirm debit'); + }) + ->cancelWith(static function () use ($mark): void { + $mark('cancel debit'); + }) + ->add() + ->participant('credit')->order(2) + ->tryWith(static function () use ($mark): string { + $mark('try credit'); + + return 'c'; + }) + ->confirmWith(static function () use ($mark, $crash): void { + if ($crash) { + exit(70); + } + $mark('confirm credit'); + }) + ->cancelWith(static function () use ($mark): void { + $mark('cancel credit'); + }) + ->add() + ->build()); + } + + public static function start(TestEngines $engines, string $scenario, string $correlationId): void + { + if ($scenario === 'tcc') { + $engines->tccs->execute('crash-tcc', [], [], $correlationId); + + return; + } + + $engines->sagas->execute('crash-saga', ['orderId' => 1], [], $correlationId); + } + + /** Runs the scenario in a child PHP process that dies inside a step; returns its exit code (70). */ + public static function crash(string $target, string $scenario, string $correlationId, string $marks): int + { + $process = new Process([PHP_BINARY, __DIR__.'/crash-orchestration.php', $target, $scenario, $correlationId, $marks], timeout: 60); + $process->run(); + + return (int) $process->getExitCode(); + } + + /** @return list */ + public static function marks(string $file): array + { + return is_file($file) ? array_values(array_filter(explode("\n", (string) file_get_contents($file)), static fn (string $line): bool => $line !== '')) : []; + } +} diff --git a/packages/orchestration/tests/Support/TestEngines.php b/packages/orchestration/tests/Support/TestEngines.php index 707ae123..98b085a8 100644 --- a/packages/orchestration/tests/Support/TestEngines.php +++ b/packages/orchestration/tests/Support/TestEngines.php @@ -24,6 +24,7 @@ use Firefly\Orchestration\OrchestrationSettings; use Firefly\Orchestration\Persistence\ExecutionPersistenceProvider; use Firefly\Orchestration\Persistence\MemoryExecutionPersistenceProvider; +use Firefly\Orchestration\Recovery\RecoveryService; use Firefly\Orchestration\Saga\Compensation\CompensationErrorHandlerFactory; use Firefly\Orchestration\Saga\Compensation\SagaCompensator; use Firefly\Orchestration\Saga\Composition\CompositionDataFlowManager; @@ -102,6 +103,8 @@ final class TestEngines public readonly DeadLetterRetrier $retrier; + public readonly RecoveryService $recovery; + /** * @param array $config * @param array $resilience @@ -176,6 +179,10 @@ private function __construct(array $config, ?ExecutionPersistenceProvider $persi ); $this->orchestrator = new Orchestrator($this->sagas, $this->tccs, $this->workflows, $this->persistence); $this->retrier = new DeadLetterRetrier($this->support->deadLetters, $this->orchestrator); + $this->recovery = new RecoveryService( + $this->persistence, $this->sagas, $this->tccs, $this->workflows, + $this->settings->recovery, $this->settings->persistence, $this->clock, $this->logger, $this->settings->backpressure->batchSize, + ); } /** diff --git a/packages/orchestration/tests/Support/crash-orchestration.php b/packages/orchestration/tests/Support/crash-orchestration.php new file mode 100644 index 00000000..ceb105c6 --- /dev/null +++ b/packages/orchestration/tests/Support/crash-orchestration.php @@ -0,0 +1,33 @@ + $arguments */ +$arguments = array_values(array_map(static fn (mixed $argument): string => is_scalar($argument) ? (string) $argument : '', is_array($_SERVER['argv'] ?? null) ? $_SERVER['argv'] : [])); +[, $target, $scenario, $correlationId, $marks] = $arguments + ['', '', '', '', '']; + +if (str_starts_with($target, 'sqlite:')) { + $connection = IntegrationDatabase::sqlite(substr($target, 7)); +} else { + [$driver, $variable] = explode('|', $target, 2) + ['', '']; + $connection = IntegrationDatabase::connection($driver, $variable); +} + +$engines = TestEngines::create(persistence: new DatabaseExecutionPersistenceProvider($connection), deadLetters: new DatabaseDeadLetterStore($connection)); +CrashScenarios::register($engines, $marks, crash: true); +CrashScenarios::start($engines, $scenario, $correlationId); + +fwrite(STDERR, "the scenario did not crash\n"); +exit(1); From 8cd844554fbb929d1bd1ea7bc6e82181a2eba95a Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 21:54:35 -0700 Subject: [PATCH 36/58] fix(orchestration): finish an interrupted workflow compensation on recovery instead of driving the workflow forward --- .../src/Workflow/Engine/WorkflowEngine.php | 69 +++++++++++++-- .../tests/Recovery/RecoveryServiceTest.php | 84 +++++++++++++++++++ 2 files changed, 145 insertions(+), 8 deletions(-) diff --git a/packages/orchestration/src/Workflow/Engine/WorkflowEngine.php b/packages/orchestration/src/Workflow/Engine/WorkflowEngine.php index 48bdf2a0..3d668c18 100644 --- a/packages/orchestration/src/Workflow/Engine/WorkflowEngine.php +++ b/packages/orchestration/src/Workflow/Engine/WorkflowEngine.php @@ -11,6 +11,7 @@ use Firefly\Orchestration\Core\Exception\InvalidExecutionStateException; use Firefly\Orchestration\Core\Exception\OrchestrationException; use Firefly\Orchestration\Core\Exception\PatternDisabledException; +use Firefly\Orchestration\Core\Exception\StepTimeoutException; use Firefly\Orchestration\Core\ExecutionContext; use Firefly\Orchestration\Core\ExecutionPattern; use Firefly\Orchestration\Core\ExecutionStatus; @@ -51,6 +52,9 @@ final class WorkflowEngine /** The variable counting the ticks in a row that could not drive an execution. */ public const string BACKOFF_FAILURES = 'tickFailures'; + /** The variable compensate() records, so recover() finishes an interrupted compensation on request as one. */ + public const string COMPENSATION_REQUESTED = 'compensationRequested'; + private const int BACKOFF_BASE_SECONDS = 60; private const int BACKOFF_MAX_SECONDS = 3600; @@ -257,8 +261,11 @@ public function resume(string $correlationId): ExecutionState /** * Re-drives a workflow a crashed process left in a recoverable status (the recovery sweep's `resume` - * action): a PENDING one is begun, a RUNNING or COMPENSATING one is driven from its last checkpoint, so the - * step that was running when the process died runs again — make workflow steps idempotent. + * action): a PENDING one is begun, a RUNNING one is driven from its last checkpoint, so the step that was + * running when the process died runs again — make workflow steps idempotent. A COMPENSATING one is never + * driven forward, since its undo is already under way: the compensation is finished instead, over the steps + * still DONE, and the workflow ends the way the interrupted settlement would have ended it (see + * finishCompensation()). */ public function recover(string $correlationId): ExecutionState { @@ -271,7 +278,11 @@ public function recover(string $correlationId): ExecutionState $definition = $this->registry->get($state->executionName); $context = ExecutionContext::fromState($state); - return $state->status === ExecutionStatus::Pending ? $this->begin($definition, $context) : $this->drive($definition, $context); + return match ($state->status) { + ExecutionStatus::Pending => $this->begin($definition, $context), + ExecutionStatus::Compensating => $this->finishCompensation($definition, $context), + default => $this->drive($definition, $context), + }; }); } @@ -308,15 +319,57 @@ public function compensate(string $correlationId): ExecutionState $definition = $this->registry->get($state->executionName); $context = ExecutionContext::fromState($state); - $this->settle($correlationId, function () use ($definition, $context): void { - $this->cancelChildren($context); - $this->executor->compensate($definition, $context); - }); + // Checkpointed COMPENSATING with the request recorded before anything is undone, so a process that + // dies part-way leaves a row recover() finishes as a compensation on request, not as a failure. + $context->setVariable(self::COMPENSATION_REQUESTED, true); + $this->support->checkpoint($context, ExecutionStatus::Compensating); - return $this->finish($definition, $context, ExecutionStatus::Cancelled, 'compensated on request'); + return $this->compensateOnRequest($definition, $context); }); } + private function compensateOnRequest(WorkflowDefinition $workflow, ExecutionContext $context): ExecutionState + { + $this->settle($context->correlationId, function () use ($workflow, $context): void { + $this->cancelChildren($context); + $this->executor->compensate($workflow, $context); + }); + + return $this->finish($workflow, $context, ExecutionStatus::Cancelled, 'compensated on request'); + } + + /** + * Finishes the compensation of a workflow a crashed process left COMPENSATING. A compensation asked for + * through compensate() is finished the same way and ends CANCELLED. Otherwise a step failure (or the + * workflow's deadline) started it, and fail() settles it again from that failed step: the steps still DONE + * are compensated, the failure is dead-lettered and the workflow ends FAILED (TIMED_OUT for a passed + * deadline). Steps already COMPENSATED are not undone twice, and no PENDING step ever runs. + */ + private function finishCompensation(WorkflowDefinition $workflow, ExecutionContext $context): ExecutionState + { + if ($context->variable(self::COMPENSATION_REQUESTED) === true) { + return $this->compensateOnRequest($workflow, $context); + } + + $outcome = null; + foreach ($workflow->stepIds() as $stepId) { + if (in_array($context->stepStatus($stepId), [StepStatus::Failed, StepStatus::TimedOut], true)) { + $outcome = WorkflowRunOutcome::failed($stepId, $context->stepError($stepId) ?? new OrchestrationException("Workflow [{$workflow->id}] step [{$stepId}] failed.")); + + break; + } + } + + $deadline = $context->deadline(); + if ($outcome === null && $deadline !== null && $this->support->now() >= $deadline) { + $outcome = WorkflowRunOutcome::failed($context->waitingOn() ?? '', new StepTimeoutException("Workflow [{$workflow->id}] passed its timeoutMs deadline of {$workflow->timeoutMs} ms."), timedOut: true); + } + + $outcome ??= WorkflowRunOutcome::failed('', new OrchestrationException("Workflow [{$workflow->id}] was interrupted while compensating.")); + + return $this->settle($context->correlationId, fn (): ExecutionState => $this->fail($workflow, $context, $outcome)); + } + /** * @param list $arguments */ diff --git a/packages/orchestration/tests/Recovery/RecoveryServiceTest.php b/packages/orchestration/tests/Recovery/RecoveryServiceTest.php index 89c34f89..28cfc57e 100644 --- a/packages/orchestration/tests/Recovery/RecoveryServiceTest.php +++ b/packages/orchestration/tests/Recovery/RecoveryServiceTest.php @@ -4,9 +4,11 @@ use Firefly\Orchestration\Core\ExecutionPattern; use Firefly\Orchestration\Core\ExecutionStatus; +use Firefly\Orchestration\Core\StepStatus; use Firefly\Orchestration\Tests\Support\ExecutionPersistenceContract; use Firefly\Orchestration\Tests\Support\TestEngines; use Firefly\Orchestration\Workflow\Builder\WorkflowBuilder; +use Firefly\Orchestration\Workflow\Engine\WorkflowEngine; it('cleans up terminal executions past the retention period and never touches waiting ones', function (): void { $engines = TestEngines::create(['persistence' => ['retention-period' => 'P1D']]); @@ -35,6 +37,88 @@ ->and($engines->persistence->findById('w-1')?->status)->toBe(ExecutionStatus::Completed); }); +/** + * Workflow w: a -> b -> c, with compensations on a and b; every run and undo is logged. + * + * @param list $log + */ +function compensatingWorkflow(TestEngines $engines, array &$log): void +{ + $engines->workflowRegistry->register(WorkflowBuilder::workflow('w') + ->step('a')->handler(static function () use (&$log): string { + $log[] = 'run a'; + + return 'a'; + })->compensation(static function () use (&$log): void { + $log[] = 'undo a'; + })->add() + ->step('b')->dependsOn('a')->handler(static function () use (&$log): string { + $log[] = 'run b'; + + return 'b'; + })->compensation(static function () use (&$log): void { + $log[] = 'undo b'; + })->add() + ->step('c')->dependsOn('b')->handler(static function () use (&$log): string { + $log[] = 'run c'; + + return 'c'; + })->add() + ->build()); +} + +it('finishes a compensation on request a crash interrupted instead of driving the workflow forward', function (): void { + $engines = TestEngines::create(['recovery' => ['action' => 'resume', 'stale-threshold' => '1m']]); + $log = []; + compensatingWorkflow($engines, $log); + $engines->persistence->save(ExecutionPersistenceContract::state('w-1', ExecutionPattern::Workflow, ExecutionStatus::Compensating, '2026-10-12T09:00:00+00:00', [ + 'executionName' => 'w', 'stepStatuses' => ['a' => 'DONE', 'b' => 'COMPENSATED'], 'stepResults' => ['a' => 'a', 'b' => 'b'], + 'completionOrder' => ['a', 'b'], 'variables' => [WorkflowEngine::COMPENSATION_REQUESTED => true], + ])); + + $report = $engines->recovery->recover(); + $state = $engines->persistence->findById('w-1'); + + expect($report->resumed)->toBe(['w-1']) + ->and($log)->toBe(['undo a']) + ->and($state?->status)->toBe(ExecutionStatus::Cancelled) + ->and($state?->failureReason)->toBe('compensated on request') + ->and($state?->stepStatuses)->toBe(['a' => StepStatus::Compensated, 'b' => StepStatus::Compensated]); +}); + +it('records a compensation on request before undoing anything, so recovery can tell it from a failure', function (): void { + $engines = TestEngines::create(); + $log = []; + compensatingWorkflow($engines, $log); + $engines->workflows->start('w', [], [], 'w-1'); + + $state = $engines->workflows->compensate('w-1'); + + expect($state->status)->toBe(ExecutionStatus::Cancelled) + ->and($state->variables[WorkflowEngine::COMPENSATION_REQUESTED] ?? null)->toBeTrue() + ->and($log)->toBe(['run a', 'run b', 'run c', 'undo b', 'undo a']); +}); + +it('finishes the compensation of a failed step a crash interrupted and ends the workflow FAILED', function (): void { + $engines = TestEngines::create(['recovery' => ['action' => 'resume', 'stale-threshold' => '1m']]); + $log = []; + compensatingWorkflow($engines, $log); + $engines->persistence->save(ExecutionPersistenceContract::state('w-1', ExecutionPattern::Workflow, ExecutionStatus::Compensating, '2026-10-12T09:00:00+00:00', [ + 'executionName' => 'w', 'stepStatuses' => ['a' => 'DONE', 'b' => 'COMPENSATED', 'c' => 'FAILED'], 'stepResults' => ['a' => 'a', 'b' => 'b'], + 'completionOrder' => ['a', 'b'], 'stepErrors' => ['c' => 'card declined'], + ])); + + $report = $engines->recovery->recover(); + $state = $engines->persistence->findById('w-1'); + + expect($report->resumed)->toBe(['w-1']) + ->and($log)->toBe(['undo a']) + ->and($state?->status)->toBe(ExecutionStatus::Failed) + ->and($state?->failureReason)->toBe('card declined') + ->and($state?->stepStatuses)->toBe(['a' => StepStatus::Compensated, 'b' => StepStatus::Compensated, 'c' => StepStatus::Failed]) + ->and($engines->deadLetterStore->findByCorrelationId('w-1'))->toHaveCount(1); +}); + it('records a recovery that fails instead of aborting the sweep', function (): void { $engines = TestEngines::create(['recovery' => ['action' => 'resume', 'stale-threshold' => '1m']]); $engines->persistence->save(ExecutionPersistenceContract::state('ghost', ExecutionPattern::Saga, ExecutionStatus::Running, '2026-10-12T09:00:00+00:00', ['executionName' => 'unregistered'])); From eee42a91e262fbe347af325cf911053014c1cb72 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 22:02:46 -0700 Subject: [PATCH 37/58] fix(orchestration): re-check a stale execution under its lock before recovery acts, so a run that finished or checkpointed while the sweep waited is skipped instead of undone or driven twice --- .../firefly-orchestration-components.php | 5 +- .../src/OrchestrationAutoConfiguration.php | 4 +- .../src/Recovery/RecoveryReport.php | 12 +- .../src/Recovery/RecoveryService.php | 70 ++++++++++-- .../tests/Recovery/RecoveryServiceTest.php | 85 +++++++++++++- .../Support/RacingExecutionPersistence.php | 107 ++++++++++++++++++ .../tests/Support/TestEngines.php | 2 +- skeleton/config/firefly.php | 5 +- 8 files changed, 271 insertions(+), 19 deletions(-) create mode 100644 packages/orchestration/tests/Support/RacingExecutionPersistence.php diff --git a/packages/orchestration/cache/firefly-orchestration-components.php b/packages/orchestration/cache/firefly-orchestration-components.php index a7f58a65..5d669143 100644 --- a/packages/orchestration/cache/firefly-orchestration-components.php +++ b/packages/orchestration/cache/firefly-orchestration-components.php @@ -435,8 +435,9 @@ 2 => 'Firefly\\Orchestration\\Saga\\Engine\\SagaEngine', 3 => 'Firefly\\Orchestration\\Tcc\\Engine\\TccEngine', 4 => 'Firefly\\Orchestration\\Workflow\\Engine\\WorkflowEngine', - 5 => 'Firefly\\Orchestration\\Core\\Clock', - 6 => 'Illuminate\\Container\\Container', + 5 => 'Firefly\\Orchestration\\Core\\ExecutionLock', + 6 => 'Firefly\\Orchestration\\Core\\Clock', + 7 => 'Illuminate\\Container\\Container', ], ], ], diff --git a/packages/orchestration/src/OrchestrationAutoConfiguration.php b/packages/orchestration/src/OrchestrationAutoConfiguration.php index c0e15968..5b4653ce 100644 --- a/packages/orchestration/src/OrchestrationAutoConfiguration.php +++ b/packages/orchestration/src/OrchestrationAutoConfiguration.php @@ -391,9 +391,9 @@ public function deadLetterRetrier(DeadLetterService $deadLetters, Orchestrator $ #[Bean] #[Lazy] #[ConditionalOnMissingBean(RecoveryService::class)] - public function recoveryService(OrchestrationSettings $settings, ExecutionPersistenceProvider $persistence, SagaEngine $sagas, TccEngine $tccs, WorkflowEngine $workflows, Clock $clock, Container $app): RecoveryService + public function recoveryService(OrchestrationSettings $settings, ExecutionPersistenceProvider $persistence, SagaEngine $sagas, TccEngine $tccs, WorkflowEngine $workflows, ExecutionLock $lock, Clock $clock, Container $app): RecoveryService { - return new RecoveryService($persistence, $sagas, $tccs, $workflows, $settings->recovery, $settings->persistence, $clock, self::logger($app), $settings->backpressure->batchSize); + return new RecoveryService($persistence, $sagas, $tccs, $workflows, $lock, $settings->recovery, $settings->persistence, $clock, self::logger($app), $settings->backpressure->batchSize); } private static function cacheStore(Container $app, ?string $store): CacheRepository diff --git a/packages/orchestration/src/Recovery/RecoveryReport.php b/packages/orchestration/src/Recovery/RecoveryReport.php index 0fef4225..6bc916b2 100644 --- a/packages/orchestration/src/Recovery/RecoveryReport.php +++ b/packages/orchestration/src/Recovery/RecoveryReport.php @@ -4,7 +4,11 @@ namespace Firefly\Orchestration\Recovery; -/** What one recovery sweep found and did; `errors` maps a correlation id to why its recovery failed. */ +/** + * What one recovery sweep found and did; `errors` maps a correlation id to why its recovery failed. `skipped` + * lists the executions the sweep selected as stale that had changed by the time it held their lock (their + * process was alive after all) and were left alone; they are not in `stale`. + */ final readonly class RecoveryReport { /** @@ -12,6 +16,7 @@ * @param list $resumed * @param list $compensated * @param array $errors + * @param list $skipped */ public function __construct( public array $stale = [], @@ -19,11 +24,12 @@ public function __construct( public array $compensated = [], public array $errors = [], public int $cleaned = 0, + public array $skipped = [], ) {} - /** @return array{stale: list, resumed: list, compensated: list, errors: array, cleaned: int} */ + /** @return array{stale: list, resumed: list, compensated: list, errors: array, cleaned: int, skipped: list} */ public function toArray(): array { - return ['stale' => $this->stale, 'resumed' => $this->resumed, 'compensated' => $this->compensated, 'errors' => $this->errors, 'cleaned' => $this->cleaned]; + return ['stale' => $this->stale, 'resumed' => $this->resumed, 'compensated' => $this->compensated, 'errors' => $this->errors, 'cleaned' => $this->cleaned, 'skipped' => $this->skipped]; } } diff --git a/packages/orchestration/src/Recovery/RecoveryService.php b/packages/orchestration/src/Recovery/RecoveryService.php index e1f2ffc6..c4e86d54 100644 --- a/packages/orchestration/src/Recovery/RecoveryService.php +++ b/packages/orchestration/src/Recovery/RecoveryService.php @@ -6,6 +6,7 @@ use DateTimeImmutable; use Firefly\Orchestration\Core\Clock; +use Firefly\Orchestration\Core\ExecutionLock; use Firefly\Orchestration\Core\ExecutionPattern; use Firefly\Orchestration\Persistence\ExecutionPersistenceProvider; use Firefly\Orchestration\Persistence\ExecutionState; @@ -31,6 +32,19 @@ * execution behind them forever; the sweep asks for the next rows instead, as WorkflowEngine::resumeDue() does, * looking at no more than ten batches' worth of rows in one sweep. * WAITING and SUSPENDED workflows are idle by design and never stale. + * + * Rows are selected without a lock, so each one is checked again under its execution lock before anything is + * done to it: it must still be the row the sweep selected — same status, same last checkpoint. A long step + * leaves its row looking stale while its process is alive and holds the lock; the sweep waits for that lock, + * and by the time it has it the run may have finished COMPLETED or CONFIRMED, or checkpointed again. Such a row + * is reported `skipped` (no longer stale), never resumed and never compensated — the engines' compensate() + * accepts a COMPLETED or CONFIRMED execution on purpose, as the manual undo API, and without this check the + * sweep would undo a business transaction that succeeded. The engine calls run inside that same lock + * (ExecutionLock is re-entrant), so nothing can checkpoint the row between the check and the recovery. + * + * What no check can tell apart is a live process whose execution lock EXPIRED (`lock.ttl`) and that has not + * checkpointed for `recovery.stale-threshold`: from the store it looks exactly like a dead one. Keep every + * step shorter than `lock.ttl`, and `stale-threshold` well above both. */ final class RecoveryService { @@ -42,6 +56,7 @@ public function __construct( private readonly SagaEngine $sagas, private readonly TccEngine $tccs, private readonly WorkflowEngine $workflows, + private readonly ExecutionLock $lock, private readonly RecoverySettings $settings, private readonly PersistenceSettings $persistenceSettings, private readonly Clock $clock, @@ -70,6 +85,7 @@ public function recover(): RecoveryReport $resumed = []; $compensated = []; $errors = []; + $skipped = []; $before = $this->staleBefore(); $seen = []; @@ -85,19 +101,28 @@ public function recover(): RecoveryReport } $seen[$id] = true; $fresh++; - $stale[] = $id; try { - if ($this->settings->action === RecoverySettings::RESUME) { - $this->resume($state); - $resumed[] = $id; - } elseif ($this->settings->action === RecoverySettings::COMPENSATE) { - $this->compensate($state); - $compensated[] = $id; - } else { + if ($this->settings->action === RecoverySettings::REPORT) { + $stale[] = $id; $this->logger->warning("Orchestration recovery found {$state->pattern->value} [{$state->executionName}] ({$id}) stale in {$state->status->value} since ".ExecutionState::instant($state->updatedAt).'; firefly.orchestration.recovery.action is report, so it is left as it is.', ['correlationId' => $id]); + } elseif ($this->recoverIfUnchanged($state)) { + $stale[] = $id; + if ($this->settings->action === RecoverySettings::RESUME) { + $resumed[] = $id; + } else { + $compensated[] = $id; + } + } else { + // Not an error and not stale: its process is alive and moved it on. It is not counted + // against the batch, and findStale() no longer returns it. + $skipped[] = $id; + $this->logger->info("Orchestration recovery left [{$id}] alone: it changed after the sweep selected it as stale, so the process running it is still alive.", ['correlationId' => $id]); + + continue; } } catch (Throwable $error) { + $stale[] = $id; $failed = true; $errors[$id] = $error->getMessage(); $this->logger->error("Orchestration recovery of [{$id}] failed: {$error->getMessage()}", ['correlationId' => $id, 'exception' => $error]); @@ -109,7 +134,7 @@ public function recover(): RecoveryReport } } while ($failed && $fresh > 0 && count($seen) < $this->batchSize * self::MAX_BATCHES); - return new RecoveryReport($stale, $resumed, $compensated, $errors, $this->cleanup()); + return new RecoveryReport($stale, $resumed, $compensated, $errors, $this->cleanup(), $skipped); } private function staleBefore(): DateTimeImmutable @@ -117,6 +142,33 @@ private function staleBefore(): DateTimeImmutable return $this->clock->now()->modify(sprintf('-%d milliseconds', (int) round($this->settings->staleThreshold * 1000))); } + /** + * Recovers $selected under its execution lock, unless the row changed since the sweep selected it. + * + * @return bool false when the row is no longer the stale one the sweep saw (it was left alone) + */ + private function recoverIfUnchanged(ExecutionState $selected): bool + { + return $this->lock->synchronized($selected->correlationId, function () use ($selected): bool { + $current = $this->persistence->findById($selected->correlationId); + if ($current === null + || $current->pattern !== $selected->pattern + || $current->status !== $selected->status + || ! $current->status->isRecoverable() + || ExecutionState::instant($current->updatedAt) !== ExecutionState::instant($selected->updatedAt)) { + return false; + } + + if ($this->settings->action === RecoverySettings::RESUME) { + $this->resume($current); + } else { + $this->compensate($current); + } + + return true; + }); + } + private function resume(ExecutionState $state): void { if ($state->pattern === ExecutionPattern::Saga) { diff --git a/packages/orchestration/tests/Recovery/RecoveryServiceTest.php b/packages/orchestration/tests/Recovery/RecoveryServiceTest.php index 28cfc57e..025a0692 100644 --- a/packages/orchestration/tests/Recovery/RecoveryServiceTest.php +++ b/packages/orchestration/tests/Recovery/RecoveryServiceTest.php @@ -5,7 +5,11 @@ use Firefly\Orchestration\Core\ExecutionPattern; use Firefly\Orchestration\Core\ExecutionStatus; use Firefly\Orchestration\Core\StepStatus; +use Firefly\Orchestration\Persistence\ExecutionState; +use Firefly\Orchestration\Saga\Builder\SagaBuilder; +use Firefly\Orchestration\Tcc\Builder\TccBuilder; use Firefly\Orchestration\Tests\Support\ExecutionPersistenceContract; +use Firefly\Orchestration\Tests\Support\RacingExecutionPersistence; use Firefly\Orchestration\Tests\Support\TestEngines; use Firefly\Orchestration\Workflow\Builder\WorkflowBuilder; use Firefly\Orchestration\Workflow\Engine\WorkflowEngine; @@ -150,5 +154,84 @@ function compensatingWorkflow(TestEngines $engines, array &$log): void $engines = TestEngines::create(['recovery' => ['enabled' => false]]); $engines->persistence->save(ExecutionPersistenceContract::state('old', ExecutionPattern::Saga, ExecutionStatus::Completed, '2026-01-01T09:00:00+00:00')); - expect($engines->recovery->recover()->toArray())->toBe(['stale' => [], 'resumed' => [], 'compensated' => [], 'errors' => [], 'cleaned' => 0]); + expect($engines->recovery->recover()->toArray())->toBe(['stale' => [], 'resumed' => [], 'compensated' => [], 'errors' => [], 'cleaned' => 0, 'skipped' => []]); +}); + +it('leaves alone an execution that finished while the sweep waited for its lock, instead of undoing it', function (ExecutionPattern $pattern, ExecutionStatus $running, ExecutionStatus $finished): void { + $persistence = new RacingExecutionPersistence; + $engines = TestEngines::create(['recovery' => ['action' => 'compensate', 'stale-threshold' => '1m']], $persistence); + $undone = []; + $engines->sagaRegistry->register(SagaBuilder::saga('order') + ->step('reserve')->handler(static fn (): string => 'h')->compensation(static function () use (&$undone): void { + $undone[] = 'release'; + })->add() + ->build()); + $engines->tccRegistry->register(TccBuilder::tcc('order')->retry(false) + ->participant('reserve')->order(1) + ->tryWith(static fn (): string => 'h') + ->confirmWith(static function (): void {}) + ->cancelWith(static function () use (&$undone): void { + $undone[] = 'cancel'; + }) + ->add() + ->build()); + $selected = ExecutionPersistenceContract::state('x-1', $pattern, $running, '2026-10-12T09:00:00+00:00'); + $persistence->save($selected); + // The live run finishes between findStale() and the sweep's reload under the execution lock. + $persistence->changeOnNextFind(ExecutionPersistenceContract::state('x-1', $pattern, $finished, '2026-10-12T10:00:00+00:00', [ + 'stepStatuses' => ['reserve' => $pattern === ExecutionPattern::Tcc ? 'CONFIRMED' : 'DONE'], + ])); + + $report = $engines->recovery->recover(); + + expect($report->skipped)->toBe(['x-1']) + ->and($report->stale)->toBe([]) + ->and($report->compensated)->toBe([]) + ->and($report->errors)->toBe([]) + ->and($undone)->toBe([]) + ->and($persistence->findById('x-1')?->status)->toBe($finished); +})->with([ + 'saga' => [ExecutionPattern::Saga, ExecutionStatus::Running, ExecutionStatus::Completed], + 'tcc' => [ExecutionPattern::Tcc, ExecutionStatus::Confirming, ExecutionStatus::Confirmed], +]); + +it('leaves alone a workflow that checkpointed again while the sweep waited for its lock, instead of driving it twice', function (string $action): void { + $persistence = new RacingExecutionPersistence; + $engines = TestEngines::create(['recovery' => ['action' => $action, 'stale-threshold' => '1m']], $persistence); + $log = []; + compensatingWorkflow($engines, $log); + $row = static fn (array $statuses): array => [ + 'executionName' => 'w', 'stepStatuses' => $statuses, 'stepResults' => ['a' => 'a'], 'completionOrder' => ['a'], + ]; + $persistence->save(ExecutionPersistenceContract::state('w-1', ExecutionPattern::Workflow, ExecutionStatus::Running, '2026-10-12T09:00:00+00:00', $row(['a' => 'DONE', 'b' => 'RUNNING']))); + // Still RUNNING, but with a fresh checkpoint: the process driving it is alive. + $persistence->changeOnNextFind(ExecutionPersistenceContract::state('w-1', ExecutionPattern::Workflow, ExecutionStatus::Running, '2026-10-12T09:59:30+00:00', $row(['a' => 'DONE', 'b' => 'DONE', 'c' => 'RUNNING']))); + + $report = $engines->recovery->recover(); + $state = $persistence->findById('w-1'); + + expect($report->skipped)->toBe(['w-1']) + ->and($report->stale)->toBe([]) + ->and($report->resumed)->toBe([]) + ->and($report->compensated)->toBe([]) + ->and($log)->toBe([]) + ->and($state?->status)->toBe(ExecutionStatus::Running) + ->and($state === null ? null : ExecutionState::instant($state->updatedAt))->toBe(ExecutionState::instant(new DateTimeImmutable('2026-10-12T09:59:30+00:00'))); +})->with(['resume', 'compensate']); + +it('still recovers a stale execution the sweep finds unchanged under its lock', function (): void { + $persistence = new RacingExecutionPersistence; + $engines = TestEngines::create(['recovery' => ['action' => 'compensate', 'stale-threshold' => '1m']], $persistence); + $log = []; + compensatingWorkflow($engines, $log); + $persistence->save(ExecutionPersistenceContract::state('w-1', ExecutionPattern::Workflow, ExecutionStatus::Running, '2026-10-12T09:00:00+00:00', [ + 'executionName' => 'w', 'stepStatuses' => ['a' => 'DONE', 'b' => 'RUNNING'], 'stepResults' => ['a' => 'a'], 'completionOrder' => ['a'], + ])); + + $report = $engines->recovery->recover(); + + expect($report->compensated)->toBe(['w-1']) + ->and($report->skipped)->toBe([]) + ->and($log)->toBe(['undo a']) + ->and($persistence->findById('w-1')?->status)->toBe(ExecutionStatus::Cancelled); }); diff --git a/packages/orchestration/tests/Support/RacingExecutionPersistence.php b/packages/orchestration/tests/Support/RacingExecutionPersistence.php new file mode 100644 index 00000000..589b85f2 --- /dev/null +++ b/packages/orchestration/tests/Support/RacingExecutionPersistence.php @@ -0,0 +1,107 @@ + */ + private array $changes = []; + + private readonly MemoryExecutionPersistenceProvider $inner; + + public function __construct() + { + $this->inner = new MemoryExecutionPersistenceProvider; + } + + public function changeOnNextFind(ExecutionState $state): void + { + $this->changes[$state->correlationId] = $state; + } + + public function save(ExecutionState $state): void + { + $this->inner->save($state); + } + + public function findById(string $correlationId): ?ExecutionState + { + $change = $this->changes[$correlationId] ?? null; + if ($change !== null) { + unset($this->changes[$correlationId]); + $this->inner->save($change); + } + + return $this->inner->findById($correlationId); + } + + public function find(ExecutionCriteria $criteria): array + { + return $this->inner->find($criteria); + } + + public function findInFlight(int $limit = 100): array + { + return $this->inner->findInFlight($limit); + } + + public function findStale(DateTimeImmutable $before, int $limit = 100): array + { + return $this->inner->findStale($before, $limit); + } + + public function findDue(DateTimeImmutable $now, int $limit = 100): array + { + return $this->inner->findDue($now, $limit); + } + + public function findBySearchAttributes(array $criteria, int $limit = 100): array + { + return $this->inner->findBySearchAttributes($criteria, $limit); + } + + public function updateStatus(string $correlationId, ExecutionStatus $status, DateTimeImmutable $now): void + { + $this->inner->updateStatus($correlationId, $status, $now); + } + + public function delete(string $correlationId): bool + { + return $this->inner->delete($correlationId); + } + + public function cleanup(DateTimeImmutable $before): int + { + return $this->inner->cleanup($before); + } + + public function countInFlight(): int + { + return $this->inner->countInFlight(); + } + + public function countByStatus(): array + { + return $this->inner->countByStatus(); + } + + public function isHealthy(): bool + { + return $this->inner->isHealthy(); + } +} diff --git a/packages/orchestration/tests/Support/TestEngines.php b/packages/orchestration/tests/Support/TestEngines.php index 98b085a8..d6490ac6 100644 --- a/packages/orchestration/tests/Support/TestEngines.php +++ b/packages/orchestration/tests/Support/TestEngines.php @@ -180,7 +180,7 @@ private function __construct(array $config, ?ExecutionPersistenceProvider $persi $this->orchestrator = new Orchestrator($this->sagas, $this->tccs, $this->workflows, $this->persistence); $this->retrier = new DeadLetterRetrier($this->support->deadLetters, $this->orchestrator); $this->recovery = new RecoveryService( - $this->persistence, $this->sagas, $this->tccs, $this->workflows, + $this->persistence, $this->sagas, $this->tccs, $this->workflows, $this->support->lock, $this->settings->recovery, $this->settings->persistence, $this->clock, $this->logger, $this->settings->backpressure->batchSize, ); } diff --git a/skeleton/config/firefly.php b/skeleton/config/firefly.php index 6ba15f54..2da14792 100644 --- a/skeleton/config/firefly.php +++ b/skeleton/config/firefly.php @@ -2639,7 +2639,10 @@ | The recovery sweep finds executions a crashed process left RUNNING, COMPENSATING, TRYING, | CONFIRMING, CANCELING or PENDING for longer than `stale-threshold`, and `action`s them: report (log | only), resume (continue from the last checkpoint — make your steps idempotent) or compensate (undo - | what completed). WAITING and SUSPENDED workflows are idle by design and never touched. + | what completed). WAITING and SUSPENDED workflows are idle by design and never touched. Each row + | is checked again under its execution lock, and one that finished or checkpointed since the sweep + | selected it is skipped. A live process whose `lock.ttl` expired and that has not checkpointed for + | `stale-threshold` looks dead, so keep every step shorter than `lock.ttl` and this well above it. | Defaults: enabled true, 1h, report. */ 'recovery' => [ From 3e164ee24ca78849258c8116b6695ef72dd4db52 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 22:25:24 -0700 Subject: [PATCH 38/58] fix(orchestration): keep a timer that came due between its wait check and the checkpoint in wakeAt, so the tick still wakes that waiting workflow --- .../src/Core/ExecutionContext.php | 36 +++++++++++++++---- .../tests/Core/ExecutionContextTest.php | 28 +++++++++++++++ 2 files changed, 58 insertions(+), 6 deletions(-) diff --git a/packages/orchestration/src/Core/ExecutionContext.php b/packages/orchestration/src/Core/ExecutionContext.php index 3c9dd578..3ce48706 100644 --- a/packages/orchestration/src/Core/ExecutionContext.php +++ b/packages/orchestration/src/Core/ExecutionContext.php @@ -81,6 +81,14 @@ final class ExecutionContext /** @var array */ private array $timers = []; + /** + * The instant each timer was last checked (timerFired()) in THIS run. Not persisted: it only tells + * wakeAt() which due timers a check actually saw fire, and every run checks its own waits again. + * + * @var array + */ + private array $timerChecks = []; + private ?string $waitingOn = null; private ?string $parentCorrelationId = null; @@ -490,16 +498,22 @@ public function hasTimer(string $timerId): bool public function scheduleTimer(string $timerId, DateTimeImmutable $fireAt): void { $this->timers[$timerId] = $fireAt; + unset($this->timerChecks[$timerId]); } public function timerFired(string $timerId, DateTimeImmutable $now): bool { - return isset($this->timers[$timerId]) && $this->timers[$timerId] <= $now; + if (! isset($this->timers[$timerId])) { + return false; + } + $this->timerChecks[$timerId] = $now; + + return $this->timers[$timerId] <= $now; } public function removeTimer(string $timerId): void { - unset($this->timers[$timerId]); + unset($this->timers[$timerId], $this->timerChecks[$timerId]); } public function waitingOn(): ?string @@ -549,13 +563,23 @@ public function setDeadline(?DateTimeImmutable $deadline): void } /** - * The earliest PENDING instant: a timer already due is being handled by the run that checkpoints now (it - * fired, and the step is still waiting on something else — a signal of a WaitForAll), so counting it - * would make the tick wake this execution every minute for nothing. + * The earliest PENDING instant. A timer a check of this run saw fire is being handled by the run that + * checkpoints now (the step is still waiting on something else — a signal of a WaitForAll), and a due timer + * nothing in this run checked belongs to a run that did not get that far (a tick back-off), so counting + * either would make the tick wake this execution on every pass for nothing. A timer is judged against the + * instant it was CHECKED, not the checkpoint instant: a wait is evaluated a moment before the checkpoint is + * written, and a timer that came due in between was never seen firing — dropping it would leave a WAITING + * execution with no wake-up, which no tick ever looks at again. Its wakeAt is then already past, and the + * next tick drives it. */ private function wakeAt(DateTimeImmutable $now): ?DateTimeImmutable { - $candidates = array_values(array_filter($this->timers, static fn (DateTimeImmutable $at): bool => $at > $now)); + $candidates = []; + foreach ($this->timers as $timerId => $at) { + if ($at > ($this->timerChecks[$timerId] ?? $now)) { + $candidates[] = $at; + } + } if ($this->deadline !== null) { $candidates[] = $this->deadline; } diff --git a/packages/orchestration/tests/Core/ExecutionContextTest.php b/packages/orchestration/tests/Core/ExecutionContextTest.php index 7083afe4..e5c05ddb 100644 --- a/packages/orchestration/tests/Core/ExecutionContextTest.php +++ b/packages/orchestration/tests/Core/ExecutionContextTest.php @@ -78,3 +78,31 @@ expect(ExecutionState::instant($ctx->toState(ExecutionStatus::Waiting, $now)->wakeAt ?? $now))->toBe('2026-10-12T10:30:00.000000+00:00') ->and($ctx->toState(ExecutionStatus::Running, $now)->wakeAt)->toBeNull(); }); + +it('keeps a timer that came due after its wait was evaluated in the wake-up instant, so the tick still wakes the workflow', function (): void { + $evaluatedAt = new DateTimeImmutable('2026-10-12T10:00:00.000000+00:00'); + $ctx = ExecutionContext::start(ExecutionPattern::Workflow, 'w', [], [], 'c', $evaluatedAt); + $ctx->scheduleTimer('cool-off', $evaluatedAt->modify('+1 milliseconds')); + + // The engine evaluates the wait at one instant and checkpoints a little later: the timer is due by then, but + // nothing saw it fire, so dropping it would leave a WAITING row no tick ever looks at again. + expect($ctx->timerFired('cool-off', $evaluatedAt))->toBeFalse() + ->and(ExecutionState::instant($ctx->toState(ExecutionStatus::Waiting, $evaluatedAt->modify('+5 milliseconds'))->wakeAt ?? $evaluatedAt)) + ->toBe('2026-10-12T10:00:00.001000+00:00'); +}); + +it('leaves out of the wake-up instant a timer a wait saw fire, and a due timer nothing evaluated', function (): void { + $now = new DateTimeImmutable('2026-10-12T10:00:00+00:00'); + $ctx = ExecutionContext::start(ExecutionPattern::Workflow, 'w', [], [], 'c', $now); + $ctx->scheduleTimer('fired', $now->modify('-1 minute')); + $ctx->scheduleTimer('unseen', $now->modify('-2 minutes')); + $ctx->scheduleTimer('later', $now->modify('+10 minutes')); + + expect($ctx->timerFired('fired', $now))->toBeTrue() + ->and(ExecutionState::instant($ctx->toState(ExecutionStatus::Waiting, $now)->wakeAt ?? $now))->toBe('2026-10-12T10:10:00.000000+00:00'); + + // Rescheduling a timer forgets what an earlier check saw: the new instant is pending again. + $ctx->scheduleTimer('fired', $now->modify('+1 minute')); + expect($ctx->timerFired('fired', $now))->toBeFalse() + ->and(ExecutionState::instant($ctx->toState(ExecutionStatus::Waiting, $now->modify('+2 minutes'))->wakeAt ?? $now))->toBe('2026-10-12T10:01:00.000000+00:00'); +}); From b87a15396d17fcc20c55ca7ac11310a9e2036cd1 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 22:25:24 -0700 Subject: [PATCH 39/58] feat(orchestration): an orchestration actuator endpoint and health component, and commands to list, show, signal, tick and retry --- .../firefly-orchestration-components.php | 37 +++++ .../src/Actuator/OrchestrationEndpoint.php | 150 ++++++++++++++++++ .../Actuator/OrchestrationHealthIndicator.php | 66 ++++++++ .../src/Console/DeadLetterCommand.php | 37 +++++ .../src/Console/ListExecutionsCommand.php | 55 +++++++ .../src/Console/OptionReader.php | 27 ++++ .../src/Console/RetryDeadLetterCommand.php | 41 +++++ .../src/Console/ShowExecutionCommand.php | 33 ++++ .../src/Console/SignalWorkflowCommand.php | 50 ++++++ .../orchestration/src/Console/TickCommand.php | 43 +++++ .../src/OrchestrationWiringProvider.php | 18 +++ .../Actuator/OrchestrationEndpointTest.php | 41 +++++ .../OrchestrationHealthIndicatorTest.php | 39 +++++ .../Console/OrchestrationCommandsTest.php | 58 +++++++ .../Support/ActuatorCapstoneTestCase.php | 31 ++++ .../Support/OrchestrationCapstoneTestCase.php | 23 +++ 16 files changed, 749 insertions(+) create mode 100644 packages/orchestration/src/Actuator/OrchestrationEndpoint.php create mode 100644 packages/orchestration/src/Actuator/OrchestrationHealthIndicator.php create mode 100644 packages/orchestration/src/Console/DeadLetterCommand.php create mode 100644 packages/orchestration/src/Console/ListExecutionsCommand.php create mode 100644 packages/orchestration/src/Console/OptionReader.php create mode 100644 packages/orchestration/src/Console/RetryDeadLetterCommand.php create mode 100644 packages/orchestration/src/Console/ShowExecutionCommand.php create mode 100644 packages/orchestration/src/Console/SignalWorkflowCommand.php create mode 100644 packages/orchestration/src/Console/TickCommand.php create mode 100644 packages/orchestration/tests/Actuator/OrchestrationEndpointTest.php create mode 100644 packages/orchestration/tests/Actuator/OrchestrationHealthIndicatorTest.php create mode 100644 packages/orchestration/tests/Console/OrchestrationCommandsTest.php create mode 100644 packages/orchestration/tests/Support/ActuatorCapstoneTestCase.php diff --git a/packages/orchestration/cache/firefly-orchestration-components.php b/packages/orchestration/cache/firefly-orchestration-components.php index 5d669143..f3fe8be3 100644 --- a/packages/orchestration/cache/firefly-orchestration-components.php +++ b/packages/orchestration/cache/firefly-orchestration-components.php @@ -6,6 +6,43 @@ return [ 0 => [ + 'class' => 'Firefly\\Orchestration\\Actuator\\OrchestrationEndpoint', + 'stereotype' => 'component', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 0, + 'qualifier' => null, + 'interfaces' => [ + 0 => 'Firefly\\Actuator\\Endpoint\\ActuatorEndpoint', + ], + 'beans' => [ + ], + 'lazy' => false, + 'dependencies' => [ + 0 => 'Firefly\\Container\\Container', + ], + ], + 1 => [ + 'class' => 'Firefly\\Orchestration\\Actuator\\OrchestrationHealthIndicator', + 'stereotype' => 'component', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 0, + 'qualifier' => null, + 'interfaces' => [ + 0 => 'Firefly\\Actuator\\Health\\ConditionalHealthIndicator', + 1 => 'Firefly\\Actuator\\Health\\HealthIndicator', + ], + 'beans' => [ + ], + 'lazy' => false, + 'dependencies' => [ + 0 => 'Firefly\\Container\\Container', + ], + ], + 2 => [ 'class' => 'Firefly\\Orchestration\\OrchestrationAutoConfiguration', 'stereotype' => 'configuration', 'name' => null, diff --git a/packages/orchestration/src/Actuator/OrchestrationEndpoint.php b/packages/orchestration/src/Actuator/OrchestrationEndpoint.php new file mode 100644 index 00000000..0d0fb1f9 --- /dev/null +++ b/packages/orchestration/src/Actuator/OrchestrationEndpoint.php @@ -0,0 +1,150 @@ +beans->has(OrchestrationSettings::class); + } + + public function handle(EndpointRequest $request): ?EndpointResponse + { + if ($request->method !== 'GET' || ! $this->enabled()) { + return null; + } + + return match ($request->subPath[0] ?? null) { + null => EndpointResponse::json($this->overview()), + 'executions' => $this->executions($request), + 'dlq' => EndpointResponse::json(['deadLetters' => array_map(static fn (DeadLetterEntry $entry): array => $entry->toArray(), $this->deadLetters()->all(self::limit($request)))]), + default => null, + }; + } + + /** @return array */ + private function overview(): array + { + $settings = $this->get(OrchestrationSettings::class); + + $sagas = []; + foreach ($this->get(SagaRegistry::class)->all() as $saga) { + $sagas[$saga->name] = ['steps' => $saga->stepIds(), 'layers' => $saga->layers, 'triggerEventType' => $saga->triggerEventType, 'layerConcurrency' => $saga->layerConcurrency]; + } + $tccs = []; + foreach ($this->get(TccRegistry::class)->all() as $tcc) { + $tccs[$tcc->name] = ['participants' => $tcc->participantIds(), 'triggerEventType' => $tcc->triggerEventType]; + } + $workflows = []; + foreach ($this->get(WorkflowRegistry::class)->all() as $workflow) { + $workflows[$workflow->id] = [ + 'steps' => $workflow->stepIds(), 'layers' => $workflow->layers, 'triggerMode' => $workflow->triggerMode->value, + 'triggerEventType' => $workflow->triggerEventType, 'queries' => array_map('strval', array_keys($workflow->queries)), + ]; + } + + return [ + 'persistence' => $settings->persistence->provider, + 'compensationPolicy' => $settings->saga->compensationPolicy->value, + 'executions' => $this->get(ExecutionPersistenceProvider::class)->countByStatus(), + 'deadLetters' => $this->deadLetters()->count(), + 'sagas' => $sagas, + 'tccs' => $tccs, + 'workflows' => $workflows, + 'triggers' => $this->get(EventGateway::class)->describe(), + 'notes' => ['Steps of one topology layer run one after another; layerConcurrency, cpuBound and a workflow step\'s async are accepted for parity and have no effect.'], + ]; + } + + private function executions(EndpointRequest $request): EndpointResponse + { + $persistence = $this->get(ExecutionPersistenceProvider::class); + $id = $request->subPath[1] ?? null; + + if ($id !== null) { + $state = $persistence->findById($id); + + return $state === null + ? EndpointResponse::json(['error' => "No orchestration execution [{$id}] is persisted."], 404) + : EndpointResponse::json($state->toArray()); + } + + $query = $request->query; + $criteria = new ExecutionCriteria( + pattern: is_string($query['pattern'] ?? null) ? ExecutionPattern::tryFrom(strtoupper($query['pattern'])) : null, + status: is_string($query['status'] ?? null) ? ExecutionStatus::tryFrom(strtoupper($query['status'])) : null, + name: is_string($query['name'] ?? null) && $query['name'] !== '' ? $query['name'] : null, + limit: self::limit($request), + ); + + return EndpointResponse::json(['executions' => array_map(static fn (ExecutionState $state): array => [ + 'correlationId' => $state->correlationId, + 'name' => $state->executionName, + 'pattern' => $state->pattern->value, + 'status' => $state->status->value, + 'updatedAt' => ExecutionState::instant($state->updatedAt), + 'waitingOn' => $state->waitingOn, + 'failureReason' => $state->failureReason, + ], $persistence->find($criteria))]); + } + + private function deadLetters(): DeadLetterService + { + return $this->get(DeadLetterService::class); + } + + private static function limit(EndpointRequest $request): int + { + $limit = $request->query['limit'] ?? 50; + + return is_numeric($limit) ? max(1, min(500, (int) $limit)) : 50; + } + + /** + * @template T of object + * + * @param class-string $class + * @return T + */ + private function get(string $class): object + { + /** @var T */ + return $this->beans->get($class); + } +} diff --git a/packages/orchestration/src/Actuator/OrchestrationHealthIndicator.php b/packages/orchestration/src/Actuator/OrchestrationHealthIndicator.php new file mode 100644 index 00000000..7a9fa75c --- /dev/null +++ b/packages/orchestration/src/Actuator/OrchestrationHealthIndicator.php @@ -0,0 +1,66 @@ +beans->has(OrchestrationSettings::class)) { + return false; + } + + $settings = $this->beans->get(OrchestrationSettings::class); + + return $settings instanceof OrchestrationSettings && $settings->healthEnabled; + } + + public function health(): Health + { + /** @var OrchestrationSettings $settings */ + $settings = $this->beans->get(OrchestrationSettings::class); + /** @var ExecutionPersistenceProvider $persistence */ + $persistence = $this->beans->get(ExecutionPersistenceProvider::class); + + if (! $persistence->isHealthy()) { + return Health::down(['provider' => $settings->persistence->provider, 'error' => 'the execution store does not answer']); + } + + try { + /** @var RecoveryService $recovery */ + $recovery = $this->beans->get(RecoveryService::class); + /** @var DeadLetterService $deadLetters */ + $deadLetters = $this->beans->get(DeadLetterService::class); + + return Health::up([ + 'provider' => $settings->persistence->provider, + 'inFlight' => $persistence->countInFlight(), + 'stale' => $settings->recovery->enabled ? count($recovery->findStale()) : 0, + 'deadLetters' => $deadLetters->count(), + ]); + } catch (Throwable $error) { + return Health::down(['provider' => $settings->persistence->provider, 'error' => $error->getMessage()]); + } + } +} diff --git a/packages/orchestration/src/Console/DeadLetterCommand.php b/packages/orchestration/src/Console/DeadLetterCommand.php new file mode 100644 index 00000000..5cab436b --- /dev/null +++ b/packages/orchestration/src/Console/DeadLetterCommand.php @@ -0,0 +1,37 @@ +stringOption('limit'); + $limit = is_numeric($limit) ? max(1, (int) $limit) : 50; + $name = $this->stringOption('name'); + $entries = $name === null ? $deadLetters->all($limit) : $deadLetters->byExecutionName($name, $limit); + + $this->table(['Id', 'Execution', 'Correlation id', 'Step', 'Error', 'Retries', 'Created'], array_map(static fn (DeadLetterEntry $entry): array => [ + $entry->id, $entry->pattern->value.' '.$entry->executionName, $entry->correlationId, $entry->stepId ?? '', + $entry->errorMessage, (string) $entry->retryCount, ExecutionState::instant($entry->createdAt), + ], $entries)); + + return self::SUCCESS; + } +} diff --git a/packages/orchestration/src/Console/ListExecutionsCommand.php b/packages/orchestration/src/Console/ListExecutionsCommand.php new file mode 100644 index 00000000..12007c93 --- /dev/null +++ b/packages/orchestration/src/Console/ListExecutionsCommand.php @@ -0,0 +1,55 @@ +stringOption('pattern'); + $status = $this->stringOption('status'); + $parsedPattern = $pattern === null ? null : ExecutionPattern::tryFrom(strtoupper($pattern)); + $parsedStatus = $status === null ? null : ExecutionStatus::tryFrom(strtoupper($status)); + + if ($pattern !== null && $parsedPattern === null) { + $this->components->error("Unknown pattern [{$pattern}]: use saga, tcc or workflow."); + + return self::FAILURE; + } + if ($status !== null && $parsedStatus === null) { + $this->components->error("Unknown status [{$status}]."); + + return self::FAILURE; + } + + $limit = $this->stringOption('limit'); + $states = $persistence->find(new ExecutionCriteria($parsedPattern, $parsedStatus, $this->stringOption('name'), is_numeric($limit) ? max(1, (int) $limit) : 50)); + + $this->table(['Correlation id', 'Pattern', 'Name', 'Status', 'Updated', 'Waiting on'], array_map(static fn (ExecutionState $state): array => [ + $state->correlationId, $state->pattern->value, $state->executionName, $state->status->value, + ExecutionState::instant($state->updatedAt), $state->waitingOn ?? '', + ], $states)); + + return self::SUCCESS; + } +} diff --git a/packages/orchestration/src/Console/OptionReader.php b/packages/orchestration/src/Console/OptionReader.php new file mode 100644 index 00000000..d46d6966 --- /dev/null +++ b/packages/orchestration/src/Console/OptionReader.php @@ -0,0 +1,27 @@ +option($name); + + return is_string($value) && $value !== '' ? $value : null; + } + + private function stringArgument(string $name): ?string + { + $value = $this->argument($name); + + return is_string($value) && $value !== '' ? $value : null; + } +} diff --git a/packages/orchestration/src/Console/RetryDeadLetterCommand.php b/packages/orchestration/src/Console/RetryDeadLetterCommand.php new file mode 100644 index 00000000..bbabd663 --- /dev/null +++ b/packages/orchestration/src/Console/RetryDeadLetterCommand.php @@ -0,0 +1,41 @@ +` — start the dead letter's definition again with its input. */ +final class RetryDeadLetterCommand extends Command +{ + use OptionReader; + + protected $signature = 'firefly:orchestration:dlq:retry {id : the dead-letter id}'; + + protected $description = 'Retry an orchestration dead letter.'; + + public function handle(DeadLetterRetrier $retrier): int + { + try { + $outcome = $retrier->retry($this->stringArgument('id') ?? ''); + } catch (OrchestrationException $refused) { + $this->components->error($refused->getMessage()); + + return self::FAILURE; + } + + $line = "{$outcome->pattern->value} [{$outcome->name}] restarted as {$outcome->correlationId}: {$outcome->status->value}."; + if ($outcome->success) { + $this->components->info($line); + + return self::SUCCESS; + } + + $this->components->error($line); + + return self::FAILURE; + } +} diff --git a/packages/orchestration/src/Console/ShowExecutionCommand.php b/packages/orchestration/src/Console/ShowExecutionCommand.php new file mode 100644 index 00000000..d42c6684 --- /dev/null +++ b/packages/orchestration/src/Console/ShowExecutionCommand.php @@ -0,0 +1,33 @@ +` — one execution's full checkpoint as JSON. */ +final class ShowExecutionCommand extends Command +{ + use OptionReader; + + protected $signature = 'firefly:orchestration:show {id : the correlation id}'; + + protected $description = 'Show one persisted orchestration execution.'; + + public function handle(ExecutionPersistenceProvider $persistence): int + { + $id = $this->stringArgument('id') ?? ''; + $state = $persistence->findById($id); + if ($state === null) { + $this->components->error("No orchestration execution [{$id}] is persisted."); + + return self::FAILURE; + } + + $this->line((string) json_encode($state->toArray(), JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_PRESERVE_ZERO_FRACTION)); + + return self::SUCCESS; + } +} diff --git a/packages/orchestration/src/Console/SignalWorkflowCommand.php b/packages/orchestration/src/Console/SignalWorkflowCommand.php new file mode 100644 index 00000000..cb9d4282 --- /dev/null +++ b/packages/orchestration/src/Console/SignalWorkflowCommand.php @@ -0,0 +1,50 @@ + [--payload=JSON]` — deliver a signal to a workflow. */ +final class SignalWorkflowCommand extends Command +{ + use OptionReader; + + protected $signature = 'firefly:orchestration:signal + {id : the workflow correlation id} + {signal : the signal name} + {--payload= : the payload, as JSON}'; + + protected $description = 'Deliver a signal to a durable workflow.'; + + public function handle(WorkflowEngine $workflows): int + { + $payload = null; + $raw = $this->stringOption('payload'); + if ($raw !== null) { + try { + $payload = json_decode($raw, true, 512, JSON_THROW_ON_ERROR); + } catch (JsonException $error) { + $this->components->error("--payload is not JSON: {$error->getMessage()}"); + + return self::FAILURE; + } + } + + try { + $state = $workflows->signal($this->stringArgument('id') ?? '', $this->stringArgument('signal') ?? '', $payload); + } catch (OrchestrationException $refused) { + $this->components->error($refused->getMessage()); + + return self::FAILURE; + } + + $this->components->info("Execution {$state->correlationId} is now {$state->status->value}".($state->waitingOn === null ? '.' : " (waiting on {$state->waitingOn}).")); + + return self::SUCCESS; + } +} diff --git a/packages/orchestration/src/Console/TickCommand.php b/packages/orchestration/src/Console/TickCommand.php new file mode 100644 index 00000000..c255e9e9 --- /dev/null +++ b/packages/orchestration/src/Console/TickCommand.php @@ -0,0 +1,43 @@ +resumeDue()); + $this->components->info("Advanced {$advanced} waiting workflow(s)."); + + if ($this->option('no-recovery') === true || ! $settings->recovery->enabled) { + return self::SUCCESS; + } + + $report = $recovery->recover(); + $this->components->info(sprintf( + 'Recovery: %d stale, %d resumed, %d compensated, %d failed, %d cleaned up.', + count($report->stale), count($report->resumed), count($report->compensated), count($report->errors), $report->cleaned, + )); + foreach ($report->errors as $id => $error) { + $this->components->warn("{$id}: {$error}"); + } + + return $report->errors === [] ? self::SUCCESS : self::FAILURE; + } +} diff --git a/packages/orchestration/src/OrchestrationWiringProvider.php b/packages/orchestration/src/OrchestrationWiringProvider.php index aec5c2e8..54cc9842 100644 --- a/packages/orchestration/src/OrchestrationWiringProvider.php +++ b/packages/orchestration/src/OrchestrationWiringProvider.php @@ -12,6 +12,12 @@ use Firefly\Orchestration\Boot\OrchestrationRegistryPass; use Firefly\Orchestration\Boot\OrchestrationSchedulePass; use Firefly\Orchestration\Boot\ScheduledExecutionsPass; +use Firefly\Orchestration\Console\DeadLetterCommand; +use Firefly\Orchestration\Console\ListExecutionsCommand; +use Firefly\Orchestration\Console\RetryDeadLetterCommand; +use Firefly\Orchestration\Console\ShowExecutionCommand; +use Firefly\Orchestration\Console\SignalWorkflowCommand; +use Firefly\Orchestration\Console\TickCommand; use Firefly\Orchestration\Manifest\OrchestrationManifest; use Firefly\Orchestration\Scanner\OrchestrationScanner; use Illuminate\Contracts\Container\Container; @@ -26,6 +32,7 @@ * cached application that has firefly/cli pays nothing; * - contributes the registry, gateway, schedule (the tick and the recovery sweep) and scheduled-execution * (#[ScheduledSaga] / #[ScheduledTcc] / #[ScheduledWorkflow]) passes; + * - registers the firefly:orchestration:* commands (list, show, signal, dlq, dlq:retry, tick) in a console; * - publishes the migration (`--tag=firefly-orchestration-migrations`) and loads it while persistence or * the DLQ is `database`. */ @@ -60,6 +67,17 @@ public function passes(): array public function boot(): void { + if ($this->app->runningInConsole()) { + $this->commands([ + ListExecutionsCommand::class, + ShowExecutionCommand::class, + SignalWorkflowCommand::class, + DeadLetterCommand::class, + RetryDeadLetterCommand::class, + TickCommand::class, + ]); + } + $migrations = dirname(__DIR__).'/database/migrations'; $this->publishes([$migrations.'/'.self::MIGRATION => $this->app->databasePath('migrations/'.self::MIGRATION)], 'firefly-orchestration-migrations'); diff --git a/packages/orchestration/tests/Actuator/OrchestrationEndpointTest.php b/packages/orchestration/tests/Actuator/OrchestrationEndpointTest.php new file mode 100644 index 00000000..3a2be0fa --- /dev/null +++ b/packages/orchestration/tests/Actuator/OrchestrationEndpointTest.php @@ -0,0 +1,41 @@ +app()->make(Orchestrator::class)->start(ExecutionPattern::Saga, 'checkout', ['orderId' => 1, 'amount' => 500]); + + $overview = $this->getJson('/actuator/orchestration')->assertOk(); + + expect($overview->json('persistence'))->toBe('database') + ->and($overview->json('sagas.checkout.layers'))->toBe([['reserve'], ['charge']]) + ->and($overview->json('sagas.checkout.triggerEventType'))->toBe('orders.placed') + ->and($overview->json('tccs.transfer.participants'))->toBe(['debit', 'credit']) + ->and($overview->json('workflows.onboarding.queries'))->toBe(['progress']) + ->and($overview->json('executions.FAILED'))->toBe(1) + ->and($overview->json('deadLetters'))->toBe(1) + ->and($overview->json('triggers.0.eventType'))->toBe('orders.placed') + ->and($overview->json('notes.0'))->toBeString()->toContain('run one after another'); +}); + +it('lists and shows executions and dead letters', function (): void { + /** @var ActuatorCapstoneTestCase $this */ + $outcome = $this->app()->make(Orchestrator::class)->start(ExecutionPattern::Saga, 'checkout', ['orderId' => 2, 'amount' => 10]); + + $list = $this->getJson('/actuator/orchestration/executions?pattern=SAGA&status=COMPLETED')->assertOk(); + $show = $this->getJson('/actuator/orchestration/executions/'.$outcome->correlationId)->assertOk(); + + expect($list->json('executions.0.correlationId'))->toBe($outcome->correlationId) + ->and($show->json('status'))->toBe('COMPLETED') + ->and($show->json('report.steps.charge.status'))->toBe('DONE'); + + $this->getJson('/actuator/orchestration/executions/nope')->assertNotFound(); + $this->getJson('/actuator/orchestration/dlq')->assertOk()->assertJson(['deadLetters' => []]); +}); diff --git a/packages/orchestration/tests/Actuator/OrchestrationHealthIndicatorTest.php b/packages/orchestration/tests/Actuator/OrchestrationHealthIndicatorTest.php new file mode 100644 index 00000000..3cbc2543 --- /dev/null +++ b/packages/orchestration/tests/Actuator/OrchestrationHealthIndicatorTest.php @@ -0,0 +1,39 @@ +getJson('/actuator/health'); + + expect($health->json('components.orchestration.status'))->toBe('UP') + ->and($health->json('components.orchestration.details'))->toBe(['provider' => 'database', 'inFlight' => 0, 'stale' => 0, 'deadLetters' => 0]); +}); + +it('reports the component DOWN when the execution store does not answer', function (): void { + /** @var ActuatorCapstoneTestCase $this */ + Schema::drop(OrchestrationSchema::EXECUTIONS); + + $health = $this->getJson('/actuator/health'); + + expect($health->json('components.orchestration.status'))->toBe('DOWN') + ->and($health->json('components.orchestration.details'))->toBe(['provider' => 'database', 'error' => 'the execution store does not answer']); +}); + +it('counts open executions and dead letters in the component details', function (): void { + /** @var ActuatorCapstoneTestCase $this */ + $this->app()->make(WorkflowEngine::class)->start('onboarding', ['email' => 'ada@example.com'], correlationId: 'health-1'); + $this->app()->make(Orchestrator::class)->start(ExecutionPattern::Saga, 'checkout', ['orderId' => 5, 'amount' => 500]); + + expect($this->getJson('/actuator/health')->json('components.orchestration.details')) + ->toBe(['provider' => 'database', 'inFlight' => 1, 'stale' => 0, 'deadLetters' => 1]); +}); diff --git a/packages/orchestration/tests/Console/OrchestrationCommandsTest.php b/packages/orchestration/tests/Console/OrchestrationCommandsTest.php new file mode 100644 index 00000000..d7fe4f3c --- /dev/null +++ b/packages/orchestration/tests/Console/OrchestrationCommandsTest.php @@ -0,0 +1,58 @@ +app()->make(Orchestrator::class)->start(ExecutionPattern::Saga, 'checkout', ['orderId' => 3, 'amount' => 10]); + + expect($this->command('firefly:orchestration:list', ['--pattern' => 'saga']))->toBe(0) + ->and($this->commandOutput())->toContain($outcome->correlationId)->toContain('COMPLETED') + ->and($this->command('firefly:orchestration:show', ['id' => $outcome->correlationId]))->toBe(0) + ->and($this->commandOutput())->toContain('"executionName": "checkout"') + ->and($this->command('firefly:orchestration:show', ['id' => 'nope']))->toBe(1) + ->and($this->commandOutput())->toContain('No orchestration execution [nope] is persisted.') + ->and($this->command('firefly:orchestration:list', ['--pattern' => 'nonsense']))->toBe(1) + ->and($this->commandOutput())->toContain('Unknown pattern [nonsense]'); +}); + +it('signals a waiting workflow and ticks its timer', function (): void { + /** @var OrchestrationCapstoneTestCase $this */ + $this->app()->make(WorkflowEngine::class)->start('onboarding', ['email' => 'grace@example.com'], correlationId: 'cmd-1'); + + expect($this->command('firefly:orchestration:signal', ['id' => 'cmd-1', 'signal' => 'email-verified', '--payload' => '{"token":"t"}']))->toBe(0) + ->and($this->commandOutput())->toContain('WAITING'); + usleep(5_000); + expect($this->command('firefly:orchestration:tick'))->toBe(0) + ->and($this->commandOutput())->toContain('Advanced 1 waiting workflow'); + + expect($this->app()->make(WorkflowEngine::class)->find('cmd-1')?->status)->toBe(ExecutionStatus::Completed) + ->and($this->command('firefly:orchestration:signal', ['id' => 'cmd-1', 'signal' => 'x', '--payload' => '{']))->toBe(1) + ->and($this->commandOutput())->toContain('--payload is not JSON'); +}); + +it('lists dead letters and retries one', function (): void { + /** @var OrchestrationCapstoneTestCase $this */ + $this->app()->make(Orchestrator::class)->start(ExecutionPattern::Saga, 'checkout', ['orderId' => 4, 'amount' => 500]); + $entry = $this->app()->make(DeadLetterService::class)->all()[0]; + + expect($this->command('firefly:orchestration:dlq'))->toBe(0) + ->and($this->commandOutput())->toContain($entry->id)->toContain('amount 500 exceeds the limit') + ->and($this->command('firefly:orchestration:dlq', ['--name' => 'onboarding']))->toBe(0) + ->and($this->commandOutput())->not->toContain($entry->id) + ->and($this->command('firefly:orchestration:dlq:retry', ['id' => $entry->id]))->toBe(1) + ->and($this->commandOutput())->toContain('FAILED'); + + expect($this->app()->make(DeadLetterService::class)->all()[0]->retryCount)->toBe(1) + ->and($this->app()->make(Ledger::class)->lines)->toContain('reserve 4'); +}); diff --git a/packages/orchestration/tests/Support/ActuatorCapstoneTestCase.php b/packages/orchestration/tests/Support/ActuatorCapstoneTestCase.php new file mode 100644 index 00000000..0905ba1a --- /dev/null +++ b/packages/orchestration/tests/Support/ActuatorCapstoneTestCase.php @@ -0,0 +1,31 @@ + */ + protected function fireflyProviders(): array + { + return [ValidationServiceProvider::class, WebServiceProvider::class, ActuatorServiceProvider::class, ActuatorWiringProvider::class, ...parent::fireflyProviders()]; + } + + /** @return array */ + protected function configOverrides(): array + { + return [ + ...parent::configOverrides(), + 'firefly.management.enabled' => true, + 'firefly.management.endpoints.web.exposure.include' => 'health,orchestration', + 'firefly.management.endpoint.health.show-details' => 'always', + ]; + } +} diff --git a/packages/orchestration/tests/Support/OrchestrationCapstoneTestCase.php b/packages/orchestration/tests/Support/OrchestrationCapstoneTestCase.php index 0856e10c..b3a7999e 100644 --- a/packages/orchestration/tests/Support/OrchestrationCapstoneTestCase.php +++ b/packages/orchestration/tests/Support/OrchestrationCapstoneTestCase.php @@ -15,6 +15,7 @@ use Firefly\Scheduling\SchedulingServiceProvider; use Firefly\Scheduling\SchedulingWiringProvider; use Firefly\Testing\FireflyDatabaseTestCase; +use Illuminate\Contracts\Console\Kernel; use Illuminate\Foundation\Application; use Illuminate\Support\Facades\DB; @@ -70,4 +71,26 @@ protected function setUp(): void OrchestrationSchema::create(DB::connection()->getSchemaBuilder()); } + + /** + * Runs an artisan command through the console Kernel and returns its exit code; commandOutput() is everything + * it printed. Kernel::call() rather than $this->artisan(), which is declared `PendingCommand|int` and matches + * each expected substring against one write at a time (one table row satisfies only the first substring). + * + * @param array $parameters + */ + public function command(string $name, array $parameters = []): int + { + return $this->kernel()->call($name, $parameters); + } + + public function commandOutput(): string + { + return $this->kernel()->output(); + } + + private function kernel(): Kernel + { + return $this->app()->make(Kernel::class); + } } From 15879d3474982cfaff0929a67e07f794807d3dd1 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 22:31:16 -0700 Subject: [PATCH 40/58] feat(orchestration): an opt-in REST surface to start, inspect, signal and retry executions --- .../src/Boot/OrchestrationRouteRegistrar.php | 74 ++++++ .../src/Http/OrchestrationController.php | 220 ++++++++++++++++++ .../src/OrchestrationWiringProvider.php | 6 +- .../Http/OrchestrationRestDisabledTest.php | 16 ++ .../tests/Http/OrchestrationRestTest.php | 91 ++++++++ .../tests/Support/RestCapstoneTestCase.php | 19 ++ 6 files changed, 424 insertions(+), 2 deletions(-) create mode 100644 packages/orchestration/src/Boot/OrchestrationRouteRegistrar.php create mode 100644 packages/orchestration/src/Http/OrchestrationController.php create mode 100644 packages/orchestration/tests/Http/OrchestrationRestDisabledTest.php create mode 100644 packages/orchestration/tests/Http/OrchestrationRestTest.php create mode 100644 packages/orchestration/tests/Support/RestCapstoneTestCase.php diff --git a/packages/orchestration/src/Boot/OrchestrationRouteRegistrar.php b/packages/orchestration/src/Boot/OrchestrationRouteRegistrar.php new file mode 100644 index 00000000..3252dd7c --- /dev/null +++ b/packages/orchestration/src/Boot/OrchestrationRouteRegistrar.php @@ -0,0 +1,74 @@ +get(` argument + * as a Config key spelled out of sight. + */ +final class OrchestrationRouteRegistrar implements BootPass +{ + public const string ROUTE_PREFIX = 'firefly.orchestration.'; + + public function phase(): BootPhase + { + return BootPhase::WiringPasses; + } + + public function order(): int + { + return 65; + } + + public function run(BootContext $context): void + { + $container = $context->container; + if (! $container->bound(OrchestrationSettings::class)) { + return; + } + + /** @var OrchestrationSettings $settings */ + $settings = $container->make(OrchestrationSettings::class); + if (! $settings->rest->enabled) { + return; + } + + /** @var Router $router */ + $router = $container->make('router'); + $base = $settings->rest->path; + $controller = static function () use ($container): OrchestrationController { + /** @var OrchestrationController */ + return $container->make(OrchestrationController::class); + }; + + $router->post($base.'/sagas/{name}', static fn (Request $request, string $name): Response => $controller()->startSaga($request, $name))->name(self::ROUTE_PREFIX.'start-saga'); + $router->post($base.'/tcc/{name}', static fn (Request $request, string $name): Response => $controller()->startTcc($request, $name))->name(self::ROUTE_PREFIX.'start-tcc'); + $router->post($base.'/workflows/{id}', static fn (Request $request, string $id): Response => $controller()->startWorkflow($request, $id))->name(self::ROUTE_PREFIX.'start-workflow'); + $router->get($base.'/executions', static fn (Request $request): Response => $controller()->executions($request))->name(self::ROUTE_PREFIX.'executions'); + $router->get($base.'/executions/{id}', static fn (string $id): Response => $controller()->execution($id))->name(self::ROUTE_PREFIX.'execution'); + $router->post($base.'/workflows/{id}/signals/{signal}', static fn (Request $request, string $id, string $signal): Response => $controller()->signal($request, $id, $signal))->name(self::ROUTE_PREFIX.'signal'); + $router->post($base.'/workflows/{id}/suspend', static fn (Request $request, string $id): Response => $controller()->suspend($request, $id))->name(self::ROUTE_PREFIX.'suspend'); + $router->post($base.'/workflows/{id}/resume', static fn (string $id): Response => $controller()->resume($id))->name(self::ROUTE_PREFIX.'resume'); + $router->post($base.'/workflows/{id}/cancel', static fn (string $id): Response => $controller()->cancel($id))->name(self::ROUTE_PREFIX.'cancel'); + $router->get($base.'/workflows/{id}/queries/{query}', static fn (string $id, string $query): Response => $controller()->query($id, $query))->name(self::ROUTE_PREFIX.'query'); + $router->get($base.'/dlq', static fn (Request $request): Response => $controller()->deadLetters($request))->name(self::ROUTE_PREFIX.'dlq'); + $router->post($base.'/dlq/{id}/retry', static fn (string $id): Response => $controller()->retryDeadLetter($id))->name(self::ROUTE_PREFIX.'dlq-retry'); + $router->delete($base.'/dlq/{id}', static fn (string $id): Response => $controller()->deleteDeadLetter($id))->name(self::ROUTE_PREFIX.'dlq-delete'); + } +} diff --git a/packages/orchestration/src/Http/OrchestrationController.php b/packages/orchestration/src/Http/OrchestrationController.php new file mode 100644 index 00000000..1ccd8bd5 --- /dev/null +++ b/packages/orchestration/src/Http/OrchestrationController.php @@ -0,0 +1,220 @@ +guard(fn (): HttpResponse => $this->started($this->orchestrator->start(ExecutionPattern::Saga, $name, self::input($request), self::headers($request)))); + } + + public function startTcc(Request $request, string $name): HttpResponse + { + return $this->guard(fn (): HttpResponse => $this->started($this->orchestrator->start(ExecutionPattern::Tcc, $name, self::input($request), self::headers($request)))); + } + + public function startWorkflow(Request $request, string $id): HttpResponse + { + return $this->guard(function () use ($request, $id): HttpResponse { + $correlationId = self::body($request)['correlationId'] ?? null; + $state = $this->orchestrator->workflows()->start($id, self::input($request), self::headers($request), is_string($correlationId) && $correlationId !== '' ? $correlationId : null); + + return $this->started(ExecutionOutcome::fromWorkflow($state)); + }); + } + + public function executions(Request $request): HttpResponse + { + return $this->guard(function () use ($request): HttpResponse { + $pattern = $request->query('pattern'); + $status = $request->query('status'); + $name = $request->query('name'); + $criteria = new ExecutionCriteria( + pattern: is_string($pattern) ? ExecutionPattern::tryFrom(strtoupper($pattern)) : null, + status: is_string($status) ? ExecutionStatus::tryFrom(strtoupper($status)) : null, + name: is_string($name) && $name !== '' ? $name : null, + limit: self::limit($request), + ); + + return new JsonResponse(['executions' => array_map(static fn (ExecutionState $state): array => [ + 'correlationId' => $state->correlationId, + 'name' => $state->executionName, + 'pattern' => $state->pattern->value, + 'status' => $state->status->value, + 'updatedAt' => ExecutionState::instant($state->updatedAt), + 'waitingOn' => $state->waitingOn, + 'failureReason' => $state->failureReason, + ], $this->persistence->find($criteria))]); + }); + } + + public function execution(string $id): HttpResponse + { + return $this->guard(fn (): HttpResponse => new JsonResponse(($this->persistence->findById($id) ?? throw new ExecutionNotFoundException($id))->toArray())); + } + + public function signal(Request $request, string $id, string $signal): HttpResponse + { + return $this->guard(fn (): HttpResponse => new JsonResponse($this->orchestrator->workflows()->signal($id, $signal, self::body($request)['payload'] ?? null)->toArray())); + } + + public function suspend(Request $request, string $id): HttpResponse + { + return $this->guard(function () use ($request, $id): HttpResponse { + $reason = self::body($request)['reason'] ?? ''; + + return new JsonResponse($this->orchestrator->workflows()->suspend($id, is_string($reason) ? $reason : '')->toArray()); + }); + } + + public function resume(string $id): HttpResponse + { + return $this->guard(fn (): HttpResponse => new JsonResponse($this->orchestrator->workflows()->resume($id)->toArray())); + } + + public function cancel(string $id): HttpResponse + { + return $this->guard(fn (): HttpResponse => new JsonResponse($this->orchestrator->workflows()->cancel($id)->toArray())); + } + + public function query(string $id, string $query): HttpResponse + { + return $this->guard(fn (): HttpResponse => new JsonResponse(['result' => $this->orchestrator->workflows()->query($id, $query)])); + } + + public function deadLetters(Request $request): HttpResponse + { + return $this->guard(fn (): HttpResponse => new JsonResponse(['deadLetters' => array_map( + static fn (DeadLetterEntry $entry): array => $entry->toArray(), + $this->deadLetters->all(self::limit($request)), + )])); + } + + public function retryDeadLetter(string $id): HttpResponse + { + return $this->guard(fn (): HttpResponse => $this->started($this->retrier->retry($id))); + } + + public function deleteDeadLetter(string $id): HttpResponse + { + return $this->guard(function () use ($id): HttpResponse { + if (! $this->deadLetters->delete($id)) { + throw new DeadLetterNotFoundException($id); + } + + return new Response('', 204); + }); + } + + private function started(ExecutionOutcome $outcome): HttpResponse + { + if (! $outcome->success) { + return new JsonResponse($outcome->toArray(), 422); + } + + return new JsonResponse($outcome->toArray(), 201, ['Location' => url($this->settings->rest->path.'/executions/'.rawurlencode($outcome->correlationId))]); + } + + /** @param Closure(): HttpResponse $action */ + private function guard(Closure $action): HttpResponse + { + try { + return $action(); + } catch (FireflyException $refused) { + return new JsonResponse(['error' => $refused->getMessage(), 'code' => $refused->errorCode()], $refused->httpStatus()); + } + } + + /** @return array */ + private static function body(Request $request): array + { + if ($request->getContent() === '') { + return []; + } + + $body = json_decode($request->getContent(), true); + if (! is_array($body) || array_is_list($body) && $body !== []) { + throw new OrchestrationException('The request body must be a JSON object.', 'ORCHESTRATION_BAD_REQUEST', 400); + } + + /** @var array $body */ + return $body; + } + + /** @return array */ + private static function input(Request $request): array + { + $input = self::body($request)['input'] ?? []; + if (! is_array($input) || array_is_list($input) && $input !== []) { + throw new OrchestrationException('"input" must be a JSON object.', 'ORCHESTRATION_BAD_REQUEST', 400); + } + + /** @var array $input */ + return $input; + } + + /** @return array */ + private static function headers(Request $request): array + { + $headers = self::body($request)['headers'] ?? []; + if (! is_array($headers) || array_is_list($headers) && $headers !== []) { + throw new OrchestrationException('"headers" must be a JSON object of strings.', 'ORCHESTRATION_BAD_REQUEST', 400); + } + + $strings = []; + foreach ($headers as $name => $value) { + if (! is_scalar($value)) { + throw new OrchestrationException('"headers" must be a JSON object of strings.', 'ORCHESTRATION_BAD_REQUEST', 400); + } + $strings[(string) $name] = (string) $value; + } + + return $strings; + } + + private static function limit(Request $request): int + { + $limit = $request->query('limit', '50'); + + return is_numeric($limit) ? max(1, min(500, (int) $limit)) : 50; + } +} diff --git a/packages/orchestration/src/OrchestrationWiringProvider.php b/packages/orchestration/src/OrchestrationWiringProvider.php index 54cc9842..c6187ee5 100644 --- a/packages/orchestration/src/OrchestrationWiringProvider.php +++ b/packages/orchestration/src/OrchestrationWiringProvider.php @@ -10,6 +10,7 @@ use Firefly\Context\Scan\AppScan; use Firefly\Orchestration\Boot\EventGatewayWiringPass; use Firefly\Orchestration\Boot\OrchestrationRegistryPass; +use Firefly\Orchestration\Boot\OrchestrationRouteRegistrar; use Firefly\Orchestration\Boot\OrchestrationSchedulePass; use Firefly\Orchestration\Boot\ScheduledExecutionsPass; use Firefly\Orchestration\Console\DeadLetterCommand; @@ -31,7 +32,8 @@ * in-process scan of `firefly.scan.paths`, then an empty manifest (the AppScan convention), lazily, so a * cached application that has firefly/cli pays nothing; * - contributes the registry, gateway, schedule (the tick and the recovery sweep) and scheduled-execution - * (#[ScheduledSaga] / #[ScheduledTcc] / #[ScheduledWorkflow]) passes; + * (#[ScheduledSaga] / #[ScheduledTcc] / #[ScheduledWorkflow]) passes, and the REST route registrar that + * mounts nothing unless firefly.orchestration.rest.enabled; * - registers the firefly:orchestration:* commands (list, show, signal, dlq, dlq:retry, tick) in a console; * - publishes the migration (`--tag=firefly-orchestration-migrations`) and loads it while persistence or * the DLQ is `database`. @@ -62,7 +64,7 @@ public function register(): void */ public function passes(): array { - return [new OrchestrationRegistryPass, new EventGatewayWiringPass, new OrchestrationSchedulePass, new ScheduledExecutionsPass]; + return [new OrchestrationRegistryPass, new EventGatewayWiringPass, new OrchestrationSchedulePass, new ScheduledExecutionsPass, new OrchestrationRouteRegistrar]; } public function boot(): void diff --git a/packages/orchestration/tests/Http/OrchestrationRestDisabledTest.php b/packages/orchestration/tests/Http/OrchestrationRestDisabledTest.php new file mode 100644 index 00000000..d1dd2412 --- /dev/null +++ b/packages/orchestration/tests/Http/OrchestrationRestDisabledTest.php @@ -0,0 +1,16 @@ +postJson('/orchestration/sagas/checkout', ['input' => ['orderId' => 1, 'amount' => 5]])->assertNotFound(); + + expect(collect($this->app()->make('router')->getRoutes()->getRoutes())->map->getName()->filter( + static fn (?string $name): bool => is_string($name) && str_starts_with($name, 'firefly.orchestration.'), + )->all())->toBe([]); +}); diff --git a/packages/orchestration/tests/Http/OrchestrationRestTest.php b/packages/orchestration/tests/Http/OrchestrationRestTest.php new file mode 100644 index 00000000..77856443 --- /dev/null +++ b/packages/orchestration/tests/Http/OrchestrationRestTest.php @@ -0,0 +1,91 @@ + $response + */ +function restId(TestResponse $response, string $path): string +{ + $id = $response->json($path); + if (! is_string($id) || $id === '') { + throw new RuntimeException("the response carried no string [{$path}]."); + } + + return $id; +} + +it('starts a saga and points at its checkpoint', function (): void { + /** @var RestCapstoneTestCase $this */ + $response = $this->postJson('/ops/orchestration/sagas/checkout', ['input' => ['orderId' => 10, 'amount' => 5]]) + ->assertCreated() + ->assertJson(['pattern' => 'SAGA', 'name' => 'checkout', 'status' => 'COMPLETED', 'success' => true]); + $id = restId($response, 'correlationId'); + + expect($response->headers->get('Location'))->toEndWith("/ops/orchestration/executions/{$id}"); + $this->getJson("/ops/orchestration/executions/{$id}")->assertOk()->assertJson(['correlationId' => $id, 'status' => 'COMPLETED']); + $this->getJson('/ops/orchestration/executions?pattern=saga')->assertOk()->assertJsonPath('executions.0.correlationId', $id); + $this->getJson('/ops/orchestration/executions/nope')->assertNotFound()->assertJson(['code' => 'ORCHESTRATION_EXECUTION_NOT_FOUND']); +}); + +it('answers 422 for a failed saga and retries and deletes its dead letter', function (): void { + /** @var RestCapstoneTestCase $this */ + $this->postJson('/ops/orchestration/sagas/checkout', ['input' => ['orderId' => 11, 'amount' => 500]]) + ->assertStatus(422) + ->assertJson(['status' => 'FAILED', 'success' => false]); + + $id = restId($this->getJson('/ops/orchestration/dlq')->assertOk(), 'deadLetters.0.id'); + + $this->postJson("/ops/orchestration/dlq/{$id}/retry")->assertStatus(422)->assertJson(['status' => 'FAILED']); + $this->deleteJson("/ops/orchestration/dlq/{$id}")->assertNotFound(); + + // The retry removed the entry it retried and the failed re-run dead-lettered itself under a fresh id. + $retried = restId($this->getJson('/ops/orchestration/dlq')->assertOk()->assertJsonCount(1, 'deadLetters'), 'deadLetters.0.id'); + expect($retried)->not->toBe($id); + + $this->deleteJson("/ops/orchestration/dlq/{$retried}")->assertNoContent(); + $this->deleteJson("/ops/orchestration/dlq/{$retried}")->assertNotFound(); + $this->postJson("/ops/orchestration/dlq/{$retried}/retry")->assertNotFound(); +}); + +it('confirms a TCC', function (): void { + /** @var RestCapstoneTestCase $this */ + $this->postJson('/ops/orchestration/tcc/transfer', ['input' => ['amount' => 25, 'to' => 'acc-2']]) + ->assertCreated() + ->assertJson(['pattern' => 'TCC', 'status' => 'CONFIRMED']); + + expect($this->app()->make(Ledger::class)->lines)->toContain('commit hold-25'); +}); + +it('drives a workflow through signals, queries, suspension and cancellation', function (): void { + /** @var RestCapstoneTestCase $this */ + $this->postJson('/ops/orchestration/workflows/onboarding', ['input' => ['email' => 'ada@example.com'], 'correlationId' => 'rest-1']) + ->assertCreated() + ->assertJson(['correlationId' => 'rest-1', 'pattern' => 'WORKFLOW', 'status' => 'WAITING']); + + $this->getJson('/ops/orchestration/workflows/rest-1/queries/progress')->assertOk()->assertExactJson(['result' => 1]); + $this->postJson('/ops/orchestration/workflows/rest-1/suspend', ['reason' => 'audit'])->assertOk()->assertJson(['status' => 'SUSPENDED']); + $this->postJson('/ops/orchestration/workflows/rest-1/resume')->assertOk()->assertJson(['status' => 'WAITING', 'waitingOn' => 'verify']); + $this->postJson('/ops/orchestration/workflows/rest-1/signals/email-verified', ['payload' => ['token' => 't']]) + ->assertOk() + ->assertJson(['status' => 'WAITING', 'waitingOn' => 'cool-off']); + $this->postJson('/ops/orchestration/workflows/rest-1/cancel')->assertOk()->assertJson(['status' => 'CANCELLED']); + $this->postJson('/ops/orchestration/workflows/rest-1/cancel')->assertStatus(409); + $this->postJson('/ops/orchestration/workflows/nope/signals/email-verified')->assertNotFound(); +}); + +it('refuses bodies that are not JSON objects and unknown definitions', function (): void { + /** @var RestCapstoneTestCase $this */ + $this->postJson('/ops/orchestration/sagas/checkout', ['input' => 'nope'])->assertStatus(400)->assertJson(['code' => 'ORCHESTRATION_BAD_REQUEST']); + $this->postJson('/ops/orchestration/sagas/nope', ['input' => []])->assertNotFound(); +}); diff --git a/packages/orchestration/tests/Support/RestCapstoneTestCase.php b/packages/orchestration/tests/Support/RestCapstoneTestCase.php new file mode 100644 index 00000000..2de95b40 --- /dev/null +++ b/packages/orchestration/tests/Support/RestCapstoneTestCase.php @@ -0,0 +1,19 @@ + */ + protected function configOverrides(): array + { + return [ + ...parent::configOverrides(), + 'firefly.orchestration.rest.enabled' => true, + 'firefly.orchestration.rest.path' => 'ops/orchestration', + ]; + } +} From 5389fe6000502d6cb2b92395972c489c47ea761a Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 22:38:48 -0700 Subject: [PATCH 41/58] feat(observability): meter and span adapters for sagas, TCC and workflows --- deptrac.yaml | 4 ++ .../firefly-observability-components.php | 24 +++++++++ .../cache/firefly-observability-context.php | 46 +++++++++++++++++ packages/observability/composer.json | 3 +- .../src/ObservabilityAutoConfiguration.php | 27 ++++++++++ .../MeterRegistryOrchestrationMetrics.php | 40 +++++++++++++++ .../TracerOrchestrationTracing.php | 50 +++++++++++++++++++ .../MeterRegistryOrchestrationMetricsTest.php | 25 ++++++++++ .../OrchestrationAdaptersBootTest.php | 49 ++++++++++++++++++ .../TracerOrchestrationTracingTest.php | 35 +++++++++++++ skeleton/config/firefly.php | 10 ++++ 11 files changed, 312 insertions(+), 1 deletion(-) create mode 100644 packages/observability/src/Orchestration/MeterRegistryOrchestrationMetrics.php create mode 100644 packages/observability/src/Orchestration/TracerOrchestrationTracing.php create mode 100644 packages/observability/tests/Orchestration/MeterRegistryOrchestrationMetricsTest.php create mode 100644 packages/observability/tests/Orchestration/OrchestrationAdaptersBootTest.php create mode 100644 packages/observability/tests/Orchestration/TracerOrchestrationTracingTest.php diff --git a/deptrac.yaml b/deptrac.yaml index 27a6f6ec..5dea99fa 100644 --- a/deptrac.yaml +++ b/deptrac.yaml @@ -439,6 +439,9 @@ deptrac: # # The => FeatureFlags edge is one class (src/FeatureFlags): MeterRegistryFeatureFlagMetrics backs the # FeatureFlagMetrics port with feature_flag_evaluations_total{flag, variant, reason} — the CqrsMetrics seam. + # + # The => Orchestration edge is one folder (src/Orchestration): the meter-registry OrchestrationMetrics and + # the Tracer-backed OrchestrationTracing — the same seam direction as Cqrs, Eda and FeatureFlags. Observability: - Kernel - Container @@ -452,6 +455,7 @@ deptrac: - Resilience - Data - FeatureFlags + - Orchestration # Admin is the browser front-end for Actuator: it renders the SAME ActuatorEndpoint beans the JSON surface # serves, resolved in-process from Actuator's registry rather than fetched over HTTP (=> Actuator), mounts diff --git a/packages/observability/cache/firefly-observability-components.php b/packages/observability/cache/firefly-observability-components.php index 445874de..abf9a4a7 100644 --- a/packages/observability/cache/firefly-observability-components.php +++ b/packages/observability/cache/firefly-observability-components.php @@ -230,6 +230,30 @@ 0 => 'Firefly\\Observability\\Tracing\\Tracer', ], ], + 10 => [ + 'method' => 'orchestrationMetrics', + 'returns' => 'Firefly\\Orchestration\\Event\\OrchestrationMetrics', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 0, + 'lazy' => false, + 'dependencies' => [ + 0 => 'Firefly\\Observability\\Metrics\\MetricsRecorder', + ], + ], + 11 => [ + 'method' => 'orchestrationTracing', + 'returns' => 'Firefly\\Orchestration\\Event\\OrchestrationTracing', + 'name' => null, + 'scope' => 'Singleton', + 'primary' => false, + 'order' => 0, + 'lazy' => false, + 'dependencies' => [ + 0 => 'Firefly\\Observability\\Tracing\\Tracer', + ], + ], ], 'lazy' => false, 'dependencies' => [ diff --git a/packages/observability/cache/firefly-observability-context.php b/packages/observability/cache/firefly-observability-context.php index a5a3a133..bd593692 100644 --- a/packages/observability/cache/firefly-observability-context.php +++ b/packages/observability/cache/firefly-observability-context.php @@ -248,6 +248,52 @@ ], ], ], + 10 => [ + 'method' => 'orchestrationMetrics', + 'conditions' => [ + 0 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnMissingBean', + 'args' => [ + 0 => 'Firefly\\Orchestration\\Event\\OrchestrationMetrics', + ], + ], + 1 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnProperty', + 'args' => [ + 0 => 'firefly.observability.metrics.enabled', + 1 => 'true', + 2 => true, + ], + ], + ], + ], + 11 => [ + 'method' => 'orchestrationTracing', + 'conditions' => [ + 0 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnMissingBean', + 'args' => [ + 0 => 'Firefly\\Orchestration\\Event\\OrchestrationTracing', + ], + ], + 1 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnProperty', + 'args' => [ + 0 => 'firefly.observability.tracing.enabled', + 1 => 'true', + 2 => false, + ], + ], + 2 => [ + 'type' => 'Firefly\\Context\\Condition\\Attributes\\ConditionalOnProperty', + 'args' => [ + 0 => 'firefly.observability.tracing.orchestration.enabled', + 1 => 'true', + 2 => true, + ], + ], + ], + ], ], ], 3 => [ diff --git a/packages/observability/composer.json b/packages/observability/composer.json index 7eb26aeb..33e3612e 100644 --- a/packages/observability/composer.json +++ b/packages/observability/composer.json @@ -1,6 +1,6 @@ { "name": "firefly/observability", - "description": "LaraFly observability: a first-party pure-PHP MeterRegistry (counter/gauge/timer) with Prometheus 0.0.4 text exposition and a Micrometer-JSON /metrics endpoint, HTTP request auto-instrumentation, the real MeterRegistry-backed CqrsMetrics recorder, a resilience circuit-breaker gauge, process metrics, a Spring/OTel-shaped Tracer port with an OpenTelemetry adapter and W3C traceparent propagation over HTTP, CQRS and EDA, trace-aware structured logging, histogram buckets. Top-of-stack, gated on a MeterRegistry bean.", + "description": "LaraFly observability: a first-party pure-PHP MeterRegistry (counter/gauge/timer) with Prometheus 0.0.4 text exposition and a Micrometer-JSON /metrics endpoint, HTTP request auto-instrumentation, the real MeterRegistry-backed CqrsMetrics recorder, a resilience circuit-breaker gauge, process metrics, a Spring/OTel-shaped Tracer port with an OpenTelemetry adapter and W3C traceparent propagation over HTTP, CQRS, EDA and orchestration, trace-aware structured logging, histogram buckets. Top-of-stack, gated on a MeterRegistry bean.", "type": "library", "license": "Apache-2.0", "homepage": "https://github.com/fireflyframework/fireflyframework-php", @@ -32,6 +32,7 @@ "firefly/eda": "*@dev", "firefly/feature-flags": "*@dev", "firefly/kernel": "*@dev", + "firefly/orchestration": "*@dev", "firefly/resilience": "*@dev", "firefly/web": "*@dev", "guzzlehttp/promises": "^2.0 || ^3.0", diff --git a/packages/observability/src/ObservabilityAutoConfiguration.php b/packages/observability/src/ObservabilityAutoConfiguration.php index 2be57f9c..6d113025 100644 --- a/packages/observability/src/ObservabilityAutoConfiguration.php +++ b/packages/observability/src/ObservabilityAutoConfiguration.php @@ -29,10 +29,14 @@ use Firefly\Observability\Metrics\MetricsRecorder; use Firefly\Observability\Metrics\NoOpMetricsRecorder; use Firefly\Observability\Metrics\SimpleMeterRegistry; +use Firefly\Observability\Orchestration\MeterRegistryOrchestrationMetrics; +use Firefly\Observability\Orchestration\TracerOrchestrationTracing; use Firefly\Observability\Prometheus\PrometheusTextFormat; use Firefly\Observability\Tracing\NoOpTracer; use Firefly\Observability\Tracing\Tracer; use Firefly\Observability\Tracing\W3CTraceContextPropagator; +use Firefly\Orchestration\Event\OrchestrationMetrics; +use Firefly\Orchestration\Event\OrchestrationTracing; use Illuminate\Container\Container; use Illuminate\Contracts\Cache\Factory; @@ -233,4 +237,27 @@ public function edaTracing(Tracer $tracer): EdaTracing { return new TracerEdaTracing($tracer, new W3CTraceContextPropagator); } + + /** + * Backs firefly/orchestration's OrchestrationMetrics port — the cqrsMetrics() precedence over + * OrchestrationAutoConfiguration's #[Order(1000)] NoOp. firefly.orchestration.metrics.enabled still decides, + * on the orchestration side, whether the engines report to it at all. + */ + #[Bean] + #[ConditionalOnMissingBean(OrchestrationMetrics::class)] + #[ConditionalOnProperty(name: 'firefly.observability.metrics.enabled', havingValue: 'true', matchIfMissing: true)] + public function orchestrationMetrics(MetricsRecorder $recorder): OrchestrationMetrics + { + return new MeterRegistryOrchestrationMetrics($recorder); + } + + /** The OrchestrationTracing drop-in — the cqrsTracing() gates, with the orchestration instrumentation switch. */ + #[Bean] + #[ConditionalOnMissingBean(OrchestrationTracing::class)] + #[ConditionalOnProperty(name: 'firefly.observability.tracing.enabled', havingValue: 'true')] + #[ConditionalOnProperty(name: 'firefly.observability.tracing.orchestration.enabled', havingValue: 'true', matchIfMissing: true)] + public function orchestrationTracing(Tracer $tracer): OrchestrationTracing + { + return new TracerOrchestrationTracing($tracer); + } } diff --git a/packages/observability/src/Orchestration/MeterRegistryOrchestrationMetrics.php b/packages/observability/src/Orchestration/MeterRegistryOrchestrationMetrics.php new file mode 100644 index 00000000..5db56f7e --- /dev/null +++ b/packages/observability/src/Orchestration/MeterRegistryOrchestrationMetrics.php @@ -0,0 +1,40 @@ +recorder->record('orchestration_executions_seconds', ['name' => $name, 'pattern' => strtolower($pattern->value), 'outcome' => $outcome], $seconds); + } + + public function recordStep(string $name, string $stepId, string $outcome, int $attempts, float $seconds): void + { + $this->recorder->record('orchestration_steps_seconds', ['name' => $name, 'step' => $stepId, 'outcome' => $outcome], $seconds); + } + + public function recordCompensation(string $name, string $stepId, string $outcome): void + { + $this->recorder->increment('orchestration_compensations_total', ['name' => $name, 'step' => $stepId, 'outcome' => $outcome]); + } + + public function recordDeadLetter(string $name): void + { + $this->recorder->increment('orchestration_dead_letters_total', ['name' => $name]); + } +} diff --git a/packages/observability/src/Orchestration/TracerOrchestrationTracing.php b/packages/observability/src/Orchestration/TracerOrchestrationTracing.php new file mode 100644 index 00000000..837e66e5 --- /dev/null +++ b/packages/observability/src/Orchestration/TracerOrchestrationTracing.php @@ -0,0 +1,50 @@ +tracer->trace(strtolower($pattern->value).' '.$name, $execution, SpanKind::Internal, [ + 'firefly.orchestration.pattern' => $pattern->value, + 'firefly.orchestration.name' => $name, + 'firefly.orchestration.correlation_id' => $correlationId, + ]); + } + + /** + * @template T + * + * @param callable(): T $step + * @return T + */ + public function traceStep(string $name, string $stepId, string $correlationId, callable $step): mixed + { + return $this->tracer->trace("{$name}.{$stepId}", $step, SpanKind::Internal, [ + 'firefly.orchestration.name' => $name, + 'firefly.orchestration.step' => $stepId, + 'firefly.orchestration.correlation_id' => $correlationId, + ]); + } +} diff --git a/packages/observability/tests/Orchestration/MeterRegistryOrchestrationMetricsTest.php b/packages/observability/tests/Orchestration/MeterRegistryOrchestrationMetricsTest.php new file mode 100644 index 00000000..b913b990 --- /dev/null +++ b/packages/observability/tests/Orchestration/MeterRegistryOrchestrationMetricsTest.php @@ -0,0 +1,25 @@ +recordExecution('checkout', ExecutionPattern::Saga, 'success', 0.4); + $metrics->recordExecution('checkout', ExecutionPattern::Saga, 'failure', 0.2); + $metrics->recordStep('checkout', 'charge', 'failure', 3, 0.1); + $metrics->recordCompensation('checkout', 'reserve', 'success'); + $metrics->recordDeadLetter('checkout'); + $metrics->recordDeadLetter('checkout'); + + expect($registry->timer('orchestration_executions_seconds', ['name' => 'checkout', 'pattern' => 'saga', 'outcome' => 'success'])->count())->toBe(1) + ->and($registry->timer('orchestration_executions_seconds', ['name' => 'checkout', 'pattern' => 'saga', 'outcome' => 'failure'])->totalTimeSeconds())->toBe(0.2) + ->and($registry->timer('orchestration_steps_seconds', ['name' => 'checkout', 'step' => 'charge', 'outcome' => 'failure'])->count())->toBe(1) + ->and($registry->counter('orchestration_compensations_total', ['name' => 'checkout', 'step' => 'reserve', 'outcome' => 'success'])->count())->toBe(1.0) + ->and($registry->counter('orchestration_dead_letters_total', ['name' => 'checkout'])->count())->toBe(2.0); +}); diff --git a/packages/observability/tests/Orchestration/OrchestrationAdaptersBootTest.php b/packages/observability/tests/Orchestration/OrchestrationAdaptersBootTest.php new file mode 100644 index 00000000..8f0c45ae --- /dev/null +++ b/packages/observability/tests/Orchestration/OrchestrationAdaptersBootTest.php @@ -0,0 +1,49 @@ + $observability */ +function bootOrchestrationObservability(array $observability): ApplicationContext +{ + return bootFireflyApp( + [ + 'firefly' => ['observability' => $observability], + 'logging' => ['default' => 'test', 'channels' => ['test' => ['driver' => 'errorlog']]], + ], + [ + ResilienceServiceProvider::class, SchedulingServiceProvider::class, SchedulingWiringProvider::class, + EdaServiceProvider::class, EdaWiringProvider::class, + ObservabilityServiceProvider::class, ObservabilityWiringProvider::class, + OrchestrationServiceProvider::class, OrchestrationWiringProvider::class, + ], + needs: ['cache'], + ); +} + +it('backs the orchestration metrics port with the meter registry unless metrics are off', function (): void { + expect(bootOrchestrationObservability([])->get(OrchestrationMetrics::class))->toBeInstanceOf(MeterRegistryOrchestrationMetrics::class) + ->and(bootOrchestrationObservability(['metrics' => ['enabled' => false]])->get(OrchestrationMetrics::class))->toBeInstanceOf(NoOpOrchestrationMetrics::class); +}); + +it('traces orchestrations only under the tracing master gate and the orchestration switch', function (): void { + expect(bootOrchestrationObservability(['tracing' => ['enabled' => true]])->get(OrchestrationTracing::class))->toBeInstanceOf(TracerOrchestrationTracing::class) + ->and(bootOrchestrationObservability([])->get(OrchestrationTracing::class))->toBeInstanceOf(NoOpOrchestrationTracing::class) + ->and(bootOrchestrationObservability(['tracing' => ['enabled' => true, 'orchestration' => ['enabled' => false]]])->get(OrchestrationTracing::class))->toBeInstanceOf(NoOpOrchestrationTracing::class); +}); diff --git a/packages/observability/tests/Orchestration/TracerOrchestrationTracingTest.php b/packages/observability/tests/Orchestration/TracerOrchestrationTracingTest.php new file mode 100644 index 00000000..8c74ade0 --- /dev/null +++ b/packages/observability/tests/Orchestration/TracerOrchestrationTracingTest.php @@ -0,0 +1,35 @@ +traceExecution('checkout', ExecutionPattern::Saga, 'c-1', fn (): string => $tracing->traceStep('checkout', 'charge', 'c-1', fn (): string => 'charged')); + + expect($result)->toBe('charged'); + + $execution = $tracer->find('saga checkout'); + $step = $tracer->find('checkout.charge'); + expect($execution?->kind)->toBe(SpanKind::Internal) + ->and($execution?->attributes)->toBe(['firefly.orchestration.pattern' => 'SAGA', 'firefly.orchestration.name' => 'checkout', 'firefly.orchestration.correlation_id' => 'c-1']) + ->and($step?->attributes)->toBe(['firefly.orchestration.name' => 'checkout', 'firefly.orchestration.step' => 'charge', 'firefly.orchestration.correlation_id' => 'c-1']) + ->and($execution?->ended)->toBeTrue(); +}); + +it('records a failing step as ERROR and rethrows', function (): void { + $tracer = new RecordingTracer; + + expect(fn () => (new TracerOrchestrationTracing($tracer))->traceStep('checkout', 'charge', 'c-2', function (): never { + throw new RuntimeException('card declined'); + }))->toThrow(RuntimeException::class, 'card declined'); + + expect($tracer->recorded()[0]->status)->toBe(SpanStatus::Error); +}); diff --git a/skeleton/config/firefly.php b/skeleton/config/firefly.php index 2da14792..83b2d5a7 100644 --- a/skeleton/config/firefly.php +++ b/skeleton/config/firefly.php @@ -1966,6 +1966,16 @@ 'eda' => [ 'enabled' => true, ], + + /* + | INTERNAL spans around every saga, TCC and workflow execution (`saga checkout`) and each of + | their steps (`checkout.charge`). Also needs firefly.orchestration.tracing.enabled. + | + | Default: true. + */ + 'orchestration' => [ + 'enabled' => true, + ], ], ], From 9308f45e9a536486cf89208455ea4580bb767324 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 22:52:00 -0700 Subject: [PATCH 42/58] docs(orchestration): the orchestration guide, its links, the observability meters and the changelog --- CHANGELOG.md | 18 ++ README.md | 9 +- docs/README.md | 7 +- docs/architecture.md | 4 +- docs/index.md | 4 +- docs/modules.md | 4 +- docs/modules/observability.md | 8 + docs/modules/orchestration.md | 304 +++++++++++++++++++++++++++++++ docs/modules/tracing.md | 2 + mkdocs.yml | 1 + packages/orchestration/README.md | 2 + 11 files changed, 350 insertions(+), 13 deletions(-) create mode 100644 docs/modules/orchestration.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f15508c..4245323b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,24 @@ All notable changes to LaraFly are documented here. This project uses CalVer (`Y ## [Unreleased] +### Added + +- **Orchestration (`firefly/orchestration`):** sagas with five compensation policies, compensation error + handlers, a builder DSL and saga composition; TCC with ordered participants, per-phase timeouts and + retries and TCC composition; and durable workflows with dependencies, conditions, signals, timers, + wait-for-all/any, child workflows, continue-as-new, queries, suspend/resume, search attributes and queued + execution. Definitions are attribute-discovered, compiled by `firefly:cache` and validated at boot. +- **Orchestration persistence and recovery:** checkpoints at every step through a `memory`, `cache` or + `database` store (PostgreSQL, MySQL, MariaDB, SQLite and SQL Server), a per-execution distributed lock, a + scheduled recovery sweep that reports, resumes or compensates stale executions, retention cleanup, and a + dead-letter queue with retry. +- **Orchestration operations:** lifecycle and step events through `firefly/eda`, event-type triggers, cron and + fixed-rate schedules, the `orchestration` actuator endpoint and health component, + `php artisan firefly:orchestration:{list,show,signal,dlq,dlq:retry,tick}` and an opt-in REST surface. +- **Observability:** `orchestration_executions_seconds`, `orchestration_steps_seconds`, + `orchestration_compensations_total` and `orchestration_dead_letters_total`, and INTERNAL spans per execution + and step behind `firefly.observability.tracing.orchestration.enabled`. + ## [26.10.1] - 2026-10-06 ### Added diff --git a/README.md b/README.md index 5abf114c..8a17d7c6 100644 --- a/README.md +++ b/README.md @@ -1066,7 +1066,7 @@ Full flag reference and generated-file contents: [CLI](docs/cli.md). One published library, `fireflyframework/larafly`, contains the components under `packages/*` and the bundled application skeleton. Its `replace` entries satisfy component requirements at the framework's version; -component names remain useful module boundaries, with their own test suites. The 33 +component names remain useful module boundaries, with their own test suites. The 34 [module guides](docs/modules/) below group them by concern: | Group | Module | Package(s) | @@ -1089,6 +1089,7 @@ component names remain useful module boundaries, with their own test suites. The | Eventing & Messaging | [Event-Driven Architecture](docs/modules/eda.md) — `EventPublisher`, `#[EventListener]`, retry/DLQ | `firefly/eda` | | Eventing & Messaging | [EDA Brokers](docs/modules/eda-brokers.md) — RabbitMQ/Postgres/Kafka adapters, the same-tx outbox | `firefly/eda-rabbitmq`, `firefly/eda-postgres`, `firefly/eda-kafka` | | Eventing & Messaging | [Messaging](docs/modules/messaging.md) — the lower-level raw-bytes broker layer | `firefly/messaging` | +| Eventing & Messaging | [Orchestration](docs/modules/orchestration.md) — sagas, TCC, durable workflows, recovery, DLQ | `firefly/orchestration` | | CQRS | [CQRS](docs/modules/cqrs.md) — `CommandBus`/`QueryBus`, the domain→integration-event bridge | `firefly/cqrs` | | Security | [Security](docs/modules/security.md) — principal model, `HttpSecurity`, `#[PreAuthorize]`, JWT/OAuth2 | `firefly/security` | | Security | [OAuth2 Client](docs/modules/security-oauth2-client.md) — OpenID Connect login with presets and discovery, PKCE, id-token validation, `OidcUser`, RP-initiated logout, client credentials, `Http::oauth2Client()` | `firefly/security-oauth2-client` | @@ -1144,10 +1145,10 @@ still ahead, accurately: round-trip test (RabbitMQ, Postgres, Kafka) gated behind Docker + a live connection string — these run on demand, not in the default `vendor/bin/pest`/CI pass; wiring a scheduled/opt-in CI lane for them is still open. -- **More actuator endpoints.** Sixteen ship today — `/health`, `/info`, `/env`, `/beans`, `/conditions`, +- **More actuator endpoints.** Seventeen ship today — `/health`, `/info`, `/env`, `/beans`, `/conditions`, `/mappings`, `/loggers`, `/scheduledtasks`, `/caches`, `/configprops`, `/metrics`, `/prometheus`, - `/process`, `/httpexchanges`, `/oauth2clients` when the authorization server is installed, and `/flags` - when feature flags are enabled — and + `/process`, `/httpexchanges`, `/oauth2clients` when the authorization server is installed, `/flags` + when feature flags are enabled, and `/orchestration` for sagas, TCCs and workflows — and `firefly.management.server.port` moves the whole surface onto a second listener, with `ManagementPortGuard` making the actuator refuse the application port and `php artisan firefly:management:serve` running that second listener in development. Spring Boot's `/refresh`, `/threaddump` and `/shutdown` are still not diff --git a/docs/README.md b/docs/README.md index 48a43e2d..58c7721d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -28,11 +28,11 @@ ## Module Guides Every module guide lives under [`modules/`](modules/), in the ten groups `mkdocs.yml`'s navigation and the -site's own [Modules](modules.md) landing page use. All 33 are listed here, and neither the list nor the +site's own [Modules](modules.md) landing page use. All 34 are listed here, and neither the list nor the number is maintained on trust: `tests/ModuleDocumentationTest.php` fails the build if a guide is missing from the navigation, from the table below, from the root `README.md`'s table, from `docs/index.md` or from the landing page, and `tests/SiteNavigationTest.php` reads that number off `docs/modules/*.md` itself, so a -thirty-fourth guide turns this page red rather than quietly making it wrong. +thirty-fifth guide turns this page red rather than quietly making it wrong. ### Foundation @@ -76,6 +76,7 @@ thirty-fourth guide turns this page red rather than quietly making it wrong. | [Event-Driven Architecture](modules/eda.md) | `firefly/eda` — broker-backed `EventPublisher`, envelopes, retry/DLQ, in-memory + queue adapters | | [EDA Brokers](modules/eda-brokers.md) | Real broker adapters behind the `EventPublisher` port — RabbitMQ, Postgres LISTEN/NOTIFY, Kafka | | [Messaging](modules/messaging.md) | `firefly/messaging` — the lower-level raw-bytes `MessageBrokerPort`, `#[MessageListener]` | +| [Orchestration](modules/orchestration.md) | `firefly/orchestration` — sagas, TCC and durable workflows with checkpoints, recovery and a dead-letter queue | ### CQRS @@ -123,7 +124,7 @@ thirty-fourth guide turns this page red rather than quietly making it wrong. | Document | Description | |----------|-------------| -| [Modules](modules.md) | The grouped index of all 33 module guides, and how a package wires itself | +| [Modules](modules.md) | The grouped index of all 34 module guides, and how a package wires itself | | [CLI Reference](cli.md) | Every `firefly:*` command and `make:firefly-*` generator, including the four contributed by capability packages | | [Laravel Comparison](laravel-comparison.md) | Side-by-side concept mapping for developers coming from plain Laravel | | [Versioning](versioning.md) | CalVer (`YY.MM.Patch`), no `version` field, how Packagist derives releases from tags | diff --git a/docs/architecture.md b/docs/architecture.md index 385e9408..ab2c8d0b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -268,10 +268,10 @@ onto every record beside the correlation and request ids, and `firefly.logging.s `json`, `ecs` or `logstash`, or empty for plain text — decides how a line is rendered, attached per channel at resolution time. -Metrics and introspection are the actuator's. Sixteen `ActuatorEndpoint` implementations ship, discovered +Metrics and introspection are the actuator's. Seventeen `ActuatorEndpoint` implementations ship, discovered into the `ActuatorRegistry` — health, info, env, beans, conditions, mappings, loggers, scheduledtasks, caches, configprops, metrics, prometheus, process, httpexchanges, oauth2clients when the authorization server is -installed, and flags when feature flags are enabled — and the sensitive ones answer **404** until named in +installed, flags when feature flags are enabled, and orchestration for sagas, TCCs and workflows — and the sensitive ones answer **404** until named in `firefly.management.endpoints.web.exposure.include`. `firefly/admin` renders those same beans server-side by resolving them **in-process from that registry**, never by fetching its own HTTP surface: one data path, and pages the JSON surface deliberately keeps unexposed. That is also why the dashboard's own URL is its entire diff --git a/docs/index.md b/docs/index.md index da69695f..e3eb4082 100644 --- a/docs/index.md +++ b/docs/index.md @@ -103,7 +103,7 @@ See [Installation](installation.md) for requirements and manual setup, and ## The modules -Every capability above ships in the root library; configuration selects optional features. The [module index](modules.md) lays all 33 +Every capability above ships in the root library; configuration selects optional features. The [module index](modules.md) lays all 34 guides out by concern, with a line on each saying what it is for, and the same grouping is the site's **Modules** tab. @@ -113,7 +113,7 @@ guides out by concern, with a line on each saying what it is for, and the same g | **Web & API** | [Web Layer](modules/web.md) · [Web Filters](modules/web-filters.md) · [OpenAPI](modules/openapi.md) | | **Resilience & Scheduling** | [Resilience](modules/resilience.md) · [Scheduling](modules/scheduling.md) | | **Data & Domain** | [Domain (DDD)](modules/domain.md) · [Data & Repositories](modules/data.md) · [Relational Data](modules/data-relational.md) · [Transactions](modules/transactional.md) | -| **Eventing & Messaging** | [EDA](modules/eda.md) · [EDA Brokers](modules/eda-brokers.md) · [Messaging](modules/messaging.md) | +| **Eventing & Messaging** | [EDA](modules/eda.md) · [EDA Brokers](modules/eda-brokers.md) · [Messaging](modules/messaging.md) · [Orchestration](modules/orchestration.md) | | **CQRS** | [Command/Query](modules/cqrs.md) | | **Security** | [Security](modules/security.md) · [OAuth2 Client](modules/security-oauth2-client.md) · [OAuth2 Authorization Server](modules/security-oauth2-server.md) | | **Operations** | [Actuator](modules/actuator.md) · [Observability](modules/observability.md) · [Tracing](modules/tracing.md) · [Logging](modules/logging.md) · [Admin Dashboard](modules/admin.md) · [Bean Graph](modules/bean-graph.md) · [Data Browser](modules/data-browser.md) · [Feature Flags](modules/feature-flags.md) | diff --git a/docs/modules.md b/docs/modules.md index 7fdb64ee..882515ff 100644 --- a/docs/modules.md +++ b/docs/modules.md @@ -1,7 +1,7 @@ # Modules LaraFly publishes one `fireflyframework/larafly` library containing the **31 component directories** under -`packages/*`, each with its own tests or metadata. The **33 guides** below explain their public surfaces. +`packages/*`, each with its own tests or metadata. The **34 guides** below explain their public surfaces. Several components carry more than one guide: `firefly/data` answers for *Data & Repositories*, *Relational Data* and *Transactions*. Laravel discovers the root library's providers, and configuration selects optional features. A bean you declare wins over the framework default. @@ -43,7 +43,7 @@ each binding in `WebServiceProvider::register()` with a `bound()` check instead,
Eventing & Messaging -[EDA](modules/eda.md) · [EDA Brokers](modules/eda-brokers.md) · [Messaging](modules/messaging.md) +[EDA](modules/eda.md) · [EDA Brokers](modules/eda-brokers.md) · [Messaging](modules/messaging.md) · [Orchestration](modules/orchestration.md) One `EventPublisher` port with in-memory, queue, RabbitMQ, Postgres and Kafka adapters, a genuine same-transaction outbox, and the lower-level raw-bytes broker layer beneath it.
diff --git a/docs/modules/observability.md b/docs/modules/observability.md index 8bb38636..caa99fc3 100644 --- a/docs/modules/observability.md +++ b/docs/modules/observability.md @@ -242,6 +242,14 @@ survivors before the next config is evaluated, so `cqrsMetrics()` registers **fi `#[ConditionalOnMissingBean(CqrsMetrics::class)]` then sees a bean already bound and backs off. No code in `firefly/cqrs` changes; the win is pure auto-configuration ordering. +`MeterRegistryOrchestrationMetrics` is the same drop-in for `firefly/orchestration`'s `OrchestrationMetrics` port, +behind the same `#[Order(500)]` precedence over `OrchestrationAutoConfiguration`'s `#[Order(1000)]` no-op: + + orchestration_executions_seconds{name, pattern, outcome} + orchestration_steps_seconds{name, step, outcome} + orchestration_compensations_total{name, step, outcome} + orchestration_dead_letters_total{name} + ## Gating — property, not bean presence Every observability bean/endpoint/filter gates on the **same** config property, diff --git a/docs/modules/orchestration.md b/docs/modules/orchestration.md new file mode 100644 index 00000000..66a67524 --- /dev/null +++ b/docs/modules/orchestration.md @@ -0,0 +1,304 @@ +# Orchestration + +`firefly/orchestration` coordinates work that spans more than one transaction. It ships three patterns, shaped +after the Java `fireflyframework-orchestration` engine: + +- **Sagas** run forward steps in dependency order and, when one fails, undo the completed ones with their + compensations under one of five compensation policies. +- **TCC** (try/confirm/cancel) reserves with every participant first, then confirms all of them or cancels + the ones that tried. +- **Durable workflows** run a step graph that can wait for signals and timers, start child workflows, answer + queries, suspend and resume, and survive a process that dies between two steps. + +Every step boundary writes an `ExecutionState` checkpoint through the `ExecutionPersistenceProvider` port, so +a recovery sweep can resume or compensate whatever a crashed process left behind. Failures land in a +dead-letter queue. Lifecycle events publish through `firefly/eda`, and an EDA event can start a definition. + +## Declaring a saga + +A saga is a bean with `#[Saga]` and one `#[SagaStep]` per forward step. `compensate` names the method that +undoes the step; `dependsOn` orders the steps into topology layers. Parameters are injected by attribute: +`#[Input]` reads the execution input, `#[FromStep]` the result of an earlier step, and an `ExecutionContext` +parameter receives the live context. + + +```php +use Firefly\Container\Attributes\Service; +use Firefly\Orchestration\Core\Argument\FromStep; +use Firefly\Orchestration\Core\Argument\Input; +use Firefly\Orchestration\Saga\Attributes\OnSagaError; +use Firefly\Orchestration\Saga\Attributes\Saga; +use Firefly\Orchestration\Saga\Attributes\SagaStep; +use Firefly\Orchestration\Saga\Attributes\StepEvent; + +#[Service] +#[Saga(name: 'checkout', triggerEventType: 'orders.placed')] +final class CheckoutSaga +{ + #[SagaStep(id: 'reserve', compensate: 'release')] + public function reserve(#[Input('orderId')] int $orderId): string + { + return "reservation-{$orderId}"; + } + + public function release(string $reservation): void {} + + #[SagaStep(id: 'charge', dependsOn: ['reserve'], compensate: 'refund', retry: 2, backoffMs: 200, timeoutMs: 5000)] + #[StepEvent(topic: 'payments', type: 'payment.captured', key: 'orderId')] + public function charge(#[Input('amount')] int $amount, #[FromStep('reserve')] string $reservation): string + { + return "receipt-for-{$reservation}"; + } + + public function refund(string $receipt): void {} + + #[OnSagaError] + public function failed(\Throwable $error): void {} +} +``` + +A compensation receives the forward step's result. Start the saga through the `Orchestrator` (or `SagaEngine` +directly): + + +```php +use Firefly\Orchestration\Core\ExecutionPattern; +use Firefly\Orchestration\Core\Orchestrator; + +final class PlaceOrder +{ + public function __construct(private readonly Orchestrator $orchestrator) {} + + /** @return array */ + public function __invoke(int $orderId, int $amount): array + { + $outcome = $this->orchestrator->start(ExecutionPattern::Saga, 'checkout', ['orderId' => $orderId, 'amount' => $amount]); + + return ['id' => $outcome->correlationId, 'status' => $outcome->status->value]; + } +} +``` + +Steps can also be declared without a class through `SagaBuilder::saga('checkout')->step('reserve')->handler(…) +->compensation(…)->add()->build()`, and registered on the `SagaRegistry`. + +### Compensation policies + +`firefly.orchestration.saga.compensation-policy` decides how completed steps are undone: + +| Policy | Behavior | +|---|---| +| `strict-sequential` (default) | Undo in reverse completion order, one at a time. | +| `grouped-parallel` | Undo layer by layer, in reverse layer order. | +| `retry-with-backoff` | Like `strict-sequential`, retrying each compensation with its `compensationRetry` and `compensationBackoffMs`. | +| `circuit-breaker` | Stop compensating once a `compensationCritical` step fails or the breaker opens, and leave the rest to recovery. | +| `best-effort` | Run every compensation and record each failure without stopping. | + +`firefly.orchestration.saga.compensation-error-handler` decides what one failed compensation does to the run: +`default`, `fail-fast`, `log-and-continue`, `retry`, `robust`, or the class name of your own +`CompensationErrorHandler` bean. + +## Declaring a TCC + +A TCC names its participants by `#[TccParticipant]` on separate beans. Each participant has a `#[TryMethod]`, a +`#[ConfirmMethod]` and a `#[CancelMethod]`; confirm and cancel receive the try result through `#[FromTry]`. +Participants try in `order`. If any try fails, every participant that tried is cancelled. + + +```php +use Firefly\Container\Attributes\Service; +use Firefly\Orchestration\Core\Argument\Input; +use Firefly\Orchestration\Tcc\Attributes\CancelMethod; +use Firefly\Orchestration\Tcc\Attributes\ConfirmMethod; +use Firefly\Orchestration\Tcc\Attributes\FromTry; +use Firefly\Orchestration\Tcc\Attributes\Tcc; +use Firefly\Orchestration\Tcc\Attributes\TccParticipant; +use Firefly\Orchestration\Tcc\Attributes\TryMethod; + +#[Tcc(name: 'transfer', timeoutMs: 10000)] +final class TransferTcc {} + +#[Service] +#[TccParticipant(tcc: 'transfer', id: 'debit', order: 1)] +final class DebitAccount +{ + #[TryMethod] + public function hold(#[Input('amount')] int $amount): string + { + return "hold-{$amount}"; + } + + #[ConfirmMethod] + public function commit(#[FromTry] string $hold): void {} + + #[CancelMethod] + public function release(#[FromTry] string $hold): void {} +} +``` + +A confirmed TCC ends `CONFIRMED`, a cancelled one `CANCELED`. With `recovery.action` set to `resume`, a +process that died in `CONFIRMING` is driven on to `CONFIRMED`, confirming only the participants that had not +confirmed yet; `compensate` cancels every participant instead. + +## Durable workflows + +A workflow is a step graph whose steps may wait. `#[WaitForSignal]` parks the execution `WAITING` until +`WorkflowEngine::signal()` delivers the signal, whose payload the step reads as the variable +`signals.`. `#[WaitForTimer]` parks it until the timer is due; the scheduled tick +(`firefly.orchestration.workflow.tick.*`) or `php artisan firefly:orchestration:tick` advances due workflows. +`#[WaitForAll]` and `#[WaitForAny]` combine signals and timers, `#[ChildWorkflow]` starts another workflow and +waits for it, `condition` skips a step when an ExpressionLanguage expression over `input`, `results`, +`variables` and `headers` is false, and `#[WorkflowQuery]` answers a read-only question about a live execution. + + +```php +use Firefly\Container\Attributes\Service; +use Firefly\Orchestration\Core\Argument\Input; +use Firefly\Orchestration\Core\Argument\Variable; +use Firefly\Orchestration\Core\ExecutionContext; +use Firefly\Orchestration\Workflow\Attributes\WaitForSignal; +use Firefly\Orchestration\Workflow\Attributes\WaitForTimer; +use Firefly\Orchestration\Workflow\Attributes\Workflow; +use Firefly\Orchestration\Workflow\Attributes\WorkflowQuery; +use Firefly\Orchestration\Workflow\Attributes\WorkflowStep; + +#[Service] +#[Workflow(id: 'onboarding')] +final class OnboardingWorkflow +{ + #[WorkflowStep(id: 'register')] + public function register(#[Input('email')] string $email, ExecutionContext $ctx): string + { + $ctx->setSearchAttribute('email', $email); + + return "user-{$email}"; + } + + #[WorkflowStep(id: 'verify', dependsOn: ['register'])] + #[WaitForSignal('email-verified', timeoutMs: 86400000)] + public function verify(#[Variable('signals.email-verified')] mixed $proof): string + { + return 'verified'; + } + + #[WorkflowStep(id: 'welcome', dependsOn: ['verify'])] + #[WaitForTimer(delayMs: 3600000, timerId: 'welcome')] + public function welcome(): string + { + return 'welcomed'; + } + + #[WorkflowQuery('progress')] + public function progress(ExecutionContext $ctx): int + { + return count($ctx->results()); + } +} +``` + + +```php +use Firefly\Orchestration\Workflow\Engine\WorkflowEngine; + +final class Onboarding +{ + public function __construct(private readonly WorkflowEngine $workflows) {} + + public function begin(string $email): string + { + return $this->workflows->start('onboarding', ['email' => $email])->correlationId; + } + + public function verified(string $correlationId, string $token): void + { + $this->workflows->signal($correlationId, 'email-verified', ['token' => $token]); + } + + public function progress(string $correlationId): mixed + { + return $this->workflows->query($correlationId, 'progress'); + } +} +``` + +`suspend()`, `resume()`, `cancel()` and `compensate()` manage a live execution; +`upsertSearchAttributes()` and `search()` find executions by attribute. A workflow with +`triggerMode: TriggerMode::Async` is started `PENDING` and run by `RunWorkflowJob` on the queue named by +`firefly.orchestration.workflow.queue.*`. A step that returns `ContinueAsNew` ends the run and starts a fresh +one with new input, linked through the `x-firefly-continued-from` header. + +## Persistence, recovery and the dead-letter queue + +`firefly.orchestration.persistence.provider` selects the store: + +| Provider | Use | +|---|---| +| `memory` (default) | Tests and single-request flows. Nothing survives the process. | +| `cache` | Any Laravel cache store (`persistence.cache-store`), keys under `persistence.key-prefix`. | +| `database` | The `firefly_orchestration_*` tables on `persistence.connection`. Proven on PostgreSQL, MySQL, MariaDB and SQLite, and on SQL Server in CI. | + +Publish the migration with `php artisan vendor:publish --tag=firefly-orchestration-migrations`; the package also +loads it while `persistence.provider` or `dlq.store` is `database`. + +Writers serialize per execution through the `firefly/scheduling` `DistributedLock` (`firefly.orchestration.lock.*`). +With `firefly.orchestration.recovery.enabled`, a scheduled sweep finds executions not updated for +`recovery.stale-threshold` and, by `recovery.action`, reports them (`report`), runs them on (`resume`) or undoes +them (`compensate`). The same sweep deletes terminal executions older than `persistence.retention-period`. + +A failed execution becomes a dead letter (`firefly.orchestration.dlq.*`). Retrying one starts its definition +again with the original input and counts the retry on the entry. + +## Events, triggers and schedules + +With `firefly.orchestration.events.enabled`, every execution publishes `EXECUTION_STARTED`, +`EXECUTION_COMPLETED`, `STEP_COMPLETED` and `STEP_FAILED` through the `EventPublisher` to +`events.default-topic` (or `orchestration`); a workflow publishes its `STEP_COMPLETED` events only with +`publishEvents: true`. A `#[StepEvent]`, a `#[TccEvent]` or a workflow step's `outputEventType` publishes that +step's completion (for a TCC participant, its confirm) to the topic and type it names, whether or not +`events.enabled` is on. A definition's `triggerEventType` registers it on the event gateway, so an EDA event of that type starts it +with the event payload as input. `#[ScheduledSaga]`, `#[ScheduledTcc]` and `#[ScheduledWorkflow]` start a +definition on a cron, fixed-rate or fixed-delay cadence through `firefly/scheduling`, guarded by its distributed lock. +Lifecycle hooks are beans implementing `OrchestrationEvents` (extend `OrchestrationEventsAdapter` to override a +few of its 26 methods). + +## Operating + +- `GET /actuator/orchestration` (expose `orchestration` in `firefly.management.endpoints.web.exposure.include`) + lists definitions with their topology layers, execution counts, dead letters and triggers; + `…/executions`, `…/executions/{id}` and `…/dlq` drill down. +- The `orchestration` health component reports the store, open and stale executions and dead letters + (`firefly.orchestration.health.enabled`). +- `php artisan firefly:orchestration:list`, `:show`, `:signal`, `:dlq`, `:dlq:retry` and `:tick`. +- With `firefly.orchestration.rest.enabled`, a REST surface under `rest.path` starts sagas, TCCs and + workflows, signals, suspends, resumes and cancels workflows, runs queries and retries or deletes dead + letters. It is an operator surface: protect it with your own middleware or network policy. +- With `firefly/observability`, executions and steps are timed (`orchestration_executions_seconds`, + `orchestration_steps_seconds`), compensations and dead letters counted + (`orchestration_compensations_total`, `orchestration_dead_letters_total`), and traced when + `firefly.observability.tracing.orchestration.enabled` is on. + +## Configuration + +Every key lives under `firefly.orchestration` in `config/firefly.php`; the skeleton documents each with its +default. + +| Keys | Purpose | +|---|---| +| `saga.enabled`, `saga.compensation-policy`, `saga.default-timeout`, `saga.compensation-error-handler` | Saga engine | +| `tcc.enabled`, `tcc.default-timeout`, `tcc.composition.enabled`, `tcc.composition.compensation-policy` | TCC engine | +| `workflow.enabled`, `workflow.queue.connection`, `workflow.queue.name`, `workflow.tick.enabled`, `workflow.tick.interval` | Workflow engine and its tick | +| `persistence.provider`, `persistence.connection`, `persistence.cache-store`, `persistence.key-prefix`, `persistence.key-ttl`, `persistence.retention-period`, `persistence.cleanup-interval` | Execution store | +| `recovery.enabled`, `recovery.stale-threshold`, `recovery.action` | Recovery sweep | +| `dlq.enabled`, `dlq.store` | Dead-letter queue | +| `events.enabled`, `events.default-topic`, `event-gateway.enabled` | EDA events and triggers | +| `rest.enabled`, `rest.path` | REST surface | +| `health.enabled`, `metrics.enabled`, `tracing.enabled` | Health component, meters, spans | +| `validation.enabled`, `validation.fail-on-warning` | Boot-time topology validation | +| `lock.ttl`, `lock.wait` | Per-execution lock | +| `backpressure.strategy`, `backpressure.batch-size`, `backpressure.max-in-flight`, `backpressure.circuit-breaker.*` | Admission control | + +## Limits + +Steps of one topology layer run one after another. `layerConcurrency`, a step's `cpuBound` and a workflow +step's `async` are accepted so definitions port from the Java engine unchanged, and have no effect; the +actuator endpoint says so in its `notes`. diff --git a/docs/modules/tracing.md b/docs/modules/tracing.md index bdf34e76..bda7ff38 100644 --- a/docs/modules/tracing.md +++ b/docs/modules/tracing.md @@ -73,6 +73,7 @@ case-insensitively, list or string values) and `inject(SpanContext): array` / `CONSUMER` `process `; `messaging.system=firefly-eda`, `messaging.destination.name`, `messaging.operation.type`, `messaging.message.id`, `firefly.eda.event_type` | `traceparent`/`tracestate` in the envelope headers, beside `x-correlation-id`; `firefly.eda.tracing.brokers.enabled=false` keeps the in-process spans while the three broker publishers stamp nothing on the transport (see [EDA](eda.md#config-keys)) | +| Orchestration | `OrchestrationTracing` seam in `firefly/orchestration` (`NoOpOrchestrationTracing` default), filled by `TracerOrchestrationTracing` | `INTERNAL` ` ` per execution and `.` per step; `firefly.orchestration.pattern`, `firefly.orchestration.name`, `firefly.orchestration.step`, `firefly.orchestration.correlation_id` | nothing to carry — in-process; also needs `firefly.orchestration.tracing.enabled` | **The id a person quotes is the trace id.** `firefly/web` reads the ids `TracingFilter` publishes above (`Firefly\Web\Trace\TraceContext`, which owns the two attribute/`Context` keys the filter's own constants @@ -144,6 +145,7 @@ and the NoOp stays: every instrumentation site costs a few method calls and publ | `http-client.enabled` | `true` | CLIENT spans + `traceparent` on the `Http` client. | | `cqrs.enabled` | `true` | INTERNAL spans on the command/query buses. | | `eda.enabled` | `true` | PRODUCER/CONSUMER spans and `traceparent` in envelopes. | +| `orchestration.enabled` | `true` | INTERNAL spans around saga, TCC and workflow executions and their steps. | ## Testing diff --git a/mkdocs.yml b/mkdocs.yml index df54cf58..d31fa15c 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -141,6 +141,7 @@ nav: - Event-Driven Architecture: modules/eda.md - EDA Brokers: modules/eda-brokers.md - Messaging: modules/messaging.md + - Orchestration: modules/orchestration.md - CQRS: - CQRS (Command/Query): modules/cqrs.md - Security: diff --git a/packages/orchestration/README.md b/packages/orchestration/README.md index 95e935a3..dee56db1 100644 --- a/packages/orchestration/README.md +++ b/packages/orchestration/README.md @@ -15,4 +15,6 @@ crashed process left behind. Failed executions are parked in a dead-letter queue lifecycle events publish through `firefly/eda`, the `orchestration` actuator endpoint and health component report what is running, and `php artisan firefly:orchestration:*` lists, shows, signals, ticks and retries. +The guide is [docs/modules/orchestration.md](../../docs/modules/orchestration.md). + Apache-2.0 © Firefly Software Solutions Inc. From 224f02a28b670a9768dafb926c40fedd97300b85 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 23:13:59 -0700 Subject: [PATCH 43/58] docs(orchestration): stop claiming SQL Server is proven until its integration tests run in CI --- CHANGELOG.md | 3 ++- docs/modules/orchestration.md | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4245323b..87bf8052 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,8 @@ All notable changes to LaraFly are documented here. This project uses CalVer (`Y wait-for-all/any, child workflows, continue-as-new, queries, suspend/resume, search attributes and queued execution. Definitions are attribute-discovered, compiled by `firefly:cache` and validated at boot. - **Orchestration persistence and recovery:** checkpoints at every step through a `memory`, `cache` or - `database` store (PostgreSQL, MySQL, MariaDB, SQLite and SQL Server), a per-execution distributed lock, a + `database` store (proven on PostgreSQL, MySQL, MariaDB and SQLite; SQL Server is written for and has + integration tests that have not yet run against a real server), a per-execution distributed lock, a scheduled recovery sweep that reports, resumes or compensates stale executions, retention cleanup, and a dead-letter queue with retry. - **Orchestration operations:** lifecycle and step events through `firefly/eda`, event-type triggers, cron and diff --git a/docs/modules/orchestration.md b/docs/modules/orchestration.md index 66a67524..800088f0 100644 --- a/docs/modules/orchestration.md +++ b/docs/modules/orchestration.md @@ -235,7 +235,7 @@ one with new input, linked through the `x-firefly-continued-from` header. |---|---| | `memory` (default) | Tests and single-request flows. Nothing survives the process. | | `cache` | Any Laravel cache store (`persistence.cache-store`), keys under `persistence.key-prefix`. | -| `database` | The `firefly_orchestration_*` tables on `persistence.connection`. Proven on PostgreSQL, MySQL, MariaDB and SQLite, and on SQL Server in CI. | +| `database` | The `firefly_orchestration_*` tables on `persistence.connection`. Proven on PostgreSQL, MySQL, MariaDB and SQLite. SQL Server takes the same `upsert()` path (`MERGE`) and has integration tests behind `FIREFLY_SQLSRV_DSN`, but they have not yet run against a real server. | Publish the migration with `php artisan vendor:publish --tag=firefly-orchestration-migrations`; the package also loads it while `persistence.provider` or `dlq.store` is `database`. From b6792ef7a0cd117765e0d0fd1fe6124bba378670 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 23:20:42 -0700 Subject: [PATCH 44/58] test(orchestration): run ASYNC workflows through a real Laravel queue in the capstone and on every database --- CHANGELOG.md | 5 ++ docs/modules/orchestration.md | 56 +++++++++++++-- .../CapstoneInlineQueuedWorkflowTest.php | 24 +++++++ .../Capstone/CapstoneQueuedWorkflowTest.php | 65 ++++++++++++++++++ .../CapstoneFixtures/StatementWorkflow.php | 35 ++++++++++ .../QueuedWorkflowIntegrationTest.php | 43 ++++++++++++ .../InlineQueuedWorkflowCapstoneTestCase.php | 21 ++++++ .../tests/Support/IntegrationDatabase.php | 22 ++++-- .../Support/OrchestrationCapstoneTestCase.php | 2 + .../QueuedWorkflowCapstoneTestCase.php | 68 +++++++++++++++++++ .../QueuedWorkflowIntegrationTestCase.php | 53 +++++++++++++++ .../tests/Support/RecordingBusDispatcher.php | 61 +++++++++++++++++ .../Workflow/Queue/WorkflowQueueTest.php | 60 ++++++++++++++++ skeleton/config/firefly.php | 3 +- 14 files changed, 507 insertions(+), 11 deletions(-) create mode 100644 packages/orchestration/tests/Capstone/CapstoneInlineQueuedWorkflowTest.php create mode 100644 packages/orchestration/tests/Capstone/CapstoneQueuedWorkflowTest.php create mode 100644 packages/orchestration/tests/CapstoneFixtures/StatementWorkflow.php create mode 100644 packages/orchestration/tests/Integration/QueuedWorkflowIntegrationTest.php create mode 100644 packages/orchestration/tests/Support/InlineQueuedWorkflowCapstoneTestCase.php create mode 100644 packages/orchestration/tests/Support/QueuedWorkflowCapstoneTestCase.php create mode 100644 packages/orchestration/tests/Support/QueuedWorkflowIntegrationTestCase.php create mode 100644 packages/orchestration/tests/Support/RecordingBusDispatcher.php create mode 100644 packages/orchestration/tests/Workflow/Queue/WorkflowQueueTest.php diff --git a/CHANGELOG.md b/CHANGELOG.md index 87bf8052..463f7719 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,11 @@ All notable changes to LaraFly are documented here. This project uses CalVer (`Y - **Orchestration operations:** lifecycle and step events through `firefly/eda`, event-type triggers, cron and fixed-rate schedules, the `orchestration` actuator endpoint and health component, `php artisan firefly:orchestration:{list,show,signal,dlq,dlq:retry,tick}` and an opt-in REST surface. +- **Orchestration queued workflows:** ASYNC workflows are now tested end to end through a real Laravel queue. + A capstone checks that the `RunWorkflowJob` is pushed on `firefly.orchestration.workflow.queue.connection` + and `.name`, that `queue:work` runs it to `COMPLETED`, that a redelivered job is a no-op, and that the + `sync` connection runs the job inside the start. An integration test runs the same path on the `database` + queue against PostgreSQL, MySQL and MariaDB (its SQL Server case has not yet run against a real server). - **Observability:** `orchestration_executions_seconds`, `orchestration_steps_seconds`, `orchestration_compensations_total` and `orchestration_dead_letters_total`, and INTERNAL spans per execution and step behind `firefly.observability.tracing.orchestration.enabled`. diff --git a/docs/modules/orchestration.md b/docs/modules/orchestration.md index 800088f0..4b98548d 100644 --- a/docs/modules/orchestration.md +++ b/docs/modules/orchestration.md @@ -222,10 +222,58 @@ final class Onboarding ``` `suspend()`, `resume()`, `cancel()` and `compensate()` manage a live execution; -`upsertSearchAttributes()` and `search()` find executions by attribute. A workflow with -`triggerMode: TriggerMode::Async` is started `PENDING` and run by `RunWorkflowJob` on the queue named by -`firefly.orchestration.workflow.queue.*`. A step that returns `ContinueAsNew` ends the run and starts a fresh -one with new input, linked through the `x-firefly-continued-from` header. +`upsertSearchAttributes()` and `search()` find executions by attribute. A step that returns `ContinueAsNew` +ends the run and starts a fresh one with new input, linked through the `x-firefly-continued-from` header. + +### Queued (ASYNC) workflows + +A workflow declared with `triggerMode: TriggerMode::Async` never runs inside `start()`. The start persists +the execution `PENDING` and dispatches a `RunWorkflowJob` that carries only the correlation id. A queue +worker then loads the execution from the persistence provider and runs it: + + +```php +#[Service] +#[Workflow(id: 'statement', triggerMode: TriggerMode::Async)] +final class StatementWorkflow +{ + public function __construct(private readonly Ledger $ledger) {} + + #[WorkflowStep(id: 'render')] + public function render(#[Input('account')] string $account): string + { + $this->ledger->record("render {$account}"); + + return "statement-{$account}"; + } + // … +} +``` + +The job is dispatched on `firefly.orchestration.workflow.queue.connection` and `.name`. When either is unset, +the application's default connection or queue is used: + + +```php +'queue' => [ + 'connection' => env('FIREFLY_ORCHESTRATION_WORKFLOW_QUEUE_CONNECTION'), + 'name' => env('FIREFLY_ORCHESTRATION_WORKFLOW_QUEUE'), +], +``` + +A worker on that connection and queue runs the job: + +```bash +php artisan queue:work database --queue=workflows +``` + +The worker is a different process, so ASYNC workflows need `cache` or `database` persistence. The `sync` +connection is the one exception: it runs the job inside `start()`, and the start returns the finished +execution. Redelivery is safe. A job delivered a second time, for example by a worker that died before it +acknowledged the job, finds the execution no longer `PENDING` and returns it unchanged, so no step runs twice. +The capstone in `packages/orchestration/tests/Capstone/CapstoneQueuedWorkflowTest.php` covers this path on +the `database` queue. Its integration twin covers it on PostgreSQL, MySQL and MariaDB. A SQL Server case is +written but has not yet run against a real server. ## Persistence, recovery and the dead-letter queue diff --git a/packages/orchestration/tests/Capstone/CapstoneInlineQueuedWorkflowTest.php b/packages/orchestration/tests/Capstone/CapstoneInlineQueuedWorkflowTest.php new file mode 100644 index 00000000..bd5d9e42 --- /dev/null +++ b/packages/orchestration/tests/Capstone/CapstoneInlineQueuedWorkflowTest.php @@ -0,0 +1,24 @@ +app()->make(WorkflowEngine::class); + + $started = $workflows->start('statement', ['account' => 'acc-3'], correlationId: 'stmt-sync'); + + expect($started->status)->toBe(ExecutionStatus::Completed) + ->and($workflows->find('stmt-sync')?->status)->toBe(ExecutionStatus::Completed) + ->and(DB::table(QueuedWorkflowCapstoneTestCase::JOBS_TABLE)->count())->toBe(0) + ->and($this->app()->make(Ledger::class)->lines)->toBe(['hook start statement', 'render acc-3', 'deliver acc-3']); +}); diff --git a/packages/orchestration/tests/Capstone/CapstoneQueuedWorkflowTest.php b/packages/orchestration/tests/Capstone/CapstoneQueuedWorkflowTest.php new file mode 100644 index 00000000..0105298d --- /dev/null +++ b/packages/orchestration/tests/Capstone/CapstoneQueuedWorkflowTest.php @@ -0,0 +1,65 @@ +app()->make(WorkflowEngine::class); + + $started = $workflows->start('statement', ['account' => 'acc-1'], correlationId: 'stmt-fake'); + + expect($this->app()->make(WorkflowDispatcher::class))->toBeInstanceOf(BusWorkflowDispatcher::class) + ->and($started->status)->toBe(ExecutionStatus::Pending) + ->and($workflows->find('stmt-fake')?->status)->toBe(ExecutionStatus::Pending) + ->and($this->app()->make(Ledger::class)->lines)->toBe([]); + Queue::assertPushedOn('workflows', RunWorkflowJob::class, static fn (RunWorkflowJob $job): bool => $job->correlationId === 'stmt-fake' && $job->connection === 'database'); + Queue::assertCount(1); +}); + +it('runs a PENDING workflow in a queue worker to COMPLETED and treats a redelivered job as a no-op', function (): void { + /** @var QueuedWorkflowCapstoneTestCase $this */ + $workflows = $this->app()->make(WorkflowEngine::class); + $ledger = $this->app()->make(Ledger::class); + + $started = $workflows->start('statement', ['account' => 'acc-2'], correlationId: 'stmt-1'); + $queued = DB::table(QueuedWorkflowCapstoneTestCase::JOBS_TABLE)->get(['queue', 'payload'])->map(static fn (object $row): array => (array) $row)->all(); + $payload = json_decode(is_string($queued[0]['payload'] ?? null) ? $queued[0]['payload'] : '', true); + + expect($started->status)->toBe(ExecutionStatus::Pending) + ->and($ledger->lines)->toBe([]) + ->and($queued)->toHaveCount(1) + ->and($queued[0]['queue'] ?? null)->toBe('workflows') + ->and(data_get($payload, 'displayName'))->toBe(RunWorkflowJob::class); + + $exit = $this->work(); + $finished = $workflows->find('stmt-1'); + + expect($exit)->toBe(0) + ->and($finished?->status)->toBe(ExecutionStatus::Completed) + ->and($finished?->stepResults)->toBe(['render' => 'statement-acc-2', 'deliver' => 'delivered']) + ->and($ledger->lines)->toBe(['hook start statement', 'render acc-2', 'deliver acc-2']) + ->and(DB::table(QueuedWorkflowCapstoneTestCase::JOBS_TABLE)->count())->toBe(0); + + // A redelivered job (a worker that died after the run but before the ack) finds a COMPLETED execution. + $this->app()->make(Dispatcher::class)->dispatch((new RunWorkflowJob('stmt-1'))->onConnection('database')->onQueue('workflows')); + $redelivered = $this->work(); + + expect($redelivered)->toBe(0) + ->and(DB::table(QueuedWorkflowCapstoneTestCase::JOBS_TABLE)->count())->toBe(0) + ->and($workflows->find('stmt-1')?->toArray())->toBe($finished?->toArray()) + ->and($ledger->lines)->toBe(['hook start statement', 'render acc-2', 'deliver acc-2']); +}); diff --git a/packages/orchestration/tests/CapstoneFixtures/StatementWorkflow.php b/packages/orchestration/tests/CapstoneFixtures/StatementWorkflow.php new file mode 100644 index 00000000..b65d5068 --- /dev/null +++ b/packages/orchestration/tests/CapstoneFixtures/StatementWorkflow.php @@ -0,0 +1,35 @@ +ledger->record("render {$account}"); + + return "statement-{$account}"; + } + + #[WorkflowStep(id: 'deliver', dependsOn: ['render'])] + public function deliver(#[Input('account')] string $account): string + { + $this->ledger->record("deliver {$account}"); + + return 'delivered'; + } +} diff --git a/packages/orchestration/tests/Integration/QueuedWorkflowIntegrationTest.php b/packages/orchestration/tests/Integration/QueuedWorkflowIntegrationTest.php new file mode 100644 index 00000000..2a8e6b3e --- /dev/null +++ b/packages/orchestration/tests/Integration/QueuedWorkflowIntegrationTest.php @@ -0,0 +1,43 @@ +app()->make(WorkflowEngine::class); + $ledger = $this->app()->make(Ledger::class); + + $started = $workflows->start('statement', ['account' => 'acc-ü'], correlationId: 'stmt-it'); + $queued = DB::table(QueuedWorkflowCapstoneTestCase::JOBS_TABLE)->pluck('queue')->all(); + + expect(DB::connection()->getDriverName())->toBe(IntegrationDatabase::servers()[$server][0]) + ->and($started->status)->toBe(ExecutionStatus::Pending) + ->and($queued)->toBe(['workflows']); + + $exit = $this->work(); + $finished = $workflows->find('stmt-it'); + + expect($exit)->toBe(0) + ->and($finished?->status)->toBe(ExecutionStatus::Completed) + ->and($finished?->stepResults)->toBe(['render' => 'statement-acc-ü', 'deliver' => 'delivered']) + ->and(DB::table(QueuedWorkflowCapstoneTestCase::JOBS_TABLE)->count())->toBe(0); + + $this->app()->make(Dispatcher::class)->dispatch((new RunWorkflowJob('stmt-it'))->onConnection('database')->onQueue('workflows')); + + expect($this->work())->toBe(0) + ->and(DB::table(QueuedWorkflowCapstoneTestCase::JOBS_TABLE)->count())->toBe(0) + ->and($workflows->find('stmt-it')?->toArray())->toBe($finished?->toArray()) + ->and($ledger->lines)->toBe(['hook start statement', 'render acc-ü', 'deliver acc-ü']); +})->with(array_keys(IntegrationDatabase::servers()))->group('integration'); diff --git a/packages/orchestration/tests/Support/InlineQueuedWorkflowCapstoneTestCase.php b/packages/orchestration/tests/Support/InlineQueuedWorkflowCapstoneTestCase.php new file mode 100644 index 00000000..a9f72f24 --- /dev/null +++ b/packages/orchestration/tests/Support/InlineQueuedWorkflowCapstoneTestCase.php @@ -0,0 +1,21 @@ + */ + protected function configOverrides(): array + { + return [ + ...parent::configOverrides(), + 'firefly.orchestration.workflow.queue.connection' => 'sync', + ]; + } +} diff --git a/packages/orchestration/tests/Support/IntegrationDatabase.php b/packages/orchestration/tests/Support/IntegrationDatabase.php index 24f550e4..754c632e 100644 --- a/packages/orchestration/tests/Support/IntegrationDatabase.php +++ b/packages/orchestration/tests/Support/IntegrationDatabase.php @@ -39,6 +39,20 @@ public static function skipReason(string $variable, string $extension): ?string } public static function connection(string $driver, string $variable): Connection + { + $capsule = new Manager; + $capsule->addConnection(self::config($driver, $variable), $variable); + + return $capsule->getConnection($variable); + } + + /** + * The Laravel connection config for one server — what connection() opens, and what a booted capstone sets + * under `database.connections.*` to run the real pipeline against that server. + * + * @return array + */ + public static function config(string $driver, string $variable): array { $dsn = getenv($variable); if (! is_string($dsn) || $dsn === '') { @@ -60,16 +74,12 @@ public static function connection(string $driver, string $variable): Connection 'password' => $parts['password'] ?? '', 'prefix' => '', ]; - $config += match ($driver) { + + return $config + match ($driver) { 'pgsql' => ['charset' => 'utf8', 'search_path' => 'public', 'sslmode' => 'prefer'], 'sqlsrv' => ['charset' => 'utf8', 'trust_server_certificate' => true], default => ['charset' => 'utf8mb4', 'collation' => 'utf8mb4_unicode_ci'], }; - - $capsule = new Manager; - $capsule->addConnection($config, $variable); - - return $capsule->getConnection($variable); } public static function sqlite(string $database = ':memory:'): Connection diff --git a/packages/orchestration/tests/Support/OrchestrationCapstoneTestCase.php b/packages/orchestration/tests/Support/OrchestrationCapstoneTestCase.php index b3a7999e..b8c0a2b6 100644 --- a/packages/orchestration/tests/Support/OrchestrationCapstoneTestCase.php +++ b/packages/orchestration/tests/Support/OrchestrationCapstoneTestCase.php @@ -69,6 +69,8 @@ protected function setUp(): void { parent::setUp(); + // Dropped first: on a real server (the integration lane's subclasses) the tables of the last run remain. + OrchestrationSchema::drop(DB::connection()->getSchemaBuilder()); OrchestrationSchema::create(DB::connection()->getSchemaBuilder()); } diff --git a/packages/orchestration/tests/Support/QueuedWorkflowCapstoneTestCase.php b/packages/orchestration/tests/Support/QueuedWorkflowCapstoneTestCase.php new file mode 100644 index 00000000..38a4fe93 --- /dev/null +++ b/packages/orchestration/tests/Support/QueuedWorkflowCapstoneTestCase.php @@ -0,0 +1,68 @@ + */ + protected function configOverrides(): array + { + return [ + ...parent::configOverrides(), + 'queue.connections.database' => [ + 'driver' => 'database', + 'connection' => null, + 'table' => self::JOBS_TABLE, + 'queue' => 'default', + 'retry_after' => 90, + 'after_commit' => false, + ], + 'queue.failed.driver' => 'null', + 'firefly.orchestration.workflow.queue.connection' => 'database', + 'firefly.orchestration.workflow.queue.name' => 'workflows', + ]; + } + + protected function setUp(): void + { + parent::setUp(); + + self::createJobsTable(DB::connection()->getSchemaBuilder()); + } + + /** Laravel's own jobs-table shape (the `queue:table` migration), dropped first so a rerun starts empty. */ + public static function createJobsTable(Builder $schema): void + { + $schema->dropIfExists(self::JOBS_TABLE); + $schema->create(self::JOBS_TABLE, static function (Blueprint $table): void { + $table->id(); + $table->string('queue')->index(); + $table->longText('payload'); + $table->unsignedTinyInteger('attempts'); + $table->unsignedInteger('reserved_at')->nullable(); + $table->unsignedInteger('available_at'); + $table->unsignedInteger('created_at'); + }); + } + + /** Runs ONE job the way a deployed worker does: `queue:work database --queue=workflows --once`. */ + public function work(): int + { + return $this->command('queue:work', ['connection' => 'database', '--queue' => 'workflows', '--once' => true, '--stop-when-empty' => true]); + } +} diff --git a/packages/orchestration/tests/Support/QueuedWorkflowIntegrationTestCase.php b/packages/orchestration/tests/Support/QueuedWorkflowIntegrationTestCase.php new file mode 100644 index 00000000..507fef0d --- /dev/null +++ b/packages/orchestration/tests/Support/QueuedWorkflowIntegrationTestCase.php @@ -0,0 +1,53 @@ +server(); + $reason = IntegrationDatabase::skipReason($variable, $extension); + if ($reason !== null) { + $this->markTestSkipped($reason); + } + + parent::setUp(); + } + + protected function resolveApplicationConfiguration($app): void + { + parent::resolveApplicationConfiguration($app); + + [$driver, $variable] = $this->server(); + /** @var Repository $config */ + $config = $app->make('config'); + $config->set('database.connections.'.self::CONNECTION, IntegrationDatabase::config($driver, $variable)); + $config->set('database.default', self::CONNECTION); + } + + /** @return array{0: string, 1: string, 2: string} */ + private function server(): array + { + $name = $this->providedData()[0] ?? null; + $servers = IntegrationDatabase::servers(); + if (! is_string($name) || ! isset($servers[$name])) { + throw new \LogicException('A queued-workflow integration test needs a dataset naming one of IntegrationDatabase::servers().'); + } + + return $servers[$name]; + } +} diff --git a/packages/orchestration/tests/Support/RecordingBusDispatcher.php b/packages/orchestration/tests/Support/RecordingBusDispatcher.php new file mode 100644 index 00000000..98ddb2d6 --- /dev/null +++ b/packages/orchestration/tests/Support/RecordingBusDispatcher.php @@ -0,0 +1,61 @@ + */ + public array $dispatched = []; + + public function dispatch($command): mixed + { + $this->dispatched[] = $command; + + return null; + } + + public function dispatchSync($command, $handler = null): mixed + { + return null; + } + + public function dispatchNow($command, $handler = null): mixed + { + return null; + } + + public function dispatchAfterResponse($command, $handler = null): void {} + + /** @param mixed $jobs */ + public function chain($jobs = null): mixed + { + return null; + } + + public function hasCommandHandler($command): bool + { + return false; + } + + public function getCommandHandler($command): bool + { + return false; + } + + /** @param array $pipes */ + public function pipeThrough(array $pipes): static + { + return $this; + } + + /** @param array $map */ + public function map(array $map): static + { + return $this; + } +} diff --git a/packages/orchestration/tests/Workflow/Queue/WorkflowQueueTest.php b/packages/orchestration/tests/Workflow/Queue/WorkflowQueueTest.php new file mode 100644 index 00000000..037abc00 --- /dev/null +++ b/packages/orchestration/tests/Workflow/Queue/WorkflowQueueTest.php @@ -0,0 +1,60 @@ +dispatch('wf-1'); + + $job = $bus->dispatched[0] ?? null; + expect($bus->dispatched)->toHaveCount(1) + ->and($job)->toBeInstanceOf(RunWorkflowJob::class) + ->and($job instanceof RunWorkflowJob ? [$job->correlationId, $job->connection, $job->queue] : [])->toBe(['wf-1', 'redis', 'workflows']); +}); + +it('leaves the connection and queue to the application defaults when neither is configured', function (): void { + $bus = new RecordingBusDispatcher; + + (new BusWorkflowDispatcher($bus))->dispatch('wf-2'); + + $job = $bus->dispatched[0] ?? null; + expect($job instanceof RunWorkflowJob ? [$job->correlationId, $job->connection, $job->queue] : [])->toBe(['wf-2', null, null]); +}); + +it('survives the queue payload round trip with its connection and queue', function (): void { + $job = (new RunWorkflowJob('wf-3'))->onConnection('database')->onQueue('workflows'); + + $restored = unserialize(serialize($job)); + + expect($restored)->toBeInstanceOf(RunWorkflowJob::class) + ->and($restored instanceof RunWorkflowJob ? [$restored->correlationId, $restored->connection, $restored->queue] : [])->toBe(['wf-3', 'database', 'workflows']); +}); + +it('runs the PENDING execution it names, and a second delivery returns it untouched', function (): void { + $engines = TestEngines::create(); + $runs = 0; + $engines->workflowRegistry->register(WorkflowBuilder::workflow('report')->triggerMode(TriggerMode::Async) + ->step('build')->handler(static function () use (&$runs): string { + $runs++; + + return 'pdf'; + })->add()->build()); + $engines->workflows->start('report', correlationId: 'wf-4'); + + (new RunWorkflowJob('wf-4'))->handle($engines->workflows); + $first = $engines->workflows->find('wf-4'); + (new RunWorkflowJob('wf-4'))->handle($engines->workflows); + + expect($first?->status)->toBe(ExecutionStatus::Completed) + ->and($engines->workflows->find('wf-4')?->toArray())->toBe($first?->toArray()) + ->and($runs)->toBe(1); +}); diff --git a/skeleton/config/firefly.php b/skeleton/config/firefly.php index 83b2d5a7..73ecba7e 100644 --- a/skeleton/config/firefly.php +++ b/skeleton/config/firefly.php @@ -2611,7 +2611,8 @@ /* | Workflows triggered ASYNC are persisted PENDING and run by a RunWorkflowJob on this queue | (null = the default connection / queue); they need `cache` or `database` persistence, because the - | worker is another process. The `tick` is a scheduled task (firefly:schedule lists it) that resumes + | worker is another process (the `sync` connection runs the job inside the start instead). A job + | delivered twice is harmless: the second delivery finds the execution no longer PENDING. The `tick` is a scheduled task (firefly:schedule lists it) that resumes | WAITING workflows whose timers or deadlines are due, under the DistributedLock; without a scheduler | run php artisan firefly:orchestration:tick from cron. Defaults: enabled true, tick every 1m. */ From ba9eef3721f0a233b18e23029d026d707dc3b2fe Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 23:25:15 -0700 Subject: [PATCH 45/58] test(orchestration): cover the #[CompensationStep] workflow form in the scanner, the capstone and on every database --- CHANGELOG.md | 5 ++ docs/modules/orchestration.md | 47 +++++++++++++++++ .../tests/Capstone/CapstoneWorkflowTest.php | 29 +++++++++++ .../CapstoneFixtures/ShipmentWorkflow.php | 51 +++++++++++++++++++ .../WorkflowCompensationIntegrationTest.php | 31 +++++++++++ .../Scanner/OrchestrationScannerTest.php | 14 +++++ .../BothCompensationFormsWorkflow.php | 24 +++++++++ .../DuplicateCompensationWorkflow.php | 25 +++++++++ .../UnknownCompensationWorkflow.php | 22 ++++++++ .../ScannerFixtures/Good/ShipmentWorkflow.php | 30 +++++++++++ .../Support/BootsOnIntegrationServer.php | 51 +++++++++++++++++++ .../OrchestrationIntegrationTestCase.php | 14 +++++ .../QueuedWorkflowIntegrationTestCase.php | 46 ++--------------- 13 files changed, 347 insertions(+), 42 deletions(-) create mode 100644 packages/orchestration/tests/CapstoneFixtures/ShipmentWorkflow.php create mode 100644 packages/orchestration/tests/Integration/WorkflowCompensationIntegrationTest.php create mode 100644 packages/orchestration/tests/ScannerFixtures/Bad/BothCompensationForms/BothCompensationFormsWorkflow.php create mode 100644 packages/orchestration/tests/ScannerFixtures/Bad/DuplicateCompensation/DuplicateCompensationWorkflow.php create mode 100644 packages/orchestration/tests/ScannerFixtures/Bad/UnknownCompensation/UnknownCompensationWorkflow.php create mode 100644 packages/orchestration/tests/ScannerFixtures/Good/ShipmentWorkflow.php create mode 100644 packages/orchestration/tests/Support/BootsOnIntegrationServer.php create mode 100644 packages/orchestration/tests/Support/OrchestrationIntegrationTestCase.php diff --git a/CHANGELOG.md b/CHANGELOG.md index 463f7719..603acef8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,11 @@ All notable changes to LaraFly are documented here. This project uses CalVer (`Y and `.name`, that `queue:work` runs it to `COMPLETED`, that a redelivered job is a no-op, and that the `sync` connection runs the job inside the start. An integration test runs the same path on the `database` queue against PostgreSQL, MySQL and MariaDB (its SQL Server case has not yet run against a real server). +- **Orchestration workflow compensation steps:** the separate-method `#[CompensationStep(compensates: …)]` + form is now tested and documented. The scanner tests check that the method becomes the step's compensation + with the step result bound implicitly, and that a duplicate, a step with both forms and an unknown step are + refused. A capstone and an integration test on PostgreSQL, MySQL and MariaDB check that a failing + discovered workflow runs it (its SQL Server case has not yet run against a real server). - **Observability:** `orchestration_executions_seconds`, `orchestration_steps_seconds`, `orchestration_compensations_total` and `orchestration_dead_letters_total`, and INTERNAL spans per execution and step behind `firefly.observability.tracing.orchestration.enabled`. diff --git a/docs/modules/orchestration.md b/docs/modules/orchestration.md index 4b98548d..d82c6361 100644 --- a/docs/modules/orchestration.md +++ b/docs/modules/orchestration.md @@ -225,6 +225,53 @@ final class Onboarding `upsertSearchAttributes()` and `search()` find executions by attribute. A step that returns `ContinueAsNew` ends the run and starts a fresh one with new input, linked through the `x-firefly-continued-from` header. +### Compensating a workflow + +When a workflow step fails (after its retries), the engine undoes every completed step that has a +compensation, newest first, dead-letters the failure and ends the execution `FAILED`. A step declares its +compensation inline, with `compensatable: true, compensationMethod: 'name'` on its `#[WorkflowStep]`, or as a +separate method marked `#[CompensationStep(compensates: 'step-id')]`, which makes the step compensatable on +its own. Either way the method receives the step's result as its implicit argument (and may take the usual +`#[Input]`, `#[Variable]` or `ExecutionContext` parameters); what it returns is recorded under +`compensationResults`. A step has one compensation: two `#[CompensationStep]` methods for one step, both forms +on one step, or a `#[CompensationStep]` naming a step the workflow does not declare are refused with a +`TopologyValidationException` when the definitions are scanned. + + +```php +use Firefly\Container\Attributes\Service; +use Firefly\Orchestration\Core\Argument\Input; +use Firefly\Orchestration\Workflow\Attributes\CompensationStep; +use Firefly\Orchestration\Workflow\Attributes\Workflow; +use Firefly\Orchestration\Workflow\Attributes\WorkflowStep; + +#[Service] +#[Workflow(id: 'shipment')] +final class ShipmentWorkflow +{ + #[WorkflowStep(id: 'book')] + public function book(#[Input('parcel')] string $parcel): string + { + return "label-{$parcel}"; + } + + #[CompensationStep(compensates: 'book')] + public function voidLabel(string $label): string + { + return "voided {$label}"; + } + + #[WorkflowStep(id: 'dispatch', dependsOn: ['book'])] + public function dispatch(#[Input('carrier')] string $carrier): string + { + return "dispatched via {$carrier}"; + } +} +``` + +`WorkflowEngine::compensate()` runs the same compensations on request for a live execution and ends it +`CANCELLED`. + ### Queued (ASYNC) workflows A workflow declared with `triggerMode: TriggerMode::Async` never runs inside `start()`. The start persists diff --git a/packages/orchestration/tests/Capstone/CapstoneWorkflowTest.php b/packages/orchestration/tests/Capstone/CapstoneWorkflowTest.php index cfd3a38f..7a036372 100644 --- a/packages/orchestration/tests/Capstone/CapstoneWorkflowTest.php +++ b/packages/orchestration/tests/Capstone/CapstoneWorkflowTest.php @@ -3,9 +3,12 @@ declare(strict_types=1); use Firefly\Orchestration\Core\ExecutionStatus; +use Firefly\Orchestration\Core\StepStatus; +use Firefly\Orchestration\Persistence\OrchestrationSchema; use Firefly\Orchestration\Tests\CapstoneFixtures\Ledger; use Firefly\Orchestration\Tests\Support\OrchestrationCapstoneTestCase; use Firefly\Orchestration\Workflow\Engine\WorkflowEngine; +use Illuminate\Support\Facades\DB; uses(OrchestrationCapstoneTestCase::class); @@ -28,3 +31,29 @@ ->and($workflows->query('onb-1', 'progress'))->toBe(3) ->and(array_map(static fn ($s): string => $s->correlationId, $workflows->search(['email' => 'ada@example.com'])))->toBe(['onb-1']); }); + +it('undoes a failed workflow through its #[CompensationStep] method, handing it the step result, and dead-letters the failure', function (): void { + /** @var OrchestrationCapstoneTestCase $this */ + $workflows = $this->app()->make(WorkflowEngine::class); + + $failed = $workflows->start('shipment', ['parcel' => 'p-7', 'carrier' => 'closed'], correlationId: 'ship-1'); + $stored = $workflows->find('ship-1'); + + expect($failed->status)->toBe(ExecutionStatus::Failed) + ->and($failed->failureReason)->toBe('carrier closed refused the parcel') + ->and($this->app()->make(Ledger::class)->lines)->toBe(['hook start shipment', 'book p-7', 'void label-p-7']) + ->and($stored?->stepStatus('book'))->toBe(StepStatus::Compensated) + ->and($stored?->compensationResults)->toBe(['book' => 'voided label-p-7']) + ->and(DB::table(OrchestrationSchema::EXECUTIONS)->where('correlation_id', 'ship-1')->value('status'))->toBe('FAILED') + ->and(DB::table(OrchestrationSchema::DEAD_LETTERS)->where('correlation_id', 'ship-1')->value('step_id'))->toBe('dispatch'); +}); + +it('leaves the #[CompensationStep] method alone when the workflow completes', function (): void { + /** @var OrchestrationCapstoneTestCase $this */ + $finished = $this->app()->make(WorkflowEngine::class)->start('shipment', ['parcel' => 'p-8', 'carrier' => 'post'], correlationId: 'ship-2'); + + expect($finished->status)->toBe(ExecutionStatus::Completed) + ->and($finished->stepStatus('book'))->toBe(StepStatus::Done) + ->and($finished->compensationResults)->toBe([]) + ->and($this->app()->make(Ledger::class)->lines)->toBe(['hook start shipment', 'book p-8', 'dispatch via post']); +}); diff --git a/packages/orchestration/tests/CapstoneFixtures/ShipmentWorkflow.php b/packages/orchestration/tests/CapstoneFixtures/ShipmentWorkflow.php new file mode 100644 index 00000000..c34f8b0b --- /dev/null +++ b/packages/orchestration/tests/CapstoneFixtures/ShipmentWorkflow.php @@ -0,0 +1,51 @@ +ledger->record("book {$parcel}"); + + return "label-{$parcel}"; + } + + #[CompensationStep(compensates: 'book')] + public function voidLabel(string $label): string + { + $this->ledger->record("void {$label}"); + + return "voided {$label}"; + } + + #[WorkflowStep(id: 'dispatch', dependsOn: ['book'])] + public function dispatch(#[Input('carrier')] string $carrier): string + { + if ($carrier === 'closed') { + throw new RuntimeException("carrier {$carrier} refused the parcel"); + } + $this->ledger->record("dispatch via {$carrier}"); + + return 'dispatched'; + } +} diff --git a/packages/orchestration/tests/Integration/WorkflowCompensationIntegrationTest.php b/packages/orchestration/tests/Integration/WorkflowCompensationIntegrationTest.php new file mode 100644 index 00000000..cf001f97 --- /dev/null +++ b/packages/orchestration/tests/Integration/WorkflowCompensationIntegrationTest.php @@ -0,0 +1,31 @@ +app()->make(WorkflowEngine::class); + + $failed = $workflows->start('shipment', ['parcel' => 'pä-7', 'carrier' => 'closed'], correlationId: 'ship-it'); + $stored = $workflows->find('ship-it'); + + expect(DB::connection()->getDriverName())->toBe(IntegrationDatabase::servers()[$server][0]) + ->and($failed->status)->toBe(ExecutionStatus::Failed) + ->and($this->app()->make(Ledger::class)->lines)->toBe(['hook start shipment', 'book pä-7', 'void label-pä-7']) + ->and($stored?->stepStatus('book'))->toBe(StepStatus::Compensated) + ->and($stored?->stepResults['book'] ?? null)->toBe('label-pä-7') + ->and($stored?->compensationResults)->toBe(['book' => 'voided label-pä-7']) + ->and(DB::table(OrchestrationSchema::EXECUTIONS)->where('correlation_id', 'ship-it')->value('status'))->toBe('FAILED') + ->and(DB::table(OrchestrationSchema::DEAD_LETTERS)->where('correlation_id', 'ship-it')->value('step_id'))->toBe('dispatch'); +})->with(array_keys(IntegrationDatabase::servers()))->group('integration'); diff --git a/packages/orchestration/tests/Scanner/OrchestrationScannerTest.php b/packages/orchestration/tests/Scanner/OrchestrationScannerTest.php index 1e8f8c32..99b0fa94 100644 --- a/packages/orchestration/tests/Scanner/OrchestrationScannerTest.php +++ b/packages/orchestration/tests/Scanner/OrchestrationScannerTest.php @@ -15,6 +15,7 @@ use Firefly\Orchestration\Tests\ScannerFixtures\Good\DebitParticipant; use Firefly\Orchestration\Tests\ScannerFixtures\Good\InventoryService; use Firefly\Orchestration\Tests\ScannerFixtures\Good\OrderSaga; +use Firefly\Orchestration\Tests\ScannerFixtures\Good\ShipmentWorkflow; use Firefly\Orchestration\Workflow\Registry\WaitDefinition; function orchestrationScanGood(): OrchestrationManifest @@ -85,6 +86,16 @@ function orchestrationScanGood(): OrchestrationManifest ->and($workflow->onError[0]->suppressError)->toBeTrue(); }); +it('compiles a #[CompensationStep] method as the compensation of the step it names, bound to that step\'s result', function (): void { + $workflow = orchestrationScanGood()->workflows()[1]; + + expect($workflow->id)->toBe('shipment') + ->and($workflow->step('book')->compensatable)->toBeTrue() + ->and($workflow->step('book')->compensation)->toEqual(new MethodInvocation(ShipmentWorkflow::class, 'voidLabel', [new ParameterBinding('label', ParameterBinding::IMPLICIT)])) + ->and($workflow->step('dispatch')->compensatable)->toBeFalse() + ->and($workflow->step('dispatch')->compensation)->toBeNull(); +}); + it('compiles the schedules', function (): void { $schedules = orchestrationScanGood()->schedules(); @@ -107,4 +118,7 @@ function orchestrationScanGood(): OrchestrationManifest ['BadCondition', 'does not parse'], ['InitialDelay', 'initialDelay'], ['OrphanExternal', "no #[Saga(name: 'ghost')]"], + ['DuplicateCompensation', 'Workflow [twice-undone] step [book] has two #[CompensationStep] methods.'], + ['UnknownCompensation', "#[CompensationStep(compensates: 'ship')] in workflow [stray-undo] names a step it does not declare."], + ['BothCompensationForms', 'Workflow [both-forms] step [book] declares compensationMethod: and a #[CompensationStep]; a step has one compensation.'], ]); diff --git a/packages/orchestration/tests/ScannerFixtures/Bad/BothCompensationForms/BothCompensationFormsWorkflow.php b/packages/orchestration/tests/ScannerFixtures/Bad/BothCompensationForms/BothCompensationFormsWorkflow.php new file mode 100644 index 00000000..d15189c3 --- /dev/null +++ b/packages/orchestration/tests/ScannerFixtures/Bad/BothCompensationForms/BothCompensationFormsWorkflow.php @@ -0,0 +1,24 @@ +server(); + $reason = IntegrationDatabase::skipReason($variable, $extension); + if ($reason !== null) { + $this->markTestSkipped($reason); + } + + parent::setUp(); + } + + protected function resolveApplicationConfiguration($app): void + { + parent::resolveApplicationConfiguration($app); + + [$driver, $variable] = $this->server(); + /** @var Repository $config */ + $config = $app->make('config'); + $config->set('database.connections.orchestration-it', IntegrationDatabase::config($driver, $variable)); + $config->set('database.default', 'orchestration-it'); + } + + /** @return array{0: string, 1: string, 2: string} */ + private function server(): array + { + $name = $this->providedData()[0] ?? null; + $servers = IntegrationDatabase::servers(); + if (! is_string($name) || ! isset($servers[$name])) { + throw new \LogicException('An orchestration integration test needs a dataset naming one of IntegrationDatabase::servers().'); + } + + return $servers[$name]; + } +} diff --git a/packages/orchestration/tests/Support/OrchestrationIntegrationTestCase.php b/packages/orchestration/tests/Support/OrchestrationIntegrationTestCase.php new file mode 100644 index 00000000..94769e63 --- /dev/null +++ b/packages/orchestration/tests/Support/OrchestrationIntegrationTestCase.php @@ -0,0 +1,14 @@ +server(); - $reason = IntegrationDatabase::skipReason($variable, $extension); - if ($reason !== null) { - $this->markTestSkipped($reason); - } - - parent::setUp(); - } - - protected function resolveApplicationConfiguration($app): void - { - parent::resolveApplicationConfiguration($app); - - [$driver, $variable] = $this->server(); - /** @var Repository $config */ - $config = $app->make('config'); - $config->set('database.connections.'.self::CONNECTION, IntegrationDatabase::config($driver, $variable)); - $config->set('database.default', self::CONNECTION); - } - - /** @return array{0: string, 1: string, 2: string} */ - private function server(): array - { - $name = $this->providedData()[0] ?? null; - $servers = IntegrationDatabase::servers(); - if (! is_string($name) || ! isset($servers[$name])) { - throw new \LogicException('A queued-workflow integration test needs a dataset naming one of IntegrationDatabase::servers().'); - } - - return $servers[$name]; - } + use BootsOnIntegrationServer; } From 236bbf66938e29d0c01a0023d2bc7a05bcf929c1 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 23:30:26 -0700 Subject: [PATCH 46/58] test(samples): register the firefly:eda:consume provider in the lumen Postgres outbox round-trip harness --- .../Integration/OutboxPostgresIntegrationTestCase.php | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/samples/lumen/tests/Integration/OutboxPostgresIntegrationTestCase.php b/samples/lumen/tests/Integration/OutboxPostgresIntegrationTestCase.php index d2d8ee32..58b86e8d 100644 --- a/samples/lumen/tests/Integration/OutboxPostgresIntegrationTestCase.php +++ b/samples/lumen/tests/Integration/OutboxPostgresIntegrationTestCase.php @@ -4,6 +4,7 @@ namespace Lumen\Tests\Integration; +use Firefly\Eda\EdaConsumerServiceProvider; use Firefly\Eda\Postgres\EdaPostgresServiceProvider; use Firefly\Eda\Postgres\Outbox\OutboxSchema; use Illuminate\Contracts\Config\Repository; @@ -32,10 +33,16 @@ public static function postgresAvailable(): bool return extension_loaded('pdo_pgsql') && getenv('FIREFLY_PG_DSN') !== false; } - /** @return list> */ + /** + * The postgres outbox plus the provider that registers firefly:eda:consume. A real app gets the consume command + * through package auto-discovery; Testbench ignores discovery, so the round-trip names it like + * EdaConsumeCommandTestCase does. + * + * @return list> + */ protected function fireflyProviders(): array { - return [...parent::fireflyProviders(), EdaPostgresServiceProvider::class]; + return [...parent::fireflyProviders(), EdaPostgresServiceProvider::class, EdaConsumerServiceProvider::class]; } /** @return array */ From 0ff0dd4c7977469ae704378c16b31f834eab2d66 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 23:45:56 -0700 Subject: [PATCH 47/58] fix(orchestration): end a saga whose compensation stopped part-way COMPENSATION_FAILED, dead-letter it and name the steps it left uncompensated --- docs/modules/orchestration.md | 2 +- .../Core/Exception/CompensationException.php | 15 +++- .../src/Core/ExecutionOutcome.php | 2 +- .../src/Core/ExecutionStatus.php | 6 +- .../src/Saga/Compensation/SagaCompensator.php | 28 ++++++-- .../src/Saga/Composition/SagaCompositor.php | 6 +- .../src/Saga/Engine/SagaEngine.php | 35 +++++++--- .../src/Saga/Engine/SagaResult.php | 13 +++- .../orchestration/tests/Core/EnumsTest.php | 2 +- .../tests/Saga/Engine/SagaEngineTest.php | 68 +++++++++++++++++++ 10 files changed, 149 insertions(+), 28 deletions(-) diff --git a/docs/modules/orchestration.md b/docs/modules/orchestration.md index d82c6361..5526cb55 100644 --- a/docs/modules/orchestration.md +++ b/docs/modules/orchestration.md @@ -91,7 +91,7 @@ Steps can also be declared without a class through `SagaBuilder::saga('checkout' | `strict-sequential` (default) | Undo in reverse completion order, one at a time. | | `grouped-parallel` | Undo layer by layer, in reverse layer order. | | `retry-with-backoff` | Like `strict-sequential`, retrying each compensation with its `compensationRetry` and `compensationBackoffMs`. | -| `circuit-breaker` | Stop compensating once a `compensationCritical` step fails or the breaker opens, and leave the rest to recovery. | +| `circuit-breaker` | Stop compensating once a `compensationCritical` step fails or the breaker opens; the saga ends `COMPENSATION_FAILED`, is dead-lettered, and lists the steps it left in `SagaResult::$uncompensatedSteps`. | | `best-effort` | Run every compensation and record each failure without stopping. | `firefly.orchestration.saga.compensation-error-handler` decides what one failed compensation does to the run: diff --git a/packages/orchestration/src/Core/Exception/CompensationException.php b/packages/orchestration/src/Core/Exception/CompensationException.php index c81bdfc3..90a33ffb 100644 --- a/packages/orchestration/src/Core/Exception/CompensationException.php +++ b/packages/orchestration/src/Core/Exception/CompensationException.php @@ -9,12 +9,21 @@ /** * Compensation stopped before every completed step was undone: an error handler answered FAIL_SAGA, or under * CIRCUIT_BREAKER the saga's breaker opened or a `compensationCritical` step failed. The message names the - * step that stopped it and becomes the execution's recorded reason. + * step that stopped it and becomes the execution's recorded reason; `stoppedAt` is that step and + * `uncompensatedSteps` every step with a compensation that was not undone, in compensation order. SagaEngine + * ends such an execution COMPENSATION_FAILED and dead-letters it. */ final class CompensationException extends OrchestrationException { - public function __construct(string $message, ?Throwable $previous = null) - { + /** + * @param list $uncompensatedSteps + */ + public function __construct( + string $message, + ?Throwable $previous = null, + public readonly ?string $stoppedAt = null, + public readonly array $uncompensatedSteps = [], + ) { parent::__construct($message, 'ORCHESTRATION_COMPENSATION_FAILED', 500, $previous); } } diff --git a/packages/orchestration/src/Core/ExecutionOutcome.php b/packages/orchestration/src/Core/ExecutionOutcome.php index 4bcb83bc..ac8b4feb 100644 --- a/packages/orchestration/src/Core/ExecutionOutcome.php +++ b/packages/orchestration/src/Core/ExecutionOutcome.php @@ -38,7 +38,7 @@ public static function fromTcc(TccResult $result): self public static function fromWorkflow(ExecutionState $state): self { - $success = ! in_array($state->status, [ExecutionStatus::Failed, ExecutionStatus::TimedOut, ExecutionStatus::Cancelled], true); + $success = ! in_array($state->status, [ExecutionStatus::Failed, ExecutionStatus::TimedOut, ExecutionStatus::Cancelled, ExecutionStatus::CompensationFailed], true); return new self($state->correlationId, ExecutionPattern::Workflow, $state->executionName, $state->status, $success, $state->toArray()); } diff --git a/packages/orchestration/src/Core/ExecutionStatus.php b/packages/orchestration/src/Core/ExecutionStatus.php index 094b02a1..f10a5073 100644 --- a/packages/orchestration/src/Core/ExecutionStatus.php +++ b/packages/orchestration/src/Core/ExecutionStatus.php @@ -6,7 +6,8 @@ /** * The lifecycle of an execution, shared by the three engines (jafly ExecutionStatus parity). Sagas use - * RUNNING → COMPENSATING → COMPLETED|FAILED; TCC uses TRYING → CONFIRMING → CONFIRMED, or CANCELING → + * RUNNING → COMPENSATING → COMPLETED|FAILED, or COMPENSATION_FAILED when the compensation stopped before every + * completed step was undone (the execution is dead-lettered for an operator); TCC uses TRYING → CONFIRMING → CONFIRMED, or CANCELING → * CANCELED|FAILED; workflows add PENDING (queued), WAITING (a signal, timer or child) and SUSPENDED. * * "Recoverable" is the set a crashed process can leave behind and the recovery sweep may act on. WAITING and @@ -29,6 +30,7 @@ enum ExecutionStatus: string case Canceling = 'CANCELING'; case Canceled = 'CANCELED'; case Compensating = 'COMPENSATING'; + case CompensationFailed = 'COMPENSATION_FAILED'; public function isTerminal(): bool { @@ -58,7 +60,7 @@ public function canResume(): bool /** @return list */ public static function terminal(): array { - return [self::Completed, self::Failed, self::Cancelled, self::TimedOut, self::Confirmed, self::Canceled]; + return [self::Completed, self::Failed, self::Cancelled, self::TimedOut, self::Confirmed, self::Canceled, self::CompensationFailed]; } /** @return list */ diff --git a/packages/orchestration/src/Saga/Compensation/SagaCompensator.php b/packages/orchestration/src/Saga/Compensation/SagaCompensator.php index f92753ab..37eb81f8 100644 --- a/packages/orchestration/src/Saga/Compensation/SagaCompensator.php +++ b/packages/orchestration/src/Saga/Compensation/SagaCompensator.php @@ -30,8 +30,9 @@ * - RETRY_WITH_BACKOFF: reverse completion order with the step's compensation retry/backoff/timeout. * - CIRCUIT_BREAKER: reverse completion order through a firefly/resilience breaker per saga; a failed * `compensationCritical` step, or an open breaker, stops the sweep and throws a CompensationException - * naming the step that stopped it and the steps left DONE. Nothing re-runs them automatically: the - * execution ends with that message as its reason, for an operator to act on. + * naming the step that stopped it and the steps left DONE. Nothing re-runs them automatically: SagaEngine + * ends the execution COMPENSATION_FAILED with that message as its reason and dead-letters it, for an + * operator to act on. * - BEST_EFFORT: reverse completion order, every compensation attempted, FAIL_SAGA ignored. * * The error handler's RETRY answer re-runs the compensation after the step's compensation backoff (under @@ -101,8 +102,8 @@ private function sequentially(SagaDefinition $saga, ExecutionContext $context, a * Compensates in reverse completion order through the saga's breaker. When the breaker refuses a step (or * opens during the sweep), or a `compensationCritical` step fails, the sweep stops and a * CompensationException names the step that stopped it and the steps left DONE. The engine records that - * message as the execution's reason, so a FAILED or CANCELLED saga never claims a compensation it did not - * finish. + * message as the execution's reason and ends it COMPENSATION_FAILED, so a saga never claims a compensation + * it did not finish. * * @param list $reversed */ @@ -175,7 +176,22 @@ private function throughBreaker(SagaDefinition $saga, ExecutionContext $context, $this->support->logger->warning($message.'.', ['correlationId' => $context->correlationId]); $this->support->checkpoint($context, ExecutionStatus::Compensating); - throw new CompensationException($message.($lastError === null ? '.' : ": {$lastError->getMessage()}"), $lastError); + throw new CompensationException($message.($lastError === null ? '.' : ": {$lastError->getMessage()}"), $lastError, $stoppedAt, $this->uncompensated($saga, $context)); + } + + /** + * The completed steps with a compensation that is not undone (still DONE, or COMPENSATION_FAILED), in + * reverse completion order. + * + * @return list + */ + private function uncompensated(SagaDefinition $saga, ExecutionContext $context): array + { + return array_values(array_filter( + array_reverse($context->completionOrder()), + static fn (string $stepId): bool => ($saga->steps[$stepId] ?? null)?->compensation !== null + && in_array($context->stepStatus($stepId), [StepStatus::Done, StepStatus::CompensationFailed], true), + )); } /** Runs one compensation; returns its error, or null when it succeeded. Throws only when `$breaker` refused admission. */ @@ -242,7 +258,7 @@ private function onFailure(SagaDefinition $saga, ExecutionContext $context, stri if ($decision === CompensationErrorResult::FailSaga) { if ($honourFailSaga) { - throw new CompensationException("Saga [{$saga->name}] stopped compensating at step [{$stepId}]: {$error->getMessage()}", $error); + throw new CompensationException("Saga [{$saga->name}] stopped compensating at step [{$stepId}]: {$error->getMessage()}", $error, $stepId, $this->uncompensated($saga, $context)); } return false; diff --git a/packages/orchestration/src/Saga/Composition/SagaCompositor.php b/packages/orchestration/src/Saga/Composition/SagaCompositor.php index 96ced9cd..35daadc2 100644 --- a/packages/orchestration/src/Saga/Composition/SagaCompositor.php +++ b/packages/orchestration/src/Saga/Composition/SagaCompositor.php @@ -34,8 +34,10 @@ * - compensate() refused before it started (the execution lock timed out, the store could not be read, the * execution was already cancelled): the execution keeps its status — a COMPLETED member stays COMPLETED, * which recovery never picks up, so the result is the only place the caller learns of it; - * - one of its compensations failed and the engine handled it (the error handler continued, FAIL_SAGA, an - * open compensation circuit): the execution ends CANCELLED with the failed steps marked in its report; + * - one of its compensations failed and the error handler continued: the execution ends CANCELLED with the + * failed steps marked in its report; + * - the compensation stopped part-way (FAIL_SAGA, an open compensation circuit, a failed critical step): the + * execution ends COMPENSATION_FAILED and is dead-lettered; * - compensate() threw after it started: the execution stays COMPENSATING, where recovery finds it. */ final class SagaCompositor diff --git a/packages/orchestration/src/Saga/Engine/SagaEngine.php b/packages/orchestration/src/Saga/Engine/SagaEngine.php index 6194e0f3..c1a40caf 100644 --- a/packages/orchestration/src/Saga/Engine/SagaEngine.php +++ b/packages/orchestration/src/Saga/Engine/SagaEngine.php @@ -113,14 +113,13 @@ public function compensate(string $correlationId): SagaResult $context = ExecutionContext::fromState($state); $this->support->checkpoint($context, ExecutionStatus::Compensating); - $reason = 'compensated on request'; try { $this->compensator->compensate($definition, $context); } catch (CompensationException $failure) { - $reason = $failure->getMessage(); + return $this->compensationFailed($definition, $context, $failure, $failure); } - return $this->finish($definition, $context, ExecutionStatus::Cancelled, null, $reason); + return $this->finish($definition, $context, ExecutionStatus::Cancelled, null, 'compensated on request'); }); } @@ -167,21 +166,39 @@ private function fail(SagaDefinition $saga, ExecutionContext $context, Throwable $this->support->deadLetter($context, $stepId, ExecutionStatus::Failed, $stepError); } - $suppressed = $compensationFailure === null && $this->support->callbacks->suppresses($saga->onError, $error); - $result = $suppressed - ? $this->finish($saga, $context, ExecutionStatus::Completed, null, null) - : $this->finish($saga, $context, ExecutionStatus::Failed, $error, $compensationFailure?->getMessage() ?? $error->getMessage()); + if ($compensationFailure !== null) { + $result = $this->compensationFailed($saga, $context, $compensationFailure, $error); + } elseif ($this->support->callbacks->suppresses($saga->onError, $error)) { + $result = $this->finish($saga, $context, ExecutionStatus::Completed, null, null); + } else { + $result = $this->finish($saga, $context, ExecutionStatus::Failed, $error, $error->getMessage()); + } $this->support->callbacks->invoke($saga->onError, $context, $result, $error); return $result; } - private function finish(SagaDefinition $saga, ExecutionContext $context, ExecutionStatus $status, ?Throwable $error, ?string $reason): SagaResult + /** + * A compensation that stopped before every completed step was undone (CompensationException) ends the saga + * COMPENSATION_FAILED, never FAILED or CANCELLED as if it had been rolled back: the steps it left are named + * in the reason and in SagaResult::$uncompensatedSteps, and the execution is dead-lettered for an operator. + */ + private function compensationFailed(SagaDefinition $saga, ExecutionContext $context, CompensationException $failure, Throwable $error): SagaResult + { + $this->support->deadLetter($context, $failure->stoppedAt, ExecutionStatus::CompensationFailed, $failure); + + return $this->finish($saga, $context, ExecutionStatus::CompensationFailed, $error, $failure->getMessage(), $failure->uncompensatedSteps); + } + + /** + * @param list $uncompensated + */ + private function finish(SagaDefinition $saga, ExecutionContext $context, ExecutionStatus $status, ?Throwable $error, ?string $reason, array $uncompensated = []): SagaResult { $now = $this->support->now(); $report = ExecutionReportBuilder::fromContext($context, $status, $reason, $now, $saga->stepIds()); - $result = SagaResult::from($saga, $context, $status, $error, $report, $now); + $result = SagaResult::from($saga, $context, $status, $error, $report, $now, $uncompensated); $this->support->checkpoint($context, $status, $reason, $report); $this->support->publish($this->support->lifecycle, OrchestrationEvent::executionCompleted($saga->name, $context->correlationId, ExecutionPattern::Saga, $status, $now)); diff --git a/packages/orchestration/src/Saga/Engine/SagaResult.php b/packages/orchestration/src/Saga/Engine/SagaResult.php index 523fe71b..9aa30675 100644 --- a/packages/orchestration/src/Saga/Engine/SagaResult.php +++ b/packages/orchestration/src/Saga/Engine/SagaResult.php @@ -16,13 +16,15 @@ /** * What SagaEngine::execute() returns (jafly SagaResult). A failed step is reported here, never thrown: check * isSuccess(), then failedSteps()/compensatedSteps()/error. `success` is false for a FAILED saga and true for - * a COMPLETED one — including one whose error an #[OnSagaError(suppressError: true)] suppressed. + * a COMPLETED one — including one whose error an #[OnSagaError(suppressError: true)] suppressed. A saga whose + * compensation stopped part-way is COMPENSATION_FAILED and lists the steps it left in `uncompensatedSteps`. */ final readonly class SagaResult { /** * @param array $steps * @param array $headers + * @param list $uncompensatedSteps the steps a COMPENSATION_FAILED saga left undone, in compensation order */ public function __construct( public string $sagaName, @@ -35,9 +37,13 @@ public function __construct( public ?ExecutionReport $report, public DateTimeImmutable $startedAt, public DateTimeImmutable $completedAt, + public array $uncompensatedSteps = [], ) {} - public static function from(SagaDefinition $saga, ExecutionContext $context, ExecutionStatus $status, ?Throwable $error, ?ExecutionReport $report, DateTimeImmutable $now): self + /** + * @param list $uncompensated + */ + public static function from(SagaDefinition $saga, ExecutionContext $context, ExecutionStatus $status, ?Throwable $error, ?ExecutionReport $report, DateTimeImmutable $now, array $uncompensated = []): self { $steps = []; foreach ($saga->stepIds() as $stepId) { @@ -53,7 +59,7 @@ public static function from(SagaDefinition $saga, ExecutionContext $context, Exe ); } - return new self($saga->name, $context->correlationId, $status, $status->isSuccess(), $error, $steps, $context->headers(), $report, $context->startedAt(), $now); + return new self($saga->name, $context->correlationId, $status, $status->isSuccess(), $error, $steps, $context->headers(), $report, $context->startedAt(), $now, $uncompensated); } public function isSuccess(): bool @@ -115,6 +121,7 @@ public function toArray(): array 'durationMs' => $this->durationMs(), 'headers' => $this->headers, 'steps' => array_map(static fn (StepOutcome $step): array => $step->toArray(), $this->steps), + 'uncompensatedSteps' => $this->uncompensatedSteps, 'report' => $this->report?->toArray(), ]; } diff --git a/packages/orchestration/tests/Core/EnumsTest.php b/packages/orchestration/tests/Core/EnumsTest.php index 36cff972..04ef851b 100644 --- a/packages/orchestration/tests/Core/EnumsTest.php +++ b/packages/orchestration/tests/Core/EnumsTest.php @@ -10,7 +10,7 @@ it('separates terminal, recoverable and suspendable execution statuses', function (): void { expect(ExecutionStatus::terminal())->toBe([ ExecutionStatus::Completed, ExecutionStatus::Failed, ExecutionStatus::Cancelled, - ExecutionStatus::TimedOut, ExecutionStatus::Confirmed, ExecutionStatus::Canceled, + ExecutionStatus::TimedOut, ExecutionStatus::Confirmed, ExecutionStatus::Canceled, ExecutionStatus::CompensationFailed, ]) ->and(ExecutionStatus::Waiting->isTerminal())->toBeFalse() ->and(ExecutionStatus::Waiting->isRecoverable())->toBeFalse() diff --git a/packages/orchestration/tests/Saga/Engine/SagaEngineTest.php b/packages/orchestration/tests/Saga/Engine/SagaEngineTest.php index 7d0ce69c..62cf35ef 100644 --- a/packages/orchestration/tests/Saga/Engine/SagaEngineTest.php +++ b/packages/orchestration/tests/Saga/Engine/SagaEngineTest.php @@ -243,3 +243,71 @@ function orchestrationOrderSaga(array &$log, bool &$declined): SagaDefinition expect($engines->sagas->execute('hello', ['who' => 'ada'])->resultOf('greet'))->toBe('hello ada'); }); + +/** + * a → b → c → d; d fails unless `$failForward` is false, and b's compensation always fails and is critical, so + * CIRCUIT_BREAKER stops before a. + * + * @param list $log + */ +function orchestrationCriticalSaga(array &$log, bool $failForward = true): SagaDefinition +{ + $builder = SagaBuilder::saga('critical'); + foreach (['a' => [], 'b' => ['a'], 'c' => ['b']] as $id => $dependsOn) { + $builder = $builder->step($id)->dependsOn(...$dependsOn) + ->handler(static fn (): string => "did {$id}") + ->compensation(static function () use ($id, &$log): string { + $log[] = "undo {$id}"; + if ($id === 'b') { + throw new RuntimeException('cannot undo b'); + } + + return "undid {$id}"; + }) + ->compensationCritical($id === 'b') + ->add(); + } + + return $builder->step('d')->dependsOn('c')->handler(static function () use ($failForward): string { + if ($failForward) { + throw new RuntimeException('d failed'); + } + + return 'did d'; + })->add()->build(); +} + +it('ends COMPENSATION_FAILED, dead-letters the execution and names the uncompensated steps when CIRCUIT_BREAKER stops a failed saga', function (): void { + $engines = TestEngines::create(['saga' => ['compensation-policy' => 'circuit-breaker']]); + $log = []; + + $result = $engines->sagas->execute(orchestrationCriticalSaga($log), correlationId: 'cb-1'); + $letters = $engines->deadLetterStore->findByCorrelationId('cb-1'); + $statuses = array_map(static fn ($entry): string => $entry->statusAtFailure->value, $letters); + + expect($log)->toBe(['undo c', 'undo b']) + ->and($result->status)->toBe(ExecutionStatus::CompensationFailed) + ->and($result->isSuccess())->toBeFalse() + ->and($result->uncompensatedSteps)->toBe(['b', 'a']) + ->and($result->compensatedSteps())->toBe(['c']) + ->and($result->toArray()['uncompensatedSteps'] ?? null)->toBe(['b', 'a']) + ->and($engines->persistence->findById('cb-1')?->status)->toBe(ExecutionStatus::CompensationFailed) + ->and($engines->persistence->findById('cb-1')?->failureReason)->toContain('[a] left uncompensated') + ->and($statuses)->toContain(ExecutionStatus::CompensationFailed->value) + ->and(array_values(array_filter($letters, static fn ($entry): bool => $entry->statusAtFailure === ExecutionStatus::CompensationFailed))[0]->stepId ?? null)->toBe('b'); +}); + +it('ends COMPENSATION_FAILED and dead-letters the execution when CIRCUIT_BREAKER stops a compensation on request', function (): void { + $engines = TestEngines::create(['saga' => ['compensation-policy' => 'circuit-breaker']]); + $log = []; + $saga = orchestrationCriticalSaga($log, failForward: false); + $engines->sagaRegistry->register($saga); + $engines->sagas->execute($saga, correlationId: 'cb-2'); + + $result = $engines->sagas->compensate('cb-2'); + + expect($result->status)->toBe(ExecutionStatus::CompensationFailed) + ->and($result->uncompensatedSteps)->toBe(['b', 'a']) + ->and($engines->persistence->findById('cb-2')?->status)->toBe(ExecutionStatus::CompensationFailed) + ->and(array_map(static fn ($entry): string => $entry->statusAtFailure->value, $engines->deadLetterStore->findByCorrelationId('cb-2')))->toBe([ExecutionStatus::CompensationFailed->value]); +}); From b9490a1dcc7c12e21a736e182e429d7694259b66 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 23:49:03 -0700 Subject: [PATCH 48/58] fix(orchestration): park a due workflow the tick fails to drive max-resume-attempts times FAILED and dead-letter it, and let database faults escape the tick --- CHANGELOG.md | 3 + docs/modules/orchestration.md | 7 ++- .../src/OrchestrationSettings.php | 1 + .../src/Settings/WorkflowSettings.php | 6 +- .../src/Workflow/Engine/WorkflowEngine.php | 56 +++++++++++++++--- .../tests/OrchestrationSettingsTest.php | 2 + .../UnreadableExecutionPersistence.php | 17 +++++- .../Workflow/Engine/WorkflowWaitsTest.php | 59 +++++++++++++++++++ skeleton/config/firefly.php | 4 ++ 9 files changed, 142 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 603acef8..6f4c7f57 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,9 @@ All notable changes to LaraFly are documented here. This project uses CalVer (`Y with the step result bound implicitly, and that a duplicate, a step with both forms and an unknown step are refused. A capstone and an integration test on PostgreSQL, MySQL and MariaDB check that a failing discovered workflow runs it (its SQL Server case has not yet run against a real server). +- **Orchestration failure parking:** a saga whose compensation stops part-way ends `COMPENSATION_FAILED` and + is dead-lettered, and a due workflow the tick fails to drive `firefly.orchestration.workflow.max-resume-attempts` + times in a row (default `5`) is parked `FAILED` and dead-lettered. - **Observability:** `orchestration_executions_seconds`, `orchestration_steps_seconds`, `orchestration_compensations_total` and `orchestration_dead_letters_total`, and INTERNAL spans per execution and step behind `firefly.observability.tracing.orchestration.enabled`. diff --git a/docs/modules/orchestration.md b/docs/modules/orchestration.md index 5526cb55..445834f3 100644 --- a/docs/modules/orchestration.md +++ b/docs/modules/orchestration.md @@ -146,6 +146,11 @@ A workflow is a step graph whose steps may wait. `#[WaitForSignal]` parks the ex `WorkflowEngine::signal()` delivers the signal, whose payload the step reads as the variable `signals.`. `#[WaitForTimer]` parks it until the timer is due; the scheduled tick (`firefly.orchestration.workflow.tick.*`) or `php artisan firefly:orchestration:tick` advances due workflows. +A due workflow the tick cannot drive (its definition is not deployed on that node, a handler throws outside a +step) is backed off, starting at one minute and doubling, so it never holds back the healthy rows behind it. +After `firefly.orchestration.workflow.max-resume-attempts` failed ticks in a row (default `5`, about fifteen +minutes) it is parked `FAILED` without compensation and dead-lettered. A database fault is not counted: it +escapes the tick, so the scheduler reports the outage. `#[WaitForAll]` and `#[WaitForAny]` combine signals and timers, `#[ChildWorkflow]` starts another workflow and waits for it, `condition` skips a step when an ExpressionLanguage expression over `input`, `results`, `variables` and `headers` is false, and `#[WorkflowQuery]` answers a read-only question about a live execution. @@ -381,7 +386,7 @@ default. |---|---| | `saga.enabled`, `saga.compensation-policy`, `saga.default-timeout`, `saga.compensation-error-handler` | Saga engine | | `tcc.enabled`, `tcc.default-timeout`, `tcc.composition.enabled`, `tcc.composition.compensation-policy` | TCC engine | -| `workflow.enabled`, `workflow.queue.connection`, `workflow.queue.name`, `workflow.tick.enabled`, `workflow.tick.interval` | Workflow engine and its tick | +| `workflow.enabled`, `workflow.queue.connection`, `workflow.queue.name`, `workflow.tick.enabled`, `workflow.tick.interval`, `workflow.max-resume-attempts` | Workflow engine and its tick | | `persistence.provider`, `persistence.connection`, `persistence.cache-store`, `persistence.key-prefix`, `persistence.key-ttl`, `persistence.retention-period`, `persistence.cleanup-interval` | Execution store | | `recovery.enabled`, `recovery.stale-threshold`, `recovery.action` | Recovery sweep | | `dlq.enabled`, `dlq.store` | Dead-letter queue | diff --git a/packages/orchestration/src/OrchestrationSettings.php b/packages/orchestration/src/OrchestrationSettings.php index 0693a7b9..9b91c67f 100644 --- a/packages/orchestration/src/OrchestrationSettings.php +++ b/packages/orchestration/src/OrchestrationSettings.php @@ -93,6 +93,7 @@ public static function fromConfig(Config $config): self queueName: self::nullableName($config->get('firefly.orchestration.workflow.queue.name'), 'firefly.orchestration.workflow.queue.name'), tickEnabled: $config->bool('firefly.orchestration.workflow.tick.enabled', true), tickInterval: $durations->get('firefly.orchestration.workflow.tick.interval', 60.0, 'a zero tick interval would schedule the timer sweep continuously'), + maxResumeAttempts: self::atLeast($config->int('firefly.orchestration.workflow.max-resume-attempts', 5), 1, 'firefly.orchestration.workflow.max-resume-attempts'), ); $provider = $config->string('firefly.orchestration.persistence.provider', 'memory'); diff --git a/packages/orchestration/src/Settings/WorkflowSettings.php b/packages/orchestration/src/Settings/WorkflowSettings.php index c84bb113..3b7363ba 100644 --- a/packages/orchestration/src/Settings/WorkflowSettings.php +++ b/packages/orchestration/src/Settings/WorkflowSettings.php @@ -4,7 +4,10 @@ namespace Firefly\Orchestration\Settings; -/** firefly.orchestration.workflow.* — the queue ASYNC starts go to, and the timer tick's cadence in seconds. */ +/** + * firefly.orchestration.workflow.* — the queue ASYNC starts go to, the timer tick's cadence in seconds, and how + * many ticks in a row may fail to drive a due execution before it is parked FAILED and dead-lettered. + */ final readonly class WorkflowSettings { public function __construct( @@ -13,5 +16,6 @@ public function __construct( public ?string $queueName, public bool $tickEnabled, public float $tickInterval, + public int $maxResumeAttempts = 5, ) {} } diff --git a/packages/orchestration/src/Workflow/Engine/WorkflowEngine.php b/packages/orchestration/src/Workflow/Engine/WorkflowEngine.php index 3d668c18..fb4e7c79 100644 --- a/packages/orchestration/src/Workflow/Engine/WorkflowEngine.php +++ b/packages/orchestration/src/Workflow/Engine/WorkflowEngine.php @@ -4,6 +4,7 @@ namespace Firefly\Orchestration\Workflow\Engine; +use Firefly\Kernel\Exception\Infrastructure\DataAccessException; use Firefly\Orchestration\Core\ClosureInvocation; use Firefly\Orchestration\Core\EngineSupport; use Firefly\Orchestration\Core\Exception\ExecutionLockedException; @@ -28,6 +29,7 @@ use Firefly\Orchestration\Workflow\Registry\WorkflowRegistry; use Illuminate\Contracts\Container\Container; use Illuminate\Support\Str; +use PDOException; use Throwable; /** @@ -167,9 +169,15 @@ public function signal(string $correlationId, string $signal, mixed $payload = n * One execution that cannot be driven (its definition was removed in a deploy, or is not deployed on this * node yet; a handler blew up outside a step) is logged and backed off (backOff()): its wake-up moves * forward, so it leaves the head of the due queue instead of coming back first on every tick and, once a - * batch of them is due, starving every healthy timer and deadline. It is not failed: a rolling deploy can - * leave a node that does not know a workflow yet, and that is no reason to end the execution. When a batch - * had to back rows off, the tick asks for the next batch, so the healthy rows behind them still move now. + * batch of them is due, starving every healthy timer and deadline. A rolling deploy can leave a node that + * does not know a workflow yet, so one failed attempt does not end the execution; the attempts are counted + * in the state (BACKOFF_FAILURES), and the one that reaches `firefly.orchestration.workflow. + * max-resume-attempts` parks it FAILED and dead-letters it (park()). When a batch had to back rows off, the + * tick asks for the next batch, so the healthy rows behind them still move now. + * + * A database fault (a PDOException, which Laravel's QueryException is, or a firefly DataAccessException) is + * not the execution's fault: it is neither counted nor swallowed, and escapes the tick so the scheduler sees + * the outage instead of a log line per row. * * @return list the executions this tick advanced */ @@ -203,14 +211,18 @@ public function resumeDue(): array $context->removeVariable(self::BACKOFF_FAILURES); return $this->drive($this->registry->get($state->executionName), $context); + } catch (ExecutionLockedException|PDOException|DataAccessException $notItsFault) { + throw $notItsFault; } catch (Throwable $error) { - $this->backOff($due->correlationId, $state); + $this->backOff($due->correlationId, $state, $error); throw $error; } }); } catch (ExecutionLockedException) { // Another process is advancing it right now; the next tick sees whatever it left. + } catch (PDOException|DataAccessException $fault) { + throw $fault; } catch (Throwable $error) { $backedOff = true; $this->support->logger->warning("Workflow [{$due->executionName}] execution [{$due->correlationId}] could not be resumed by the tick: {$error->getMessage()}", [ @@ -719,9 +731,10 @@ private function applyDeadline(WorkflowDefinition $workflow, ExecutionContext $c * to BACKOFF_MAX_SECONDS with every tick in a row that fails on it. It is written on the latest checkpoint * (the failed drive may have written one) and only while that is still WAITING; the deadline is left out of * the wake-up on purpose, since a deadline already past would keep the row at the head of the queue. The - * next drive that gets through clears the back-off and recomputes the wake-up from the definition. + * next drive that gets through clears the back-off and recomputes the wake-up from the definition. The + * attempt that reaches `max-resume-attempts` parks the execution instead (park()). */ - private function backOff(string $correlationId, ExecutionState $before): void + private function backOff(string $correlationId, ExecutionState $before, Throwable $cause): void { try { $state = $this->support->persistence->findById($correlationId) ?? $before; @@ -731,10 +744,16 @@ private function backOff(string $correlationId, ExecutionState $before): void $previous = $before->variables[self::BACKOFF_FAILURES] ?? 0; $failures = (is_int($previous) ? $previous : 0) + 1; - $delay = min(self::BACKOFF_MAX_SECONDS, self::BACKOFF_BASE_SECONDS * 2 ** min($failures - 1, 16)); - $context = ExecutionContext::fromState($state); $context->setVariable(self::BACKOFF_FAILURES, $failures); + + if ($failures >= $this->settings->maxResumeAttempts) { + $this->park($state, $context, $cause, $failures); + + return; + } + + $delay = min(self::BACKOFF_MAX_SECONDS, self::BACKOFF_BASE_SECONDS * 2 ** min($failures - 1, 16)); $context->scheduleTimer(self::BACKOFF_TIMER, $this->support->now()->modify(sprintf('+%d seconds', $delay))); $this->support->checkpoint($context, ExecutionStatus::Waiting, $state->failureReason); } catch (Throwable $error) { @@ -745,6 +764,27 @@ private function backOff(string $correlationId, ExecutionState $before): void } } + /** + * Ends an execution the tick failed to drive `$attempts` ticks in a row: FAILED, with a report and no wake-up + * (only a WAITING row has one), so it leaves the due queue for good; dead-lettered with the last error for an operator to retry, + * and its parent woken like by any other failed child. Nothing is compensated: the definition may be gone, + * and the steps it completed are named in the report and the dead letter. + */ + private function park(ExecutionState $state, ExecutionContext $context, Throwable $cause, int $attempts): void + { + $reason = "Workflow [{$state->executionName}] could not be resumed by the tick after {$attempts} attempts (".$cause::class.'); parked FAILED.'; + $stepIds = $this->registry->has($state->executionName) + ? $this->registry->get($state->executionName)->stepIds() + : array_map('strval', array_keys($state->stepStatuses)); + $now = $this->support->now(); + $parked = $this->support->checkpoint($context, ExecutionStatus::Failed, $reason, ExecutionReportBuilder::fromContext($context, ExecutionStatus::Failed, $reason, $now, $stepIds)); + $this->support->deadLetter($context, $state->waitingOn, ExecutionStatus::Failed, $cause); + $this->support->logger->warning($reason, ['correlationId' => $state->correlationId, 'exceptionClass' => $cause::class]); + $this->support->publish($this->support->lifecycle, OrchestrationEvent::executionCompleted($state->executionName, $state->correlationId, ExecutionPattern::Workflow, ExecutionStatus::Failed, $now)); + $this->support->events->onCompleted($state->executionName, $state->correlationId, ExecutionPattern::Workflow, false, $this->support->elapsedMs($context)); + $this->notifyParent($parked); + } + /** * The first checkpoint is an upsert, so starting with a correlation id that is already taken would reset that * execution to RUNNING and run every step again (SagaEngine and TccEngine refuse it the same way). diff --git a/packages/orchestration/tests/OrchestrationSettingsTest.php b/packages/orchestration/tests/OrchestrationSettingsTest.php index e0e3a9ff..48205d60 100644 --- a/packages/orchestration/tests/OrchestrationSettingsTest.php +++ b/packages/orchestration/tests/OrchestrationSettingsTest.php @@ -23,6 +23,7 @@ function orchestrationSettings(array $orchestration): OrchestrationSettings ->and($settings->saga->compensationErrorHandler)->toBe('default') ->and($settings->tcc->defaultTimeout)->toBe(30.0) ->and($settings->workflow->tickInterval)->toBe(60.0) + ->and($settings->workflow->maxResumeAttempts)->toBe(5) ->and($settings->persistence->provider)->toBe('memory') ->and($settings->persistence->keyPrefix)->toBe('orchestration:') ->and($settings->persistence->keyTtl)->toBeNull() @@ -76,6 +77,7 @@ function orchestrationSettings(array $orchestration): OrchestrationSettings [['backpressure' => ['batch-size' => 0]], 'firefly.orchestration.backpressure.batch-size'], [['saga' => ['default-timeout' => '-5s']], 'firefly.orchestration.saga.default-timeout'], [['workflow' => ['tick' => ['interval' => '0']]], 'firefly.orchestration.workflow.tick.interval'], + [['workflow' => ['max-resume-attempts' => 0]], 'firefly.orchestration.workflow.max-resume-attempts'], [['rest' => ['path' => '/']], 'firefly.orchestration.rest.path'], [['saga' => ['compensation-policy' => 'optimistic']], 'firefly.orchestration.saga.compensation-policy'], ]); diff --git a/packages/orchestration/tests/Support/UnreadableExecutionPersistence.php b/packages/orchestration/tests/Support/UnreadableExecutionPersistence.php index 19c0f8cf..cb62be93 100644 --- a/packages/orchestration/tests/Support/UnreadableExecutionPersistence.php +++ b/packages/orchestration/tests/Support/UnreadableExecutionPersistence.php @@ -4,6 +4,7 @@ namespace Firefly\Orchestration\Tests\Support; +use Closure; use DateTimeImmutable; use Firefly\Orchestration\Core\ExecutionStatus; use Firefly\Orchestration\Persistence\ExecutionCriteria; @@ -11,10 +12,13 @@ use Firefly\Orchestration\Persistence\ExecutionState; use Firefly\Orchestration\Persistence\MemoryExecutionPersistenceProvider; use RuntimeException; +use Throwable; /** * An in-memory store that, once refuseLoadsOf() names an execution, throws when findById() reads one of that - * execution's states — a store that fails part-way through, after the execution was written. + * execution's states — a store that fails part-way through, after the execution was written. `$fault` builds + * what it throws (a RuntimeException by default), so a test can stand in a database fault such as a + * PDOException. */ final class UnreadableExecutionPersistence implements ExecutionPersistenceProvider { @@ -23,8 +27,15 @@ final class UnreadableExecutionPersistence implements ExecutionPersistenceProvid /** @var list */ private array $refused = []; - public function __construct() + /** @var Closure(string): Throwable */ + private readonly Closure $fault; + + /** + * @param (Closure(string): Throwable)|null $fault + */ + public function __construct(?Closure $fault = null) { + $this->fault = $fault ?? static fn (string $correlationId): Throwable => new RuntimeException("cannot read [{$correlationId}]"); $this->inner = new MemoryExecutionPersistenceProvider; } @@ -48,7 +59,7 @@ public function findById(string $correlationId): ?ExecutionState { $state = $this->inner->findById($correlationId); if ($state !== null && in_array($state->executionName, $this->refused, true)) { - throw new RuntimeException("cannot read [{$correlationId}]"); + throw ($this->fault)($correlationId); } return $state; diff --git a/packages/orchestration/tests/Workflow/Engine/WorkflowWaitsTest.php b/packages/orchestration/tests/Workflow/Engine/WorkflowWaitsTest.php index 95d5547a..178381f6 100644 --- a/packages/orchestration/tests/Workflow/Engine/WorkflowWaitsTest.php +++ b/packages/orchestration/tests/Workflow/Engine/WorkflowWaitsTest.php @@ -7,7 +7,9 @@ use Firefly\Orchestration\Core\ExecutionStatus; use Firefly\Orchestration\Core\StepStatus; use Firefly\Orchestration\Tests\Support\TestEngines; +use Firefly\Orchestration\Tests\Support\UnreadableExecutionPersistence; use Firefly\Orchestration\Workflow\Builder\WorkflowBuilder; +use Firefly\Orchestration\Workflow\Engine\WorkflowEngine; use Firefly\Orchestration\Workflow\Registry\SignalWait; use Firefly\Orchestration\Workflow\Registry\TimerWait; @@ -150,3 +152,60 @@ ->and($again)->toBe([]) ->and(count($after->logger->messages('warning')))->toBe(5); }); + +it('parks a due execution it cannot drive FAILED and dead-letters it after max-resume-attempts, without holding back a healthy one', function (): void { + $timer = static fn (string $id) => WorkflowBuilder::workflow($id) + ->step('pause')->waitForTimer(60_000)->add() + ->step('go')->dependsOn('pause')->handler(static fn (): string => 'went')->add() + ->build(); + $config = ['backpressure' => ['batch-size' => 1], 'workflow' => ['max-resume-attempts' => 3]]; + $before = TestEngines::create($config); + $before->workflowRegistry->register($timer('retired')); + $before->workflowRegistry->register($timer('kept')); + $before->workflows->start('retired', correlationId: 'a-poison'); + $before->workflows->start('kept', correlationId: 'b-healthy'); + + $after = TestEngines::create($config, persistence: $before->persistence); + $after->workflowRegistry->register($timer('kept')); + $after->clock->advance(61.0); + $first = $after->workflows->resumeDue(); + $attempts = [$after->workflows->find('a-poison')?->variables[WorkflowEngine::BACKOFF_FAILURES] ?? null]; + $after->clock->advance(61.0); + $after->workflows->resumeDue(); + $attempts[] = $after->workflows->find('a-poison')?->variables[WorkflowEngine::BACKOFF_FAILURES] ?? null; + $waitingBeforeLast = $after->workflows->find('a-poison')?->status; + $after->clock->advance(121.0); + $after->workflows->resumeDue(); + $parked = $after->workflows->find('a-poison'); + $letters = $after->deadLetterStore->findByCorrelationId('a-poison'); + $after->clock->advance(3600.0); + + expect(array_map(static fn ($state): string => $state->correlationId, $first))->toBe(['b-healthy']) + ->and($after->workflows->find('b-healthy')?->status)->toBe(ExecutionStatus::Completed) + ->and($attempts)->toBe([1, 2]) + ->and($waitingBeforeLast)->toBe(ExecutionStatus::Waiting) + ->and($parked?->status)->toBe(ExecutionStatus::Failed) + ->and($parked?->wakeAt)->toBeNull() + ->and($parked?->failureReason)->toContain('3 attempts') + ->and($parked?->report)->not->toBeNull() + ->and($letters)->toHaveCount(1) + ->and($letters[0]->statusAtFailure ?? null)->toBe(ExecutionStatus::Failed) + ->and($letters[0]->stepId ?? null)->toBe('pause') + ->and($after->workflows->resumeDue())->toBe([]); +}); + +it('lets a database fault escape the tick instead of logging it and leaving the execution WAITING', function (): void { + $store = new UnreadableExecutionPersistence(static fn (string $id): Throwable => new PDOException("SQLSTATE[08006] connection to the store lost reading [{$id}]")); + $engines = TestEngines::create(persistence: $store); + $engines->workflowRegistry->register(WorkflowBuilder::workflow('cooldown') + ->step('pause')->waitForTimer(60_000)->add() + ->step('go')->dependsOn('pause')->handler(static fn (): string => 'went')->add() + ->build()); + $engines->workflows->start('cooldown', correlationId: 'db-1'); + $store->refuseLoadsOf('cooldown'); + $engines->clock->advance(61.0); + + expect(fn () => $engines->workflows->resumeDue())->toThrow(PDOException::class, 'connection to the store lost') + ->and($store->peek('db-1')?->variables)->not->toHaveKey(WorkflowEngine::BACKOFF_FAILURES) + ->and($engines->logger->messages('warning'))->toBe([]); +}); diff --git a/skeleton/config/firefly.php b/skeleton/config/firefly.php index 73ecba7e..270c1e77 100644 --- a/skeleton/config/firefly.php +++ b/skeleton/config/firefly.php @@ -2615,6 +2615,9 @@ | delivered twice is harmless: the second delivery finds the execution no longer PENDING. The `tick` is a scheduled task (firefly:schedule lists it) that resumes | WAITING workflows whose timers or deadlines are due, under the DistributedLock; without a scheduler | run php artisan firefly:orchestration:tick from cron. Defaults: enabled true, tick every 1m. + | A due workflow the tick cannot drive (its definition is not deployed, a handler throws outside a + | step) is backed off from 1m, doubling; after `max-resume-attempts` ticks in a row (default 5, about + | 15 minutes, at least 1) it is parked FAILED and dead-lettered. Database faults are never counted. */ 'workflow' => [ 'enabled' => env('FIREFLY_ORCHESTRATION_WORKFLOW_ENABLED', true), @@ -2626,6 +2629,7 @@ 'enabled' => env('FIREFLY_ORCHESTRATION_TICK_ENABLED', true), 'interval' => env('FIREFLY_ORCHESTRATION_TICK_INTERVAL', '1m'), ], + 'max-resume-attempts' => (int) env('FIREFLY_ORCHESTRATION_WORKFLOW_MAX_RESUME_ATTEMPTS', 5), ], /* From f4a76849966c73d9a3bcc0a7bd1c387d97fc909e Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 23:49:45 -0700 Subject: [PATCH 49/58] fix(orchestration): log a tick failure by exception class, execution and correlation id, and keep the error message at debug level --- .../src/Workflow/Engine/WorkflowEngine.php | 23 ++++++++++++------- .../tests/Support/RecordingLogger.php | 13 +++++++++-- .../Workflow/Engine/WorkflowWaitsTest.php | 21 +++++++++++++++++ 3 files changed, 47 insertions(+), 10 deletions(-) diff --git a/packages/orchestration/src/Workflow/Engine/WorkflowEngine.php b/packages/orchestration/src/Workflow/Engine/WorkflowEngine.php index fb4e7c79..70e6c53d 100644 --- a/packages/orchestration/src/Workflow/Engine/WorkflowEngine.php +++ b/packages/orchestration/src/Workflow/Engine/WorkflowEngine.php @@ -225,10 +225,7 @@ public function resumeDue(): array throw $fault; } catch (Throwable $error) { $backedOff = true; - $this->support->logger->warning("Workflow [{$due->executionName}] execution [{$due->correlationId}] could not be resumed by the tick: {$error->getMessage()}", [ - 'exception' => $error, - 'correlationId' => $due->correlationId, - ]); + $this->logTickFailure("Workflow [{$due->executionName}] execution [{$due->correlationId}] could not be resumed by the tick", $due, $error); } } } while ($backedOff && $fresh > 0 && count($advanced) < $this->batchSize); @@ -757,13 +754,23 @@ private function backOff(string $correlationId, ExecutionState $before, Throwabl $context->scheduleTimer(self::BACKOFF_TIMER, $this->support->now()->modify(sprintf('+%d seconds', $delay))); $this->support->checkpoint($context, ExecutionStatus::Waiting, $state->failureReason); } catch (Throwable $error) { - $this->support->logger->warning("Workflow execution [{$correlationId}] could not be backed off by the tick: {$error->getMessage()}", [ - 'exception' => $error, - 'correlationId' => $correlationId, - ]); + $this->logTickFailure("Workflow [{$before->executionName}] execution [{$correlationId}] could not be backed off by the tick", $before, $error); } } + /** + * Logs a failure of the tick without the error's message at warning level: a message can carry what a + * handler or a driver put in it (a query, a token, a payload), and warnings are what log shipping keeps. The + * warning names the exception class, the execution and its correlation id; the message and the exception + * itself go to debug, for whoever turns it on. + */ + private function logTickFailure(string $what, ExecutionState $execution, Throwable $error): void + { + $context = ['correlationId' => $execution->correlationId, 'executionName' => $execution->executionName, 'exceptionClass' => $error::class]; + $this->support->logger->warning($what.' ('.$error::class.').', $context); + $this->support->logger->debug($what.': '.$error->getMessage(), $context + ['exception' => $error]); + } + /** * Ends an execution the tick failed to drive `$attempts` ticks in a row: FAILED, with a report and no wake-up * (only a WAITING row has one), so it leaves the due queue for good; dead-lettered with the last error for an operator to retry, diff --git a/packages/orchestration/tests/Support/RecordingLogger.php b/packages/orchestration/tests/Support/RecordingLogger.php index 21998afd..77da9b00 100644 --- a/packages/orchestration/tests/Support/RecordingLogger.php +++ b/packages/orchestration/tests/Support/RecordingLogger.php @@ -10,7 +10,7 @@ /** Records every log line so a test can assert that a contained failure was reported, not swallowed. */ final class RecordingLogger extends AbstractLogger { - /** @var list */ + /** @var list}> */ public array $records = []; /** @@ -18,7 +18,16 @@ final class RecordingLogger extends AbstractLogger */ public function log($level, string|Stringable $message, array $context = []): void { - $this->records[] = ['level' => is_string($level) ? $level : 'unknown', 'message' => (string) $message]; + $this->records[] = ['level' => is_string($level) ? $level : 'unknown', 'message' => (string) $message, 'context' => $context]; + } + + /** @return list> the context of every line logged at `$level` */ + public function contexts(string $level): array + { + return array_values(array_map( + static fn (array $record): array => $record['context'], + array_filter($this->records, static fn (array $record): bool => $record['level'] === $level), + )); } /** @return list */ diff --git a/packages/orchestration/tests/Workflow/Engine/WorkflowWaitsTest.php b/packages/orchestration/tests/Workflow/Engine/WorkflowWaitsTest.php index 178381f6..81c25a17 100644 --- a/packages/orchestration/tests/Workflow/Engine/WorkflowWaitsTest.php +++ b/packages/orchestration/tests/Workflow/Engine/WorkflowWaitsTest.php @@ -209,3 +209,24 @@ ->and($store->peek('db-1')?->variables)->not->toHaveKey(WorkflowEngine::BACKOFF_FAILURES) ->and($engines->logger->messages('warning'))->toBe([]); }); + +it('logs a due execution the tick cannot drive by exception class, name and correlation id, keeping the error message to debug', function (): void { + $store = new UnreadableExecutionPersistence(static fn (string $id): Throwable => new RuntimeException("token=s3cr3t while reading [{$id}]")); + $engines = TestEngines::create(persistence: $store); + $engines->workflowRegistry->register(WorkflowBuilder::workflow('cooldown') + ->step('pause')->waitForTimer(60_000)->add() + ->build()); + $engines->workflows->start('cooldown', correlationId: 'log-1'); + $store->refuseLoadsOf('cooldown'); + $engines->clock->advance(61.0); + + $engines->workflows->resumeDue(); + $warnings = $engines->logger->messages('warning'); + $contexts = $engines->logger->contexts('warning'); + + expect($warnings)->toHaveCount(1) + ->and($warnings[0] ?? '')->toContain('cooldown')->toContain('log-1')->toContain(RuntimeException::class)->not->toContain('s3cr3t') + ->and($contexts[0] ?? [])->not->toHaveKey('exception') + ->and($contexts[0]['correlationId'] ?? null)->toBe('log-1') + ->and(implode("\n", $engines->logger->messages('debug')))->toContain('s3cr3t'); +}); From 5591c6ddc19ccc34d80778309247b6fdc4085679 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 23:51:13 -0700 Subject: [PATCH 50/58] fix(orchestration): count nested drives and settlements of one workflow by depth, so a nested call no longer clears the outer call's mark --- .../src/Workflow/Engine/WorkflowEngine.php | 40 +++++++++++++--- .../Workflow/Engine/WorkflowLifecycleTest.php | 46 +++++++++++++++++++ 2 files changed, 80 insertions(+), 6 deletions(-) diff --git a/packages/orchestration/src/Workflow/Engine/WorkflowEngine.php b/packages/orchestration/src/Workflow/Engine/WorkflowEngine.php index 70e6c53d..5bde76c9 100644 --- a/packages/orchestration/src/Workflow/Engine/WorkflowEngine.php +++ b/packages/orchestration/src/Workflow/Engine/WorkflowEngine.php @@ -61,10 +61,13 @@ final class WorkflowEngine private const int BACKOFF_MAX_SECONDS = 3600; - /** @var array the executions this process is driving right now */ + /** + * @var array the executions this process is driving right now, by nesting depth: the lock is + * reentrant, so one execution can be driven again from inside its own drive + */ private array $driving = []; - /** @var array the executions this process is cancelling or compensating right now */ + /** @var array the executions this process is cancelling or compensating right now, by nesting depth */ private array $settling = []; public function __construct( @@ -465,7 +468,7 @@ private function begin(WorkflowDefinition $workflow, ExecutionContext $context): private function drive(WorkflowDefinition $workflow, ExecutionContext $context): ExecutionState { - $this->driving[$context->correlationId] = true; + self::enter($this->driving, $context->correlationId); try { $this->applyDeadline($workflow, $context); @@ -494,7 +497,7 @@ private function drive(WorkflowDefinition $workflow, ExecutionContext $context): }; } } finally { - unset($this->driving[$context->correlationId]); + self::leave($this->driving, $context->correlationId); } } @@ -676,11 +679,36 @@ private function wakeParent(string $parentId, ExecutionState $child): void */ private function settle(string $correlationId, callable $work): mixed { - $this->settling[$correlationId] = true; + self::enter($this->settling, $correlationId); try { return $work(); } finally { - unset($this->settling[$correlationId]); + self::leave($this->settling, $correlationId); + } + } + + /** + * Marks `$correlationId` one level deeper in `$marks`. A depth, not a flag: a nested call for the same + * execution (a compensation that cancels its own workflow, under the reentrant lock) must not clear the + * outer call's mark when it returns. + * + * @param array $marks + */ + private static function enter(array &$marks, string $correlationId): void + { + $marks[$correlationId] = ($marks[$correlationId] ?? 0) + 1; + } + + /** + * @param array $marks + */ + private static function leave(array &$marks, string $correlationId): void + { + $depth = ($marks[$correlationId] ?? 1) - 1; + if ($depth > 0) { + $marks[$correlationId] = $depth; + } else { + unset($marks[$correlationId]); } } diff --git a/packages/orchestration/tests/Workflow/Engine/WorkflowLifecycleTest.php b/packages/orchestration/tests/Workflow/Engine/WorkflowLifecycleTest.php index 4717ef8b..189c9dd1 100644 --- a/packages/orchestration/tests/Workflow/Engine/WorkflowLifecycleTest.php +++ b/packages/orchestration/tests/Workflow/Engine/WorkflowLifecycleTest.php @@ -4,9 +4,11 @@ use Firefly\Orchestration\Core\Exception\InvalidExecutionStateException; use Firefly\Orchestration\Core\ExecutionContext; +use Firefly\Orchestration\Core\ExecutionPattern; use Firefly\Orchestration\Core\ExecutionStatus; use Firefly\Orchestration\Tests\Support\TestEngines; use Firefly\Orchestration\Workflow\Builder\WorkflowBuilder; +use Firefly\Orchestration\Workflow\Engine\WorkflowEngine; function orchestrationLifecycleEngines(): TestEngines { @@ -87,3 +89,47 @@ function orchestrationLifecycleEngines(): TestEngines ->and($upserted->wakeAt)->toEqual($waiting->wakeAt) ->and($engines->workflows->find('w-2')?->wakeAt)->toEqual($waiting->wakeAt); }); + +it('keeps an execution marked as settling while a nested cancel of the same execution returns', function (): void { + $engines = TestEngines::create(); + $engine = $engines->workflows; + $marked = []; + $peek = static fn (string $id): bool => (bool) Closure::bind(fn (): bool => isset($this->settling[$id]), $engine, WorkflowEngine::class)(); + $engines->workflowRegistry->register(WorkflowBuilder::workflow('undo') + ->step('a')->handler(static fn (): string => 'did a')->compensation(static function () use ($engine, $peek, &$marked): void { + // A compensation that cancels its own workflow re-enters settle() for the same id under the + // reentrant lock; returning from it must not clear the outer compensation's mark. + $engine->cancel('h-1'); + $marked[] = $peek('h-1'); + })->add() + ->build()); + $engine->start('undo', correlationId: 'h-1'); + + $engine->compensate('h-1'); + + expect($marked)->toBe([true]) + ->and($peek('h-1'))->toBeFalse(); +}); + +it('keeps an execution marked as driving while a nested drive of the same execution returns', function (): void { + $engines = TestEngines::create(); + $engine = $engines->workflows; + $inner = WorkflowBuilder::workflow('inner')->step('noop')->handler(static fn (): string => 'ok')->add()->build(); + $marked = []; + $peek = static fn (string $id): bool => (bool) Closure::bind(fn (): bool => isset($this->driving[$id]), $engine, WorkflowEngine::class)(); + $outer = WorkflowBuilder::workflow('outer') + ->step('nest')->handler(static function () use ($engine, $engines, $inner, $peek, &$marked): string { + Closure::bind(function () use ($inner, $engines): void { + $this->drive($inner, ExecutionContext::start(ExecutionPattern::Workflow, 'inner', [], [], 'h-2', $engines->clock->now())); + }, $engine, WorkflowEngine::class)(); + $marked[] = $peek('h-2'); + + return 'nested'; + })->add() + ->build(); + + $engine->start($outer, correlationId: 'h-2'); + + expect($marked)->toBe([true]) + ->and($peek('h-2'))->toBeFalse(); +}); From cc89c782a3e3a0670a4e0bd230929f294a5ea337 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 23:53:03 -0700 Subject: [PATCH 51/58] fix(orchestration): replay the child completions a failed cancel or compensation held back, so the parent does not stay WAITING on a child it already cancelled --- .../src/Workflow/Engine/WorkflowEngine.php | 56 ++++++++++++++++++- .../Workflow/Engine/WorkflowChildrenTest.php | 35 ++++++++++++ 2 files changed, 88 insertions(+), 3 deletions(-) diff --git a/packages/orchestration/src/Workflow/Engine/WorkflowEngine.php b/packages/orchestration/src/Workflow/Engine/WorkflowEngine.php index 5bde76c9..8310a9d6 100644 --- a/packages/orchestration/src/Workflow/Engine/WorkflowEngine.php +++ b/packages/orchestration/src/Workflow/Engine/WorkflowEngine.php @@ -70,6 +70,12 @@ final class WorkflowEngine /** @var array the executions this process is cancelling or compensating right now, by nesting depth */ private array $settling = []; + /** + * @var array> by settling parent, the finished children whose wake-up + * notifyParent() held back while that parent was settling + */ + private array $heldBack = []; + public function __construct( private readonly WorkflowRegistry $registry, private readonly WorkflowExecutor $executor, @@ -624,7 +630,13 @@ private function finish(WorkflowDefinition $workflow, ExecutionContext $context, private function notifyParent(ExecutionState $child): void { $parentId = $child->parentCorrelationId; - if ($parentId === null || isset($this->driving[$parentId]) || isset($this->settling[$parentId])) { + if ($parentId === null || isset($this->driving[$parentId])) { + return; + } + if (isset($this->settling[$parentId])) { + // Held back, not dropped: if the settlement fails before it moves the parent on, settle() replays it. + $this->heldBack[$parentId][$child->correlationId] = $child; + return; } @@ -672,6 +684,13 @@ private function wakeParent(string $parentId, ExecutionState $child): void * through notifyParent(): the lock is reentrant, and the parent is still WAITING on that child until * `$work` has moved it on. * + * When `$work` throws before it got there (the cancel of a later child failed, say), the children it + * already finished would leave the parent WAITING on a child that is gone, with nothing left to wake it. + * Their held-back wake-ups are replayed once the outermost settlement of the parent has ended, so the + * parent sees them like any other child completion; then the original error is rethrown. A replay that + * fails itself is logged and leaves the parent where the replay stopped (a COMPENSATING parent is + * recoverable). + * * @template T * * @param callable(): T $work @@ -681,9 +700,40 @@ private function settle(string $correlationId, callable $work): mixed { self::enter($this->settling, $correlationId); try { - return $work(); - } finally { + $result = $work(); + } catch (Throwable $error) { self::leave($this->settling, $correlationId); + $this->replayHeldBack($correlationId); + + throw $error; + } + + self::leave($this->settling, $correlationId); + if (! isset($this->settling[$correlationId])) { + unset($this->heldBack[$correlationId]); + } + + return $result; + } + + private function replayHeldBack(string $parentId): void + { + if (isset($this->settling[$parentId])) { + return; // an outer settlement of the same parent is still under way; it decides + } + + $children = $this->heldBack[$parentId] ?? []; + unset($this->heldBack[$parentId]); + foreach ($children as $child) { + try { + $this->notifyParent($child); + } catch (Throwable $error) { + $this->support->logger->warning("Workflow execution [{$child->correlationId}] finished while its parent [{$parentId}] was settling, and the parent could not be woken after that settlement failed (".$error::class.').', [ + 'correlationId' => $child->correlationId, + 'parentCorrelationId' => $parentId, + 'exceptionClass' => $error::class, + ]); + } } } diff --git a/packages/orchestration/tests/Workflow/Engine/WorkflowChildrenTest.php b/packages/orchestration/tests/Workflow/Engine/WorkflowChildrenTest.php index 21695c62..d9118b52 100644 --- a/packages/orchestration/tests/Workflow/Engine/WorkflowChildrenTest.php +++ b/packages/orchestration/tests/Workflow/Engine/WorkflowChildrenTest.php @@ -2,10 +2,12 @@ declare(strict_types=1); +use Firefly\Orchestration\Core\Exception\ExecutionLockedException; use Firefly\Orchestration\Core\ExecutionLock; use Firefly\Orchestration\Core\ExecutionStatus; use Firefly\Orchestration\Core\StepStatus; use Firefly\Orchestration\Core\TriggerMode; +use Firefly\Orchestration\Persistence\ExecutionState; use Firefly\Orchestration\Tests\Support\ScriptedLock; use Firefly\Orchestration\Tests\Support\TestEngines; use Firefly\Orchestration\Workflow\Builder\WorkflowBuilder; @@ -289,3 +291,36 @@ function orchestrationPayingParentEngines(array &$log): TestEngines ->and(data_get($parent?->stepResults, 'spawn.results'))->toBe(['tick' => 'ticked', 'approve' => 'approved']) ->and($engines->workflows->find(is_string($cancelLastRun) ? $cancelLastRun : '')?->status)->toBe(ExecutionStatus::Cancelled); }); + +it('replays the completion of a child a failed cancel already cancelled, so the parent does not stay WAITING on it', function (): void { + $lock = new ScriptedLock; + $engines = TestEngines::create(['lock' => ['wait' => 0]], lock: $lock); + $engines->workflowRegistry->register(WorkflowBuilder::workflow('check')->triggerMode(TriggerMode::Async) + ->step('run')->handler(static fn (): string => 'checked')->add()->build()); + $engines->workflowRegistry->register(WorkflowBuilder::workflow('parent') + ->step('fire')->childWorkflow('check', waitForCompletion: false)->handler(static fn (): array => [])->add() + ->step('wait')->dependsOn('fire')->childWorkflow('check')->handler(static fn (): array => [])->add() + ->build()); + $waiting = $engines->workflows->start('parent', correlationId: 'p-g'); + $fired = $waiting->children['fire'] ?? ''; + $awaited = $waiting->children['wait'] ?? ''; + + // The awaited child is cancelled first (a store need not keep the children in start order), then the + // cancel of the fired one fails because another process holds it. + $data = $waiting->toArray(); + $data['children'] = ['wait' => $awaited, 'fire' => $fired]; + $engines->persistence->save(ExecutionState::fromArray($data)); + $lock->holdElsewhere(ExecutionLock::name($fired)); + + expect(fn () => $engines->workflows->cancel('p-g'))->toThrow(ExecutionLockedException::class); + $afterFailedCancel = $engines->workflows->find('p-g'); + + $lock->releaseElsewhere(ExecutionLock::name($fired)); + $recovered = $engines->workflows->recover('p-g'); + + expect($engines->workflows->find($awaited)?->status)->toBe(ExecutionStatus::Cancelled) + ->and($afterFailedCancel?->status)->not->toBe(ExecutionStatus::Waiting) + ->and($afterFailedCancel?->status->isRecoverable())->toBeTrue() + ->and($recovered->status)->toBe(ExecutionStatus::Failed) + ->and($engines->workflows->find($fired)?->status)->toBe(ExecutionStatus::Cancelled); +}); From 5ad50e3cc79c368f8822d9eed562d9919854429e Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Tue, 13 Oct 2026 00:47:08 -0700 Subject: [PATCH 52/58] test(orchestration): read integration DSNs through IntegrationDsn and run every lane of the CI matrix IntegrationDatabase now delegates to Firefly\Testing\Integration\IntegrationDsn instead of parsing FIREFLY_* variables by hand, iterates IntegrationDsn::lanes() (the four servers plus a sqlite file) and skips with IntegrationDsn::skipReason(), so each CI integration job runs its own lane. The two-worker interleaving test runs on the server lanes only, and sets SQL Server's LOCK_TIMEOUT unprepared, since a prepared SET is undone when its statement ends. --- .../CrashRecoveryIntegrationTest.php | 10 +- .../DatabaseDeadLetterIntegrationTest.php | 14 +-- .../DatabasePersistenceIntegrationTest.php | 60 +++++----- .../DatabaseSagaIntegrationTest.php | 18 +-- .../QueuedWorkflowIntegrationTest.php | 6 +- .../WorkflowCompensationIntegrationTest.php | 6 +- .../Support/BootsOnIntegrationServer.php | 24 ++-- .../tests/Support/IntegrationDatabase.php | 105 +++++++++--------- .../tests/Support/crash-orchestration.php | 6 +- 9 files changed, 127 insertions(+), 122 deletions(-) diff --git a/packages/orchestration/tests/Integration/CrashRecoveryIntegrationTest.php b/packages/orchestration/tests/Integration/CrashRecoveryIntegrationTest.php index 06718ded..79f059a6 100644 --- a/packages/orchestration/tests/Integration/CrashRecoveryIntegrationTest.php +++ b/packages/orchestration/tests/Integration/CrashRecoveryIntegrationTest.php @@ -9,15 +9,15 @@ use Firefly\Orchestration\Tests\Support\IntegrationDatabase; use Firefly\Orchestration\Tests\Support\TestEngines; -foreach (IntegrationDatabase::servers() as $server => [$driver, $variable, $extension]) { +foreach (IntegrationDatabase::lanes() as $lane) { foreach (['resume' => ['reserve', 'charge'], 'compensate' => ['reserve', 'release']] as $action => $expected) { - it("recovers a saga killed mid-step on {$server} with action {$action}", function () use ($driver, $variable, $action, $expected): void { - $connection = IntegrationDatabase::fresh(IntegrationDatabase::connection($driver, $variable)); + it("recovers a saga killed mid-step on {$lane} with action {$action}", function () use ($lane, $action, $expected): void { + $connection = IntegrationDatabase::fresh(IntegrationDatabase::connection($lane)); $marks = sys_get_temp_dir().'/firefly-orchestration-crash-'.bin2hex(random_bytes(6)).'.marks'; $correlationId = "crash-{$action}-".bin2hex(random_bytes(4)); try { - expect(CrashScenarios::crash("{$driver}|{$variable}", 'saga', $correlationId, $marks))->toBe(70); + expect(CrashScenarios::crash(IntegrationDatabase::crashTarget($lane), 'saga', $correlationId, $marks))->toBe(70); $engines = TestEngines::create(['recovery' => ['action' => $action, 'stale-threshold' => '1m']], new DatabaseExecutionPersistenceProvider($connection), new DatabaseDeadLetterStore($connection)); CrashScenarios::register($engines, $marks, crash: false); @@ -29,6 +29,6 @@ } finally { $connection->disconnect(); } - })->group('integration')->skip(IntegrationDatabase::skipReason($variable, $extension) !== null, (string) IntegrationDatabase::skipReason($variable, $extension)); + })->group('integration')->skip(IntegrationDatabase::skipReason($lane) !== null, (string) IntegrationDatabase::skipReason($lane)); } } diff --git a/packages/orchestration/tests/Integration/DatabaseDeadLetterIntegrationTest.php b/packages/orchestration/tests/Integration/DatabaseDeadLetterIntegrationTest.php index 66be7630..54b0cef0 100644 --- a/packages/orchestration/tests/Integration/DatabaseDeadLetterIntegrationTest.php +++ b/packages/orchestration/tests/Integration/DatabaseDeadLetterIntegrationTest.php @@ -7,20 +7,20 @@ use Firefly\Orchestration\Tests\Support\DeadLetterStoreContract; use Firefly\Orchestration\Tests\Support\IntegrationDatabase; -foreach (IntegrationDatabase::servers() as $server => [$driver, $variable, $extension]) { +foreach (IntegrationDatabase::lanes() as $lane) { foreach (DeadLetterStoreContract::cases() as $case => $assert) { - it("{$server} dead-letter store {$case}", function () use ($assert, $driver, $variable): void { - $connection = IntegrationDatabase::fresh(IntegrationDatabase::connection($driver, $variable)); + it("{$lane} dead-letter store {$case}", function () use ($assert, $lane): void { + $connection = IntegrationDatabase::fresh(IntegrationDatabase::connection($lane)); try { $assert(new DatabaseDeadLetterStore($connection)); } finally { $connection->disconnect(); } - })->group('integration')->skip(IntegrationDatabase::skipReason($variable, $extension) !== null, (string) IntegrationDatabase::skipReason($variable, $extension)); + })->group('integration')->skip(IntegrationDatabase::skipReason($lane) !== null, (string) IntegrationDatabase::skipReason($lane)); } - it("keeps an error message longer than a MySQL TEXT column on {$server}", function () use ($driver, $variable): void { - $connection = IntegrationDatabase::fresh(IntegrationDatabase::connection($driver, $variable)); + it("keeps an error message longer than a MySQL TEXT column on {$lane}", function () use ($lane): void { + $connection = IntegrationDatabase::fresh(IntegrationDatabase::connection($lane)); try { $store = new DatabaseDeadLetterStore($connection); $entry = DeadLetterEntry::fromArray(['errorMessage' => str_repeat('é', 70_000)] + DeadLetterStoreContract::entry('d-1')->toArray()); @@ -31,5 +31,5 @@ } finally { $connection->disconnect(); } - })->group('integration')->skip(IntegrationDatabase::skipReason($variable, $extension) !== null, (string) IntegrationDatabase::skipReason($variable, $extension)); + })->group('integration')->skip(IntegrationDatabase::skipReason($lane) !== null, (string) IntegrationDatabase::skipReason($lane)); } diff --git a/packages/orchestration/tests/Integration/DatabasePersistenceIntegrationTest.php b/packages/orchestration/tests/Integration/DatabasePersistenceIntegrationTest.php index 83421278..5f387b40 100644 --- a/packages/orchestration/tests/Integration/DatabasePersistenceIntegrationTest.php +++ b/packages/orchestration/tests/Integration/DatabasePersistenceIntegrationTest.php @@ -8,27 +8,49 @@ use Firefly\Orchestration\Tests\Support\ExecutionPersistenceContract; use Firefly\Orchestration\Tests\Support\IntegrationDatabase; -foreach (IntegrationDatabase::servers() as $server => [$driver, $variable, $extension]) { +foreach (IntegrationDatabase::lanes() as $lane) { foreach (ExecutionPersistenceContract::cases() as $case => $assert) { - it("{$server} database persistence {$case}", function () use ($assert, $driver, $variable): void { - $connection = IntegrationDatabase::fresh(IntegrationDatabase::connection($driver, $variable)); + it("{$lane} database persistence {$case}", function () use ($assert, $lane): void { + $connection = IntegrationDatabase::fresh(IntegrationDatabase::connection($lane)); try { $assert(new DatabaseExecutionPersistenceProvider($connection)); } finally { $connection->disconnect(); } - })->group('integration')->skip(IntegrationDatabase::skipReason($variable, $extension) !== null, (string) IntegrationDatabase::skipReason($variable, $extension)); + })->group('integration')->skip(IntegrationDatabase::skipReason($lane) !== null, (string) IntegrationDatabase::skipReason($lane)); } - it("saves two new executions with search attributes from two workers at once without a gap-lock wait on {$server}", function () use ($driver, $variable): void { - $first = IntegrationDatabase::fresh(IntegrationDatabase::connection($driver, $variable)); - $second = IntegrationDatabase::connection($driver, $variable); + it("keeps full-width ids, names and search attributes and refuses wider ones on {$lane}", function () use ($lane): void { + $connection = IntegrationDatabase::fresh(IntegrationDatabase::connection($lane)); + try { + $provider = new DatabaseExecutionPersistenceProvider($connection); + $id = str_repeat('i', 64); + $state = ExecutionPersistenceContract::state($id, overrides: ['executionName' => str_repeat('ñ', 191), 'searchAttributes' => [str_repeat('k', 128) => str_repeat('é', 191)]]); + + $provider->save($state); + + expect($provider->findById($id)?->toArray())->toBe($state->toArray()) + ->and(ExecutionPersistenceContract::ids($provider->findBySearchAttributes([str_repeat('k', 128) => str_repeat('é', 191)])))->toBe([$id]) + ->and(fn () => $provider->save(ExecutionPersistenceContract::state('wide', overrides: ['searchAttributes' => ['customer' => str_repeat('é', 192)]])))->toThrow(OrchestrationException::class, 'searchAttributes.customer') + ->and($connection->table(OrchestrationSchema::EXECUTIONS)->count())->toBe(1); + } finally { + $connection->disconnect(); + } + })->group('integration')->skip(IntegrationDatabase::skipReason($lane) !== null, (string) IntegrationDatabase::skipReason($lane)); +} + +// The interleaving below needs a server with row-level locking; sqlite has one writer per file, so there it is only a wait. +foreach (IntegrationDatabase::serverLanes() as $lane) { + it("saves two new executions with search attributes from two workers at once without a gap-lock wait on {$lane}", function () use ($lane): void { + $first = IntegrationDatabase::fresh(IntegrationDatabase::connection($lane)); + $second = IntegrationDatabase::connection($lane); try { // A lock wait (the first half of the MySQL/MariaDB deadlock) surfaces in one second instead of hanging. - match ($driver) { + match ($lane) { 'mysql', 'mariadb' => $second->statement('SET SESSION innodb_lock_wait_timeout = 1'), 'pgsql' => $second->statement("SET lock_timeout = '1s'"), - 'sqlsrv' => $second->statement('SET LOCK_TIMEOUT 1000'), + // A prepared SET is undone when its statement ends on SQL Server, so it is sent unprepared there. + 'sqlsrv' => $second->unprepared('SET LOCK_TIMEOUT 1000'), default => null, }; $one = new DatabaseExecutionPersistenceProvider($first); @@ -59,23 +81,5 @@ $first->disconnect(); $second->disconnect(); } - })->group('integration')->skip(IntegrationDatabase::skipReason($variable, $extension) !== null, (string) IntegrationDatabase::skipReason($variable, $extension)); - - it("keeps full-width ids, names and search attributes and refuses wider ones on {$server}", function () use ($driver, $variable): void { - $connection = IntegrationDatabase::fresh(IntegrationDatabase::connection($driver, $variable)); - try { - $provider = new DatabaseExecutionPersistenceProvider($connection); - $id = str_repeat('i', 64); - $state = ExecutionPersistenceContract::state($id, overrides: ['executionName' => str_repeat('ñ', 191), 'searchAttributes' => [str_repeat('k', 128) => str_repeat('é', 191)]]); - - $provider->save($state); - - expect($provider->findById($id)?->toArray())->toBe($state->toArray()) - ->and(ExecutionPersistenceContract::ids($provider->findBySearchAttributes([str_repeat('k', 128) => str_repeat('é', 191)])))->toBe([$id]) - ->and(fn () => $provider->save(ExecutionPersistenceContract::state('wide', overrides: ['searchAttributes' => ['customer' => str_repeat('é', 192)]])))->toThrow(OrchestrationException::class, 'searchAttributes.customer') - ->and($connection->table(OrchestrationSchema::EXECUTIONS)->count())->toBe(1); - } finally { - $connection->disconnect(); - } - })->group('integration')->skip(IntegrationDatabase::skipReason($variable, $extension) !== null, (string) IntegrationDatabase::skipReason($variable, $extension)); + })->group('integration')->skip(IntegrationDatabase::skipReason($lane) !== null, (string) IntegrationDatabase::skipReason($lane)); } diff --git a/packages/orchestration/tests/Integration/DatabaseSagaIntegrationTest.php b/packages/orchestration/tests/Integration/DatabaseSagaIntegrationTest.php index 578f5a70..152bea51 100644 --- a/packages/orchestration/tests/Integration/DatabaseSagaIntegrationTest.php +++ b/packages/orchestration/tests/Integration/DatabaseSagaIntegrationTest.php @@ -10,9 +10,9 @@ use Firefly\Orchestration\Tests\Support\TestEngines; use Firefly\Orchestration\Workflow\Builder\WorkflowBuilder; -foreach (IntegrationDatabase::servers() as $server => [$driver, $variable, $extension]) { - it("runs, compensates and dead-letters a saga, and parks and wakes a workflow, on {$server}", function () use ($driver, $variable): void { - $connection = IntegrationDatabase::fresh(IntegrationDatabase::connection($driver, $variable)); +foreach (IntegrationDatabase::lanes() as $lane) { + it("runs, compensates and dead-letters a saga, and parks and wakes a workflow, on {$lane}", function () use ($lane): void { + $connection = IntegrationDatabase::fresh(IntegrationDatabase::connection($lane)); try { $engines = TestEngines::create(persistence: new DatabaseExecutionPersistenceProvider($connection), deadLetters: new DatabaseDeadLetterStore($connection)); $saga = SagaBuilder::saga('order') @@ -22,19 +22,19 @@ $engines->workflowRegistry->register(WorkflowBuilder::workflow('approval') ->step('approve')->waitForTimer(60_000, 'deadline')->add()->build()); - $failed = $engines->sagas->execute($saga, ['orderId' => 7], correlationId: "it-{$variable}-saga"); - $waiting = $engines->workflows->start('approval', correlationId: "it-{$variable}-wf"); + $failed = $engines->sagas->execute($saga, ['orderId' => 7], correlationId: "it-{$lane}-saga"); + $waiting = $engines->workflows->start('approval', correlationId: "it-{$lane}-wf"); $engines->clock->advance(61.0); $woken = $engines->workflows->resumeDue(); expect($failed->status)->toBe(ExecutionStatus::Failed) - ->and($engines->persistence->findById("it-{$variable}-saga")?->compensationResults)->toBe(['reserve' => 'released']) - ->and($engines->persistence->findById("it-{$variable}-saga")?->stepResults['reserve'] ?? null)->toBe(['hold' => 'h-1', 'note' => 'café']) - ->and($engines->deadLetterStore->findByCorrelationId("it-{$variable}-saga")[0]->stepId ?? null)->toBe('charge') + ->and($engines->persistence->findById("it-{$lane}-saga")?->compensationResults)->toBe(['reserve' => 'released']) + ->and($engines->persistence->findById("it-{$lane}-saga")?->stepResults['reserve'] ?? null)->toBe(['hold' => 'h-1', 'note' => 'café']) + ->and($engines->deadLetterStore->findByCorrelationId("it-{$lane}-saga")[0]->stepId ?? null)->toBe('charge') ->and($waiting->status)->toBe(ExecutionStatus::Waiting) ->and($woken[0]->status ?? null)->toBe(ExecutionStatus::Completed); } finally { $connection->disconnect(); } - })->group('integration')->skip(IntegrationDatabase::skipReason($variable, $extension) !== null, (string) IntegrationDatabase::skipReason($variable, $extension)); + })->group('integration')->skip(IntegrationDatabase::skipReason($lane) !== null, (string) IntegrationDatabase::skipReason($lane)); } diff --git a/packages/orchestration/tests/Integration/QueuedWorkflowIntegrationTest.php b/packages/orchestration/tests/Integration/QueuedWorkflowIntegrationTest.php index 2a8e6b3e..487deb5b 100644 --- a/packages/orchestration/tests/Integration/QueuedWorkflowIntegrationTest.php +++ b/packages/orchestration/tests/Integration/QueuedWorkflowIntegrationTest.php @@ -14,7 +14,7 @@ uses(QueuedWorkflowIntegrationTestCase::class); -it('queues an ASYNC workflow on the database queue, runs it in a worker and ignores a redelivery', function (string $server): void { +it('queues an ASYNC workflow on the database queue, runs it in a worker and ignores a redelivery', function (string $lane): void { /** @var QueuedWorkflowIntegrationTestCase $this */ $workflows = $this->app()->make(WorkflowEngine::class); $ledger = $this->app()->make(Ledger::class); @@ -22,7 +22,7 @@ $started = $workflows->start('statement', ['account' => 'acc-ü'], correlationId: 'stmt-it'); $queued = DB::table(QueuedWorkflowCapstoneTestCase::JOBS_TABLE)->pluck('queue')->all(); - expect(DB::connection()->getDriverName())->toBe(IntegrationDatabase::servers()[$server][0]) + expect(DB::connection()->getDriverName())->toBe(IntegrationDatabase::driver($lane)) ->and($started->status)->toBe(ExecutionStatus::Pending) ->and($queued)->toBe(['workflows']); @@ -40,4 +40,4 @@ ->and(DB::table(QueuedWorkflowCapstoneTestCase::JOBS_TABLE)->count())->toBe(0) ->and($workflows->find('stmt-it')?->toArray())->toBe($finished?->toArray()) ->and($ledger->lines)->toBe(['hook start statement', 'render acc-ü', 'deliver acc-ü']); -})->with(array_keys(IntegrationDatabase::servers()))->group('integration'); +})->with(IntegrationDatabase::lanes())->group('integration'); diff --git a/packages/orchestration/tests/Integration/WorkflowCompensationIntegrationTest.php b/packages/orchestration/tests/Integration/WorkflowCompensationIntegrationTest.php index cf001f97..8a1d7928 100644 --- a/packages/orchestration/tests/Integration/WorkflowCompensationIntegrationTest.php +++ b/packages/orchestration/tests/Integration/WorkflowCompensationIntegrationTest.php @@ -13,14 +13,14 @@ uses(OrchestrationIntegrationTestCase::class); -it('runs a #[CompensationStep] method with the step result when a discovered workflow fails, and stores it on the server', function (string $server): void { +it('runs a #[CompensationStep] method with the step result when a discovered workflow fails, and stores it on the server', function (string $lane): void { /** @var OrchestrationIntegrationTestCase $this */ $workflows = $this->app()->make(WorkflowEngine::class); $failed = $workflows->start('shipment', ['parcel' => 'pä-7', 'carrier' => 'closed'], correlationId: 'ship-it'); $stored = $workflows->find('ship-it'); - expect(DB::connection()->getDriverName())->toBe(IntegrationDatabase::servers()[$server][0]) + expect(DB::connection()->getDriverName())->toBe(IntegrationDatabase::driver($lane)) ->and($failed->status)->toBe(ExecutionStatus::Failed) ->and($this->app()->make(Ledger::class)->lines)->toBe(['hook start shipment', 'book pä-7', 'void label-pä-7']) ->and($stored?->stepStatus('book'))->toBe(StepStatus::Compensated) @@ -28,4 +28,4 @@ ->and($stored?->compensationResults)->toBe(['book' => 'voided label-pä-7']) ->and(DB::table(OrchestrationSchema::EXECUTIONS)->where('correlation_id', 'ship-it')->value('status'))->toBe('FAILED') ->and(DB::table(OrchestrationSchema::DEAD_LETTERS)->where('correlation_id', 'ship-it')->value('step_id'))->toBe('dispatch'); -})->with(array_keys(IntegrationDatabase::servers()))->group('integration'); +})->with(IntegrationDatabase::lanes())->group('integration'); diff --git a/packages/orchestration/tests/Support/BootsOnIntegrationServer.php b/packages/orchestration/tests/Support/BootsOnIntegrationServer.php index b0f98f5f..db3e5f2b 100644 --- a/packages/orchestration/tests/Support/BootsOnIntegrationServer.php +++ b/packages/orchestration/tests/Support/BootsOnIntegrationServer.php @@ -8,17 +8,16 @@ /** * Boots an orchestration capstone against a REAL server: the test's dataset names one of - * IntegrationDatabase::servers(), and that server becomes the application's default connection — so the + * IntegrationDatabase::lanes(), and that lane's database becomes the application's default connection — so the * execution checkpoints, the dead letters, the search attributes and (for the queued capstone) the `database` - * queue's jobs table all live on it, exactly as in a deployed application. A server whose DSN variable or PDO - * driver is missing SKIPS before the application boots. + * queue's jobs table all live on it, exactly as in a deployed application. A lane whose DSN variable or PDO driver + * is missing SKIPS, with IntegrationDsn's reason, before the application boots. */ trait BootsOnIntegrationServer { protected function setUp(): void { - [, $variable, $extension] = $this->server(); - $reason = IntegrationDatabase::skipReason($variable, $extension); + $reason = IntegrationDatabase::skipReason($this->lane()); if ($reason !== null) { $this->markTestSkipped($reason); } @@ -30,22 +29,19 @@ protected function resolveApplicationConfiguration($app): void { parent::resolveApplicationConfiguration($app); - [$driver, $variable] = $this->server(); /** @var Repository $config */ $config = $app->make('config'); - $config->set('database.connections.orchestration-it', IntegrationDatabase::config($driver, $variable)); + $config->set('database.connections.orchestration-it', IntegrationDatabase::config($this->lane())); $config->set('database.default', 'orchestration-it'); } - /** @return array{0: string, 1: string, 2: string} */ - private function server(): array + private function lane(): string { - $name = $this->providedData()[0] ?? null; - $servers = IntegrationDatabase::servers(); - if (! is_string($name) || ! isset($servers[$name])) { - throw new \LogicException('An orchestration integration test needs a dataset naming one of IntegrationDatabase::servers().'); + $lane = $this->providedData()[0] ?? null; + if (! is_string($lane) || ! in_array($lane, IntegrationDatabase::lanes(), true)) { + throw new \LogicException('An orchestration integration test needs a dataset naming one of IntegrationDatabase::lanes().'); } - return $servers[$name]; + return $lane; } } diff --git a/packages/orchestration/tests/Support/IntegrationDatabase.php b/packages/orchestration/tests/Support/IntegrationDatabase.php index 754c632e..07530aa0 100644 --- a/packages/orchestration/tests/Support/IntegrationDatabase.php +++ b/packages/orchestration/tests/Support/IntegrationDatabase.php @@ -5,83 +5,75 @@ namespace Firefly\Orchestration\Tests\Support; use Firefly\Orchestration\Persistence\OrchestrationSchema; +use Firefly\Testing\Integration\IntegrationDsn; use Illuminate\Database\Capsule\Manager; use Illuminate\Database\Connection; -use RuntimeException; /** - * Real database connections for the integration lane, from the program's DSN variables - * (`host=…;port=…;dbname=…;user=…;password=…`). A missing variable or PDO driver SKIPS, never fails. W1 moves - * this parsing into packages/testing; until this branch merges it, it lives here (spec decision D16). - * fresh() drops and recreates the three orchestration tables only — never a database. + * Real database connections for the orchestration integration tests, one per lane of the CI matrix + * (IntegrationDsn::lanes(): the four servers plus a sqlite FILE). Every DSN is read by firefly/testing's + * IntegrationDsn, so a lane whose variable or PDO driver is missing skips with IntegrationDsn::skipReason(), and a + * CI job that exports only its own DSN runs exactly its own lane (and sqlite). fresh() drops and recreates the three + * orchestration tables only — never a database. */ final class IntegrationDatabase { - /** @return array */ - public static function servers(): array + /** @return list every lane of the integration matrix */ + public static function lanes(): array { - return [ - 'PostgreSQL' => ['pgsql', 'FIREFLY_PG_DSN', 'pdo_pgsql'], - 'MySQL' => ['mysql', 'FIREFLY_MYSQL_DSN', 'pdo_mysql'], - 'MariaDB' => ['mariadb', 'FIREFLY_MARIADB_DSN', 'pdo_mysql'], - 'SQL Server' => ['sqlsrv', 'FIREFLY_SQLSRV_DSN', 'pdo_sqlsrv'], - ]; + return IntegrationDsn::lanes(); } - public static function skipReason(string $variable, string $extension): ?string + /** + * The lanes that are a separate server process with row-level locking — what a test of two concurrent sessions + * interleaving inside one open transaction needs. sqlite has one writer per database file, so a second session's + * write while the first holds its transaction is not an interleaving there, only a wait. + * + * @return list + */ + public static function serverLanes(): array { - $dsn = getenv($variable); - if (! is_string($dsn) || $dsn === '') { - return "Set {$variable} to an isolated server."; - } + return array_values(array_filter(IntegrationDsn::lanes(), static fn (string $lane): bool => $lane !== 'sqlite')); + } - return extension_loaded($extension) ? null : "The {$extension} extension is not loaded."; + public static function skipReason(string $lane): ?string + { + return IntegrationDsn::skipReason($lane); } - public static function connection(string $driver, string $variable): Connection + /** The Laravel driver name a connection on $lane reports. */ + public static function driver(string $lane): string { - $capsule = new Manager; - $capsule->addConnection(self::config($driver, $variable), $variable); + return IntegrationDsn::DATABASES[$lane]['driver']; + } - return $capsule->getConnection($variable); + public static function connection(string $lane): Connection + { + return IntegrationDsn::connect($lane, 'orchestration-'.$lane, self::overrides($lane)); } /** - * The Laravel connection config for one server — what connection() opens, and what a booted capstone sets - * under `database.connections.*` to run the real pipeline against that server. + * The Laravel connection config for one lane — what connection() opens, and what a booted capstone sets under + * `database.connections.*` to run the real pipeline against that server. * * @return array */ - public static function config(string $driver, string $variable): array + public static function config(string $lane): array { - $dsn = getenv($variable); - if (! is_string($dsn) || $dsn === '') { - throw new RuntimeException("{$variable} is not set."); - } - - $parts = []; - foreach (explode(';', $dsn) as $pair) { - [$key, $value] = array_pad(explode('=', $pair, 2), 2, ''); - $parts[trim($key)] = trim($value); - } - - $config = [ - 'driver' => $driver, - 'host' => $parts['host'] ?? '127.0.0.1', - 'port' => (int) ($parts['port'] ?? 0), - 'database' => $parts['dbname'] ?? '', - 'username' => $parts['user'] ?? '', - 'password' => $parts['password'] ?? '', - 'prefix' => '', - ]; + return IntegrationDsn::connectionConfig($lane, self::overrides($lane)); + } - return $config + match ($driver) { - 'pgsql' => ['charset' => 'utf8', 'search_path' => 'public', 'sslmode' => 'prefer'], - 'sqlsrv' => ['charset' => 'utf8', 'trust_server_certificate' => true], - default => ['charset' => 'utf8mb4', 'collation' => 'utf8mb4_unicode_ci'], - }; + /** + * What CrashScenarios::crash() hands the child process so it opens the same database: the sqlite lane's file as + * `sqlite:` (the form the unit-level crash test uses too), a server lane as `lane:` — the child + * inherits the environment, so it reads the same DSN. + */ + public static function crashTarget(string $lane): string + { + return $lane === 'sqlite' ? 'sqlite:'.IntegrationDsn::sqlitePath('orchestration') : 'lane:'.$lane; } + /** An in-process sqlite database for the unit tests (`:memory:` unless a file is named). */ public static function sqlite(string $database = ':memory:'): Connection { $capsule = new Manager; @@ -97,4 +89,15 @@ public static function fresh(Connection $connection): Connection return $connection; } + + /** + * The sqlite lane's one file per process, shared by every connection a test opens — and by the crash process + * CrashScenarios spawns, which is handed the same path. + * + * @return array + */ + private static function overrides(string $lane): array + { + return $lane === 'sqlite' ? ['database' => IntegrationDsn::sqlitePath('orchestration')] : []; + } } diff --git a/packages/orchestration/tests/Support/crash-orchestration.php b/packages/orchestration/tests/Support/crash-orchestration.php index ceb105c6..1078da79 100644 --- a/packages/orchestration/tests/Support/crash-orchestration.php +++ b/packages/orchestration/tests/Support/crash-orchestration.php @@ -20,9 +20,11 @@ if (str_starts_with($target, 'sqlite:')) { $connection = IntegrationDatabase::sqlite(substr($target, 7)); +} elseif (str_starts_with($target, 'lane:')) { + $connection = IntegrationDatabase::connection(substr($target, 5)); } else { - [$driver, $variable] = explode('|', $target, 2) + ['', '']; - $connection = IntegrationDatabase::connection($driver, $variable); + fwrite(STDERR, "unknown target [{$target}]\n"); + exit(1); } $engines = TestEngines::create(persistence: new DatabaseExecutionPersistenceProvider($connection), deadLetters: new DatabaseDeadLetterStore($connection)); From 24be842697470c5f14bae16651e1a800e37e85c6 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Tue, 13 Oct 2026 00:47:08 -0700 Subject: [PATCH 53/58] fix(orchestration): write search attributes by exact primary key, so a save on SQL Server no longer waits on another execution's open checkpoint The multi-row upsert ran as a MERGE that SQL Server can execute as a scan of the whole attributes table, and under READ COMMITTED that scan blocked on the rows another worker's uncommitted save held. save() now updates each kept key and inserts the new ones by primary key, touching only the execution's own rows on every server. --- .../DatabaseExecutionPersistenceProvider.php | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/packages/orchestration/src/Persistence/DatabaseExecutionPersistenceProvider.php b/packages/orchestration/src/Persistence/DatabaseExecutionPersistenceProvider.php index ead087f2..0a33c862 100644 --- a/packages/orchestration/src/Persistence/DatabaseExecutionPersistenceProvider.php +++ b/packages/orchestration/src/Persistence/DatabaseExecutionPersistenceProvider.php @@ -23,7 +23,10 @@ * nothing (every first save) takes a gap lock, and two workers saving new executions whose ids share an index * gap then deadlock on each other's INSERT (MySQL and MariaDB error 1213). save() instead reads the stored * keys with a plain select, deletes only the obsolete ones by exact primary key (record locks, no gap - * locks) and upserts the new set on [correlation_id, attribute_key]. A top-level save() also retries a + * locks), updates each kept key by exact primary key and inserts the new ones. The attributes are not upserted: + * SQL Server runs a multi-row upsert as a MERGE that can scan the whole table, and under READ COMMITTED that scan + * waits on every row another worker's open checkpoint holds — so a save of one execution blocked behind a save + * of another. Statements keyed on this execution's own primary keys touch only its own rows on every server. A top-level save() also retries a * deadlock or serialization failure up to ATTEMPTS times; inside a caller's transaction Laravel cannot * retry a nested block, so the error reaches the caller, whose own transaction decides what to do. */ @@ -67,7 +70,8 @@ public function save(ExecutionState $state): void $this->connection->transaction(function () use ($row, $state, $attributes): void { $this->connection->table(OrchestrationSchema::EXECUTIONS)->upsert([$row], ['correlation_id'], self::UPDATED); - $obsolete = array_values(array_diff($this->attributeKeys($state->correlationId), array_column($attributes, 'attribute_key'))); + $stored = $this->attributeKeys($state->correlationId); + $obsolete = array_values(array_diff($stored, array_column($attributes, 'attribute_key'))); if ($obsolete !== []) { $this->connection->table(OrchestrationSchema::SEARCH_ATTRIBUTES) ->where('correlation_id', $state->correlationId) @@ -75,8 +79,21 @@ public function save(ExecutionState $state): void ->delete(); } - if ($attributes !== []) { - $this->connection->table(OrchestrationSchema::SEARCH_ATTRIBUTES)->upsert($attributes, ['correlation_id', 'attribute_key'], ['attribute_value']); + $new = []; + foreach ($attributes as $attribute) { + if (! in_array($attribute['attribute_key'], $stored, true)) { + $new[] = $attribute; + + continue; + } + $this->connection->table(OrchestrationSchema::SEARCH_ATTRIBUTES) + ->where('correlation_id', $attribute['correlation_id']) + ->where('attribute_key', $attribute['attribute_key']) + ->update(['attribute_value' => $attribute['attribute_value']]); + } + + if ($new !== []) { + $this->connection->table(OrchestrationSchema::SEARCH_ATTRIBUTES)->insert($new); } }, self::ATTEMPTS); } From a5394ed056833c787d14b3482f9c4c2e5689e4a3 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Tue, 13 Oct 2026 00:52:14 -0700 Subject: [PATCH 54/58] docs(orchestration): state that SQL Server is proven by the CI integration lane, and add orchestration to the database support matrix --- CHANGELOG.md | 13 ++++++------- docs/database-support.md | 2 ++ docs/modules/orchestration.md | 6 +++--- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 766f96ba..92f3ea66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,10 +12,9 @@ All notable changes to LaraFly are documented here. This project uses CalVer (`Y wait-for-all/any, child workflows, continue-as-new, queries, suspend/resume, search attributes and queued execution. Definitions are attribute-discovered, compiled by `firefly:cache` and validated at boot. - **Orchestration persistence and recovery:** checkpoints at every step through a `memory`, `cache` or - `database` store (proven on PostgreSQL, MySQL, MariaDB and SQLite; SQL Server is written for and has - integration tests that have not yet run against a real server), a per-execution distributed lock, a - scheduled recovery sweep that reports, resumes or compensates stale executions, retention cleanup, and a - dead-letter queue with retry. + `database` store (proven on PostgreSQL, MySQL, MariaDB, SQL Server and SQLite by the CI integration lanes), + a per-execution distributed lock, a scheduled recovery sweep that reports, resumes or compensates stale + executions, retention cleanup, and a dead-letter queue with retry. - **Orchestration operations:** lifecycle and step events through `firefly/eda`, event-type triggers, cron and fixed-rate schedules, the `orchestration` actuator endpoint and health component, `php artisan firefly:orchestration:{list,show,signal,dlq,dlq:retry,tick}` and an opt-in REST surface. @@ -23,12 +22,12 @@ All notable changes to LaraFly are documented here. This project uses CalVer (`Y A capstone checks that the `RunWorkflowJob` is pushed on `firefly.orchestration.workflow.queue.connection` and `.name`, that `queue:work` runs it to `COMPLETED`, that a redelivered job is a no-op, and that the `sync` connection runs the job inside the start. An integration test runs the same path on the `database` - queue against PostgreSQL, MySQL and MariaDB (its SQL Server case has not yet run against a real server). + queue against PostgreSQL, MySQL, MariaDB, SQL Server and a SQLite file, one CI integration lane each. - **Orchestration workflow compensation steps:** the separate-method `#[CompensationStep(compensates: …)]` form is now tested and documented. The scanner tests check that the method becomes the step's compensation with the step result bound implicitly, and that a duplicate, a step with both forms and an unknown step are - refused. A capstone and an integration test on PostgreSQL, MySQL and MariaDB check that a failing - discovered workflow runs it (its SQL Server case has not yet run against a real server). + refused. A capstone and an integration test on PostgreSQL, MySQL, MariaDB, SQL Server and a SQLite file + check that a failing discovered workflow runs it. - **Orchestration failure parking:** a saga whose compensation stops part-way ends `COMPENSATION_FAILED` and is dead-lettered, and a due workflow the tick fails to drive `firefly.orchestration.workflow.max-resume-attempts` times in a row (default `5`) is parked `FAILED` and dead-lettered. diff --git a/docs/database-support.md b/docs/database-support.md index c218cca9..a3c79638 100644 --- a/docs/database-support.md +++ b/docs/database-support.md @@ -20,6 +20,8 @@ on a temporary file. | `#[Transactional(timeout:)]` at the server | `SET LOCAL statement_timeout` | `max_execution_time` (reads) + `innodb_lock_wait_timeout` | `max_statement_time` + `innodb_lock_wait_timeout` | `SET LOCK_TIMEOUT` (lock waits) | busy timeout | | `#[Lock(PESSIMISTIC_WRITE / READ)]` | ✓ `FOR UPDATE` / `FOR SHARE` | ✓ `FOR UPDATE` / `LOCK IN SHARE MODE` | ✓ | ✓ `UPDLOCK` / `HOLDLOCK` hints | — (compiles to a plain read) | | Optimistic locking (`HasOptimisticLock`) | ✓ | ✓ | ✓ | ✓ | ✓ | +| Orchestration checkpoints, search attributes, recovery and dead letters (`firefly.orchestration.persistence.provider=database`) | ✓ | ✓ | ✓ | ✓ | ✓ | +| ASYNC workflows on the `database` queue | ✓ | ✓ | ✓ | ✓ | ✓ | How the outbox claim is chosen, and why every strategy is safe with any number of workers, is explained in [EDA Database Outbox](modules/eda-database.md#claim-strategies). Servers older than the versions above fall back to diff --git a/docs/modules/orchestration.md b/docs/modules/orchestration.md index 445834f3..75d9c20e 100644 --- a/docs/modules/orchestration.md +++ b/docs/modules/orchestration.md @@ -324,8 +324,8 @@ connection is the one exception: it runs the job inside `start()`, and the start execution. Redelivery is safe. A job delivered a second time, for example by a worker that died before it acknowledged the job, finds the execution no longer `PENDING` and returns it unchanged, so no step runs twice. The capstone in `packages/orchestration/tests/Capstone/CapstoneQueuedWorkflowTest.php` covers this path on -the `database` queue. Its integration twin covers it on PostgreSQL, MySQL and MariaDB. A SQL Server case is -written but has not yet run against a real server. +the `database` queue. Its integration twin covers it on PostgreSQL, MySQL, MariaDB, SQL Server and a SQLite +file, one CI integration lane each. ## Persistence, recovery and the dead-letter queue @@ -335,7 +335,7 @@ written but has not yet run against a real server. |---|---| | `memory` (default) | Tests and single-request flows. Nothing survives the process. | | `cache` | Any Laravel cache store (`persistence.cache-store`), keys under `persistence.key-prefix`. | -| `database` | The `firefly_orchestration_*` tables on `persistence.connection`. Proven on PostgreSQL, MySQL, MariaDB and SQLite. SQL Server takes the same `upsert()` path (`MERGE`) and has integration tests behind `FIREFLY_SQLSRV_DSN`, but they have not yet run against a real server. | +| `database` | The `firefly_orchestration_*` tables on `persistence.connection`. Proven on PostgreSQL, MySQL, MariaDB, SQL Server and SQLite by the CI integration lanes (see the [database support matrix](../database-support.md)). | Publish the migration with `php artisan vendor:publish --tag=firefly-orchestration-migrations`; the package also loads it while `persistence.provider` or `dlq.store` is `database`. From de0739cb9e0bc21dd139142a745f4a3122c673f3 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Tue, 13 Oct 2026 00:53:33 -0700 Subject: [PATCH 55/58] docs(cli): list the six firefly:orchestration commands and recount the guides --- docs/README.md | 2 +- docs/cli.md | 6 ++++++ docs/contributing.md | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/README.md b/docs/README.md index dc3d5710..2931547a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -127,7 +127,7 @@ thirty-seventh guide turns this page red rather than quietly making it wrong. | Document | Description | |----------|-------------| | [Modules](modules.md) | The grouped index of all 36 module guides, and how a package wires itself | -| [CLI Reference](cli.md) | Every `firefly:*` command and `make:firefly-*` generator, including the seven contributed by capability packages | +| [CLI Reference](cli.md) | Every `firefly:*` command and `make:firefly-*` generator, including the thirteen contributed by capability packages | | [Database Support Matrix](database-support.md) | What each database-backed capability does on PostgreSQL, MySQL, MariaDB, SQL Server and SQLite, and which CI lane proves it | | [Laravel Comparison](laravel-comparison.md) | Side-by-side concept mapping for developers coming from plain Laravel | | [Versioning](versioning.md) | CalVer (`YY.MM.Patch`), no `version` field, how Packagist derives releases from tags | diff --git a/docs/cli.md b/docs/cli.md index d3650dca..a430f505 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -333,4 +333,10 @@ command is part of that capability rather than of the console. | `firefly:eda:dead-letters:replay` | `firefly/eda` | Republishes dead-lettered envelopes (`firefly.eda.dead-letter.store=database`) through the bound `EventPublisher`, oldest first, to their original destination, removing each one that republished. `--id=*`, `--destination=`, `--event-type=`, `--limit=100`, `--dry-run`, `--keep`; exit 1 when any entry failed. Registered only under that `database` store. | | `firefly:messaging:dead-letters:replay` | `firefly/messaging` | Republishes dead-lettered messages (`firefly.messaging.dead-letter.store=database`) through the bound `MessageBrokerPort` to their original topic. `--id=*`, `--topic=`, `--limit=100`, `--dry-run`, `--keep`; exit 1 when any entry failed. Registered only under that `database` store. | | `firefly:outbox:relay` | `firefly/eda-database` | **Optional.** Forwards committed `firefly_eda_outbox` rows to the downstream broker named by `firefly.eda.database.relay.downstream-provider` (alias `firefly.eda.postgres.relay.downstream_provider`) (claim → publish → mark `PUBLISHED`/`FAILED`); any number of relay processes may run — each batch is leased. `--max-messages=`, `--time-limit=`, `--sleep=1` (seconds between empty batches), `--batch-size=50`. In-process delivery is `firefly:eda:consume`'s job, not this command's, so an app that only needs `#[EventListener]` handlers never runs it; an unset or misconfigured `downstream-provider` fails loudly before a single row is claimed. See [EDA Database Outbox](modules/eda-database.md). | +| `firefly:orchestration:list` | `firefly/orchestration` | Lists persisted saga, TCC and workflow executions. `--pattern=saga\|tcc\|workflow`, `--status=` (e.g. `failed`, `waiting`), `--name=` (the saga/TCC name or workflow id), `--limit=50`. See [Orchestration](modules/orchestration.md). | +| `firefly:orchestration:show` | `firefly/orchestration` | Shows one persisted execution by correlation id: its status, steps and report. | +| `firefly:orchestration:signal` | `firefly/orchestration` | Delivers a signal to a durable workflow: `{id} {signal}`, with `--payload=` as JSON. | +| `firefly:orchestration:dlq` | `firefly/orchestration` | Lists orchestration dead letters. `--name=`, `--limit=50`. | +| `firefly:orchestration:dlq:retry` | `firefly/orchestration` | Retries one orchestration dead letter by id; the execution restarts from scratch. | +| `firefly:orchestration:tick` | `firefly/orchestration` | Resumes due workflows (signals, timers) and runs the recovery sweep; `--no-recovery` only resumes. The same work runs as a `#[Scheduled]` task under the `DistributedLock` port. | | `firefly:flags` | `firefly/feature-flags` | Runs the `flags` management operations in-process: `list`, `show {key}`, `evaluate {key}` (`--context=`, `--targeting-key=`), and the writes `enable`, `disable`, `default-variant {key} {variant}`, `put {key} --file=`, `delete {key}` (each with `--expected-version=`). `--json` prints the management JSON, including `{error,message}` on a refusal. Writes need `firefly.feature-flags.management.writes` and a flag store; the actor is the principal, else `cli:`. An accepted write with pending local refresh exits successfully and asks the operator to check visibility again. | diff --git a/docs/contributing.md b/docs/contributing.md index cf230ec6..dd1a0f58 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -245,7 +245,7 @@ required to land on a document that exists; `navigation.indexes` is required to section really has an `index.md`/`README.md` for it to bind, because the feature is inert otherwise and the first explanation of the tab row was wrong in precisely that way; the package and guide counts quoted in prose — on `README.md`, `docs/README.md`, `docs/index.md` and `docs/modules.md` — are compared against -`glob()` over `packages/*` and `docs/modules/*.md` rather than proof-read, so a thirty-third guide fails the +`glob()` over `packages/*` and `docs/modules/*.md` rather than proof-read, so a thirty-seventh guide fails the build instead of turning four sentences into lies no diff touched; and the two wiring idioms `docs/modules.md` names are held to the split the packages really have, exemplar files included. From 44a6e45a5e40a1ebe41ac5fcae3421ba207df234 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Tue, 13 Oct 2026 01:00:43 -0700 Subject: [PATCH 56/58] test(testing): open every SQL Server integration connection unpooled, so no test inherits another's session settings --- packages/testing/src/Integration/IntegrationDsn.php | 4 +++- packages/testing/tests/Integration/IntegrationDsnTest.php | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/testing/src/Integration/IntegrationDsn.php b/packages/testing/src/Integration/IntegrationDsn.php index d34cd9bb..218c88ee 100644 --- a/packages/testing/src/Integration/IntegrationDsn.php +++ b/packages/testing/src/Integration/IntegrationDsn.php @@ -173,7 +173,9 @@ public static function connectionConfigFor(string $driver, string $variable, arr $config = match ($driver) { 'pgsql' => [...$config, 'charset' => 'utf8', 'search_path' => $parts['search_path'] ?? 'public'], 'mysql', 'mariadb' => [...$config, 'charset' => 'utf8mb4', 'collation' => 'utf8mb4_unicode_ci'], - 'sqlsrv' => [...$config, 'charset' => 'utf8', 'encrypt' => $parts['encrypt'] ?? 'yes', 'trust_server_certificate' => $parts['trust_server_certificate'] ?? 'false'], + // pooling off: the ODBC driver would otherwise hand a later test the same physical session, SET options + // (a short LOCK_TIMEOUT) included, so one test's session state could decide another's outcome. + 'sqlsrv' => [...$config, 'charset' => 'utf8', 'encrypt' => $parts['encrypt'] ?? 'yes', 'trust_server_certificate' => $parts['trust_server_certificate'] ?? 'false', 'pooling' => false], default => $config, }; diff --git a/packages/testing/tests/Integration/IntegrationDsnTest.php b/packages/testing/tests/Integration/IntegrationDsnTest.php index f5d9fba8..8ed93b53 100644 --- a/packages/testing/tests/Integration/IntegrationDsnTest.php +++ b/packages/testing/tests/Integration/IntegrationDsnTest.php @@ -84,7 +84,7 @@ function integrationDsnRemoveSqlite(string $path): void 'password' => 'pw', 'prefix' => '', 'charset' => 'utf8', 'search_path' => 'public', ])->and(IntegrationDsn::connectionConfigFor('mariadb', 'FIREFLY_TEST_DSN')['charset'])->toBe('utf8mb4') ->and(IntegrationDsn::connectionConfigFor('sqlsrv', 'FIREFLY_TEST_DSN'))->toMatchArray([ - 'driver' => 'sqlsrv', 'encrypt' => 'yes', 'trust_server_certificate' => 'false', + 'driver' => 'sqlsrv', 'encrypt' => 'yes', 'trust_server_certificate' => 'false', 'pooling' => false, ])->and(IntegrationDsn::connectionConfigFor('mysql', 'FIREFLY_TEST_DSN', ['database' => 'other'])['database'])->toBe('other'); }); From 4e0b17deba3a25396e7f5f12f5a10053dda10a26 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Tue, 13 Oct 2026 01:06:39 -0700 Subject: [PATCH 57/58] test(data): say what the deadlock test's parent session saw when the exactly-one-loser check fails --- .../PersistenceExceptionTranslationIntegrationTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/data/tests/Integration/PersistenceExceptionTranslationIntegrationTest.php b/packages/data/tests/Integration/PersistenceExceptionTranslationIntegrationTest.php index cef091b4..acda037e 100644 --- a/packages/data/tests/Integration/PersistenceExceptionTranslationIntegrationTest.php +++ b/packages/data/tests/Integration/PersistenceExceptionTranslationIntegrationTest.php @@ -128,7 +128,7 @@ function translationShortLockWait(Connection $connection, string $lane): void $racerSaw = trim($racer->getOutput()); // Exactly one side lost, and it was told so in the kernel's own vocabulary. - expect($parent instanceof DeadlockLoserDataAccessException xor str_ends_with($racerSaw, 'deadlock'))->toBeTrue('parent: '.($parent === null ? 'committed' : $parent::class)."; racer: {$racerSaw}") + expect($parent instanceof DeadlockLoserDataAccessException xor str_ends_with($racerSaw, 'deadlock'))->toBeTrue('parent: '.($parent === null ? 'committed' : $parent::class.' — '.$parent->getMessage().' — previous: '.($parent->getPrevious()?->getMessage() ?? 'none')).'; parent lock_timeout: '.json_encode($connection->selectOne('select @@lock_timeout as value')).'; racer: '.$racerSaw) ->and($parent === null || $parent instanceof DeadlockLoserDataAccessException)->toBeTrue(); } finally { $racer->stop(); From a8bc39fe430ace9a1363f99b32fca19cd46b25a8 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Tue, 13 Oct 2026 01:10:54 -0700 Subject: [PATCH 58/58] test(integration): put SQL Server's lock wait back after each test that shortens it, since the runner's driver manager pools sessions --- ...stenceExceptionTranslationIntegrationTest.php | 16 +++++++++++++++- .../RepositoryLockingIntegrationTest.php | 4 ++++ packages/data/tests/Support/deadlock-racer.php | 3 +++ .../DatabasePersistenceIntegrationTest.php | 3 +++ .../testing/src/Integration/IntegrationDsn.php | 4 +--- .../tests/Integration/IntegrationDsnTest.php | 2 +- 6 files changed, 27 insertions(+), 5 deletions(-) diff --git a/packages/data/tests/Integration/PersistenceExceptionTranslationIntegrationTest.php b/packages/data/tests/Integration/PersistenceExceptionTranslationIntegrationTest.php index acda037e..672cdb74 100644 --- a/packages/data/tests/Integration/PersistenceExceptionTranslationIntegrationTest.php +++ b/packages/data/tests/Integration/PersistenceExceptionTranslationIntegrationTest.php @@ -61,6 +61,18 @@ function translationShortLockWait(Connection $connection, string $lane): void }; } +/** + * Put the server's own lock wait back. The ODBC driver manager on the CI runner pools SQL Server sessions, so a + * session-level SET outlives disconnect() and would decide a later test's outcome (the deadlock test below saw a + * 300 ms lock timeout and lost by timing out instead of by being chosen as the deadlock victim). + */ +function translationDefaultLockWait(Connection $connection, string $lane): void +{ + if ($lane === 'sqlsrv') { + $connection->unprepared('set lock_timeout -1'); + } +} + foreach (IntegrationDsn::lanes() as $lane) { it("translates integrity, grammar and lock-wait failures on {$lane}", function () use ($lane): void { $overrides = $lane === 'sqlite' ? ['database' => IntegrationDsn::sqlitePath('translation'), 'busy_timeout' => 300] : []; @@ -83,6 +95,7 @@ function translationShortLockWait(Connection $connection, string $lane): void translationShortLockWait($b, $lane); $waited = translationOf($b, static fn () => $b->table('firefly_translation_parents')->where('id', 1)->update(['name' => 'waiter'])); $a->rollBack(); + translationDefaultLockWait($b, $lane); expect($waited)->toBeInstanceOf(CannotAcquireLockException::class); } finally { @@ -98,6 +111,7 @@ function translationShortLockWait(Connection $connection, string $lane): void foreach (['pgsql', 'mysql', 'mariadb', 'sqlsrv'] as $lane) { it("translates the deadlock victim on {$lane}, whichever process the server picks", function () use ($lane): void { $connection = IntegrationDsn::connect($lane, 'deadlock-parent'); + translationDefaultLockWait($connection, $lane); // a pooled session may carry another test's lock timeout $schema = $connection->getSchemaBuilder(); $schema->dropIfExists('firefly_deadlock_probe'); $schema->create('firefly_deadlock_probe', static function (Blueprint $table): void { @@ -128,7 +142,7 @@ function translationShortLockWait(Connection $connection, string $lane): void $racerSaw = trim($racer->getOutput()); // Exactly one side lost, and it was told so in the kernel's own vocabulary. - expect($parent instanceof DeadlockLoserDataAccessException xor str_ends_with($racerSaw, 'deadlock'))->toBeTrue('parent: '.($parent === null ? 'committed' : $parent::class.' — '.$parent->getMessage().' — previous: '.($parent->getPrevious()?->getMessage() ?? 'none')).'; parent lock_timeout: '.json_encode($connection->selectOne('select @@lock_timeout as value')).'; racer: '.$racerSaw) + expect($parent instanceof DeadlockLoserDataAccessException xor str_ends_with($racerSaw, 'deadlock'))->toBeTrue('parent: '.($parent === null ? 'committed' : $parent::class.' — '.$parent->getMessage().' — previous: '.($parent->getPrevious()?->getMessage() ?? 'none')).($lane === 'sqlsrv' ? '; parent lock_timeout: '.json_encode($connection->selectOne('select @@lock_timeout as value')) : '').'; racer: '.$racerSaw) ->and($parent === null || $parent instanceof DeadlockLoserDataAccessException)->toBeTrue(); } finally { $racer->stop(); diff --git a/packages/data/tests/Integration/RepositoryLockingIntegrationTest.php b/packages/data/tests/Integration/RepositoryLockingIntegrationTest.php index 616aa271..0199e6d4 100644 --- a/packages/data/tests/Integration/RepositoryLockingIntegrationTest.php +++ b/packages/data/tests/Integration/RepositoryLockingIntegrationTest.php @@ -70,6 +70,10 @@ function lockingWriteFromOther(Connection $other, string $lane): ?Throwable $other->table('records')->where('id', 1)->update(['status' => 'stolen']); } catch (Throwable $e) { return (new PersistenceExceptionTranslator)->translate($e, $other->getDriverName()); + } finally { + if ($lane === 'sqlsrv') { + $other->unprepared('set lock_timeout -1'); // pooled SQL Server sessions keep session SETs past disconnect() + } } return null; diff --git a/packages/data/tests/Support/deadlock-racer.php b/packages/data/tests/Support/deadlock-racer.php index c7338c5f..d5b0cfee 100644 --- a/packages/data/tests/Support/deadlock-racer.php +++ b/packages/data/tests/Support/deadlock-racer.php @@ -16,6 +16,9 @@ $lane = $argv[1] ?? ''; $go = $argv[2] ?? ''; $connection = IntegrationDsn::connect($lane, 'deadlock-racer'); +if ($lane === 'sqlsrv') { + $connection->unprepared('set lock_timeout -1'); // a pooled session may carry another test's lock timeout +} $connection->beginTransaction(); $connection->table('firefly_deadlock_probe')->where('id', 2)->update(['name' => 'racer']); diff --git a/packages/orchestration/tests/Integration/DatabasePersistenceIntegrationTest.php b/packages/orchestration/tests/Integration/DatabasePersistenceIntegrationTest.php index 5f387b40..b156e2df 100644 --- a/packages/orchestration/tests/Integration/DatabasePersistenceIntegrationTest.php +++ b/packages/orchestration/tests/Integration/DatabasePersistenceIntegrationTest.php @@ -75,6 +75,9 @@ ->and(ExecutionPersistenceContract::ids($two->findBySearchAttributes(['customer' => 'barbara'])))->toBe(['a2']) ->and($first->table(OrchestrationSchema::SEARCH_ATTRIBUTES)->whereIn('correlation_id', ['a1', 'a2'])->count())->toBe(3); } finally { + if ($lane === 'sqlsrv') { + $second->unprepared('SET LOCK_TIMEOUT -1'); // pooled SQL Server sessions keep session SETs past disconnect() + } if ($first->transactionLevel() > 0) { $first->rollBack(); } diff --git a/packages/testing/src/Integration/IntegrationDsn.php b/packages/testing/src/Integration/IntegrationDsn.php index 218c88ee..d34cd9bb 100644 --- a/packages/testing/src/Integration/IntegrationDsn.php +++ b/packages/testing/src/Integration/IntegrationDsn.php @@ -173,9 +173,7 @@ public static function connectionConfigFor(string $driver, string $variable, arr $config = match ($driver) { 'pgsql' => [...$config, 'charset' => 'utf8', 'search_path' => $parts['search_path'] ?? 'public'], 'mysql', 'mariadb' => [...$config, 'charset' => 'utf8mb4', 'collation' => 'utf8mb4_unicode_ci'], - // pooling off: the ODBC driver would otherwise hand a later test the same physical session, SET options - // (a short LOCK_TIMEOUT) included, so one test's session state could decide another's outcome. - 'sqlsrv' => [...$config, 'charset' => 'utf8', 'encrypt' => $parts['encrypt'] ?? 'yes', 'trust_server_certificate' => $parts['trust_server_certificate'] ?? 'false', 'pooling' => false], + 'sqlsrv' => [...$config, 'charset' => 'utf8', 'encrypt' => $parts['encrypt'] ?? 'yes', 'trust_server_certificate' => $parts['trust_server_certificate'] ?? 'false'], default => $config, }; diff --git a/packages/testing/tests/Integration/IntegrationDsnTest.php b/packages/testing/tests/Integration/IntegrationDsnTest.php index 8ed93b53..f5d9fba8 100644 --- a/packages/testing/tests/Integration/IntegrationDsnTest.php +++ b/packages/testing/tests/Integration/IntegrationDsnTest.php @@ -84,7 +84,7 @@ function integrationDsnRemoveSqlite(string $path): void 'password' => 'pw', 'prefix' => '', 'charset' => 'utf8', 'search_path' => 'public', ])->and(IntegrationDsn::connectionConfigFor('mariadb', 'FIREFLY_TEST_DSN')['charset'])->toBe('utf8mb4') ->and(IntegrationDsn::connectionConfigFor('sqlsrv', 'FIREFLY_TEST_DSN'))->toMatchArray([ - 'driver' => 'sqlsrv', 'encrypt' => 'yes', 'trust_server_certificate' => 'false', 'pooling' => false, + 'driver' => 'sqlsrv', 'encrypt' => 'yes', 'trust_server_certificate' => 'false', ])->and(IntegrationDsn::connectionConfigFor('mysql', 'FIREFLY_TEST_DSN', ['database' => 'other'])['database'])->toBe('other'); });